Skip to content

Optimize: Skip module resolution if Puppetfile matches upstream #243

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
xorpaul wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Optimize: Skip module resolution if Puppetfile matches upstream #243

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 13 additions & 12 deletions g10k.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -140,18 +140,19 @@ type Source struct {

// Puppetfile contains the key value pairs from the Puppetfile
type Puppetfile struct {
forgeBaseURL string
forgeCacheTTL time.Duration
forgeModules map[string]ForgeModule
gitModules map[string]GitModule
privateKey string
source string
sourceBranch string
workDir string
gitDir string
gitURL string
moduleDirs []string
controlRepoBranch string
forgeBaseURL string
forgeCacheTTL time.Duration
forgeModules map[string]ForgeModule
gitModules map[string]GitModule
privateKey string
source string
sourceBranch string
workDir string
gitDir string
gitURL string
moduleDirs []string
controlRepoBranch string
upstreamPuppetfileMatches bool
}

// ForgeModule contains information (Version, Name, Author, md5 checksum, file size of the tar.gz archive, Forge BaseURL if custom) about a Puppetlabs Forge module
Expand Down
23 changes: 23 additions & 0 deletions git.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,7 @@
}

// check if git URL does match NO_PROXY
disableHttpProxy := false

Check failure on line 127 in git.go

View workflow job for this annotation

GitHub Actions / Test on go version and macOS-latest 

var disableHttpProxy should be disableHTTPProxy

Check failure on line 127 in git.go

View workflow job for this annotation

GitHub Actions / Test on go version and ubuntu-latest 

var disableHttpProxy should be disableHTTPProxy

Check failure on line 127 in git.go

View workflow job for this annotation

GitHub Actions / Test on go version and macOS-latest 

var disableHttpProxy should be disableHTTPProxy

Check failure on line 127 in git.go

View workflow job for this annotation

GitHub Actions / Test on go version and ubuntu-latest 

var disableHttpProxy should be disableHTTPProxy
if matchGitRemoteURLNoProxy(gitModule.git) {
disableHttpProxy = true
}
Expand Down Expand Up @@ -240,7 +240,30 @@
purgeWholeEnvDir = true
} else {
purgeWholeEnvDir = false

// calculate the checksum of the deployed Puppetfile
deployedPuppetfile := filepath.Join(targetDir, "Puppetfile")
deployedPuppetfileChecksum := ""
if fileExists(deployedPuppetfile) {
deployedPuppetfileChecksum = getSha256sumFile(deployedPuppetfile)
}

// calculate the checksum of the upstream Puppetfile
upstreamPuppetfileChecksum := getSha256sumString(executeResult.output)

if deployedPuppetfileChecksum == upstreamPuppetfileChecksum {
Debugf("Puppetfile checksum match for " + targetDir)
// we can signal to skip module resolution if only the Puppetfile hasn't changed
// but we can't easily return this info from here to resolvePuppetEnvironment
// So we might need to store this state somewhere
mutex.Lock()
Debugf("Setting PuppetfileMatch for env: " + correspondingPuppetEnvironment)
needSyncEnvs[correspondingPuppetEnvironment+":PuppetfileMatch"] = empty
mutex.Unlock()
}

lines := strings.Split(executeResult.output, "\n")

for _, line := range lines {
if m := reModuledir.FindStringSubmatch(line); len(m) > 1 {
// moduledir CLI parameter override
Expand Down
7 changes: 7 additions & 0 deletions helper.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -169,7 +169,7 @@
}
}

func executeCommand(command string, commandDir string, timeout int, allowFail bool, disableHttpProxy bool) ExecResult {

Check failure on line 172 in helper.go

View workflow job for this annotation

GitHub Actions / Test on go version and macOS-latest 

func parameter disableHttpProxy should be disableHTTPProxy

Check failure on line 172 in helper.go

View workflow job for this annotation

GitHub Actions / Test on go version and ubuntu-latest 

func parameter disableHttpProxy should be disableHTTPProxy

Check failure on line 172 in helper.go

View workflow job for this annotation

GitHub Actions / Test on go version and macOS-latest 

func parameter disableHttpProxy should be disableHTTPProxy

Check failure on line 172 in helper.go

View workflow job for this annotation

GitHub Actions / Test on go version and ubuntu-latest 

func parameter disableHttpProxy should be disableHTTPProxy
if len(commandDir) > 0 {
Debugf("Executing " + command + " in cwd " + commandDir)
} else {
Expand Down Expand Up @@ -272,6 +272,13 @@
return hex.EncodeToString(h.Sum(nil))
}

// getSha256sumString return the SHA256 hash sum of the given string
func getSha256sumString(s string) string {
h := sha256.New()
h.Write([]byte(s))
return hex.EncodeToString(h.Sum(nil))
}

// moveFile uses io.Copy to create a copy of the given file https://stackoverflow.com/a/50741908/682847
func moveFile(sourcePath, destPath string, deleteSourceFileToggle bool) error {
inputFile, err := os.Open(sourcePath)
Expand Down Expand Up @@ -345,7 +352,7 @@
func stripComponent(component string, env string) string {
if regexp.MustCompile(`^/.*/$`).MatchString(component) {
return regexp.MustCompile(component[1:len(component)-1]).ReplaceAllString(env, "")
} else {

Check failure on line 355 in helper.go

View workflow job for this annotation

GitHub Actions / Test on go version and macOS-latest 

if block ends with a return statement, so drop this else and outdent its block

Check failure on line 355 in helper.go

View workflow job for this annotation

GitHub Actions / Test on go version and ubuntu-latest 

if block ends with a return statement, so drop this else and outdent its block

Check failure on line 355 in helper.go

View workflow job for this annotation

GitHub Actions / Test on go version and macOS-latest 

if block ends with a return statement, so drop this else and outdent its block

Check failure on line 355 in helper.go

View workflow job for this annotation

GitHub Actions / Test on go version and ubuntu-latest 

if block ends with a return statement, so drop this else and outdent its block
return strings.TrimPrefix(env, component)
}
}
Expand Down
7 changes: 7 additions & 0 deletions puppetfile.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -168,6 +168,8 @@ func resolvePuppetEnvironment(tags bool, outputNameTag string) {
pf := filepath.Join(targetDir, "Puppetfile")
mutex.Lock()
allBasedirs[sa.Basedir] = true
// check if the Puppetfile content from the upstream repository matches the one in the deployed environment
_, pfMatch := needSyncEnvs[env+":PuppetfileMatch"]
mutex.Unlock()
if !fileExists(pf) {
Debugf("resolvePuppetEnvironment(): Skipping branch " + source + "_" + branch + " because " + pf + " does not exist")
Expand All @@ -183,6 +185,7 @@ func resolvePuppetEnvironment(tags bool, outputNameTag string) {
}
} else {
puppetfile := readPuppetfile(pf, sa.PrivateKey, source, branch, sa.ForceForgeVersions, false)
puppetfile.upstreamPuppetfileMatches = pfMatch
puppetfile.workDir = normalizeDir(targetDir)
puppetfile.controlRepoBranch = branch
puppetfile.gitDir = workDir
Expand Down Expand Up @@ -248,6 +251,10 @@ func resolvePuppetfile(allPuppetfiles map[string]Puppetfile) {
for env, pf := range allPuppetfiles {
Debugf("Resolving branch " + env + " of source " + pf.source)
//fmt.Println(pf)
if pf.upstreamPuppetfileMatches && !force {
Debugf("Skipping resolution of branch " + env + " of source " + pf.source + " because Puppetfile content has not changed")
continue
}
for gitName, gitModule := range pf.gitModules {
if len(moduleParam) > 0 {
if gitName != moduleParam {
Expand Down