From e89bca05696c89c1a2c1a06616ec1b4d17e28347 Mon Sep 17 00:00:00 2001 From: Andreas Paul Date: Wed, 21 Jan 2026 17:52:15 +0100 Subject: [PATCH] Optimize: Skip module resolution if Puppetfile matches upstream When updating a control repository, if the Puppetfile in the upstream commit is identical to the one currently deployed, we can skip the expensive module resolution and extraction steps. This change: 1. Calculates SHA256 checksums of the deployed Puppetfile and the upstream Puppetfile content (retrieved via git show). 2. Signals a match via the needSyncEnvs map. 3. Checks this signal in resolvePuppetfile() to bypass module processing unless -force is used. This significantly speeds up deployments where only control repository content (like manifests or hiera data) changes but module dependencies remain static. Fixes #233 --- g10k.go | 25 +++++++++++++------------ git.go | 23 +++++++++++++++++++++++ helper.go | 7 +++++++ puppetfile.go | 7 +++++++ 4 files changed, 50 insertions(+), 12 deletions(-) diff --git a/g10k.go b/g10k.go index 9d1ea4f..87154ed 100755 --- a/g10k.go +++ b/g10k.go @@ -140,18 +140,19 @@ type Source struct { // Puppetfile contains the key value pairs from the Puppetfile type Puppetfile struct { - forgeBaseURL string - forgeCacheTTL time.Duration - forgeModules map[string]ForgeModule - gitModules map[string]GitModule - privateKey string - source string - sourceBranch string - workDir string - gitDir string - gitURL string - moduleDirs []string - controlRepoBranch string + forgeBaseURL string + forgeCacheTTL time.Duration + forgeModules map[string]ForgeModule + gitModules map[string]GitModule + privateKey string + source string + sourceBranch string + workDir string + gitDir string + gitURL string + moduleDirs []string + controlRepoBranch string + upstreamPuppetfileMatches bool } // ForgeModule contains information (Version, Name, Author, md5 checksum, file size of the tar.gz archive, Forge BaseURL if custom) about a Puppetlabs Forge module diff --git a/git.go b/git.go index 356d417..b874e20 100644 --- a/git.go +++ b/git.go @@ -240,7 +240,30 @@ func syncToModuleDir(gitModule GitModule, srcDir string, targetDir string, corre purgeWholeEnvDir = true } else { purgeWholeEnvDir = false + + // calculate the checksum of the deployed Puppetfile + deployedPuppetfile := filepath.Join(targetDir, "Puppetfile") + deployedPuppetfileChecksum := "" + if fileExists(deployedPuppetfile) { + deployedPuppetfileChecksum = getSha256sumFile(deployedPuppetfile) + } + + // calculate the checksum of the upstream Puppetfile + upstreamPuppetfileChecksum := getSha256sumString(executeResult.output) + + if deployedPuppetfileChecksum == upstreamPuppetfileChecksum { + Debugf("Puppetfile checksum match for " + targetDir) + // we can signal to skip module resolution if only the Puppetfile hasn't changed + // but we can't easily return this info from here to resolvePuppetEnvironment + // So we might need to store this state somewhere + mutex.Lock() + Debugf("Setting PuppetfileMatch for env: " + correspondingPuppetEnvironment) + needSyncEnvs[correspondingPuppetEnvironment+":PuppetfileMatch"] = empty + mutex.Unlock() + } + lines := strings.Split(executeResult.output, "\n") + for _, line := range lines { if m := reModuledir.FindStringSubmatch(line); len(m) > 1 { // moduledir CLI parameter override diff --git a/helper.go b/helper.go index f56ea75..9090bcf 100644 --- a/helper.go +++ b/helper.go @@ -272,6 +272,13 @@ func getSha256sumFile(file string) string { return hex.EncodeToString(h.Sum(nil)) } +// getSha256sumString return the SHA256 hash sum of the given string +func getSha256sumString(s string) string { + h := sha256.New() + h.Write([]byte(s)) + return hex.EncodeToString(h.Sum(nil)) +} + // moveFile uses io.Copy to create a copy of the given file https://stackoverflow.com/a/50741908/682847 func moveFile(sourcePath, destPath string, deleteSourceFileToggle bool) error { inputFile, err := os.Open(sourcePath) diff --git a/puppetfile.go b/puppetfile.go index 1b146b9..e600ded 100644 --- a/puppetfile.go +++ b/puppetfile.go @@ -168,6 +168,8 @@ func resolvePuppetEnvironment(tags bool, outputNameTag string) { pf := filepath.Join(targetDir, "Puppetfile") mutex.Lock() allBasedirs[sa.Basedir] = true + // check if the Puppetfile content from the upstream repository matches the one in the deployed environment + _, pfMatch := needSyncEnvs[env+":PuppetfileMatch"] mutex.Unlock() if !fileExists(pf) { Debugf("resolvePuppetEnvironment(): Skipping branch " + source + "_" + branch + " because " + pf + " does not exist") @@ -183,6 +185,7 @@ func resolvePuppetEnvironment(tags bool, outputNameTag string) { } } else { puppetfile := readPuppetfile(pf, sa.PrivateKey, source, branch, sa.ForceForgeVersions, false) + puppetfile.upstreamPuppetfileMatches = pfMatch puppetfile.workDir = normalizeDir(targetDir) puppetfile.controlRepoBranch = branch puppetfile.gitDir = workDir @@ -248,6 +251,10 @@ func resolvePuppetfile(allPuppetfiles map[string]Puppetfile) { for env, pf := range allPuppetfiles { Debugf("Resolving branch " + env + " of source " + pf.source) //fmt.Println(pf) + if pf.upstreamPuppetfileMatches && !force { + Debugf("Skipping resolution of branch " + env + " of source " + pf.source + " because Puppetfile content has not changed") + continue + } for gitName, gitModule := range pf.gitModules { if len(moduleParam) > 0 { if gitName != moduleParam {