From 42da147b6586c8998005adae44d8a22a28db8076 Mon Sep 17 00:00:00 2001 From: Mat Sutcliffe Date: Thu, 21 Feb 2019 00:27:26 +0000 Subject: [PATCH] Add CLCACHE_RECACHE feature Just like in ccache, this puts clcache into a "write-only" mode. Objects will be stored in the cache, but objects already in the cache will not be reused. The real compiler is always used. This is useful in CI when you have a release branch that you want to always use the real compiler, and feature branches that you want to reap the benefit of the cache generated from the release branch. In that case you would enable CLCACHE_RECACHE only when building from the release branch. --- README.asciidoc | 4 ++++ clcache/__main__.py | 9 ++++++++- tests/test_integration.py | 22 ++++++++++++++++++++++ 3 files changed, 34 insertions(+), 1 deletion(-) diff --git a/README.asciidoc b/README.asciidoc index a0f4259f..86d93590 100644 --- a/README.asciidoc +++ b/README.asciidoc @@ -65,6 +65,10 @@ CLCACHE_LOG:: CLCACHE_DISABLE:: Setting this variable will disable 'clcache.py' completely. The script will relay all calls to the real compiler. +CLCACHE_RECACHE:: + Setting this variable will prevent clcache from retrieving files stored in its + cache. It will still store object files in the cache (possibly overwriting + those already present) but they will always be rebuilt using the real compiler. CLCACHE_HARDLINK:: If this variable is set, cached object files won't be copied to their final location. Instead, hard links pointing to the cached object files diff --git a/clcache/__main__.py b/clcache/__main__.py index 94af73f1..8433513b 100644 --- a/clcache/__main__.py +++ b/clcache/__main__.py @@ -377,6 +377,8 @@ def cachedObjectName(self, key): return os.path.join(self.cacheEntryDir(key), CompilerArtifactsSection.OBJECT_FILE) def hasEntry(self, key): + if 'CLCACHE_RECACHE' in os.environ: + return False return os.path.exists(self.cacheEntryDir(key)) def setEntry(self, key, artifacts): @@ -395,6 +397,10 @@ def setEntry(self, key, artifacts): if artifacts.stderr != '': setCachedCompilerConsoleOutput(os.path.join(tempEntryDir, CompilerArtifactsSection.STDERR_FILE), artifacts.stderr) + if 'CLCACHE_RECACHE' in os.environ: + if os.path.exists(cacheEntryDir): + rmtree(cacheEntryDir) + size = -1 # Replace the full cache entry atomically os.replace(tempEntryDir, cacheEntryDir) return size @@ -1493,7 +1499,8 @@ def addObjectToCache(stats, cache, cachekey, artifacts): size = cache.setEntry(cachekey, artifacts) if size is None: size = os.path.getsize(artifacts.objectFilePath) - stats.registerCacheEntry(size) + if size >= 0: # negative size means an existing cache entry was overwritten + stats.registerCacheEntry(size) with cache.configuration as cfg: return stats.currentCacheSize() >= cfg.maximumCacheSize() diff --git a/tests/test_integration.py b/tests/test_integration.py index ced806ec..e0b44eaf 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -263,6 +263,28 @@ def testHitsSimple(self): newHits = stats.numCacheHits() self.assertEqual(newHits, oldHits + 1) + @pytest.mark.skipif("CLCACHE_MEMCACHED" in os.environ, + reason="recache on memcached not implemented") + def testRecacheMiss(self): + with cd(os.path.join(ASSETS_DIR, "hits-and-misses")): + cmd = CLCACHE_CMD + ["/nologo", "/EHsc", "/c", 'hit.cpp'] + subprocess.check_call(cmd) # Ensure it has been compiled before + + customEnv = dict(os.environ, CLCACHE_RECACHE="1") + cache = clcache.Cache() + with cache.statistics as stats: + oldHits = stats.numCacheHits() + oldMiss = stats.numCacheMisses() + oldSize = stats.currentCacheSize() + subprocess.check_call(cmd, env=customEnv) # This must miss now + with cache.statistics as stats: + newHits = stats.numCacheHits() + newMiss = stats.numCacheMisses() + newSize = stats.currentCacheSize() + self.assertEqual(newHits, oldHits) + self.assertEqual(newMiss, oldMiss + 1) + self.assertEqual(newSize, oldSize) # No change in size + def testAlternatingHeadersHit(self): with cd(os.path.join(ASSETS_DIR, "hits-and-misses")), tempfile.TemporaryDirectory() as tempDir: cache = clcache.Cache(tempDir)