From 4e807e7f281ab963c163375d5df1f6eb855ff779 Mon Sep 17 00:00:00 2001 From: Hubert Hesse Date: Tue, 23 May 2017 23:18:54 +0200 Subject: [PATCH 1/3] Move cache to storage module --- clcache.py | 156 +-------------------------------------------------- storage.py | 160 ++++++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 159 insertions(+), 157 deletions(-) diff --git a/clcache.py b/clcache.py index 530fdb04..55e0cc5b 100644 --- a/clcache.py +++ b/clcache.py @@ -12,7 +12,6 @@ import cProfile import codecs import concurrent.futures -import contextlib import errno import hashlib import json @@ -25,6 +24,7 @@ import sys import threading from tempfile import TemporaryFile +from storage import Cache VERSION = "4.1.0-dev" @@ -199,18 +199,6 @@ def getManifest(self, manifestHash): return None -@contextlib.contextmanager -def allSectionsLocked(repository): - sections = list(repository.sections()) - for section in sections: - section.lock.acquire() - try: - yield - finally: - for section in sections: - section.lock.release() - - class ManifestRepository(object): # Bump this counter whenever the current manifest file format changes. # E.g. changing the file format from {'oldkey': ...} to {'newkey': ...} requires @@ -479,148 +467,6 @@ def _normalizedCommandLine(cmdline): return [arg for arg in cmdline if not (arg[0] in "/-" and arg[1:].startswith(argsToStrip))] -class CacheFileStrategy(object): - def __init__(self, cacheDirectory=None): - self.dir = cacheDirectory - if not self.dir: - try: - self.dir = os.environ["CLCACHE_DIR"] - except KeyError: - self.dir = os.path.join(os.path.expanduser("~"), "clcache") - - manifestsRootDir = os.path.join(self.dir, "manifests") - ensureDirectoryExists(manifestsRootDir) - self.manifestRepository = ManifestRepository(manifestsRootDir) - - compilerArtifactsRootDir = os.path.join(self.dir, "objects") - ensureDirectoryExists(compilerArtifactsRootDir) - self.compilerArtifactsRepository = CompilerArtifactsRepository(compilerArtifactsRootDir) - - self.configuration = Configuration(os.path.join(self.dir, "config.txt")) - self.statistics = Statistics(os.path.join(self.dir, "stats.txt")) - - def __str__(self): - return "Disk cache at {}".format(self.dir) - - @property - @contextlib.contextmanager - def lock(self): - with allSectionsLocked(self.manifestRepository), \ - allSectionsLocked(self.compilerArtifactsRepository), \ - self.statistics.lock: - yield - - def lockFor(self, key): - assert isinstance(self.compilerArtifactsRepository.section(key).lock, CacheLock) - return self.compilerArtifactsRepository.section(key).lock - - def manifestLockFor(self, key): - return self.manifestRepository.section(key).lock - - def getEntry(self, key): - return self.compilerArtifactsRepository.section(key).getEntry(key) - - def setEntry(self, key, value): - self.compilerArtifactsRepository.section(key).setEntry(key, value) - - def pathForObject(self, key): - return self.compilerArtifactsRepository.section(key).cachedObjectName(key) - - def directoryForCache(self, key): - return self.compilerArtifactsRepository.section(key).cacheEntryDir(key) - - def deserializeCacheEntry(self, key, objectData): - path = self.pathForObject(key) - ensureDirectoryExists(self.directoryForCache(key)) - with open(path, 'wb') as f: - f.write(objectData) - return path - - def hasEntry(self, cachekey): - return self.compilerArtifactsRepository.section(cachekey).hasEntry(cachekey) - - def setManifest(self, manifestHash, manifest): - self.manifestRepository.section(manifestHash).setManifest(manifestHash, manifest) - - def getManifest(self, manifestHash): - return self.manifestRepository.section(manifestHash).getManifest(manifestHash) - - def clean(self, stats, maximumSize): - currentSize = stats.currentCacheSize() - if currentSize < maximumSize: - return - - # Free at least 10% to avoid cleaning up too often which - # is a big performance hit with large caches. - effectiveMaximumSizeOverall = maximumSize * 0.9 - - # Split limit in manifests (10 %) and objects (90 %) - effectiveMaximumSizeManifests = effectiveMaximumSizeOverall * 0.1 - effectiveMaximumSizeObjects = effectiveMaximumSizeOverall - effectiveMaximumSizeManifests - - # Clean manifests - currentSizeManifests = self.manifestRepository.clean(effectiveMaximumSizeManifests) - - # Clean artifacts - currentCompilerArtifactsCount, currentCompilerArtifactsSize = self.compilerArtifactsRepository.clean( - effectiveMaximumSizeObjects) - - stats.setCacheSize(currentCompilerArtifactsSize + currentSizeManifests) - stats.setNumCacheEntries(currentCompilerArtifactsCount) - - -class Cache(object): - def __init__(self, cacheDirectory=None): - if os.environ.get("CLCACHE_MEMCACHED"): - from storage import CacheFileWithMemcacheFallbackStrategy - self.strategy = CacheFileWithMemcacheFallbackStrategy(os.environ.get("CLCACHE_MEMCACHED"), - cacheDirectory=cacheDirectory) - else: - self.strategy = CacheFileStrategy(cacheDirectory=cacheDirectory) - - def __str__(self): - return str(self.strategy) - - @property - def lock(self): - return self.strategy.lock - - @contextlib.contextmanager - def manifestLockFor(self, key): - with self.strategy.manifestLockFor(key): - yield - - @property - def configuration(self): - return self.strategy.configuration - - @property - def statistics(self): - return self.strategy.statistics - - def clean(self, stats, maximumSize): - return self.strategy.clean(stats, maximumSize) - - @contextlib.contextmanager - def lockFor(self, key): - with self.strategy.lockFor(key): - yield - - def getEntry(self, key): - return self.strategy.getEntry(key) - - def setEntry(self, key, value): - self.strategy.setEntry(key, value) - - def hasEntry(self, cachekey): - return self.strategy.hasEntry(cachekey) - - def setManifest(self, manifestHash, manifest): - self.strategy.setManifest(manifestHash, manifest) - - def getManifest(self, manifestHash): - return self.strategy.getManifest(manifestHash) - class PersistentJSONDict(object): def __init__(self, fileName): diff --git a/storage.py b/storage.py index d98d8c9a..7651aeca 100644 --- a/storage.py +++ b/storage.py @@ -1,11 +1,167 @@ import contextlib +import os from pymemcache.client.base import Client from pymemcache.serde import (python_memcache_serializer, python_memcache_deserializer) -from clcache import CacheFileStrategy, getStringHash, printTraceStatement, CompilerArtifacts, \ - CACHE_COMPILER_OUTPUT_STORAGE_CODEC +from clcache import CacheLock, CompilerArtifacts, CompilerArtifactsRepository, Configuration, ManifestRepository, \ + Statistics, getStringHash, printTraceStatement, ensureDirectoryExists, CACHE_COMPILER_OUTPUT_STORAGE_CODEC + + +@contextlib.contextmanager +def allSectionsLocked(repository): + sections = list(repository.sections()) + for section in sections: + section.lock.acquire() + try: + yield + finally: + for section in sections: + section.lock.release() + + +class Cache(object): + def __init__(self, cacheDirectory=None): + if os.environ.get("CLCACHE_MEMCACHED"): + from storage import CacheFileWithMemcacheFallbackStrategy + self.strategy = CacheFileWithMemcacheFallbackStrategy(os.environ.get("CLCACHE_MEMCACHED"), + cacheDirectory=cacheDirectory) + else: + self.strategy = CacheFileStrategy(cacheDirectory=cacheDirectory) + + def __str__(self): + return str(self.strategy) + + @property + def lock(self): + return self.strategy.lock + + @contextlib.contextmanager + def manifestLockFor(self, key): + with self.strategy.manifestLockFor(key): + yield + + @property + def configuration(self): + return self.strategy.configuration + + @property + def statistics(self): + return self.strategy.statistics + + def clean(self, stats, maximumSize): + return self.strategy.clean(stats, maximumSize) + + @contextlib.contextmanager + def lockFor(self, key): + with self.strategy.lockFor(key): + yield + + def getEntry(self, key): + return self.strategy.getEntry(key) + + def setEntry(self, key, value): + self.strategy.setEntry(key, value) + + def hasEntry(self, cachekey): + return self.strategy.hasEntry(cachekey) + + def setManifest(self, manifestHash, manifest): + self.strategy.setManifest(manifestHash, manifest) + + def getManifest(self, manifestHash): + return self.strategy.getManifest(manifestHash) + + +class CacheFileStrategy(object): + def __init__(self, cacheDirectory=None): + self.dir = cacheDirectory + if not self.dir: + try: + self.dir = os.environ["CLCACHE_DIR"] + except KeyError: + self.dir = os.path.join(os.path.expanduser("~"), "clcache") + + manifestsRootDir = os.path.join(self.dir, "manifests") + ensureDirectoryExists(manifestsRootDir) + self.manifestRepository = ManifestRepository(manifestsRootDir) + + compilerArtifactsRootDir = os.path.join(self.dir, "objects") + ensureDirectoryExists(compilerArtifactsRootDir) + self.compilerArtifactsRepository = CompilerArtifactsRepository(compilerArtifactsRootDir) + + self.configuration = Configuration(os.path.join(self.dir, "config.txt")) + self.statistics = Statistics(os.path.join(self.dir, "stats.txt")) + + def __str__(self): + return "Disk cache at {}".format(self.dir) + + @property + @contextlib.contextmanager + def lock(self): + with allSectionsLocked(self.manifestRepository), \ + allSectionsLocked(self.compilerArtifactsRepository), \ + self.statistics.lock: + yield + + def lockFor(self, key): + assert isinstance(self.compilerArtifactsRepository.section(key).lock, CacheLock) + return self.compilerArtifactsRepository.section(key).lock + + def manifestLockFor(self, key): + return self.manifestRepository.section(key).lock + + def getEntry(self, key): + return self.compilerArtifactsRepository.section(key).getEntry(key) + + def setEntry(self, key, value): + self.compilerArtifactsRepository.section(key).setEntry(key, value) + + def pathForObject(self, key): + return self.compilerArtifactsRepository.section(key).cachedObjectName(key) + + def directoryForCache(self, key): + return self.compilerArtifactsRepository.section(key).cacheEntryDir(key) + + def deserializeCacheEntry(self, key, objectData): + path = self.pathForObject(key) + ensureDirectoryExists(self.directoryForCache(key)) + with open(path, 'wb') as f: + f.write(objectData) + return path + + def hasEntry(self, cachekey): + return self.compilerArtifactsRepository.section(cachekey).hasEntry(cachekey) + + def setManifest(self, manifestHash, manifest): + self.manifestRepository.section(manifestHash).setManifest(manifestHash, manifest) + + def getManifest(self, manifestHash): + return self.manifestRepository.section(manifestHash).getManifest(manifestHash) + + def clean(self, stats, maximumSize): + currentSize = stats.currentCacheSize() + if currentSize < maximumSize: + return + + # Free at least 10% to avoid cleaning up too often which + # is a big performance hit with large caches. + effectiveMaximumSizeOverall = maximumSize * 0.9 + + # Split limit in manifests (10 %) and objects (90 %) + effectiveMaximumSizeManifests = effectiveMaximumSizeOverall * 0.1 + effectiveMaximumSizeObjects = effectiveMaximumSizeOverall - effectiveMaximumSizeManifests + + # Clean manifests + currentSizeManifests = self.manifestRepository.clean(effectiveMaximumSizeManifests) + + # Clean artifacts + currentCompilerArtifactsCount, currentCompilerArtifactsSize = self.compilerArtifactsRepository.clean( + effectiveMaximumSizeObjects) + + stats.setCacheSize(currentCompilerArtifactsSize + currentSizeManifests) + stats.setNumCacheEntries(currentCompilerArtifactsCount) class CacheDummyLock(object): From 625a3e0b2574acac87dff4a170a6a953ea0ab81a Mon Sep 17 00:00:00 2001 From: Hubert Hesse Date: Tue, 23 May 2017 23:25:05 +0200 Subject: [PATCH 2/3] Move Configuration class to storage module --- clcache.py | 25 ------------------------- storage.py | 28 ++++++++++++++++++++++++++-- unittests.py | 3 +-- 3 files changed, 27 insertions(+), 29 deletions(-) diff --git a/clcache.py b/clcache.py index 55e0cc5b..6f7defd8 100644 --- a/clcache.py +++ b/clcache.py @@ -500,31 +500,6 @@ def __eq__(self, other): return type(self) is type(other) and self.__dict__ == other.__dict__ -class Configuration(object): - _defaultValues = {"MaximumCacheSize": 1073741824} # 1 GiB - - def __init__(self, configurationFile): - self._configurationFile = configurationFile - self._cfg = None - - def __enter__(self): - self._cfg = PersistentJSONDict(self._configurationFile) - for setting, defaultValue in self._defaultValues.items(): - if setting not in self._cfg: - self._cfg[setting] = defaultValue - return self - - def __exit__(self, typ, value, traceback): - # Does not write to disc when unchanged - self._cfg.save() - - def maximumCacheSize(self): - return self._cfg["MaximumCacheSize"] - - def setMaximumCacheSize(self, size): - self._cfg["MaximumCacheSize"] = size - - class Statistics(object): CALLS_WITH_INVALID_ARGUMENT = "CallsWithInvalidArgument" CALLS_WITHOUT_SOURCE_FILE = "CallsWithoutSourceFile" diff --git a/storage.py b/storage.py index 7651aeca..1fd0bec9 100644 --- a/storage.py +++ b/storage.py @@ -5,7 +5,7 @@ from pymemcache.serde import (python_memcache_serializer, python_memcache_deserializer) -from clcache import CacheLock, CompilerArtifacts, CompilerArtifactsRepository, Configuration, ManifestRepository, \ +from clcache import CacheLock, CompilerArtifacts, CompilerArtifactsRepository, PersistentJSONDict, ManifestRepository, \ Statistics, getStringHash, printTraceStatement, ensureDirectoryExists, CACHE_COMPILER_OUTPUT_STORAGE_CODEC @@ -21,10 +21,34 @@ def allSectionsLocked(repository): section.lock.release() +class Configuration(object): + _defaultValues = {"MaximumCacheSize": 1073741824} # 1 GiB + + def __init__(self, configurationFile): + self._configurationFile = configurationFile + self._cfg = None + + def __enter__(self): + self._cfg = PersistentJSONDict(self._configurationFile) + for setting, defaultValue in self._defaultValues.items(): + if setting not in self._cfg: + self._cfg[setting] = defaultValue + return self + + def __exit__(self, typ, value, traceback): + # Does not write to disc when unchanged + self._cfg.save() + + def maximumCacheSize(self): + return self._cfg["MaximumCacheSize"] + + def setMaximumCacheSize(self, size): + self._cfg["MaximumCacheSize"] = size + + class Cache(object): def __init__(self, cacheDirectory=None): if os.environ.get("CLCACHE_MEMCACHED"): - from storage import CacheFileWithMemcacheFallbackStrategy self.strategy = CacheFileWithMemcacheFallbackStrategy(os.environ.get("CLCACHE_MEMCACHED"), cacheDirectory=cacheDirectory) else: diff --git a/unittests.py b/unittests.py index 05190df1..170be90d 100644 --- a/unittests.py +++ b/unittests.py @@ -19,7 +19,6 @@ from clcache import ( CommandLineAnalyzer, CompilerArtifactsRepository, - Configuration, Manifest, ManifestEntry, ManifestRepository, @@ -34,7 +33,7 @@ NoSourceFileError, PersistentJSONDict, ) -from storage import CacheMemcacheStrategy +from storage import CacheMemcacheStrategy, Configuration ASSETS_DIR = os.path.join("tests", "unittests") From 322e7622203238e395bfcd73108d74707cec999b Mon Sep 17 00:00:00 2001 From: Hubert Hesse Date: Tue, 23 May 2017 23:36:41 +0200 Subject: [PATCH 3/3] Rename cache to storage --- clcache.py | 6 +++--- integrationtests.py | 46 ++++++++++++++++++++++----------------------- performancetests.py | 2 +- storage.py | 28 +++++++++++++-------------- unittests.py | 28 +++++++++++++-------------- 5 files changed, 55 insertions(+), 55 deletions(-) diff --git a/clcache.py b/clcache.py index 6f7defd8..13ca8306 100644 --- a/clcache.py +++ b/clcache.py @@ -24,7 +24,7 @@ import sys import threading from tempfile import TemporaryFile -from storage import Cache +from storage import Storage VERSION = "4.1.0-dev" @@ -1326,7 +1326,7 @@ def main(): """.strip().format(VERSION)) return 0 - cache = Cache() + cache = Storage() if len(sys.argv) == 2 and sys.argv[1] == "-s": with cache.lock: @@ -1467,7 +1467,7 @@ def scheduleJobs(cache, compiler, cmdLine, environment, sourceFiles, objectFiles def processSingleSource(compiler, cmdLine, sourceFile, objectFile, environment): try: assert objectFile is not None - cache = Cache() + cache = Storage() if 'CLCACHE_NODIRECT' in os.environ: return processNoDirect(cache, objectFile, compiler, cmdLine, environment) diff --git a/integrationtests.py b/integrationtests.py index 33400443..480e412d 100644 --- a/integrationtests.py +++ b/integrationtests.py @@ -235,7 +235,7 @@ def testHitsSimple(self): cmd = CLCACHE_CMD + ["/nologo", "/EHsc", "/c", 'hit.cpp'] subprocess.check_call(cmd) # Ensure it has been compiled before - cache = clcache.Cache() + cache = clcache.Storage() with cache.statistics as stats: oldHits = stats.numCacheHits() subprocess.check_call(cmd) # This must hit now @@ -245,7 +245,7 @@ def testHitsSimple(self): def testAlternatingHeadersHit(self): with cd(os.path.join(ASSETS_DIR, "hits-and-misses")), tempfile.TemporaryDirectory() as tempDir: - cache = clcache.Cache(tempDir) + cache = clcache.Storage(tempDir) customEnv = dict(os.environ, CLCACHE_DIR=tempDir) baseCmd = CLCACHE_CMD + ["/nologo", "/EHsc", "/c"] @@ -296,7 +296,7 @@ def testAlternatingHeadersHit(self): def testRemovedHeader(self): with cd(os.path.join(ASSETS_DIR, "hits-and-misses")), tempfile.TemporaryDirectory() as tempDir: - cache = clcache.Cache(tempDir) + cache = clcache.Storage(tempDir) customEnv = dict(os.environ, CLCACHE_DIR=tempDir) baseCmd = CLCACHE_CMD + ["/nologo", "/EHsc", "/c"] @@ -347,7 +347,7 @@ def testRemovedHeader(self): def testAlternatingTransitiveHeader(self): with cd(os.path.join(ASSETS_DIR, "hits-and-misses")), tempfile.TemporaryDirectory() as tempDir: - cache = clcache.Cache(tempDir) + cache = clcache.Storage(tempDir) customEnv = dict(os.environ, CLCACHE_DIR=tempDir) baseCmd = CLCACHE_CMD + ["/nologo", "/EHsc", "/c"] @@ -398,7 +398,7 @@ def testAlternatingTransitiveHeader(self): def testRemovedTransitiveHeader(self): with cd(os.path.join(ASSETS_DIR, "hits-and-misses")), tempfile.TemporaryDirectory() as tempDir: - cache = clcache.Cache(tempDir) + cache = clcache.Storage(tempDir) customEnv = dict(os.environ, CLCACHE_DIR=tempDir) baseCmd = CLCACHE_CMD + ["/nologo", "/EHsc", "/c"] @@ -449,7 +449,7 @@ def testRemovedTransitiveHeader(self): def testAlternatingIncludeOrder(self): with cd(os.path.join(ASSETS_DIR, "hits-and-misses")), tempfile.TemporaryDirectory() as tempDir: - cache = clcache.Cache(tempDir) + cache = clcache.Storage(tempDir) customEnv = dict(os.environ, CLCACHE_DIR=tempDir) baseCmd = CLCACHE_CMD + ["/nologo", "/EHsc", "/c"] @@ -509,7 +509,7 @@ def testAlternatingIncludeOrder(self): def testRepeatedIncludes(self): with cd(os.path.join(ASSETS_DIR, "hits-and-misses")), tempfile.TemporaryDirectory() as tempDir: - cache = clcache.Cache(tempDir) + cache = clcache.Storage(tempDir) customEnv = dict(os.environ, CLCACHE_DIR=tempDir) baseCmd = CLCACHE_CMD + ["/nologo", "/EHsc", "/c"] @@ -660,7 +660,7 @@ def testObsoleteHeaderDisappears(self): with cd(os.path.join(ASSETS_DIR, "header-miss-obsolete")), tempfile.TemporaryDirectory() as tempDir: customEnv = dict(os.environ, CLCACHE_DIR=tempDir) compileCmd = CLCACHE_CMD + ["/I.", "/nologo", "/EHsc", "/c", "main.cpp"] - cache = clcache.Cache(tempDir) + cache = clcache.Storage(tempDir) with open("A.h", "w") as header: header.write('#define INFO 1337\n') @@ -731,7 +731,7 @@ def testParallel(self): # Compile first time self._buildAll() - cache = clcache.Cache() + cache = clcache.Storage() with cache.statistics as stats: hits = stats.numCacheHits() misses = stats.numCacheMisses() @@ -740,7 +740,7 @@ def testParallel(self): # Compile second time self._buildAll() - cache = clcache.Cache() + cache = clcache.Storage() with cache.statistics as stats: hits = stats.numCacheHits() misses = stats.numCacheMisses() @@ -748,7 +748,7 @@ def testParallel(self): def testHitViaMpSequential(self): with cd(os.path.join(ASSETS_DIR, "parallel")), tempfile.TemporaryDirectory() as tempDir: - cache = clcache.Cache(tempDir) + cache = clcache.Storage(tempDir) customEnv = self._createEnv(tempDir) @@ -777,7 +777,7 @@ def testHitViaMpSequential(self): def testHitsViaMpConcurrent(self): with cd(os.path.join(ASSETS_DIR, "parallel")), tempfile.TemporaryDirectory() as tempDir: - cache = clcache.Cache(tempDir) + cache = clcache.Storage(tempDir) customEnv = self._createEnv(tempDir) @@ -808,7 +808,7 @@ def testHitsViaMpConcurrent(self): def testOutput(self): with cd(os.path.join(ASSETS_DIR, "parallel")), tempfile.TemporaryDirectory() as tempDir: sources = glob.glob("*.cpp") - clcache.Cache(tempDir) + clcache.Storage(tempDir) customEnv = self._createEnv(tempDir) cmd = CLCACHE_CMD + ["/nologo", "/EHsc", "/c"] mpFlag = "/MP" + str(len(sources)) @@ -825,7 +825,7 @@ class TestRunParallel(RunParallelBase, unittest.TestCase): class TestMultipleSources(unittest.TestCase): def testTwo(self): with cd(os.path.join(ASSETS_DIR, "mutiple-sources")), tempfile.TemporaryDirectory() as tempDir: - cache = clcache.Cache(tempDir) + cache = clcache.Storage(tempDir) customEnv = dict(os.environ, CLCACHE_DIR=tempDir) baseCmd = CLCACHE_CMD + ["/nologo", "/EHsc", "/c"] @@ -850,7 +850,7 @@ def testTwo(self): def testFive(self): with cd(os.path.join(ASSETS_DIR, "mutiple-sources")), tempfile.TemporaryDirectory() as tempDir: - cache = clcache.Cache(tempDir) + cache = clcache.Storage(tempDir) customEnv = dict(os.environ, CLCACHE_DIR=tempDir) baseCmd = CLCACHE_CMD + ["/nologo", "/EHsc", "/c"] @@ -898,7 +898,7 @@ def _clearCache(self): subprocess.check_call(CLCACHE_CMD + ["-C"]) def testClearIdempotency(self): - cache = clcache.Cache() + cache = clcache.Storage() self._clearCache() with cache.statistics as stats: @@ -914,7 +914,7 @@ def testClearIdempotency(self): @pytest.mark.skipif("CLCACHE_MEMCACHED" in os.environ, reason="clearing on memcached not implemented") def testClearPostcondition(self): - cache = clcache.Cache() + cache = clcache.Storage() # Compile a random file to populate cache cmd = CLCACHE_CMD + ["/nologo", "/EHsc", "/c", os.path.join(ASSETS_DIR, "fibonacci.cpp")] @@ -989,7 +989,7 @@ def testHitsSimple(self): ["/nologo", "/E", "/EP"], ] - cache = clcache.Cache() + cache = clcache.Storage() with cache.statistics as stats: oldPreprocessorCalls = stats.numCallsForPreprocessing() @@ -1005,7 +1005,7 @@ class TestNoDirectCalls(RunParallelBase, unittest.TestCase): env = dict(os.environ, CLCACHE_NODIRECT="1") def testPreprocessorFailure(self): - cache = clcache.Cache() + cache = clcache.Storage() oldStats = copy.copy(cache.statistics) cmd = CLCACHE_CMD + ["/nologo", "/c", "doesnotexist.cpp"] @@ -1019,7 +1019,7 @@ def testHit(self): self.assertEqual(subprocess.call(cmd, env=self.env), 0) - cache = clcache.Cache() + cache = clcache.Storage() with cache.statistics as stats: oldHits = stats.numCacheHits() @@ -1040,7 +1040,7 @@ def setUp(self): for buildDir in ["builddir_a", "builddir_b"]: shutil.copytree(self.projectDir, buildDir) - self.cache = clcache.Cache(self.clcacheDir) + self.cache = clcache.Storage(self.clcacheDir) def tearDown(self): os.chdir(self.savedCwd) @@ -1139,7 +1139,7 @@ def testEvictedObject(self): subprocess.check_call(cmd, env=customEnv) # Remove object - cache = clcache.Cache(tempDir) + cache = clcache.Storage(tempDir) clcache.cleanCache(cache) self.assertEqual(subprocess.call(cmd, env=customEnv), 0) @@ -1153,7 +1153,7 @@ def testEvictedManifest(self): subprocess.check_call(cmd, env=customEnv) # Remove manifest - cache = clcache.Cache(tempDir) + cache = clcache.Storage(tempDir) clcache.clearCache(cache) self.assertEqual(subprocess.call(cmd, env=customEnv), 0) diff --git a/performancetests.py b/performancetests.py index 16d62410..b6a867ae 100644 --- a/performancetests.py +++ b/performancetests.py @@ -53,7 +53,7 @@ def testConcurrentHitsScaling(self): with tempfile.TemporaryDirectory() as tempDir: customEnv = dict(os.environ, CLCACHE_DIR=tempDir) - cache = clcache.Cache(tempDir) + cache = clcache.Storage(tempDir) with cache.statistics as stats: self.assertEqual(stats.numCacheHits(), 0) diff --git a/storage.py b/storage.py index 1fd0bec9..2ffe107c 100644 --- a/storage.py +++ b/storage.py @@ -46,13 +46,13 @@ def setMaximumCacheSize(self, size): self._cfg["MaximumCacheSize"] = size -class Cache(object): +class Storage(object): def __init__(self, cacheDirectory=None): if os.environ.get("CLCACHE_MEMCACHED"): - self.strategy = CacheFileWithMemcacheFallbackStrategy(os.environ.get("CLCACHE_MEMCACHED"), - cacheDirectory=cacheDirectory) + self.strategy = FileAndMemcacheStrategy(os.environ.get("CLCACHE_MEMCACHED"), + cacheDirectory=cacheDirectory) else: - self.strategy = CacheFileStrategy(cacheDirectory=cacheDirectory) + self.strategy = FileStrategy(cacheDirectory=cacheDirectory) def __str__(self): return str(self.strategy) @@ -98,7 +98,7 @@ def getManifest(self, manifestHash): return self.strategy.getManifest(manifestHash) -class CacheFileStrategy(object): +class FileStrategy(object): def __init__(self, cacheDirectory=None): self.dir = cacheDirectory if not self.dir: @@ -196,9 +196,9 @@ def __exit__(self, typ, value, traceback): pass -class CacheMemcacheStrategy(object): +class MemcacheStrategy(object): def __init__(self, server, cacheDirectory=None, manifestPrefix='manifests_', objectPrefix='objects_'): - self.fileStrategy = CacheFileStrategy(cacheDirectory=cacheDirectory) + self.fileStrategy = FileStrategy(cacheDirectory=cacheDirectory) # XX Memcache Strategy should be independent self.lock = CacheDummyLock() @@ -210,7 +210,7 @@ def __init__(self, server, cacheDirectory=None, manifestPrefix='manifests_', obj self.connect(server) def connect(self, server): - server = CacheMemcacheStrategy.splitHosts(server) + server = MemcacheStrategy.splitHosts(server) assert server, "{} is not a suitable server".format(server) if len(server) == 1: clientClass = Client @@ -250,7 +250,7 @@ def splitHosts(hosts): :param hosts: A string in the format of HOST:PORT[,HOST:PORT] :return: a list [(HOST, int(PORT)), ..] of tuples that can be consumed by socket.connect() """ - return [CacheMemcacheStrategy.splitHost(h) for h in hosts.split(',')] + return [MemcacheStrategy.splitHost(h) for h in hosts.split(',')] def __str__(self): return "Remote Memcache @{} object-prefix: {}".format(self.server, self.objectPrefix) @@ -336,12 +336,12 @@ def clean(self, stats, maximumSize): maximumSize) -class CacheFileWithMemcacheFallbackStrategy(object): +class FileAndMemcacheStrategy(object): def __init__(self, server, cacheDirectory=None, manifestPrefix='manifests_', objectPrefix='objects_'): - self.localCache = CacheFileStrategy(cacheDirectory=cacheDirectory) - self.remoteCache = CacheMemcacheStrategy(server, cacheDirectory=cacheDirectory, - manifestPrefix=manifestPrefix, - objectPrefix=objectPrefix) + self.localCache = FileStrategy(cacheDirectory=cacheDirectory) + self.remoteCache = MemcacheStrategy(server, cacheDirectory=cacheDirectory, + manifestPrefix=manifestPrefix, + objectPrefix=objectPrefix) def __str__(self): return "CacheFileWithMemcacheFallbackStrategy local({}) and remote({})".format(self.localCache, diff --git a/unittests.py b/unittests.py index 170be90d..64e5b26b 100644 --- a/unittests.py +++ b/unittests.py @@ -33,7 +33,7 @@ NoSourceFileError, PersistentJSONDict, ) -from storage import CacheMemcacheStrategy, Configuration +from storage import MemcacheStrategy, Configuration ASSETS_DIR = os.path.join("tests", "unittests") @@ -1031,7 +1031,7 @@ def testSetGet(self): from clcache import CompilerArtifacts, getStringHash with tempfile.TemporaryDirectory() as tempDir: - memcache = CacheMemcacheStrategy("localhost", cacheDirectory=tempDir) + memcache = MemcacheStrategy("localhost", cacheDirectory=tempDir) memcache.client = MockMemcacheClient(allow_unicode_keys=True) key = getStringHash("hello") memcache.fileStrategy.lockFor = memcache.lockFor @@ -1059,31 +1059,31 @@ def testSetGet(self): def testArgumentParsing(self): with self.assertRaises(ValueError): - CacheMemcacheStrategy.splitHosts("") + MemcacheStrategy.splitHosts("") - self.assertEqual(CacheMemcacheStrategy.splitHosts("localhost"), [("localhost", 11211)]) - self.assertEqual(CacheMemcacheStrategy.splitHosts("localhost:123"), [("localhost", 123)]) + self.assertEqual(MemcacheStrategy.splitHosts("localhost"), [("localhost", 11211)]) + self.assertEqual(MemcacheStrategy.splitHosts("localhost:123"), [("localhost", 123)]) with self.assertRaises(ValueError): - CacheMemcacheStrategy.splitHosts("localhost:123:") + MemcacheStrategy.splitHosts("localhost:123:") with self.assertRaises(ValueError): - CacheMemcacheStrategy.splitHosts("localhost:123:") + MemcacheStrategy.splitHosts("localhost:123:") - self.assertEqual(CacheMemcacheStrategy.splitHosts("localhost.local, example.local"), + self.assertEqual(MemcacheStrategy.splitHosts("localhost.local, example.local"), [("localhost.local", 11211), ("example.local", 11211)]) - self.assertEqual(CacheMemcacheStrategy.splitHosts("localhost.local:12345, example.local"), + self.assertEqual(MemcacheStrategy.splitHosts("localhost.local:12345, example.local"), [("localhost.local", 12345), ("example.local", 11211)]) with self.assertRaises(ValueError): - CacheMemcacheStrategy.splitHosts("localhost.local:123456") + MemcacheStrategy.splitHosts("localhost.local:123456") with self.assertRaises(ValueError): - CacheMemcacheStrategy.splitHosts("localhost.local:123456, example.local") + MemcacheStrategy.splitHosts("localhost.local:123456, example.local") with self.assertRaises(ValueError): - CacheMemcacheStrategy.splitHosts("localhost.local:12345,") + MemcacheStrategy.splitHosts("localhost.local:12345,") with self.assertRaises(ValueError): - CacheMemcacheStrategy.splitHosts("localhost.local,12345:") + MemcacheStrategy.splitHosts("localhost.local,12345:") with self.assertRaises(ValueError): - CacheMemcacheStrategy.splitHosts("localhost.local;12345:") + MemcacheStrategy.splitHosts("localhost.local;12345:") if __name__ == '__main__':