Add msbuild tracker support

Detecting of tracker presence by TRACKER_ENABLED environ.

Move reading of stats.txt and manifests out of main thread to avoid
tracking.

Using ProcessPoolExecutor to make it trackable.

Author: izmmisha

clcache/__main__.py

@@ -163,6 +163,22 @@ def __str__(self):
         return repr(self.message)
 
 
+def isTrackerEnabled():
+    return 'TRACKER_ENABLED' in os.environ
+
+def untrackable(func):
+    if not isTrackerEnabled():
+        return func
+
+    if not hasattr(untrackable, "untrackableExecutor"):
+        setattr(untrackable, "untrackableExecutor", concurrent.futures.ThreadPoolExecutor(max_workers=1))
+
+    def executeNotInMainThread(*args, **kwargs):
+        future = untrackable.untrackableExecutor.submit(func, *args, **kwargs) # pylint: disable=no-member
+        return future.result()
+
+    return executeNotInMainThread
+
 class Manifest:
     def __init__(self, entries=None):
         if entries is None:
@@ -203,6 +219,7 @@ def setManifest(self, manifestHash, manifest):
             jsonobject = {'entries': entries}
             json.dump(jsonobject, outFile, sort_keys=True, indent=2)
 
+    @untrackable
     def getManifest(self, manifestHash):
         fileName = self.manifestPath(manifestHash)
         if not os.path.exists(fileName):
@@ -741,6 +758,7 @@ def __init__(self, statsFile):
         self._stats = None
         self.lock = CacheLock.forPath(self._statsFile)
 
+    @untrackable
     def __enter__(self):
         self._stats = PersistentJSONDict(self._statsFile)
         for k in Statistics.RESETTABLE_KEYS | Statistics.NON_RESETTABLE_KEYS:
@@ -1645,7 +1663,13 @@ def scheduleJobs(cache: Any, compiler: str, cmdLine: List[str], environment: Any
 
     exitCode = 0
     cleanupRequired = False
-    with concurrent.futures.ThreadPoolExecutor(max_workers=jobCount(cmdLine)) as executor:
+
+    if isTrackerEnabled():
+        poolExecutor = concurrent.futures.ProcessPoolExecutor
+    else:
+        poolExecutor = concurrent.futures.ThreadPoolExecutor
+
+    with poolExecutor(max_workers=min(jobCount(cmdLine), len(objectFiles))) as executor:
         jobs = []
         for (srcFile, srcLanguage), objFile in zip(sourceFiles, objectFiles):
             jobCmdLine = baseCmdLine + [srcLanguage + srcFile]
@@ -1770,6 +1794,7 @@ def ensureArtifactsExist(cache, cachekey, reason, objectFile, compilerResult, ex
 
 
 if __name__ == '__main__':
+    multiprocessing.freeze_support()
     if 'CLCACHE_PROFILE' in os.environ:
         INVOCATION_HASH = getStringHash(','.join(sys.argv))
         cProfile.run('main()', filename='clcache-{}.prof'.format(INVOCATION_HASH))