diff --git a/.gitignore b/.gitignore index 5c77ab2..5288ff5 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,6 @@ develop-eggs parts dist build + +.* +!.gitignore diff --git a/collective/eggproxy/config.py b/collective/eggproxy/config.py index 014868f..207360f 100644 --- a/collective/eggproxy/config.py +++ b/collective/eggproxy/config.py @@ -19,6 +19,7 @@ from ConfigParser import ConfigParser from ConfigParser import NoSectionError +import logging # First try: User-specific config file CONFIG_FILE = os.path.join(os.path.expanduser('~'), 'eggproxy.conf') @@ -30,18 +31,27 @@ if not os.path.exists(CONFIG_FILE): CONFIG_FILE = '/etc/eggproxy.conf' -#print "Using config file", CONFIG_FILE +print "Using config file", CONFIG_FILE config = ConfigParser() config.add_section("eggproxy") config.set("eggproxy", "eggs_directory", "/var/www") +config.set("eggproxy", "create_eggs_directory", "0") config.set("eggproxy", "index", 'http://pypi.python.org/simple') config.set("eggproxy", "update_interval", '24') config.set("eggproxy", "port", '8888') config.set("eggproxy", "always_refresh", '0') config.set("eggproxy", "timeout", '3') +config.set("eggproxy", "logging", "0") if os.path.exists(CONFIG_FILE): config.readfp(open(CONFIG_FILE)) + + if config.getboolean('eggproxy', 'logging'): + logging.basicConfig( + level=logging.DEBUG, + format='%(asctime)-6s: %(name)30s - %(levelname)s - %(message)s', + ) + # Check for old [default] section that fails with python2.6 had thus has # been changed to [eggproxy] in 0.4 try: diff --git a/collective/eggproxy/utils.py b/collective/eggproxy/utils.py index f5ca9c7..fc8c484 100644 --- a/collective/eggproxy/utils.py +++ b/collective/eggproxy/utils.py @@ -33,6 +33,9 @@ from pkg_resources import Requirement from collective.eggproxy.config import config +import logging + +log=logging.getLogger('collective.eggproxy.utils') ALWAYS_REFRESH = config.getboolean('eggproxy', 'always_refresh') EGGS_DIR = config.get("eggproxy", "eggs_directory") @@ -234,8 +237,19 @@ def updateEggFor(self, package_name, eggname, eggs_dir=EGGS_DIR): filename, md5 = egg_info_for_url(dist.location) if filename == eggname: tmp = tempfile.gettempdir() - tmp_location = self.index.download(dist.location, tmp) - shutil.move(tmp_location, file_path) - return + try: + tmp_location = self.index.download(dist.location, tmp) + log.debug('Downloaded %s\n' % dist.location) + + # BUG: + # 2 instances are downloading "traits" package. + # First instance removes the file + # from /tmp with "shutil.move", + # second instance can not find it -> + # "shutil.move" raises exception + shutil.move(tmp_location, file_path) + return + except Exception, err: + log.debug('Error downloading %s: \n\t%s\n' % (dist.location, err)) raise ValueError, "Egg '%s' not found in index" % eggname diff --git a/collective/eggproxy/wsgi.py b/collective/eggproxy/wsgi.py index 76f7122..5184a63 100644 --- a/collective/eggproxy/wsgi.py +++ b/collective/eggproxy/wsgi.py @@ -27,6 +27,9 @@ from collective.eggproxy import IndexProxy from collective.eggproxy import PackageNotFound from collective.eggproxy.config import config +import logging + +logger = logging.getLogger(__name__) ALWAYS_REFRESH = config.getboolean('eggproxy', 'always_refresh') if ALWAYS_REFRESH: @@ -46,8 +49,13 @@ def __init__(self, index_url=None, eggs_dir=None): if not eggs_dir: eggs_dir = config.get('eggproxy', 'eggs_directory') if not os.path.isdir(eggs_dir): - print 'You must create the %r directory' % eggs_dir - sys.exit() + create_eggs_directory = config.getboolean('eggproxy', 'create_eggs_directory') + if create_eggs_directory: + print 'Creating the %r directory' % eggs_dir + os.makedirs(eggs_dir) + else: + print 'You must create the %r directory' % eggs_dir + sys.exit() self.eggs_index_proxy = IndexProxy(PackageIndex(index_url=index_url)) self.eggs_dir = eggs_dir @@ -106,12 +114,24 @@ def checkPackageIndex(self, package_name): def checkEggFor(self, package_name, eggname): filename = os.path.join(self.eggs_dir, package_name, eggname) + logger.debug('Asking for package:%s egg:%s' % (package_name, eggname)) if not os.path.exists(filename): + logger.debug('Not in cache, start downloading..') try: self.eggs_index_proxy.updateEggFor(package_name, eggname, eggs_dir=self.eggs_dir) - except ValueError: + except ValueError as e: + logger.debug('Download error:%s' % (e)) + if not os.path.exists(filename): + open(filename, 'w').write('') + logger.debug('Writing empty file at:%s' % (filename)) return None + else: + logger.debug('Found in cache:%s' % (filename)) + + if os.path.getsize(filename) == 0: + return None + return filename @@ -141,7 +161,15 @@ def standalone(): port = config.get('eggproxy', 'port') # 0.2.0 way of starting the httpserver, but using the config'ed port # number instead of a hardcoded 8888. - httpserver.serve(EggProxyApp(), host='127.0.0.1', port=port) + httpserver.serve(EggProxyApp(), + host='127.0.0.1', + port=port, + socket_timeout=5*60, + use_threadpool=False, + threadpool_workers=10, + threadpool_options=None, + request_queue_size=5 + ) # Post-0.2.0 way of starting the server using hardcoded config by means of # the package-internal .ini file. This does not allow starting it on a # different port, so I [reinout] commented it out for now. @@ -149,3 +177,7 @@ def standalone(): #this_dir = os.path.dirname(__file__) #sys.argv.extend(['serve', os.path.join(this_dir, 'wsgi.ini')]) #paste.script.command.run() + + +if __name__ == '__main__': + standalone()