dead links are not cached

.gitignore

@@ -6,3 +6,6 @@ develop-eggs
 parts
 dist
 build
+
+.*
+!.gitignore

collective/eggproxy/config.py

@@ -19,6 +19,7 @@
 from ConfigParser import ConfigParser
 from ConfigParser import NoSectionError
 
+import logging
 
 # First try: User-specific config file
 CONFIG_FILE = os.path.join(os.path.expanduser('~'), 'eggproxy.conf')
@@ -30,18 +31,27 @@
 if not os.path.exists(CONFIG_FILE):
     CONFIG_FILE = '/etc/eggproxy.conf'
 
-#print "Using config file", CONFIG_FILE
+print "Using config file", CONFIG_FILE
 config = ConfigParser()
 config.add_section("eggproxy")
 config.set("eggproxy", "eggs_directory", "/var/www")
+config.set("eggproxy", "create_eggs_directory", "0")
 config.set("eggproxy", "index", 'http://pypi.python.org/simple')
 config.set("eggproxy", "update_interval", '24')
 config.set("eggproxy", "port", '8888')
 config.set("eggproxy", "always_refresh", '0')
 config.set("eggproxy", "timeout", '3')
+config.set("eggproxy", "logging", "0")
 
 if os.path.exists(CONFIG_FILE):
     config.readfp(open(CONFIG_FILE))
+
+    if config.getboolean('eggproxy', 'logging'):
+        logging.basicConfig(
+            level=logging.DEBUG,
+            format='%(asctime)-6s: %(name)30s - %(levelname)s - %(message)s',
+            )
+
     # Check for old [default] section that fails with python2.6 had thus has
     # been changed to [eggproxy] in 0.4
     try:

collective/eggproxy/utils.py

@@ -33,6 +33,9 @@
 
 from pkg_resources import Requirement
 from collective.eggproxy.config import config
+import logging
+
+log=logging.getLogger('collective.eggproxy.utils')
 
 ALWAYS_REFRESH = config.getboolean('eggproxy', 'always_refresh')
 EGGS_DIR = config.get("eggproxy", "eggs_directory")
@@ -234,8 +237,19 @@ def updateEggFor(self, package_name, eggname, eggs_dir=EGGS_DIR):
             filename, md5 = egg_info_for_url(dist.location)
             if filename == eggname:
                 tmp = tempfile.gettempdir()
-                tmp_location = self.index.download(dist.location, tmp)
-                shutil.move(tmp_location, file_path)
-                return
+                try:
+                    tmp_location = self.index.download(dist.location, tmp)
+                    log.debug('Downloaded %s\n' % dist.location)
+
+                    # BUG:
+                    # 2 instances are downloading "traits" package.
+                    # First instance removes the file
+                    #   from /tmp with "shutil.move",
+                    # second instance can not find it ->
+                    # "shutil.move" raises exception
+                    shutil.move(tmp_location, file_path)
+                    return
+                except Exception, err:
+                    log.debug('Error downloading %s: \n\t%s\n' % (dist.location, err))
 
         raise ValueError, "Egg '%s' not found in index" % eggname

collective/eggproxy/wsgi.py

@@ -27,6 +27,9 @@
 from collective.eggproxy import IndexProxy
 from collective.eggproxy import PackageNotFound
 from collective.eggproxy.config import config
+import logging
+
+logger = logging.getLogger(__name__)
 
 ALWAYS_REFRESH = config.getboolean('eggproxy', 'always_refresh')
 if ALWAYS_REFRESH:
@@ -46,8 +49,13 @@ def __init__(self, index_url=None, eggs_dir=None):
         if not eggs_dir:
             eggs_dir = config.get('eggproxy', 'eggs_directory')
         if not os.path.isdir(eggs_dir):
-            print 'You must create the %r directory' % eggs_dir
-            sys.exit()
+            create_eggs_directory = config.getboolean('eggproxy', 'create_eggs_directory')
+            if create_eggs_directory:
+                print 'Creating the %r directory' % eggs_dir
+                os.makedirs(eggs_dir)
+            else:
+                print 'You must create the %r directory' % eggs_dir
+                sys.exit()
         self.eggs_index_proxy = IndexProxy(PackageIndex(index_url=index_url))
         self.eggs_dir = eggs_dir
 
@@ -106,12 +114,24 @@ def checkPackageIndex(self, package_name):
 
     def checkEggFor(self, package_name, eggname):
         filename = os.path.join(self.eggs_dir, package_name, eggname)
+        logger.debug('Asking for package:%s egg:%s' % (package_name, eggname))
         if not os.path.exists(filename):
+            logger.debug('Not in cache, start downloading..')
             try:
                 self.eggs_index_proxy.updateEggFor(package_name, eggname,
                                                    eggs_dir=self.eggs_dir)
-            except ValueError:
+            except ValueError as e:
+                logger.debug('Download error:%s' % (e))
+                if not os.path.exists(filename):
+                    open(filename, 'w').write('')
+                    logger.debug('Writing empty file at:%s' % (filename))
                 return None
+        else:
+            logger.debug('Found in cache:%s' % (filename))
+
+        if os.path.getsize(filename) == 0:
+            return None
+
         return filename
 
 
@@ -141,11 +161,23 @@ def standalone():
     port = config.get('eggproxy', 'port')
     # 0.2.0 way of starting the httpserver, but using the config'ed port
     # number instead of a hardcoded 8888.
-    httpserver.serve(EggProxyApp(), host='127.0.0.1', port=port)
+    httpserver.serve(EggProxyApp(), 
+                     host='127.0.0.1', 
+                     port=port,
+                    socket_timeout=5*60,
+                    use_threadpool=False, 
+                    threadpool_workers=10,
+                    threadpool_options=None, 
+                    request_queue_size=5
+                    )
     # Post-0.2.0 way of starting the server using hardcoded config by means of
     # the package-internal .ini file. This does not allow starting it on a
     # different port, so I [reinout] commented it out for now.
     #import paste.script.command
     #this_dir = os.path.dirname(__file__)
     #sys.argv.extend(['serve', os.path.join(this_dir, 'wsgi.ini')])
     #paste.script.command.run()
+
+
+if __name__ == '__main__':
+    standalone()