Hdf5 handler

.travis.yml

@@ -1,8 +1,8 @@
 language: python
-python:
-  - "2.7"
+env:
+  - TOXENV=py27
+  - TOXENV=py33
 install:
-  - pip install .
+  - pip install tox
 script:
-  - nosetests
-  - pyqi
+  - tox

ChangeLog.md

@@ -4,6 +4,8 @@ pyqi ChangeLog
 pyqi 0.3.1-dev
 --------------
 
+* added an HDF5 implicit dataset extender
+* native python 3 support
 * painless profiling: just set the environment variable PYQI_PROFILE_COMMAND
 
 pyqi 0.3.1

doc/index.rst

@@ -6,7 +6,7 @@ What is pyqi?
 
 pyqi (canonically pronounced *pie chee*) is a Python framework designed to support wrapping general *commands* in multiple types of *interfaces*, including at the command line, HTML, and API levels. 
 
-pyqi's only requirement is a working Python 2.7 installation.
+pyqi's only requirement is a working Python 2.7 or 3.3 installation.
 
 Why should I care?
 ------------------

pyqi/commands/make_optparse.py

@@ -10,7 +10,7 @@
 
 from __future__ import division
 from operator import attrgetter
-from pyqi.core.command import (Command, CommandIn, CommandOut, 
+from pyqi.core.command import (Command, CommandIn, CommandOut,
     ParameterCollection)
 from pyqi.commands.code_header_generator import CodeHeaderGenerator
 
@@ -81,7 +81,7 @@
     #                # value will be made available to Handler. This name
     #                # can be either an underscored or dashed version of the
     #                # option name (e.g., 'output_fp' or 'output-fp')
-    #                InputName='output-fp'), 
+    #                InputName='output-fp'),
     #
     # An example option that does not map to a CommandIn.
     # OptparseResult(Parameter=cmd_out_lookup('some_other_result'),
@@ -112,7 +112,7 @@
 class MakeOptparse(CodeHeaderGenerator):
     BriefDescription = "Consume a Command, stub out an optparse configuration"
     LongDescription = """Construct and stub out the basic optparse configuration for a given Command. This template provides comments and examples of what to fill in."""
-    
+
     CommandIns = ParameterCollection(
         CodeHeaderGenerator.CommandIns.Parameters + [
         CommandIn(Name='command', DataType=Command,
@@ -155,8 +155,9 @@ def run(self, **kwargs):
                 action = 'store'
                 data_type = cmdin.DataType
 
-            fmt = {'name':cmdin.Name, 'datatype':data_type, 'action':action,
-                   'required':str(cmdin.Required),
+            fmt = {'name':cmdin.Name,
+                   'datatype':getattr(data_type, '__name__', None),
+                   'action':action, 'required':str(cmdin.Required),
                    'help':cmdin.Description, 'default_block':default_block}
             cmdin_formatted.append(input_format % fmt)
 

pyqi/core/command.py

@@ -135,9 +135,9 @@ def __call__(self, **kwargs):
 
         try:
             result = self.run(**kwargs)
-        except Exception:
+        except Exception as e:
             self._logger.fatal('Error executing command: %s' % self_str)
-            raise
+            raise e
         else:
             self._logger.info('Completed command: %s' % self_str)
 

pyqi/core/hdf5.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python
+
+#-----------------------------------------------------------------------------
+# Copyright (c) 2013, The BiPy Development Team.
+#
+# Distributed under the terms of the Modified BSD License.
+#
+# The full license is in the file COPYING.txt, distributed with this software.
+#-----------------------------------------------------------------------------
+from __future__ import division
+
+__credits__ = ["Daniel McDonald"]
+
+try:
+    import h5py
+except ImportError:
+    raise ImportError("h5py is required for functionality in this module")
+
+VLENSTR = h5py.special_dtype(vlen=str)
+
+class AutoExtendHDF5(object):
+    """Allow for implicitly extendable datasets"""
+    def __init__(self, f):
+        self.f = f
+        self._known_datasets = []
+
+    def __del__(self):
+        for n in self._known_datasets:
+            self._finalize(n)
+
+    def create_dataset(self, name, dtype):
+        """Create a tracked dataset that will automatically reshape"""
+        self.f.create_dataset(name, shape=(1,), maxshape=(None,), 
+                              chunks=True, dtype=dtype)
+        self.f[name].attrs['next_item'] = 0 # idx where next item can get written
+        self._known_datasets.append(name)
+
+    def extend(self, name, data, growth_factor=1):
+        """Extend an automatically growable dataset"""
+        n_data_items = len(data)
+        next_item = self.f[name].attrs['next_item']
+
+        # resize as needed
+        if (next_item + n_data_items) >= self.f[name].size:
+            new_size = next_item + n_data_items
+            new_size += int(new_size * growth_factor)
+            self.f[name].resize((new_size,))
+
+        # store the data
+        start = next_item
+        end = next_item + len(data)
+        self.f[name][start:end] = data
+        self.f[name].attrs['next_item'] = end
+
+    def _finalize(self, name):
+        """Resize a dataset to its correct size"""
+        actual_size = self.f[name].attrs.get('next_item', None)
+
+        if actual_size is None:
+            return
+
+        self.f[name].resize((actual_size,))
+        del self.f[name].attrs['next_item']

pyqi/core/interface.py

@@ -13,7 +13,6 @@
 
 import importlib
 from sys import exit, stderr
-from ConfigParser import SafeConfigParser
 from glob import glob
 from os.path import basename, dirname, expanduser, join
 from pyqi.core.exception import IncompetentDeveloperError
@@ -193,13 +192,13 @@ class InterfaceInputOption(InterfaceOption):
         "str": str,
         "int": int,
         "float": float,
-        "long": long,
+        "long": int, # for python 3 compatibility as long is dropped
         "complex": complex,
         "tuple": tuple,
         "dict": dict,
         "list": list,
         "set": set,
-        "unicode": unicode,
+        "unicode": str, # for python 3 compatibility as all strings are unicode
         "frozenset": frozenset
     }
 
@@ -298,9 +297,9 @@ def get_command_config(command_config_module, cmd, exit_on_failure=True):
     try:
         cmd_cfg = importlib.import_module('.'.join([command_config_module,
                                                     python_cmd_name]))
-    except ImportError, e:
+    except ImportError as e:
         error_msg = str(e)
-
+        
         if exit_on_failure:
             stderr.write("Unable to import the command configuration for "
                          "%s:\n" % cmd)

pyqi/core/interfaces/html/__init__.py

@@ -14,7 +14,15 @@
 import os
 import types
 import os.path
-from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
+import sys
+
+from pyqi.util import get_version_string, is_py2
+
+if is_py2():
+    from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
+else:
+    from http.server import BaseHTTPRequestHandler, HTTPServer
+
 from cgi import parse_header, parse_multipart, parse_qs, FieldStorage
 from copy import copy
 from glob import glob
@@ -24,7 +32,6 @@
 from pyqi.core.factory import general_factory
 from pyqi.core.exception import IncompetentDeveloperError
 from pyqi.core.command import Parameter
-from pyqi.util import get_version_string
 
 class HTMLResult(InterfaceOutputOption):
     """Base class for results for an HTML config file"""
@@ -36,7 +43,7 @@ def __init__(self, MIMEType=None, **kwargs):
 
 class HTMLDownload(HTMLResult):
     """Result class for downloading a file from the server"""
-    def __init__(self, FileExtension=None, FilenameLookup=None, DefaultFilename=None, 
+    def __init__(self, FileExtension=None, FilenameLookup=None, DefaultFilename=None,
             MIMEType='application/octet-stream', **kwargs):
         super(HTMLDownload, self).__init__(MIMEType=MIMEType, **kwargs)
         self.FileExtension = FileExtension
@@ -56,7 +63,6 @@ class HTMLInputOption(InterfaceInputOption):
         bool: lambda x: x.value == "True",
         int: lambda x: int(x.value),
         float: lambda x: float(x.value),
-        long: lambda x: long(x.value),
         complex: lambda x: complex(x.value),
         "upload_file": lambda x: x.file,
         "multiple_choice": lambda x: x.value
@@ -76,17 +82,17 @@ def get_html(self, prefix, value=""):
         """Return the HTML needed for user input given a default value"""
         if (not value) and (self.Default is not None):
             value = self.Default
-            
+
         input_name = prefix + self.Name
         string_input = lambda: '<input type="text" name="%s" value="%s"/>' % (input_name, value)
         number_input = lambda: '<input type="number" name="%s" value="%s"/>' % (input_name, value)
 
-        #html input files cannot have default values. 
+        #html input files cannot have default values.
         #If the html interface worked as a data service, this would be possible as submit would be ajax.
         upload_input = lambda: '<input type="file" name="%s" />' % input_name
         mchoice_input = lambda: ''.join(
-            [ ('(%s<input type="radio" name="%s" value="%s" %s/>)' 
-                    % (choice, input_name, choice, 'checked="true"' if value == choice else '')) 
+            [ ('(%s<input type="radio" name="%s" value="%s" %s/>)'
+                    % (choice, input_name, choice, 'checked="true"' if value == choice else ''))
                 for choice in self.Choices ]
         )
 
@@ -96,7 +102,6 @@ def get_html(self, prefix, value=""):
             bool: mchoice_input,
             int: number_input,
             float: number_input,
-            long: number_input,
             complex: string_input,
             "multiple_choice": mchoice_input,
             "upload_file": upload_input
@@ -110,7 +115,7 @@ def get_html(self, prefix, value=""):
                         self.Help,
                        '</td></tr><tr><td>&nbsp;</td></tr>'
                        ])
-   
+
     def _validate_option(self):
         if self.Type not in self._type_handlers:
             raise IncompetentDeveloperError("Unsupported Type in HTMLInputOption: %s" % self.Type)
@@ -130,7 +135,7 @@ def _validate_option(self):
 class HTMLInterface(Interface):
     """An HTML interface"""
     #Relative mapping wasn't working on a collegue's MacBook when pyqi was run outside of it's directory
-    #Until I understand why that was the case and how to fix it, I am putting the style css here. 
+    #Until I understand why that was the case and how to fix it, I am putting the style css here.
     #This is not a permanent solution.
     css_style = '\n'.join([
         'html, body {',
@@ -189,7 +194,7 @@ def __init__(self, input_prefix="pyqi_", **kwargs):
         self._html_input_prefix = input_prefix
         self._html_interface_input = {}
         super(HTMLInterface, self).__init__(**kwargs)
-    
+
     #Override
     def __call__(self, in_, *args, **kwargs):
         self._the_in_validator(in_)
@@ -199,14 +204,14 @@ def __call__(self, in_, *args, **kwargs):
                     'type': 'error',
                     'errors': errors
                 }
-        else:        
+        else:
             cmd_result = self.CmdInstance(**cmd_input)
-            self._the_out_validator(cmd_result)       
+            self._the_out_validator(cmd_result)
             return self._output_handler(cmd_result)
 
     def _validate_inputs_outputs(self, inputs, outputs):
-        super(HTMLInterface, self)._validate_inputs_outputs(inputs, outputs)  
-        
+        super(HTMLInterface, self)._validate_inputs_outputs(inputs, outputs)
+
         if len(outputs) > 1:
             raise IncompetentDeveloperError("There can be only one... output")
 
@@ -241,7 +246,7 @@ def _input_handler(self, in_, *args, **kwargs):
         formatted_input = {}
 
         for key in in_:
-            mod_key = key[ len(self._html_input_prefix): ] 
+            mod_key = key[ len(self._html_input_prefix): ]
             formatted_input[mod_key] = in_[key]
             if not formatted_input[mod_key].value:
                 formatted_input[mod_key] = None
@@ -319,11 +324,11 @@ def _output_handler(self, results):
             if output.InputName is None:
                 handled_results = output.Handler(rk, results[rk])
             else:
-                handled_results = output.Handler(rk, results[rk], 
+                handled_results = output.Handler(rk, results[rk],
                     self._html_interface_input[output.InputName])
         else:
             handled_results = results[rk]
-    
+
         if isinstance(output, HTMLDownload):
             return self._output_download_handler(output, handled_results)
 
@@ -413,7 +418,7 @@ def route(self, path, output_writer):
                 self.send_header('Content-type', 'text/html')
                 self.end_headers()
                 output_writer(self.wfile.write)
-                
+
                 self.wfile.close()
                 self._unrouted = False;
 
@@ -426,7 +431,7 @@ def command_route(self, command):
                 self.send_header('Content-type', 'text/html')
                 self.end_headers()
                 cmd_obj.command_page_writer(self.wfile.write, [], {})
-                
+
                 self.wfile.close()
                 self._unrouted = False
 
@@ -441,14 +446,14 @@ def post_route(self, command, postvars):
                         'type':'error',
                         'errors':[e]
                     }
-                
+
                 if result['type'] == 'error':
                     self.send_response(400)
                     self.send_header('Content-type', 'text/html')
                     self.end_headers()
                     cmd_obj.command_page_writer(self.wfile.write, result['errors'], postvars)
 
-                elif result['type'] == 'page':       
+                elif result['type'] == 'page':
                     self.send_response(200)
                     self.send_header('Content-type', result['mime_type'])
                     self.end_headers()
@@ -460,7 +465,7 @@ def post_route(self, command, postvars):
                     self.send_header('Content-disposition', 'attachment; filename='+result['filename'])
                     self.end_headers()
                     self.wfile.write(result['contents'])
-                    
+
                 self.wfile.close()
                 self._unrouted = False
 
@@ -508,8 +513,8 @@ def do_POST(self):
 def start_server(port, module):
     """Start a server for the HTMLInterface on the specified port"""
     interface_server = HTTPServer(("", port), get_http_handler(module))
-    print "-- Starting server at http://localhost:%d --" % port
-    print "To close the server, type 'ctrl-c' into this window."
+    print("-- Starting server at http://localhost:%d --" % port)
+    print("To close the server, type 'ctrl-c' into this window.")
     try:
         interface_server.serve_forever()
 

pyqi/core/interfaces/optparse/__init__.py

@@ -13,7 +13,6 @@
                "Jose Antonio Navas Molina"]
 
 import os
-import types
 from copy import copy
 from glob import glob
 from os.path import abspath, exists, isdir, isfile, split
@@ -436,7 +435,7 @@ def _check_multiple_choice(self):
             if self.mchoices is None:
                 raise OptionError(
                     "must supply a list of mchoices for type '%s'" % self.type, self)
-            elif type(self.mchoices) not in (types.TupleType, types.ListType):
+            elif type(self.mchoices) not in (tuple, list):
                 raise OptionError(
                     "choices must be a list of strings ('%s' supplied)"
                     % str(type(self.mchoices)).split("'")[1], self)