From cb74d613e94282b6d943bbf66392e2fd0d0a3e85 Mon Sep 17 00:00:00 2001 From: Daniel McDonald Date: Sun, 22 Dec 2013 08:46:24 -0700 Subject: [PATCH 1/7] support for automatic resized hdf5 datasets --- pyqi/core/hdf5.py | 63 +++++++++++++++++++++++ tests/test_core/test_hdf5.py | 97 ++++++++++++++++++++++++++++++++++++ 2 files changed, 160 insertions(+) create mode 100644 pyqi/core/hdf5.py create mode 100644 tests/test_core/test_hdf5.py diff --git a/pyqi/core/hdf5.py b/pyqi/core/hdf5.py new file mode 100644 index 0000000..e233046 --- /dev/null +++ b/pyqi/core/hdf5.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python + +#----------------------------------------------------------------------------- +# Copyright (c) 2013, The BiPy Development Team. +# +# Distributed under the terms of the Modified BSD License. +# +# The full license is in the file COPYING.txt, distributed with this software. +#----------------------------------------------------------------------------- +from __future__ import division + +__credits__ = ["Daniel McDonald"] + +try: + import h5py +except ImportError: + raise ImportError("h5py is required for functionality in this module") + +VLENSTR = h5py.special_dtype(vlen=str) + +class AutoExtendHDF5(object): + """Allow for implicitly extendable datasets""" + def __init__(self, f): + self.f = f + self._known_datasets = [] + + def __del__(self): + for n in self._known_datasets: + self._finalize(n) + + def create_dataset(self, name, dtype): + """Create a tracked dataset that will automatically reshape""" + self.f.create_dataset(name, shape=(1,), maxshape=(None,), + chunks=True, dtype=dtype) + self.f[name].attrs['next_item'] = 0 # idx where next item can get written + self._known_datasets.append(name) + + def extend(self, name, data, growth_factor=1): + """Extend an automatically growable dataset""" + n_data_items = len(data) + next_item = self.f[name].attrs['next_item'] + + # resize as needed + if (next_item + n_data_items) >= self.f[name].size: + new_size = next_item + n_data_items + new_size += int(new_size * growth_factor) + self.f[name].resize((new_size,)) + + # store the data + start = next_item + end = next_item + len(data) + self.f[name][start:end] = data + self.f[name].attrs['next_item'] = end + + def _finalize(self, name): + """Resize a dataset to its correct size""" + actual_size = self.f[name].attrs.get('next_item', None) + + if actual_size is None: + return + + self.f[name].resize((actual_size,)) + del self.f[name].attrs['next_item'] diff --git a/tests/test_core/test_hdf5.py b/tests/test_core/test_hdf5.py new file mode 100644 index 0000000..179ce0b --- /dev/null +++ b/tests/test_core/test_hdf5.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python + +#----------------------------------------------------------------------------- +# Copyright (c) 2013, The BiPy Development Team. +# +# Distributed under the terms of the Modified BSD License. +# +# The full license is in the file COPYING.txt, distributed with this software. +#----------------------------------------------------------------------------- +from __future__ import division + +__credits__ = ["Daniel McDonald"] + +from unittest import TestCase, main +from os import remove +from numpy import array, hstack +from pyqi.core.hdf5 import AutoExtendHDF5 +import h5py + +class HDF5AutoExtendTests(TestCase): + def setUp(self): + self.hdf5_file = h5py.File('_test_file.hdf5','w') + self.obj = AutoExtendHDF5(self.hdf5_file) + + def tearDown(self): + remove('_test_file.hdf5') + + def test_create_dataset(self): + """Create something""" + self.obj.create_dataset('test_group/test_ds', int) + self.assertEqual(array([0],int), self.obj.f['test_group/test_ds'][:]) + self.assertEqual(self.obj._known_datasets, ['test_group/test_ds']) + + def test_extend(self): + """Check the implicit resizing""" + name = 'test_group/test_ds' + fetch = lambda x: x[name][:x[name].attrs['next_item']] + size = lambda x: x[name].size + + self.obj.create_dataset(name, int) + ds1 = array([1,2,3,4]) + ds2 = array([5,10,20,50]) + ds3 = array([44,44,44,123,123]) + ds4 = array([1]) + + self.obj.extend(name, ds1) + self.assertTrue((fetch(self.obj.f) == ds1).all()) + self.assertEqual(self.obj.f[name].attrs['next_item'], 4) + self.assertEqual(size(self.obj.f), 8) + + self.obj.extend(name, ds2) + exp = hstack([ds1, ds2]) + obs = fetch(self.obj.f) + self.assertTrue((obs == exp).all()) + self.assertEqual(self.obj.f[name].attrs['next_item'], 8) + self.assertEqual(size(self.obj.f), 16) + + self.obj.extend(name, ds3) + exp = hstack([ds1, ds2, ds3]) + obs = fetch(self.obj.f) + self.assertTrue((obs == exp).all()) + self.assertEqual(self.obj.f[name].attrs['next_item'], 13) + self.assertEqual(size(self.obj.f), 16) + + self.obj.extend(name, ds4) + exp = hstack([ds1, ds2, ds3, ds4]) + obs = fetch(self.obj.f) + self.assertTrue((obs == exp).all()) + self.assertEqual(self.obj.f[name].attrs['next_item'], 14) + self.assertEqual(size(self.obj.f), 16) + + def test_finalize(self): + """Make sure we can finalize a dataset""" + name = 'test_group/test_ds' + fetch = lambda x: x[name][:] + size = lambda x: x[name].size + self.obj.create_dataset(name, int) + self.obj.extend(name, array([1,2,3])) + self.obj.extend(name, array([1,2,3])) + self.obj.extend(name, array([1,2,3])) + self.obj.extend(name, array([1,2,3])) + self.obj.extend(name, array([1,2,3])) + + self.assertEqual(size(self.obj.f), 24) + exp = array([1,2,3,1,2,3,1,2,3,1,2,3,1,2,3,0,0,0,0,0,0,0,0,0]) + obs = fetch(self.obj.f) + self.assertTrue((obs == exp).all()) + + self.obj._finalize(name) + exp = array([1,2,3] * 5) + obs = fetch(self.obj.f) + self.assertTrue((obs == exp).all()) + + self.assertFalse('next_item' in self.obj.f[name].attrs) + +if __name__ == '__main__': + main() From ee3ff38269de16905ef579240d1d8044b66c3fd6 Mon Sep 17 00:00:00 2001 From: Daniel McDonald Date: Sun, 22 Dec 2013 12:00:50 -0700 Subject: [PATCH 2/7] added changelog message --- ChangeLog.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ChangeLog.md b/ChangeLog.md index 457bc11..3110d9c 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -1,6 +1,11 @@ pyqi ChangeLog ============== +pyqi 0.3.1-dev +-------------- + +* added an HDF5 implicit dataset extender + pyqi 0.3.1 ---------- From 29314a7d6c75d3033d3474891d3acabc6b2a969a Mon Sep 17 00:00:00 2001 From: Daniel McDonald Date: Wed, 19 Mar 2014 11:58:21 -0600 Subject: [PATCH 3/7] skipIf decorator added --- tests/test_core/test_hdf5.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/tests/test_core/test_hdf5.py b/tests/test_core/test_hdf5.py index 179ce0b..fe3f737 100644 --- a/tests/test_core/test_hdf5.py +++ b/tests/test_core/test_hdf5.py @@ -11,12 +11,20 @@ __credits__ = ["Daniel McDonald"] +import unittest + from unittest import TestCase, main from os import remove from numpy import array, hstack from pyqi.core.hdf5 import AutoExtendHDF5 -import h5py +try: + import h5py + h5py_missing = False +except ImportError: + h5py_missing = True + +@unittest.skipIf(h5py_missing, "h5py is not present") class HDF5AutoExtendTests(TestCase): def setUp(self): self.hdf5_file = h5py.File('_test_file.hdf5','w') @@ -80,9 +88,9 @@ def test_finalize(self): self.obj.extend(name, array([1,2,3])) self.obj.extend(name, array([1,2,3])) self.obj.extend(name, array([1,2,3])) - + self.assertEqual(size(self.obj.f), 24) - exp = array([1,2,3,1,2,3,1,2,3,1,2,3,1,2,3,0,0,0,0,0,0,0,0,0]) + exp = array([1,2,3,1,2,3,1,2,3,1,2,3,1,2,3,0,0,0,0,0,0,0,0,0]) obs = fetch(self.obj.f) self.assertTrue((obs == exp).all()) @@ -90,7 +98,7 @@ def test_finalize(self): exp = array([1,2,3] * 5) obs = fetch(self.obj.f) self.assertTrue((obs == exp).all()) - + self.assertFalse('next_item' in self.obj.f[name].attrs) if __name__ == '__main__': From dc6fa685d11f89d767d4bbeeefecebda05b2afff Mon Sep 17 00:00:00 2001 From: Daniel McDonald Date: Wed, 19 Mar 2014 11:59:37 -0600 Subject: [PATCH 4/7] merge with master --- .travis.yml | 8 +++ ChangeLog.md | 1 + README.md | 2 +- doc/install/index.rst | 2 +- pyqi/commands/make_bash_completion.py | 3 +- pyqi/commands/make_command.py | 4 +- pyqi/commands/make_release.py | 5 +- pyqi/commands/serve_html_interface.py | 9 ++-- pyqi/core/interfaces/html/__init__.py | 4 +- pyqi/core/interfaces/html/input_handler.py | 19 +++++++ pyqi/core/interfaces/html/output_handler.py | 4 ++ .../interfaces/optparse/output_handler.py | 25 ++++++++- pyqi/util.py | 3 +- scripts/pyqi | 13 ++++- .../test_html/test_input_handler.py | 52 +++++++++++++++++++ .../test_html/test_output_handler.py | 30 +++++++++++ 16 files changed, 169 insertions(+), 15 deletions(-) create mode 100644 .travis.yml create mode 100644 tests/test_core/test_interfaces/test_html/test_input_handler.py create mode 100644 tests/test_core/test_interfaces/test_html/test_output_handler.py diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..e049b27 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,8 @@ +language: python +python: + - "2.7" +install: + - pip install . +script: + - nosetests + - pyqi diff --git a/ChangeLog.md b/ChangeLog.md index 3110d9c..1b06311 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -5,6 +5,7 @@ pyqi 0.3.1-dev -------------- * added an HDF5 implicit dataset extender +* painless profiling: just set the environment variable PYQI_PROFILE_COMMAND pyqi 0.3.1 ---------- diff --git a/README.md b/README.md index f45e368..bf40f46 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ pyqi: expose your interface =========================== -[![Build Status](http://ci.qiime.org/job/pyqi/badge/icon)](http://ci.qiime.org/job/pyqi/) +[![Build Status](https://travis-ci.org/bipy/pyqi.png?branch=master)](https://travis-ci.org/bipy/pyqi) pyqi (canonically pronounced *pie chee*) is designed to support wrapping general commands in multiple types of interfaces, including at the command line, HTML, and API levels. We're currently in the early stages of development, and there is a lot to be done. We're very interested in having beta users, and we fully embrace collaborative development, so if you're interested in using or developing pyqi, you should get in touch. For now, you can direct questions to gregcaporaso@gmail.com. diff --git a/doc/install/index.rst b/doc/install/index.rst index a79a8fe..b72fa94 100644 --- a/doc/install/index.rst +++ b/doc/install/index.rst @@ -27,7 +27,7 @@ If you decided not to install pyqi using pip, you can install it manually with t * To use the release version of pyqi, you can download it from `here `_. The latest release is |release|. After downloading, unzip the file with ``tar -xzf pyqi-x.y.z.tar.gz`` and change to the new ``pyqi-x.y.z`` directory, where x.y.z corresponds to the downloaded version. -* To use the latest development version of pyqi you can download it from our `GitHub repository `_ using ``git clone git@github.com:bipy/pyqi.git``. After downloading, change to the new ``pyqi`` directory. +* To use the latest development version of pyqi you can download it from our `GitHub repository `_ using ``git clone git://github.com/bipy/pyqi.git``. After downloading, change to the new ``pyqi`` directory. * Next, run ``python setup.py install``. That's it! diff --git a/pyqi/commands/make_bash_completion.py b/pyqi/commands/make_bash_completion.py index 6d55195..f82b8a7 100644 --- a/pyqi/commands/make_bash_completion.py +++ b/pyqi/commands/make_bash_completion.py @@ -58,7 +58,8 @@ def _get_cfg_module(desc): class BashCompletion(Command): BriefDescription = "Construct a bash completion script" - LongDescription = """Construct a bash tab completion script that will search through available commands and options""" + LongDescription = ("Construct a bash tab completion script that will search" + " through available commands and options") CommandIns = ParameterCollection([ CommandIn(Name='command_config_module', DataType=str, diff --git a/pyqi/commands/make_command.py b/pyqi/commands/make_command.py index 0179a5f..75a5489 100644 --- a/pyqi/commands/make_command.py +++ b/pyqi/commands/make_command.py @@ -58,7 +58,9 @@ def test_run(self): class MakeCommand(CodeHeaderGenerator): BriefDescription = "Construct a stubbed out Command object" - LongDescription = """This command is intended to construct the basics of a Command object so that a developer can dive straight into the implementation of the command""" + LongDescription = ("This command is intended to construct the basics of a " + "Command object so that a developer can dive straight into the " + "implementation of the command") CommandIns = ParameterCollection( CodeHeaderGenerator.CommandIns.Parameters + [ diff --git a/pyqi/commands/make_release.py b/pyqi/commands/make_release.py index 4474fcf..51103b4 100644 --- a/pyqi/commands/make_release.py +++ b/pyqi/commands/make_release.py @@ -64,7 +64,8 @@ def _parse_changelog(self, pkg_name): if change_info: break - match = re.search(r'released on (\w+\s+\d+\w+\s+\d+)', change_info) + match = re.search(r'released on (\w+\s+\d+\w+\s+\d+)', + change_info) if match is None: continue @@ -163,7 +164,7 @@ def _make_git_tag(self, tag): stdout, stderr, retval = pyqi_system_call(cmd, shell=False, dry_run=not self.RealRun) if retval is not 0: - self._fail("Could not git tag, \nSTDOUT:\n%s\n\nSTDERR:\n%s", stdout, + self._fail("Could not git tag, \nSTDOUT:\n%s\n\nSTDERR:\n%s",stdout, stderr) def _get_git_branch(self): diff --git a/pyqi/commands/serve_html_interface.py b/pyqi/commands/serve_html_interface.py index dc63b78..7376f92 100644 --- a/pyqi/commands/serve_html_interface.py +++ b/pyqi/commands/serve_html_interface.py @@ -17,19 +17,22 @@ class ServeHTMLInterface(Command): BriefDescription = "Start the HTMLInterface server" - LongDescription = "Start the HTMLInterface server and load the provided interface_module and port" + LongDescription = ("Start the HTMLInterface server and load the provided " + "interface_module and port") CommandIns = ParameterCollection([ CommandIn(Name='port', DataType=int, Description='The port to run the server on', Required=False, Default=8080), CommandIn(Name='interface_module', DataType=str, - Description='The module to serve the interface for', Required=True) + Description='The module to serve the interface for', + Required=True) ]) CommandOuts = ParameterCollection([ CommandOut(Name='result',DataType=str, - Description='Signals the termination of the HTMLInterface server') + Description='Signals the termination of the HTMLInterface ' + 'server') ]) def run(self, **kwargs): diff --git a/pyqi/core/interfaces/html/__init__.py b/pyqi/core/interfaces/html/__init__.py index 6f48d53..eff4675 100644 --- a/pyqi/core/interfaces/html/__init__.py +++ b/pyqi/core/interfaces/html/__init__.py @@ -58,7 +58,7 @@ class HTMLInputOption(InterfaceInputOption): float: lambda x: float(x.value), long: lambda x: long(x.value), complex: lambda x: complex(x.value), - "upload_file": lambda x: x.file.read(), + "upload_file": lambda x: x.file, "multiple_choice": lambda x: x.value } @@ -120,7 +120,7 @@ def _validate_option(self): if self.Choices is None: raise IncompetentDeveloperError( "must supply a list of Choices for type '%s'" % self.type, self) - elif type(self.Choices) not in (_type_handlers.TupleType, types.ListType): + elif type(self.Choices) not in (types.TupleType, types.ListType): raise IncompetentDeveloperError( "choices must be a list of strings ('%s' supplied)" % str(type(self.Choices)).split("'")[1], self) diff --git a/pyqi/core/interfaces/html/input_handler.py b/pyqi/core/interfaces/html/input_handler.py index df85848..7cedd9f 100644 --- a/pyqi/core/interfaces/html/input_handler.py +++ b/pyqi/core/interfaces/html/input_handler.py @@ -10,3 +10,22 @@ #----------------------------------------------------------------------------- __credits__ = ["Evan Bolyen"] + +from pyqi.core.exception import IncompetentDeveloperError + +def load_file_lines(option_value): + """Return a list of strings, one per line in the file. + + Each line will have leading and trailing whitespace stripped from it. + """ + if not hasattr(option_value, 'read'): + raise IncompetentDeveloperError("Input type must be a file object.") + + return [line.strip() for line in option_value] + +def load_file_contents(option_value): + """Return the contents of a file as a single string.""" + if not hasattr(option_value, 'read'): + raise IncompetentDeveloperError("Input type must be a file object.") + + return option_value.read() diff --git a/pyqi/core/interfaces/html/output_handler.py b/pyqi/core/interfaces/html/output_handler.py index 0858d8e..b24ad00 100644 --- a/pyqi/core/interfaces/html/output_handler.py +++ b/pyqi/core/interfaces/html/output_handler.py @@ -14,3 +14,7 @@ def newline_list_of_strings(result_key, data, option_value=None): """Return a string from a list of strings while appending newline""" return "\n".join(data) + +def html_list_of_strings(result_key, data, option_value=None): + """Return a string from a list of strings while appending an html break""" + return "
".join(data) diff --git a/pyqi/core/interfaces/optparse/output_handler.py b/pyqi/core/interfaces/optparse/output_handler.py index 1f3175c..94d7075 100644 --- a/pyqi/core/interfaces/optparse/output_handler.py +++ b/pyqi/core/interfaces/optparse/output_handler.py @@ -21,7 +21,7 @@ #----------------------------------------------------------------------------- __credits__ = ["Daniel McDonald", "Greg Caporaso", "Doug Wendel", - "Jai Ram Rideout", "Evan Bolyen"] + "Jai Ram Rideout", "Evan Bolyen", "Adam Robbins-Pianka"] from pyqi.core.exception import IncompetentDeveloperError import os @@ -73,3 +73,26 @@ def print_string(result_key, data, option_value=None): A newline will be printed before the data""" print "" print data + +def write_or_print_string(result_key, data, option_value=None): + """Write a string to a file. + + A newline will be added to the end of the file. If no file is supplied, + then the output will be printed to stdout instead. + """ + if option_value is None: + print_string(result_key, data, option_value) + else: + write_string(result_key, data, option_value) + +def write_or_print_list_of_strings(result_key, data, option_value=None): + """Write a list of strings to a file, one per line. + + A newline will be added to the end of the file. If no file is supplied, + then the output will be printed to stdout instead. + """ + if option_value is None: + print_list_of_strings(result_key, data, option_value) + else: + write_list_of_strings(result_key, data, option_value) + diff --git a/pyqi/util.py b/pyqi/util.py index 80f0663..b82bad6 100644 --- a/pyqi/util.py +++ b/pyqi/util.py @@ -69,7 +69,8 @@ def remove_files(list_of_filepaths, error_on_missing=True): missing.append(fp) if error_on_missing and missing: - raise OSError, "Some filepaths were not accessible: %s" % '\t'.join(missing) + raise OSError, "Some filepaths were not accessible: %s" % '\t'.join( + missing) def old_to_new_command(driver_name, project_title, local_argv): """Deprecate an old-style script. diff --git a/scripts/pyqi b/scripts/pyqi index dc6750b..2f3ebf3 100755 --- a/scripts/pyqi +++ b/scripts/pyqi @@ -19,12 +19,15 @@ __email__ = "mcdonadt@colorado.edu" import importlib import textwrap +import cProfile +import pstats from sys import argv, exit, stderr from pyqi.core.interface import get_command_names, get_command_config from pyqi.core.interfaces.optparse import optparse_main, optparse_factory from pyqi.util import get_version_string from os.path import basename from string import ljust +from os import environ ### we actually have some flexibility here to make the driver interface agnostic as well @@ -168,10 +171,16 @@ if __name__ == '__main__': help_(cmd_cfg_mod, help_cmd) else: assert_command_exists(cmd_name, command_names, driver_name) - + # see the note about crying about tears. argv[0] = ' '.join([driver_name, cmd_name]) cmd_obj = get_cmd_obj(cmd_cfg_mod, cmd_name) # execute FTW - optparse_main(cmd_obj, argv[1:]) + if 'PYQI_PROFILE_COMMAND' in environ: + stats_f = "%s.stats" % cmd_name + cProfile.run("optparse_main(cmd_obj, argv[1:])", stats_f) + stats = pstats.Stats(stats_f) + stats.strip_dirs().sort_stats('cumul').print_stats(25) + else: + optparse_main(cmd_obj, argv[1:]) diff --git a/tests/test_core/test_interfaces/test_html/test_input_handler.py b/tests/test_core/test_interfaces/test_html/test_input_handler.py new file mode 100644 index 0000000..e22dafe --- /dev/null +++ b/tests/test_core/test_interfaces/test_html/test_input_handler.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python + +#----------------------------------------------------------------------------- +# Copyright (c) 2013, The BiPy Development Team. +# +# Distributed under the terms of the Modified BSD License. +# +# The full license is in the file COPYING.txt, distributed with this software. +#----------------------------------------------------------------------------- + +__credits__ = ["Evan Bolyen"] + +from StringIO import StringIO +from unittest import TestCase, main +from pyqi.core.exception import IncompetentDeveloperError +from pyqi.core.interfaces.html.input_handler import (load_file_lines, + load_file_contents) + +class HTMLInputHandlerTests(TestCase): + def setUp(self): + self.file_like_object = StringIO() + #Note the whitespace, this tests strip() + self.file_like_object.write("This is line 1\n") + self.file_like_object.write(" This is line 2\n") + self.file_like_object.write("This is line 3 \n") + #Place it at the beginning of the file again + self.file_like_object.seek(0) + + def tearDown(self): + self.file_like_object.close() + + def test_load_file_lines(self): + """Correctly returns file lines as a list of strings""" + # can't load a string, etc... + self.assertRaises(IncompetentDeveloperError, load_file_lines, 'This is not a file') + result = load_file_lines(self.file_like_object) + self.assertEqual(result, + ["This is line 1", + "This is line 2", + "This is line 3"]) + + def test_load_file_contents(self): + """Correctly returns file contents""" + # can't load a string, etc... + self.assertRaises(IncompetentDeveloperError, load_file_contents, 'This is not a file') + + result = load_file_contents(self.file_like_object) + #Note the whitespace + self.assertEqual(result, "This is line 1\n This is line 2\nThis is line 3 \n") + +if __name__ == '__main__': + main() diff --git a/tests/test_core/test_interfaces/test_html/test_output_handler.py b/tests/test_core/test_interfaces/test_html/test_output_handler.py new file mode 100644 index 0000000..abb59d1 --- /dev/null +++ b/tests/test_core/test_interfaces/test_html/test_output_handler.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python + +#----------------------------------------------------------------------------- +# Copyright (c) 2013, The BiPy Development Team. +# +# Distributed under the terms of the Modified BSD License. +# +# The full license is in the file COPYING.txt, distributed with this software. +#----------------------------------------------------------------------------- + +__credits__ = ["Evan Bolyen"] + +from unittest import TestCase, main +from pyqi.core.interfaces.html.output_handler import (newline_list_of_strings, + html_list_of_strings) + +class HTMLOutputHandlerTests(TestCase): + + def test_newline_list_of_strings(self): + """Correctly returns a list of strings to delimited by '\n'.""" + result = newline_list_of_strings('foo', ['bar','bay','baz']) + self.assertEqual(result, 'bar\nbay\nbaz') + + def test_html_list_of_strings(self): + """Correctly returns a list of strings delimited by '
'.""" + result = html_list_of_strings('foo', ['bar','bay','baz']) + self.assertEqual(result, 'bar
bay
baz') + +if __name__ == '__main__': + main() From 22af4624f32d62aededcb50df7e93a45337fd088 Mon Sep 17 00:00:00 2001 From: Daniel McDonald Date: Wed, 19 Mar 2014 12:01:02 -0600 Subject: [PATCH 5/7] conflict fix --- pyqi/core/interfaces/html/__init__.py | 53 +++++++++++-------- .../test_optparse/test_output_handler.py | 11 ++++ 2 files changed, 43 insertions(+), 21 deletions(-) diff --git a/pyqi/core/interfaces/html/__init__.py b/pyqi/core/interfaces/html/__init__.py index eff4675..0d0840e 100644 --- a/pyqi/core/interfaces/html/__init__.py +++ b/pyqi/core/interfaces/html/__init__.py @@ -14,7 +14,19 @@ import os import types import os.path +<<<<<<< Updated upstream from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer +======= +import sys + +from pyqi.util import get_version_string, is_py2 + +if is_py2(): + from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer +else: + from http.server import BaseHTTPRequestHandler, HTTPServer + +>>>>>>> Stashed changes from cgi import parse_header, parse_multipart, parse_qs, FieldStorage from copy import copy from glob import glob @@ -24,7 +36,6 @@ from pyqi.core.factory import general_factory from pyqi.core.exception import IncompetentDeveloperError from pyqi.core.command import Parameter -from pyqi.util import get_version_string class HTMLResult(InterfaceOutputOption): """Base class for results for an HTML config file""" @@ -36,7 +47,7 @@ def __init__(self, MIMEType=None, **kwargs): class HTMLDownload(HTMLResult): """Result class for downloading a file from the server""" - def __init__(self, FileExtension=None, FilenameLookup=None, DefaultFilename=None, + def __init__(self, FileExtension=None, FilenameLookup=None, DefaultFilename=None, MIMEType='application/octet-stream', **kwargs): super(HTMLDownload, self).__init__(MIMEType=MIMEType, **kwargs) self.FileExtension = FileExtension @@ -76,17 +87,17 @@ def get_html(self, prefix, value=""): """Return the HTML needed for user input given a default value""" if (not value) and (self.Default is not None): value = self.Default - + input_name = prefix + self.Name string_input = lambda: '' % (input_name, value) number_input = lambda: '' % (input_name, value) - #html input files cannot have default values. + #html input files cannot have default values. #If the html interface worked as a data service, this would be possible as submit would be ajax. upload_input = lambda: '' % input_name mchoice_input = lambda: ''.join( - [ ('(%s)' - % (choice, input_name, choice, 'checked="true"' if value == choice else '')) + [ ('(%s)' + % (choice, input_name, choice, 'checked="true"' if value == choice else '')) for choice in self.Choices ] ) @@ -110,7 +121,7 @@ def get_html(self, prefix, value=""): self.Help, ' ' ]) - + def _validate_option(self): if self.Type not in self._type_handlers: raise IncompetentDeveloperError("Unsupported Type in HTMLInputOption: %s" % self.Type) @@ -130,7 +141,7 @@ def _validate_option(self): class HTMLInterface(Interface): """An HTML interface""" #Relative mapping wasn't working on a collegue's MacBook when pyqi was run outside of it's directory - #Until I understand why that was the case and how to fix it, I am putting the style css here. + #Until I understand why that was the case and how to fix it, I am putting the style css here. #This is not a permanent solution. css_style = '\n'.join([ 'html, body {', @@ -189,7 +200,7 @@ def __init__(self, input_prefix="pyqi_", **kwargs): self._html_input_prefix = input_prefix self._html_interface_input = {} super(HTMLInterface, self).__init__(**kwargs) - + #Override def __call__(self, in_, *args, **kwargs): self._the_in_validator(in_) @@ -199,14 +210,14 @@ def __call__(self, in_, *args, **kwargs): 'type': 'error', 'errors': errors } - else: + else: cmd_result = self.CmdInstance(**cmd_input) - self._the_out_validator(cmd_result) + self._the_out_validator(cmd_result) return self._output_handler(cmd_result) def _validate_inputs_outputs(self, inputs, outputs): - super(HTMLInterface, self)._validate_inputs_outputs(inputs, outputs) - + super(HTMLInterface, self)._validate_inputs_outputs(inputs, outputs) + if len(outputs) > 1: raise IncompetentDeveloperError("There can be only one... output") @@ -241,7 +252,7 @@ def _input_handler(self, in_, *args, **kwargs): formatted_input = {} for key in in_: - mod_key = key[ len(self._html_input_prefix): ] + mod_key = key[ len(self._html_input_prefix): ] formatted_input[mod_key] = in_[key] if not formatted_input[mod_key].value: formatted_input[mod_key] = None @@ -319,11 +330,11 @@ def _output_handler(self, results): if output.InputName is None: handled_results = output.Handler(rk, results[rk]) else: - handled_results = output.Handler(rk, results[rk], + handled_results = output.Handler(rk, results[rk], self._html_interface_input[output.InputName]) else: handled_results = results[rk] - + if isinstance(output, HTMLDownload): return self._output_download_handler(output, handled_results) @@ -413,7 +424,7 @@ def route(self, path, output_writer): self.send_header('Content-type', 'text/html') self.end_headers() output_writer(self.wfile.write) - + self.wfile.close() self._unrouted = False; @@ -426,7 +437,7 @@ def command_route(self, command): self.send_header('Content-type', 'text/html') self.end_headers() cmd_obj.command_page_writer(self.wfile.write, [], {}) - + self.wfile.close() self._unrouted = False @@ -441,14 +452,14 @@ def post_route(self, command, postvars): 'type':'error', 'errors':[e] } - + if result['type'] == 'error': self.send_response(400) self.send_header('Content-type', 'text/html') self.end_headers() cmd_obj.command_page_writer(self.wfile.write, result['errors'], postvars) - elif result['type'] == 'page': + elif result['type'] == 'page': self.send_response(200) self.send_header('Content-type', result['mime_type']) self.end_headers() @@ -460,7 +471,7 @@ def post_route(self, command, postvars): self.send_header('Content-disposition', 'attachment; filename='+result['filename']) self.end_headers() self.wfile.write(result['contents']) - + self.wfile.close() self._unrouted = False diff --git a/tests/test_core/test_interfaces/test_optparse/test_output_handler.py b/tests/test_core/test_interfaces/test_optparse/test_output_handler.py index c4c9a82..49f52f0 100644 --- a/tests/test_core/test_interfaces/test_optparse/test_output_handler.py +++ b/tests/test_core/test_interfaces/test_optparse/test_output_handler.py @@ -13,7 +13,18 @@ import os import sys +<<<<<<< Updated upstream from StringIO import StringIO +======= + +from pyqi.util import is_py2 + +if is_py2(): + from StringIO import StringIO +else: + from io import StringIO + +>>>>>>> Stashed changes from shutil import rmtree from tempfile import mkdtemp from unittest import TestCase, main From 1787afeba73d41b4e8a0b20243f31af148f7214c Mon Sep 17 00:00:00 2001 From: Daniel McDonald Date: Wed, 19 Mar 2014 12:03:15 -0600 Subject: [PATCH 6/7] conflict resolution --- pyqi/core/interfaces/html/__init__.py | 4 ---- .../test_interfaces/test_optparse/test_output_handler.py | 4 ---- 2 files changed, 8 deletions(-) diff --git a/pyqi/core/interfaces/html/__init__.py b/pyqi/core/interfaces/html/__init__.py index 0d0840e..dc4d07b 100644 --- a/pyqi/core/interfaces/html/__init__.py +++ b/pyqi/core/interfaces/html/__init__.py @@ -14,9 +14,6 @@ import os import types import os.path -<<<<<<< Updated upstream -from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer -======= import sys from pyqi.util import get_version_string, is_py2 @@ -26,7 +23,6 @@ else: from http.server import BaseHTTPRequestHandler, HTTPServer ->>>>>>> Stashed changes from cgi import parse_header, parse_multipart, parse_qs, FieldStorage from copy import copy from glob import glob diff --git a/tests/test_core/test_interfaces/test_optparse/test_output_handler.py b/tests/test_core/test_interfaces/test_optparse/test_output_handler.py index 49f52f0..22a687c 100644 --- a/tests/test_core/test_interfaces/test_optparse/test_output_handler.py +++ b/tests/test_core/test_interfaces/test_optparse/test_output_handler.py @@ -13,9 +13,6 @@ import os import sys -<<<<<<< Updated upstream -from StringIO import StringIO -======= from pyqi.util import is_py2 @@ -24,7 +21,6 @@ else: from io import StringIO ->>>>>>> Stashed changes from shutil import rmtree from tempfile import mkdtemp from unittest import TestCase, main From df59bff0b8e485a0373f3e622b188aa846acebf3 Mon Sep 17 00:00:00 2001 From: Daniel McDonald Date: Wed, 19 Mar 2014 16:02:50 -0600 Subject: [PATCH 7/7] fixing conflicts --- .travis.yml | 10 ++++---- ChangeLog.md | 1 + doc/index.rst | 2 +- pyqi/commands/make_optparse.py | 11 +++++---- pyqi/core/command.py | 4 ++-- pyqi/core/interface.py | 9 ++++--- pyqi/core/interfaces/html/__init__.py | 6 ++--- pyqi/core/interfaces/optparse/__init__.py | 3 +-- .../interfaces/optparse/output_handler.py | 6 ++--- pyqi/util.py | 24 ++++++++++++------- scripts/pyqi | 22 ++++++++--------- setup.py | 14 +++-------- .../test_code_header_generator.py | 6 ++--- .../test_make_bash_completion.py | 2 +- tests/test_commands/test_make_command.py | 4 ++-- tests/test_commands/test_make_optparse.py | 10 ++++---- tests/test_core/test_interface.py | 14 ++++++++--- .../test_html/test_input_handler.py | 12 +++++++--- .../test_optparse/test_init.py | 16 +++++++------ tests/test_util.py | 1 + tox.ini | 8 ++++--- 21 files changed, 100 insertions(+), 85 deletions(-) diff --git a/.travis.yml b/.travis.yml index e049b27..c531075 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,8 +1,8 @@ language: python -python: - - "2.7" +env: + - TOXENV=py27 + - TOXENV=py33 install: - - pip install . + - pip install tox script: - - nosetests - - pyqi + - tox diff --git a/ChangeLog.md b/ChangeLog.md index 1b06311..4561dce 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -5,6 +5,7 @@ pyqi 0.3.1-dev -------------- * added an HDF5 implicit dataset extender +* native python 3 support * painless profiling: just set the environment variable PYQI_PROFILE_COMMAND pyqi 0.3.1 diff --git a/doc/index.rst b/doc/index.rst index 5a7bcea..de999ea 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -6,7 +6,7 @@ What is pyqi? pyqi (canonically pronounced *pie chee*) is a Python framework designed to support wrapping general *commands* in multiple types of *interfaces*, including at the command line, HTML, and API levels. -pyqi's only requirement is a working Python 2.7 installation. +pyqi's only requirement is a working Python 2.7 or 3.3 installation. Why should I care? ------------------ diff --git a/pyqi/commands/make_optparse.py b/pyqi/commands/make_optparse.py index b99930a..6c26323 100644 --- a/pyqi/commands/make_optparse.py +++ b/pyqi/commands/make_optparse.py @@ -10,7 +10,7 @@ from __future__ import division from operator import attrgetter -from pyqi.core.command import (Command, CommandIn, CommandOut, +from pyqi.core.command import (Command, CommandIn, CommandOut, ParameterCollection) from pyqi.commands.code_header_generator import CodeHeaderGenerator @@ -81,7 +81,7 @@ # # value will be made available to Handler. This name # # can be either an underscored or dashed version of the # # option name (e.g., 'output_fp' or 'output-fp') - # InputName='output-fp'), + # InputName='output-fp'), # # An example option that does not map to a CommandIn. # OptparseResult(Parameter=cmd_out_lookup('some_other_result'), @@ -112,7 +112,7 @@ class MakeOptparse(CodeHeaderGenerator): BriefDescription = "Consume a Command, stub out an optparse configuration" LongDescription = """Construct and stub out the basic optparse configuration for a given Command. This template provides comments and examples of what to fill in.""" - + CommandIns = ParameterCollection( CodeHeaderGenerator.CommandIns.Parameters + [ CommandIn(Name='command', DataType=Command, @@ -155,8 +155,9 @@ def run(self, **kwargs): action = 'store' data_type = cmdin.DataType - fmt = {'name':cmdin.Name, 'datatype':data_type, 'action':action, - 'required':str(cmdin.Required), + fmt = {'name':cmdin.Name, + 'datatype':getattr(data_type, '__name__', None), + 'action':action, 'required':str(cmdin.Required), 'help':cmdin.Description, 'default_block':default_block} cmdin_formatted.append(input_format % fmt) diff --git a/pyqi/core/command.py b/pyqi/core/command.py index b497a15..cf0dc83 100644 --- a/pyqi/core/command.py +++ b/pyqi/core/command.py @@ -135,9 +135,9 @@ def __call__(self, **kwargs): try: result = self.run(**kwargs) - except Exception: + except Exception as e: self._logger.fatal('Error executing command: %s' % self_str) - raise + raise e else: self._logger.info('Completed command: %s' % self_str) diff --git a/pyqi/core/interface.py b/pyqi/core/interface.py index c2e1cd1..252ab1f 100644 --- a/pyqi/core/interface.py +++ b/pyqi/core/interface.py @@ -13,7 +13,6 @@ import importlib from sys import exit, stderr -from ConfigParser import SafeConfigParser from glob import glob from os.path import basename, dirname, expanduser, join from pyqi.core.exception import IncompetentDeveloperError @@ -193,13 +192,13 @@ class InterfaceInputOption(InterfaceOption): "str": str, "int": int, "float": float, - "long": long, + "long": int, # for python 3 compatibility as long is dropped "complex": complex, "tuple": tuple, "dict": dict, "list": list, "set": set, - "unicode": unicode, + "unicode": str, # for python 3 compatibility as all strings are unicode "frozenset": frozenset } @@ -298,9 +297,9 @@ def get_command_config(command_config_module, cmd, exit_on_failure=True): try: cmd_cfg = importlib.import_module('.'.join([command_config_module, python_cmd_name])) - except ImportError, e: + except ImportError as e: error_msg = str(e) - + if exit_on_failure: stderr.write("Unable to import the command configuration for " "%s:\n" % cmd) diff --git a/pyqi/core/interfaces/html/__init__.py b/pyqi/core/interfaces/html/__init__.py index dc4d07b..87c05f0 100644 --- a/pyqi/core/interfaces/html/__init__.py +++ b/pyqi/core/interfaces/html/__init__.py @@ -63,7 +63,6 @@ class HTMLInputOption(InterfaceInputOption): bool: lambda x: x.value == "True", int: lambda x: int(x.value), float: lambda x: float(x.value), - long: lambda x: long(x.value), complex: lambda x: complex(x.value), "upload_file": lambda x: x.file, "multiple_choice": lambda x: x.value @@ -103,7 +102,6 @@ def get_html(self, prefix, value=""): bool: mchoice_input, int: number_input, float: number_input, - long: number_input, complex: string_input, "multiple_choice": mchoice_input, "upload_file": upload_input @@ -515,8 +513,8 @@ def do_POST(self): def start_server(port, module): """Start a server for the HTMLInterface on the specified port""" interface_server = HTTPServer(("", port), get_http_handler(module)) - print "-- Starting server at http://localhost:%d --" % port - print "To close the server, type 'ctrl-c' into this window." + print("-- Starting server at http://localhost:%d --" % port) + print("To close the server, type 'ctrl-c' into this window.") try: interface_server.serve_forever() diff --git a/pyqi/core/interfaces/optparse/__init__.py b/pyqi/core/interfaces/optparse/__init__.py index dd43c8c..85de1fc 100644 --- a/pyqi/core/interfaces/optparse/__init__.py +++ b/pyqi/core/interfaces/optparse/__init__.py @@ -13,7 +13,6 @@ "Jose Antonio Navas Molina"] import os -import types from copy import copy from glob import glob from os.path import abspath, exists, isdir, isfile, split @@ -436,7 +435,7 @@ def _check_multiple_choice(self): if self.mchoices is None: raise OptionError( "must supply a list of mchoices for type '%s'" % self.type, self) - elif type(self.mchoices) not in (types.TupleType, types.ListType): + elif type(self.mchoices) not in (tuple, list): raise OptionError( "choices must be a list of strings ('%s' supplied)" % str(type(self.mchoices)).split("'")[1], self) diff --git a/pyqi/core/interfaces/optparse/output_handler.py b/pyqi/core/interfaces/optparse/output_handler.py index 94d7075..fe0fcb2 100644 --- a/pyqi/core/interfaces/optparse/output_handler.py +++ b/pyqi/core/interfaces/optparse/output_handler.py @@ -65,14 +65,14 @@ def print_list_of_strings(result_key, data, option_value=None): ``result_key`` and ``option_value`` are ignored. """ for line in data: - print line + print(line) def print_string(result_key, data, option_value=None): """Print the string A newline will be printed before the data""" - print "" - print data + print("") + print(data) def write_or_print_string(result_key, data, option_value=None): """Write a string to a file. diff --git a/pyqi/util.py b/pyqi/util.py index b82bad6..7408492 100644 --- a/pyqi/util.py +++ b/pyqi/util.py @@ -15,21 +15,29 @@ import importlib from os import remove from os.path import split, splitext -import sys -from subprocess import Popen, PIPE, STDOUT +import sys +from subprocess import Popen, PIPE + from pyqi.core.log import StdErrLogger from pyqi.core.exception import MissingVersionInfoError +def is_py2(): + """Check if we're using Python 2""" + if sys.version_info.major == 2: + return True + else: + return False + def pyqi_system_call(cmd, shell=True, dry_run=False): """Call cmd and return (stdout, stderr, return_value). - cmd: can be either a string containing the command to be run, or a + cmd: can be either a string containing the command to be run, or a sequence of strings that are the tokens of the command. - shell: value passed directly to Popen (default: True). See Python's + shell: value passed directly to Popen (default: True). See Python's subprocess.Popen for a description of the shell parameter and how cmd is interpreted differently based on its value. dry_run: if True, print cmd and return ("", "", 0) (default: False) - + This function is ported from QIIME (http://www.qiime.org), previously named qiime_system_call. QIIME is a GPL project, but we obtained permission from the authors of this function to port it to pyqi (and keep it under @@ -48,7 +56,7 @@ def pyqi_system_call(cmd, shell=True, dry_run=False): universal_newlines=True, stdout=PIPE, stderr=PIPE) - # communicate pulls all stdout/stderr from the PIPEs to + # communicate pulls all stdout/stderr from the PIPEs to # avoid blocking -- don't remove this line! stdout, stderr = proc.communicate() return_value = proc.returncode @@ -69,8 +77,8 @@ def remove_files(list_of_filepaths, error_on_missing=True): missing.append(fp) if error_on_missing and missing: - raise OSError, "Some filepaths were not accessible: %s" % '\t'.join( - missing) + raise OSError("Some filepaths were not accessible: %s" % '\t'.join( + missing)) def old_to_new_command(driver_name, project_title, local_argv): """Deprecate an old-style script. diff --git a/scripts/pyqi b/scripts/pyqi index 2f3ebf3..16b5393 100755 --- a/scripts/pyqi +++ b/scripts/pyqi @@ -26,7 +26,6 @@ from pyqi.core.interface import get_command_names, get_command_config from pyqi.core.interfaces.optparse import optparse_main, optparse_factory from pyqi.util import get_version_string from os.path import basename -from string import ljust from os import environ ### we actually have some flexibility here to make the driver interface agnostic as well @@ -54,26 +53,25 @@ def usage(cmd_cfg_mod, command_names): desc_limit = TERM_WIDTH - (INDENT + max_cmd + INDENT) cmd_end = INDENT + max_cmd + INDENT - print "usage: %s []" % argv[0] - print - print "The currently available commands are:" + print("usage: %s []\n" % argv[0]) + print("The currently available commands are:") # format: # indent command indent description for c, desc in valid_cmds: - cmd_formatted = ljust(''.join([' ' * INDENT, c]), cmd_end) - print ''.join([cmd_formatted, desc[:desc_limit]]) + s = ''.join([' ' * INDENT, c]) + cmd_formatted = s.ljust(cmd_end) + print(''.join([cmd_formatted, desc[:desc_limit]])) if invalid_cmds: - print - print "The following commands could not be loaded:" + print("\nThe following commands could not be loaded:") for c, error_msg in invalid_cmds: - cmd_formatted = ljust(''.join([' ' * INDENT, c]), cmd_end) - print ''.join([cmd_formatted, 'Error: %s' % error_msg]) + s = ''.join([' ' * INDENT, c]) + cmd_formatted = s.ljust(cmd_end) + print(''.join([cmd_formatted, 'Error: %s' % error_msg])) - print - print "See '%s help ' for more information on a specific command." % argv[0] + print("\nSee '%s help ' for more information on a specific command." % argv[0]) exit(0) def get_cmd_obj(cmd_cfg_mod, cmd): diff --git a/setup.py b/setup.py index 0667db3..07769fd 100644 --- a/setup.py +++ b/setup.py @@ -21,14 +21,6 @@ from glob import glob import sys -# from https://wiki.python.org/moin/PortingPythonToPy3k -try: - # python 3.x - from distutils.command.build_py import build_py_2to3 as build_py -except ImportError: - # python 2.x - from distutils.command.build_py import build_py - # classes/classifiers code adapted from Celery: # https://github.com/celery/celery/blob/master/setup.py # @@ -41,6 +33,7 @@ Topic :: Software Development :: User Interfaces Programming Language :: Python Programming Language :: Python :: 2.7 + Programming Language :: Python :: 3.3 Programming Language :: Python :: Implementation :: CPython Operating System :: OS Independent Operating System :: POSIX @@ -50,14 +43,13 @@ # Verify Python version ver = '.'.join(map(str, [sys.version_info.major, sys.version_info.minor])) -if ver not in ['2.7']: - sys.stderr.write("Only Python >=2.7 and <3.0 is supported.") +if ver not in ['2.7', '3.3']: + sys.stderr.write("Only Python 2.7 and 3.3 are supported.") sys.exit(1) long_description = """pyqi (canonically pronounced pie chee) is a Python framework designed to support wrapping general commands in multiple types of interfaces, including at the command line, HTML, and API levels.""" setup(name='pyqi', - cmdclass={'build_py':build_py}, version=__version__, license=__license__, description='pyqi: expose your interface', diff --git a/tests/test_commands/test_code_header_generator.py b/tests/test_commands/test_code_header_generator.py index b1c91e7..dc1cc38 100644 --- a/tests/test_commands/test_code_header_generator.py +++ b/tests/test_commands/test_code_header_generator.py @@ -24,7 +24,7 @@ def test_run(self): obs = self.cmd(author='bob', email='bob@bob.bob', license='very permissive license', copyright='what\'s that?', version='1.0') - self.assertEqual(obs.keys(), ['result']) + self.assertEqual(list(obs.keys()), ['result']) obs = obs['result'] self.assertEqual('\n'.join(obs), exp_header1) @@ -34,14 +34,14 @@ def test_run(self): license='very permissive license', copyright='what\'s that?', version='1.0', credits=['another person', 'another another person']) - self.assertEqual(obs.keys(), ['result']) + self.assertEqual(list(obs.keys()), ['result']) obs = obs['result'] self.assertEqual('\n'.join(obs), exp_header2) # With no arguments obs = self.cmd() - self.assertEqual(obs.keys(), ['result']) + self.assertEqual(list(obs.keys()), ['result']) obs = obs['result'] self.assertEqual('\n'.join(obs), exp_header3) diff --git a/tests/test_commands/test_make_bash_completion.py b/tests/test_commands/test_make_bash_completion.py index aad1c08..2305da2 100644 --- a/tests/test_commands/test_make_bash_completion.py +++ b/tests/test_commands/test_make_bash_completion.py @@ -75,7 +75,7 @@ def test_run(self): params = {'command_config_module':self.temp_module_name, 'driver_name':'pyqi'} obs = self.cmd(**params) - self.assertEqual(obs.keys(), ['result']) + self.assertEqual(list(obs.keys()), ['result']) self.assertEqual(obs['result'], outputandstuff) diff --git a/tests/test_commands/test_make_command.py b/tests/test_commands/test_make_command.py index 94a680c..25f083a 100644 --- a/tests/test_commands/test_make_command.py +++ b/tests/test_commands/test_make_command.py @@ -24,7 +24,7 @@ def test_run_command_code_generation(self): obs = self.cmd(name='Test', author='bob', email='bob@bob.bob', license='very permissive license', copyright='what\'s that?', version='1.0') - self.assertEqual(obs.keys(), ['result']) + self.assertEqual(list(obs.keys()), ['result']) obs = obs['result'] self.assertEqual(obs, exp_command_code1.splitlines()) @@ -35,7 +35,7 @@ def test_run_test_code_generation(self): license='very permissive license', copyright='what\'s that?', version='1.0', credits=['another person'], test_code=True) - self.assertEqual(obs.keys(), ['result']) + self.assertEqual(list(obs.keys()), ['result']) obs = obs['result'] self.assertEqual('\n'.join(obs), exp_test_code1) diff --git a/tests/test_commands/test_make_optparse.py b/tests/test_commands/test_make_optparse.py index d36dc2f..ae30658 100644 --- a/tests/test_commands/test_make_optparse.py +++ b/tests/test_commands/test_make_optparse.py @@ -22,7 +22,7 @@ def setUp(self): def test_run(self): exp = win_text - + pc = CommandIn(Name='DUN', Required=True, DataType=str, Description="") bool_param = CommandIn(Name='imabool', DataType=bool, Description='zero or one', Required=False) @@ -39,9 +39,9 @@ class stubby: 'copyright': 'what\'s that?', 'version': '1.0' }) - + self.assertEqual(obs['result'], exp.splitlines()) - + win_text = """#!/usr/bin/env python from __future__ import division @@ -105,7 +105,7 @@ class stubby: # Help='output filepath') OptparseOption(Parameter=cmd_in_lookup('DUN'), - Type=, + Type=str, Action='store', # default is 'store', change if desired Handler=None, # must be defined if desired ShortName=None, # must be defined if desired @@ -136,7 +136,7 @@ class stubby: # # value will be made available to Handler. This name # # can be either an underscored or dashed version of the # # option name (e.g., 'output_fp' or 'output-fp') - # InputName='output-fp'), + # InputName='output-fp'), # # An example option that does not map to a CommandIn. # OptparseResult(Parameter=cmd_out_lookup('some_other_result'), diff --git a/tests/test_core/test_interface.py b/tests/test_core/test_interface.py index 63e05c0..b0b292e 100644 --- a/tests/test_core/test_interface.py +++ b/tests/test_core/test_interface.py @@ -12,14 +12,17 @@ __credits__ = ["Greg Caporaso", "Daniel McDonald", "Doug Wendel", "Jai Ram Rideout"] +import sys + from unittest import TestCase, main from pyqi.core.interface import get_command_names, get_command_config +from pyqi.util import is_py2 import pyqi.interfaces.optparse.config.make_bash_completion class TopLevelTests(TestCase): def test_get_command_names(self): """Test that command names are returned from a config directory.""" - exp = ['make-bash-completion', 'make-command', 'make-optparse', + exp = ['make-bash-completion', 'make-command', 'make-optparse', 'make-release', 'serve-html-interface'] obs = get_command_names('pyqi.interfaces.optparse.config') self.assertEqual(obs, exp) @@ -40,8 +43,13 @@ def test_get_command_config(self): 'hopefully.nonexistent.python.module', 'umm', exit_on_failure=False) self.assertEqual(cmd_cfg, None) - self.assertEqual(error_msg, 'No module named hopefully.nonexistent.' - 'python.module.umm') + + py2_err = 'No module named hopefully.nonexistent.python.module.umm' + py3_err = "No module named 'hopefully'" + if is_py2(): + self.assertEqual(error_msg, py2_err) + else: + self.assertEqual(error_msg, py3_err) if __name__ == '__main__': diff --git a/tests/test_core/test_interfaces/test_html/test_input_handler.py b/tests/test_core/test_interfaces/test_html/test_input_handler.py index e22dafe..7891672 100644 --- a/tests/test_core/test_interfaces/test_html/test_input_handler.py +++ b/tests/test_core/test_interfaces/test_html/test_input_handler.py @@ -10,8 +10,14 @@ __credits__ = ["Evan Bolyen"] -from StringIO import StringIO +import sys from unittest import TestCase, main + +if sys.version_info.major == 2: + from StringIO import StringIO +else: + from io import StringIO + from pyqi.core.exception import IncompetentDeveloperError from pyqi.core.interfaces.html.input_handler import (load_file_lines, load_file_contents) @@ -34,8 +40,8 @@ def test_load_file_lines(self): # can't load a string, etc... self.assertRaises(IncompetentDeveloperError, load_file_lines, 'This is not a file') result = load_file_lines(self.file_like_object) - self.assertEqual(result, - ["This is line 1", + self.assertEqual(result, + ["This is line 1", "This is line 2", "This is line 3"]) diff --git a/tests/test_core/test_interfaces/test_optparse/test_init.py b/tests/test_core/test_interfaces/test_optparse/test_init.py index 9e85823..4ecffe0 100644 --- a/tests/test_core/test_interfaces/test_optparse/test_init.py +++ b/tests/test_core/test_interfaces/test_optparse/test_init.py @@ -100,7 +100,7 @@ def test_validate_inputs(self): def test_input_handler(self): obs = self.interface._input_handler(['--c','foo']) - self.assertEqual(obs.items(), [('c', 'foo')]) + self.assertEqual(list(obs.items()), [('c', 'foo')]) def test_build_usage_lines(self): obs = self.interface._build_usage_lines([]) @@ -177,8 +177,10 @@ def setUp(self): self._dirs_to_clean_up = [] def tearDown(self): - map(remove, self._paths_to_clean_up) - map(rmdir, self._dirs_to_clean_up) + for p in self._paths_to_clean_up: + remove(p) + for d in self._dirs_to_clean_up: + rmdir(d) def test_check_existing_filepath(self): # Check that returns the correct value when the file exists @@ -199,8 +201,8 @@ def test_check_existing_filepath(self): def test_check_existing_filepaths(self): # Check that returns a list with the paths, in the same order as # the input comma separated list - tmp_f1, tmp_path1 = mkstemp(prefix='pyqi_tmp_') - tmp_f2, tmp_path2 = mkstemp(prefix='pyqi_tmp_') + tmp_f1, tmp_path1 = mkstemp(prefix='pyqi_tmp_testf') + tmp_f2, tmp_path2 = mkstemp(prefix='pyqi_tmp_testf') self._paths_to_clean_up = [tmp_path1, tmp_path2] option = PyqiOption('-f', '--files_test', type='existing_filepaths') exp = [tmp_path1, tmp_path2] @@ -245,8 +247,8 @@ def test_check_existing_dirpath(self): def test_check_existing_dirpaths(self): # Check that returns a list with the paths, in the same order as the # input comma separated list - tmp_dirpath1 = mkdtemp(prefix='pyqi_tmp_') - tmp_dirpath2 = mkdtemp(prefix='pyqi_tmp_') + tmp_dirpath1 = mkdtemp(prefix='pyqi_tmp_testd_') + tmp_dirpath2 = mkdtemp(prefix='pyqi_tmp_testd_') self._dirs_to_clean_up = [tmp_dirpath1, tmp_dirpath2] option = PyqiOption('-d', '--dirs_test', type='existing_dirpaths') exp = [tmp_dirpath1, tmp_dirpath2] diff --git a/tests/test_util.py b/tests/test_util.py index 30462b5..f751196 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -17,6 +17,7 @@ from pyqi.util import get_version_string from pyqi.core.exception import MissingVersionInfoError + class UtilTests(TestCase): def test_get_version_string(self): """Test extracting a version string given a module string.""" diff --git a/tox.ini b/tox.ini index b1547c6..1b99b7a 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,7 @@ [tox] -envlist = py27 +envlist = py27,py33 [testenv] -deps=nose -commands=nosetests +deps=nose +commands= + nosetests + pyqi