diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 00000000..81fca2ea --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,34 @@ +name: Run Sumatra tests + +on: [push, pull_request] + +jobs: + build: + + strategy: + matrix: + os: [ubuntu-22.04, macos-latest] + python-version: ["3.8", "3.9", "3.11", "3.12"] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + - name: Install + run: | + python -m pip install --upgrade pip + pip install -r ci/requirements.txt + pip install -r ci/requirements-test.txt + pip install . + pip freeze + - name: Lint with flake8 + run: | + # stop the build if there are Python syntax errors or undefined names + #flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + # exit-zero treats all errors as warnings. + #flake8 . --count --exit-zero --max-complexity=10 --max-line-length=119 --statistics + - name: Test with pytest + run: | + pytest test/unittests diff --git a/.gitignore b/.gitignore index 317341cd..e759ab2e 100644 --- a/.gitignore +++ b/.gitignore @@ -14,4 +14,7 @@ MANIFEST .pydevproject .komodoproject .idea -*.egg-info \ No newline at end of file +*.egg-info +env +env-ci +test/example_repositories/mercurial/hg/wcache diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index ae38afd3..00000000 --- a/.travis.yml +++ /dev/null @@ -1,44 +0,0 @@ -language: python -sudo: required -python: - - "2.7" - - "3.6" - - "3.7" -env: - global: - - PATH=/home/travis/miniconda/bin:$PATH -install: - - sudo apt-get update - # We do this conditionally because it saves us some downloading if the - # version is the same. - - echo $TRAVIS_PYTHON_VERSION - - if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then - wget https://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh; - else - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh; - fi - - bash miniconda.sh -b -p $HOME/miniconda - - export PATH="$HOME/miniconda/bin:$PATH" - - hash -r - - conda config --set always_yes yes --set changeps1 no - - conda update -q conda - # Useful for debugging any issues with conda - - conda info -a - - - conda create -n testenv --yes python=$TRAVIS_PYTHON_VERSION numpy scipy libgfortran-ng matplotlib nose pillow - - source activate testenv - - pip install -r ci/requirements.txt - - pip install -r ci/requirements-test.txt - - pip install -r ci/requirements-$TRAVIS_PYTHON_VERSION.txt - - pip install . - - - curl -OL http://raw.github.com/craigcitro/r-travis/master/scripts/travis-tool.sh - - chmod 755 ./travis-tool.sh - - ./travis-tool.sh bootstrap -# command to run tests -script: - nosetests -v --nologcapture --with-coverage --cover-package=sumatra test/unittests test/system/test_ircr.py -after_failure: - - ./travis-tool.sh dump_logs -after_success: - coveralls diff --git a/LICENSE b/LICENSE index 94c079ff..f3d60970 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2015 The Sumatra authors and contributors +Copyright (c) 2025 The Sumatra authors and contributors All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/README.rst b/README.rst index f5322154..47f249e2 100644 --- a/README.rst +++ b/README.rst @@ -26,35 +26,35 @@ For documentation, see http://packages.python.org/Sumatra/ and http://neuralense Functionality: - * launch simulations and analyses, and record various pieces of information, - including: +* launch simulations and analyses, and record various pieces of information, + including: - - the executable (identity, version) - - the script (identity, version) - - the parameters - - the duration (execution time) - - console output - - links to all data (whether in files, in a database, etc.) produced by - the simulation/analysis - - the reason for doing the simulation/analysis - - the outcome of the simulation/analysis + - the executable (identity, version) + - the script (identity, version) + - the parameters + - the duration (execution time) + - console output + - links to all data (whether in files, in a database, etc.) produced by + the simulation/analysis + - the reason for doing the simulation/analysis + - the outcome of the simulation/analysis - * allow browsing/searching/visualising the results of previous experiments - * allow the re-running of previous simulations/analyses with automatic - verification that the results are unchanged - * launch single or batch experiments, serial or parallel +* allow browsing/searching/visualising the results of previous experiments +* allow the re-running of previous simulations/analyses with automatic + verification that the results are unchanged +* launch single or batch experiments, serial or parallel ============ Requirements ============ -Sumatra requires Python versions 2.7, 3.4, 3.5 or 3.6. The web interface requires -Django (>= 1.8) and the django-tagging package. +Sumatra requires Python version 3.4 or later The web interface requires +Django (>= 2.2) and the django-tagging package. Sumatra requires that you keep your own code in a version control system (currently Subversion, Mercurial, Git and Bazaar are supported). If you are already using Bazaar there is nothing else to install. If you -are using Subversion you will need to install the pysvn package. If you using +are using Subversion you will need to install the pysvn package. If you are using Git, you will need to install git-python bindings, and for Mercurial install hg-api. @@ -62,25 +62,18 @@ Git, you will need to install git-python bindings, and for Mercurial install hg- Installation ============ -These instructions are for Unix, Mac OS X. They may work on Windows as well, but +These instructions are for Unix and Mac OS. They may work on Windows as well, but it hasn't been thoroughly tested. -If you have downloaded the source package, Sumatra-0.7.0.tar.gz:: - - $ tar xzf Sumatra-0.7.0.tar.gz - $ cd Sumatra-0.7.0 - # python setup.py install - -The last step may have to be done as root. - - -Alternatively, you can install directly from the Python Package Index:: +The easiest way to install is with pip:: $ pip install sumatra -or:: +Alternatively, you can download the source package, Sumatra-0.8.0.tar.gz from the Python Package Index:: - $ easy_install sumatra + $ tar xzf Sumatra-0.8.0.tar.gz + $ cd Sumatra-0.8.0 + $ python setup.py install You will also need to install Python bindings for the version control system(s) you use, e.g.:: @@ -92,8 +85,8 @@ You will also need to install Python bindings for the version control system(s) Code status =========== -.. image:: https://travis-ci.org/open-research/sumatra.png?branch=master - :target: https://travis-ci.org/open-research/sumatra +.. image:: https://github.com/open-research/sumatra/actions/workflows/tests/badge.svg + :target: https://github.com/open-research/sumatra/actions/workflows/tests.yml :alt: Unit Test Status .. image:: https://coveralls.io/repos/open-research/sumatra/badge.svg diff --git a/bin/smtweb b/bin/smtweb index 072d42b4..ac310664 100755 --- a/bin/smtweb +++ b/bin/smtweb @@ -68,8 +68,16 @@ def main(argv): INSTALLED_APPS=db_config._settings["INSTALLED_APPS"] + ['sumatra.web'], ROOT_URLCONF='sumatra.web.urls', STATIC_URL='/static/', - TEMPLATE_DIRS=(os.path.join(os.getcwd(), ".smt", "templates"), - os.path.join(root_dir, "templates"),), + TEMPLATES=[ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'APP_DIRS': False, + 'DIRS': [ + os.path.join(os.getcwd(), ".smt", "templates"), + os.path.join(root_dir, "templates") + ] + }, + ], MIDDLEWARE_CLASSES=tuple(), READ_ONLY = options.read_only, SERVERSIDE = options.serverside, diff --git a/ci/requirements-2.7.txt b/ci/requirements-2.7.txt deleted file mode 100644 index dea603dc..00000000 --- a/ci/requirements-2.7.txt +++ /dev/null @@ -1,5 +0,0 @@ -# additional requirements for Python 2.7 -pathlib -configparser -# optional, if you use Bazaar -bzr diff --git a/ci/requirements-3.6.txt b/ci/requirements-3.6.txt deleted file mode 100644 index 9edef4fb..00000000 --- a/ci/requirements-3.6.txt +++ /dev/null @@ -1,2 +0,0 @@ -# Additional requirements for Python 3.5 -pyyaml # this is already in requirements.txt, but pip doesn't allow empty requirements files diff --git a/ci/requirements-3.7.txt b/ci/requirements-3.7.txt deleted file mode 100644 index 9edef4fb..00000000 --- a/ci/requirements-3.7.txt +++ /dev/null @@ -1,2 +0,0 @@ -# Additional requirements for Python 3.5 -pyyaml # this is already in requirements.txt, but pip doesn't allow empty requirements files diff --git a/ci/requirements-test.txt b/ci/requirements-test.txt index 83a4b0f8..dd1c1334 100644 --- a/ci/requirements-test.txt +++ b/ci/requirements-test.txt @@ -3,7 +3,7 @@ sarge numpy scipy matplotlib -nose pillow -coverage -coveralls +pytest<8 +pytest-cov +flake8 diff --git a/ci/requirements.txt b/ci/requirements.txt index bccc6834..9d2166bf 100644 --- a/ci/requirements.txt +++ b/ci/requirements.txt @@ -1,12 +1,13 @@ # Base requirements for running Sumatra, for all supported versions of Python -Django>=1.8 +setuptools # needed for Python 3.12, since distutils removed +Django<3 django-tagging>=0.4 httplib2 jinja2 docutils parameters -future # optional requirements, depending on which version control systems you use +mercurial hgapi GitPython>=0.3.6 # optional requirements, depending on which serialization formats you want @@ -15,4 +16,4 @@ pyyaml #dexml #fs # optional, for PostgreSQL support -#psycopg2 +#psycopg2 \ No newline at end of file diff --git a/doc/build_reference.py b/doc/build_reference.py index 1a3d2029..a0b48c6d 100644 --- a/doc/build_reference.py +++ b/doc/build_reference.py @@ -1,6 +1,3 @@ -from __future__ import unicode_literals -from future import standard_library -standard_library.install_aliases() import sys from sumatra import commands from io import StringIO @@ -24,16 +21,15 @@ sys.stdout = sys.__stdout__ -f = open("command_reference.txt", "w") -f.write("=====================\n") -f.write("smt command reference\n") -f.write("=====================\n\n") - -for mode in modes: - sio = usage[mode] - f.write(mode + '\n') - f.write('-'*len(mode) + '\n::\n\n ') - sio.seek(0) - f.write(" ".join(sio.readlines()) + '\n') - sio.close() -f.close() +with open("command_reference.txt", "w") as f: + f.write("=====================\n") + f.write("smt command reference\n") + f.write("=====================\n\n") + + for mode in modes: + sio = usage[mode] + f.write(mode + '\n') + f.write('-'*len(mode) + '\n::\n\n ') + sio.seek(0) + f.write(" ".join(sio.readlines()) + '\n') + sio.close() diff --git a/doc/conf.py b/doc/conf.py index 689b07f4..24ed3abb 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -11,7 +11,6 @@ # All configuration values have a default; values that are commented out # serve to show the default. -from __future__ import unicode_literals import sys, os # If extensions (or modules to document with autodoc) are in another directory, @@ -40,7 +39,7 @@ # General information about the project. project = 'Sumatra' authors = 'Sumatra authors and contributors' -copyright = '2009-2015 ' + authors +copyright = '2009-2020, 2024-2025 ' + authors # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the diff --git a/doc/developers_guide.txt b/doc/developers_guide.txt index 65d0aa9b..76f548ed 100644 --- a/doc/developers_guide.txt +++ b/doc/developers_guide.txt @@ -3,26 +3,24 @@ Developers' guide ================= These instructions are for developing on a Unix-like platform, e.g. Linux or -Mac OS X, with the bash shell. +Mac OS. Requirements ------------ - * Python_ 2.7, 3.4, 3.5 or 3.6 - * Django_ >= 1.8 - * django-tagging_ >= 0.3 - * parameters >= 0.2.1 - * nose_ >= 0.11.4 - * future >= 0.14 - * if using Python < 3.4, pathlib >= 1.0.0 + * Python_ 3.8 or later + * Django_ >= 2.2 + * django-tagging_ + * parameters + * pytest * docutils * Jinja2 Optional: - * mpi4py_ >= 1.2.2 - * coverage_ >= 3.3.1 (for measuring test coverage) + * mpi4py_ + * pytest-cov (for measuring test coverage) * httplib2 (for the remote record store) * GitPython (for Git support) * mercurial and hgapi (for Mercurial support) @@ -71,7 +69,7 @@ Before you make any changes, run the test suite to make sure all the tests pass on your system:: $ cd sumatra/test/unittests - $ nosetests + $ pytest You will see some error messages, but don't worry - these are just tests of Sumatra's error handling. At the end, if you see "OK", then all the tests @@ -80,8 +78,6 @@ passed, otherwise it will report how many tests failed or produced errors. If any of the tests fail, check out the `continuous integration server`_ to see if these are "known" failures, otherwise please `open a bug report`_. -(many thanks to the `NEST Initiative`_ for hosting the CI server). - Writing tests ------------- @@ -93,7 +89,7 @@ check that the test now passes. To see how well the tests cover the code base, run:: - $ nosetests --coverage --cover-package=sumatra --cover-erase + $ pytest --cov=sumatra Committing your changes @@ -118,7 +114,7 @@ Coding standards and style -------------------------- All code should conform as much as possible to `PEP 8`_, and should run with -Python 2.7, 3.4, 3.5 and 3.6. Lines should be no longer than 99 characters. +Python 3.8 or later. Lines should be no longer than 119 characters. Reviewing pull requests @@ -138,8 +134,7 @@ Things to check for: * Do all public functions/classes have docstrings? * Are there tests for all new/changed functionality? * Has the documentation been updated? - * Has the Travis CI build passed? - * Is the syntax compatible with both Python 2 and 3? (even if we don't yet support Python 3, any new code should try to do so) + * Has the GitHub Actions CI build passed? * Is there any redundant or duplicate code? * Is the code as modular as possible? * Is there any commented out code, or print statements used for debugging? @@ -148,7 +143,7 @@ Things to check for: .. _Python: https://www.python.org .. _Django: https://www.djangoproject.com/ .. _django-tagging: http://code.google.com/p/django-tagging/ -.. _nose: https://nose.readthedocs.org/en/latest/ +.. _pytest: https://docs.pytest.org .. _Distribute: https://pypi.python.org/pypi/distribute .. _mpi4py: http://mpi4py.scipy.org/ .. _tox: http://codespeak.net/tox/ @@ -157,6 +152,6 @@ Things to check for: .. _`issue tracker`: https://github.com/open-research/sumatra/issues .. _virtualenv: http://www.virtualenv.org .. _`Sumatra repository on Github`: https://github.com/open-research/sumatra -.. _`continuous integration server`: https://qa.nest-initiative.org/view/Sumatra/job/sumatra/ +.. _`continuous integration server`: https://github.com/open-research/sumatra/actions .. _`NEST Initiative`: http://www.nest-initiative.org/ .. _`open a bug report`: https://github.com/open-research/sumatra/issues/new diff --git a/doc/installation.txt b/doc/installation.txt index 664c665f..9c6adc56 100644 --- a/doc/installation.txt +++ b/doc/installation.txt @@ -3,7 +3,7 @@ Installation ============ To run Sumatra you will need Python installed on your machine. If you are running -Linux or OS X, you almost certainly already have it. If you don't have Python, +Linux or Mac OS, you almost certainly already have it. If you don't have Python, you can install it from `python.org`_, or install one of the "value-added" distributions aimed at scientific users of Python: `Enthought`_, `Python(x,y)`_ or Anaconda_. @@ -14,11 +14,10 @@ The easiest way to install Sumatra is directly from the `Python Package Index`_ $ pip install sumatra -Alternatively, you can download the Sumatra package from either PyPI or the -`INCF Software Centre`_ and install it as follows:: +Alternatively, you can download the Sumatra package from PyPI and install it as follows:: - $ tar xzf Sumatra-0.7.0.tar.gz - $ cd Sumatra-0.7.0 + $ tar xzf Sumatra-0.8.0.tar.gz + $ cd Sumatra-0.8.0 # python setup.py install The last step may need to be run as root, or using sudo, although in general @@ -73,7 +72,6 @@ is recommended. .. _Git: http://git-scm.com/ .. _Bazaar: http://bazaar.canonical.com/ .. _`Python Package Index`: https://pypi.python.org/pypi/Sumatra/ -.. _`INCF Software Centre`: http://software.incf.org/software/sumatra/download .. _`django-tagging`: https://pypi.python.org/pypi/django-tagging/ .. _`pysvn bindings`: http://pysvn.tigris.org/project_downloads.html .. _`GitPython`: https://pypi.python.org/pypi/GitPython/ diff --git a/doc/using_the_api_example.py b/doc/using_the_api_example.py index fb9c1264..f6d53c92 100644 --- a/doc/using_the_api_example.py +++ b/doc/using_the_api_example.py @@ -1,4 +1,3 @@ -from __future__ import unicode_literals import numpy import sys import time diff --git a/doc/using_the_api_example2.py b/doc/using_the_api_example2.py index ab9a8055..75a995a7 100644 --- a/doc/using_the_api_example2.py +++ b/doc/using_the_api_example2.py @@ -1,4 +1,3 @@ -from __future__ import unicode_literals import numpy import sys from sumatra.parameters import build_parameters diff --git a/setup.py b/setup.py index adda6c56..30d92f81 100644 --- a/setup.py +++ b/setup.py @@ -27,8 +27,8 @@ def get_tip_revision(self, path=os.getcwd()): return repo.head.commit.hexsha[:7] -install_requires = ['Django>=1.8, <2', 'django-tagging', 'httplib2', - 'docutils', 'jinja2', 'parameters', 'future'] +install_requires = ['Django>=2, <3', 'django-tagging', 'httplib2', + 'docutils', 'jinja2', 'parameters'] major_python_version, minor_python_version, _, _, _ = sys.version_info if major_python_version < 3 or (major_python_version == 3 and minor_python_version < 4): install_requires.append('pathlib') @@ -50,7 +50,7 @@ def get_tip_revision(self, path=os.getcwd()): 'formatting/latex_template.tex', 'external_scripts/script_introspect.R']}, scripts = ['bin/smt', 'bin/smtweb', 'bin/smt-complete.sh'], author = "Sumatra authors and contributors", - author_email = "andrew.davison@unic.cnrs-gif.fr", + author_email = "andrew.davison@cnrs.fr", description = "A tool for automated tracking of computation-based scientific projects", long_description = open('README.rst').read(), license = "BSD 2 clause", @@ -64,10 +64,7 @@ def get_tip_revision(self, path=os.getcwd()): 'Natural Language :: English', 'Operating System :: OS Independent', 'Programming Language :: Python', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3.4', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3' 'Topic :: Scientific/Engineering'], cmdclass = {'sdist': sdist_git}, install_requires = install_requires, diff --git a/sumatra/__init__.py b/sumatra/__init__.py index b0026cd9..73d6960a 100644 --- a/sumatra/__init__.py +++ b/sumatra/__init__.py @@ -1,4 +1,3 @@ -from __future__ import unicode_literals __all__ = ['commands', 'datastore', 'formatting', 'launch', 'parameters', 'programs', 'projects', 'records', 'recordstore', 'versioncontrol', 'dependency_finder', 'web', 'decorators', 'publishing', diff --git a/sumatra/commands.py b/sumatra/commands.py index 14f06712..3637214f 100644 --- a/sumatra/commands.py +++ b/sumatra/commands.py @@ -4,12 +4,9 @@ Each command corresponds to a function in this module. -:copyright: Copyright 2006-2015 by the Sumatra team, see doc/authors.txt +:copyright: Copyright 2006-2020, 2024 by the Sumatra team, see doc/authors.txt :license: BSD 2-clause, see LICENSE for details. """ -from __future__ import print_function -from __future__ import unicode_literals -from builtins import str import os.path import sys @@ -248,7 +245,7 @@ def configure(argv): parser.add_argument('-c', '--on-changed', help="may be 'store-diff' or 'error': the action to take if the code in the repository or any of the dependencies has changed.", choices=['store-diff', 'error']) parser.add_argument('-g', '--labelgenerator', choices=['timestamp', 'uuid'], metavar='OPTION', help="specify which method Sumatra should use to generate labels (options: timestamp, uuid)") parser.add_argument('-t', '--timestamp_format', help="the timestamp format given to strftime") - parser.add_argument('-L', '--launch_mode', choices=['serial', 'distributed', 'slurm-mpi'], help="how computations should be launched.") + parser.add_argument('-L', '--launch_mode', choices=['serial', 'serial-tqdm', 'distributed', 'slurm-mpi'], help="how computations should be launched.") parser.add_argument('-o', '--launch_mode_options', help="extra options for the given launch mode, to be given in quotes with a leading space, e.g. ' --foo=3'") parser.add_argument('-p', '--plain', dest='plain', action='store_true', help="pass arguments to the 'run' command straight through to the program. Otherwise arguments of the form name=value can be used to overwrite default parameter values.") parser.add_argument('--no-plain', dest='plain', action='store_false', help="arguments to the 'run' command of the form name=value will overwrite default parameter values. This is the opposite of the --plain option.") @@ -538,9 +535,8 @@ def comment(argv): args = parser.parse_args(argv) if args.file: - f = open(args.comment, 'r') - comment = f.read() - f.close() + with open(args.comment, 'r') as f: + comment = f.read() else: comment = args.comment @@ -686,9 +682,8 @@ def upgrade(argv): project.record_store.clear() filename = "%s/records_export.json" % backup_dir if os.path.exists(filename): - f = open(filename) - project.record_store.import_(project.name, f.read()) - f.close() + with open(filename) as f: + project.record_store.import_(project.name, f.read()) else: print("Record file not found") sys.exit(1) diff --git a/sumatra/core.py b/sumatra/core.py index 761c462d..59e5ff42 100644 --- a/sumatra/core.py +++ b/sumatra/core.py @@ -1,15 +1,9 @@ """ -:copyright: Copyright 2006-2015 by the Sumatra team, see doc/authors.txt +:copyright: Copyright 2006-2020, 2024 by the Sumatra team, see doc/authors.txt :license: BSD 2-clause, see LICENSE for details. """ -from __future__ import unicode_literals -from builtins import str -from future.standard_library import install_aliases -install_aliases() -from builtins import object -from future.utils import with_metaclass import socket import sys @@ -20,7 +14,6 @@ from collections import OrderedDict from urllib.request import urlopen from urllib.error import URLError -import warnings import re @@ -111,7 +104,7 @@ def __call__(cls, *args, **kwargs): return cls.__instance -class _Registry(with_metaclass(SingletonType, object)): +class _Registry(metaclass=SingletonType): def __init__(self): self._components = {} diff --git a/sumatra/datastore/__init__.py b/sumatra/datastore/__init__.py index e25a8d0f..4e4039b4 100644 --- a/sumatra/datastore/__init__.py +++ b/sumatra/datastore/__init__.py @@ -22,10 +22,9 @@ constructor arguments. -:copyright: Copyright 2006-2015 by the Sumatra team, see doc/authors.txt +:copyright: Copyright 2006-2020, 2024 by the Sumatra team, see doc/authors.txt :license: BSD 2-clause, see LICENSE for details. """ -from __future__ import unicode_literals from .base import DataStore, DataKey, IGNORE_DIGEST from .filesystem import FileSystemDataStore diff --git a/sumatra/datastore/archivingfs.py b/sumatra/datastore/archivingfs.py index 6dc926ef..2fac28f7 100644 --- a/sumatra/datastore/archivingfs.py +++ b/sumatra/datastore/archivingfs.py @@ -3,12 +3,10 @@ tar files, then retrieved from the tar files. -:copyright: Copyright 2006-2015 by the Sumatra team, see doc/authors.txt +:copyright: Copyright 2006-2020, 2024 by the Sumatra team, see doc/authors.txt :license: BSD 2-clause, see LICENSE for details. """ -from __future__ import with_statement -from __future__ import unicode_literals import os import tarfile import shutil diff --git a/sumatra/datastore/base.py b/sumatra/datastore/base.py index 7c6d6db8..4a6f23b2 100644 --- a/sumatra/datastore/base.py +++ b/sumatra/datastore/base.py @@ -1,11 +1,9 @@ """ -:copyright: Copyright 2006-2015 by the Sumatra team, see doc/authors.txt +:copyright: Copyright 2006-2020, 2024 by the Sumatra team, see doc/authors.txt :license: BSD 2-clause, see LICENSE for details. """ -from __future__ import unicode_literals -from builtins import object import hashlib import os.path @@ -65,6 +63,9 @@ def contains_path(self, path): """Does the store contain a data item with the given path?""" raise NotImplementedError + def get_type(self): + return self.__class__.__name__ + class DataKey(object): """ diff --git a/sumatra/datastore/davfs.py b/sumatra/datastore/davfs.py index 48eeb3ac..e04d6d40 100644 --- a/sumatra/datastore/davfs.py +++ b/sumatra/datastore/davfs.py @@ -1,9 +1,6 @@ ''' Datastore via remote webdav connection ''' -from __future__ import unicode_literals -from future import standard_library -standard_library.install_aliases() import os import tarfile diff --git a/sumatra/datastore/filesystem.py b/sumatra/datastore/filesystem.py index cbee941a..94581007 100644 --- a/sumatra/datastore/filesystem.py +++ b/sumatra/datastore/filesystem.py @@ -2,10 +2,9 @@ Datastore based on files written to and retrieved from a local filesystem. -:copyright: Copyright 2006-2015 by the Sumatra team, see doc/authors.txt +:copyright: Copyright 2006-2020, 2024 by the Sumatra team, see doc/authors.txt :license: BSD 2-clause, see LICENSE for details. """ -from __future__ import unicode_literals import os import datetime @@ -36,12 +35,11 @@ def __init__(self, path, store, creation=None): self.mimetype, self.encoding = mimetypes.guess_type(self.full_path) def get_content(self, max_length=None): - f = open(self.full_path, 'rb') - if max_length: - content = f.read(max_length) - else: - content = f.read() - f.close() + with open(self.full_path, 'rb') as f: + if max_length: + content = f.read(max_length) + else: + content = f.read() return content content = property(fget=get_content) @@ -52,9 +50,8 @@ def sorted_content(self): cmd = "sort %s > %s" % (self.full_path, sorted_path) job = Popen(cmd, shell=True) job.wait() - f = open(sorted_path, 'rb') - content = f.read() - f.close() + with open(sorted_path, 'rb') as f: + content = f.read() if len(content) != self.size: # sort adds a \n if the file does not end with one assert len(content) == self.size + 1 content = content[:-1] @@ -73,6 +70,8 @@ class FileSystemDataStore(DataStore): data_item_class = DataFile def __init__(self, root): + if root: + root = os.path.expanduser(root) self.root = os.path.abspath(root or "./Data") def __str__(self): @@ -88,11 +87,7 @@ def __get_root(self): return self._root def __set_root(self, value): - try: - path = Path(value) - except TypeError: - # This can happen in Python2 if 'value' is a subclass of string - path = Path(unicode(value)) + path = Path(value) self._root = value if not path.exists(): try: diff --git a/sumatra/datastore/mirroredfs.py b/sumatra/datastore/mirroredfs.py index c213c006..ad12a8c8 100644 --- a/sumatra/datastore/mirroredfs.py +++ b/sumatra/datastore/mirroredfs.py @@ -6,13 +6,11 @@ user to take care of this. -:copyright: Copyright 2006-2015 by the Sumatra team, see doc/authors.txt +:copyright: Copyright 2006-2020, 2024 by the Sumatra team, see doc/authors.txt :license: BSD 2-clause, see LICENSE for details. """ -from __future__ import unicode_literals -from future.standard_library import install_aliases -install_aliases() +import datetime import os import mimetypes from urllib.request import urlopen diff --git a/sumatra/decorators.py b/sumatra/decorators.py index ccd422f9..649dfeab 100644 --- a/sumatra/decorators.py +++ b/sumatra/decorators.py @@ -8,12 +8,10 @@ def main([parameters and other args...]): -:copyright: Copyright 2006-2015 by the Sumatra team, see doc/authors.txt +:copyright: Copyright 2006-2020, 2024 by the Sumatra team, see doc/authors.txt :license: BSD 2-clause, see LICENSE for details. """ -from __future__ import unicode_literals -from builtins import str import time from sumatra.programs import PythonExecutable from sumatra.parameters import ParameterSet, SimpleParameterSet diff --git a/sumatra/dependency_finder/__init__.py b/sumatra/dependency_finder/__init__.py index 6458ff56..104138de 100644 --- a/sumatra/dependency_finder/__init__.py +++ b/sumatra/dependency_finder/__init__.py @@ -9,12 +9,10 @@ under version control. -:copyright: Copyright 2006-2015 by the Sumatra team, see doc/authors.txt +:copyright: Copyright 2006-2020, 2024 by the Sumatra team, see doc/authors.txt :license: BSD 2-clause, see LICENSE for details. """ -from __future__ import with_statement -from __future__ import unicode_literals import warnings from sumatra.dependency_finder import neuron, python, genesis, matlab, r diff --git a/sumatra/dependency_finder/core.py b/sumatra/dependency_finder/core.py index 33e4ab44..4f3624e9 100644 --- a/sumatra/dependency_finder/core.py +++ b/sumatra/dependency_finder/core.py @@ -17,11 +17,9 @@ series of functions in turn. -:copyright: Copyright 2006-2015 by the Sumatra team, see doc/authors.txt +:copyright: Copyright 2006-2020, 2024 by the Sumatra team, see doc/authors.txt :license: BSD 2-clause, see LICENSE for details. """ -from __future__ import unicode_literals -from builtins import object import os from sumatra import versioncontrol diff --git a/sumatra/dependency_finder/genesis.py b/sumatra/dependency_finder/genesis.py index e4071023..dfc49e74 100644 --- a/sumatra/dependency_finder/genesis.py +++ b/sumatra/dependency_finder/genesis.py @@ -23,14 +23,10 @@ find_version() -:copyright: Copyright 2006-2015 by the Sumatra team, see doc/authors.txt +:copyright: Copyright 2006-2020, 2024 by the Sumatra team, see doc/authors.txt :license: BSD 2-clause, see LICENSE for details. """ -from __future__ import with_statement -from __future__ import print_function -from __future__ import unicode_literals -from builtins import range import re import os from sumatra.dependency_finder import core diff --git a/sumatra/dependency_finder/matlab.py b/sumatra/dependency_finder/matlab.py index eb47bb3a..b05322bd 100644 --- a/sumatra/dependency_finder/matlab.py +++ b/sumatra/dependency_finder/matlab.py @@ -1,10 +1,9 @@ """ -:copyright: Copyright 2006-2015 by the Sumatra team, see doc/authors.txt +:copyright: Copyright 2006-2020, 2024 by the Sumatra team, see doc/authors.txt :license: BSD 2-clause, see LICENSE for details. """ -from __future__ import unicode_literals import os import re @@ -36,15 +35,15 @@ def save_dependencies(cmd, filename): def find_dependencies(filename, executable): #ifile = os.path.join(os.getcwd(), 'depfun.data') - file_data = (open('depfun.data', 'r')) - content = file_data.read() - paths = re.split('1: ', content)[2:] - list_deps = [] - for path in paths: - if os.name == 'posix': - list_data = path.split('/') - else: - list_data = path.split('\\') - list_deps.append(Dependency(list_data[-2], path.split('\n')[0])) - file_data.close() # TODO: find version of external toolboxes + with open('depfun.data', 'r') as file_data: + content = file_data.read() + paths = re.split('1: ', content)[2:] + list_deps = [] + for path in paths: + if os.name == 'posix': + list_data = path.split('/') + else: + list_data = path.split('\\') + list_deps.append(Dependency(list_data[-2], path.split('\n')[0])) + # TODO: find version of external toolboxes return list_deps diff --git a/sumatra/dependency_finder/neuron.py b/sumatra/dependency_finder/neuron.py index 0de0c659..a8661beb 100644 --- a/sumatra/dependency_finder/neuron.py +++ b/sumatra/dependency_finder/neuron.py @@ -24,12 +24,10 @@ find_version() -:copyright: Copyright 2006-2015 by the Sumatra team, see doc/authors.txt +:copyright: Copyright 2006-2020, 2024 by the Sumatra team, see doc/authors.txt :license: BSD 2-clause, see LICENSE for details. """ -from __future__ import with_statement -from __future__ import unicode_literals import re import os from sumatra.dependency_finder import core diff --git a/sumatra/dependency_finder/python.py b/sumatra/dependency_finder/python.py index 5e5e7072..3460b275 100644 --- a/sumatra/dependency_finder/python.py +++ b/sumatra/dependency_finder/python.py @@ -30,14 +30,10 @@ find_version() -:copyright: Copyright 2006-2015 by the Sumatra team, see doc/authors.txt +:copyright: Copyright 2006-2020, 2024 by the Sumatra team, see doc/authors.txt :license: BSD 2-clause, see LICENSE for details. """ -from __future__ import with_statement -from __future__ import print_function -from __future__ import unicode_literals -from builtins import str import os import sys from modulefinder import Module diff --git a/sumatra/dependency_finder/r.py b/sumatra/dependency_finder/r.py index e17f77d3..04afd688 100644 --- a/sumatra/dependency_finder/r.py +++ b/sumatra/dependency_finder/r.py @@ -1,20 +1,31 @@ """ -:copyright: Copyright 2006-2015 by the Sumatra team, see doc/authors.txt +:copyright: Copyright 2006-2020, 2024 by the Sumatra team, see doc/authors.txt :license: BSD 2-clause, see LICENSE for details. """ -from __future__ import unicode_literals import subprocess -import pkg_resources from sumatra.dependency_finder import core +import sys +if sys.version_info >= (3, 9): + import importlib.resources as importlib_resources +elif sys.version_info >= (3, 7): + import importlib_resources # Backport +else: + # pkg_resources is much slower than import_resources, and just doesn’t work with newer Python + # Migration to importlib based on https://importlib-resources.readthedocs.io/en/latest/migration.html + import pkg_resources + importlib_resources = None + package_split_str = 'pkg::\n' element_split_str = '\n' name_value_split_str = ':' -r_script_to_find_deps = pkg_resources.resource_filename("sumatra", "external_scripts/script_introspect.R") - +if importlib_resources: + r_script_to_find_deps = importlib_resources.files("sumatra") / "external_scripts/script_introspect.R" +else: + r_script_to_find_deps = pkg_resources.resource_filename("sumatra", "external_scripts/script_introspect.R") class Dependency(core.BaseDependency): @@ -38,7 +49,7 @@ def _get_r_dependencies(executable_path, rscriptfile, depfinder=r_script_to_find Rscript executable rscriptfile : path script file to be evaluated - rscriptfile : depfinder + depfinder : importlib.resources.abc.Traversable R script that finds dependencies pkg_split : str delimit packages in output @@ -59,12 +70,19 @@ def _get_r_dependencies(executable_path, rscriptfile, depfinder=r_script_to_find Raises ------ """ - parglist = [executable_path, depfinder, - rscriptfile, pkg_split, el_split, nv_split] - p = subprocess.Popen(parglist, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - result = p.wait() - output = p.stdout.read().decode("utf-8") - # import pdb; pdb.set_trace() + if importlib_resources: + with importlib_resources.as_file(depfinder) as depfinder_path: + parglist = [executable_path, depfinder_path, + rscriptfile, pkg_split, el_split, nv_split] + p = subprocess.Popen(parglist, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + result = p.wait() + output = p.stdout.read().decode("utf-8") + else: # Python < 3.7 + parglist = [executable_path, depfinder_path, + rscriptfile, pkg_split, el_split, nv_split] + p = subprocess.Popen(parglist, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + result = p.wait() + output = p.stdout.read().decode("utf-8") return result, output diff --git a/sumatra/formatting/__init__.py b/sumatra/formatting/__init__.py index 3a97c4e5..6ff14c0f 100644 --- a/sumatra/formatting/__init__.py +++ b/sumatra/formatting/__init__.py @@ -4,17 +4,13 @@ formats: currently text or HTML. -:copyright: Copyright 2006-2015 by the Sumatra team, see doc/authors.txt +:copyright: Copyright 2006-2020, 2024 by the Sumatra team, see doc/authors.txt :license: BSD 2-clause, see LICENSE for details. """ -from __future__ import unicode_literals -from builtins import zip -from builtins import str -from builtins import object import json import textwrap -import cgi +import html import re from ..core import component, component_type, get_registered_components import parameters @@ -22,7 +18,6 @@ import os - fields = ['label', 'timestamp', 'reason', 'outcome', 'duration', 'repository', 'main_file', 'version', 'script_arguments', 'executable', 'parameters', 'input_data', 'launch_mode', 'output_data', @@ -465,7 +460,7 @@ def long(self): def format_record(record): output = "
%s
\n
\n
\n" % record.label for field in fields: - output += "
%s
%s
\n" % (field, cgi.escape(str(getattr(record, field)))) + output += "
%s
%s
\n" % (field, html.escape(str(getattr(record, field)))) output += "
\n
" return output return "
\n" + "\n".join(format_record(record) for record in self.records) + "\n
" @@ -475,7 +470,7 @@ def table(self): Return detailed information about a list of records as an HTML table. """ def format_record(record): - return " \n " + "\n ".join(cgi.escape(str(getattr(record, field))) for field in fields) + " \n " + return " \n " + "\n ".join(html.escape(str(getattr(record, field))) for field in fields) + " \n " return "\n" + \ " \n \n \n \n" + \ "\n".join(format_record(record) for record in self.records) + \ diff --git a/sumatra/launch.py b/sumatra/launch.py index af048f82..b2da77ff 100644 --- a/sumatra/launch.py +++ b/sumatra/launch.py @@ -3,13 +3,9 @@ obtaining information about the platform(s) on which the simulations are run. -:copyright: Copyright 2006-2015 by the Sumatra team, see doc/authors.txt +:copyright: Copyright 2006-2020, 2024 by the Sumatra team, see doc/authors.txt :license: BSD 2-clause, see LICENSE for details. """ -from __future__ import print_function -from __future__ import unicode_literals -from builtins import range -from builtins import object import platform import socket @@ -64,7 +60,7 @@ class LaunchMode(object): required_attributes = ("check_files", "generate_command") def __init__(self, working_directory=None, options=None): - self.working_directory = working_directory or os.getcwd() + self.working_directory = os.path.expanduser(working_directory or os.getcwd()) self.options = options def __getstate__(self): @@ -92,7 +88,7 @@ def generate_command(self, paths): """Return a string containing the command to be launched.""" raise NotImplementedError("must be impemented by sub-classes") - def run(self, executable, main_file, arguments, append_label=None): + def run(self, executable, main_file, arguments, append_label=None, capture_stderr=True): """ Run a computation in a shell, with the given executable, script and arguments. If `append_label` is provided, it is appended to the @@ -107,7 +103,7 @@ def run(self, executable, main_file, arguments, append_label=None): dependencies in order to avoid opening of Matlab shell two times ''' result, output = save_dependencies(cmd, main_file) else: - result, output = tee.system2(cmd, cwd=self.working_directory, stdout=True) # cwd only relevant for local launch, not for MPI, for example + result, output = tee.system2(cmd, cwd=self.working_directory, stdout=True, capture_stderr=capture_stderr) # cwd only relevant for local launch, not for MPI, for example self.stdout_stderr = "".join(output) return result @@ -154,6 +150,9 @@ def get_platform_information(self): version=platform.version())] # maybe add system time? + def get_type(self): + return self.__class__.__name__ + @component class SerialLaunchMode(LaunchMode): @@ -188,6 +187,20 @@ def generate_command(self, executable, main_file, arguments): generate_command.__doc__ = LaunchMode.generate_command.__doc__ +@component +class SerialTqdmLaunchMode(SerialLaunchMode): + """ + Enable running with a tqdm progress bar. + Effectively this launch mode just disables the capture of stderr, which tqdm uses + for its output. + """ + name = "serial-tqdm" + + def run(self, *args, **kwargs): + return super().run(*args, capture_stderr=False, **kwargs) + run.__doc__ = LaunchMode.run.__doc__ + + @component class DistributedLaunchMode(LaunchMode): """ diff --git a/sumatra/parameters.py b/sumatra/parameters.py index 101723d1..6d642383 100644 --- a/sumatra/parameters.py +++ b/sumatra/parameters.py @@ -24,28 +24,18 @@ handles parameter files in YAML format -:copyright: Copyright 2006-2015 by the Sumatra team, see doc/authors.txt +:copyright: Copyright 2006-2020, 2024 by the Sumatra team, see doc/authors.txt :license: BSD 2-clause, see LICENSE for details. """ -from __future__ import with_statement, absolute_import -from __future__ import unicode_literals -from future import standard_library -standard_library.install_aliases() -from builtins import str -from builtins import object import os.path import shutil import abc import re from itertools import filterfalse from pathlib import Path -try: - from StringIO import StringIO # this is necessary because Python2-ConfigParser can't handle unicode -except ImportError: # Python 3 - from io import StringIO -from future.utils import with_metaclass -from configparser import SafeConfigParser, MissingSectionHeaderError, NoOptionError +from io import StringIO +from configparser import ConfigParser, MissingSectionHeaderError, NoOptionError import json try: import yaml @@ -59,12 +49,12 @@ @component_type -class ParameterSet(with_metaclass(abc.ABCMeta, object)): +class ParameterSet(metaclass=abc.ABCMeta): required_attributes = ("update", "save") list_pattern = re.compile(r'^\s*\[.*\]\s*$') tuple_pattern = re.compile(r'^\s*\(.*\)\s*$') if yaml_loaded: - casts = (yaml.load, ) # good behavior for all bool, at cost of dependency + casts = (yaml.safe_load, ) # good behavior for all bool, at cost of dependency else: casts = tuple() @@ -149,11 +139,11 @@ def __init__(self, initialiser): try: if os.path.exists(initialiser): with open(initialiser) as fid: - self.values = yaml.load(fid) + self.values = yaml.safe_load(fid) self.source_file = initialiser else: if initialiser: - self.values = yaml.load(initialiser) + self.values = yaml.safe_load(initialiser) else: self.values = {} except yaml.YAMLError: @@ -382,7 +372,7 @@ def update(self, E, **F): @component -class ConfigParserParameterSet(SafeConfigParser, ParameterSet): +class ConfigParserParameterSet(ConfigParser, ParameterSet): """ Handles parameter files in traditional config file format, as parsed by the standard Python ConfigParser module. Note that this format does not @@ -396,7 +386,7 @@ def __init__(self, initialiser): """ Create a new parameter set from a file or string. """ - SafeConfigParser.__init__(self) + ConfigParser.__init__(self) try: if os.path.exists(initialiser): self.read(initialiser) @@ -404,7 +394,7 @@ def __init__(self, initialiser): else: input = StringIO(str(initialiser)) # configparser has some problems with unicode. Using str() is a crude, and probably partial fix. input.seek(0) - self.readfp(input) + self.read_file(input) except MissingSectionHeaderError: raise SyntaxError("Initialiser contains no section headers") @@ -426,11 +416,6 @@ def __eq__(self, other): def __ne__(self, other): return not self.__eq__(other) - def __deepcopy__(self, memo): - # deepcopy of a SafeConfigParser fails under Python 2.7, so we - # implement this simple version which avoids copying SRE_Pattern objects - return ConfigParserParameterSet(self.pretty()) - def keys(self): return (section for section in self.sections()) diff --git a/sumatra/pfi.py b/sumatra/pfi.py index ac5c054f..12a8f5e1 100644 --- a/sumatra/pfi.py +++ b/sumatra/pfi.py @@ -4,10 +4,9 @@ This script should be placed somewhere on the user's path. -:copyright: Copyright 2006-2015 by the Sumatra team, see doc/authors.txt +:copyright: Copyright 2006-2020, 2024 by the Sumatra team, see doc/authors.txt :license: BSD 2-clause, see LICENSE for details. """ -from __future__ import unicode_literals from mpi4py import MPI import platform diff --git a/sumatra/programs.py b/sumatra/programs.py index 1f7dc783..f817f6ad 100644 --- a/sumatra/programs.py +++ b/sumatra/programs.py @@ -29,14 +29,10 @@ executable file or a script file that can be run with a given tool. -:copyright: Copyright 2006-2015 by the Sumatra team, see doc/authors.txt +:copyright: Copyright 2006-2020, 2024 by the Sumatra team, see doc/authors.txt :license: BSD 2-clause, see LICENSE for details. """ -from __future__ import with_statement -from __future__ import print_function -from __future__ import unicode_literals -from builtins import object import os.path import re import sys diff --git a/sumatra/projects.py b/sumatra/projects.py index 36e4b6c9..6568a990 100644 --- a/sumatra/projects.py +++ b/sumatra/projects.py @@ -17,15 +17,9 @@ load_project() - read project information from the working directory and return a Project object. -:copyright: Copyright 2006-2015 by the Sumatra team, see doc/authors.txt +:copyright: Copyright 2006-2020, 2024 by the Sumatra team, see doc/authors.txt :license: BSD 2-clause, see LICENSE for details. """ -from __future__ import print_function -from __future__ import unicode_literals -from future import standard_library -standard_library.install_aliases() -from builtins import str -from builtins import object import os import re @@ -147,15 +141,14 @@ def save(self): else: # Default value for unrecognised parameters attr = None - if hasattr(attr, "__getstate__"): + if hasattr(attr, "__getstate__") and attr.__getstate__() is not None: state[name] = {'type': attr.__class__.__module__ + "." + attr.__class__.__name__} for key, value in attr.__getstate__().items(): state[name][key] = value else: state[name] = attr - f = open(_get_project_file(self.path), 'w') # should check if file exists? - json.dump(state, f, indent=2) - f.close() + with open(_get_project_file(self.path), 'w') as f: # should check if file exists? + json.dump(state, f, indent=2) def info(self): """Show some basic information about the project.""" @@ -359,7 +352,7 @@ def add_comment(self, label, comment, replace=False): record = self.record_store.get(self.name, label) except Exception as e: raise Exception("%s. label=<%s>" % (e, label)) - if replace or record.outcome is "": + if replace or record.outcome == "": record.outcome = comment else: record.outcome = record.outcome + "\n" + comment @@ -389,9 +382,8 @@ def export(self): # copy the project data shutil.copy(".smt/project", ".smt/project_export.json") # export the record data - f = open(".smt/records_export.json", 'w') - f.write(self.record_store.export(self.name)) - f.close() + with open(".smt/records_export.json", 'w') as f: + f.write(self.record_store.export(self.name)) def repeat(self, original_label, new_label=None): if original_label == 'last': @@ -470,9 +462,9 @@ def remove_plugins(self, *plugins): def _load_project_from_json(path): - f = open(_get_project_file(path), 'r') - data = json.load(f) - f.close() + path = os.path.expanduser(path) + with open(_get_project_file(path), 'r') as f: + data = json.load(f) prj = Project.__new__(Project) prj.path = path for key, value in data.items(): @@ -498,9 +490,9 @@ def _load_project_from_json(path): def _load_project_from_pickle(path): # earlier versions of Sumatra saved Projects using pickle - f = open(_get_project_file(path), 'r') - prj = pickle.load(f) - f.close() + path = os.path.expanduser(path) + with open(_get_project_file(path), 'r') as f: + prj = pickle.load(f) return prj @@ -513,7 +505,7 @@ def load_project(path=None): if not path: p = os.getcwd() else: - p = os.path.abspath(path) + p = os.path.abspath(os.path.expanduser(path)) while not os.path.isdir(os.path.join(p, ".smt")): oldp, p = p, os.path.dirname(p) if p == oldp: diff --git a/sumatra/publishing/latex/includefigure.py b/sumatra/publishing/latex/includefigure.py index e172161b..19e14f71 100644 --- a/sumatra/publishing/latex/includefigure.py +++ b/sumatra/publishing/latex/includefigure.py @@ -1,18 +1,14 @@ """ -:copyright: Copyright 2006-2015 by the Sumatra team, see doc/authors.txt +:copyright: Copyright 2006-2020, 2024 by the Sumatra team, see doc/authors.txt :license: BSD 2-clause, see LICENSE for details. """ -from __future__ import print_function -from __future__ import unicode_literals -from future import standard_library -standard_library.install_aliases() import sys import os import logging -from configparser import SafeConfigParser +from configparser import ConfigParser from sumatra.publishing.utils import determine_project, determine_record_store, \ determine_project_name, get_image, \ record_link_url, get_record_label_and_image_path @@ -43,7 +39,7 @@ def generate_latex_command(sumatra_options, graphics_options): os.makedirs(LOCAL_IMAGE_CACHE) local_filename = image.save_copy(LOCAL_IMAGE_CACHE) - include_graphics_cmd = "\includegraphics" + include_graphics_cmd = r"\includegraphics" if graphics_options: include_graphics_cmd += "[%s]" % ",".join("%s=%s" % item for item in graphics_options.items()) include_graphics_cmd += "{%s}" % local_filename @@ -51,7 +47,7 @@ def generate_latex_command(sumatra_options, graphics_options): # if record_store is web-accessible, wrap the image in a hyperlink if hasattr(record_store, 'server_url'): target = record_link_url(record_store.server_url, project_name, record_label) - cmd = "\href{%s}{%s}" % (target, include_graphics_cmd) + cmd = r"\href{%s}{%s}" % (target, include_graphics_cmd) else: cmd = include_graphics_cmd @@ -59,7 +55,7 @@ def generate_latex_command(sumatra_options, graphics_options): def read_config(filename): - config = SafeConfigParser() + config = ConfigParser() config.read(filename) return dict(config.items("sumatra")), dict(config.items("graphics")) diff --git a/sumatra/publishing/sphinxext/__init__.py b/sumatra/publishing/sphinxext/__init__.py index d29f3c8c..95f84849 100644 --- a/sumatra/publishing/sphinxext/__init__.py +++ b/sumatra/publishing/sphinxext/__init__.py @@ -1,11 +1,9 @@ """ -:copyright: Copyright 2006-2015 by the Sumatra team, see doc/authors.txt +:copyright: Copyright 2006-2020, 2024 by the Sumatra team, see doc/authors.txt :license: BSD 2-clause, see LICENSE for details. """ -from __future__ import unicode_literals - from .sumatra_rst import SumatraImage, smt_link_role diff --git a/sumatra/publishing/sphinxext/sumatra_rst.py b/sumatra/publishing/sphinxext/sumatra_rst.py index 82709dc4..54cbad3d 100644 --- a/sumatra/publishing/sphinxext/sumatra_rst.py +++ b/sumatra/publishing/sphinxext/sumatra_rst.py @@ -14,10 +14,9 @@ The project name and recordstore directive are optional if rst2xxxx is used in a Sumatra project directory -:copyright: Copyright 2006-2015 by the Sumatra team, see doc/authors.txt +:copyright: Copyright 2006-2020, 2024 by the Sumatra team, see doc/authors.txt :license: BSD 2-clause, see LICENSE for details. """ -from __future__ import unicode_literals from docutils.parsers.rst import directives, states from docutils.parsers.rst.directives.images import Image @@ -142,8 +141,8 @@ def run(self): reference_node = nodes.reference(refuri=data) elif target_type == 'refname': reference_node = nodes.reference( - refname=fully_normalize_name(data), - name=whitespace_normalize_name(data)) + refname=nodes.fully_normalize_name(data), + name=nodes.whitespace_normalize_name(data)) reference_node.indirect_reference_name = data self.state.document.note_refname(reference_node) else: # malformed target diff --git a/sumatra/publishing/utils.py b/sumatra/publishing/utils.py index b318733e..b39d689d 100644 --- a/sumatra/publishing/utils.py +++ b/sumatra/publishing/utils.py @@ -2,11 +2,9 @@ Utility functions for use in publishing modules -:copyright: Copyright 2006-2015 by the Sumatra team, see doc/authors.txt +:copyright: Copyright 2006-2020, 2024 by the Sumatra team, see doc/authors.txt :license: BSD 2-clause, see LICENSE for details. """ -from __future__ import unicode_literals -from builtins import object import os import errno diff --git a/sumatra/records.py b/sumatra/records.py index 9d9f6cdf..64e45506 100644 --- a/sumatra/records.py +++ b/sumatra/records.py @@ -11,12 +11,9 @@ new_record() method of Project. -:copyright: Copyright 2006-2015 by the Sumatra team, see doc/authors.txt +:copyright: Copyright 2006-2020, 2024 by the Sumatra team, see doc/authors.txt :license: BSD 2-clause, see LICENSE for details. """ -from __future__ import print_function -from __future__ import unicode_literals -from builtins import object, str from datetime import datetime import time @@ -301,7 +298,8 @@ def __init__(self, recordA, recordB, self.main_file_differs = recordA.main_file != recordB.main_file self.version_differs = recordA.version != recordB.version for rec in recordA, recordB: - if rec.parameters: + # hasattr guard is for some malformed custom parameter type + if rec.parameters and hasattr(rec.parameters, 'pop'): rec.parameters.pop("sumatra_label", 1) self.parameters_differ = recordA.parameters != recordB.parameters self.script_arguments_differ = recordA.script_arguments != recordB.script_arguments diff --git a/sumatra/recordstore/__init__.py b/sumatra/recordstore/__init__.py index 45509cd5..42e4f209 100644 --- a/sumatra/recordstore/__init__.py +++ b/sumatra/recordstore/__init__.py @@ -11,10 +11,9 @@ http_store - provides the HttpRecordStore class -:copyright: Copyright 2006-2015 by the Sumatra team, see doc/authors.txt +:copyright: Copyright 2006-2020, 2024 by the Sumatra team, see doc/authors.txt :license: BSD 2-clause, see LICENSE for details. """ -from __future__ import unicode_literals from . import serialization from .base import RecordStore diff --git a/sumatra/recordstore/base.py b/sumatra/recordstore/base.py index 17b226b9..bfbe40a7 100644 --- a/sumatra/recordstore/base.py +++ b/sumatra/recordstore/base.py @@ -2,11 +2,9 @@ Provides base RecordStore class. -:copyright: Copyright 2006-2015 by the Sumatra team, see doc/authors.txt +:copyright: Copyright 2006-2020, 2024 by the Sumatra team, see doc/authors.txt :license: BSD 2-clause, see LICENSE for details. """ -from __future__ import unicode_literals -from builtins import object from sumatra.recordstore import serialization from sumatra.formatting import get_formatter diff --git a/sumatra/recordstore/django_store/__init__.py b/sumatra/recordstore/django_store/__init__.py index 73ed7cd0..5fd64b60 100644 --- a/sumatra/recordstore/django_store/__init__.py +++ b/sumatra/recordstore/django_store/__init__.py @@ -5,22 +5,15 @@ SQLite or PostgreSQL. -:copyright: Copyright 2006-2015 by the Sumatra team, see doc/authors.txt +:copyright: Copyright 2006-2020, 2024 by the Sumatra team, see doc/authors.txt :license: BSD 2-clause, see LICENSE for details. """ -from __future__ import absolute_import -from __future__ import unicode_literals -from future.standard_library import install_aliases -install_aliases() -from builtins import range -from builtins import object - import os import shutil from warnings import warn from textwrap import dedent -import imp +import importlib import django.conf as django_conf from django.core import management import django @@ -31,7 +24,7 @@ # Check that django-tagging is available. It would be better to try importing # it, but that seems to mess with Django's internals. -imp.find_module("tagging") +importlib.util.find_spec("tagging") def db_id(db): @@ -143,6 +136,7 @@ class DjangoRecordStore(RecordStore): """ def __init__(self, db_file='.smt/records'): + db_file = os.path.expanduser(db_file) self._db_label = db_config.add_database(db_file) self._db_file = db_file diff --git a/sumatra/recordstore/django_store/models.py b/sumatra/recordstore/django_store/models.py index cfa8137d..b5978e46 100644 --- a/sumatra/recordstore/django_store/models.py +++ b/sumatra/recordstore/django_store/models.py @@ -2,12 +2,9 @@ Definition of database tables and object retrieval for the DjangoRecordStore. -:copyright: Copyright 2006-2015 by the Sumatra team, see doc/authors.txt +:copyright: Copyright 2006-2020, 2024 by the Sumatra team, see doc/authors.txt :license: BSD 2-clause, see LICENSE for details. """ -from __future__ import unicode_literals -from builtins import str -from builtins import object import json from django.db import models @@ -15,7 +12,7 @@ from sumatra.datastore import get_data_store from datetime import datetime import django -from distutils.version import LooseVersion +from packaging.version import parse as parse_version from sumatra.core import get_registered_components import warnings with warnings.catch_warnings(): @@ -71,8 +68,8 @@ class Project(BaseModel): id = models.SlugField(primary_key=True) name = models.CharField(max_length=200) description = models.TextField(blank=True) - columns = ["Label", "Date/Time", "Reason", "Outcome", - "Input data", "Output data", "Duration", "Processes", + columns = ["Label", "Date/Time", "Reason", "Outcome", + "Input data", "Output data", "Duration", "Processes", "Executable", "Main", "Version", "Arguments", "Tags"] @@ -130,7 +127,7 @@ class Meta(object): class Repository(BaseModel): # the following should be unique together. type = models.CharField(max_length=100) - if LooseVersion(django.get_version()) < LooseVersion('1.5'): + if parse_version(django.get_version()) < parse_version('1.5'): url = models.URLField(verify_exists=False) upstream = models.URLField(verify_exists=False, null=True, blank=True) else: @@ -202,7 +199,7 @@ class DataKey(BaseModel): creation = models.DateTimeField(null=True, blank=True) metadata = models.TextField(blank=True) output_from_record = models.ForeignKey('Record', related_name='output_data', - null=True) + null=True, on_delete=models.CASCADE) class Meta(object): ordering = ('path',) @@ -235,7 +232,7 @@ class PlatformInformation(BaseModel): processor = models.CharField(max_length=100) release = models.CharField(max_length=100) system_name = models.CharField(max_length=20) - version = models.CharField(max_length=100) + version = models.CharField(max_length=200) def to_sumatra(self): pi = {} @@ -249,15 +246,15 @@ class Record(BaseModel): db_id = models.AutoField(primary_key=True) # django-tagging needs an integer as primary key - see http://code.google.com/p/django-tagging/issues/detail?id=15 reason = models.TextField(blank=True) duration = models.FloatField(null=True) - executable = models.ForeignKey(Executable, null=True, blank=True) # null and blank for the search. If user doesn't want to specify the executable during the search - repository = models.ForeignKey(Repository, null=True, blank=True) # null and blank for the search. + executable = models.ForeignKey(Executable, null=True, blank=True, on_delete=models.PROTECT) # null and blank for the search. If user doesn't want to specify the executable during the search + repository = models.ForeignKey(Repository, null=True, blank=True, on_delete=models.PROTECT) # null and blank for the search. main_file = models.CharField(max_length=100) version = models.CharField(max_length=50) - parameters = models.ForeignKey(ParameterSet) + parameters = models.ForeignKey(ParameterSet, on_delete=models.PROTECT) input_data = models.ManyToManyField(DataKey, related_name="input_to_records") - launch_mode = models.ForeignKey(LaunchMode) - datastore = models.ForeignKey(Datastore) - input_datastore = models.ForeignKey(Datastore, related_name="input_to_records") + launch_mode = models.ForeignKey(LaunchMode, on_delete=models.PROTECT) + datastore = models.ForeignKey(Datastore, on_delete=models.PROTECT) + input_datastore = models.ForeignKey(Datastore, related_name="input_to_records", on_delete=models.PROTECT) outcome = models.TextField(blank=True) timestamp = models.DateTimeField() tags = tagging.fields.TagField() @@ -265,7 +262,7 @@ class Record(BaseModel): platforms = models.ManyToManyField(PlatformInformation) diff = models.TextField(blank=True) user = models.CharField(max_length=100) - project = models.ForeignKey(Project, null=True) + project = models.ForeignKey(Project, null=True, on_delete=models.PROTECT) script_arguments = models.TextField(blank=True) stdout_stderr = models.TextField(blank=True) repeats = models.CharField(max_length=100, null=True, blank=True) diff --git a/sumatra/recordstore/http_store.py b/sumatra/recordstore/http_store.py index 0382db0c..1764231e 100644 --- a/sumatra/recordstore/http_store.py +++ b/sumatra/recordstore/http_store.py @@ -14,12 +14,9 @@ The required JSON structure can be seen in recordstore.serialization. -:copyright: Copyright 2006-2015 by the Sumatra team, see doc/authors.txt +:copyright: Copyright 2006-2020, 2024 by the Sumatra team, see doc/authors.txt :license: BSD 2-clause, see LICENSE for details. """ -from __future__ import unicode_literals -from future import standard_library -standard_library.install_aliases() from warnings import warn from urllib.parse import urlparse, urlunparse diff --git a/sumatra/recordstore/serialization.py b/sumatra/recordstore/serialization.py index fc038920..7157b8f3 100644 --- a/sumatra/recordstore/serialization.py +++ b/sumatra/recordstore/serialization.py @@ -2,11 +2,9 @@ Handles serialization/deserialization of record store contents to/from JSON. -:copyright: Copyright 2006-2015 by the Sumatra team, see doc/authors.txt +:copyright: Copyright 2006-2020, 2024 by the Sumatra team, see doc/authors.txt :license: BSD 2-clause, see LICENSE for details. """ -from __future__ import unicode_literals -from builtins import str import json from datetime import datetime diff --git a/sumatra/recordstore/shelve_store.py b/sumatra/recordstore/shelve_store.py index e8d0dee5..6e976ba8 100644 --- a/sumatra/recordstore/shelve_store.py +++ b/sumatra/recordstore/shelve_store.py @@ -2,11 +2,9 @@ Handles storage of simulation/analysis records based on the Python standard shelve module. -:copyright: Copyright 2006-2015 by the Sumatra team, see doc/authors.txt +:copyright: Copyright 2006-2020, 2024 by the Sumatra team, see doc/authors.txt :license: BSD 2-clause, see LICENSE for details. """ -from __future__ import unicode_literals -from builtins import str import os import shutil @@ -41,6 +39,7 @@ class ShelveRecordStore(RecordStore): """ def __init__(self, shelf_name=".smt/records"): + shelf_name = os.path.expanduser(shelf_name) self._shelf_name = shelf_name # Some shelve backends add an extension to the filename, and more than one # file may be created. So that the file(s) can be deleted, we need to try diff --git a/sumatra/tee.py b/sumatra/tee.py index 4453f590..476479a6 100644 --- a/sumatra/tee.py +++ b/sumatra/tee.py @@ -2,9 +2,7 @@ # encoding: utf-8 # Author: sorin sbarnea # License: public domain -from __future__ import print_function -from __future__ import unicode_literals -from builtins import str + import logging, sys, signal, subprocess, types, os, codecs, platform try: from time import process_time @@ -54,7 +52,7 @@ def system3(cmd): tf.close() return result, stdout_stderr -def system2(cmd, cwd=None, logger=_sentinel, stdout=_sentinel, log_command=_sentinel, timing=_sentinel): +def system2(cmd, cwd=None, logger=_sentinel, stdout=_sentinel, log_command=_sentinel, timing=_sentinel, capture_stderr=True): #def tee(cmd, cwd=None, logger=tee_logger, console=tee_console): """ This is a simple placement for os.system() or subprocess.Popen() that simulates how Unix tee() works - logging stdout/stderr using logging @@ -130,11 +128,16 @@ def nop(msg): if cwd is not None and not os.path.isdir(cwd): os.makedirs(cwd) # this throws exception if fails + if capture_stderr: + stderr = subprocess.STDOUT + else: + stderr = False + # samarkanov: commented 'quote_command' deliberately # reason: if I have 'quote_command' Sumatra does not work in Windows (it encloses the command in quotes. I did not understand why should we quote) # I have never catched "The input line is too long" (yet?) # cmd = quote_command(cmd) - p = subprocess.Popen(cmd, cwd=cwd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=(platform.system() == 'Linux')) + p = subprocess.Popen(cmd, cwd=cwd, shell=True, stdout=subprocess.PIPE, stderr=stderr, close_fds=(platform.system() == 'Linux')) if(log_command): mylogger("Running: %s" % cmd) try: diff --git a/sumatra/users.py b/sumatra/users.py index 952d3ba2..9f1f93f5 100644 --- a/sumatra/users.py +++ b/sumatra/users.py @@ -1,10 +1,9 @@ """ Find information about the current user. -:copyright: Copyright 2006-2015 by the Sumatra team, see doc/authors.txt +:copyright: Copyright 2006-2020, 2024 by the Sumatra team, see doc/authors.txt :license: BSD 2-clause, see LICENSE for details. """ -from __future__ import unicode_literals from os.path import expanduser, join, exists import json diff --git a/sumatra/versioncontrol/__init__.py b/sumatra/versioncontrol/__init__.py index 6b8754f6..fe716cff 100644 --- a/sumatra/versioncontrol/__init__.py +++ b/sumatra/versioncontrol/__init__.py @@ -29,10 +29,9 @@ object if so. -:copyright: Copyright 2006-2015 by the Sumatra team, see doc/authors.txt +:copyright: Copyright 2006-2020, 2024 by the Sumatra team, see doc/authors.txt :license: BSD 2-clause, see LICENSE for details. """ -from __future__ import unicode_literals import sys import os.path diff --git a/sumatra/versioncontrol/_bazaar.py b/sumatra/versioncontrol/_bazaar.py index 32850358..e8448147 100644 --- a/sumatra/versioncontrol/_bazaar.py +++ b/sumatra/versioncontrol/_bazaar.py @@ -8,14 +8,9 @@ BazaarRepository -:copyright: Copyright 2006-2015 by the Sumatra team, see doc/authors.txt +:copyright: Copyright 2006-2020, 2024 by the Sumatra team, see doc/authors.txt :license: BSD 2-clause, see LICENSE for details. """ -from __future__ import absolute_import -from __future__ import unicode_literals -from future import standard_library -standard_library.install_aliases() -from builtins import str from bzrlib.branch import Branch from bzrlib.workingtree import WorkingTree diff --git a/sumatra/versioncontrol/_git.py b/sumatra/versioncontrol/_git.py index 74c7b447..766d5c1f 100644 --- a/sumatra/versioncontrol/_git.py +++ b/sumatra/versioncontrol/_git.py @@ -8,21 +8,16 @@ GitRepository -:copyright: Copyright 2006-2015 by the Sumatra team, see doc/authors.txt +:copyright: Copyright 2006-2020, 2024 by the Sumatra team, see doc/authors.txt :license: BSD 2-clause, see LICENSE for details. """ -from __future__ import print_function -from __future__ import absolute_import -from __future__ import unicode_literals -from future import standard_library -standard_library.install_aliases() import logging import git import os import shutil import tempfile -from distutils.version import LooseVersion +from packaging.version import parse as parse_version from configparser import NoSectionError, NoOptionError try: from git.errors import InvalidGitRepositoryError, NoSuchPathError @@ -41,7 +36,7 @@ def check_version(): "GitPython not installed. There is a 'git' package, but it is not " "GitPython (https://pypi.python.org/pypi/GitPython/)") minimum_version = '0.3.5' - if LooseVersion(git.__version__) < LooseVersion(minimum_version): + if parse_version(git.__version__) < parse_version(minimum_version): raise VersionControlError( "Your Git Python binding is too old. You require at least " "version {0}. You can install the latest version e.g. via " @@ -84,7 +79,7 @@ def current_version(self): def use_version(self, version): logger.debug("Using git version: %s" % version) - if version is not 'master': + if version != 'master': assert not self.has_changed() g = git.Git(self.path) g.checkout(version) diff --git a/sumatra/versioncontrol/_mercurial.py b/sumatra/versioncontrol/_mercurial.py index 71571ee3..7bc7035d 100644 --- a/sumatra/versioncontrol/_mercurial.py +++ b/sumatra/versioncontrol/_mercurial.py @@ -8,12 +8,9 @@ MercurialRepository -:copyright: Copyright 2006-2015 by the Sumatra team, see doc/authors.txt +:copyright: Copyright 2006-2020, 2024 by the Sumatra team, see doc/authors.txt :license: BSD 2-clause, see LICENSE for details. """ -from __future__ import print_function -from __future__ import absolute_import -from __future__ import unicode_literals import hgapi import os diff --git a/sumatra/versioncontrol/_subversion.py b/sumatra/versioncontrol/_subversion.py index ef743f4e..f5113a7f 100644 --- a/sumatra/versioncontrol/_subversion.py +++ b/sumatra/versioncontrol/_subversion.py @@ -8,14 +8,9 @@ SubversionRepository -:copyright: Copyright 2006-2015 by the Sumatra team, see doc/authors.txt +:copyright: Copyright 2006-2020, 2024 by the Sumatra team, see doc/authors.txt :license: BSD 2-clause, see LICENSE for details. """ -from __future__ import absolute_import -from __future__ import unicode_literals -from future import standard_library -standard_library.install_aliases() -from builtins import str import pysvn import os diff --git a/sumatra/versioncontrol/base.py b/sumatra/versioncontrol/base.py index e39551fc..dec27526 100644 --- a/sumatra/versioncontrol/base.py +++ b/sumatra/versioncontrol/base.py @@ -2,11 +2,9 @@ Define the base classes for the Sumatra version control abstraction layer. -:copyright: Copyright 2006-2015 by the Sumatra team, see doc/authors.txt +:copyright: Copyright 2006-2020, 2024 by the Sumatra team, see doc/authors.txt :license: BSD 2-clause, see LICENSE for details. """ -from __future__ import unicode_literals -from builtins import object import os.path from sumatra.core import component_type @@ -33,7 +31,7 @@ class Repository(object): def __init__(self, url, upstream=None): if url == ".": - url = os.path.abspath(url) + url = os.path.abspath(os.path.expanduser(url)) self.url = url self.upstream = upstream diff --git a/sumatra/web/__init__.py b/sumatra/web/__init__.py index e26c5edd..6708409d 100644 --- a/sumatra/web/__init__.py +++ b/sumatra/web/__init__.py @@ -2,4 +2,3 @@ The web sub-package provides the Sumatra web interface. It is based on the Django framework and requires a DjangoRecordStore to be used for record storage. """ -from __future__ import unicode_literals diff --git a/sumatra/web/templatetags/filters.py b/sumatra/web/templatetags/filters.py index b8f35cc7..c6902455 100644 --- a/sumatra/web/templatetags/filters.py +++ b/sumatra/web/templatetags/filters.py @@ -1,9 +1,8 @@ """ -:copyright: Copyright 2006-2015 by the Sumatra team, see doc/authors.txt +:copyright: Copyright 2006-2020, 2024 by the Sumatra team, see doc/authors.txt :license: BSD 2-clause, see LICENSE for details. """ -from __future__ import unicode_literals import os from django import template diff --git a/sumatra/web/urls.py b/sumatra/web/urls.py index 6ce7e6e5..dfce76df 100644 --- a/sumatra/web/urls.py +++ b/sumatra/web/urls.py @@ -1,44 +1,60 @@ """ Define URL dispatching for the Sumatra web interface. -:copyright: Copyright 2006-2015 by the Sumatra team, see doc/authors.txt +:copyright: Copyright 2006-2020, 2024 by the Sumatra team, see doc/authors.txt :license: BSD 2-clause, see LICENSE for details. """ -from __future__ import unicode_literals -from django.conf.urls import patterns +from django.urls import re_path from django.contrib.staticfiles.urls import staticfiles_urlpatterns from sumatra.projects import Project from sumatra.records import Record -from sumatra.web.views import (ProjectListView, ProjectDetailView, RecordListView, - RecordDetailView, DataListView, DataDetailView, - ImageListView, SettingsView, DiffView) +from sumatra.web.views import ( + ProjectListView, + ProjectDetailView, + RecordListView, + RecordDetailView, + DataListView, + DataDetailView, + ImageListView, + SettingsView, + DiffView, + image_thumbgrid, + parameter_list, + delete_records, + compare_records, + show_script, + datatable_record, + datatable_data, + datatable_image, + show_content +) P = { - 'project': Project.valid_name_pattern, - 'label': Record.valid_name_pattern, + "project": Project.valid_name_pattern, + "label": Record.valid_name_pattern, } -urlpatterns = patterns('', - (r'^$', ProjectListView.as_view()), - (r'^settings/$', SettingsView.as_view()), - (r'^%(project)s/$' % P, RecordListView.as_view()), - (r'^%(project)s/about/$' % P, ProjectDetailView.as_view()), - (r'^%(project)s/data/$' % P, DataListView.as_view()), - (r'^%(project)s/image/$' % P, ImageListView.as_view()), - (r'^%(project)s/image/thumbgrid$' % P, 'sumatra.web.views.image_thumbgrid'), - (r'^%(project)s/parameter$' % P, 'sumatra.web.views.parameter_list'), - (r'^%(project)s/delete/$' % P, 'sumatra.web.views.delete_records'), - (r'^%(project)s/compare/$' % P, 'sumatra.web.views.compare_records'), - (r'^%(project)s/%(label)s/$' % P, RecordDetailView.as_view()), - (r'^%(project)s/%(label)s/diff$' % P, DiffView.as_view()), - (r'^%(project)s/%(label)s/diff/(?P[\w_]+)*$' % P, DiffView.as_view()), - (r'^%(project)s/%(label)s/script$' % P, 'sumatra.web.views.show_script'), - (r'^%(project)s/data/datafile$' % P, DataDetailView.as_view()), - (r'^%(project)s/datatable/record$' % P, 'sumatra.web.views.datatable_record'), - (r'^%(project)s/datatable/data$' % P, 'sumatra.web.views.datatable_data'), - (r'^%(project)s/datatable/image$' % P, 'sumatra.web.views.datatable_image'), - (r'^data/(?P\d+)$', 'sumatra.web.views.show_content'), - ) +urlpatterns = [ + re_path(r"^$", ProjectListView.as_view()), + re_path(r"^settings/$", SettingsView.as_view()), + re_path(r"^%(project)s/$" % P, RecordListView.as_view()), + re_path(r"^%(project)s/about/$" % P, ProjectDetailView.as_view()), + re_path(r"^%(project)s/data/$" % P, DataListView.as_view()), + re_path(r"^%(project)s/image/$" % P, ImageListView.as_view()), + re_path(r"^%(project)s/image/thumbgrid$" % P, image_thumbgrid), + re_path(r"^%(project)s/parameter$" % P, parameter_list), + re_path(r"^%(project)s/delete/$" % P, delete_records), + re_path(r"^%(project)s/compare/$" % P, compare_records), + re_path(r"^%(project)s/%(label)s/$" % P, RecordDetailView.as_view()), + re_path(r"^%(project)s/%(label)s/diff$" % P, DiffView.as_view()), + re_path(r"^%(project)s/%(label)s/diff/(?P[\w_]+)*$" % P, DiffView.as_view()), + re_path(r"^%(project)s/%(label)s/script$" % P, show_script), + re_path(r"^%(project)s/data/datafile$" % P, DataDetailView.as_view()), + re_path(r"^%(project)s/datatable/record$" % P, datatable_record), + re_path(r"^%(project)s/datatable/data$" % P, datatable_data), + re_path(r"^%(project)s/datatable/image$" % P, datatable_image), + re_path(r"^data/(?P\d+)$", show_content), +] urlpatterns += staticfiles_urlpatterns() diff --git a/sumatra/web/views.py b/sumatra/web/views.py index 4ae27cd4..d81ed782 100644 --- a/sumatra/web/views.py +++ b/sumatra/web/views.py @@ -1,13 +1,9 @@ """ Defines views for the Sumatra web interface. -:copyright: Copyright 2006-2015 by the Sumatra team, see doc/authors.txt +:copyright: Copyright 2006-2020, 2024 by the Sumatra team, see doc/authors.txt :license: BSD 2-clause, see LICENSE for details. """ -from __future__ import print_function -from __future__ import absolute_import -from __future__ import unicode_literals -from builtins import str import parameters import mimetypes @@ -185,9 +181,11 @@ def handle_plain_text(self, context, content): return context def handle_zipfile(self, context, content): + import io import zipfile - if zipfile.is_zipfile(path): - zf = zipfile.ZipFile(path, 'r') + fp = io.StringIO(content) + if zipfile.is_zipfile(fp): + zf = zipfile.ZipFile(fp, 'r') contents = zf.namelist() zf.close() context["content"] = "\n".join(contents) diff --git a/test/build_example_project.py b/test/build_example_project.py index 946d698d..f533fa04 100644 --- a/test/build_example_project.py +++ b/test/build_example_project.py @@ -1,6 +1,3 @@ -from __future__ import print_function -from __future__ import absolute_import -from __future__ import unicode_literals import sys, os from .smt_test import run_test diff --git a/test/build_fake_store.py b/test/build_fake_store.py index 1a91d4ea..db8aa146 100644 --- a/test/build_fake_store.py +++ b/test/build_fake_store.py @@ -1,5 +1,3 @@ -from __future__ import unicode_literals -from builtins import range from sumatra.projects import Project from sumatra.records import Record from sumatra.recordstore import django_store @@ -11,7 +9,7 @@ import random serial = SerialLaunchMode() -executable = PythonExecutable("/usr/bin/python", version="2.7") +executable = PythonExecutable("/usr/bin/python", version="3.11") repos = GitRepository('.') datastore = FileSystemDataStore("/path/to/datastore") project = Project("test_project", diff --git a/test/example_projects/python/main.py b/test/example_projects/python/main.py index 6d361eda..c99d1653 100644 --- a/test/example_projects/python/main.py +++ b/test/example_projects/python/main.py @@ -1,10 +1,20 @@ -from __future__ import unicode_literals -from past.builtins import execfile import numpy import sys __version__ = "1.2.3a" + +def execfile(filepath, globals=None, locals=None): + if globals is None: + globals = {} + globals.update({ + "__file__": filepath, + "__name__": "__main__", + }) + with open(filepath, 'rb') as fp: + exec(compile(fp.read(), filepath, 'exec'), globals, locals) + + def get_version(): # version numbers are deliberately different, for testing purposes return (1, 2, "3b") diff --git a/test/example_repositories/bazaar/main.py b/test/example_repositories/bazaar/main.py index 6d361eda..7bc248ed 100644 --- a/test/example_repositories/bazaar/main.py +++ b/test/example_repositories/bazaar/main.py @@ -1,10 +1,19 @@ -from __future__ import unicode_literals -from past.builtins import execfile import numpy import sys __version__ = "1.2.3a" +def execfile(filepath, globals=None, locals=None): + if globals is None: + globals = {} + globals.update({ + "__file__": filepath, + "__name__": "__main__", + }) + with open(filepath, 'rb') as fp: + exec(compile(fp.read(), filepath, 'exec'), globals, locals) + + def get_version(): # version numbers are deliberately different, for testing purposes return (1, 2, "3b") diff --git a/test/system/fixtures/Dockerfile.postgres b/test/system/fixtures/Dockerfile.postgres index c0945d01..5f3292e0 100644 --- a/test/system/fixtures/Dockerfile.postgres +++ b/test/system/fixtures/Dockerfile.postgres @@ -5,12 +5,12 @@ # # Usage: docker build -t postgresql_test -f Dockerfile.postgres . -FROM debian:jessie -MAINTAINER andrew.davison@unic.cnrs-gif.fr +FROM debian:bullseye +MAINTAINER andrew.davison@cnrs.fr RUN apt-get update -RUN apt-get -y -q install python-software-properties software-properties-common -RUN apt-get -y -q install postgresql-9.4 postgresql-client-9.4 postgresql-contrib-9.4 +RUN apt-get -y -q install software-properties-common +RUN apt-get -y -q install postgresql-13 postgresql-client-13 postgresql-contrib-13 USER postgres RUN /etc/init.d/postgresql start &&\ @@ -18,9 +18,9 @@ RUN /etc/init.d/postgresql start &&\ createdb -O docker sumatra_test # Adjust PostgreSQL configuration so that remote connections to the -# database are possible. -RUN echo "host all all 0.0.0.0/0 md5" >> /etc/postgresql/9.4/main/pg_hba.conf -RUN echo "listen_addresses='*'" >> /etc/postgresql/9.4/main/postgresql.conf +# database are possible. +RUN echo "host all all 0.0.0.0/0 md5" >> /etc/postgresql/13/main/pg_hba.conf +RUN echo "listen_addresses='*'" >> /etc/postgresql/13/main/postgresql.conf EXPOSE 5432 @@ -28,4 +28,4 @@ EXPOSE 5432 VOLUME ["/etc/postgresql", "/var/log/postgresql", "/var/lib/postgresql"] # Set the default command to run when starting the container -CMD ["/usr/lib/postgresql/9.4/bin/postgres", "-D", "/var/lib/postgresql/9.4/main", "-c", "config_file=/etc/postgresql/9.4/main/postgresql.conf"] +CMD ["/usr/lib/postgresql/13/bin/postgres", "-D", "/var/lib/postgresql/13/main", "-c", "config_file=/etc/postgresql/13/main/postgresql.conf"] diff --git a/test/system/fixtures/Dockerfile.webdav b/test/system/fixtures/Dockerfile.webdav index e7f934ee..0c6af63b 100644 --- a/test/system/fixtures/Dockerfile.webdav +++ b/test/system/fixtures/Dockerfile.webdav @@ -4,8 +4,8 @@ # # Usage: docker build -t webdav_test -f Dockerfile.webdav . -FROM debian:jessie -MAINTAINER andrew.davison@unic.cnrs-gif.fr +FROM debian:bullseye +MAINTAINER andrew.davison@cnrs.fr RUN apt-get update RUN apt-get -y -q install apache2 diff --git a/test/system/test_http_store.py b/test/system/test_http_store.py index 3f2daa5c..b5f140fb 100644 --- a/test/system/test_http_store.py +++ b/test/system/test_http_store.py @@ -1,152 +1,132 @@ """ Tests of the HttpRecordStore -Usage: - nosetests -v test_http_store.py -or: - python test_http_store.py """ -from __future__ import print_function -from __future__ import unicode_literals -from future import standard_library -from builtins import input -standard_library.install_aliases() - +from functools import partial import os -from urllib.parse import urlparse +import shutil +import tempfile try: import docker - if "DOCKER_HOST" in os.environ: - have_docker = True - else: - have_docker = False + have_docker = True except ImportError: have_docker = False -import utils -from utils import (setup, teardown as default_teardown, run_test, build_command, assert_file_exists, assert_in_output, - assert_config, assert_label_equal, assert_records, edit_parameters, +from utils import (run_test, build_command, assert_in_output, + assert_label_equal, assert_records, edit_parameters, expected_short_list, substitute_labels) -from functools import partial -#repository = "https://bitbucket.org/apdavison/ircr2013" -repository = "/Users/andrew/dev/ircr2013" -image = "apdavison/sumatra-server-v4" +import pytest -ctr = None -dkr = None +#repository = "https://github.com/apdavison/ircr2013" +repository = "/Users/adavison/projects/projectglass-git" +DOCKER_IMAGE = "apdavison/sumatra-server-v4" -def get_url(): - info = dkr.containers()[0] - assert info["Image"] == image - host = urlparse(dkr.base_url).hostname - return "{0}:{1}".format(host, info["Ports"][0]["PublicPort"]) +def get_url(ctr): + ctr.reload() # required to get auto-assigned ports + info = ctr.ports["80/tcp"][0] + return f"{info['HostIp']}:{info['HostPort']}" -def start_server(): +@pytest.fixture(scope="module") +def server(): """ Launch a Docker container running Sumatra Server, return the URL. - Requires the "apdavison/sumatra-server-v4" image from the Docker Index. + Requires the "apdavison/sumatra-server-v4" DOCKER_IMAGE from the Docker Index. """ - global ctr, dkr - env = docker.utils.kwargs_from_env(assert_hostname=False) - dkr = docker.Client(timeout=60, **env) - # docker run --rm -P --name smtserve apdavison/sumatra-server-v4 - ctr = dkr.create_container(image, command=None, hostname=None, user=None, - detach=False, stdin_open=False, tty=False, - ports=None, environment=None, dns=None, volumes=None, - volumes_from=None, network_disabled=False, name=None, - entrypoint=None, cpu_shares=None, working_dir=None) - dkr.start(ctr, publish_all_ports=True) - utils.env["url"] = get_url() - - -def teardown(): - global ctr, dkr - default_teardown() - if dkr is not None: - dkr.stop(ctr) - -##utils.env["url"] = "127.0.0.1:8642" - - -test_steps = [ - start_server, - ("Get the example code", - "hg clone %s ." % repository, - assert_in_output, "updating to branch default"), - ("Set up a Sumatra project", - build_command("smt init --store=http://testuser:abc123@{}/records/ -m glass_sem_analysis.py -e python -d Data -i . ProjectGlass", "url")), - ("Run the ``glass_sem_analysis.py`` script with Sumatra", - "smt run -r 'initial run' default_parameters MV_HFV_012.jpg", - assert_in_output, ("2416.86315789 60.0", "histogram.png")), - ("Comment on the outcome", - "smt comment 'works fine'"), - edit_parameters("default_parameters", "no_filter", "filter_size", 1), - ("Run with changed parameters and user-defined label", - "smt run -l example_label -r 'No filtering' no_filter MV_HFV_012.jpg", # TODO: assert(results have changed) - assert_in_output, "phases.png", - assert_label_equal, "example_label"), - ("Change parameters from the command line", - "smt run -r 'Trying a different colourmap' default_parameters MV_HFV_012.jpg phases_colourmap=hot"), # assert(results have changed) - ("Add another comment", - "smt comment 'The default colourmap is nicer'"), #TODO add a comment to an older record (e.g. this colourmap is nicer than 'hot')") - ("Add tags on the command line", - build_command("smt tag mytag {0} {1}", "labels")), - ("Review previous computations - get a list of labels", - "smt list", - assert_in_output, expected_short_list), - ("Review previous computations in detail", - "smt list -l", - assert_records, substitute_labels([ - {'label': 0, 'executable_name': 'Python', 'outcome': 'works fine', 'reason': 'initial run', - 'version': '6038f9c500d1', 'vcs': 'Mercurial', 'script_arguments': ' MV_HFV_012.jpg', - 'main_file': 'glass_sem_analysis.py'}, # TODO: add checking of parameters - {'label': 1, 'outcome': '', 'reason': 'No filtering'}, - {'label': 2, 'outcome': 'The default colourmap is nicer', 'reason': 'Trying a different colourmap'}, - ])), - # ("Filter the output of ``smt list`` based on tag", - # "smt list mytag", - # #assert(list is correct) - # ), - # ("Export Sumatra records as JSON.", - # "smt export", - # assert_file_exists, ".smt/records_export.json"), - # ("Change to a local record store", - # "smt configure --store=sumatra.sqlite"), - # ("Check the list of labels is unchanged", - # "smt list", - # assert_in_output, expected_short_list), - # ("Run another computation, which will only be captured by the local record store", - # "smt repeat --label=repeated_example example_label"), - # ("Switch back to the remote record store", - # build_command("smt configure --store=http://testuser:abc123@{}/records/", "url")), - # ("Check that all records are listed", - # "smt list -l", - # assert_records, substitute_labels([ - # {'label': 0, 'executable_name': 'Python', 'outcome': 'works fine', 'reason': 'initial run', - # 'version': '6038f9c500d1', 'vcs': 'Mercurial', 'script_args': ' MV_HFV_012.jpg', - # 'main': 'glass_sem_analysis.py'}, # TODO: add checking of parameters - # {'label': 1, 'outcome': '', 'reason': 'No filtering'}, - # {'label': 2, 'outcome': 'The default colourmap is nicer', 'reason': 'Trying a different colourmap'}, - # {'label': 3, 'outcome': 'The new record exactly matches the original.'}, - # ])), -] - - -def test_all(): - """Test generator for Nose.""" + dkr = docker.from_env(timeout=60) + ctr = dkr.containers.run(DOCKER_IMAGE, detach=True, publish_all_ports=True) + container_url = get_url(ctr) + yield container_url + ctr.stop() + + +def test_all(server): + """Run a series of Sumatra commands""" + + temporary_dir = os.path.realpath(tempfile.mkdtemp()) + working_dir = os.path.join(temporary_dir, "sumatra_exercise") + os.mkdir(working_dir) + env = { + "labels": [], + "working_dir": working_dir, + "url": server + } + + test_steps = [ + ("Get the example code", + "git clone %s ." % repository, + assert_in_output, "done."), + ("Set up a Sumatra project", + build_command("smt init --store=http://testuser:abc123@{}/records/ -m glass_sem_analysis.py -e python -d Data -i . ProjectGlass", "url")), + ("Run the ``glass_sem_analysis.py`` script with Sumatra", + "smt run -r 'initial run' default_parameters MV_HFV_012.jpg", + assert_in_output, ("2416.86315789", "histogram.png")), + ("Comment on the outcome", + "smt comment 'works fine'"), + edit_parameters("default_parameters", "no_filter", "filter_size", 1, working_dir), + ("Run with changed parameters and user-defined label", + "smt run -l example_label -r 'No filtering' no_filter MV_HFV_012.jpg", # TODO: assert(results have changed) + assert_in_output, "phases.png", + assert_label_equal, "example_label"), + ("Change parameters from the command line", + "smt run -r 'Trying a different colourmap' default_parameters MV_HFV_012.jpg phases_colourmap=hot"), # assert(results have changed) + ("Add another comment", + "smt comment 'The default colourmap is nicer'"), #TODO add a comment to an older record (e.g. this colourmap is nicer than 'hot')") + ("Add tags on the command line", + build_command("smt tag mytag {0} {1}", "labels")), + ("Review previous computations - get a list of labels", + "smt list", + assert_in_output, expected_short_list), + ("Review previous computations in detail", + "smt list -l", + assert_records, substitute_labels([ + {'label': 0, 'executable_name': 'Python', 'outcome': 'works fine', 'reason': 'initial run', + 'version': '6038f9c500d1', 'vcs': 'Mercurial', 'script_arguments': ' MV_HFV_012.jpg', + 'main_file': 'glass_sem_analysis.py'}, # TODO: add checking of parameters + {'label': 1, 'outcome': '', 'reason': 'No filtering'}, + {'label': 2, 'outcome': 'The default colourmap is nicer', 'reason': 'Trying a different colourmap'}, + ])), + # ("Filter the output of ``smt list`` based on tag", + # "smt list mytag", + # #assert(list is correct) + # ), + # ("Export Sumatra records as JSON.", + # "smt export", + # assert_file_exists, ".smt/records_export.json"), + # ("Change to a local record store", + # "smt configure --store=sumatra.sqlite"), + # ("Check the list of labels is unchanged", + # "smt list", + # assert_in_output, expected_short_list), + # ("Run another computation, which will only be captured by the local record store", + # "smt repeat --label=repeated_example example_label"), + # ("Switch back to the remote record store", + # build_command("smt configure --store=http://testuser:abc123@{}/records/", "url")), + # ("Check that all records are listed", + # "smt list -l", + # assert_records, substitute_labels([ + # {'label': 0, 'executable_name': 'Python', 'outcome': 'works fine', 'reason': 'initial run', + # 'version': '6038f9c500d1', 'vcs': 'Mercurial', 'script_args': ' MV_HFV_012.jpg', + # 'main': 'glass_sem_analysis.py'}, # TODO: add checking of parameters + # {'label': 1, 'outcome': '', 'reason': 'No filtering'}, + # {'label': 2, 'outcome': 'The default colourmap is nicer', 'reason': 'Trying a different colourmap'}, + # {'label': 3, 'outcome': 'The new record exactly matches the original.'}, + # ])), + ] + for step in test_steps: if callable(step): step() else: - test = partial(*tuple([run_test] + list(step[1:]))) - test.description = step[0] - yield test + run_test(*step[1:], env=env) + + shutil.rmtree(temporary_dir) + # Still to test: # @@ -158,19 +138,3 @@ def test_all(): #.. repeats #.. moving forwards and backwards in history #.. upgrades (needs Docker) - - -if __name__ == '__main__': - # Run the tests without using Nose. - setup() - for step in test_steps: - if callable(step): - step() - else: - print(step[0]) # description - run_test(*step[1:]) - response = input("Do you want to delete the temporary directory (default: yes)? ") - if response not in ["n", "N", "no", "No"]: - teardown() - else: - print("Temporary directory %s not removed" % utils.temporary_dir) diff --git a/test/system/test_ircr.py b/test/system/test_ircr.py index d64d2a88..96bb17fa 100644 --- a/test/system/test_ircr.py +++ b/test/system/test_ircr.py @@ -5,125 +5,120 @@ electron microscope (SEM) images of glass samples. This example was taken from an online SciPy tutorial at http://scipy-lectures.github.com/intro/summary-exercises/image-processing.html -Usage: - nosetests -v test_ircr.py -or: - python test_ircr.py """ -from __future__ import print_function -from __future__ import unicode_literals -from builtins import input # Requirements: numpy, scipy, matplotlib, mercurial, sarge import os from datetime import datetime -import utils -from utils import (setup, teardown, run_test, build_command, assert_file_exists, assert_in_output, +import shutil +import tempfile +from utils import (run_test, build_command, assert_file_exists, assert_in_output, assert_config, assert_label_equal, assert_records, assert_return_code, edit_parameters, expected_short_list, substitute_labels) -from functools import partial import re -repository = "https://bitbucket.org/apdavison/ircr2013" -#repository = "/Volumes/USERS/andrew/dev/ircr2013" # during development -#repository = "/Users/andrew/dev/ircr2013" +#repository = "https://github.com/apdavison/ircr2013" +repository = "/Users/adavison/projects/projectglass-git" -def modify_script(filename): - def wrapped(): - with open(os.path.join(utils.working_dir, filename), 'r') as fp: - script = fp.readlines() - with open(os.path.join(utils.working_dir, filename), 'w') as fp: - for line in script: - if "print(mean_bubble_size, median_bubble_size)" in line: - fp.write('print("Mean:", mean_bubble_size)\n') - fp.write('print("Median:", median_bubble_size)\n') - else: - fp.write(line) - return wrapped +def test_all(): + """Test a sequence of smt commands""" + temporary_dir = os.path.realpath(tempfile.mkdtemp()) + working_dir = os.path.join(temporary_dir, "sumatra_exercise") + os.mkdir(working_dir) -test_steps = [ - ("Get the example code", - "hg clone %s ." % repository, - assert_in_output, "updating to branch default"), - ("Run the computation without Sumatra", - "python glass_sem_analysis.py default_parameters MV_HFV_012.jpg", - assert_in_output, re.compile(r"2416\.863[0-9]* 60\.0"), - assert_file_exists, os.path.join("Data", datetime.now().strftime("%Y%m%d")), # Data subdirectory contains another subdirectory labelled with today's date) - ), # assert(subdirectory contains three image files). - ("Set up a Sumatra project", - "smt init -d Data -i . ProjectGlass", - assert_in_output, "Sumatra project successfully set up"), - ("Run the ``glass_sem_analysis.py`` script with Sumatra", - "smt run -e python -m glass_sem_analysis.py -r 'initial run' default_parameters MV_HFV_012.jpg", - assert_in_output, (re.compile(r"2416\.863[0-9]* 60\.0"), "histogram.png")), - ("Comment on the outcome", - "smt comment 'works fine'"), - ("Set defaults", - "smt configure -e python -m glass_sem_analysis.py"), - ("Look at the current configuration of the project", - "smt info", - assert_config, {"project_name": "ProjectGlass", "executable": "Python", "main": "glass_sem_analysis.py", - "code_change": "error"}), - edit_parameters("default_parameters", "no_filter", "filter_size", 1), - ("Run with changed parameters and user-defined label", - "smt run -l example_label -r 'No filtering' no_filter MV_HFV_012.jpg", # TODO: assert(results have changed) - assert_in_output, "phases.png", - assert_label_equal, "example_label"), - ("Change parameters from the command line", - "smt run -r 'Trying a different colourmap' default_parameters MV_HFV_012.jpg phases_colourmap=hot"), # assert(results have changed) - ("Add another comment", - "smt comment 'The default colourmap is nicer'"), #TODO add a comment to an older record (e.g. this colourmap is nicer than 'hot')") - ("Add tags on the command line", - build_command("smt tag mytag {0} {1}", "labels")), - modify_script("glass_sem_analysis.py"), - ("Run the modified code", - "smt run -r 'Added labels to output' default_parameters MV_HFV_012.jpg", - assert_return_code, 1, - assert_in_output, "Code has changed, please commit your changes"), - ("Commit changes...", - "hg commit -m 'Added labels to output' -u testuser"), - ("...then run again", - "smt run -r 'Added labels to output' default_parameters MV_HFV_012.jpg"), # assert(output has changed as expected) - #TODO: make another change to the Python script - ("Change configuration to store diff", - "smt configure --on-changed=store-diff"), - ("Run with store diff", - "smt run -r 'made a change' default_parameters MV_HFV_012.jpg"), # assert(code runs, stores diff) - ("Review previous computations - get a list of labels", - "smt list", - assert_in_output, expected_short_list), - ("Review previous computations in detail", - "smt list -l", - assert_records, substitute_labels([ - {'label': 0, 'executable_name': 'Python', 'outcome': 'works fine', 'reason': 'initial run', - 'version': '6038f9c500d1', 'vcs': 'Mercurial', 'script_arguments': ' MV_HFV_012.jpg', - 'main_file': 'glass_sem_analysis.py'}, # TODO: add checking of parameters - {'label': 1, 'outcome': '', 'reason': 'No filtering'}, - {'label': 2, 'outcome': 'The default colourmap is nicer', 'reason': 'Trying a different colourmap'}, - {'label': 3, 'outcome': '', 'reason': 'Added labels to output'}, - {'label': 4, 'outcome': '', 'reason': 'made a change'}, # TODO: add checking of diff - ])), - ("Filter the output of ``smt list`` based on tag", - "smt list mytag", - #assert(list is correct) - ), - ("Export Sumatra records as JSON.", - "smt export", - assert_file_exists, ".smt/records_export.json"), -] + def modify_script(filename): + def wrapped(): + with open(os.path.join(working_dir, filename), 'r') as fp: + script = fp.readlines() + with open(os.path.join(working_dir, filename), 'w') as fp: + for line in script: + if "print(mean_bubble_size, median_bubble_size)" in line: + fp.write('print("Mean:", mean_bubble_size)\n') + fp.write('print("Median:", median_bubble_size)\n') + else: + fp.write(line) + return wrapped + test_steps = [ + ("Get the example code", + "git clone %s ." % repository, + assert_in_output, "done."), + ("Run the computation without Sumatra", + "python glass_sem_analysis.py default_parameters MV_HFV_012.jpg", + assert_in_output, re.compile(r"2416\.863[0-9]* 60\.0"), + assert_file_exists, os.path.join(working_dir, "Data", datetime.now().strftime("%Y%m%d")), # Data subdirectory contains another subdirectory labelled with today's date) + ), # assert(subdirectory contains three image files). + ("Set up a Sumatra project", + "smt init -d Data -i . ProjectGlass", + assert_in_output, "Sumatra project successfully set up"), + ("Run the ``glass_sem_analysis.py`` script with Sumatra", + "smt run -e python -m glass_sem_analysis.py -r 'initial run' default_parameters MV_HFV_012.jpg", + assert_in_output, (re.compile(r"2416\.863[0-9]* 60\.0"), "histogram.png")), + ("Comment on the outcome", + "smt comment 'works fine'"), + ("Set defaults", + "smt configure -e python -m glass_sem_analysis.py"), + ("Look at the current configuration of the project", + "smt info", + assert_config, {"project_name": "ProjectGlass", "executable": "Python", "main": "glass_sem_analysis.py", + "code_change": "error"}), + edit_parameters("default_parameters", "no_filter", "filter_size", 1, working_dir), + ("Run with changed parameters and user-defined label", + "smt run -l example_label -r 'No filtering' no_filter MV_HFV_012.jpg", # TODO: assert(results have changed) + assert_in_output, "phases.png", + assert_label_equal, "example_label"), + ("Change parameters from the command line", + "smt run -r 'Trying a different colourmap' default_parameters MV_HFV_012.jpg phases_colourmap=hot"), # assert(results have changed) + ("Add another comment", + "smt comment 'The default colourmap is nicer'"), #TODO add a comment to an older record (e.g. this colourmap is nicer than 'hot')") + ("Add tags on the command line", + build_command("smt tag mytag {0} {1}", "labels")), + modify_script("glass_sem_analysis.py"), + ("Run the modified code", + "smt run -r 'Added labels to output' default_parameters MV_HFV_012.jpg", + assert_return_code, 1, + assert_in_output, "Code has changed, please commit your changes"), + ("Commit changes...", + "git commit -a -m 'Added labels to output' --author 'testuser '"), + ("...then run again", + "smt run -r 'Added labels to output' default_parameters MV_HFV_012.jpg"), # assert(output has changed as expected) + #TODO: make another change to the Python script + ("Change configuration to store diff", + "smt configure --on-changed=store-diff"), + ("Run with store diff", + "smt run -r 'made a change' default_parameters MV_HFV_012.jpg"), # assert(code runs, stores diff) + ("Review previous computations - get a list of labels", + "smt list", + assert_in_output, expected_short_list), + ("Review previous computations in detail", + "smt list -l", + assert_records, substitute_labels([ + {'label': 0, 'executable_name': 'Python', 'outcome': 'works fine', 'reason': 'initial run', + 'version': 'e74b39374b0a1a401848b05ba9c86042aac4d8e4', 'vcs': 'Git', 'script_arguments': ' MV_HFV_012.jpg', + 'main_file': 'glass_sem_analysis.py'}, # TODO: add checking of parameters + {'label': 1, 'outcome': '', 'reason': 'No filtering'}, + {'label': 2, 'outcome': 'The default colourmap is nicer', 'reason': 'Trying a different colourmap'}, + {'label': 3, 'outcome': '', 'reason': 'Added labels to output'}, + {'label': 4, 'outcome': '', 'reason': 'made a change'}, # TODO: add checking of diff + ])), + ("Filter the output of ``smt list`` based on tag", + "smt list mytag", + #assert(list is correct) + ), + ("Export Sumatra records as JSON.", + "smt export", + assert_file_exists, os.path.join(working_dir, ".smt/records_export.json")), + ] -def test_all(): - """Test generator for Nose.""" + env = {"working_dir": working_dir, "labels": []} for step in test_steps: if callable(step): step() else: - test = partial(*tuple([run_test] + list(step[1:]))) - test.description = step[0] - yield test + run_test(step[1], *step[2:], env=env) + shutil.rmtree(temporary_dir) # Still to test: # @@ -135,19 +130,3 @@ def test_all(): #.. repeats #.. moving forwards and backwards in history #.. upgrades (needs Docker) - - -if __name__ == '__main__': - # Run the tests without using Nose. - setup() - for step in test_steps: - if callable(step): - step() - else: - print(step[0]) # description - run_test(*step[1:]) - response = input("Do you want to delete the temporary directory (default: yes)? ") - if response not in ["n", "N", "no", "No"]: - teardown() - else: - print("Temporary directory %s not removed" % utils.temporary_dir) diff --git a/test/system/test_postgres.py b/test/system/test_postgres.py index e4fdb9f7..3861f538 100644 --- a/test/system/test_postgres.py +++ b/test/system/test_postgres.py @@ -1,55 +1,39 @@ """ Tests using a PostgreSQL-based record store. - -Usage: - nosetests -v test_postgres.py -or: - python test_postgres.py """ -from __future__ import print_function -from __future__ import unicode_literals -from future import standard_library -standard_library.install_aliases() -from builtins import input import os from urllib.parse import urlparse try: import docker - if "DOCKER_HOST" in os.environ: - have_docker = True - else: - have_docker = False + have_docker = True except ImportError: have_docker = False from unittest import SkipTest +import shutil +import tempfile import utils -from utils import setup, teardown as default_teardown, run_test, build_command +from utils import run_test, build_command try: import psycopg2 have_psycopg2 = True except ImportError: have_psycopg2 = False -ctr = None -dkr = None -image = "postgresql_test" - +import pytest -def create_script(): - with open(os.path.join(utils.working_dir, "main.py"), "w") as fp: - fp.write("print('Hello world!')\n") +DOCKER_IMAGE = "postgresql_test" -def get_url(): - info = dkr.containers()[0] - assert info["Image"] == image - host = urlparse(dkr.base_url).hostname - return "{0}:{1}".format(host, info["Ports"][0]["PublicPort"]) +def get_url(ctr): + ctr.reload() # required to get auto-assigned ports + info = ctr.ports["5432/tcp"][0] + return f"{info['HostIp']}:{info['HostPort']}" -def start_pg_container(): +@pytest.fixture(scope="module") +def pg_container(): """ Launch a Docker container running PostgreSQL, return the URL. @@ -57,68 +41,50 @@ def start_pg_container(): docker build -t postgresql_test - < fixtures/Dockerfile.postgres """ - global ctr, dkr - env = docker.utils.kwargs_from_env(assert_hostname=False) - dkr = docker.Client(timeout=60, **env) - # docker run -rm -P -name pg_test postgresql_test - ctr = dkr.create_container(image, command=None, hostname=None, user=None, - detach=False, stdin_open=False, tty=False, - ports=None, environment=None, dns=None, volumes=None, - volumes_from=None, network_disabled=False, name=None, - entrypoint=None, cpu_shares=None, working_dir=None) - dkr.start(ctr, publish_all_ports=True) - utils.env["url"] = get_url() - - -def teardown(): - global ctr, dkr - default_teardown() - if dkr is not None: - dkr.stop(ctr) - - -test_steps = [ - create_script, - ("Create repository", "git init"), - ("Add main file", "git add main.py"), - ("Commit main file", "git commit -m 'initial'"), - start_pg_container, - ("Set up a Sumatra project", - build_command("smt init --store=postgres://docker:docker@{}/sumatra_test -m main.py -e python MyProject", "url")), - #"smt init -m main.py -e python MyProject"), - ("Show project configuration", "smt info"), # TODO: add assert - ("Run a computation", "smt run") -] + dkr = docker.from_env() + ctr = dkr.containers.run(DOCKER_IMAGE, detach=True, publish_all_ports=True) + container_url = get_url(ctr) + yield container_url + ctr.stop() # TODO: add test skips where docker, psycopg2 not found -def test_all(): - """Test generator for Nose.""" +def test_all(pg_container): + """Run a series of Sumatra commands""" if not have_psycopg2: raise SkipTest("Tests require psycopg2") if not have_docker: raise SkipTest("Tests require docker") + temporary_dir = os.path.realpath(tempfile.mkdtemp()) + working_dir = os.path.join(temporary_dir, "sumatra_exercise") + os.mkdir(working_dir) + env = { + "labels": [], + "working_dir": working_dir, + "url": pg_container + } + + def create_script(): + with open(os.path.join(working_dir, "main.py"), "w") as fp: + fp.write("print('Hello world!')\n") + + test_steps = [ + create_script, + ("Create repository", "git init"), + ("Add main file", "git add main.py"), + ("Commit main file", "git commit -m 'initial'"), + ("Set up a Sumatra project", + build_command("smt init --store=postgres://docker:docker@{}/sumatra_test -m main.py -e python MyProject", "url")), + #"smt init -m main.py -e python MyProject"), + ("Show project configuration", "smt info"), # TODO: add assert + ("Run a computation", "smt run") + ] for step in test_steps: if callable(step): step() else: - run_test.description = step[0] - yield tuple([run_test] + list(step[1:])) + run_test(*step[1:], env=env) - -if __name__ == '__main__': - # Run the tests without using Nose. - setup() - for step in test_steps: - if callable(step): - step() - else: - print(step[0]) # description - run_test(*step[1:]) - response = input("Do you want to delete the temporary directory (default: yes)? ") - if response not in ["n", "N", "no", "No"]: - teardown() - else: - print("Temporary directory %s not removed" % utils.temporary_dir) + shutil.rmtree(temporary_dir) diff --git a/test/system/test_webdav.py b/test/system/test_webdav.py index 7f2bd769..31a0e70a 100644 --- a/test/system/test_webdav.py +++ b/test/system/test_webdav.py @@ -1,19 +1,11 @@ """ Tests using a PostgreSQL-based record store. - -Usage: - nosetests -v test_webdav.py -or: - python test_webdav.py """ -from __future__ import print_function -from __future__ import unicode_literals -from future import standard_library -standard_library.install_aliases() -from builtins import input import os +import shutil +import tempfile from urllib.parse import urlparse try: import docker @@ -21,27 +13,21 @@ except ImportError: have_docker = False from unittest import SkipTest -import utils -from utils import setup, teardown as default_teardown, run_test, build_command - -ctr = None -dkr = None -IMAGE = "webdav_test" +from utils import run_test, build_command +import pytest -def create_script(): - with open(os.path.join(utils.working_dir, "main.py"), "w") as fp: - fp.write("print('Hello world!')\n") +DOCKER_IMAGE = "webdav_test" -def get_url(): - info = dkr.containers()[0] - assert info["Image"] == IMAGE - host = urlparse(dkr.base_url).hostname - return "{0}:{1}".format(host, info["Ports"][0]["PublicPort"]) +def get_url(ctr): + ctr.reload() # required to get auto-assigned ports + info = ctr.ports["80/tcp"][0] + return f"{info['HostIp']}:{info['HostPort']}" -def start_webdav_container(): +@pytest.fixture(scope="module") +def server(): """ Launch a Docker container running apache/webdav, return the URL. @@ -53,63 +39,48 @@ def start_webdav_container(): possible since we need to ADD an apache config file. See also: http://blog.mx17.net/2014/02/12/dockerfile-file-directory-error-using-add/ """ - global ctr, dkr - env = docker.utils.kwargs_from_env(assert_hostname=False) - dkr = docker.Client(timeout=60, **env) - ctr = dkr.create_container(IMAGE, command=None, hostname=None, user=None, - detach=False, stdin_open=False, tty=False, - ports=[80], environment=None, dns=None, volumes=None, - volumes_from=None, network_disabled=False, name=None, - entrypoint=None, cpu_shares=None, working_dir=None) - dkr.start(ctr, port_bindings={80: 8080}) - print(get_url()) - utils.env["url"] = get_url() - - -def teardown(): - global ctr, dkr - default_teardown() - if dkr is not None: - dkr.stop(ctr) - - -test_steps = [ - create_script, - ("Create repository", "git init"), - ("Add main file", "git add main.py"), - ("Commit main file", "git commit -m 'initial'"), - start_webdav_container, - ("Set up a Sumatra project", - build_command("smt init -W=http://sumatra:sumatra@{}/webdav/ -m main.py -e python MyProject", "url")), - #"smt init -m main.py -e python MyProject"), - ("Show project configuration", "smt info"), # TODO: add assert - ("Run a computation", "smt run") -] - - -def test_all(): - """Test generator for Nose.""" + dkr = docker.from_env(timeout=60) + ctr = dkr.containers.run(DOCKER_IMAGE, detach=True, publish_all_ports=True) + container_url = get_url(ctr) + yield container_url + ctr.stop() + + + +def test_all(server): + """Run a series of Sumatra commands""" if not have_docker: raise SkipTest("Tests require docker") - for step in test_steps: - if callable(step): - step() - else: - run_test.description = step[0] - yield tuple([run_test] + list(step[1:])) + temporary_dir = os.path.realpath(tempfile.mkdtemp()) + working_dir = os.path.join(temporary_dir, "sumatra_exercise") + os.mkdir(working_dir) + env = { + "labels": [], + "working_dir": working_dir, + "url": server + } + + def create_script(): + with open(os.path.join(working_dir, "main.py"), "w") as fp: + fp.write("print('Hello world!')\n") + + test_steps = [ + create_script, + ("Create repository", "git init"), + ("Add main file", "git add main.py"), + ("Commit main file", "git commit -m 'initial'"), + ("Set up a Sumatra project", + build_command("smt init -W=http://sumatra:sumatra@{}/webdav/ -m main.py -e python MyProject", "url")), + ("Show project configuration", "smt info"), # TODO: add assert + ("Run a computation", "smt run") + ] -if __name__ == '__main__': - # Run the tests without using Nose. - setup() for step in test_steps: if callable(step): step() else: - print(step[0]) # description - run_test(*step[1:]) - response = input("Do you want to delete the temporary directory (default: yes)? ") - if response not in ["n", "N", "no", "No"]: - teardown() - else: - print("Temporary directory %s not removed" % utils.temporary_dir) + run_test.description = step[0] + run_test(*step[1:], env=env) + + shutil.rmtree(temporary_dir) diff --git a/test/system/test_webui.py b/test/system/test_webui.py index 54ce96be..4c025279 100644 --- a/test/system/test_webui.py +++ b/test/system/test_webui.py @@ -1,42 +1,39 @@ """ Tests of the web browser interface (smtweb) using Selenium - """ -from __future__ import print_function -from __future__ import unicode_literals import os +import shutil +import tempfile from time import sleep -from builtins import input -from unittest import SkipTest try: from selenium import webdriver + from selenium.webdriver.common.by import By have_selenium = True except ImportError: have_selenium = False -from subprocess import PIPE import sarge -from nose.tools import assert_equal, assert_dict_contains_subset, assert_in +import pytest + +from utils import (run_test, build_command, assert_in_output, assert_label_equal, + edit_parameters, substitute_labels) -import utils -from utils import (setup as default_setup, teardown as default_teardown, - run, run_test, build_command, assert_file_exists, assert_in_output, - assert_config, assert_label_equal, assert_records, assert_return_code, - edit_parameters, expected_short_list, substitute_labels) +pytest.mark.skipif(not have_selenium, reason="Tests require Selenium") -repository = "https://bitbucket.org/apdavison/ircr2013" -#repository = "/Users/andrew/dev/ircr2013" +#repository = "https://github.com/apdavison/ircr2013" +repository = "/Users/adavison/projects/projectglass-git" -def modify_script(filename): + +def modify_script(filename, working_dir): def wrapped(): - with open(os.path.join(utils.working_dir, filename), 'r') as fp: + with open(os.path.join(working_dir, filename), 'r') as fp: script = fp.readlines() - with open(os.path.join(utils.working_dir, filename), 'w') as fp: + with open(os.path.join(working_dir, filename), 'w') as fp: for line in script: if "print(mean_bubble_size, median_bubble_size)" in line: fp.write('print("Mean:", mean_bubble_size)\n') @@ -46,125 +43,142 @@ def wrapped(): return wrapped -setup_steps = [ - ("Get the example code", - "hg clone %s ." % repository, - assert_in_output, "updating to branch default"), - ("Set up a Sumatra project", - "smt init -d Data -i . -e python -m glass_sem_analysis.py --on-changed=store-diff ProjectGlass", - assert_in_output, "Sumatra project successfully set up"), - ("Run the ``glass_sem_analysis.py`` script with Sumatra", - "smt run -r 'initial run' default_parameters MV_HFV_012.jpg", - assert_in_output, ("2416.86315789 60.0", "histogram.png")), - ("Comment on the outcome", - "smt comment 'works fine'"), - edit_parameters("default_parameters", "no_filter", "filter_size", 1), - ("Run with changed parameters and user-defined label", - "smt run -l example_label -r 'No filtering' no_filter MV_HFV_012.jpg", - assert_in_output, "phases.png", - assert_label_equal, "example_label"), - ("Change parameters from the command line", - "smt run -r 'Trying a different colourmap' default_parameters MV_HFV_012.jpg phases_colourmap=hot"), - ("Add another comment", - "smt comment 'The default colourmap is nicer'"), # TODO add a comment to an older record (e.g. this colourmap is nicer than 'hot')") - ("Add tags on the command line", - build_command("smt tag mytag {0} {1}", "labels")), - modify_script("glass_sem_analysis.py"), - ("Run the modified code", - "smt run -r 'Added labels to output' default_parameters MV_HFV_012.jpg"), -] - - -def setup(): - global server, driver - if not have_selenium: - raise SkipTest("Tests require Selenium") - default_setup() +@pytest.fixture(scope="module") +def env(): + return {"labels": []} + + +@pytest.fixture(scope="module") +def server(env): + temporary_dir = os.path.realpath(tempfile.mkdtemp()) + working_dir = os.path.join(temporary_dir, "sumatra_exercise") + os.mkdir(working_dir) + env["working_dir"] = working_dir + + setup_steps = [ + ("Get the example code", + "git clone %s ." % repository, + assert_in_output, "done."), + ("Set up a Sumatra project", + "smt init -d Data -i . -e python -m glass_sem_analysis.py --on-changed=store-diff ProjectGlass", + assert_in_output, "Sumatra project successfully set up"), + ("Run the ``glass_sem_analysis.py`` script with Sumatra", + "smt run -r 'initial run' default_parameters MV_HFV_012.jpg", + assert_in_output, ("2416.86315789", "histogram.png")), + ("Comment on the outcome", + "smt comment 'works fine'"), + edit_parameters("default_parameters", "no_filter", "filter_size", 1, working_dir), + ("Run with changed parameters and user-defined label", + "smt run -l example_label -r 'No filtering' no_filter MV_HFV_012.jpg", + assert_in_output, "phases.png", + assert_label_equal, "example_label"), + ("Change parameters from the command line", + "smt run -r 'Trying a different colourmap' default_parameters MV_HFV_012.jpg phases_colourmap=hot"), + ("Add another comment", + "smt comment 'The default colourmap is nicer'"), # TODO add a comment to an older record (e.g. this colourmap is nicer than 'hot')") + ("Add tags on the command line", + build_command("smt tag mytag {0} {1}", "labels")), + modify_script("glass_sem_analysis.py", working_dir), + ("Run the modified code", + "smt run -r 'Added labels to output' default_parameters MV_HFV_012.jpg"), + ] + for step in setup_steps: if callable(step): step() else: print(step[0]) # description - run_test(*step[1:]) + run_test(*step[1:], env=env) - server = sarge.Command("smtweb -p 8765 --no-browser", cwd=utils.working_dir, + print(f"Running smtweb in {working_dir}") + server = sarge.Command("smtweb -p 8765 --no-browser", cwd=working_dir, stdout=sarge.Capture(), stderr=sarge.Capture()) - server.run(async=True) - driver = webdriver.Firefox() - - -def teardown(): - driver.close() + server.run(async_=True) + yield server server.terminate() - default_teardown() + shutil.rmtree(temporary_dir) + + +@pytest.fixture(scope="module") +def driver(server): + # note that for now we have to use Chrome, as `test_comparison_view` gives + # a scrolling error with Firefox + # These bug reports seem to be relevant: + # - https://github.com/mozilla/geckodriver/issues/776#issuecomment-355086173 + # - https://github.com/robotframework/SeleniumLibrary/issues/1780 + options = webdriver.ChromeOptions() + options.add_argument("--headless") + driver = webdriver.Chrome(options=options) + yield driver + driver.close() -def test_start_page(): +def test_start_page(driver, server, env): driver.get("http://127.0.0.1:8765") # on homepage - assert_equal(driver.title, "List of projects") + assert driver.title == "List of projects" # assert there is one project, named "ProjectGlass" - projects = driver.find_elements_by_tag_name("h3") - assert_equal(len(projects), 1) - assert_equal(projects[0].text, "ProjectGlass") + projects = driver.find_elements(By.TAG_NAME, "h3") + assert len(projects) == 1 + assert projects[0].text == "ProjectGlass" # click on ProjectGlass --> record list projects[0].click() - assert_equal(driver.title, "ProjectGlass: List of records") - assert_equal(driver.current_url, "http://127.0.0.1:8765/ProjectGlass/") + assert driver.title == "ProjectGlass: List of records" + assert driver.current_url == "http://127.0.0.1:8765/ProjectGlass/" -def test_record_list(): +def test_record_list(driver, env): driver.get("http://127.0.0.1:8765/ProjectGlass/") # assert there are four records - rows = driver.find_elements_by_tag_name('tr') - assert_equal(len(rows), 4 + 1) # first row is the header - column_headers = [elem.text for elem in rows[0].find_elements_by_tag_name('th')] + rows = driver.find_elements(By.TAG_NAME, 'tr') + assert len(rows) == 4 + 1 # first row is the header + column_headers = [elem.text for elem in rows[0].find_elements(By.TAG_NAME, 'th')] # assert the labels are correct and that the reason and outcome fields are correct expected_content = substitute_labels([ {'label': 0, 'outcome': 'works fine', 'reason': 'initial run', - 'version': '6038f9c...', 'main': 'glass_sem_analysis.py'}, + 'version': 'e74b39374…', 'main': 'glass_sem_analysis.py'}, {'label': 1, 'outcome': '', 'reason': 'No filtering'}, {'label': 2, 'outcome': 'The default colourmap is nicer', 'reason': 'Trying a different colourmap'}, - {'label': 3, 'outcome': '', 'reason': 'Added labels to output', 'version': '6038f9c...*'}])(utils.env) + {'label': 3, 'outcome': '', 'reason': 'Added labels to output', 'version': 'e74b39374…*'}])(env) for row, expected in zip(rows[1:], reversed(expected_content)): - cells = row.find_elements_by_tag_name('td') + cells = row.find_elements(By.TAG_NAME, 'td') label = cells[0].text - assert_equal(row.get_attribute('id'), label) + assert row.get_attribute('id') == label actual = dict((key.lower(), cell.text) for key, cell in zip(column_headers, cells)) - assert_dict_contains_subset(expected, actual) + assert actual == actual | expected # this uses the dictionary unary operator -def test_column_settings_dialog(): +def test_column_settings_dialog(driver): driver.get("http://127.0.0.1:8765/ProjectGlass/") # test the column settings dialog - row0 = driver.find_element_by_tag_name('tr') - column_headers = [elem.text for elem in row0.find_elements_by_tag_name('th')] - cog = driver.find_element_by_class_name("glyphicon-cog") + row0 = driver.find_element(By.TAG_NAME, 'tr') + column_headers = [elem.text for elem in row0.find_elements(By.TAG_NAME, 'th')] + cog = driver.find_element(By.CLASS_NAME, "glyphicon-cog") cog.click() sleep(0.5) - options = driver.find_elements_by_class_name("checkbox") - displayed_columns = [option.text for option in options if option.find_element_by_tag_name("input").is_selected()] - assert_equal(displayed_columns, column_headers[1:]) # can't turn off "Label" column + options = driver.find_elements(By.CLASS_NAME, "checkbox") + displayed_columns = [option.text for option in options if option.find_element(By.TAG_NAME, "input").is_selected()] + assert displayed_columns == column_headers[1:] # can't turn off "Label" column # turn on all columns for option in options: - checkbox = option.find_element_by_tag_name("input") + checkbox = option.find_element(By.TAG_NAME, "input") if not checkbox.is_selected(): checkbox.click() - apply_button, = [elem for elem in driver.find_elements_by_tag_name("button") if elem.text == "Apply"] + apply_button, = [elem for elem in driver.find_elements(By.TAG_NAME, "button") if elem.text == "Apply"] apply_button.click() sleep(0.5) - column_headers = [elem.text for elem in row0.find_elements_by_tag_name('th')] - assert_equal(column_headers, - ["Label", "Date/Time", "Reason", "Outcome", "Input data", "Output data", - "Duration", "Processes", "Executable", "Main", "Version", "Arguments", "Tags"]) + column_headers = [elem.text for elem in row0.find_elements(By.TAG_NAME, 'th')] + assert column_headers == [ + "Label", "Date/Time", "Reason", "Outcome", "Input data", "Output data", + "Duration", "Processes", "Executable", "Main", "Version", "Arguments", "Tags"] -def test_comparison_view(): +def test_comparison_view(driver, env): driver.get("http://127.0.0.1:8765/ProjectGlass/") # test that "Compare selected" gives an error message with no records selected - alert = driver.find_element_by_id("alert") + alert = driver.find_element(By.ID, "alert") assert not alert.is_displayed() - compare_button, = [elem for elem in driver.find_elements_by_tag_name("button") if "Compare" in elem.text] + compare_button, = [elem for elem in driver.find_elements(By.TAG_NAME, "button") if "Compare" in elem.text] compare_button.click() sleep(0.5) assert alert.is_displayed() @@ -174,49 +188,29 @@ def test_comparison_view(): assert not alert.is_displayed() # select two records and click on compare selected - rows = driver.find_elements_by_tag_name('tr') - target_records = utils.env["labels"][::2] + rows = driver.find_elements(By.TAG_NAME, 'tr') + target_records = env["labels"][::2] for row in rows[1:]: if row.get_attribute("id") in target_records: + #row.location_once_scrolled_into_view + #driver.execute_script("arguments[0].scrollIntoView();", row) row.click() # scroll back to the top of the screen driver.execute_script("window.scrollTo(0, 0)") compare_button.click() # assert go to comparison page - assert_in("compare", driver.current_url) + assert "compare" in driver.current_url -def test_data_detail_view(): +def test_data_detail_view(driver, env): driver.get("http://127.0.0.1:8765/ProjectGlass/") - rows = driver.find_elements_by_tag_name('tr') - rows[1].find_element_by_tag_name('td').find_element_by_tag_name('a').click() - assert_equal(driver.current_url, "http://127.0.0.1:8765/ProjectGlass/{}/".format(utils.env["labels"][-1])) - - dl = driver.find_element_by_tag_name('dl') - general_attributes = dict(zip((item.text for item in dl.find_elements_by_tag_name("dt")), - (item.text for item in dl.find_elements_by_tag_name("dd")))) - assert_equal(general_attributes["Code version:"], '6038f9c500d1* (diff)') - assert_in("Added labels to output", general_attributes["Reason:"]) - - -if __name__ == '__main__': - # Run the tests without using Nose. - setup() - try: - test_start_page() - test_record_list() - test_column_settings_dialog() - test_comparison_view() - test_data_detail_view() - # test filter by tags - # test editing reason - # test "Add outcome" button - # test deleting records - except Exception as err: - print(err) - response = input("Do you want to delete the temporary directory (default: yes)? ") - if response not in ["n", "N", "no", "No"]: - teardown() - else: - print("Temporary directory %s not removed" % utils.temporary_dir) \ No newline at end of file + rows = driver.find_elements(By.TAG_NAME, 'tr') + rows[1].find_element(By.TAG_NAME, 'td').find_element(By.TAG_NAME, 'a').click() + assert driver.current_url == "http://127.0.0.1:8765/ProjectGlass/{}/".format(env["labels"][-1]) + + dl = driver.find_element(By.TAG_NAME, 'dl') + general_attributes = dict(zip((item.text for item in dl.find_elements(By.TAG_NAME, "dt")), + (item.text for item in dl.find_elements(By.TAG_NAME, "dd")))) + assert general_attributes["Code version:"] == 'e74b39374b0a1a401848b05ba9c86042aac4d8e4* (diff)' + assert "Added labels to output" in general_attributes["Reason:"] diff --git a/test/system/utils.py b/test/system/utils.py index 85365a71..8f29953c 100644 --- a/test/system/utils.py +++ b/test/system/utils.py @@ -1,10 +1,6 @@ """ Utility functions for writing system tests. """ -from __future__ import print_function -from __future__ import unicode_literals -from builtins import zip -from builtins import str import os.path import re @@ -13,18 +9,16 @@ import shutil import sarge -DEBUG = False -temporary_dir = None -working_dir = None -env = {} +import pytest +DEBUG = True -label_pattern = re.compile("Record label for this run: '(?P
" + "".join(field.title() for field in fields) + "