diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index d30a6720..3a7f2b2a 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -20,7 +20,7 @@ jobs: uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v3 with: python-version: '3.9' diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index fd4d5abc..fcda7737 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -18,28 +18,21 @@ jobs: os: [ubuntu-latest] python-version: ['3.7', '3.8', '3.9', '3.10'] cratedb-version: ['nightly'] - sqla-version: ['1.4.37'] + sqla-version: ['1.3.24'] fail-fast: false - env: - CRATEDB_VERSION: ${{ matrix.cratedb-version }} - SQLALCHEMY_VERSION: ${{ matrix.sqla-version }} steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} - - name: Invoke tests + - name: Install dependencies run: | + ./devtools/setup_ci.sh --cratedb-version=${{ matrix.cratedb-version }} --sqlalchemy-version=${{ matrix.sqla-version }} - # Bootstrap environment. - source bootstrap.sh - - # Propagate build matrix information. - ./devtools/setup_ci.sh - - # Invoke validation tasks. - flake8 src bin - coverage run bin/test -vv1 + - name: Invoke tests + run: | + bin/flake8 + bin/coverage run bin/test -vv1 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b005723b..99984d86 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -12,7 +12,7 @@ jobs: - uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v3 with: python-version: '3.9' diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 4e5fe1e8..bfab598c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -16,14 +16,11 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-latest, macos-latest] + os: ['macos-10.15', 'macos-11', 'macos-12'] python-version: ['3.7', '3.8', '3.9', '3.10'] cratedb-version: ['4.8.0'] sqla-version: ['1.3.24', '1.4.37'] - fail-fast: true - env: - CRATEDB_VERSION: ${{ matrix.cratedb-version }} - SQLALCHEMY_VERSION: ${{ matrix.sqla-version }} + fail-fast: false steps: - uses: actions/checkout@v3 @@ -32,21 +29,26 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Invoke tests + - name: Adjust environment for macOS run: | + brew install gnu-getopt + echo "/usr/local/opt/gnu-getopt/bin" >> $GITHUB_PATH - # Bootstrap environment. - source bootstrap.sh - - # Propagate build matrix information. - ./devtools/setup_ci.sh + - name: Debug environment on macOS + run: | + sw_vers -productVersion + export SYSTEM_VERSION_COMPAT=1 + sw_vers -productVersion - # Invoke validation tasks. - flake8 src bin - coverage run bin/test -vv1 + - name: Install dependencies + run: | + ./devtools/setup_ci.sh --cratedb-version=${{ matrix.cratedb-version }} --sqlalchemy-version=${{ matrix.sqla-version }} - # Set the stage for the Codecov step. - coverage xml + - name: Invoke tests + run: | + bin/flake8 + bin/coverage run bin/test -vv1 + bin/coverage xml # https://github.com/codecov/codecov-action - name: Upload coverage results to Codecov diff --git a/.gitignore b/.gitignore index aeca2d6b..e62679a3 100644 --- a/.gitignore +++ b/.gitignore @@ -6,9 +6,7 @@ coverage.xml .tox/ *.DS_Store *.pyc -bin/* -!bin/test -!bin/sphinx +bin/ build/ crate-python.iml crate.egg-info diff --git a/DEVELOP.rst b/DEVELOP.rst index a6e8ff3c..d3d7efd4 100644 --- a/DEVELOP.rst +++ b/DEVELOP.rst @@ -5,22 +5,27 @@ Developer guide Setup ===== -To start things off, bootstrap the sandbox environment:: +This project uses buildout_ to set up the development environment. - source bootstrap.sh +To start things off, create a Python virtualenv and install buildout:: -This command should automatically install all prerequisites for the development -sandbox and drop you into the virtualenv, ready for invoking further commands. + python3 -m venv .venv + source .venv/bin/activate + # Workaround for Python 3.5 + python -m pip install --upgrade "setuptools>=31,<51" + + pip install zc.buildout==2.13.4 + +Then, run:: + + buildout -N Running tests ============= All tests will be invoked using the Python interpreter that was used when -creating the Python virtualenv. The test runner is `zope.testrunner`_. - -Some examples are outlined below. In order to learn about more details, -see, for example, `useful command-line options for zope-testrunner`_. +creating the Python virtualenv. The test runner is zope.testrunner_. Run all tests:: @@ -36,18 +41,18 @@ Ignore specific test directories:: ./bin/test -vvvv --ignore_dir=testing -You can run the tests against multiple Python interpreters with `tox`_:: +You can run the tests against multiple Python interpreters with tox_:: - tox + ./bin/tox To do this, you will need the respective Python interpreter versions available on your ``$PATH``. To run against a single interpreter, you can also invoke:: - tox -e py37 + ./bin/tox -e py37 -*Note*: Before running the tests, make sure to stop all CrateDB instances which +*Note*: before running the tests, make sure to stop all CrateDB instances which are listening on the default CrateDB transport port to avoid side effects with the test layer. @@ -116,6 +121,5 @@ nothing special you need to do to get the live docs to update. .. _Sphinx: http://sphinx-doc.org/ .. _tox: http://testrun.org/tox/latest/ .. _twine: https://pypi.python.org/pypi/twine -.. _useful command-line options for zope-testrunner: https://pypi.org/project/zope.testrunner/#some-useful-command-line-options-to-get-you-started +.. _zope.testrunner: https://pypi.python.org/pypi/zope.testrunner/4.4.1 .. _versions hosted on ReadTheDocs: https://readthedocs.org/projects/crate-python/versions/ -.. _zope.testrunner: https://pypi.org/project/zope.testrunner/ diff --git a/base.cfg b/base.cfg new file mode 100644 index 00000000..e470e47b --- /dev/null +++ b/base.cfg @@ -0,0 +1,49 @@ +[buildout] +develop = . +extends = versions.cfg +versions = versions +show-picked-versions = true +parts = test + crate + scripts + coverage + linter + +[scripts] +recipe = zc.recipe.egg:script +interpreter = py +eggs = crate + crate [test,sqlalchemy] + +[coverage] +recipe = zc.recipe.egg +eggs = createcoverage + +[crate:linux] +recipe = hexagonit.recipe.download +url = https://cdn.crate.io/downloads/releases/crate-${versions:crate_server}.tar.gz +strip-top-level-dir = true + +[crate:macosx] +recipe = hexagonit.recipe.download +url = https://cdn.crate.io/downloads/releases/cratedb/x64_mac/crate-${versions:crate_server}.tar.gz +strip-top-level-dir = true + +[crate:windows] +recipe = hexagonit.recipe.download +url = https://cdn.crate.io/downloads/releases/cratedb/x64_windows/crate-${versions:crate_server}.zip +strip-top-level-dir = true + +[test] +relative-paths = true +recipe = zc.recipe.testrunner +defaults = ['--auto-color'] +eggs = crate [test,sqlalchemy] + +[linter] +recipe = zc.recipe.egg:script +eggs = flake8 +scripts = flake8 +initialization = + sys.argv.extend(['--count', + '${buildout:directory}/src']) diff --git a/bin/sphinx b/bin/sphinx deleted file mode 100755 index 31463469..00000000 --- a/bin/sphinx +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env python -import os -import sys - -join = os.path.join -base = os.path.dirname(os.path.abspath(os.path.realpath(__file__))) -base = os.path.dirname(base) - - -sys.argv.extend(['-b', 'html', '-E', 'docs', './out/html']) -import sphinx.cmd.build # noqa:E402 - - -if __name__ == '__main__': - sys.exit(sphinx.cmd.build.main()) diff --git a/bin/test b/bin/test deleted file mode 100755 index bc720824..00000000 --- a/bin/test +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env python -import os -import sys -import zope.testrunner - -join = os.path.join -base = os.path.dirname(os.path.abspath(os.path.realpath(__file__))) -base = os.path.dirname(base) - - -sys.argv[0] = os.path.abspath(sys.argv[0]) - -if __name__ == '__main__': - sys.exit(zope.testrunner.run((['--auto-color', '--verbose']) + [ - '--test-path', join(base, 'src'), - ])) diff --git a/bootstrap.sh b/bootstrap.sh deleted file mode 100644 index 0ad9a914..00000000 --- a/bootstrap.sh +++ /dev/null @@ -1,98 +0,0 @@ -#!/bin/bash -# -# Bootstrap sandbox environment for crate-python -# -# - Create a Python virtualenv -# - Install all dependency packages and modules -# - Install package in editable mode -# - Drop user into an activated virtualenv -# -# Synopsis:: -# -# source bootstrap.sh -# - - -# Trace all invocations. -# set -x - -# Default variables. -BUILDOUT_VERSION=${BUILDOUT_VERSION:-2.13.7} -CRATEDB_VERSION=${CRATEDB_VERSION:-4.8.1} -SQLALCHEMY_VERSION=${SQLALCHEMY_VERSION:-1.4.37} - - -function print_header() { - printf '=%.0s' {1..42}; echo - echo "$1" - printf '=%.0s' {1..42}; echo -} - -function ensure_virtualenv() { - # Create a Python virtualenv with current version of Python 3. - # TODO: Maybe take `pyenv` into account. - if [[ ! -d .venv ]]; then - python3 -m venv .venv - fi -} - -function activate_virtualenv() { - # Activate Python virtualenv. - source .venv/bin/activate -} - -function before_setup() { - - # When `wheel` is installed, Python will build `wheel` packages from all - # acquired `sdist` packages and will store them into `~/.cache/pip`, where - # they will be picked up by the caching machinery and will be reused on - # subsequent invocations when run on CI. This makes a *significant* - # difference on total runtime on CI, it is about 2x faster. - # - # Otherwise, there will be admonitions like: - # Using legacy 'setup.py install' for foobar, since package 'wheel' is - # not installed. - # - pip install wheel - - # Install Buildout with designated version, allowing pre-releases. - pip install --pre "zc.buildout==${BUILDOUT_VERSION}" - -} - -function setup_package() { - - # Install package in editable mode. - pip install --editable=.[sqlalchemy,test,doc] - -} - -function run_buildout() { - buildout -N -} - -function finalize() { - - # Some steps before dropping into the activated virtualenv. - echo - echo "Sandbox environment ready" - echo -n "Using SQLAlchemy version: " - python -c 'import sqlalchemy; print(sqlalchemy.__version__)' - echo - -} - -function main() { - ensure_virtualenv - activate_virtualenv - before_setup - setup_package - run_buildout - finalize -} - -function lint() { - flake8 "$@" src bin -} - -main diff --git a/buildout.cfg b/buildout.cfg index edd92a7f..172f1803 100644 --- a/buildout.cfg +++ b/buildout.cfg @@ -1,20 +1,24 @@ [buildout] -extends = versions.cfg -versions = versions -show-picked-versions = true -parts = crate +extends = base.cfg +parts += tox + sphinx + scripts -[crate:linux] -recipe = hexagonit.recipe.download -url = https://cdn.crate.io/downloads/releases/crate-${versions:crate_server}.tar.gz -strip-top-level-dir = true +[scripts] +recipe = zc.recipe.egg:script +eggs = wheel + docutils + twine -[crate:macosx] -recipe = hexagonit.recipe.download -url = https://cdn.crate.io/downloads/releases/cratedb/x64_mac/crate-${versions:crate_server}.tar.gz -strip-top-level-dir = true +[tox] +recipe = gp.recipe.tox -[crate:windows] -recipe = hexagonit.recipe.download -url = https://cdn.crate.io/downloads/releases/cratedb/x64_windows/crate-${versions:crate_server}.zip -strip-top-level-dir = true +[sphinx] +recipe = zc.recipe.egg:script +eggs = sphinx + crate-docs-theme +relative-paths=true +scripts = sphinx-build=sphinx +initialization = + sys.argv.extend(['-N', '-q', '-b', 'html', + '-E', 'docs', '${buildout:directory}/out/html']) diff --git a/devtools/create_tag.sh b/devtools/create_tag.sh index 1ee0f68d..7a53cca1 100755 --- a/devtools/create_tag.sh +++ b/devtools/create_tag.sh @@ -48,7 +48,7 @@ fi # check if tag to create has already been created WORKING_DIR=`dirname $0` -VERSION=`python setup.py --version` +VERSION=`$WORKING_DIR/../bin/py setup.py --version` EXISTS=`git tag | grep $VERSION` if [ "$VERSION" == "$EXISTS" ] diff --git a/devtools/setup_ci.sh b/devtools/setup_ci.sh index e4d96299..b61f4da1 100755 --- a/devtools/setup_ci.sh +++ b/devtools/setup_ci.sh @@ -2,33 +2,71 @@ set -e +function args() { + options=$(getopt --long cratedb-version: --long sqlalchemy-version: -- "$@") + [ $? -eq 0 ] || { + echo "Incorrect options provided" + exit 1 + } + eval set -- "$options" + while true; do + case "$1" in + --cratedb-version) + shift; + cratedb_version=$1 + ;; + --sqlalchemy-version) + shift; + sqlalchemy_version=$1 + ;; + --) + shift + break + ;; + esac + shift + done +} + function main() { + # Read command line arguments. + args $0 "$@" + # Sanity checks. - [ -z ${CRATEDB_VERSION} ] && { + [ -z ${cratedb_version} ] && { echo "--cratedb-version must be given" - echo "Environment variable 'CRATEDB_VERSION' needed" exit 1 } - [ -z ${SQLALCHEMY_VERSION} ] && { - echo "Environment variable 'SQLALCHEMY_VERSION' needed" + [ -z ${sqlalchemy_version} ] && { + echo "--sqlalchemy-version must be given" exit 1 } # Let's go. - echo "Invoking tests with CrateDB ${CRATEDB_VERSION} and SQLAlchemy ${SQLALCHEMY_VERSION}" + echo "Invoking tests with CrateDB ${cratedb_version} and SQLAlchemy ${sqlalchemy_version}" - # Install designated SQLAlchemy version. - pip install "sqlalchemy==${SQLALCHEMY_VERSION}" + python -m pip install --upgrade pip + + pip install --upgrade wheel + pip install "zc.buildout>=2,<3" + + # Replace SQLAlchemy version. + sed -ir "s/SQLAlchemy.*/SQLAlchemy = ${sqlalchemy_version}/g" versions.cfg # Replace CrateDB version. - if [ ${CRATEDB_VERSION} = "nightly" ]; then - sed -ir "s/releases/releases\/nightly/g" buildout.cfg + if [ ${cratedb_version} = "nightly" ]; then + sed -ir "s/releases/releases\/nightly/g" base.cfg sed -ir "s/crate_server.*/crate_server = latest/g" versions.cfg else - sed -ir "s/crate_server.*/crate_server = ${CRATEDB_VERSION}/g" versions.cfg + sed -ir "s/crate_server.*/crate_server = ${cratedb_version}/g" versions.cfg fi + # Switch to enable compatibility with older versions of macOS. + export SYSTEM_VERSION_COMPAT=1 + + buildout -vv -n -c base.cfg + } main "$@" diff --git a/setup.cfg b/setup.cfg index d5ca9e8e..12d4730e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -2,4 +2,4 @@ universal = 1 [flake8] -ignore = E501, C901, W504 +ignore = E501,C901, W504 diff --git a/setup.py b/setup.py index 41aed1bb..28eea262 100644 --- a/setup.py +++ b/setup.py @@ -64,17 +64,10 @@ def read(path): ] }, extras_require=dict( - sqlalchemy=['sqlalchemy>=1.0,<1.5', - 'geojson>=2.5.0'], - test=['tox>=3,<4', - 'zope.testing>=4,<5', - 'zope.testrunner>=5,<6', + test=['zope.testing>=4,<5', 'zc.customdoctests>=1.0.1,<2', - 'createcoverage>=1,<2', - 'stopit>=1.1.2,<2', - 'flake8>=4,<5'], - doc=['sphinx>=3,<4', - 'crate-docs-theme'], + 'stopit>=1.1.2,<2'], + sqlalchemy=['sqlalchemy>=1.0,<1.5', 'geojson>=2.5.0'] ), python_requires='>=3.4', install_requires=requirements, diff --git a/versions.cfg b/versions.cfg index ec56d408..2a91d62b 100644 --- a/versions.cfg +++ b/versions.cfg @@ -1,4 +1,65 @@ [versions] crate_server = 4.8.0 +flake8 = 3.7.9 +mccabe = 0.6.1 +pep8 = 1.7.1 +pyflakes = 2.1.1 +Jinja2 = 2.11.3 +MarkupSafe = 1.1.1 +Pygments = 2.8.1 +Sphinx = 3.5.2 +SQLAlchemy = 1.3.23 +geojson = 2.5.0 +coverage = 5.5 +crate-docs-theme = 0.13.3 +createcoverage = 1.5 +docutils = 0.16 hexagonit.recipe.download = 1.7.1 +py = 1.10.0 +tox = 3.23.0 +twine = 3.4.0 +pkginfo = 1.7.0 +zc.customdoctests = 1.0.1 +zc.recipe.egg = 2.0.7 +zc.recipe.testrunner = 2.2 +zope.testing = 4.9 +stopit = 1.1.2 + +# Required by: +# clint==0.5.1 +args = 0.1.0 + +# Required by: +# twine==1.8.1 +clint = 0.5.1 + +# Required by: +# zope.testrunner==4.4.3 +six = 1.15.0 + +# Required by: +# crate-docs-theme==0.3.6 +sphinxcontrib-plantuml = 0.19 + +# Required by: +# crate==0.12.3 +urllib3 = 1.26.4 + +# Required by: +# zope.testing==4.1.3 +# zope.testrunner==4.4.3 +zope.exceptions = 4.4 + +# Required by: +# zope.testing==4.1.3 +# zope.testrunner==4.4.3 +zope.interface = 5.2.0 + +# Required by: +# zc.recipe.testrunner==2.0.0 +zope.testrunner = 5.2 +gp.recipe.tox = 0.4 + +requests = 2.25.1 +requests-toolbelt = 0.9.1