Modernize sandbox: Reduce footprint of Buildout, fix macOS >= 11, improve CI

.github/workflows/docs.yml

@@ -20,7 +20,7 @@ jobs:
         uses: actions/checkout@v3
 
       - name: Set up Python
-        uses: actions/setup-python@v3
+        uses: actions/setup-python@v4
         with:
           python-version: '3.9'
 

.github/workflows/nightly.yml

@@ -18,21 +18,28 @@ jobs:
         os: [ubuntu-latest]
         python-version: ['3.7', '3.8', '3.9', '3.10']
         cratedb-version: ['nightly']
-        sqla-version: ['1.3.24']
+        sqla-version: ['1.4.37']
       fail-fast: false
+    env:
+      CRATEDB_VERSION: ${{ matrix.cratedb-version }}
+      SQLALCHEMY_VERSION: ${{ matrix.sqla-version }}
 
     steps:
       - uses: actions/checkout@v3
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v3
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
 
-      - name: Install dependencies
-        run: |
-          ./devtools/setup_ci.sh --cratedb-version=${{ matrix.cratedb-version }} --sqlalchemy-version=${{ matrix.sqla-version }}
-
       - name: Invoke tests
         run: |
-          bin/flake8
-          bin/coverage run bin/test -vv1
+
+          # Bootstrap environment.
+          source bootstrap.sh
+
+          # Propagate build matrix information.
+          ./devtools/setup_ci.sh
+
+          # Invoke validation tasks.
+          flake8 src bin
+          coverage run bin/test -vv1

.github/workflows/release.yml

@@ -12,7 +12,7 @@ jobs:
       - uses: actions/checkout@v3
 
       - name: Set up Python
-        uses: actions/setup-python@v3
+        uses: actions/setup-python@v4
         with:
           python-version: '3.9'
 

.github/workflows/tests.yml

@@ -21,29 +21,32 @@ jobs:
         cratedb-version: ['4.8.0']
         sqla-version: ['1.3.24', '1.4.37']
       fail-fast: true
+    env:
+      CRATEDB_VERSION: ${{ matrix.cratedb-version }}
+      SQLALCHEMY_VERSION: ${{ matrix.sqla-version }}
 
     steps:
       - uses: actions/checkout@v3
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v3
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
 
-      - name: Adjust environment for macOS
-        if: matrix.os == 'macos-latest'
+      - name: Invoke tests
         run: |
-          brew install gnu-getopt
-          echo "/usr/local/opt/gnu-getopt/bin" >> $GITHUB_PATH
 
-      - name: Install dependencies
-        run: |
-          ./devtools/setup_ci.sh --cratedb-version=${{ matrix.cratedb-version }} --sqlalchemy-version=${{ matrix.sqla-version }}
+          # Bootstrap environment.
+          source bootstrap.sh
 
-      - name: Invoke tests
-        run: |
-          bin/flake8
-          bin/coverage run bin/test -vv1
-          bin/coverage xml
+          # Propagate build matrix information.
+          ./devtools/setup_ci.sh
+
+          # Invoke validation tasks.
+          flake8 src bin
+          coverage run bin/test -vv1
+
+          # Set the stage for the Codecov step.
+          coverage xml
 
       # https://github.com/codecov/codecov-action
       - name: Upload coverage results to Codecov

.gitignore

@@ -6,7 +6,9 @@ coverage.xml
 .tox/
 *.DS_Store
 *.pyc
-bin/
+bin/*
+!bin/test
+!bin/sphinx
 build/
 crate-python.iml
 crate.egg-info

DEVELOP.rst

@@ -5,27 +5,22 @@ Developer guide
 Setup
 =====
 
-This project uses buildout_ to set up the development environment.
+To start things off, bootstrap the sandbox environment::
 
-To start things off, create a Python virtualenv and install buildout::
+    source bootstrap.sh
 
-    python3 -m venv .venv
-    source .venv/bin/activate
+This command should automatically install all prerequisites for the development
+sandbox and drop you into the virtualenv, ready for invoking further commands.
 
-    # Workaround for Python 3.5
-    python -m pip install --upgrade "setuptools>=31,<51"
-
-    pip install zc.buildout==2.13.4
-
-Then, run::
-
-    buildout -N
 
 Running tests
 =============
 
 All tests will be invoked using the Python interpreter that was used when
-creating the Python virtualenv. The test runner is zope.testrunner_.
+creating the Python virtualenv. The test runner is `zope.testrunner`_.
+
+Some examples are outlined below. In order to learn about more details,
+see, for example, `useful command-line options for zope-testrunner`_.
 
 Run all tests::
 
@@ -41,18 +36,18 @@ Ignore specific test directories::
 
     ./bin/test -vvvv --ignore_dir=testing
 
-You can run the tests against multiple Python interpreters with tox_::
+You can run the tests against multiple Python interpreters with `tox`_::
 
-    ./bin/tox
+    tox
 
 To do this, you will need the respective Python interpreter versions available
 on your ``$PATH``.
 
 To run against a single interpreter, you can also invoke::
 
-    ./bin/tox -e py37
+    tox -e py37
 
-*Note*: before running the tests, make sure to stop all CrateDB instances which
+*Note*: Before running the tests, make sure to stop all CrateDB instances which
 are listening on the default CrateDB transport port to avoid side effects with
 the test layer.
 
@@ -121,5 +116,6 @@ nothing special you need to do to get the live docs to update.
 .. _Sphinx: http://sphinx-doc.org/
 .. _tox: http://testrun.org/tox/latest/
 .. _twine: https://pypi.python.org/pypi/twine
-.. _zope.testrunner: https://pypi.python.org/pypi/zope.testrunner/4.4.1
+.. _useful command-line options for zope-testrunner: https://pypi.org/project/zope.testrunner/#some-useful-command-line-options-to-get-you-started
 .. _versions hosted on ReadTheDocs: https://readthedocs.org/projects/crate-python/versions/
+.. _zope.testrunner: https://pypi.org/project/zope.testrunner/

bin/sphinx

@@ -0,0 +1,15 @@
+#!/usr/bin/env python
+import os
+import sys
+
+join = os.path.join
+base = os.path.dirname(os.path.abspath(os.path.realpath(__file__)))
+base = os.path.dirname(base)
+
+
+sys.argv.extend(['-b', 'html', '-E', 'docs', './out/html'])
+import sphinx.cmd.build  # noqa:E402
+
+
+if __name__ == '__main__':
+    sys.exit(sphinx.cmd.build.main())

bin/test

@@ -0,0 +1,16 @@
+#!/usr/bin/env python
+import os
+import sys
+import zope.testrunner
+
+join = os.path.join
+base = os.path.dirname(os.path.abspath(os.path.realpath(__file__)))
+base = os.path.dirname(base)
+
+
+sys.argv[0] = os.path.abspath(sys.argv[0])
+
+if __name__ == '__main__':
+    sys.exit(zope.testrunner.run((['--auto-color', '--verbose']) + [
+        '--test-path', join(base, 'src'),
+        ]))

bootstrap.sh

@@ -0,0 +1,98 @@
+#!/bin/bash
+#
+# Bootstrap sandbox environment for crate-python
+#
+# - Create a Python virtualenv
+# - Install all dependency packages and modules
+# - Install package in editable mode
+# - Drop user into an activated virtualenv
+#
+# Synopsis::
+#
+#     source bootstrap.sh
+#
+
+
+# Trace all invocations.
+# set -x
+
+# Default variables.
+BUILDOUT_VERSION=${BUILDOUT_VERSION:-2.13.7}
+CRATEDB_VERSION=${CRATEDB_VERSION:-4.8.1}
+SQLALCHEMY_VERSION=${SQLALCHEMY_VERSION:-1.4.37}
+
+
+function print_header() {
+    printf '=%.0s' {1..42}; echo
+    echo "$1"
+    printf '=%.0s' {1..42}; echo
+}
+
+function ensure_virtualenv() {
+    # Create a Python virtualenv with current version of Python 3.
+    # TODO: Maybe take `pyenv` into account.
+    if [[ ! -d .venv ]]; then
+        python3 -m venv .venv
+    fi
+}
+
+function activate_virtualenv() {
+    # Activate Python virtualenv.
+    source .venv/bin/activate
+}
+
+function before_setup() {
+
+    # When `wheel` is installed, Python will build `wheel` packages from all
+    # acquired `sdist` packages and will store them into `~/.cache/pip`, where
+    # they will be picked up by the caching machinery and will be reused on
+    # subsequent invocations when run on CI. This makes a *significant*
+    # difference on total runtime on CI, it is about 2x faster.
+    #
+    # Otherwise, there will be admonitions like:
+    #   Using legacy 'setup.py install' for foobar, since package 'wheel' is
+    #   not installed.
+    #
+    pip install wheel
+
+    # Install Buildout with designated version, allowing pre-releases.
+    pip install --pre "zc.buildout==${BUILDOUT_VERSION}"
+
+}
+
+function setup_package() {
+
+    # Install package in editable mode.
+    pip install --editable=.[sqlalchemy,test,doc]
+
+}
+
+function run_buildout() {
+    buildout -N
+}
+
+function finalize() {
+
+    # Some steps before dropping into the activated virtualenv.
+    echo
+    echo "Sandbox environment ready"
+    echo -n "Using SQLAlchemy version: "
+    python -c 'import sqlalchemy; print(sqlalchemy.__version__)'
+    echo
+
+}
+
+function main() {
+    ensure_virtualenv
+    activate_virtualenv
+    before_setup
+    setup_package
+    run_buildout
+    finalize
+}
+
+function lint() {
+    flake8 "$@" src bin
+}
+
+main