New screen19 pipeline

.azure-pipelines/azure-pipelines.yml

@@ -0,0 +1,130 @@
+trigger:
+  branches:
+    include:
+    - '*'
+  tags:
+    include:
+    - '*'
+
+stages:
+- stage: static
+  displayName: Static Analysis
+  jobs:
+  - job: checks
+    displayName: static code analysis
+    pool:
+      vmImage: ubuntu-22.04
+    steps:
+      # Use Python >=3.8 for syntax validation
+    - task: UsePythonVersion@0
+      displayName: Set up python
+      inputs:
+        versionSpec: 3.8
+
+      # Run syntax validation on a shallow clone
+    - bash: |
+        libtbx.python "$(Pipeline.Workspace)/.azure-pipelines/syntax-validation.py"
+      displayName: Syntax validation
+
+      # Run flake8 validation on a shallow clone
+    - bash: |
+        libtbx.pip install flake8
+        libtbx.python "$(Pipeline.Workspace)/.azure-pipelines/flake8-validation.py"
+      displayName: Flake8 validation
+
+- stage: build
+  displayName: Build
+  dependsOn:
+  jobs:
+  - job: build
+    displayName: build package
+    pool:
+      vmImage: ubuntu-22.04
+    steps:
+    - task: UsePythonVersion@0
+      displayName: Set up python
+      inputs:
+        versionSpec: 3.10
+
+    - bash: |
+        $(Pipeline.Workspace)/.azure-pipelines/setup-base
+      displayName: Set up DIALS
+
+    - bash: |
+        set -ex
+        libtbx.python setup.py sdist bdist_wheel
+        mkdir -p dist/pypi
+        shopt -s extglob
+        mv -v dist/!(pypi) dist/pypi
+        git archive HEAD | gzip > dist/repo-source.tar.gz
+        ls -laR dist
+      displayName: Build python package
+
+    - task: PublishBuildArtifacts@1
+      displayName: Store artifact
+      inputs:
+        pathToPublish: dist/
+        artifactName: package
+
+    - bash: libtbx.python "$(Pipeline.Workspace)/setup.py" checkdocs
+      displayName: Check package description
+
+- stage: tests
+  displayName: Run unit tests
+  dependsOn:
+  - static
+  - build
+  jobs:
+  - job: linux
+    pool:
+      vmImage: ubuntu-22.04
+    strategy:
+      matrix:
+        python38:
+          PYTHON_VERSION: 3.8
+        python39:
+          PYTHON_VERSION: 3.9
+        python310:
+          PYTHON_VERSION: 3.10
+    steps:
+    - template: ci.yml
+
+- stage: deploy
+  displayName: Publish release
+  dependsOn:
+  - tests
+  condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/'))
+  jobs:
+  - job: pypi
+    displayName: Publish pypi release
+    pool:
+      vmImage: ubuntu-22.04
+    steps:
+    - checkout: none
+
+    - task: UsePythonVersion@0
+      displayName: Set up python
+      inputs:
+        versionSpec: 3.8
+
+    - task: DownloadBuildArtifacts@0
+      displayName: Get pre-built package
+      inputs:
+        buildType: current
+        downloadType: single
+        artifactName: package
+        downloadPath: $(System.ArtifactsDirectory)
+
+    - script: |
+        libtbx.pip install -U pip
+        libtbx.pip install twine
+      displayName: Install twine
+
+    - task: TwineAuthenticate@1
+      displayName: Set up credentials
+      inputs:
+        pythonUploadServiceConnection: pypi-screen19
+
+    - bash: |
+        libtbx.python -m twine upload -r pypi-screen19 --config-file $(PYPIRC_PATH) $(System.ArtifactsDirectory)/package/pypi/*.tar.gz $(System.ArtifactsDirectory)/package/pypi/*.whl
+      displayName: Publish package

.azure-pipelines/ci.yml

@@ -0,0 +1,41 @@
+steps:
+- checkout: none
+
+- task: UsePythonVersion@0
+  inputs:
+    versionSpec: $(PYTHON_VERSION)
+  displayName: Use Python $(PYTHON_VERSION)
+
+- task: DownloadBuildArtifacts@0
+  displayName: Get pre-built package
+  inputs:
+    buildType: current
+    downloadType: single
+    artifactName: package
+    downloadPath: $(System.ArtifactsDirectory)
+
+- task: ExtractFiles@1
+  displayName: Checkout sources
+  inputs:
+    archiveFilePatterns: $(System.ArtifactsDirectory)/package/repo-source.tar.gz
+    destinationFolder: $(Pipeline.Workspace)/src
+
+- script: |
+    set -eux
+    libtbx.pip install -r "$(Pipeline.Workspace)/src/requirements_dev.txt"
+    libtbx.pip install "$(Pipeline.Workspace)/src"
+  displayName: Install package
+
+- script: |
+    PYTHONDEVMODE=1 pytest -ra --cov=screen19 --cov-report=xml --cov-branch
+  displayName: Run tests
+  workingDirectory: $(Pipeline.Workspace)/src
+
+- bash: |
+    curl -Os https://uploader.codecov.io/latest/linux/codecov
+    chmod +x codecov
+    ./codecov -B $(Build.SourceBranchName) -C $(Build.SourceVersion) -t $(CODECOV_TOKEN) -n "Python $(PYTHON_VERSION) $(Agent.OS)"
+  displayName: Publish coverage stats
+  continueOnError: true
+  workingDirectory: $(Pipeline.Workspace)/src
+  timeoutInMinutes: 2

.azure-pipelines/flake8-validation.py

@@ -0,0 +1,43 @@
+import os
+import subprocess
+
+# Flake8 validation
+failures = 0
+try:
+    flake8 = subprocess.run(
+        [
+            "flake8",
+            "--exit-zero",
+            "--max-line-length=88",
+            "--select=E401,E711,E712,E713,E714,E721,E722,E901,F401,F402,F403,F405,F631,F632,F633,F811,F812,F821,F822,F841,F901,W191,W291,W292,W293,W602,W603,W604,W605,W606",
+        ],
+        capture_output=True,
+        check=True,
+        encoding="latin-1",
+        timeout=300,
+    )
+except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as e:
+    print(
+        "##vso[task.logissue type=error;]flake8 validation failed with",
+        str(e.__class__.__name__),
+    )
+    print(e.stdout)
+    print(e.stderr)
+    print("##vso[task.complete result=Failed;]flake8 validation failed")
+    exit()
+else:
+    for line in flake8.stdout.split("\n"):
+        if ":" not in line:
+            continue
+        filename, lineno, column, error = line.split(":", maxsplit=3)
+        errcode, error = error.strip().split(" ", maxsplit=1)
+        filename = os.path.normpath(filename)
+        failures += 1
+        print(
+            f"##vso[task.logissue type=error;sourcepath={filename};"
+            f"linenumber={lineno};columnnumber={column};code={errcode};]" + error
+        )
+
+if failures:
+    print(f"##vso[task.logissue type=warning]Found {failures} flake8 violation(s)")
+    print(f"##vso[task.complete result=Failed;]Found {failures} flake8 violation(s)")

.travis/run-flake8 → .azure-pipelines/run-flake8

@@ -6,7 +6,7 @@ CHANGED_FILES=$(git diff --name-only --diff-filter=AM $TRAVIS_BRANCH...HEAD -- "
 # List is empty if this is not a pull request run or if no python files have changed
 [ -z "$CHANGED_FILES" ] && echo There are no relevant changes. Skipping test. && exit 0
 
-pip install -q flake8
+libtbx.pip install -q flake8
 echo
 echo Running flake8 on changed files
-flake8 --select=E401,E711,E712,E713,E714,E721,E722,E901,F401,F402,F403,F405,F631,F632,F633,F811,F812,F821,F822,F841,F901,W191,W291,W292,W293,W602,W603,W604,W605,W606 $CHANGED_FILES
+libtbx.flake8 --select=E401,E711,E712,E713,E714,E721,E722,E901,F401,F402,F403,F405,F631,F632,F633,F811,F812,F821,F822,F841,F901,W191,W291,W292,W293,W602,W603,W604,W605,W606 $CHANGED_FILES

.travis/setup-base → .azure-pipelines/setup-base

@@ -2,12 +2,6 @@
 
 EXPECTED_CACHE_REVISION=20191115-python3
 
-if [ "$TRAVIS_EVENT_TYPE" == "cron" ]; then
-  echo -e "\e[31;1mCron job build detected. Invalidating cache.\e[0m"
-  rm -f $HOME/build_dials/.cache_valid
-  rm -f $HOME/build_dials/.build_complete
-fi
-
 if [ -f $HOME/build_dials/.cache_valid ] && [ "$EXPECTED_CACHE_REVISION" == "$(cat $HOME/build_dials/.cache_valid)" ]; then
   echo -e "\e[1mCache probably valid\e[0m"
 else

.azure-pipelines/syntax-validation.py

@@ -0,0 +1,32 @@
+from __future__ import annotations
+
+import ast
+import os
+import sys
+
+print("Python", sys.version, "\n")
+
+failures = 0
+
+for base, _, files in os.walk("."):
+    for f in files:
+        if not f.endswith(".py"):
+            continue
+        filename = os.path.normpath(os.path.join(base, f))
+        try:
+            with open(filename) as fh:
+                ast.parse(fh.read())
+        except SyntaxError as se:
+            failures += 1
+            print(
+                f"##vso[task.logissue type=error;sourcepath={filename};"
+                f"linenumber={se.lineno};columnnumber={se.offset};]"
+                f"SyntaxError: {se.msg}"
+            )
+            print(" " + se.text + " " * se.offset + "^")
+            print(f"SyntaxError: {se.msg} in {filename} line {se.lineno}")
+            print()
+
+if failures:
+    print(f"##vso[task.logissue type=warning]Found {failures} syntax error(s)")
+    print(f"##vso[task.complete result=Failed;]Found {failures} syntax error(s)")

.pre-commit-config.yaml

@@ -6,7 +6,7 @@ repos:
   rev: v2.38.2
   hooks:
   - id: pyupgrade
-    args: ["--py36-plus"]
+    args: ["--py38-plus"]
 
 # Automatically sort imports with isort
 - repo: https://github.com/PyCQA/isort
@@ -23,8 +23,8 @@ repos:
     args: [--safe, --quiet]
 
 # Enforce style with Flake8
-- repo: https://gitlab.com/pycqa/flake8
-  rev: 3.9.2
+- repo: https://github.com/pycqa/flake8
+  rev: 4.0.1
   hooks:
   - id: flake8
     args: ['--max-line-length=88', '--select=E401,E711,E712,E713,E714,E721,E722,E901,F401,F402,F403,F405,F631,F632,F633,F811,F812,F821,F822,F841,F901,W191,W291,W292,W293,W602,W603,W604,W605,W606']

requirements_dev.txt

@@ -0,0 +1,2 @@
+pytest
+pytest-cov