Attempting workflow rework

.github/workflows/regression-test.yml

@@ -0,0 +1,54 @@
+# This workflow will install Python dependencies, run tests and lint with a single version of Python
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
+
+name: Regression Test
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v3
+      with:
+        python-version: "3.10"
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install flake8 pytest pytest-reportlog black
+        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+    - name: Checkout PR branch
+      uses: actions/checkout@v4
+      with:
+        ref: ${{ github.event.pull_request.head.ref }}
+    - name: Test PR branch with pytest
+      run: |
+        pytest --report-log=pr_test_results.json
+        find . -name "*.pyc" | xargs rm
+      continue-on-error: true
+    - name: Checkout main
+      uses: actions/checkout@v4
+      with:
+        ref: main
+        path: main
+    - name: Test main
+      run: |
+        pytest ./main --report-log=main_test_results.json
+      continue-on-error: true
+    - name: Print test results
+      run: |
+        cat pr_test_results.json
+        cat main_test_results.json
+    - name: Compare results
+      run: |
+        python main.py pr_test_results.json main_test_results.json

main.py

@@ -0,0 +1,47 @@
+import json
+import pathlib
+from collections import defaultdict
+import sys
+
+
+def main():
+    pull_request_dict = defaultdict(list)
+    destination_branch_dict = defaultdict(list)
+    pull_request_fp = pathlib.Path(sys.argv[1])
+    destination_branch_fp = pathlib.Path(sys.argv[2])
+    with open(pull_request_fp, "r") as f:
+        pull_request_results = f.read().splitlines()[3:-1]  # Trim unnecessary entries
+        for item in pull_request_results:
+            print(f"ITEM: {item}")
+            json_data = json.loads(item)
+            if "nodeid" in json_data.keys():
+                pull_request_dict[json_data["outcome"]].append(json_data["nodeid"])
+    with open(destination_branch_fp, "r") as f:
+        destination_branch_results = f.read().splitlines()[
+            3:-1
+        ]  # Trim unnecessary entries
+        for item in destination_branch_results:
+            json_data = json.loads(item)
+            if "nodeid" in json_data.keys():
+                destination_branch_dict[json_data["outcome"]].append(
+                    json_data["nodeid"]
+                )
+    regressions = []
+    warnings = []
+    for item in destination_branch_dict["passed"]:
+        if item in pull_request_dict["failed"]:
+            regressions.append(item)
+        elif item in pull_request_dict["skipped"]:
+            warnings.append(item)
+    print(f"{len(warnings)} tests now skipped/xfailed that weren't previously")
+    for warning in warnings:
+        print(f"Warning: {warning}")
+    print(f"{len(regressions)} regressions were found")
+    for regression in regressions:
+        print(f"Regression: {regression}")
+    if len(regressions) > 0:
+        exit(1)
+
+
+if __name__ == "__main__":
+    main()

pytest.ini

@@ -0,0 +1,2 @@
+[pytest]
+testpaths=*_test.py

sanity_test.py

@@ -0,0 +1,5 @@
+def test_nothing():
+    assert False
+
+def test_new_passing_test():
+    assert True