From afb4678c6100d9301362bf13d120096a34d06c1f Mon Sep 17 00:00:00 2001 From: austinleblanc Date: Wed, 4 Jun 2025 21:48:17 -0400 Subject: [PATCH 1/2] Moved existing workflows to .github/workflows/old, added regression-test.yml and main.py (regression checker) --- .github/workflows/{ => old}/alert-discord.yml | 0 .github/workflows/{ => old}/build-next.yml | 0 .../workflows/{ => old}/gh-pr-assignees.yml | 0 .../workflows/{ => old}/gh-projects-qa.yml | 0 .github/workflows/{ => old}/gh-submodules.yml | 0 .github/workflows/{ => old}/gh-sync.yml | 0 .../{ => old}/gh-test-downstream.yml | 0 .../{ => old}/meta-regression-analysis.yml | 0 .github/workflows/{ => old}/publish-ghcr.yml | 0 .../workflows/{ => old}/publish-py-pypi.yml | 0 .../workflows/{ => old}/publish-storybook.yml | 0 .../workflows/{ => old}/publish-ts-npm.yml | 0 .github/workflows/{ => old}/test-py-lint.yml | 0 .../workflows/{ => old}/test-py-pytest.yml | 0 .github/workflows/{ => old}/test-pytest.yml | 0 .../workflows/{ => old}/test-storybook.yml | 0 .github/workflows/{ => old}/test-ts-lint.yml | 0 .github/workflows/regression-test.yml | 48 +++++++++++++++++++ main.py | 37 ++++++++++++++ pytest.ini | 2 + sanity_test.py | 5 ++ 21 files changed, 92 insertions(+) rename .github/workflows/{ => old}/alert-discord.yml (100%) rename .github/workflows/{ => old}/build-next.yml (100%) rename .github/workflows/{ => old}/gh-pr-assignees.yml (100%) rename .github/workflows/{ => old}/gh-projects-qa.yml (100%) rename .github/workflows/{ => old}/gh-submodules.yml (100%) rename .github/workflows/{ => old}/gh-sync.yml (100%) rename .github/workflows/{ => old}/gh-test-downstream.yml (100%) rename .github/workflows/{ => old}/meta-regression-analysis.yml (100%) rename .github/workflows/{ => old}/publish-ghcr.yml (100%) rename .github/workflows/{ => old}/publish-py-pypi.yml (100%) rename .github/workflows/{ => old}/publish-storybook.yml (100%) rename .github/workflows/{ => old}/publish-ts-npm.yml (100%) rename .github/workflows/{ => old}/test-py-lint.yml (100%) rename .github/workflows/{ => old}/test-py-pytest.yml (100%) rename .github/workflows/{ => old}/test-pytest.yml (100%) rename .github/workflows/{ => old}/test-storybook.yml (100%) rename .github/workflows/{ => old}/test-ts-lint.yml (100%) create mode 100644 .github/workflows/regression-test.yml create mode 100644 main.py create mode 100644 pytest.ini create mode 100644 sanity_test.py diff --git a/.github/workflows/alert-discord.yml b/.github/workflows/old/alert-discord.yml similarity index 100% rename from .github/workflows/alert-discord.yml rename to .github/workflows/old/alert-discord.yml diff --git a/.github/workflows/build-next.yml b/.github/workflows/old/build-next.yml similarity index 100% rename from .github/workflows/build-next.yml rename to .github/workflows/old/build-next.yml diff --git a/.github/workflows/gh-pr-assignees.yml b/.github/workflows/old/gh-pr-assignees.yml similarity index 100% rename from .github/workflows/gh-pr-assignees.yml rename to .github/workflows/old/gh-pr-assignees.yml diff --git a/.github/workflows/gh-projects-qa.yml b/.github/workflows/old/gh-projects-qa.yml similarity index 100% rename from .github/workflows/gh-projects-qa.yml rename to .github/workflows/old/gh-projects-qa.yml diff --git a/.github/workflows/gh-submodules.yml b/.github/workflows/old/gh-submodules.yml similarity index 100% rename from .github/workflows/gh-submodules.yml rename to .github/workflows/old/gh-submodules.yml diff --git a/.github/workflows/gh-sync.yml b/.github/workflows/old/gh-sync.yml similarity index 100% rename from .github/workflows/gh-sync.yml rename to .github/workflows/old/gh-sync.yml diff --git a/.github/workflows/gh-test-downstream.yml b/.github/workflows/old/gh-test-downstream.yml similarity index 100% rename from .github/workflows/gh-test-downstream.yml rename to .github/workflows/old/gh-test-downstream.yml diff --git a/.github/workflows/meta-regression-analysis.yml b/.github/workflows/old/meta-regression-analysis.yml similarity index 100% rename from .github/workflows/meta-regression-analysis.yml rename to .github/workflows/old/meta-regression-analysis.yml diff --git a/.github/workflows/publish-ghcr.yml b/.github/workflows/old/publish-ghcr.yml similarity index 100% rename from .github/workflows/publish-ghcr.yml rename to .github/workflows/old/publish-ghcr.yml diff --git a/.github/workflows/publish-py-pypi.yml b/.github/workflows/old/publish-py-pypi.yml similarity index 100% rename from .github/workflows/publish-py-pypi.yml rename to .github/workflows/old/publish-py-pypi.yml diff --git a/.github/workflows/publish-storybook.yml b/.github/workflows/old/publish-storybook.yml similarity index 100% rename from .github/workflows/publish-storybook.yml rename to .github/workflows/old/publish-storybook.yml diff --git a/.github/workflows/publish-ts-npm.yml b/.github/workflows/old/publish-ts-npm.yml similarity index 100% rename from .github/workflows/publish-ts-npm.yml rename to .github/workflows/old/publish-ts-npm.yml diff --git a/.github/workflows/test-py-lint.yml b/.github/workflows/old/test-py-lint.yml similarity index 100% rename from .github/workflows/test-py-lint.yml rename to .github/workflows/old/test-py-lint.yml diff --git a/.github/workflows/test-py-pytest.yml b/.github/workflows/old/test-py-pytest.yml similarity index 100% rename from .github/workflows/test-py-pytest.yml rename to .github/workflows/old/test-py-pytest.yml diff --git a/.github/workflows/test-pytest.yml b/.github/workflows/old/test-pytest.yml similarity index 100% rename from .github/workflows/test-pytest.yml rename to .github/workflows/old/test-pytest.yml diff --git a/.github/workflows/test-storybook.yml b/.github/workflows/old/test-storybook.yml similarity index 100% rename from .github/workflows/test-storybook.yml rename to .github/workflows/old/test-storybook.yml diff --git a/.github/workflows/test-ts-lint.yml b/.github/workflows/old/test-ts-lint.yml similarity index 100% rename from .github/workflows/test-ts-lint.yml rename to .github/workflows/old/test-ts-lint.yml diff --git a/.github/workflows/regression-test.yml b/.github/workflows/regression-test.yml new file mode 100644 index 0000000..8216332 --- /dev/null +++ b/.github/workflows/regression-test.yml @@ -0,0 +1,48 @@ +# This workflow will install Python dependencies, run tests and lint with a single version of Python +# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python + +name: Regression Test + +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] + +permissions: + contents: read + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - name: Set up Python 3.10 + uses: actions/setup-python@v3 + with: + python-version: "3.10" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install flake8 pytest pytest-reportlog black + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + - name: Test PR branch with pytest + run: | + pytest --report-log=pr_test_results.json + - name: Checkout main + uses: actions/checkout@v4 + with: + ref: main + path: main + - name: Test main + run: | + pytest --report-log=main_test_results.json + - name: Print test results + run: | + cat pr_test_results.json + cat main_test_results.json + - name: Compare results + run: | + python main.py pr_test_results.json main_test_results.json \ No newline at end of file diff --git a/main.py b/main.py new file mode 100644 index 0000000..df210bb --- /dev/null +++ b/main.py @@ -0,0 +1,37 @@ +import json +import pathlib +from collections import defaultdict +import sys + + +def main(): + source_dict = defaultdict(list) + dest_dict = defaultdict(list) + source_fp = pathlib.Path(sys.argv[1]) + dest_fp = pathlib.Path(sys.argv[2]) + with open(source_fp, "r") as f: + source_results = f.read().splitlines()[3:-1] # Trim unnecessary entries + for item in source_results: + print(f"ITEM: {item}") + json_data = json.loads(item) + if "nodeid" in json_data.keys(): + source_dict[json_data["outcome"]].append(json_data["nodeid"]) + with open(dest_fp, "r") as f: + dest_results = f.read().splitlines()[3:-1] # Trim unnecessary entries + for item in dest_results: + json_data = json.loads(item) + if "nodeid" in json_data.keys(): + dest_dict[json_data["outcome"]].append(json_data["nodeid"]) + regressions = [] + for item in dest_dict["passed"]: + if item not in source_dict["passed"]: + regressions.append(item) + print(f"{len(regressions)} regressions were found") + for item in regressions: + print(f"Regression {item}") + if len(regressions) > 0: + exit(1) + + +if __name__ == "__main__": + main() diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..99844fa --- /dev/null +++ b/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +testpaths=*_test.py diff --git a/sanity_test.py b/sanity_test.py new file mode 100644 index 0000000..b01dc18 --- /dev/null +++ b/sanity_test.py @@ -0,0 +1,5 @@ +def test_nothing(): + assert False + +def test_new_passing_test(): + assert True \ No newline at end of file From 37595d73fa4750d0049aca3bdda614def2fe1b9c Mon Sep 17 00:00:00 2001 From: austinleblanc Date: Fri, 6 Jun 2025 12:35:15 -0400 Subject: [PATCH 2/2] Updates to regression checking logic/workflow --- .github/workflows/regression-test.yml | 10 +++++-- main.py | 42 +++++++++++++++++---------- 2 files changed, 34 insertions(+), 18 deletions(-) diff --git a/.github/workflows/regression-test.yml b/.github/workflows/regression-test.yml index 8216332..fde8896 100644 --- a/.github/workflows/regression-test.yml +++ b/.github/workflows/regression-test.yml @@ -16,7 +16,6 @@ jobs: build: runs-on: ubuntu-latest - steps: - uses: actions/checkout@v4 - name: Set up Python 3.10 @@ -28,9 +27,15 @@ jobs: python -m pip install --upgrade pip pip install flake8 pytest pytest-reportlog black if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + - name: Checkout PR branch + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.ref }} - name: Test PR branch with pytest run: | pytest --report-log=pr_test_results.json + find . -name "*.pyc" | xargs rm + continue-on-error: true - name: Checkout main uses: actions/checkout@v4 with: @@ -38,7 +43,8 @@ jobs: path: main - name: Test main run: | - pytest --report-log=main_test_results.json + pytest ./main --report-log=main_test_results.json + continue-on-error: true - name: Print test results run: | cat pr_test_results.json diff --git a/main.py b/main.py index df210bb..bf707f8 100644 --- a/main.py +++ b/main.py @@ -5,30 +5,40 @@ def main(): - source_dict = defaultdict(list) - dest_dict = defaultdict(list) - source_fp = pathlib.Path(sys.argv[1]) - dest_fp = pathlib.Path(sys.argv[2]) - with open(source_fp, "r") as f: - source_results = f.read().splitlines()[3:-1] # Trim unnecessary entries - for item in source_results: + pull_request_dict = defaultdict(list) + destination_branch_dict = defaultdict(list) + pull_request_fp = pathlib.Path(sys.argv[1]) + destination_branch_fp = pathlib.Path(sys.argv[2]) + with open(pull_request_fp, "r") as f: + pull_request_results = f.read().splitlines()[3:-1] # Trim unnecessary entries + for item in pull_request_results: print(f"ITEM: {item}") json_data = json.loads(item) if "nodeid" in json_data.keys(): - source_dict[json_data["outcome"]].append(json_data["nodeid"]) - with open(dest_fp, "r") as f: - dest_results = f.read().splitlines()[3:-1] # Trim unnecessary entries - for item in dest_results: + pull_request_dict[json_data["outcome"]].append(json_data["nodeid"]) + with open(destination_branch_fp, "r") as f: + destination_branch_results = f.read().splitlines()[ + 3:-1 + ] # Trim unnecessary entries + for item in destination_branch_results: json_data = json.loads(item) if "nodeid" in json_data.keys(): - dest_dict[json_data["outcome"]].append(json_data["nodeid"]) + destination_branch_dict[json_data["outcome"]].append( + json_data["nodeid"] + ) regressions = [] - for item in dest_dict["passed"]: - if item not in source_dict["passed"]: + warnings = [] + for item in destination_branch_dict["passed"]: + if item in pull_request_dict["failed"]: regressions.append(item) + elif item in pull_request_dict["skipped"]: + warnings.append(item) + print(f"{len(warnings)} tests now skipped/xfailed that weren't previously") + for warning in warnings: + print(f"Warning: {warning}") print(f"{len(regressions)} regressions were found") - for item in regressions: - print(f"Regression {item}") + for regression in regressions: + print(f"Regression: {regression}") if len(regressions) > 0: exit(1)