From 7bb86f1f4124787dcd3c4fce5b68a41e73be27be Mon Sep 17 00:00:00 2001 From: zhizhongpu <84325421+zhizhongpu@users.noreply.github.com> Date: Wed, 18 Mar 2026 17:53:03 -0600 Subject: [PATCH 01/19] #140 bd migrate in github actions --- .github/README.md | 7 + .github/actions/timed-check/action.yml | 17 ++ .github/check_eps_savefig.py | 147 ++++++++++++++++++ .github/check_newlines.py | 38 +++++ .github/check_sconscript_log.py | 27 ++++ .github/check_sconscripts.py | 113 ++++++++++++++ .github/post_check_results.py | 63 ++++++++ .../post_template_issue_thread_pr_close.md | 10 ++ .github/post_template_pr_thread_pr_close.md | 9 ++ .github/scripts/pr_close_comment.py | 47 ++++++ .github/workflows/checks.yml | 94 +++++++++++ .github/workflows/pr-close-comment.yml | 36 +++++ 12 files changed, 608 insertions(+) create mode 100644 .github/README.md create mode 100644 .github/actions/timed-check/action.yml create mode 100644 .github/check_eps_savefig.py create mode 100644 .github/check_newlines.py create mode 100644 .github/check_sconscript_log.py create mode 100644 .github/check_sconscripts.py create mode 100644 .github/post_check_results.py create mode 100644 .github/post_template_issue_thread_pr_close.md create mode 100644 .github/post_template_pr_thread_pr_close.md create mode 100644 .github/scripts/pr_close_comment.py create mode 100644 .github/workflows/checks.yml create mode 100644 .github/workflows/pr-close-comment.yml diff --git a/.github/README.md b/.github/README.md new file mode 100644 index 0000000..7e971e9 --- /dev/null +++ b/.github/README.md @@ -0,0 +1,7 @@ +# GitHub Actions Checks + +## Adding a new check + +1. Add a step to `checks.yml` with `id: check_` +2. Add `CHECK__OUTCOME` and `CHECK__TIME` env vars to the `Post results` step in `checks.yml` +3. Add `("Display Name", "check_")` to `CHECKS` in `post_check_results.py` diff --git a/.github/actions/timed-check/action.yml b/.github/actions/timed-check/action.yml new file mode 100644 index 0000000..ac7841a --- /dev/null +++ b/.github/actions/timed-check/action.yml @@ -0,0 +1,17 @@ +name: Timed Python Check +inputs: + script: + required: true +outputs: + time: + description: Elapsed time in seconds + value: ${{ steps.run.outputs.time }} +runs: + using: composite + steps: + - id: run + shell: bash + run: | + s=$(date +%s%N); python ${{ inputs.script }} || e=$? + awk "BEGIN{printf \"time=%.3f\n\",($(date +%s%N)-$s)/1e9}" >> $GITHUB_OUTPUT + exit ${e:-0} diff --git a/.github/check_eps_savefig.py b/.github/check_eps_savefig.py new file mode 100644 index 0000000..e864ceb --- /dev/null +++ b/.github/check_eps_savefig.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 +import os +import re +import sys + +def IsExcludedPath(path, excluded): + norm = os.path.normpath(path) + return any(norm == e or norm.startswith(e + os.sep) for e in excluded) + +def IsHidden(name): + return name.startswith(".") + +def IsIgnoredDir(name): + return IsHidden(name) or name == "__pycache__" + +def ReadFile(path): + try: + with open(path, "r", encoding="utf-8") as fh: + return fh.read() + except Exception: + return None + +def CheckEpsSavefig(file_path, content): + """ + Check if file contains .savefig(*eps*) without remove_eps_info( + Returns list of problematic lines with line numbers + """ + problems = [] + lines = content.split('\n') + + # Pattern to match .savefig with eps format (both single and double quotes) + eps_savefig_patterns = [ + r'\.savefig\([^)]*[\'"].*eps.*[\'"][^)]*\)', # matches .savefig(...'...eps...'...) + r'\.savefig\([^)]*format\s*=\s*[\'"]eps[\'"][^)]*\)' # matches format='eps' or format="eps" + ] + + has_remove_eps_info = 'remove_eps_info(' in content + + for line_num, line in enumerate(lines, 1): + for pattern in eps_savefig_patterns: + if re.search(pattern, line, re.IGNORECASE): + if not has_remove_eps_info: + problems.append(f"Line {line_num}: {line.strip()}") + break + + return problems + +def CollectEpsProblems(root, excluded): + """Walk through Python files and check for EPS savefig issues""" + problems = [] + + for dir_path, dir_names, file_names in os.walk(root): + if IsExcludedPath(dir_path, excluded): + dir_names[:] = [] + continue + + dir_names[:] = [d for d in dir_names if not IsIgnoredDir(d)] + + for file_name in file_names: + if not file_name.endswith('.py'): + continue + + if IsHidden(file_name): + continue + + file_path = os.path.join(dir_path, file_name) + content = ReadFile(file_path) + + if content is None: + continue + + file_problems = CheckEpsSavefig(file_path, content) + if file_problems: + problems.append({ + 'file': file_path, + 'issues': file_problems + }) + + return problems + +def CheckEpsCreationDate(content): + """ + Check if any line in an EPS file starts with %%CreationDate. + Returns True if it does. + """ + return any(line.startswith("%%CreationDate") for line in content.splitlines()) + +def CollectEpsCreationDateProblems(root, excluded): + """Walk through EPS files and report those that contain %%CreationDate""" + eps_files = [] + + for dir_path, dir_names, file_names in os.walk(root): + if IsExcludedPath(dir_path, excluded): + dir_names[:] = [] + continue + + dir_names[:] = [d for d in dir_names if not IsIgnoredDir(d)] + + for file_name in file_names: + if not file_name.endswith('.eps'): + continue + + if IsHidden(file_name): + continue + + eps_path = os.path.join(dir_path, file_name) + content = ReadFile(eps_path) + + if content is None: + continue + + if CheckEpsCreationDate(content): + eps_files.append(eps_path) + + return eps_files + +def main(): + source_root = "source" + output_root = "output" + excluded = ["source/lib", "source/raw", "source/scrape"] + + problems = CollectEpsProblems(source_root, excluded) + creationdate_eps_files = CollectEpsCreationDateProblems(output_root, []) + + if creationdate_eps_files: + print("EPS files containing %%CreationDate:") + for eps_file in creationdate_eps_files: + print(f" {eps_file}") + print("") + return 1 + + if problems: + print("EPS savefig check failed!") + print("\nPython files using .savefig(*eps*) without remove_eps_info():") + for problem in problems: + print(f"\nFile: {problem['file']}") + for issue in problem['issues']: + print(f" {issue}") + print("\nTo fix: Add 'from source.lib.JMSLab.remove_eps_info import remove_eps_info'") + print("and call 'remove_eps_info(filename)' after each EPS savefig.") + return 1 + else: + print("EPS savefig check: all checks passed.") + return 0 + +if __name__ == "__main__": + sys.exit(main()) \ No newline at end of file diff --git a/.github/check_newlines.py b/.github/check_newlines.py new file mode 100644 index 0000000..2f23cd1 --- /dev/null +++ b/.github/check_newlines.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +import subprocess +import sys +from pathlib import Path + +EXCLUDED_EXT = {".log", ".txt"} + +INCLUDED_DIRS = {"source", "output"} + +def GetTrackedFiles(): + return [Path(p) for p in subprocess.run(["git","ls-files"], stdout=subprocess.PIPE, check=True).stdout.decode().splitlines() if p.strip() and Path(p).parts[0] in INCLUDED_DIRS] + +def IsBinary(p): + try: + return b"\0" in p.read_bytes()[:4096] + except: + return True + +def NeedsNewline(p): + try: + d = p.read_bytes() + return len(d)==0 or not d.endswith(b"\n") + except: + return False + +def ProcessFiles(): + missing = [p for p in GetTrackedFiles() if p.exists() and p.suffix not in EXCLUDED_EXT and not IsBinary(p) and NeedsNewline(p)] + if not missing: + print("No files missing trailing newlines."); return 0 + print("Files missing trailing newline:") + for p in missing: print(" -", p) + return 1 + +def Main(): + return ProcessFiles() + +if __name__=="__main__": + sys.exit(Main()) diff --git a/.github/check_sconscript_log.py b/.github/check_sconscript_log.py new file mode 100644 index 0000000..96742db --- /dev/null +++ b/.github/check_sconscript_log.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python3 +import sys +from pathlib import Path + +TARGET = "terminated because of errors." + +def Main(): + bad = [] + for p in Path(".").rglob("**/*.log"): + try: + if TARGET in p.read_text(errors="replace"): + bad.append(p) + except: + pass + + if not bad: + print("No log files contain the error string.") + return 0 + + print("Problematic log files:") + for p in bad: + print(" -", p) + + return 1 + +if __name__ == "__main__": + sys.exit(Main()) diff --git a/.github/check_sconscripts.py b/.github/check_sconscripts.py new file mode 100644 index 0000000..be10771 --- /dev/null +++ b/.github/check_sconscripts.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +import os +import re +import sys +from pathlib import Path + +ROOT = Path("source") +EXCLUDED = {Path("source/lib"), Path("source/raw"), Path("source/scrape")} +PAPER_DIR = Path("source/paper") +PAPER_EXTS = {".bib", ".tex", ".lyx"} + + +def Main(): + missing_dirs, missing_mentions = CollectProblems() + missing_in_sconstruct = SourceFoldersMissingInSConstruct() + any_problems = bool(missing_dirs or missing_mentions or missing_in_sconstruct) + if any_problems: + print("SConscript/SConstruct summary of missing items:") + if missing_dirs: + print("\nFolders missing SConscript:") + for p in missing_dirs: + print(p) + if missing_mentions: + print("\nFiles not mentioned in their SConscript:") + for p in missing_mentions: + print(p) + if missing_in_sconstruct: + print("\nTop-level source folders missing from root SConstruct:") + for p in missing_in_sconstruct: + print(p) + else: + print("SConscript/SConstruct summary: all checks passed.") + return 1 if any_problems else 0 + + +def CollectProblems(): + missing_dirs = [] + missing_mentions = [] + for dir_path, dir_names, file_names in os.walk(ROOT): + dir_path = Path(dir_path) + if IsExcluded(dir_path): + dir_names[:] = [] + continue + dir_names[:] = sorted(d for d in dir_names if not IsIgnored(d)) + file_names = [f for f in file_names if not IsIgnored(f)] + if dir_path == ROOT or (not dir_names and not file_names): + continue + content = Read(dir_path / "SConscript") + if content is None: + parent_content = Read(dir_path.parent / "SConscript") + if parent_content is None: + missing_dirs.append(dir_path) + continue + for f in sorted(f for f in file_names if f != "SConscript"): + if ShouldCheck(dir_path, f) and not IsMentioned(content, f, dir_path): + missing_mentions.append(f"{dir_path} -> {f}") + for subdir in dir_names: + if re.search(rf"\b{re.escape(subdir)}\b", content): + continue + subdir_path = dir_path / subdir + try: + subfiles = sorted( + e for e in os.listdir(subdir_path) + if not IsIgnored(e) and (subdir_path / e).is_file() and ShouldCheck(subdir_path, e) + ) + except Exception: + missing_dirs.append(subdir_path) + continue + for f in subfiles: + if not IsMentioned(content, f, subdir_path): + missing_mentions.append(f"{dir_path} -> {subdir}/{f}") + return missing_dirs, missing_mentions + + +def IsExcluded(dir_path): + return any(dir_path == e or dir_path.is_relative_to(e) for e in EXCLUDED) + + +def IsIgnored(name): + return name.startswith(".") or name == "__pycache__" + + +def ShouldCheck(dir_path, name): + if dir_path.is_relative_to(PAPER_DIR): + return Path(name).suffix in PAPER_EXTS + return True + + +def Read(path): + try: + return Path(path).read_text(encoding="utf-8") + except Exception: + return None + + +def IsMentioned(content, name, dir_path): + rel = dir_path.relative_to(ROOT).as_posix() + return f"#source/{rel}/{name}" in content + + +def SourceFoldersMissingInSConstruct(): + content = Read("SConstruct") + folders = sorted( + e for e in os.listdir(ROOT) + if (ROOT / e).is_dir() and not IsIgnored(e) and not IsExcluded(ROOT / e) + ) + if content is None: + return folders + return [f for f in folders if f"source/{f}/SConscript" not in content] + + +if __name__ == "__main__": + sys.exit(Main()) diff --git a/.github/post_check_results.py b/.github/post_check_results.py new file mode 100644 index 0000000..f66162b --- /dev/null +++ b/.github/post_check_results.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +import os +import subprocess +import sys + +CHECKS = [ + ("SCons DAG", "check_scons"), + ("Newlines", "check_newlines"), + ("EPS data", "check_eps"), + ("Build log", "check_scons_log"), +] + +def Main(): + repo = os.environ["GITHUB_REPOSITORY"] + run_id = os.environ["GITHUB_RUN_ID"] + + print("Runtime:") + rows, failed = CollectResults() + + if "--post" in sys.argv: + PostResults(repo, run_id, rows, failed) + + if failed: + print(f"Failed checks: {', '.join(failed)}") + return 1 + return 0 + +def CollectResults(): + rows, failed = [], [] + for name, step_id in CHECKS: + key = step_id.upper() + outcome = os.environ.get(f"{key}_OUTCOME", "skipped") + time = os.environ.get(f"{key}_TIME", "") + print(f" {step_id}: {time}s") + if outcome == "skipped": + rows.append(f"| {name} | SKIP | |") + elif outcome == "success": + rows.append(f"| {name} | ✅ | {time}s |") + else: + failed.append(name) + rows.append(f"| {name} | ❌ | {time}s |") + return rows, failed + +def PostResults(repo, run_id, rows, failed): + run_url = f"https://github.com/{repo}/actions/runs/{run_id}" + table = "\n".join(["| Check | Result | Time |", "|-------|--------|------|", *rows]) + body = f"**Check Results** ([run details]({run_url}))\n\n{table}" + pr_num = os.environ["PR_NUMBER"] + pr_sha = os.environ["PR_SHA"] + subprocess.run([ + "gh", "api", f"repos/{repo}/issues/{pr_num}/comments", + "--method", "POST", "-f", f"body={body}", + ], check=True) + state = "failure" if failed else "success" + desc = f"Failed: {', '.join(failed)}" if failed else "All checks passed" + subprocess.run([ + "gh", "api", f"repos/{repo}/statuses/{pr_sha}", + "-f", f"state={state}", "-f", "context=Checks", + "-f", f"description={desc}", "-f", f"target_url={run_url}", + ], check=True) + +if __name__ == "__main__": + sys.exit(Main()) diff --git a/.github/post_template_issue_thread_pr_close.md b/.github/post_template_issue_thread_pr_close.md new file mode 100644 index 0000000..dc49fbf --- /dev/null +++ b/.github/post_template_issue_thread_pr_close.md @@ -0,0 +1,10 @@ +### Summary + +In this issue, we: + +- WHAT DID YOU DO? + +Deliverables: +- WHAT DID YOU BUILD? IN PERMALINK + +[Optional] Issue folder is PERMALINK diff --git a/.github/post_template_pr_thread_pr_close.md b/.github/post_template_pr_thread_pr_close.md new file mode 100644 index 0000000..037083e --- /dev/null +++ b/.github/post_template_pr_thread_pr_close.md @@ -0,0 +1,9 @@ +Thanks for closing the PR. + +You need to: +- [ ] edit the issue summary post above +- [ ] update other issue branches using `main` +- [ ] update the datastore in the cloud +- [ ] delete this issue branch + +If you have questions, please ask by tagging the reviewers of this PR. diff --git a/.github/scripts/pr_close_comment.py b/.github/scripts/pr_close_comment.py new file mode 100644 index 0000000..36360ec --- /dev/null +++ b/.github/scripts/pr_close_comment.py @@ -0,0 +1,47 @@ +import os +import re +from pathlib import Path +from github import Github + +GITHUB_TOKEN = os.environ["GITHUB_TOKEN"] +REPO_NAME = os.environ["REPO"] +PR_NUMBER = int(os.environ["PR_NUMBER"]) +PR_AUTHOR = os.environ["PR_AUTHOR"] +BRANCH_NAME = os.environ["BRANCH_NAME"] +LAST_COMMIT_SHA = os.environ["LAST_COMMIT_SHA"] + +ISSUE_TEMPLATE = Path(".github/post_template_issue_thread_pr_close.md") +PR_TEMPLATE = Path(".github/post_template_pr_thread_pr_close.md") + + +def Main(): + repo = Github(GITHUB_TOKEN).get_repo(REPO_NAME) + pr = repo.get_pull(PR_NUMBER) + + issue_comment_url = PostIssueComment(repo) + PostPrComment(pr, issue_comment_url) + + +def PostIssueComment(repo): + issue_match = re.match(r"^(\d+)", BRANCH_NAME) + if not issue_match: + return None + + issue_number = int(issue_match.group(1)) + issue_body = ISSUE_TEMPLATE.read_text() + f"\n\nLast commit in issue branch: {LAST_COMMIT_SHA}" + + comment = repo.get_issue(issue_number).create_comment(issue_body) + return comment.html_url + + +def PostPrComment(pr, issue_comment_url): + pr_body = PR_TEMPLATE.read_text() + + if issue_comment_url: + pr_body = f"[Issue summary]({issue_comment_url})\n\n{pr_body}" + + pr.as_issue().create_comment(f"@{PR_AUTHOR} {pr_body}") + + +if __name__ == "__main__": + Main() diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml new file mode 100644 index 0000000..e614d2d --- /dev/null +++ b/.github/workflows/checks.yml @@ -0,0 +1,94 @@ +name: Checks + +on: + push: + branches: + - "**" + issue_comment: + types: [created] + +jobs: + checks: + if: | + (github.event_name == 'push' && contains(github.event.head_commit.message, '[run-actions-all]')) || + (github.event_name == 'issue_comment' && github.event.issue.pull_request && contains(github.event.comment.body, '/run-actions-')) + runs-on: ubuntu-latest + timeout-minutes: 1 + permissions: + contents: write + statuses: write + pull-requests: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + ref: ${{ github.event_name == 'issue_comment' && format('refs/pull/{0}/head', github.event.issue.number) || github.sha }} + + - name: Post pending commit status + if: github.event_name == 'issue_comment' + env: + GH_TOKEN: ${{ github.token }} + run: | + PR_SHA=$(gh api repos/${{ github.repository }}/pulls/${{ github.event.issue.number }} --jq '.head.sha') + echo "PR_SHA=$PR_SHA" >> $GITHUB_ENV + gh api repos/${{ github.repository }}/statuses/$PR_SHA \ + -f state=pending \ + -f context="Checks" \ + -f description="Running..." \ + -f target_url="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.x" + + - id: check_scons + name: Check SCons DAG for missing dependencies + if: github.event_name == 'push' || contains(github.event.comment.body, '/run-actions-dag') || contains(github.event.comment.body, '/run-actions-all') + continue-on-error: true + uses: ./.github/actions/timed-check + with: + script: .github/check_sconscripts.py + + - id: check_newlines + name: Check missing newlines + if: github.event_name == 'push' || contains(github.event.comment.body, '/run-actions-newlines') || contains(github.event.comment.body, '/run-actions-all') + continue-on-error: true + uses: ./.github/actions/timed-check + with: + script: .github/check_newlines.py + + - id: check_eps + name: Check extraneous EPS data + if: github.event_name == 'push' || contains(github.event.comment.body, '/run-actions-eps') || contains(github.event.comment.body, '/run-actions-all') + continue-on-error: true + uses: ./.github/actions/timed-check + with: + script: .github/check_eps_savefig.py + + - id: check_scons_log + name: Check build failure + if: github.event_name == 'push' || contains(github.event.comment.body, '/run-actions-log') || contains(github.event.comment.body, '/run-actions-all') + continue-on-error: true + uses: ./.github/actions/timed-check + with: + script: .github/check_sconscript_log.py + + - name: Post results + if: always() + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.issue.number }} + PR_SHA: ${{ env.PR_SHA }} + CHECK_SCONS_OUTCOME: ${{ steps.check_scons.outcome }} + CHECK_SCONS_TIME: ${{ steps.check_scons.outputs.time }} + CHECK_NEWLINES_OUTCOME: ${{ steps.check_newlines.outcome }} + CHECK_NEWLINES_TIME: ${{ steps.check_newlines.outputs.time }} + CHECK_EPS_OUTCOME: ${{ steps.check_eps.outcome }} + CHECK_EPS_TIME: ${{ steps.check_eps.outputs.time }} + CHECK_SCONS_LOG_OUTCOME: ${{ steps.check_scons_log.outcome }} + CHECK_SCONS_LOG_TIME: ${{ steps.check_scons_log.outputs.time }} + run: | + python .github/post_check_results.py \ + ${{ contains(github.event.comment.body, '--post') && '--post' || '' }} diff --git a/.github/workflows/pr-close-comment.yml b/.github/workflows/pr-close-comment.yml new file mode 100644 index 0000000..3f787b1 --- /dev/null +++ b/.github/workflows/pr-close-comment.yml @@ -0,0 +1,36 @@ +# Post a comment when a PR is closed (merged or unmerged) + +name: PR Close Comment + +on: + pull_request: + types: [closed] + +permissions: + issues: write + pull-requests: write + +jobs: + comment: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install dependencies + run: pip install PyGithub + + - name: Comment on PR close + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REPO: ${{ github.repository }} + PR_NUMBER: ${{ github.event.pull_request.number }} + PR_AUTHOR: ${{ github.event.pull_request.user.login }} + BRANCH_NAME: ${{ github.event.pull_request.head.ref }} + LAST_COMMIT_SHA: ${{ github.event.pull_request.head.sha }} + run: python .github/scripts/pr_close_comment.py From 07324157cea08d8afaab13ed84ef5bc9d1a4375a Mon Sep 17 00:00:00 2001 From: jmshapir Date: Thu, 19 Mar 2026 14:39:44 -0400 Subject: [PATCH 02/19] Adds datastore to workflow for #140 --- docs/workflow.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/workflow.md b/docs/workflow.md index 1b142a3..aee04bf 100644 --- a/docs/workflow.md +++ b/docs/workflow.md @@ -29,7 +29,7 @@ We suggest the following adaptation of [Github flow](https://docs.github.com/en/ * the issue subfolder (ephemeral deliverables) * the latest version of the issue branch prior to merging (all deliverables). * **Link** to the summary comment in the pull request. - * **Update** all open branches from `main`. + * **Update** all open branches from `main` and (if needed) the datastore. * If you encounter merge conflicts you cannot resolve, check with the _assignee(s)_ of the corresponding issue(s). * **Prioritize** work in the order older pull requests > newer pull requests > older issues > newer issues. * Age is defined by github numbering. From d4fc0d9f5f843e673b9b57e6379723a24c1e3bae Mon Sep 17 00:00:00 2001 From: jmshapir Date: Thu, 19 Mar 2026 14:43:03 -0400 Subject: [PATCH 03/19] Revisions to message for #140 --- .github/post_template_pr_thread_pr_close.md | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/.github/post_template_pr_thread_pr_close.md b/.github/post_template_pr_thread_pr_close.md index 037083e..2299dc1 100644 --- a/.github/post_template_pr_thread_pr_close.md +++ b/.github/post_template_pr_thread_pr_close.md @@ -1,9 +1,3 @@ -Thanks for closing the PR. +Thanks for closing this pull. -You need to: -- [ ] edit the issue summary post above -- [ ] update other issue branches using `main` -- [ ] update the datastore in the cloud -- [ ] delete this issue branch - -If you have questions, please ask by tagging the reviewers of this PR. +Before leaving the pull, please be sure you have completed all the required steps in the [workflow](https://github.com/JMSLab/Template/blob/main/docs/workflow.md)! From 6003fe3f040221f1d0af1ee9f70fd3aa701d5e2d Mon Sep 17 00:00:00 2001 From: jmshapir Date: Thu, 19 Mar 2026 14:49:26 -0400 Subject: [PATCH 04/19] Documents tests for #140 --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index b20b4a7..055ebfa 100644 --- a/README.md +++ b/README.md @@ -91,6 +91,14 @@ env.Stata(target, source) - For tips on batch-specifying targets and sources, see [./docs/batch_specifying.md](./docs/batch_specifying.md). +### Automation + +The repository is prebuilt with some automated testing using [Github Actions](./github). + +To run all tests, add `[run-actions-all]` to a commit message or type `/run-actions-all` in a comment. + +To run a particular test, type `/run-actions-NAMEOFTEST` in a comment (e.g., `/run-actions-log` to run [this test](https://github.com/JMSLab/Template/blob/140-automating-pr-reviews-using-github-actions/.github/check_sconscript_log.py). + ### Citations and expectations for usage This template is based on [gslab-econ/Template/v4.1.3](https://github.com/gslab-econ/template/releases/tag/4.1.3) and [gslab-python/v4.1.4](https://github.com/gslab-econ/gslab_python/releases/tag/v4.1.4). From 90ccf639d9e0634f432040eb1eac057e6f5b1055 Mon Sep 17 00:00:00 2001 From: zhizhongpu <84325421+zhizhongpu@users.noreply.github.com> Date: Thu, 19 Mar 2026 14:42:59 -0600 Subject: [PATCH 05/19] #140 bd extra line --- .github/check_eps_savefig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/check_eps_savefig.py b/.github/check_eps_savefig.py index e864ceb..423214b 100644 --- a/.github/check_eps_savefig.py +++ b/.github/check_eps_savefig.py @@ -144,4 +144,4 @@ def main(): return 0 if __name__ == "__main__": - sys.exit(main()) \ No newline at end of file + sys.exit(main()) From df96e64bb269990855ea78139ceb0ba7e4700eb5 Mon Sep 17 00:00:00 2001 From: zhizhongpu <84325421+zhizhongpu@users.noreply.github.com> Date: Thu, 19 Mar 2026 14:45:29 -0600 Subject: [PATCH 06/19] #140 rename README --- .github/{README.md => readme.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/{README.md => readme.md} (100%) diff --git a/.github/README.md b/.github/readme.md similarity index 100% rename from .github/README.md rename to .github/readme.md From 79ee7b972def17f5e901a2505d84e03c21035dd9 Mon Sep 17 00:00:00 2001 From: zhizhongpu <84325421+zhizhongpu@users.noreply.github.com> Date: Thu, 19 Mar 2026 14:51:15 -0600 Subject: [PATCH 07/19] #140 reorg rename github_action_checks.md --- .github/{readme.md => github_action_checks.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/{readme.md => github_action_checks.md} (100%) diff --git a/.github/readme.md b/.github/github_action_checks.md similarity index 100% rename from .github/readme.md rename to .github/github_action_checks.md From 46e832037f7c530036d8cbb10a16b5a2967b010b Mon Sep 17 00:00:00 2001 From: Chris Liao <33707455+liaochris@users.noreply.github.com> Date: Thu, 19 Mar 2026 18:58:58 -0400 Subject: [PATCH 08/19] Improve how we add actions [run-actions-all] #141 --- .github/actions/timed-check/action.yml | 19 ++++++----- .github/checks.json | 6 ++++ .github/github_action_checks.md | 7 ++-- .github/post_check_results.py | 46 ++++++++++++++------------ .github/workflows/checks.yml | 12 +++---- 5 files changed, 48 insertions(+), 42 deletions(-) create mode 100644 .github/checks.json diff --git a/.github/actions/timed-check/action.yml b/.github/actions/timed-check/action.yml index ac7841a..0d03791 100644 --- a/.github/actions/timed-check/action.yml +++ b/.github/actions/timed-check/action.yml @@ -2,16 +2,17 @@ name: Timed Python Check inputs: script: required: true -outputs: - time: - description: Elapsed time in seconds - value: ${{ steps.run.outputs.time }} + display: + required: true runs: using: composite steps: - - id: run - shell: bash + - shell: bash run: | - s=$(date +%s%N); python ${{ inputs.script }} || e=$? - awk "BEGIN{printf \"time=%.3f\n\",($(date +%s%N)-$s)/1e9}" >> $GITHUB_OUTPUT - exit ${e:-0} + mkdir -p "$RUNNER_TEMP/check_results" + s=$(date +%s%N) + python ${{ inputs.script }} && outcome=success || outcome=failure + time=$(awk "BEGIN{printf \"%.3f\",($(date +%s%N)-$s)/1e9}") + printf '{"outcome":"%s","time":"%s"}' "$outcome" "$time" \ + > "$RUNNER_TEMP/check_results/${{ inputs.display }}.json" + [ "$outcome" = "success" ] diff --git a/.github/checks.json b/.github/checks.json new file mode 100644 index 0000000..4811be2 --- /dev/null +++ b/.github/checks.json @@ -0,0 +1,6 @@ +[ + {"name": "SCons DAG"}, + {"name": "Newlines"}, + {"name": "EPS data"}, + {"name": "Build log"} +] diff --git a/.github/github_action_checks.md b/.github/github_action_checks.md index 7e971e9..046b96b 100644 --- a/.github/github_action_checks.md +++ b/.github/github_action_checks.md @@ -2,6 +2,7 @@ ## Adding a new check -1. Add a step to `checks.yml` with `id: check_` -2. Add `CHECK__OUTCOME` and `CHECK__TIME` env vars to the `Post results` step in `checks.yml` -3. Add `("Display Name", "check_")` to `CHECKS` in `post_check_results.py` +1. Add a step to `checks.yml` with a unique `id` and `display` (e.g. `id: check_newlines`, `display: Newlines`) +2. Add `{"name": "Newlines"}` (i.e. matching `display`) to `checks.json` + +The order of entries in `checks.json` controls the row order in the results table. diff --git a/.github/post_check_results.py b/.github/post_check_results.py index f66162b..20c39c6 100644 --- a/.github/post_check_results.py +++ b/.github/post_check_results.py @@ -1,14 +1,10 @@ #!/usr/bin/env python3 +import json import os import subprocess import sys -CHECKS = [ - ("SCons DAG", "check_scons"), - ("Newlines", "check_newlines"), - ("EPS data", "check_eps"), - ("Build log", "check_scons_log"), -] +CHECKS_JSON = os.path.join(os.path.dirname(__file__), 'checks.json') def Main(): repo = os.environ["GITHUB_REPOSITORY"] @@ -26,27 +22,33 @@ def Main(): return 0 def CollectResults(): + checks = json.load(open(CHECKS_JSON)) + results_dir = os.path.join(os.environ["RUNNER_TEMP"], "check_results") rows, failed = [], [] - for name, step_id in CHECKS: - key = step_id.upper() - outcome = os.environ.get(f"{key}_OUTCOME", "skipped") - time = os.environ.get(f"{key}_TIME", "") - print(f" {step_id}: {time}s") - if outcome == "skipped": - rows.append(f"| {name} | SKIP | |") - elif outcome == "success": - rows.append(f"| {name} | ✅ | {time}s |") + for check in checks: + name = check["name"] + result_file = os.path.join(results_dir, f"{name}.json") + if os.path.exists(result_file): + result = json.load(open(result_file)) + outcome = result["outcome"] + time = result["time"] + print(f" {name}: {time}s") + if outcome == "success": + rows.append(f"| {name} | ✅ | {time}s |") + else: + failed.append(name) + rows.append(f"| {name} | ❌ | {time}s |") else: - failed.append(name) - rows.append(f"| {name} | ❌ | {time}s |") + print(f" {name}: skipped") + rows.append(f"| {name} | SKIP | |") return rows, failed def PostResults(repo, run_id, rows, failed): - run_url = f"https://github.com/{repo}/actions/runs/{run_id}" - table = "\n".join(["| Check | Result | Time |", "|-------|--------|------|", *rows]) - body = f"**Check Results** ([run details]({run_url}))\n\n{table}" - pr_num = os.environ["PR_NUMBER"] - pr_sha = os.environ["PR_SHA"] + run_url = f"https://github.com/{repo}/actions/runs/{run_id}" + table = "\n".join(["| Check | Result | Time |", "|-------|--------|------|", *rows]) + body = f"**Check Results** ([run details]({run_url}))\n\n{table}" + pr_num = os.environ["PR_NUMBER"] + pr_sha = os.environ["PR_SHA"] subprocess.run([ "gh", "api", f"repos/{repo}/issues/{pr_num}/comments", "--method", "POST", "-f", f"body={body}", diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index e614d2d..c0c70e7 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -50,6 +50,7 @@ jobs: uses: ./.github/actions/timed-check with: script: .github/check_sconscripts.py + display: SCons DAG - id: check_newlines name: Check missing newlines @@ -58,6 +59,7 @@ jobs: uses: ./.github/actions/timed-check with: script: .github/check_newlines.py + display: Newlines - id: check_eps name: Check extraneous EPS data @@ -66,6 +68,7 @@ jobs: uses: ./.github/actions/timed-check with: script: .github/check_eps_savefig.py + display: EPS data - id: check_scons_log name: Check build failure @@ -74,6 +77,7 @@ jobs: uses: ./.github/actions/timed-check with: script: .github/check_sconscript_log.py + display: Build log - name: Post results if: always() @@ -81,14 +85,6 @@ jobs: GH_TOKEN: ${{ github.token }} PR_NUMBER: ${{ github.event.issue.number }} PR_SHA: ${{ env.PR_SHA }} - CHECK_SCONS_OUTCOME: ${{ steps.check_scons.outcome }} - CHECK_SCONS_TIME: ${{ steps.check_scons.outputs.time }} - CHECK_NEWLINES_OUTCOME: ${{ steps.check_newlines.outcome }} - CHECK_NEWLINES_TIME: ${{ steps.check_newlines.outputs.time }} - CHECK_EPS_OUTCOME: ${{ steps.check_eps.outcome }} - CHECK_EPS_TIME: ${{ steps.check_eps.outputs.time }} - CHECK_SCONS_LOG_OUTCOME: ${{ steps.check_scons_log.outcome }} - CHECK_SCONS_LOG_TIME: ${{ steps.check_scons_log.outputs.time }} run: | python .github/post_check_results.py \ ${{ contains(github.event.comment.body, '--post') && '--post' || '' }} From bf0edefa0bd2b332844e8ff2e4db00ed2cf0ec71 Mon Sep 17 00:00:00 2001 From: Chris Liao <33707455+liaochris@users.noreply.github.com> Date: Thu, 19 Mar 2026 19:10:50 -0400 Subject: [PATCH 09/19] reorganize folder structure #141 --- .github/checks.json | 6 ----- .github/{ => checks}/check_eps_savefig.py | 0 .github/{ => checks}/check_newlines.py | 0 .github/{ => checks}/check_sconscript_log.py | 0 .github/{ => checks}/check_sconscripts.py | 0 .github/checks/checks.json | 6 +++++ .github/github_action_checks.md | 2 +- .github/helper_scripts/parse_commands.py | 8 +++++++ .../post_check_results.py | 6 +++-- .../pr_close_comment.py | 0 .github/workflows/checks.yml | 24 ++++++++++++------- .github/workflows/pr-close-comment.yml | 2 +- 12 files changed, 35 insertions(+), 19 deletions(-) delete mode 100644 .github/checks.json rename .github/{ => checks}/check_eps_savefig.py (100%) rename .github/{ => checks}/check_newlines.py (100%) rename .github/{ => checks}/check_sconscript_log.py (100%) rename .github/{ => checks}/check_sconscripts.py (100%) create mode 100644 .github/checks/checks.json create mode 100644 .github/helper_scripts/parse_commands.py rename .github/{ => helper_scripts}/post_check_results.py (88%) rename .github/{scripts => helper_scripts}/pr_close_comment.py (100%) diff --git a/.github/checks.json b/.github/checks.json deleted file mode 100644 index 4811be2..0000000 --- a/.github/checks.json +++ /dev/null @@ -1,6 +0,0 @@ -[ - {"name": "SCons DAG"}, - {"name": "Newlines"}, - {"name": "EPS data"}, - {"name": "Build log"} -] diff --git a/.github/check_eps_savefig.py b/.github/checks/check_eps_savefig.py similarity index 100% rename from .github/check_eps_savefig.py rename to .github/checks/check_eps_savefig.py diff --git a/.github/check_newlines.py b/.github/checks/check_newlines.py similarity index 100% rename from .github/check_newlines.py rename to .github/checks/check_newlines.py diff --git a/.github/check_sconscript_log.py b/.github/checks/check_sconscript_log.py similarity index 100% rename from .github/check_sconscript_log.py rename to .github/checks/check_sconscript_log.py diff --git a/.github/check_sconscripts.py b/.github/checks/check_sconscripts.py similarity index 100% rename from .github/check_sconscripts.py rename to .github/checks/check_sconscripts.py diff --git a/.github/checks/checks.json b/.github/checks/checks.json new file mode 100644 index 0000000..a0f40dc --- /dev/null +++ b/.github/checks/checks.json @@ -0,0 +1,6 @@ +[ + {"name": "SCons DAG", "command": "/run-actions-dag"}, + {"name": "Newlines", "command": "/run-actions-newlines"}, + {"name": "EPS data", "command": "/run-actions-eps"}, + {"name": "Build log", "command": "/run-actions-log"} +] diff --git a/.github/github_action_checks.md b/.github/github_action_checks.md index 046b96b..126391c 100644 --- a/.github/github_action_checks.md +++ b/.github/github_action_checks.md @@ -3,6 +3,6 @@ ## Adding a new check 1. Add a step to `checks.yml` with a unique `id` and `display` (e.g. `id: check_newlines`, `display: Newlines`) -2. Add `{"name": "Newlines"}` (i.e. matching `display`) to `checks.json` +2. Add `{"name": "Newlines", "command": "/run-actions-newlines"}` (i.e. matching `display`) to `checks.json` The order of entries in `checks.json` controls the row order in the results table. diff --git a/.github/helper_scripts/parse_commands.py b/.github/helper_scripts/parse_commands.py new file mode 100644 index 0000000..b300b75 --- /dev/null +++ b/.github/helper_scripts/parse_commands.py @@ -0,0 +1,8 @@ +import json, os +checks = json.load(open(os.path.join(os.path.dirname(__file__), '../checks/checks.json'))) +comment = os.environ.get('COMMENT_BODY', '') +run = [c['name'] for c in checks + if os.environ.get('GITHUB_EVENT_NAME') == 'push' + or '/run-actions-all' in comment + or c.get('command', '') in comment] +print(json.dumps(run)) diff --git a/.github/post_check_results.py b/.github/helper_scripts/post_check_results.py similarity index 88% rename from .github/post_check_results.py rename to .github/helper_scripts/post_check_results.py index 20c39c6..d920bb4 100644 --- a/.github/post_check_results.py +++ b/.github/helper_scripts/post_check_results.py @@ -4,7 +4,7 @@ import subprocess import sys -CHECKS_JSON = os.path.join(os.path.dirname(__file__), 'checks.json') +CHECKS_JSON = os.path.join(os.path.dirname(__file__), '../checks/checks.json') def Main(): repo = os.environ["GITHUB_REPOSITORY"] @@ -44,9 +44,11 @@ def CollectResults(): return rows, failed def PostResults(repo, run_id, rows, failed): + checks = json.load(open(CHECKS_JSON)) run_url = f"https://github.com/{repo}/actions/runs/{run_id}" table = "\n".join(["| Check | Result | Time |", "|-------|--------|------|", *rows]) - body = f"**Check Results** ([run details]({run_url}))\n\n{table}" + commands = " · ".join(f"`{c['command']}`" for c in checks if "command" in c) + " · `/run-actions-all`" + body = f"**Check Results** ([run details]({run_url}))\n\n{table}\n\nRun individually: {commands}" pr_num = os.environ["PR_NUMBER"] pr_sha = os.environ["PR_SHA"] subprocess.run([ diff --git a/.github/scripts/pr_close_comment.py b/.github/helper_scripts/pr_close_comment.py similarity index 100% rename from .github/scripts/pr_close_comment.py rename to .github/helper_scripts/pr_close_comment.py diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index c0c70e7..ee63ccd 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -43,40 +43,46 @@ jobs: with: python-version: "3.x" + - id: parse_commands + env: + COMMENT_BODY: ${{ github.event.comment.body }} + run: echo "run=$(python .github/helper_scripts/parse_commands.py)" >> $GITHUB_OUTPUT + shell: bash + - id: check_scons name: Check SCons DAG for missing dependencies - if: github.event_name == 'push' || contains(github.event.comment.body, '/run-actions-dag') || contains(github.event.comment.body, '/run-actions-all') + if: contains(fromJSON(steps.parse_commands.outputs.run), 'SCons DAG') continue-on-error: true uses: ./.github/actions/timed-check with: - script: .github/check_sconscripts.py + script: .github/checks/check_sconscripts.py display: SCons DAG - id: check_newlines name: Check missing newlines - if: github.event_name == 'push' || contains(github.event.comment.body, '/run-actions-newlines') || contains(github.event.comment.body, '/run-actions-all') + if: contains(fromJSON(steps.parse_commands.outputs.run), 'Newlines') continue-on-error: true uses: ./.github/actions/timed-check with: - script: .github/check_newlines.py + script: .github/checks/check_newlines.py display: Newlines - id: check_eps name: Check extraneous EPS data - if: github.event_name == 'push' || contains(github.event.comment.body, '/run-actions-eps') || contains(github.event.comment.body, '/run-actions-all') + if: contains(fromJSON(steps.parse_commands.outputs.run), 'EPS data') continue-on-error: true uses: ./.github/actions/timed-check with: - script: .github/check_eps_savefig.py + script: .github/checks/check_eps_savefig.py display: EPS data - id: check_scons_log name: Check build failure - if: github.event_name == 'push' || contains(github.event.comment.body, '/run-actions-log') || contains(github.event.comment.body, '/run-actions-all') + if: contains(fromJSON(steps.parse_commands.outputs.run), 'Build log') continue-on-error: true uses: ./.github/actions/timed-check with: - script: .github/check_sconscript_log.py + script: .github/checks/check_sconscript_log.py display: Build log - name: Post results @@ -86,5 +92,5 @@ jobs: PR_NUMBER: ${{ github.event.issue.number }} PR_SHA: ${{ env.PR_SHA }} run: | - python .github/post_check_results.py \ + python .github/helper_scripts/post_check_results.py \ ${{ contains(github.event.comment.body, '--post') && '--post' || '' }} diff --git a/.github/workflows/pr-close-comment.yml b/.github/workflows/pr-close-comment.yml index 3f787b1..9a0ddf7 100644 --- a/.github/workflows/pr-close-comment.yml +++ b/.github/workflows/pr-close-comment.yml @@ -33,4 +33,4 @@ jobs: PR_AUTHOR: ${{ github.event.pull_request.user.login }} BRANCH_NAME: ${{ github.event.pull_request.head.ref }} LAST_COMMIT_SHA: ${{ github.event.pull_request.head.sha }} - run: python .github/scripts/pr_close_comment.py + run: python .github/helper_scripts/pr_close_comment.py From 76cd84d5607b5d462f679971823238b8b2acfcc3 Mon Sep 17 00:00:00 2001 From: Chris Liao <33707455+liaochris@users.noreply.github.com> Date: Thu, 19 Mar 2026 19:14:53 -0400 Subject: [PATCH 10/19] new url #141 --- .github/github_action_checks.md | 4 ++-- README.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/github_action_checks.md b/.github/github_action_checks.md index 126391c..cc37aaf 100644 --- a/.github/github_action_checks.md +++ b/.github/github_action_checks.md @@ -2,7 +2,7 @@ ## Adding a new check -1. Add a step to `checks.yml` with a unique `id` and `display` (e.g. `id: check_newlines`, `display: Newlines`) -2. Add `{"name": "Newlines", "command": "/run-actions-newlines"}` (i.e. matching `display`) to `checks.json` +1. Add a step to `workflows/checks.yml` with a unique `id` and `display` (e.g. `id: check_newlines`, `display: Newlines`), and a `script` pointing to your script in `checks/` +2. Add `{"name": "Newlines", "command": "/run-actions-newlines"}` (i.e. matching `display`) to `checks/checks.json` The order of entries in `checks.json` controls the row order in the results table. diff --git a/README.md b/README.md index 055ebfa..8e5d5e1 100644 --- a/README.md +++ b/README.md @@ -97,7 +97,7 @@ The repository is prebuilt with some automated testing using [Github Actions](./ To run all tests, add `[run-actions-all]` to a commit message or type `/run-actions-all` in a comment. -To run a particular test, type `/run-actions-NAMEOFTEST` in a comment (e.g., `/run-actions-log` to run [this test](https://github.com/JMSLab/Template/blob/140-automating-pr-reviews-using-github-actions/.github/check_sconscript_log.py). +To run a particular test, type `/run-actions-NAMEOFTEST` in a comment (e.g., `/run-actions-log` to run [this test](https://github.com/JMSLab/Template/blob/master/.github/checks/check_sconscript_log.py). ### Citations and expectations for usage From 729443dca07912999bbf1cf58d10f6274eea6a6c Mon Sep 17 00:00:00 2001 From: Chris Liao <33707455+liaochris@users.noreply.github.com> Date: Thu, 19 Mar 2026 19:21:05 -0400 Subject: [PATCH 11/19] remove footer i dislike #141 --- .github/helper_scripts/post_check_results.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/helper_scripts/post_check_results.py b/.github/helper_scripts/post_check_results.py index d920bb4..6246e47 100644 --- a/.github/helper_scripts/post_check_results.py +++ b/.github/helper_scripts/post_check_results.py @@ -44,11 +44,9 @@ def CollectResults(): return rows, failed def PostResults(repo, run_id, rows, failed): - checks = json.load(open(CHECKS_JSON)) run_url = f"https://github.com/{repo}/actions/runs/{run_id}" table = "\n".join(["| Check | Result | Time |", "|-------|--------|------|", *rows]) - commands = " · ".join(f"`{c['command']}`" for c in checks if "command" in c) + " · `/run-actions-all`" - body = f"**Check Results** ([run details]({run_url}))\n\n{table}\n\nRun individually: {commands}" + body = f"**Check Results** ([run details]({run_url}))\n\n{table}" pr_num = os.environ["PR_NUMBER"] pr_sha = os.environ["PR_SHA"] subprocess.run([ From f39e9c0d5c7709d0ae95312610f5e6eb56d4d98c Mon Sep 17 00:00:00 2001 From: Chris Liao <33707455+liaochris@users.noreply.github.com> Date: Thu, 19 Mar 2026 19:23:11 -0400 Subject: [PATCH 12/19] improve code quality #141 --- .github/helper_scripts/parse_commands.py | 32 ++++++++++++++++++------ 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/.github/helper_scripts/parse_commands.py b/.github/helper_scripts/parse_commands.py index b300b75..650eb97 100644 --- a/.github/helper_scripts/parse_commands.py +++ b/.github/helper_scripts/parse_commands.py @@ -1,8 +1,24 @@ -import json, os -checks = json.load(open(os.path.join(os.path.dirname(__file__), '../checks/checks.json'))) -comment = os.environ.get('COMMENT_BODY', '') -run = [c['name'] for c in checks - if os.environ.get('GITHUB_EVENT_NAME') == 'push' - or '/run-actions-all' in comment - or c.get('command', '') in comment] -print(json.dumps(run)) +#!/usr/bin/env python3 +import json +import os +import sys +from pathlib import Path + +def LoadChecks(): + checks_path = Path(__file__).parent.parent / 'checks' / 'checks.json' + return json.loads(checks_path.read_text()) + +def ParseCommands(): + checks = LoadChecks() + comment = os.environ.get('COMMENT_BODY', '') + return [c['name'] for c in checks + if os.environ.get('GITHUB_EVENT_NAME') == 'push' + or '/run-actions-all' in comment + or c.get('command', '') in comment] + +def Main(): + print(json.dumps(ParseCommands())) + return 0 + +if __name__ == '__main__': + sys.exit(Main()) From 3a55df2ffe88afcd9e4e849d40f79130f13989ae Mon Sep 17 00:00:00 2001 From: Chris Liao <33707455+liaochris@users.noreply.github.com> Date: Thu, 19 Mar 2026 19:41:58 -0400 Subject: [PATCH 13/19] minor improvements [run-actions-all] #141 thank you claude senior swe --- .github/actions/timed-check/action.yml | 2 +- .github/checks/check_newlines.py | 4 ++-- .github/checks/check_sconscript_log.py | 2 +- .github/helper_scripts/post_check_results.py | 6 ++++-- .github/workflows/checks.yml | 1 - .github/workflows/pytest.yml | 4 ++-- 6 files changed, 10 insertions(+), 9 deletions(-) diff --git a/.github/actions/timed-check/action.yml b/.github/actions/timed-check/action.yml index 0d03791..8294b6a 100644 --- a/.github/actions/timed-check/action.yml +++ b/.github/actions/timed-check/action.yml @@ -11,7 +11,7 @@ runs: run: | mkdir -p "$RUNNER_TEMP/check_results" s=$(date +%s%N) - python ${{ inputs.script }} && outcome=success || outcome=failure + python "${{ inputs.script }}" && outcome=success || outcome=failure time=$(awk "BEGIN{printf \"%.3f\",($(date +%s%N)-$s)/1e9}") printf '{"outcome":"%s","time":"%s"}' "$outcome" "$time" \ > "$RUNNER_TEMP/check_results/${{ inputs.display }}.json" diff --git a/.github/checks/check_newlines.py b/.github/checks/check_newlines.py index 2f23cd1..59cd9b1 100644 --- a/.github/checks/check_newlines.py +++ b/.github/checks/check_newlines.py @@ -13,14 +13,14 @@ def GetTrackedFiles(): def IsBinary(p): try: return b"\0" in p.read_bytes()[:4096] - except: + except Exception: return True def NeedsNewline(p): try: d = p.read_bytes() return len(d)==0 or not d.endswith(b"\n") - except: + except Exception: return False def ProcessFiles(): diff --git a/.github/checks/check_sconscript_log.py b/.github/checks/check_sconscript_log.py index 96742db..67310ea 100644 --- a/.github/checks/check_sconscript_log.py +++ b/.github/checks/check_sconscript_log.py @@ -10,7 +10,7 @@ def Main(): try: if TARGET in p.read_text(errors="replace"): bad.append(p) - except: + except Exception: pass if not bad: diff --git a/.github/helper_scripts/post_check_results.py b/.github/helper_scripts/post_check_results.py index 6246e47..ec2d1da 100644 --- a/.github/helper_scripts/post_check_results.py +++ b/.github/helper_scripts/post_check_results.py @@ -22,14 +22,16 @@ def Main(): return 0 def CollectResults(): - checks = json.load(open(CHECKS_JSON)) + with open(CHECKS_JSON) as f: + checks = json.load(f) results_dir = os.path.join(os.environ["RUNNER_TEMP"], "check_results") rows, failed = [], [] for check in checks: name = check["name"] result_file = os.path.join(results_dir, f"{name}.json") if os.path.exists(result_file): - result = json.load(open(result_file)) + with open(result_file) as f: + result = json.load(f) outcome = result["outcome"] time = result["time"] print(f" {name}: {time}s") diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index ee63ccd..db190e2 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -15,7 +15,6 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 1 permissions: - contents: write statuses: write pull-requests: write diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index f7c5d65..c0d400c 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -20,9 +20,9 @@ jobs: os: [ ubuntu-latest, windows-latest, macos-latest ] python-version: [ "3.10", "3.11", "3.12", "3.13", "3.14" ] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install dependencies From 02b212745b4045d542a605d73c1ac9f5b23885df Mon Sep 17 00:00:00 2001 From: Chris Liao <33707455+liaochris@users.noreply.github.com> Date: Thu, 19 Mar 2026 19:44:20 -0400 Subject: [PATCH 14/19] more swe fixes #141 [run-actions-all] --- .github/checks/check_eps_savefig.py | 12 ++++---- .github/helper_scripts/pr_close_comment.py | 32 +++++++++++----------- 2 files changed, 23 insertions(+), 21 deletions(-) diff --git a/.github/checks/check_eps_savefig.py b/.github/checks/check_eps_savefig.py index 423214b..2064960 100644 --- a/.github/checks/check_eps_savefig.py +++ b/.github/checks/check_eps_savefig.py @@ -34,15 +34,17 @@ def CheckEpsSavefig(file_path, content): r'\.savefig\([^)]*format\s*=\s*[\'"]eps[\'"][^)]*\)' # matches format='eps' or format="eps" ] - has_remove_eps_info = 'remove_eps_info(' in content - + eps_lines = [] for line_num, line in enumerate(lines, 1): for pattern in eps_savefig_patterns: if re.search(pattern, line, re.IGNORECASE): - if not has_remove_eps_info: - problems.append(f"Line {line_num}: {line.strip()}") + eps_lines.append(f"Line {line_num}: {line.strip()}") break - + + remove_count = content.count('remove_eps_info(') + if len(eps_lines) != remove_count: + problems.extend(eps_lines) + return problems def CollectEpsProblems(root, excluded): diff --git a/.github/helper_scripts/pr_close_comment.py b/.github/helper_scripts/pr_close_comment.py index 36360ec..c73cc7c 100644 --- a/.github/helper_scripts/pr_close_comment.py +++ b/.github/helper_scripts/pr_close_comment.py @@ -3,44 +3,44 @@ from pathlib import Path from github import Github -GITHUB_TOKEN = os.environ["GITHUB_TOKEN"] -REPO_NAME = os.environ["REPO"] -PR_NUMBER = int(os.environ["PR_NUMBER"]) -PR_AUTHOR = os.environ["PR_AUTHOR"] -BRANCH_NAME = os.environ["BRANCH_NAME"] -LAST_COMMIT_SHA = os.environ["LAST_COMMIT_SHA"] - ISSUE_TEMPLATE = Path(".github/post_template_issue_thread_pr_close.md") PR_TEMPLATE = Path(".github/post_template_pr_thread_pr_close.md") def Main(): - repo = Github(GITHUB_TOKEN).get_repo(REPO_NAME) - pr = repo.get_pull(PR_NUMBER) + github_token = os.environ["GITHUB_TOKEN"] + repo_name = os.environ["REPO"] + pr_number = int(os.environ["PR_NUMBER"]) + pr_author = os.environ["PR_AUTHOR"] + branch_name = os.environ["BRANCH_NAME"] + last_commit_sha = os.environ["LAST_COMMIT_SHA"] + + repo = Github(github_token).get_repo(repo_name) + pr = repo.get_pull(pr_number) - issue_comment_url = PostIssueComment(repo) - PostPrComment(pr, issue_comment_url) + issue_comment_url = PostIssueComment(repo, branch_name, last_commit_sha) + PostPrComment(pr, pr_author, issue_comment_url) -def PostIssueComment(repo): - issue_match = re.match(r"^(\d+)", BRANCH_NAME) +def PostIssueComment(repo, branch_name, last_commit_sha): + issue_match = re.match(r"^(\d+)", branch_name) if not issue_match: return None issue_number = int(issue_match.group(1)) - issue_body = ISSUE_TEMPLATE.read_text() + f"\n\nLast commit in issue branch: {LAST_COMMIT_SHA}" + issue_body = ISSUE_TEMPLATE.read_text() + f"\n\nLast commit in issue branch: {last_commit_sha}" comment = repo.get_issue(issue_number).create_comment(issue_body) return comment.html_url -def PostPrComment(pr, issue_comment_url): +def PostPrComment(pr, pr_author, issue_comment_url): pr_body = PR_TEMPLATE.read_text() if issue_comment_url: pr_body = f"[Issue summary]({issue_comment_url})\n\n{pr_body}" - pr.as_issue().create_comment(f"@{PR_AUTHOR} {pr_body}") + pr.as_issue().create_comment(f"@{pr_author} {pr_body}") if __name__ == "__main__": From 523f6186e998e82707082c16df5c57946ce126dc Mon Sep 17 00:00:00 2001 From: Chris Liao <33707455+liaochris@users.noreply.github.com> Date: Thu, 19 Mar 2026 19:48:31 -0400 Subject: [PATCH 15/19] remove node js warnings #141 [run-actions-all] --- .github/workflows/checks.yml | 4 ++-- .github/workflows/pr-close-comment.yml | 4 ++-- .github/workflows/pytest.yml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index db190e2..bd246bc 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -20,7 +20,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v6 with: ref: ${{ github.event_name == 'issue_comment' && format('refs/pull/{0}/head', github.event.issue.number) || github.sha }} @@ -38,7 +38,7 @@ jobs: -f target_url="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" - name: Setup Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: "3.x" diff --git a/.github/workflows/pr-close-comment.yml b/.github/workflows/pr-close-comment.yml index 9a0ddf7..4a66d97 100644 --- a/.github/workflows/pr-close-comment.yml +++ b/.github/workflows/pr-close-comment.yml @@ -15,10 +15,10 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: '3.12' diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index c0d400c..b33128d 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -20,9 +20,9 @@ jobs: os: [ ubuntu-latest, windows-latest, macos-latest ] python-version: [ "3.10", "3.11", "3.12", "3.13", "3.14" ] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} - name: Install dependencies From 9f81d0ba5c1f89cf39b2f86b5915d0e5e32833d7 Mon Sep 17 00:00:00 2001 From: jmshapir Date: Fri, 20 Mar 2026 08:11:42 -0400 Subject: [PATCH 16/19] Rename for #141 --- .github/{github_action_checks.md => readme_for_checks.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/{github_action_checks.md => readme_for_checks.md} (100%) diff --git a/.github/github_action_checks.md b/.github/readme_for_checks.md similarity index 100% rename from .github/github_action_checks.md rename to .github/readme_for_checks.md From 7f63852d6f44d06eb299c68790999a0983df99ff Mon Sep 17 00:00:00 2001 From: jmshapir Date: Fri, 20 Mar 2026 08:14:49 -0400 Subject: [PATCH 17/19] Readme for #141 --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8e5d5e1..6b2694b 100644 --- a/README.md +++ b/README.md @@ -93,11 +93,11 @@ env.Stata(target, source) ### Automation -The repository is prebuilt with some automated testing using [Github Actions](./github). +The repository is prebuilt with some automated testing using [Github Actions](./.github). To run all tests, add `[run-actions-all]` to a commit message or type `/run-actions-all` in a comment. -To run a particular test, type `/run-actions-NAMEOFTEST` in a comment (e.g., `/run-actions-log` to run [this test](https://github.com/JMSLab/Template/blob/master/.github/checks/check_sconscript_log.py). +To run a particular test, type `/run-actions-NAMEOFTEST` in a comment (e.g., `/run-actions-log` to run [this test](./.github/checks/check_sconscript_log.py); see [commands](./.github/checks/checks.json) for others). ### Citations and expectations for usage From 847de72d0dcb6cb65f20a4fc6403f41db3d3657e Mon Sep 17 00:00:00 2001 From: jmshapir Date: Fri, 20 Mar 2026 08:19:38 -0400 Subject: [PATCH 18/19] Clarifies docs for #141 --- .github/post_template_pr_thread_pr_close.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/post_template_pr_thread_pr_close.md b/.github/post_template_pr_thread_pr_close.md index 2299dc1..a4488dd 100644 --- a/.github/post_template_pr_thread_pr_close.md +++ b/.github/post_template_pr_thread_pr_close.md @@ -1,3 +1,6 @@ Thanks for closing this pull. -Before leaving the pull, please be sure you have completed all the required steps in the [workflow](https://github.com/JMSLab/Template/blob/main/docs/workflow.md)! +Before leaving the pull, please be sure you have completed all the required steps in the [workflow](https://github.com/JMSLab/Template/blob/main/docs/workflow.md). + +This includes filling in the issue summary linked at the top of this comment. + From c44ac14dd6300eff8addd0cd1b53cc1f9bd13441 Mon Sep 17 00:00:00 2001 From: Chris Liao <33707455+liaochris@users.noreply.github.com> Date: Fri, 20 Mar 2026 15:34:50 -0400 Subject: [PATCH 19/19] [run-actions-all] #141 improve style, remove eps output check --- .github/checks/check_eps_savefig.py | 118 ++++--------------- .github/checks/check_sconscript_log.py | 2 +- .github/helper_scripts/post_check_results.py | 37 +++--- .github/helper_scripts/pr_close_comment.py | 4 +- .github/workflows/checks.yml | 3 +- .github/workflows/pr-close-comment.yml | 4 +- 6 files changed, 51 insertions(+), 117 deletions(-) diff --git a/.github/checks/check_eps_savefig.py b/.github/checks/check_eps_savefig.py index 2064960..7d89454 100644 --- a/.github/checks/check_eps_savefig.py +++ b/.github/checks/check_eps_savefig.py @@ -20,116 +20,50 @@ def ReadFile(path): except Exception: return None -def CheckEpsSavefig(file_path, content): - """ - Check if file contains .savefig(*eps*) without remove_eps_info( - Returns list of problematic lines with line numbers - """ - problems = [] - lines = content.split('\n') - - # Pattern to match .savefig with eps format (both single and double quotes) +def WalkFiles(root, excluded, extension): + for dir_path, dir_names, file_names in os.walk(root): + if IsExcludedPath(dir_path, excluded): + dir_names[:] = [] + continue + + dir_names[:] = [d for d in dir_names if not IsIgnoredDir(d)] + + for file_name in file_names: + if file_name.endswith(extension) and not IsHidden(file_name): + yield os.path.join(dir_path, file_name) + +def CheckEpsSavefig(content): eps_savefig_patterns = [ - r'\.savefig\([^)]*[\'"].*eps.*[\'"][^)]*\)', # matches .savefig(...'...eps...'...) - r'\.savefig\([^)]*format\s*=\s*[\'"]eps[\'"][^)]*\)' # matches format='eps' or format="eps" + r'\.savefig\([^)]*[\'"].*eps.*[\'"][^)]*\)', + r'\.savefig\([^)]*format\s*=\s*[\'"]eps[\'"][^)]*\)' ] - + eps_lines = [] - for line_num, line in enumerate(lines, 1): + for line_num, line in enumerate(content.split('\n'), 1): for pattern in eps_savefig_patterns: if re.search(pattern, line, re.IGNORECASE): eps_lines.append(f"Line {line_num}: {line.strip()}") break remove_count = content.count('remove_eps_info(') - if len(eps_lines) != remove_count: - problems.extend(eps_lines) - - return problems + return eps_lines if len(eps_lines) != remove_count else [] def CollectEpsProblems(root, excluded): - """Walk through Python files and check for EPS savefig issues""" problems = [] - - for dir_path, dir_names, file_names in os.walk(root): - if IsExcludedPath(dir_path, excluded): - dir_names[:] = [] + for file_path in WalkFiles(root, excluded, '.py'): + content = ReadFile(file_path) + if content is None: continue - - dir_names[:] = [d for d in dir_names if not IsIgnoredDir(d)] - - for file_name in file_names: - if not file_name.endswith('.py'): - continue - - if IsHidden(file_name): - continue - - file_path = os.path.join(dir_path, file_name) - content = ReadFile(file_path) - - if content is None: - continue - - file_problems = CheckEpsSavefig(file_path, content) - if file_problems: - problems.append({ - 'file': file_path, - 'issues': file_problems - }) - + file_problems = CheckEpsSavefig(content) + if file_problems: + problems.append({'file': file_path, 'issues': file_problems}) return problems -def CheckEpsCreationDate(content): - """ - Check if any line in an EPS file starts with %%CreationDate. - Returns True if it does. - """ - return any(line.startswith("%%CreationDate") for line in content.splitlines()) - -def CollectEpsCreationDateProblems(root, excluded): - """Walk through EPS files and report those that contain %%CreationDate""" - eps_files = [] - - for dir_path, dir_names, file_names in os.walk(root): - if IsExcludedPath(dir_path, excluded): - dir_names[:] = [] - continue - - dir_names[:] = [d for d in dir_names if not IsIgnoredDir(d)] - - for file_name in file_names: - if not file_name.endswith('.eps'): - continue - - if IsHidden(file_name): - continue - - eps_path = os.path.join(dir_path, file_name) - content = ReadFile(eps_path) - - if content is None: - continue - - if CheckEpsCreationDate(content): - eps_files.append(eps_path) - - return eps_files - -def main(): +def Main(): source_root = "source" - output_root = "output" excluded = ["source/lib", "source/raw", "source/scrape"] problems = CollectEpsProblems(source_root, excluded) - creationdate_eps_files = CollectEpsCreationDateProblems(output_root, []) - - if creationdate_eps_files: - print("EPS files containing %%CreationDate:") - for eps_file in creationdate_eps_files: - print(f" {eps_file}") - print("") - return 1 if problems: print("EPS savefig check failed!") @@ -146,4 +80,4 @@ def main(): return 0 if __name__ == "__main__": - sys.exit(main()) + sys.exit(Main()) diff --git a/.github/checks/check_sconscript_log.py b/.github/checks/check_sconscript_log.py index 67310ea..f3e12c9 100644 --- a/.github/checks/check_sconscript_log.py +++ b/.github/checks/check_sconscript_log.py @@ -6,7 +6,7 @@ def Main(): bad = [] - for p in Path(".").rglob("**/*.log"): + for p in Path(".").rglob("*.log"): try: if TARGET in p.read_text(errors="replace"): bad.append(p) diff --git a/.github/helper_scripts/post_check_results.py b/.github/helper_scripts/post_check_results.py index ec2d1da..cbf7b32 100644 --- a/.github/helper_scripts/post_check_results.py +++ b/.github/helper_scripts/post_check_results.py @@ -3,8 +3,9 @@ import os import subprocess import sys +from pathlib import Path -CHECKS_JSON = os.path.join(os.path.dirname(__file__), '../checks/checks.json') +CHECKS_JSON = Path(__file__).parent.parent / 'checks' / 'checks.json' def Main(): repo = os.environ["GITHUB_REPOSITORY"] @@ -22,27 +23,25 @@ def Main(): return 0 def CollectResults(): - with open(CHECKS_JSON) as f: - checks = json.load(f) - results_dir = os.path.join(os.environ["RUNNER_TEMP"], "check_results") + checks = json.loads(CHECKS_JSON.read_text()) + results_dir = Path(os.environ["RUNNER_TEMP"]) / "check_results" rows, failed = [], [] + STATUS = {"success": "✅", "failure": "❌"} for check in checks: - name = check["name"] - result_file = os.path.join(results_dir, f"{name}.json") - if os.path.exists(result_file): - with open(result_file) as f: - result = json.load(f) - outcome = result["outcome"] - time = result["time"] - print(f" {name}: {time}s") - if outcome == "success": - rows.append(f"| {name} | ✅ | {time}s |") - else: - failed.append(name) - rows.append(f"| {name} | ❌ | {time}s |") + check_name = check["name"] + result_file = results_dir / f"{check_name}.json" + if result_file.exists(): + result = json.loads(result_file.read_text()) + outcome = result["outcome"] + elapsed = result["time"] + print(f" {check_name}: {elapsed}s") + status_icon = STATUS.get(outcome, "❌") + if outcome != "success": + failed.append(check_name) + rows.append(f"| {check_name} | {status_icon} | {elapsed}s |") else: - print(f" {name}: skipped") - rows.append(f"| {name} | SKIP | |") + print(f" {check_name}: skipped") + rows.append(f"| {check_name} | SKIP | |") return rows, failed def PostResults(repo, run_id, rows, failed): diff --git a/.github/helper_scripts/pr_close_comment.py b/.github/helper_scripts/pr_close_comment.py index c73cc7c..7d93594 100644 --- a/.github/helper_scripts/pr_close_comment.py +++ b/.github/helper_scripts/pr_close_comment.py @@ -1,5 +1,6 @@ import os import re +import sys from pathlib import Path from github import Github @@ -20,6 +21,7 @@ def Main(): issue_comment_url = PostIssueComment(repo, branch_name, last_commit_sha) PostPrComment(pr, pr_author, issue_comment_url) + return 0 def PostIssueComment(repo, branch_name, last_commit_sha): @@ -44,4 +46,4 @@ def PostPrComment(pr, pr_author, issue_comment_url): if __name__ == "__main__": - Main() + sys.exit(Main()) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index bd246bc..44e3cc6 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -43,6 +43,7 @@ jobs: python-version: "3.x" - id: parse_commands + name: Parse requested checks env: COMMENT_BODY: ${{ github.event.comment.body }} run: echo "run=$(python .github/helper_scripts/parse_commands.py)" >> $GITHUB_OUTPUT @@ -76,7 +77,7 @@ jobs: display: EPS data - id: check_scons_log - name: Check build failure + name: Check SCons build log for errors if: contains(fromJSON(steps.parse_commands.outputs.run), 'Build log') continue-on-error: true uses: ./.github/actions/timed-check diff --git a/.github/workflows/pr-close-comment.yml b/.github/workflows/pr-close-comment.yml index 4a66d97..6f5db5e 100644 --- a/.github/workflows/pr-close-comment.yml +++ b/.github/workflows/pr-close-comment.yml @@ -1,5 +1,3 @@ -# Post a comment when a PR is closed (merged or unmerged) - name: PR Close Comment on: @@ -20,7 +18,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v6 with: - python-version: '3.12' + python-version: "3.x" - name: Install dependencies run: pip install PyGithub