Skip to content

Commit db131b3

Browse files
jmfernandezjmfernandez
committed
Several CI workflows additions and improvements.
..."borrowed" from other projects I'm working on.
1 parent af0611f commit db131b3

File tree

4 files changed

+220
-50
lines changed

4 files changed

+220
-50
lines changed

‎.github/dependabot.yml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
version: 2
2+
updates:
3+
# Maintain dependencies for GitHub Actions
4+
- package-ecosystem: "github-actions"
5+
directory: "/"
6+
schedule:
7+
# Check for updates to GitHub Actions every week
8+
interval: "weekly"
9+
groups:
10+
github-actions:
11+
patterns:
12+
- "*"
13+

‎.github/workflows/build_n_deploy.yml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,9 @@ jobs:
99
runs-on: ubuntu-latest
1010
steps:
1111
- name: Checkout repo
12-
uses: actions/checkout@v4
12+
uses: actions/checkout@v5
1313
- name: Setup Python environment
14-
uses: actions/setup-python@v5
14+
uses: actions/setup-python@v6
1515
with:
1616
python-version: '3.x'
1717
- name: Install development dependencies
@@ -21,7 +21,7 @@ jobs:
2121
- name: Build packages to be uploaded
2222
run: python -m build
2323
- name: pypi-publish
24-
uses: pypa/gh-action-pypi-publish@v1.12.4
24+
uses: pypa/gh-action-pypi-publish@v1.13.0
2525
with:
2626
user: __token__
2727
password: ${{ secrets.PYPI_API_TOKEN }}

‎.github/workflows/pip-audit.yml‎

Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
name: pip-audit
2+
3+
on:
4+
workflow_dispatch:
5+
schedule:
6+
- cron: "0 12 * * 1"
7+
8+
jobs:
9+
pip-audit:
10+
runs-on: ubuntu-latest
11+
strategy:
12+
matrix:
13+
python-version: [ "3.10", "3.11", "3.12", "3.13" ]
14+
name: pip-audit python ${{ matrix.python-version }}
15+
steps:
16+
- uses: actions/checkout@v5
17+
- uses: actions/setup-python@v6
18+
with:
19+
python-version: ${{ matrix.python-version }}
20+
cache: 'pip'
21+
cache-dependency-path: |
22+
requirements.txt
23+
architecture: x64
24+
- name: 'Install requirements (standard or constraints ${{ matrix.python-version }})'
25+
run: |
26+
python -mvenv /tmp/PIPAUDIT
27+
source /tmp/PIPAUDIT/bin/activate
28+
pip install --upgrade pip wheel
29+
pip install pip-audit
30+
# - name: 'Freeze Python ${{ matrix.python-version }} constraints'
31+
# run: |
32+
# pip freeze > constraints-${{ matrix.python-version }}.txt
33+
- id: gen-cve-output
34+
run: |
35+
source /tmp/PIPAUDIT/bin/activate
36+
set +e
37+
pip-audit --desc=on --progress-spinner=off -r constraints-${{ matrix.python-version }}.txt --no-deps --disable-pip -f markdown -o /tmp/report-before.md
38+
refreeze=$?
39+
set -e
40+
41+
if [ "$refreeze" != 0 ] ; then
42+
deactivate
43+
python -mvenv /tmp/PIPFREEZE
44+
source /tmp/PIPFREEZE/bin/activate
45+
pip install --upgrade pip wheel
46+
pip install -r requirements.txt
47+
pip freeze > constraints-${{ matrix.python-version }}.txt
48+
49+
# Re-audit the populated environment
50+
deactivate
51+
source /tmp/PIPAUDIT/bin/activate
52+
set +e
53+
pip-audit --desc=on --progress-spinner=off -r constraints-${{ matrix.python-version }}.txt --no-deps --disable-pip -f markdown -o /tmp/report-after.md
54+
auditres=$?
55+
set -e
56+
57+
if [ "$auditres" = 0 ] ; then
58+
echo "# Fixed dependency issues for Python ${{ matrix.python-version }}" > audit-report-${{ matrix.python-version }}.md
59+
cat /tmp/report-before.md >> audit-report-${{ matrix.python-version }}.md
60+
else
61+
# Time to emit the report
62+
echo "# Dependency issues not solved for Python ${{ matrix.python-version }}" > audit-report-${{ matrix.python-version }}.md
63+
cat /tmp/report-after.md >> audit-report-${{ matrix.python-version }}.md
64+
fi
65+
cat audit-report-${{ matrix.python-version }}.md >> "$GITHUB_STEP_SUMMARY"
66+
fi
67+
- uses: actions/upload-artifact@v4
68+
with:
69+
name: audit-${{ matrix.python-version }}
70+
retention-days: 2
71+
path: |
72+
constraints-${{ matrix.python-version }}.txt
73+
audit-report-${{ matrix.python-version }}.md
74+
75+
pull_request_changes:
76+
# Do this only when it is not a pull request validation
77+
if: github.event_name != 'pull_request'
78+
runs-on: ubuntu-latest
79+
name: Pull request with the newly generated contents
80+
needs:
81+
- pip-audit
82+
steps:
83+
- name: Get analysis timestamp
84+
id: timestamp
85+
run: echo "timestamp=$(date -Is)" >> "$GITHUB_OUTPUT"
86+
- uses: actions/checkout@v5
87+
- uses: actions/download-artifact@v5
88+
id: download
89+
with:
90+
pattern: audit-*
91+
merge-multiple: true
92+
path: changes-dir
93+
- name: Move artifacts to their right place
94+
id: move
95+
run: |
96+
skip=true
97+
if [ -d "${{steps.download.outputs.download-path}}" ] ; then
98+
for con in "${{steps.download.outputs.download-path}}"/constraints-*.txt ; do
99+
case "$con" in
100+
*/constraints-\*.txt)
101+
break
102+
;;
103+
*)
104+
cp -p "$con" .
105+
skip=false
106+
;;
107+
esac
108+
done
109+
for aud in "${{steps.download.outputs.download-path}}"/audit-report-*.md ; do
110+
case "$aud" in
111+
*/audit-report-\*.md)
112+
touch pull-body.md
113+
break
114+
;;
115+
*)
116+
cat "$aud" >> pull-body.md
117+
;;
118+
esac
119+
done
120+
fi
121+
ls -l
122+
echo "skip=$skip" >> "$GITHUB_OUTPUT"
123+
- name: Create Pull Request
124+
id: cpr
125+
uses: peter-evans/create-pull-request@v7
126+
if: steps.move.outputs.skip == 'false'
127+
with:
128+
title: Updated constraints due security reasons (triggered on ${{ steps.timestamp.outputs.timestamp }} by ${{ github.sha }})
129+
branch: create-pull-request/patch-audit-constraints
130+
add-paths: constraints-*.txt
131+
delete-branch: true
132+
commit-message: "[create-pull-request] Automatically updated constraints due security reasons"
133+
body-path: pull-body.md
134+
- name: Check outputs
135+
if: ${{ steps.cpr.outputs.pull-request-number }}
136+
run: |
137+
echo "Pull Request Number - ${{ steps.cpr.outputs.pull-request-number }}" >> "$GITHUB_STEP_SUMMARY"
138+
echo "Pull Request URL - ${{ steps.cpr.outputs.pull-request-url }}" >> "$GITHUB_STEP_SUMMARY"

‎.github/workflows/pre-commit.yml‎

Lines changed: 66 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -15,10 +15,10 @@ jobs:
1515
python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12", "3.13" ]
1616
name: Pre-commit python ${{ matrix.python-version }}
1717
steps:
18-
- uses: actions/checkout@v4
18+
- uses: actions/checkout@v5
1919
with:
2020
fetch-depth: 100
21-
- uses: actions/setup-python@v5
21+
- uses: actions/setup-python@v6
2222
id: cachepy
2323
with:
2424
python-version: ${{ matrix.python-version }}
@@ -28,32 +28,35 @@ jobs:
2828
mypy-requirements.txt
2929
dev-requirements.txt
3030
architecture: x64
31-
- name: Changed requirements.txt
32-
id: changed-requirements-txt
33-
uses: tj-actions/changed-files@v44
34-
with:
35-
files: requirements.txt
36-
37-
- name: 'Install requirements (standard)'
38-
run: |
39-
pip install --upgrade pip wheel
40-
pip install -r requirements.txt
41-
- name: 'Install requirements (standard with freeze) or constraints (Python ${{ matrix.python-version }})'
31+
- name: 'Install requirements (standard or constraints ${{ matrix.python-version }})'
4232
run: |
4333
pip install --upgrade pip wheel
44-
if [ ${{ steps.changed-requirements-txt.outputs.any_changed }} != 'true' ] && [ -f constraints-${{ matrix.python-version }}.txt ] ; then
45-
pip install -r requirements.txt -c constraints-${{ matrix.python-version }}.txt
34+
constraints_file="constraints-${{ matrix.python-version }}.txt"
35+
regen_constraints=
36+
if [ -f "$constraints_file" ] ; then
37+
at="$(git --no-pager log -p -1 "--format=tformat:%at" --no-patch -- "$constraints_file")"
38+
dat="$(git --no-pager log -p -1 "--format=tformat:%at" --no-patch -- "requirements.txt")"
39+
if [ "$at" -lt "$dat" ] ; then
40+
regen_constraints=true
41+
fi
4642
else
43+
regen_constraints=true
44+
fi
45+
if [ -n "$regen_constraints" ] ; then
4746
pip install -r requirements.txt
48-
pip freeze > constraints-${{ matrix.python-version }}.txt
47+
pip freeze > "$constraints_file"
48+
grep -vF git+ "$constraints_file" > "$constraints_file"-relaxed
49+
else
50+
grep -vF git+ "$constraints_file" > "$constraints_file"-relaxed
51+
pip install -r requirements.txt -c "$constraints_file"-relaxed
4952
fi
50-
- name: 'Install dev requirements'
53+
- name: 'Install development requirements'
5154
run: |
52-
pip install -r dev-requirements.txt -r mypy-requirements.txt -c constraints-${{ matrix.python-version }}.txt
55+
pip install -r dev-requirements.txt -r mypy-requirements.txt -c constraints-${{ matrix.python-version }}.txt-relaxed
5356
- name: MyPy cache
5457
uses: actions/cache@v4
5558
with:
56-
path: .mypy_cache/${{ matrix.python-version }}
59+
path: '.mypy_cache/[0-9]*'
5760
key: mypy-${{ matrix.python-version }}
5861
- name: 'pre-commit'
5962
uses: pre-commit/action@v3.0.1
@@ -65,18 +68,25 @@ jobs:
6568
# if: ${{ matrix.python-version == '3.6' }}
6669
# with:
6770
# extra_args: --all -c .pre-commit-config-gh-${{ matrix.python-version }}.yaml
71+
- name: Get transitive dependencies licences
72+
id: license_check_print_report
73+
# uses: pilosus/action-pip-license-checker@v1.0.0
74+
# continue-on-error: true
75+
uses: pilosus/action-pip-license-checker@v3.1.0
76+
with:
77+
requirements: constraints-${{ matrix.python-version }}.txt
6878
- name: Check transitive dependencies licences
6979
id: license_check_report
7080
# uses: pilosus/action-pip-license-checker@v1.0.0
7181
# continue-on-error: true
72-
uses: pilosus/action-pip-license-checker@v2.0.0
82+
uses: pilosus/action-pip-license-checker@v3.1.0
7383
with:
7484
requirements: constraints-${{ matrix.python-version }}.txt
7585
fail: 'StrongCopyleft'
7686
exclude: 'pylint.*'
7787
- name: Print licences report
7888
if: ${{ always() }}
79-
run: echo "${{ steps.license_check_report.outputs.report }}"
89+
run: echo "${{ steps.license_check_print_report.outputs.report }}"
8090
- uses: actions/upload-artifact@v4
8191
with:
8292
name: constraints-artifacts-${{ matrix.python-version }}
@@ -90,10 +100,10 @@ jobs:
90100
python-version: [ "3.7" ]
91101
name: Pre-commit python ${{ matrix.python-version }}
92102
steps:
93-
- uses: actions/checkout@v4
103+
- uses: actions/checkout@v5
94104
with:
95105
fetch-depth: 100
96-
- uses: actions/setup-python@v5
106+
- uses: actions/setup-python@v6
97107
id: cachepy
98108
with:
99109
python-version: ${{ matrix.python-version }}
@@ -103,32 +113,34 @@ jobs:
103113
mypy-requirements.txt
104114
dev-requirements.txt
105115
architecture: x64
106-
- name: Changed requirements.txt
107-
id: changed-requirements-txt
108-
uses: tj-actions/changed-files@v44
109-
with:
110-
files: requirements.txt
111-
112-
- name: 'Install requirements (standard)'
113-
run: |
114-
pip install --upgrade pip wheel
115-
pip install -r requirements.txt
116-
- name: 'Install requirements (standard with freeze) or constraints (Python ${{ matrix.python-version }})'
116+
- name: 'Install requirements (standard or constraints ${{ matrix.python-version }})'
117117
run: |
118118
pip install --upgrade pip wheel
119-
if [ ${{ steps.changed-requirements-txt.outputs.any_changed }} != 'true' ] && [ -f constraints-${{ matrix.python-version }}.txt ] ; then
120-
pip install -r requirements.txt -c constraints-${{ matrix.python-version }}.txt
119+
constraints_file="constraints-${{ matrix.python-version }}.txt"
120+
regen_constraints=
121+
if [ -f "$constraints_file" ] ; then
122+
at="$(git --no-pager log -p -1 "--format=tformat:%at" --no-patch -- "$constraints_file")"
123+
dat="$(git --no-pager log -p -1 "--format=tformat:%at" --no-patch -- "requirements.txt")"
124+
if [ "$at" -lt "$dat" ] ; then
125+
regen_constraints=true
126+
fi
121127
else
128+
regen_constraints=true
129+
fi
130+
if [ -n "$regen_constraints" ] ; then
122131
pip install -r requirements.txt
123-
pip freeze > constraints-${{ matrix.python-version }}.txt
132+
pip freeze > "$constraints_file"
133+
grep -vF git+ "$constraints_file" > "$constraints_file"-relaxed
134+
else
135+
grep -vF git+ "$constraints_file" > "$constraints_file"-relaxed
136+
pip install -r requirements.txt -c "$constraints_file"-relaxed
124137
fi
125-
- name: 'Install dev requirements'
126-
run: |
127-
pip install -r dev-requirements.txt -r mypy-requirements.txt -c constraints-${{ matrix.python-version }}.txt
138+
- run: |
139+
pip install -r dev-requirements.txt -r mypy-requirements.txt -c constraints-${{ matrix.python-version }}.txt-relaxed
128140
- name: MyPy cache
129141
uses: actions/cache@v4
130142
with:
131-
path: .mypy_cache/${{ matrix.python-version }}
143+
path: '.mypy_cache/[0-9]*'
132144
key: mypy-${{ matrix.python-version }}
133145
- name: 'pre-commit'
134146
uses: pre-commit/action@v3.0.1
@@ -140,18 +152,25 @@ jobs:
140152
# if: ${{ matrix.python-version == '3.6' }}
141153
# with:
142154
# extra_args: --all -c .pre-commit-config-gh-${{ matrix.python-version }}.yaml
155+
- name: Get transitive dependencies licences
156+
id: license_check_print_report
157+
# uses: pilosus/action-pip-license-checker@v1.0.0
158+
# continue-on-error: true
159+
uses: pilosus/action-pip-license-checker@v3.1.0
160+
with:
161+
requirements: constraints-${{ matrix.python-version }}.txt
143162
- name: Check transitive dependencies licences
144163
id: license_check_report
145164
# uses: pilosus/action-pip-license-checker@v1.0.0
146165
# continue-on-error: true
147-
uses: pilosus/action-pip-license-checker@v2.0.0
166+
uses: pilosus/action-pip-license-checker@v3.1.0
148167
with:
149168
requirements: constraints-${{ matrix.python-version }}.txt
150169
fail: 'StrongCopyleft'
151-
exclude: 'pylint.*'
170+
exclude: '(?i)^(pylint|dulwich).*'
152171
- name: Print licences report
153172
if: ${{ always() }}
154-
run: echo "${{ steps.license_check_report.outputs.report }}"
173+
run: echo "${{ steps.license_check_print_report.outputs.report }}"
155174
- uses: actions/upload-artifact@v4
156175
with:
157176
name: constraints-artifacts-${{ matrix.python-version }}
@@ -169,8 +188,8 @@ jobs:
169188
- name: Get analysis timestamp
170189
id: timestamp
171190
run: echo "timestamp=$(date -Is)" >> "$GITHUB_OUTPUT"
172-
- uses: actions/checkout@v4
173-
- uses: actions/download-artifact@v4
191+
- uses: actions/checkout@v5
192+
- uses: actions/download-artifact@v5
174193
id: download
175194
with:
176195
pattern: constraints-artifacts-*
@@ -199,7 +218,7 @@ jobs:
199218
uses: peter-evans/create-pull-request@v7
200219
if: steps.move.outputs.skip == 'false'
201220
with:
202-
title: Updated constraints (triggered by ${{ github.sha }})
221+
title: Updated constraints (triggered on ${{ steps.timestamp.outputs.timestamp }} by ${{ github.sha }})
203222
add-paths: constraints-*.txt
204223
delete-branch: true
205224
commit-message: "[create-pull-request] Automatically commit updated contents (constraints)"

0 commit comments

Comments
 (0)