Skip to content

Make Python optional & Modernise Github Actions #484

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
fosslinux wants to merge 2 commits into from
Closed

Make Python optional & Modernise Github Actions #484

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c9fa9a0
Make Python bootstrap optional
fosslinux Dec 17, 2024
86bd8de
Modernise the Github Actions workflow & test Python
fosslinux Dec 17, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
126 changes: 82 additions & 44 deletions .github/workflows/bwrap.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,21 +13,21 @@ on:

jobs:
pass1:
name: Run up to Linux build under bubblewrap
name: Run up to bash under bubblewrap
runs-on: ubuntu-latest
steps:
- name: Install bubblewrap
run: sudo apt install bubblewrap
- name: Checkout repo
uses: actions/checkout@v3
uses: actions/checkout@v4
with:
submodules: recursive
# There is a strange bug(?) in nongnu, when you clone a git repository
# against a commit != HEAD with depth=1, it errors out.
fetch-depth: 0
- name: Query cache for sources
id: cache
uses: actions/cache/restore@v3
uses: actions/cache/restore@v4
with:
path: |
distfiles
Expand All @@ -37,50 +37,42 @@ jobs:
run: ./download-distfiles.sh
- name: Cache sources
if: steps.cache.outputs.cache-hit != 'true'
uses: actions/cache/save@v3
uses: actions/cache/save@v4
with:
path: |
distfiles
key: cache-${{ hashFiles('steps/*/sources') }}
- name: Run bootstrap
run: ./rootfs.py --bwrap --external-sources --build-kernels --cores 2 --internal-ci pass1
- name: Archive created packages
if: failure() # archive failed builds progress
uses: actions/upload-artifact@v3
with:
name: packages
path: target/external/repo/**
- name: Tar pass1 image
run: tar -cf pass1_image.tar target
- name: Archive pass1_image
uses: actions/upload-artifact@v3
- name: Create early-preseed.tar
run: rm -rf target/external target/steps && tar -C target/ -cf early-preseed.tar .
- name: Archive early-preseed
uses: actions/upload-artifact@v4
with:
name: internal_pass1_image
path: pass1_image.tar
name: early-preseed
path: early-preseed.tar

pass2:
name: Run up to Python bootstrap under bubblewrap
name: Run up to just before GCC 10 under bubblewrap
needs: pass1
runs-on: ubuntu-latest
steps:
- name: Install bubblewrap
run: sudo apt install bubblewrap
- name: Checkout repo
uses: actions/checkout@v3
uses: actions/checkout@v4
with:
submodules: recursive
# There is a strange bug(?) in nongnu, when you clone a git repository
# against a commit != HEAD with depth=1, it errors out.
fetch-depth: 0
- name: Get pass1_image
uses: actions/download-artifact@v3
- name: Get early_preseed
uses: actions/download-artifact@v4
with:
name: internal_pass1_image
- name: Extract pass1_image
run: tar -xf pass1_image.tar
name: early-preseed
- name: Query cache for sources
id: cache
uses: actions/cache/restore@v3
uses: actions/cache/restore@v4
with:
path: |
distfiles
Expand All @@ -90,26 +82,18 @@ jobs:
run: ./download-distfiles.sh
- name: Cache sources
if: steps.cache.outputs.cache-hit != 'true'
uses: actions/cache/save@v3
uses: actions/cache/save@v4
with:
path: |
distfiles
key: cache-${{ hashFiles('steps/*/sources') }}
- name: Run bootstrap
run: ./rootfs.py --bwrap --external-sources --build-kernels --cores 2 --internal-ci pass2
run: ./rootfs.py --bwrap --external-sources --build-kernels --cores 2 --internal-ci pass2 --early-preseed early-preseed.tar
- name: Archive created packages
if: failure() # archive failed builds progress
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v4
with:
name: internal_packages_pass2
path: target/external/repo/**
- name: Tar pass2 image
run: tar -cf pass2_image.tar target
- name: Archive pass2_image
uses: actions/upload-artifact@v3
with:
name: internal_pass2_image
path: pass2_image.tar

pass3:
name: Run remaining builds under bubblewrap
Expand All @@ -119,21 +103,24 @@ jobs:
- name: Install bubblewrap
run: sudo apt install bubblewrap
- name: Checkout repo
uses: actions/checkout@v3
uses: actions/checkout@v4
with:
submodules: recursive
# There is a strange bug(?) in nongnu, when you clone a git repository
# against a commit != HEAD with depth=1, it errors out.
fetch-depth: 0
- name: Get pass2_image
uses: actions/download-artifact@v3
- name: Get early_preseed
uses: actions/download-artifact@v4
with:
name: early-preseed
- name: Get pass2 packages
uses: actions/download-artifact@v4
with:
name: internal_pass2_image
- name: Extract pass2_image
run: tar -xf pass2_image.tar
path: repo/
- name: Query cache for sources
id: cache
uses: actions/cache/restore@v3
uses: actions/cache/restore@v4
with:
path: |
distfiles
Expand All @@ -143,16 +130,67 @@ jobs:
run: ./download-distfiles.sh
- name: Cache sources
if: steps.cache.outputs.cache-hit != 'true'
uses: actions/cache/save@v3
uses: actions/cache/save@v4
with:
path: |
distfiles
key: cache-${{ hashFiles('steps/*/sources') }}
- name: Run bootstrap
run: ./rootfs.py --bwrap --external-sources --build-kernels --cores 2 --internal-ci pass3
run: ./rootfs.py --bwrap --external-sources --build-kernels --cores 2 --internal-ci pass3 --early-preseed early_preseed.tar --repo repo
- name: Archive created packages
if: always() # archive both failed and successful builds
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v4
with:
name: packages
path: target/external/repo/**

python:
name: Run the Python bootstrap
needs: pass3
runs-on: ubuntu-latest
steps:
- name: Install bubblewrap
run: sudo apt install bubblewrap
- name: Checkout repo
uses: actions/checkout@v4
with:
submodules: recursive
# There is a strange bug(?) in nongnu, when you clone a git repository
# against a commit != HEAD with depth=1, it errors out.
fetch-depth: 0
- name: Get early_preseed
uses: actions/download-artifact@v4
with:
name: early-preseed
- name: Get full set of packages
uses: actions/download-artifact@v4
with:
name: packages
path: repo/
- name: Query cache for sources
id: cache
uses: actions/cache/restore@v4
with:
path: |
distfiles
key: cache-${{ hashFiles('steps/*/sources') }}
- name: Get sources
if: steps.cache.outputs.cache-hit != 'true'
run: ./download-distfiles.sh
- name: Cache sources
if: steps.cache.outputs.cache-hit != 'true'
uses: actions/cache/save@v4
with:
path: |
distfiles
key: cache-${{ hashFiles('steps/*/sources') }}
- name: Run bootstrap
run: ./rootfs.py --bwrap --external-sources --build-kernels --cores 2 --early-preseed early_preseed.tar --repo repo --extras python
- name: Remove non-new packages
run: comm -12 <(ls -1 repo) <(ls -1 target/external/repo) | xargs -I{} rm target/external/repo/{}
- name: Archive Python packages
if: always() # archive both failed and successful builds
uses: actions/upload-artifact@v4
with:
name: packages
path: target/external/repo/**
19 changes: 13 additions & 6 deletions rootfs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,8 @@ def create_configuration_file(args):
config.write("DISK=sda1\n")
config.write("KERNEL_BOOTSTRAP=False\n")
config.write(f"BUILD_KERNELS={args.update_checksums or args.build_kernels}\n")
for extra in args.extras.split(","):
config.write(f"EXTRA_{extra.upper()}=True\n")
config.write(f"CONFIGURATOR={args.configurator}\n")

# pylint: disable=too-many-statements,too-many-branches
Expand Down Expand Up @@ -101,6 +103,8 @@ def main():
parser.add_argument("--internal-ci", help="INTERNAL for github CI")
parser.add_argument("-s", "--swap", help="Swap space to allocate in Linux",
default=0)
parser.add_argument("-x", "--extras", help="Comma-separated list of extra bootstraps to run (python)",
Copy link
Collaborator

@stikonas stikonas Dec 17, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think choices = ['python'] with nargs="+" would work better. You would not need to do the checks bellow then.

default="")

# QEMU arguments
parser.add_argument("-q", "--qemu", help="Use QEMU",
Expand Down Expand Up @@ -165,6 +169,12 @@ def check_types():
else:
args.swap = 0

# Check extra bootstraps
valid = ["python"]
for extra in args.extras.split(","):
if extra != "" and extra not in valid:
raise ValueError(f"{extra} is not a known bootstrap")

# Set constant umask
os.umask(0o022)

Expand Down Expand Up @@ -210,13 +220,10 @@ def bootstrap(args, generator, target, size):

elif args.bwrap:
init = '/init'
if not args.internal_ci or args.internal_ci == "pass1":
generator.prepare(target, using_kernel=False)
generator.prepare(target, using_kernel=False)

arch = stage0_arch_map.get(args.arch, args.arch)
init = os.path.join(os.sep, 'bootstrap-seeds', 'POSIX', arch, 'kaem-optional-seed')
else:
generator.reuse(target)
arch = stage0_arch_map.get(args.arch, args.arch)
init = os.path.join(os.sep, 'bootstrap-seeds', 'POSIX', arch, 'kaem-optional-seed')

run('env', '-i', 'bwrap', '--unshare-user',
'--uid', '0',
Expand Down
7 changes: 7 additions & 0 deletions steps/configurator
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,13 @@ this correctly, otherwise you risk overwriting an existing disk on your
system. LIVE-BOOTSTRAP TAKES NO LIABILITY FOR ANY DATA LOSS RESULTING FROM
ITS USE.

m /extras _ _ _ Extra bootstraps
Various extra language or package bootstraps that are not core to
live-bootstrap, but are an optional part of it.

o /extras/python EXTRA_PYTHON bool False Python bootstrap
Bootstraps Python.

m /sysinfo _ _ _ System information
Details about your specific system and the environment live-bootstrap is
running in.
Expand Down
26 changes: 13 additions & 13 deletions steps/manifest
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ build: bzip2-1.0.8
build: coreutils-5.0
build: byacc-20240109
build: bash-2.05b
jump: break ( INTERNAL_CI == pass1 )
improve: setup_repo
improve: update_env
improve: merged_usr
Expand Down Expand Up @@ -127,7 +128,6 @@ build: kexec-tools-2.0.22 ( BUILD_LINUX == True )
improve: clean_sources
improve: clean_artifacts
build: linux-4.14.341-openela ( BUILD_LINUX == True )
jump: break ( INTERNAL_CI == pass1 )
improve: populate_device_nodes
jump: linux ( CHROOT == False )
jump: move_disk ( KERNEL_BOOTSTRAP == True )
Expand Down Expand Up @@ -181,18 +181,6 @@ build: grep-3.7
build: sed-4.8
build: autogen-5.18.16
build: musl-1.2.4
build: python-2.0.1
build: python-2.0.1
build: python-2.3.7
build: python-2.3.7
build: python-2.5.6
build: python-3.1.5
build: python-3.1.5
build: python-3.3.7
build: python-3.4.10
build: python-3.8.16
build: python-3.11.1
uninstall: python-2.0.1 python-2.3.7 python-3.1.5 python-3.3.7 python-3.4.10 python-3.8.16
jump: break ( INTERNAL_CI == pass1 ) # scripts are generated in pass1
build: gcc-10.4.0
build: binutils-2.41
Expand All @@ -207,6 +195,18 @@ build: gzip-1.13
build: diffutils-3.10
build: gawk-5.3.0
build: m4-1.4.19
build: python-2.0.1 ( EXTRA_PYTHON == True )
build: python-2.0.1 ( EXTRA_PYTHON == True )
build: python-2.3.7 ( EXTRA_PYTHON == True )
build: python-2.3.7 ( EXTRA_PYTHON == True )
build: python-2.5.6 ( EXTRA_PYTHON == True )
build: python-3.1.5 ( EXTRA_PYTHON == True )
build: python-3.1.5 ( EXTRA_PYTHON == True )
build: python-3.3.7 ( EXTRA_PYTHON == True )
build: python-3.4.10 ( EXTRA_PYTHON == True )
build: python-3.8.16 ( EXTRA_PYTHON == True )
build: python-3.11.1 ( EXTRA_PYTHON == True )
uninstall: python-2.0.1 python-2.3.7 python-3.1.5 python-3.3.7 python-3.4.10 python-3.8.16 ( EXTRA_PYTHON == True )
improve: cleanup_filesystem
improve: null_time ( FORCE_TIMESTAMPS == True )
improve: update_checksums ( UPDATE_CHECKSUMS == True )
Expand Down
Loading