diff --git a/.github/workflows/bwrap.yml b/.github/workflows/bwrap.yml index 66cea846..414dc0cf 100644 --- a/.github/workflows/bwrap.yml +++ b/.github/workflows/bwrap.yml @@ -13,13 +13,13 @@ on: jobs: pass1: - name: Run up to Linux build under bubblewrap + name: Run up to bash under bubblewrap runs-on: ubuntu-latest steps: - name: Install bubblewrap run: sudo apt install bubblewrap - name: Checkout repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: submodules: recursive # There is a strange bug(?) in nongnu, when you clone a git repository @@ -27,7 +27,7 @@ jobs: fetch-depth: 0 - name: Query cache for sources id: cache - uses: actions/cache/restore@v3 + uses: actions/cache/restore@v4 with: path: | distfiles @@ -37,50 +37,42 @@ jobs: run: ./download-distfiles.sh - name: Cache sources if: steps.cache.outputs.cache-hit != 'true' - uses: actions/cache/save@v3 + uses: actions/cache/save@v4 with: path: | distfiles key: cache-${{ hashFiles('steps/*/sources') }} - name: Run bootstrap run: ./rootfs.py --bwrap --external-sources --build-kernels --cores 2 --internal-ci pass1 - - name: Archive created packages - if: failure() # archive failed builds progress - uses: actions/upload-artifact@v3 - with: - name: packages - path: target/external/repo/** - - name: Tar pass1 image - run: tar -cf pass1_image.tar target - - name: Archive pass1_image - uses: actions/upload-artifact@v3 + - name: Create early-preseed.tar + run: rm -rf target/external target/steps && tar -C target/ -cf early-preseed.tar . + - name: Archive early-preseed + uses: actions/upload-artifact@v4 with: - name: internal_pass1_image - path: pass1_image.tar + name: early-preseed + path: early-preseed.tar pass2: - name: Run up to Python bootstrap under bubblewrap + name: Run up to just before GCC 10 under bubblewrap needs: pass1 runs-on: ubuntu-latest steps: - name: Install bubblewrap run: sudo apt install bubblewrap - name: Checkout repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: submodules: recursive # There is a strange bug(?) in nongnu, when you clone a git repository # against a commit != HEAD with depth=1, it errors out. fetch-depth: 0 - - name: Get pass1_image - uses: actions/download-artifact@v3 + - name: Get early_preseed + uses: actions/download-artifact@v4 with: - name: internal_pass1_image - - name: Extract pass1_image - run: tar -xf pass1_image.tar + name: early-preseed - name: Query cache for sources id: cache - uses: actions/cache/restore@v3 + uses: actions/cache/restore@v4 with: path: | distfiles @@ -90,26 +82,18 @@ jobs: run: ./download-distfiles.sh - name: Cache sources if: steps.cache.outputs.cache-hit != 'true' - uses: actions/cache/save@v3 + uses: actions/cache/save@v4 with: path: | distfiles key: cache-${{ hashFiles('steps/*/sources') }} - name: Run bootstrap - run: ./rootfs.py --bwrap --external-sources --build-kernels --cores 2 --internal-ci pass2 + run: ./rootfs.py --bwrap --external-sources --build-kernels --cores 2 --internal-ci pass2 --early-preseed early-preseed.tar - name: Archive created packages - if: failure() # archive failed builds progress - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: internal_packages_pass2 path: target/external/repo/** - - name: Tar pass2 image - run: tar -cf pass2_image.tar target - - name: Archive pass2_image - uses: actions/upload-artifact@v3 - with: - name: internal_pass2_image - path: pass2_image.tar pass3: name: Run remaining builds under bubblewrap @@ -119,21 +103,24 @@ jobs: - name: Install bubblewrap run: sudo apt install bubblewrap - name: Checkout repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: submodules: recursive # There is a strange bug(?) in nongnu, when you clone a git repository # against a commit != HEAD with depth=1, it errors out. fetch-depth: 0 - - name: Get pass2_image - uses: actions/download-artifact@v3 + - name: Get early_preseed + uses: actions/download-artifact@v4 + with: + name: early-preseed + - name: Get pass2 packages + uses: actions/download-artifact@v4 with: name: internal_pass2_image - - name: Extract pass2_image - run: tar -xf pass2_image.tar + path: repo/ - name: Query cache for sources id: cache - uses: actions/cache/restore@v3 + uses: actions/cache/restore@v4 with: path: | distfiles @@ -143,16 +130,67 @@ jobs: run: ./download-distfiles.sh - name: Cache sources if: steps.cache.outputs.cache-hit != 'true' - uses: actions/cache/save@v3 + uses: actions/cache/save@v4 with: path: | distfiles key: cache-${{ hashFiles('steps/*/sources') }} - name: Run bootstrap - run: ./rootfs.py --bwrap --external-sources --build-kernels --cores 2 --internal-ci pass3 + run: ./rootfs.py --bwrap --external-sources --build-kernels --cores 2 --internal-ci pass3 --early-preseed early_preseed.tar --repo repo - name: Archive created packages if: always() # archive both failed and successful builds - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 + with: + name: packages + path: target/external/repo/** + + python: + name: Run the Python bootstrap + needs: pass3 + runs-on: ubuntu-latest + steps: + - name: Install bubblewrap + run: sudo apt install bubblewrap + - name: Checkout repo + uses: actions/checkout@v4 + with: + submodules: recursive + # There is a strange bug(?) in nongnu, when you clone a git repository + # against a commit != HEAD with depth=1, it errors out. + fetch-depth: 0 + - name: Get early_preseed + uses: actions/download-artifact@v4 + with: + name: early-preseed + - name: Get full set of packages + uses: actions/download-artifact@v4 + with: + name: packages + path: repo/ + - name: Query cache for sources + id: cache + uses: actions/cache/restore@v4 + with: + path: | + distfiles + key: cache-${{ hashFiles('steps/*/sources') }} + - name: Get sources + if: steps.cache.outputs.cache-hit != 'true' + run: ./download-distfiles.sh + - name: Cache sources + if: steps.cache.outputs.cache-hit != 'true' + uses: actions/cache/save@v4 + with: + path: | + distfiles + key: cache-${{ hashFiles('steps/*/sources') }} + - name: Run bootstrap + run: ./rootfs.py --bwrap --external-sources --build-kernels --cores 2 --early-preseed early_preseed.tar --repo repo --extras python + - name: Remove non-new packages + run: comm -12 <(ls -1 repo) <(ls -1 target/external/repo) | xargs -I{} rm target/external/repo/{} + - name: Archive Python packages + if: always() # archive both failed and successful builds + uses: actions/upload-artifact@v4 with: name: packages path: target/external/repo/** diff --git a/rootfs.py b/rootfs.py index 1cfeab94..a2918d35 100755 --- a/rootfs.py +++ b/rootfs.py @@ -49,6 +49,8 @@ def create_configuration_file(args): config.write("DISK=sda1\n") config.write("KERNEL_BOOTSTRAP=False\n") config.write(f"BUILD_KERNELS={args.update_checksums or args.build_kernels}\n") + for extra in args.extras.split(","): + config.write(f"EXTRA_{extra.upper()}=True\n") config.write(f"CONFIGURATOR={args.configurator}\n") # pylint: disable=too-many-statements,too-many-branches @@ -101,6 +103,8 @@ def main(): parser.add_argument("--internal-ci", help="INTERNAL for github CI") parser.add_argument("-s", "--swap", help="Swap space to allocate in Linux", default=0) + parser.add_argument("-x", "--extras", help="Comma-separated list of extra bootstraps to run (python)", + default="") # QEMU arguments parser.add_argument("-q", "--qemu", help="Use QEMU", @@ -165,6 +169,12 @@ def check_types(): else: args.swap = 0 + # Check extra bootstraps + valid = ["python"] + for extra in args.extras.split(","): + if extra != "" and extra not in valid: + raise ValueError(f"{extra} is not a known bootstrap") + # Set constant umask os.umask(0o022) @@ -210,13 +220,10 @@ def bootstrap(args, generator, target, size): elif args.bwrap: init = '/init' - if not args.internal_ci or args.internal_ci == "pass1": - generator.prepare(target, using_kernel=False) + generator.prepare(target, using_kernel=False) - arch = stage0_arch_map.get(args.arch, args.arch) - init = os.path.join(os.sep, 'bootstrap-seeds', 'POSIX', arch, 'kaem-optional-seed') - else: - generator.reuse(target) + arch = stage0_arch_map.get(args.arch, args.arch) + init = os.path.join(os.sep, 'bootstrap-seeds', 'POSIX', arch, 'kaem-optional-seed') run('env', '-i', 'bwrap', '--unshare-user', '--uid', '0', diff --git a/steps/configurator b/steps/configurator index ff97fe23..bb0eddf1 100644 --- a/steps/configurator +++ b/steps/configurator @@ -40,6 +40,13 @@ this correctly, otherwise you risk overwriting an existing disk on your system. LIVE-BOOTSTRAP TAKES NO LIABILITY FOR ANY DATA LOSS RESULTING FROM ITS USE. +m /extras _ _ _ Extra bootstraps +Various extra language or package bootstraps that are not core to +live-bootstrap, but are an optional part of it. + +o /extras/python EXTRA_PYTHON bool False Python bootstrap +Bootstraps Python. + m /sysinfo _ _ _ System information Details about your specific system and the environment live-bootstrap is running in. diff --git a/steps/manifest b/steps/manifest index 0f4a4cfb..b2fa828c 100644 --- a/steps/manifest +++ b/steps/manifest @@ -52,6 +52,7 @@ build: bzip2-1.0.8 build: coreutils-5.0 build: byacc-20240109 build: bash-2.05b +jump: break ( INTERNAL_CI == pass1 ) improve: setup_repo improve: update_env improve: merged_usr @@ -127,7 +128,6 @@ build: kexec-tools-2.0.22 ( BUILD_LINUX == True ) improve: clean_sources improve: clean_artifacts build: linux-4.14.341-openela ( BUILD_LINUX == True ) -jump: break ( INTERNAL_CI == pass1 ) improve: populate_device_nodes jump: linux ( CHROOT == False ) jump: move_disk ( KERNEL_BOOTSTRAP == True ) @@ -181,18 +181,6 @@ build: grep-3.7 build: sed-4.8 build: autogen-5.18.16 build: musl-1.2.4 -build: python-2.0.1 -build: python-2.0.1 -build: python-2.3.7 -build: python-2.3.7 -build: python-2.5.6 -build: python-3.1.5 -build: python-3.1.5 -build: python-3.3.7 -build: python-3.4.10 -build: python-3.8.16 -build: python-3.11.1 -uninstall: python-2.0.1 python-2.3.7 python-3.1.5 python-3.3.7 python-3.4.10 python-3.8.16 jump: break ( INTERNAL_CI == pass1 ) # scripts are generated in pass1 build: gcc-10.4.0 build: binutils-2.41 @@ -207,6 +195,18 @@ build: gzip-1.13 build: diffutils-3.10 build: gawk-5.3.0 build: m4-1.4.19 +build: python-2.0.1 ( EXTRA_PYTHON == True ) +build: python-2.0.1 ( EXTRA_PYTHON == True ) +build: python-2.3.7 ( EXTRA_PYTHON == True ) +build: python-2.3.7 ( EXTRA_PYTHON == True ) +build: python-2.5.6 ( EXTRA_PYTHON == True ) +build: python-3.1.5 ( EXTRA_PYTHON == True ) +build: python-3.1.5 ( EXTRA_PYTHON == True ) +build: python-3.3.7 ( EXTRA_PYTHON == True ) +build: python-3.4.10 ( EXTRA_PYTHON == True ) +build: python-3.8.16 ( EXTRA_PYTHON == True ) +build: python-3.11.1 ( EXTRA_PYTHON == True ) +uninstall: python-2.0.1 python-2.3.7 python-3.1.5 python-3.3.7 python-3.4.10 python-3.8.16 ( EXTRA_PYTHON == True ) improve: cleanup_filesystem improve: null_time ( FORCE_TIMESTAMPS == True ) improve: update_checksums ( UPDATE_CHECKSUMS == True )