dev 6.13 rebase

.github/workflows/build.yml

@@ -12,96 +12,301 @@ on:
   workflow_dispatch:
 
 jobs:
+  prebuild:
+    runs-on: rehosting-arc
+    outputs:
+      matrix: ${{ steps.find_targets.outputs.matrix }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          ref: ${{ github.ref }}
+
+      - name: Trust Harbor's self-signed certificate
+        run: |
+          echo "Fetching certificate from ${{ secrets.REHOSTING_ARC_REGISTRY }}"
+          openssl s_client -showcerts -connect ${{ secrets.REHOSTING_ARC_REGISTRY }}:443 < /dev/null 2>/dev/null | openssl x509 -outform PEM | sudo tee /usr/local/share/ca-certificates/harbor.crt > /dev/null
+          sudo update-ca-certificates
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          driver-opts: |
+            image=moby/buildkit:master
+            network=host
+          buildkitd-config-inline: |
+            [registry."${{ secrets.REHOSTING_ARC_REGISTRY }}"]
+              insecure = true
+              http = true
+
+      - name: Log git revisions of all linux projects
+        run: |
+          echo "Main repo revision:" && git rev-parse HEAD
+          echo
+          echo "Submodule revisions:" && git submodule status
+          echo
+          echo "Full submodule SHAs:" && git submodule foreach 'echo $name: $(git rev-parse HEAD)'
+
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: rehosting
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Log in to Rehosting Arc Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{secrets.REHOSTING_ARC_REGISTRY}}
+          username: ${{ secrets.REHOSTING_ARC_REGISTRY_USER }}
+          password: ${{ secrets.REHOSTING_ARC_REGISTRY_PASSWORD }}
+
+      # Build this first, build.sh will use it later
+      - name: Build kernel_builder docker image
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          push: true
+          tags: |
+            ${{secrets.REHOSTING_ARC_REGISTRY}}/pandare/kernel_builder:${{ github.ref_name }}
+            ${{secrets.REHOSTING_ARC_REGISTRY}}/pandare/kernel_builder:latest
+          build-args: |
+            REGISTRY=${{ secrets.REHOSTING_ARC_REGISTRY }}/proxy
+          cache-from: |
+            type=registry,ref=${{secrets.REHOSTING_ARC_REGISTRY}}/pandare/kernel_builder:cache,mode=max
+            type=registry,ref=${{secrets.REHOSTING_ARC_REGISTRY}}/pandare/kernel_builderr:cache-PR-${{github.event.number}},mode=max
+          cache-to: |
+            type=registry,ref=${{secrets.REHOSTING_ARC_REGISTRY}}/pandare/kernel_builder:cache,mode=max
+            type=registry,ref=${{secrets.REHOSTING_ARC_REGISTRY}}/pandare/kernel_builder:cache-PR-${{ github.event.number }}
+
+      - name: Setup shared Linux kernel sources
+        run: |
+          set -eux
+          BASE_REPO_DIR="/home/runner/_shared/linux"
+          BASE_REPO_URL="https://github.com/rehosting/linux"
+          # Clone bare base repo if missing
+          if [ ! -d "$BASE_REPO_DIR" ]; then
+            echo "Cloning bare base repo to $BASE_REPO_DIR"
+            git clone --bare "$BASE_REPO_URL" "$BASE_REPO_DIR"
+          fi
+          # Always fetch latest from upstream
+          git --git-dir="$BASE_REPO_DIR" fetch --all
+
+          # Find all needed kernel versions from configs
+          needed_versions=()
+          for version_dir in configs/*/; do
+            version=$(basename "$version_dir")
+            needed_versions+=("$version")
+          done
+
+          # For each needed version, create a unique per-run clone and checkout correct revision
+          for version in "${needed_versions[@]}"; do
+            CLONE_DIR="/home/runner/_shared/linux-$version-$GITHUB_RUN_ID"
+            if [ ! -d "$CLONE_DIR" ]; then
+              echo "Cloning $version to $CLONE_DIR"
+              git clone --reference "$BASE_REPO_DIR" "$BASE_REPO_DIR" "$CLONE_DIR"
+            fi
+
+            # Get submodule path and revision from .gitmodules and ls-tree
+            SUBMODULE_PATH="linux/$version"
+            SUBMODULE_REV=$(git ls-tree HEAD "$SUBMODULE_PATH" | awk '{print $3}')
+            if [ -z "$SUBMODULE_REV" ]; then
+              echo "No submodule revision found for $SUBMODULE_PATH"
+              continue
+            fi
+
+            # Checkout the correct revision
+            cd "$CLONE_DIR"
+            git fetch --all
+            git checkout "$SUBMODULE_REV"
+            cd -
+          done
+
+          echo "Available kernel clones:"
+          ls -la /home/runner/_shared/
+
+      - name: Find valid target/version combinations
+        id: find_targets
+        run: |
+          COMBINATIONS=()
+          for version_dir in configs/*/; do
+            version=$(basename "$version_dir")
+            for config_file in "$version_dir"*; do
+              if [[ -f "$config_file" && ! "$config_file" =~ \.inc$ ]]; then
+                target=$(basename "$config_file")
+                COMBINATIONS+=("\"${target}@${version}\"")
+              fi
+            done
+          done
+          MATRIX=$(IFS=,; echo "[${COMBINATIONS[*]}]")
+          echo "matrix=$MATRIX" >> $GITHUB_OUTPUT
+          echo "Found valid combinations: $MATRIX"
+
   build:
-    # Only publish on tags. run git tag vX and git push origin vX
-    # runs-on: self-hosted
-    runs-on: ubuntu-latest
+    needs: prebuild
+    runs-on: rehosting-arc
     if: github.event.pull_request.draft == false
 
-
     strategy:
       matrix:
-        target: [armel, arm64, mipsel, mipseb, mips64el, mips64eb, x86_64]
-        version: ["4.10"] # XXX: quotes are necessary, otherwise 4.10 -> 4.1
+        target_version: ${{ fromJSON(needs.prebuild.outputs.matrix) }}
 
     steps:
-      - uses: actions/checkout@v4 # Clones to $GITHUB_WORKSPACE
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-          #submodules: 'true'
+          ref: ${{ github.ref }}
 
-      # Instead of getting submodules with checkout, we can do it manually to control depth.
-      # We don't want a full Linux history
-      - name: Pull kernel source
-        run: git submodule update --init --depth 1
+      - name: Extract target and version
+        id: extract
+        run: |
+          TARGET_VERSION="${{ matrix.target_version }}"
+          TARGET="${TARGET_VERSION%@*}"
+          VERSION="${TARGET_VERSION#*@}"
+          echo "target=$TARGET" >> $GITHUB_OUTPUT
+          echo "version=$VERSION" >> $GITHUB_OUTPUT
+          echo "Building target: $TARGET, version: $VERSION"
 
-      - name: Cache kernel build objects
-        uses: actions/cache@v4
-        with:
-          path: |
-            cache
-          key: ${{ runner.os }}-kernel-build-${{ matrix.kernel_version }}-${{ matrix.target }}-${{ hashFiles('**/Dockerfile', '**/*.sh') }}
-          restore-keys: |
-            ${{ runner.os }}-kernel-build-${{ matrix.kernel_version }}-${{ matrix.target }}-
-            ${{ runner.os }}-kernel-build-${{ matrix.kernel_version }}-
-            ${{ runner.os }}-kernel-build-
-
-      - name: Lint configs
-        run: ./build.sh --targets ${{ matrix.target }} --versions ${{ matrix.version }} --config-only
-
-      - name: Build Kernel for ${{ matrix.target }}
-        run: ./build.sh --targets ${{ matrix.target }} --versions ${{ matrix.version }}
-
-      # Temporarily store each target's build output
-      - name: Save ${{ matrix.target }} ${{ matrix.version }} build output
-        uses: actions/upload-artifact@v4
+      - name: Trust Harbor's self-signed certificate
+        run: |
+          echo "Fetching certificate from ${{ secrets.REHOSTING_ARC_REGISTRY }}"
+          openssl s_client -showcerts -connect ${{ secrets.REHOSTING_ARC_REGISTRY }}:443 < /dev/null 2>/dev/null | openssl x509 -outform PEM | sudo tee /usr/local/share/ca-certificates/harbor.crt > /dev/null
+          sudo update-ca-certificates
+
+      - name: Log in to Rehosting Arc Registry
+        uses: docker/login-action@v3
         with:
-          name: build-output-${{ matrix.target }}.${{ matrix.version }}
-          path: kernels-latest.tar.gz
+          registry: ${{secrets.REHOSTING_ARC_REGISTRY}}
+          username: ${{ secrets.REHOSTING_ARC_REGISTRY_USER }}
+          password: ${{ secrets.REHOSTING_ARC_REGISTRY_PASSWORD }}
+
+      - name: Pull prebuilt image
+        run: |
+          docker pull ${{ secrets.REHOSTING_ARC_REGISTRY }}/rehosting/linux_builder:${{ github.ref_name }}
+          docker tag ${{ secrets.REHOSTING_ARC_REGISTRY }}/rehosting/linux_builder:${{ github.ref_name }} rehosting/linux_builder:${{ github.ref_name }}
+
+      - uses: oras-project/setup-oras@v1.2.3
+
+      - name: Install pigz for parallel compression
+        run: |
+          sudo apt-get update && sudo apt-get install -y pigz
+
+      - name: Build Kernel for ${{ steps.extract.outputs.target }}
+        run: |
+            set -eux
+            VERSION="${{ steps.extract.outputs.version }}"
+            CLONE_DIR="/home/runner/_shared/linux-$VERSION-$GITHUB_RUN_ID"
+            if [ ! -d "$CLONE_DIR" ]; then
+              echo "Kernel clone $CLONE_DIR not found!"
+              exit 1
+            fi
+
+            mkdir -p cache
+            docker run --rm \
+              -v $PWD/cache:/tmp/build \
+              -v $PWD/configs:/app/configs:ro \
+              -v $PWD/_in_container_build.sh:/app/_in_container_build.sh:ro \
+              -v $CLONE_DIR:/app/linux/${VERSION}:ro \
+              ${{ secrets.REHOSTING_ARC_REGISTRY }}/pandare/kernel_builder:${{ github.ref_name }} \
+              bash -c "./_in_container_build.sh false ${VERSION} ${{ steps.extract.outputs.target }} false false false true && cp kernels-latest.tar.gz /tmp/build/kernels-latest.tar.gz && cp /app/kernel-devel-all.tar.gz /tmp/build/kernel-devel-all.tar.gz"
+            mv cache/kernels-latest.tar.gz kernels-latest.tar.gz
+            mv cache/kernel-devel-all.tar.gz kernel-devel-all.tar.gz
+            tar -cf - kernels-latest.tar.gz kernel-devel-all.tar.gz | pigz -p $(nproc) > build-output-${{ steps.extract.outputs.target }}.${VERSION}.tar.gz
+
+      - name: Push build artifact to registry with oras
+        run: |
+          oras push ${{ secrets.REHOSTING_ARC_REGISTRY }}/rehosting/linux_builder_artifact:${{ steps.extract.outputs.target }}-${{ steps.extract.outputs.version }} build-output-${{ steps.extract.outputs.target }}.${{ steps.extract.outputs.version }}.tar.gz:application/gzip
 
   aggregate:
     if: startsWith(github.ref, 'refs/tags/v') || github.event_name == 'workflow_dispatch'
     needs: build
-    runs-on: ubuntu-latest
+    runs-on: rehosting-arc
+    env:
+      MATRIX_VERSIONS: ${{ toJSON(needs.build.strategy.matrix.version) }}
     permissions:
       actions: write
       contents: write
     steps:
-      - name: Download all build artifacts
-        uses: actions/download-artifact@v4
+      - name: Trust Harbor's self-signed certificate
+        run: |
+          echo "Fetching certificate from ${{ secrets.REHOSTING_ARC_REGISTRY }}"
+          openssl s_client -showcerts -connect ${{ secrets.REHOSTING_ARC_REGISTRY }}:443 < /dev/null 2>/dev/null | openssl x509 -outform PEM | sudo tee /usr/local/share/ca-certificates/harbor.crt > /dev/null
+          sudo update-ca-certificates
+
+      - name: Log in to Rehosting Arc Registry
+        uses: docker/login-action@v3
         with:
-          path: downloaded-kernels
+          registry: ${{secrets.REHOSTING_ARC_REGISTRY}}
+          username: ${{ secrets.REHOSTING_ARC_REGISTRY_USER }}
+          password: ${{ secrets.REHOSTING_ARC_REGISTRY_PASSWORD }}
+      - uses: oras-project/setup-oras@v1.2.3
+
+      - name: Pull all build artifacts from registry with oras
+        run: |
+          mkdir -p downloaded-kernels
+          for target in armel arm64 mipsel mipseb mips64el mips64eb powerpc powerpc64 loongarch64 riscv64 x86_64; do
+            for version in 6.13; do
+              tag="$target-$version"
+              oras pull ${{ secrets.REHOSTING_ARC_REGISTRY }}/rehosting/linux_builder_artifact:$tag -o downloaded-kernels || echo "No artifact for $tag"
+            done
+          done
+          echo "[DEBUG] Downloaded build artifacts:"
+          ls -lh downloaded-kernels
+
+      - name: Extract all build outputs
+        run: |
+          rm -rf extracted-kernels && mkdir extracted-kernels
+          for archive in $(find downloaded-kernels -name "build-output-*.tar.gz"); do
+            archver=$(basename "$archive" | sed 's/build-output-\(.*\)\.tar\.gz/\1/')
+            mkdir -p extracted-kernels/$archver
+            echo "[DEBUG] Extracting $archive to extracted-kernels/$archver"
+            tar -xzf "$archive" -C extracted-kernels/$archver
+          done
+          echo "[DEBUG] Extracted build outputs:"
+          find extracted-kernels
 
       - name: Combine all kernels into a single archive
         run: |
           rm -rf combined-kernels && mkdir combined-kernels
-          # Extract each kernels-latest.tar.gz archive
-          for archive in $(find downloaded-kernels -name "*.tar.gz"); do
+          echo "[DEBUG] Found kernels-latest.tar.gz files:"
+          find extracted-kernels -name "kernels-latest.tar.gz"
+          # Recursively extract all kernels-latest.tar.gz archives for all architectures/versions
+          find extracted-kernels -name "kernels-latest.tar.gz" | while read archive; do
+            echo "[DEBUG] Extracting $archive into combined-kernels"
             tar -xzf "$archive" -C combined-kernels
           done
+          echo "[DEBUG] Contents of combined-kernels after extraction:"
+          find combined-kernels
 
           # Combine OSI profiles for each kernel version
-          # otherwise the extracted kernel configs willc lobber each other
+          # otherwise the extracted kernel configs will clobber each other
           if [ -d combined-kernels/kernels/4.10 ] ; then
-            for archive in $(find downloaded-kernels -name "*.tar.gz"); do
+            find extracted-kernels -name "kernels-latest.tar.gz" | while read archive; do
               tar -O -xf "$archive" "kernels/4.10/osi.config";
             done > combined-kernels/kernels/4.10/osi.config
           fi
 
-          if [ -d combined-kernels/kernels/6.7 ] ; then
-            for archive in $(find downloaded-kernels -name "*.tar.gz"); do
-              tar -O -xf "$archive" "kernels/6.7/osi.config";
-            done > combined-kernels/kernels/6.7/osi.config
+          if [ -d combined-kernels/kernels/6.13 ] ; then
+            find extracted-kernels -name "kernels-latest.tar.gz" | while read archive; do
+              tar -O -xf "$archive" "kernels/6.13/osi.config";
+            done > combined-kernels/kernels/6.13/osi.config
           fi
 
-
-
           # Create a new single archive from the combined content
           tar -czvf kernels-latest.tar.gz -C combined-kernels .
 
+      - name: Aggregate all kernel-devel artifacts
+        run: |
+          mkdir -p kernel-devel-all
+          for archive in $(find extracted-kernels -name "kernel-devel-all.tar.gz"); do
+            tar -xzf "$archive" -C kernel-devel-all
+          done
+          tar -czvf kernel-devel-all.tar.gz -C kernel-devel-all .
+
       - name: Create and publish release
         uses: softprops/action-gh-release@v1
         with:
-          files: kernels-latest.tar.gz
+          files: |
+            kernels-latest.tar.gz
+            kernel-devel-all.tar.gz
           token: ${{ secrets.GITHUB_TOKEN }}
-          tag_name: ${{ github.ref }}
+          tag_name: ${{ github.ref_name }}

.github/workflows/clear_cache.yml

@@ -0,0 +1,30 @@
+name: Clear kernel cache
+
+on:
+  workflow_dispatch:
+
+jobs:
+  clear-cache:
+    runs-on: rehosting-arc
+    steps:
+      - name: Trust Harbor's self-signed certificate
+        run: |
+          echo "Fetching certificate from ${{ secrets.REHOSTING_ARC_REGISTRY }}"
+          openssl s_client -showcerts -connect ${{ secrets.REHOSTING_ARC_REGISTRY }}:443 < /dev/null 2>/dev/null | openssl x509 -outform PEM | sudo tee /usr/local/share/ca-certificates/harbor.crt > /dev/null
+          sudo update-ca-certificates
+      - uses: oras-project/setup-oras@v1.2.3
+      - name: Log in to Rehosting Arc Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ secrets.REHOSTING_ARC_REGISTRY }}
+          username: ${{ secrets.REHOSTING_ARC_REGISTRY_USER }}
+          password: ${{ secrets.REHOSTING_ARC_REGISTRY_PASSWORD }}
+      - name: Delete all kernel cache images
+        run: |
+          set -e
+          repo="${{ secrets.REHOSTING_ARC_REGISTRY }}/rehosting/linux_builder_cache"
+          tags=$(oras repo tags $repo)
+          for tag in $tags; do
+            echo "Deleting $repo:$tag"
+            oras repo rm $repo:$tag || echo "Failed to delete $repo:$tag"
+          done

.gitmodules

@@ -3,6 +3,6 @@
 	url = https://github.com/rehosting/linux.git
 	branch = main_4.10
 [submodule "linux_6.7"]
-	path = linux/6.7
+	path = linux/6.13
 	url = https://github.com/rehosting/linux.git
 	branch = main_6.7