Run evals in decentralized python client test

.github/actions/wait-for-garnix/action.yaml

@@ -12,6 +12,17 @@ inputs:
 runs:
   using: 'composite'
   steps:
+    - name: Install GitHub CLI
+      shell: bash
+      run: |
+        if ! command -v gh &> /dev/null; then
+          echo "Installing GitHub CLI..."
+          sudo apt update
+          sudo apt install gh -y
+        else
+          echo "GitHub CLI already installed"
+        fi
+
     - name: Wait for All Garnix checks
       shell: bash
       env:
@@ -26,13 +37,13 @@ runs:
         for i in $(seq 1 $TOTAL_ATTEMPTS); do
           if [ -z "$GARNIX_SUITE_ID" ]; then
             GARNIX_SUITE_ID=$(gh api repos/${{ github.repository }}/commits/$SHA/check-suites --jq '.check_suites[] | select(.app.name == "Garnix CI") | .id')
-            
+
             if [ -z "$GARNIX_SUITE_ID" ]; then
               echo "No Garnix CI check suite found yet, waiting... (attempt $i/$TOTAL_ATTEMPTS)"
               sleep 10
               continue
             fi
-            
+
             echo "Found Garnix CI check suite: $GARNIX_SUITE_ID"
           fi
 

.github/workflows/solana-integration-test-run.yml

@@ -4,6 +4,11 @@ on:
     branches: [main]
   pull_request:
     branches: [main, '**']
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   # First, build the validator image and cache it
   build-validator:
@@ -36,3 +41,65 @@ jobs:
     with:
       test-name: ${{ matrix.test-name }}
     secrets: inherit
+
+  decentralized-integration-python-test:
+    runs-on: self-hosted
+    needs: build-validator
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Get Validator Image from cache
+        id: cache-validator
+        uses: actions/cache/restore@v4
+        with:
+          path: validator-image.tar.gz
+          key: validator-image-${{ runner.os }}-${{ hashFiles('shared/coordinator/src/coordinator.rs', 'architectures/decentralized/solana-coordinator/**/*.rs', 'architectures/decentralized/solana-coordinator/**/*.toml', 'architectures/decentralized/solana-coordinator/Cargo.lock', 'architectures/decentralized/solana-authorizer/**/*.rs', 'architectures/decentralized/solana-authorizer/**/*.toml', 'architectures/decentralized/solana-authorizer/Cargo.lock', 'docker/test/psyche_solana_validator_entrypoint.sh', 'nix/docker.nix', 'flake.lock') }}
+          fail-on-cache-miss: true
+
+      - name: Load Validator Image
+        run: |
+          echo "Loading validator image from cache"
+          docker load < validator-image.tar.gz
+          docker images | grep psyche-solana-test-validator
+
+          echo "Disk usage after loading validator"
+          df -h
+
+      - name: Clean up validator tar file
+        run: |
+          # Remove the compressed validator image to free up disk space
+          rm -f validator-image.tar.gz
+          echo "Disk usage after removing validator tar"
+          df -h
+
+      - name: Download Solana Test Client Python Image
+        run: |
+          echo "Disk space before client build"
+          df -h
+
+          sleep 500
+          # Calculate the derivation hash
+          echo "Calculating derivation path"
+          DRV_PATH=$(nix eval --raw .#docker-psyche-solana-test-client.drvPath)
+          echo "Derivation path: $DRV_PATH"
+
+          OUT_PATH=$(nix derivation show $DRV_PATH | jq -r '.[].outputs.out.path')
+          echo "Output path: $OUT_PATH"
+
+          # download from Garnix cache first
+          echo "Attempting to fetch from Garnix cache"
+          nix-store --realise $OUT_PATH --option substitute true
+
+          # Load the image into Docker
+          $OUT_PATH | docker load
+
+          echo "Disk space after client build"
+          df -h
+
+      - name: Run decentralized integration test
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+        run: |
+          nix develop .#dev-python --command bash -c "cargo test --release --features python,parallelism -p psyche-decentralized-testing --test integration_tests -- --nocapture test_big_model_with_sidecars test_torchtitan_p2p_model_share"