feat: move witness generation outside mutex

.github/workflows/test-e2e-multi_pp.yml

@@ -7,6 +7,7 @@ on:
   workflow_dispatch: {}
 
 jobs:
+
   test-e2e-multi_pp:
     strategy:
       fail-fast: false

.github/workflows/test-e2e-pp.yml

@@ -0,0 +1,106 @@
+name: Test e2e pp
+on: 
+  push:
+    branches:
+      - '**'
+  workflow_dispatch: {}
+
+
+jobs:
+  build-cdk-image:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+
+    - uses: actions/setup-go@v5
+      with:
+        go-version: 1.23.x
+
+    - name: Build cdk docker image
+      run: make build-docker
+
+    - name: Save cdk image to archive
+      run: docker save --output /tmp/cdk.tar cdk
+
+    - name: Upload archive
+      uses: actions/upload-artifact@v4
+      with:
+        name: cdk
+        path: /tmp/cdk.tar
+
+  test-e2e:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    needs: build-cdk-image
+    strategy:
+      fail-fast: false
+      matrix:
+        e2e-group:
+          - "fork12-pessimistic"
+    steps:
+    - name: Checkout Code
+      uses: actions/checkout@v4
+
+    - name: Checkout Kurtosis CDK
+      uses: actions/checkout@v4
+      with:
+        repository: 0xPolygon/kurtosis-cdk
+        path: kurtosis-cdk
+        ref: v0.2.31
+
+    - name: Install Kurtosis CDK tools
+      uses: ./kurtosis-cdk/.github/actions/setup-kurtosis-cdk
+
+    - name: Install polycli
+      run: |
+        POLYCLI_VERSION="${{ vars.POLYCLI_VERSION }}"
+        tmp_dir=$(mktemp -d) 
+        curl -L "https://github.com/0xPolygon/polygon-cli/releases/download/${POLYCLI_VERSION}/polycli_${POLYCLI_VERSION}_linux_amd64.tar.gz" | tar -xz -C "$tmp_dir" 
+        mv "$tmp_dir"/* /usr/local/bin/polycli 
+        rm -rf "$tmp_dir"
+        sudo chmod +x /usr/local/bin/polycli
+        /usr/local/bin/polycli version
+
+    - name: Setup Bats and bats libs
+      uses: bats-core/bats-action@2.0.0
+
+    - name: Download cdk archive
+      uses: actions/download-artifact@v4
+      with:
+        name: cdk
+        path: /tmp
+
+    - name: Load cdk image
+      run: |
+        docker load --input /tmp/cdk.tar
+        docker image ls -a
+
+    - name: Run e2e tests
+      run: make test-e2e-${{ matrix.e2e-group }}
+      working-directory: test
+      env:
+        KURTOSIS_FOLDER: ${{ github.workspace }}/kurtosis-cdk
+        BATS_LIB_PATH: /usr/lib/
+        agglayer_prover_sp1_key: ${{ secrets.SP1_PRIVATE_KEY }}
+
+
+    - name: Dump enclave logs
+      if: failure()
+      run: kurtosis dump ./dump
+
+    - name: Generate archive name
+      if: failure()
+      run: |
+        archive_name="dump_run_with_args_${{matrix.e2e-group}}_${{ github.run_id }}"
+        echo "ARCHIVE_NAME=${archive_name}" >> "$GITHUB_ENV"
+        echo "Generated archive name: ${archive_name}"
+        kurtosis service exec cdk cdk-node-001 'cat /etc/cdk/cdk-node-config.toml' > ./dump/cdk-node-config.toml
+
+    - name: Upload logs
+      if: failure()
+      uses: actions/upload-artifact@v4
+      with:
+        name: ${{ env.ARCHIVE_NAME }}
+        path: ./dump

.github/workflows/test-e2e.yml

@@ -42,7 +42,6 @@ jobs:
           - "fork11-rollup"
           - "fork12-validium"
           - "fork12-rollup"
-          - "fork12-pessimistic"
     steps:
     - name: Checkout Code
       uses: actions/checkout@v4
@@ -52,7 +51,7 @@ jobs:
       with:
         repository: 0xPolygon/kurtosis-cdk
         path: kurtosis-cdk
-        ref: v0.2.31
+        ref: v0.3.3
 
     - name: Install Kurtosis CDK tools
       uses: ./kurtosis-cdk/.github/actions/setup-kurtosis-cdk

aggregator/aggregator.go

@@ -1053,7 +1053,7 @@ func (a *Aggregator) getVerifiedBatchAccInputHash(ctx context.Context, batchNumb
 
 func (a *Aggregator) getAndLockBatchToProve(
 	ctx context.Context, prover ProverInterface,
-) (*state.Batch, []byte, *state.Proof, error) {
+) (*state.Batch, *state.Proof, error) {
 	proverID := prover.ID()
 	proverName := prover.Name()
 
@@ -1069,7 +1069,7 @@ func (a *Aggregator) getAndLockBatchToProve(
 	// Get last virtual batch number from L1
 	lastVerifiedBatchNumber, err := a.etherman.GetLatestVerifiedBatchNum()
 	if err != nil {
-		return nil, nil, nil, err
+		return nil, nil, err
 	}
 
 	proofExists := true
@@ -1082,7 +1082,7 @@ func (a *Aggregator) getAndLockBatchToProve(
 		if err != nil {
 			tmpLogger.Infof("Error checking proof exists for batch %d", batchNumberToVerify)
 
-			return nil, nil, nil, err
+			return nil, nil, err
 		}
 
 		if proofExists {
@@ -1094,7 +1094,7 @@ func (a *Aggregator) getAndLockBatchToProve(
 				err := a.storage.CleanupGeneratedProofs(ctx, math.MaxInt, nil)
 				if err != nil {
 					tmpLogger.Infof("Error cleaning up generated proofs for batch %d", batchNumberToVerify)
-					return nil, nil, nil, err
+					return nil, nil, err
 				}
 				batchNumberToVerify--
 				break
@@ -1105,38 +1105,45 @@ func (a *Aggregator) getAndLockBatchToProve(
 	// Check if the batch has been sequenced
 	sequence, err := a.l1Syncr.GetSequenceByBatchNumber(ctx, batchNumberToVerify)
 	if err != nil && !errors.Is(err, entities.ErrNotFound) {
-		return nil, nil, nil, err
+		return nil, nil, err
 	}
 
 	// Not found, so it it not possible to verify the batch yet
 	if sequence == nil || errors.Is(err, entities.ErrNotFound) {
 		tmpLogger.Infof("Sequencing event for batch %d has not been synced yet, "+
 			"so it is not possible to verify it yet. Waiting ...", batchNumberToVerify)
 
-		return nil, nil, nil, state.ErrNotFound
+		return nil, nil, state.ErrNotFound
 	}
 
 	stateSequence := state.Sequence{
 		FromBatchNumber: sequence.FromBatchNumber,
 		ToBatchNumber:   sequence.ToBatchNumber,
 	}
 
+	// Store the sequence in aggregator DB
+	err = a.storage.AddSequence(ctx, stateSequence, nil)
+	if err != nil {
+		tmpLogger.Infof("Error storing sequence for batch %d", batchNumberToVerify)
+		return nil, nil, err
+	}
+
 	// Get Batch from L1 Syncer
 	virtualBatch, err := a.l1Syncr.GetVirtualBatchByBatchNumber(a.ctx, batchNumberToVerify)
 	if err != nil && !errors.Is(err, entities.ErrNotFound) {
 		a.logger.Errorf("Error getting virtual batch: %v", err)
-		return nil, nil, nil, err
+		return nil, nil, err
 	} else if errors.Is(err, entities.ErrNotFound) {
 		a.logger.Infof("Virtual batch %d has not been synced yet, "+
 			"so it is not possible to verify it yet. Waiting ...", batchNumberToVerify)
-		return nil, nil, nil, state.ErrNotFound
+		return nil, nil, state.ErrNotFound
 	}
 
 	// Get Batch from RPC
 	rpcBatch, err := a.rpcClient.GetBatch(batchNumberToVerify)
 	if err != nil {
 		a.logger.Errorf("error getting batch %d from RPC: %v.", batchNumberToVerify, err)
-		return nil, nil, nil, err
+		return nil, nil, err
 	}
 
 	// Compare BatchL2Data from virtual batch and rpcBatch (skipping injected batch (1))
@@ -1157,7 +1164,7 @@ func (a *Aggregator) getAndLockBatchToProve(
 	oldAccInputHash := a.getAccInputHash(batchNumberToVerify - 1)
 	if oldAccInputHash == (common.Hash{}) && batchNumberToVerify > 1 {
 		tmpLogger.Warnf("AccInputHash for previous batch (%d) is not in memory. Waiting ...", batchNumberToVerify-1)
-		return nil, nil, nil, state.ErrNotFound
+		return nil, nil, state.ErrNotFound
 	}
 
 	forcedBlockHashL1 := rpcBatch.ForcedBlockHashL1()
@@ -1167,7 +1174,7 @@ func (a *Aggregator) getAndLockBatchToProve(
 		l1Block, err := a.l1Syncr.GetL1BlockByNumber(ctx, virtualBatch.BlockNumber)
 		if err != nil {
 			a.logger.Errorf("Error getting l1 block: %v", err)
-			return nil, nil, nil, err
+			return nil, nil, err
 		}
 
 		forcedBlockHashL1 = l1Block.ParentHash
@@ -1213,34 +1220,9 @@ func (a *Aggregator) getAndLockBatchToProve(
 		ForkID:          a.cfg.ForkId,
 	}
 
-	// Request the witness from the server, if it is busy just keep looping until it is available
-	start := time.Now()
-	witness, err := a.rpcClient.GetWitness(batchNumberToVerify, a.cfg.UseFullWitness)
-	for err != nil {
-		if errors.Is(err, rpc.ErrBusy) {
-			a.logger.Debugf(
-				"Witness server is busy, retrying get witness for batch %d in %v",
-				batchNumberToVerify, a.cfg.RetryTime.Duration,
-			)
-		} else {
-			a.logger.Errorf("Failed to get witness for batch %d, err: %v", batchNumberToVerify, err)
-		}
-		time.Sleep(a.cfg.RetryTime.Duration)
-		witness, err = a.rpcClient.GetWitness(batchNumberToVerify, a.cfg.UseFullWitness)
-	}
-	end := time.Now()
-	a.logger.Debugf("Time to get witness for batch %d: %v", batchNumberToVerify, end.Sub(start))
-
-	// Store the sequence in aggregator DB
-	err = a.storage.AddSequence(ctx, stateSequence, nil)
-	if err != nil {
-		tmpLogger.Infof("Error storing sequence for batch %d", batchNumberToVerify)
-
-		return nil, nil, nil, err
-	}
-
 	// All the data required to generate a proof is ready
-	tmpLogger.Infof("All information to generate proof for batch %d is ready", virtualBatch.BatchNumber)
+	tmpLogger.Infof("All information to generate proof for batch %d is ready. "+
+		"Witness will be requested.", virtualBatch.BatchNumber)
 	tmpLogger = tmpLogger.WithFields("batch", virtualBatch.BatchNumber)
 
 	now := time.Now().Round(time.Microsecond)
@@ -1257,10 +1239,10 @@ func (a *Aggregator) getAndLockBatchToProve(
 	if err != nil {
 		tmpLogger.Errorf("Failed to add batch proof to DB for batch %d, err: %v", virtualBatch.BatchNumber, err)
 
-		return nil, nil, nil, err
+		return nil, nil, err
 	}
 
-	return stateBatch, witness, proof, nil
+	return stateBatch, proof, nil
 }
 
 func (a *Aggregator) tryGenerateBatchProof(ctx context.Context, prover ProverInterface) (bool, error) {
@@ -1271,7 +1253,7 @@ func (a *Aggregator) tryGenerateBatchProof(ctx context.Context, prover ProverInt
 	)
 	tmpLogger.Debug("tryGenerateBatchProof start")
 
-	batchToProve, witness, proof, err0 := a.getAndLockBatchToProve(ctx, prover)
+	batchToProve, proof, err0 := a.getAndLockBatchToProve(ctx, prover)
 	if errors.Is(err0, state.ErrNotFound) || errors.Is(err0, entities.ErrNotFound) {
 		// nothing to proof, swallow the error
 		tmpLogger.Debug("Nothing to generate proof")
@@ -1281,6 +1263,11 @@ func (a *Aggregator) tryGenerateBatchProof(ctx context.Context, prover ProverInt
 		return false, err0
 	}
 
+	// Request Witness
+	tmpLogger.Infof("Requesting witness for batch %d", batchToProve.BatchNumber)
+	witness := a.getWitness(batchToProve.BatchNumber)
+	tmpLogger.Infof("Witness received for batch %d", batchToProve.BatchNumber)
+
 	tmpLogger = tmpLogger.WithFields("batch", batchToProve.BatchNumber)
 
 	var (
@@ -1362,6 +1349,28 @@ func (a *Aggregator) tryGenerateBatchProof(ctx context.Context, prover ProverInt
 	return true, nil
 }
 
+func (a *Aggregator) getWitness(batchNumber uint64) []byte {
+	// Request the witness from the server, if it is busy just keep looping until it is available
+	start := time.Now()
+	witness, err := a.rpcClient.GetWitness(batchNumber, a.cfg.UseFullWitness)
+	for err != nil {
+		if errors.Is(err, rpc.ErrBusy) {
+			a.logger.Debugf(
+				"Witness server is busy, retrying get witness for batch %d in %v",
+				batchNumber, a.cfg.RetryTime.Duration,
+			)
+		} else {
+			a.logger.Errorf("Failed to get witness for batch %d, err: %v", batchNumber, err)
+		}
+		time.Sleep(a.cfg.RetryTime.Duration)
+		witness, err = a.rpcClient.GetWitness(batchNumber, a.cfg.UseFullWitness)
+	}
+	end := time.Now()
+	a.logger.Debugf("Time to get witness for batch %d: %v", batchNumber, end.Sub(start))
+
+	return witness
+}
+
 func (a *Aggregator) performSanityChecks(tmpLogger *log.Logger, stateRoot, accInputHash common.Hash,
 	batchToProve *state.Batch) {
 	// Sanity Check: state root from the proof must match the one from the batch

test/combinations/fork11-rollup.yml

@@ -2,14 +2,12 @@ deployment_stages:
   deploy_l2_contracts: true
 
 args:
-  agglayer_image: ghcr.io/agglayer/agglayer:0.2.0-rc.19
   zkevm_contracts_image: leovct/zkevm-contracts:v7.0.0-rc.2-fork.11-patch.1
-  zkevm_prover_image: hermeznetwork/zkevm-prover:v7.0.2-fork.11
-  zkevm_node_image: hermeznetwork/zkevm-node:v0.7.0-fork11-RC1
+  agglayer_image: ghcr.io/agglayer/agglayer:0.2.0-rc.19
+  zkevm_prover_image: hermeznetwork/zkevm-prover:v7.0.4-hotfix.2-fork.11
+  zkevm_node_image: hermeznetwork/zkevm-node:v0.7.0-fork11
   cdk_node_image: cdk:latest
   gas_token_enabled: true
   gas_token_address: ""
   consensus_contract_type: rollup
   sequencer_type: erigon
-  # This is just a workaround, since the Kurtosis is deploying the agglayer even in the FEP setup
-  agglayer_prover_sp1_key: {{.agglayer_prover_sp1_key}}

test/combinations/fork12-cdk-validium.yml

@@ -2,13 +2,10 @@ deployment_stages:
   deploy_l2_contracts: true
 
 args:
-  zkevm_contracts_image: leovct/zkevm-contracts:v8.0.0-rc.4-fork.12-patch.1
-  zkevm_prover_image: hermeznetwork/zkevm-prover:v8.0.0-RC12-fork.12
-  cdk_erigon_node_image: hermeznetwork/cdk-erigon:v2.61.14-RC1-amd64
+  zkevm_contracts_image: leovct/zkevm-contracts:v9.0.0-rc.6-pp-fork.12
+  zkevm_prover_image: hermeznetwork/zkevm-prover:v8.0.0-RC16-fork.12
   cdk_node_image: cdk:latest
   gas_token_enabled: true
   gas_token_address: ""
   consensus_contract_type: cdk-validium
   sequencer_type: erigon
-  # This is just a workaround, since the Kurtosis is deploying the agglayer even in the FEP setup
-  agglayer_prover_sp1_key: {{.agglayer_prover_sp1_key}}

test/combinations/fork12-pessimistic-multi-attach-second-cdk.yml

@@ -25,7 +25,6 @@ args:
   zkevm_l2_proofsigner_address: "0xf1a661D7b601Ec46a040f57193cC99aB8c4132FA"
   zkevm_l2_proofsigner_private_key: "0xc7fe3a006d75ba9326d9792523385abb49057c66aee0b8b4248821a89713f975"
 
-
   cdk_node_image: cdk:latest
   cdk_erigon_node_image: hermeznetwork/cdk-erigon:v2.61.14-RC1-amd64
   zkevm_contracts_image: leovct/zkevm-contracts:v9.0.0-rc.5-pp-fork.12
@@ -37,4 +36,3 @@ args:
   zkevm_use_real_verifier: false
   enable_normalcy: true
   verifier_program_vkey: 0x00766aa16a6efe4ac05c0fe21d4b50f9631dbd1a2663a982da861427085ea2ea
-