diff --git a/.circleci/continue/main.yml b/.circleci/continue/main.yml index 0ff0402b5fcc2..16358bedccd0d 100644 --- a/.circleci/continue/main.yml +++ b/.circleci/continue/main.yml @@ -1824,10 +1824,6 @@ jobs: op-acceptance-tests: parameters: - gate: - description: The gate to run. Reads package list from op-acceptance-tests/gates/.txt. If empty, runs all tests. - type: string - default: "" l2_cl_kind: description: "L2 consensus layer client (op-node or kona-node)" type: string @@ -1881,7 +1877,7 @@ jobs: working_directory: op-acceptance-tests no_output_timeout: <> command: | - LOG_LEVEL=info just acceptance-test "<>" + LOG_LEVEL=info just acceptance-test - run: name: Print results (summary) working_directory: op-acceptance-tests @@ -2778,10 +2774,9 @@ workflows: - cannon-prestate - rust-binaries-for-sysgo - go-binaries-for-sysgo - # IN-MEMORY (base gate) - op-node/op-reth + # IN-MEMORY - op-node/op-reth - op-acceptance-tests: name: memory-all-opn-op-reth - gate: "base" l2_el_kind: op-reth no_output_timeout: 120m context: @@ -2793,10 +2788,9 @@ workflows: - cannon-prestate - rust-binaries-for-sysgo - go-binaries-for-sysgo - # IN-MEMORY (base gate) - kona/op-reth + # IN-MEMORY - kona/op-reth - op-acceptance-tests: name: memory-all-kona-op-reth - gate: "base" l2_cl_kind: kona-node l2_el_kind: op-reth no_output_timeout: 120m diff --git a/op-acceptance-tests/gates/base.txt b/op-acceptance-tests/gates/base.txt deleted file mode 100644 index 6f73a09ba864a..0000000000000 --- a/op-acceptance-tests/gates/base.txt +++ /dev/null @@ -1,12 +0,0 @@ -./op-acceptance-tests/tests/base -./op-acceptance-tests/tests/base/deposit -./op-acceptance-tests/tests/base/chain -./op-acceptance-tests/tests/ecotone -./op-acceptance-tests/tests/fjord -./op-acceptance-tests/tests/isthmus -./op-acceptance-tests/tests/isthmus/operator_fee -./op-acceptance-tests/tests/isthmus/withdrawal_root -./op-acceptance-tests/tests/isthmus/erc20_bridge -./op-acceptance-tests/tests/isthmus/pectra -./op-acceptance-tests/tests/jovian/bpo2 -./op-acceptance-tests/tests/jovian/pectra diff --git a/op-acceptance-tests/justfile b/op-acceptance-tests/justfile index d06c358a30b90..c1a55bc15b182 100644 --- a/op-acceptance-tests/justfile +++ b/op-acceptance-tests/justfile @@ -36,16 +36,11 @@ build-deps: test *args: build-deps cd {{REPO_ROOT}} && go test -count=1 -timeout 30m {{args}} -# Run acceptance tests, optionally filtered by a gate (reads packages from gates/.txt) -acceptance-test gate="": build-deps +# Run acceptance tests +acceptance-test: #!/usr/bin/env bash set -euo pipefail - GATE="{{gate}}" - if [[ -n "$GATE" ]]; then - echo -e "DEVNET: in-memory, GATE: $GATE\n" - else - echo -e "DEVNET: in-memory, MODE: all tests\n" - fi + echo -e "DEVNET: in-memory\n" LOG_LEVEL="$(echo "${LOG_LEVEL:-info}" | grep -E '^(debug|info|warn|error)$' || echo 'info')" echo "LOG_LEVEL: $LOG_LEVEL" @@ -70,7 +65,7 @@ acceptance-test gate="": build-deps JOBS="${ACCEPTANCE_TEST_JOBS:-$DEFAULT_JOBS}" TEST_PARALLEL="${ACCEPTANCE_TEST_PARALLEL:-$DEFAULT_TEST_PARALLEL}" - TEST_TIMEOUT="${ACCEPTANCE_TEST_TIMEOUT:-2h}" + TEST_TIMEOUT="${ACCEPTANCE_TEST_TIMEOUT:-30m}" echo "CPU COUNT: $CPU_COUNT" echo "PACKAGE JOBS: $JOBS" echo "TEST PARALLELISM: $TEST_PARALLEL" @@ -96,25 +91,7 @@ acceptance-test gate="": build-deps grep -RIn "MarkFlaky(" ./op-acceptance-tests/tests > "$FLAKY_REPORT" || true fi - # Determine which packages to test - if [[ -n "$GATE" ]]; then - GATE_FILE="{{REPO_ROOT}}/op-acceptance-tests/gates/${GATE}.txt" - if [[ ! -f "$GATE_FILE" ]]; then - echo "error: gate file not found: $GATE_FILE" >&2 - exit 1 - fi - # Read non-empty, non-comment lines from the gate file - TEST_PACKAGES=() - while IFS= read -r line; do - line="${line%%#*}" # strip comments - line="${line// /}" # strip whitespace - [[ -z "$line" ]] && continue - TEST_PACKAGES+=("$line") - done < "$GATE_FILE" - echo "Gate '$GATE': ${#TEST_PACKAGES[@]} packages" - else - TEST_PACKAGES=("./op-acceptance-tests/tests/...") - fi + TEST_PACKAGES=("./op-acceptance-tests/tests/...") set +e LOG_LEVEL="${LOG_LEVEL}" {{REPO_ROOT}}/ops/scripts/gotestsum-split.sh \ diff --git a/op-acceptance-tests/tests/base/conductor/leadership_transfer_test.go b/op-acceptance-tests/tests/base/conductor/leadership_transfer_test.go index 9f396a6e3d0f1..769c678736699 100644 --- a/op-acceptance-tests/tests/base/conductor/leadership_transfer_test.go +++ b/op-acceptance-tests/tests/base/conductor/leadership_transfer_test.go @@ -10,6 +10,7 @@ import ( "github.com/ethereum-optimism/optimism/op-devstack/devtest" "github.com/ethereum-optimism/optimism/op-devstack/dsl" "github.com/ethereum-optimism/optimism/op-devstack/presets" + "github.com/ethereum-optimism/optimism/op-devstack/sysgo" "github.com/ethereum-optimism/optimism/op-service/testlog" "github.com/ethereum/go-ethereum/log" "github.com/stretchr/testify/require" @@ -23,6 +24,12 @@ type conductorWithInfo struct { // TestConductorLeadershipTransfer checks if the leadership transfer works correctly on the conductors func TestConductorLeadershipTransfer(gt *testing.T) { t := devtest.ParallelT(gt) + // Example error with kona-node: + // + // --- FAIL: TestConductorLeadershipTransfer (63.04s) + // panic: interface conversion: sysgo.L2CLNode is *sysgo.KonaNode, not *sysgo.OpNode [recovered] + // panic: interface conversion: sysgo.L2CLNode is *sysgo.KonaNode, not *sysgo.OpNode + sysgo.SkipOnKonaNode(t, "not supported") logger := testlog.Logger(t, log.LevelInfo).With("Test", "TestConductorLeadershipTransfer") sys := presets.NewMinimalWithConductors(t) diff --git a/op-acceptance-tests/tests/depreqres/reqressyncdisabled/divergence/divergence_test.go b/op-acceptance-tests/tests/depreqres/reqressyncdisabled/divergence/divergence_test.go index 6a50903502673..c4c2e604e0f14 100644 --- a/op-acceptance-tests/tests/depreqres/reqressyncdisabled/divergence/divergence_test.go +++ b/op-acceptance-tests/tests/depreqres/reqressyncdisabled/divergence/divergence_test.go @@ -7,6 +7,7 @@ import ( "github.com/ethereum-optimism/optimism/op-acceptance-tests/tests/depreqres/common" "github.com/ethereum-optimism/optimism/op-devstack/devtest" "github.com/ethereum-optimism/optimism/op-devstack/presets" + "github.com/ethereum-optimism/optimism/op-devstack/sysgo" "github.com/ethereum-optimism/optimism/op-node/rollup/sync" "github.com/ethereum-optimism/optimism/op-service/eth" "github.com/ethereum-optimism/optimism/op-supervisor/supervisor/types" @@ -19,6 +20,15 @@ import ( // the corresponding block, and therefore does not serve it. func TestCLELDivergence(gt *testing.T) { t := devtest.ParallelT(gt) + // Example error with kona-node: + // + // assertions.go:387: ERROR[03-31|10:19:50.846] + // assertions.go:387: Error Trace: /Users/josh/repos/optimism/op-acceptance-tests/tests/depreqres/reqressyncdisabled/divergence/divergence_test.go:34 + // assertions.go:387: Error: Not equal: + // assertions.go:387: expected: 0x1 + // assertions.go:387: actual : 0x0 + // assertions.go:387: Test: TestCLELDivergence + sysgo.SkipOnKonaNode(t, "not supported") sys := presets.NewSingleChainMultiNodeWithoutP2PWithoutCheck(t, common.ReqRespSyncDisabledOpts(sync.ELSync)...) require := t.Require() l := t.Logger() diff --git a/op-acceptance-tests/tests/flashblocks/flashblocks_stream_test.go b/op-acceptance-tests/tests/flashblocks/flashblocks_stream_test.go index e29196631d08d..ea8e012c930e9 100644 --- a/op-acceptance-tests/tests/flashblocks/flashblocks_stream_test.go +++ b/op-acceptance-tests/tests/flashblocks/flashblocks_stream_test.go @@ -11,6 +11,7 @@ import ( "github.com/ethereum-optimism/optimism/op-devstack/devtest" "github.com/ethereum-optimism/optimism/op-devstack/presets" + "github.com/ethereum-optimism/optimism/op-devstack/sysgo" "github.com/ethereum-optimism/optimism/op-service/log/logfilter" "github.com/ethereum-optimism/optimism/op-service/logmods" "github.com/ethereum/go-ethereum/log" @@ -22,6 +23,23 @@ const maxExpectedFlashblocks = 20 // TestFlashblocksStream checks we can connect to the flashblocks stream across multiple CL backends. func TestFlashblocksStream(gt *testing.T) { t := devtest.ParallelT(gt) + // Example error with kona-node: + // + // assertions.go:387: ERROR[03-30|22:44:52.250] + // assertions.go:387: Error Trace: /home/circleci/project/op-devstack/sysgo/l2_cl_kona.go:99 + // assertions.go:387: /home/circleci/project/op-devstack/sysgo/mixed_runtime.go:456 + // assertions.go:387: /home/circleci/project/op-devstack/sysgo/singlechain_build.go:182 + // assertions.go:387: /home/circleci/project/op-devstack/sysgo/singlechain_build.go:276 + // assertions.go:387: /home/circleci/project/op-devstack/sysgo/singlechain_flashblocks.go:36 + // assertions.go:387: /home/circleci/project/op-devstack/sysgo/singlechain_runtime.go:105 + // assertions.go:387: /home/circleci/project/op-devstack/sysgo/singlechain_flashblocks.go:53 + // assertions.go:387: /home/circleci/project/op-devstack/presets/flashblocks.go:43 + // assertions.go:387: /home/circleci/project/op-acceptance-tests/tests/flashblocks/flashblocks_stream_test.go:26 + // assertions.go:387: Error: Received unexpected error: + // assertions.go:387: context deadline exceeded + // assertions.go:387: Test: TestFlashblocksStream + // assertions.go:387: Messages: need user RPC + sysgo.SkipOnKonaNode(t, "not supported (fail to get user rpc)") logger := t.Logger() sys := presets.NewSingleChainWithFlashblocks(t) filterHandler, ok := logmods.FindHandler[logfilter.FilterHandler](logger.Handler()) diff --git a/op-acceptance-tests/tests/flashblocks/flashblocks_transfer_test.go b/op-acceptance-tests/tests/flashblocks/flashblocks_transfer_test.go index a612a65e60fc3..76fad22b19398 100644 --- a/op-acceptance-tests/tests/flashblocks/flashblocks_transfer_test.go +++ b/op-acceptance-tests/tests/flashblocks/flashblocks_transfer_test.go @@ -10,6 +10,7 @@ import ( "github.com/ethereum-optimism/optimism/op-devstack/devtest" "github.com/ethereum-optimism/optimism/op-devstack/presets" + "github.com/ethereum-optimism/optimism/op-devstack/sysgo" "github.com/ethereum-optimism/optimism/op-service/eth" "github.com/ethereum-optimism/optimism/op-service/txplan" "github.com/stretchr/testify/require" @@ -32,6 +33,23 @@ type timedMessage struct { // - That Flashblock's time in nanoseconds must be before the approximated transaction confirmation time recorded previously. func TestFlashblocksTransfer(gt *testing.T) { t := devtest.ParallelT(gt) + // Example error with kona-node: + // + // assertions.go:387: ERROR[03-30|22:44:52.250] + // assertions.go:387: Error Trace: /op-devstack/sysgo/l2_cl_kona.go:99 + // assertions.go:387: /op-devstack/sysgo/mixed_runtime.go:456 + // assertions.go:387: /op-devstack/sysgo/singlechain_build.go:182 + // assertions.go:387: /op-devstack/sysgo/singlechain_build.go:276 + // assertions.go:387: /op-devstack/sysgo/singlechain_flashblocks.go:36 + // assertions.go:387: /op-devstack/sysgo/singlechain_runtime.go:105 + // assertions.go:387: /op-devstack/sysgo/singlechain_flashblocks.go:53 + // assertions.go:387: /op-devstack/presets/flashblocks.go:43 + // assertions.go:387: /op-acceptance-tests/tests/flashblocks/flashblocks_transfer_test.go:38 + // assertions.go:387: Error: Received unexpected error: + // assertions.go:387: context deadline exceeded + // assertions.go:387: Test: TestFlashblocksTransfer + // assertions.go:387: Messages: need user RPC + sysgo.SkipOnKonaNode(t, "not supported (fail to get user rpc)") logger := t.Logger() tracer := t.Tracer() ctx := t.Ctx() diff --git a/op-acceptance-tests/tests/proofs/cannon/step_test.go b/op-acceptance-tests/tests/proofs/cannon/step_test.go index addf858771d5c..f358e51643523 100644 --- a/op-acceptance-tests/tests/proofs/cannon/step_test.go +++ b/op-acceptance-tests/tests/proofs/cannon/step_test.go @@ -5,12 +5,14 @@ import ( "github.com/ethereum-optimism/optimism/op-devstack/devtest" "github.com/ethereum-optimism/optimism/op-devstack/dsl/proofs" + "github.com/ethereum-optimism/optimism/op-devstack/sysgo" "github.com/ethereum-optimism/optimism/op-service/eth" "github.com/ethereum-optimism/optimism/op-supervisor/supervisor/types" ) func TestExecuteStep_Cannon(gt *testing.T) { t := devtest.ParallelT(gt) + sysgo.SkipOnOpReth(t, "not supported (timeout)") sys := newSystem(t) l1User := sys.FunderL1.NewFundedEOA(eth.ThousandEther) @@ -28,6 +30,19 @@ func TestExecuteStep_Cannon(gt *testing.T) { func TestExecuteStep_CannonKona(gt *testing.T) { t := devtest.ParallelT(gt) + // Example error with kona-node: + // + // assertions.go:387: ERROR[03-31|10:33:38.442] + // assertions.go:387: Error Trace: /optimism/op-devstack/dsl/proofs/game_helper.go:263 + // assertions.go:387: /optimism/op-devstack/dsl/proofs/game_helper.go:196 + // assertions.go:387: /optimism/op-devstack/dsl/proofs/fault_dispute_game.go:124 + // assertions.go:387: /optimism/op-acceptance-tests/tests/proofs/cannon/step_test.go:40 + // assertions.go:387: /.local/share/mise/installs/go/1.24.13/src/runtime/asm_arm64.s:1223 + // assertions.go:387: Error: Received unexpected error: + // assertions.go:387: failed to get safe head at L1 block 0xe926b953777546729b2607011590354a6a42dc35e36b7ea68c9dd4b470261541:6: not found + // assertions.go:387: Test: TestExecuteStep_CannonKona + // assertions.go:387: Messages: Failed to get honest root claim + sysgo.SkipOnKonaNode(t, "not supported") sys := newSystem(t) l1User := sys.FunderL1.NewFundedEOA(eth.ThousandEther) diff --git a/op-acceptance-tests/tests/safeheaddb_clsync/safeheaddb_test.go b/op-acceptance-tests/tests/safeheaddb_clsync/safeheaddb_test.go index da441258a0ef3..8699482b0b08b 100644 --- a/op-acceptance-tests/tests/safeheaddb_clsync/safeheaddb_test.go +++ b/op-acceptance-tests/tests/safeheaddb_clsync/safeheaddb_test.go @@ -13,6 +13,18 @@ import ( func TestPreserveDatabaseOnCLResync(gt *testing.T) { t := devtest.ParallelT(gt) + // Example error with op-reth: + // + // assertions.go:387: ERROR[03-31|09:40:50.281] + // assertions.go:387: Error Trace: /optimism/op-devstack/sysgo/l2_el_p2p_util.go:61 + // assertions.go:387: /optimism/op-devstack/dsl/l2_el.go:281 + // assertions.go:387: /optimism/op-acceptance-tests/tests/safeheaddb_clsync/safeheaddb_test.go:42 + // assertions.go:387: Error: Received unexpected error: + // assertions.go:387: context deadline exceeded + // assertions.go:387: Test: TestPreserveDatabaseOnCLResync + // assertions.go:387: Messages: The peer was not connected + // assertions.go:387: + sysgo.SkipOnOpReth(t, "not supported (peering issue)") sys := presets.NewSingleChainMultiNode(t, presets.WithGlobalL2CLOption(sysgo.L2CLOptionFn(func(p devtest.T, _ sysgo.ComponentTarget, cfg *sysgo.L2CLConfig) { cfg.SequencerSyncMode = sync.CLSync diff --git a/op-acceptance-tests/tests/safeheaddb_elsync/safeheaddb_test.go b/op-acceptance-tests/tests/safeheaddb_elsync/safeheaddb_test.go index 5ddfe16c5d09a..3ed7357687c8d 100644 --- a/op-acceptance-tests/tests/safeheaddb_elsync/safeheaddb_test.go +++ b/op-acceptance-tests/tests/safeheaddb_elsync/safeheaddb_test.go @@ -22,6 +22,18 @@ func newSingleChainMultiNodeELSync(t devtest.T) *presets.SingleChainMultiNode { func TestTruncateDatabaseOnELResync(gt *testing.T) { t := devtest.ParallelT(gt) + // Example error with op-reth: + // + // assertions.go:387: ERROR[03-31|09:41:28.788] + // assertions.go:387: Error Trace: /optimism/op-devstack/sysgo/l2_el_p2p_util.go:61 + // assertions.go:387: /optimism/op-devstack/dsl/l2_el.go:281 + // assertions.go:387: /optimism/op-acceptance-tests/tests/safeheaddb_elsync/safeheaddb_test.go:44 + // assertions.go:387: Error: Received unexpected error: + // assertions.go:387: dial tcp 127.0.0.1:55994: i/o timeout + // assertions.go:387: Test: TestTruncateDatabaseOnELResync + // assertions.go:387: Messages: The peer was not connected + // assertions.go:387: + sysgo.SkipOnOpReth(t, "not supported (peering issue)") sys := newSingleChainMultiNodeELSync(t) dsl.CheckAll(t, @@ -51,6 +63,16 @@ func TestTruncateDatabaseOnELResync(gt *testing.T) { func TestNotTruncateDatabaseOnRestartWithExistingDatabase(gt *testing.T) { t := devtest.ParallelT(gt) + // Example error with kona-node: + // + // assertions.go:387: ERROR[03-31|10:35:59.154] + // assertions.go:387: Error Trace: /Users/josh/repos/optimism/op-devstack/dsl/safedb.go:22 + // assertions.go:387: /Users/josh/repos/optimism/op-devstack/dsl/l2_cl.go:432 + // assertions.go:387: /Users/josh/repos/optimism/op-acceptance-tests/tests/safeheaddb_elsync/safeheaddb_test.go:74 + // assertions.go:387: Error: Expected value not to be nil. + // assertions.go:387: Test: TestNotTruncateDatabaseOnRestartWithExistingDatabase + // assertions.go:387: Messages: no safe head data available at L1 block 4 + sysgo.SkipOnKonaNode(t, "not supported") sys := newSingleChainMultiNodeELSync(t) dsl.CheckAll(t, diff --git a/op-acceptance-tests/tests/sync/clsync/gap_clp2p/sync_test.go b/op-acceptance-tests/tests/sync/clsync/gap_clp2p/sync_test.go index d56828d53d062..fb16729e4feab 100644 --- a/op-acceptance-tests/tests/sync/clsync/gap_clp2p/sync_test.go +++ b/op-acceptance-tests/tests/sync/clsync/gap_clp2p/sync_test.go @@ -4,6 +4,7 @@ import ( "testing" "github.com/ethereum-optimism/optimism/op-devstack/devtest" + "github.com/ethereum-optimism/optimism/op-devstack/sysgo" "github.com/ethereum-optimism/optimism/op-service/eth" "github.com/ethereum-optimism/optimism/op-supervisor/supervisor/types" ) @@ -11,6 +12,18 @@ import ( // TestSyncAfterInitialELSync tests that blocks received out of order would be processed in order when running in CL sync mode. Note that this is not going to happen when running in EL sync mode, which relies on healthy ELP2P, something that is disabled in this test. func TestSyncAfterInitialELSync(gt *testing.T) { t := devtest.ParallelT(gt) + // Example error with kona-node: + // + // assertions.go:387: ERROR[03-31|10:38:08.992] + // "\n\tError Trace:\t/optimism/op-devstack/dsl/l2_el.go:192\n\t + // \t\t\t\t/optimism/op-acceptance-tests/tests/sync/clsync/gap_clp2p/sync_test.go:46 + // \n\tError: + // \tReceived unexpected error:\n\t + // \toperation failed permanently after 2 attempts: expected head for label=latest to advance to target=5, + // but got current=2 + // \tTest: \tTestSyncAfterInitialELSync\n" + + sysgo.SkipOnKonaNode(t, "not supported") sys := newGapCLP2PSystem(t) require := t.Require() diff --git a/op-acceptance-tests/tests/sync/elsync/gap_clp2p/sync_test.go b/op-acceptance-tests/tests/sync/elsync/gap_clp2p/sync_test.go index 2dd083f7cee1f..19954dfe819aa 100644 --- a/op-acceptance-tests/tests/sync/elsync/gap_clp2p/sync_test.go +++ b/op-acceptance-tests/tests/sync/elsync/gap_clp2p/sync_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/ethereum-optimism/optimism/op-devstack/devtest" + "github.com/ethereum-optimism/optimism/op-devstack/sysgo" "github.com/ethereum-optimism/optimism/op-service/eth" "github.com/ethereum-optimism/optimism/op-supervisor/supervisor/types" ) @@ -46,6 +47,16 @@ func TestReachUnsafeTipByAppendingUnsafePayload(gt *testing.T) { // while maintaining correct Engine API semantics. func TestCLUnsafeNotRewoundOnInvalidDuringELSync(gt *testing.T) { t := devtest.ParallelT(gt) + // Example error with op-reth: + // + // assertions.go:387: ERROR[03-31|09:41:58.089] + // assertions.go:387: Error Trace: /optimism/op-devstack/dsl/l2_cl.go:279 + // assertions.go:387: /optimism/op-acceptance-tests/tests/sync/elsync/gap_clp2p/sync_test.go:96 + // assertions.go:387: Error: Received unexpected error: + // assertions.go:387: expected head not to advance: unsafe + // assertions.go:387: Test: TestCLUnsafeNotRewoundOnInvalidDuringELSync + // assertions.go:387: + sysgo.SkipOnOpReth(t, "not supported") sys := newGapCLP2PSystem(t) logger := t.Logger() require := t.Require() diff --git a/op-acceptance-tests/tests/sync/elsync/gap_elp2p/sync_test.go b/op-acceptance-tests/tests/sync/elsync/gap_elp2p/sync_test.go index ae2a15255e7c4..574c69be0ad63 100644 --- a/op-acceptance-tests/tests/sync/elsync/gap_elp2p/sync_test.go +++ b/op-acceptance-tests/tests/sync/elsync/gap_elp2p/sync_test.go @@ -4,6 +4,7 @@ import ( "testing" "github.com/ethereum-optimism/optimism/op-devstack/devtest" + "github.com/ethereum-optimism/optimism/op-devstack/sysgo" "github.com/ethereum-optimism/optimism/op-service/eth" "github.com/ethereum-optimism/optimism/op-supervisor/supervisor/types" "github.com/ethereum/go-ethereum" @@ -55,6 +56,17 @@ import ( // - With ELP2P enabled, repeated FCU attempts eventually validate and advance the canonical chain. func TestL2ELP2PCanonicalChainAdvancedByFCU(gt *testing.T) { t := devtest.ParallelT(gt) + // Example error with op-reth: + // + // assertions.go:387: ERROR[03-31|09:58:15.522] + // assertions.go:387: Error Trace: /optimism/op-devstack/dsl/l2_el.go:64 + // assertions.go:387: /optimism/op-acceptance-tests/tests/sync/elsync/gap_elp2p/sync_test.go:111 + // assertions.go:387: Error: Received unexpected error: + // assertions.go:387: failed to determine block-hash of hash 0x8b94830f261ef4568bc2ba248f52b27f9a7c366e8157794c93bdd05ca4735564, could not get payload: not found + // assertions.go:387: Test: TestL2ELP2PCanonicalChainAdvancedByFCU + // assertions.go:387: Messages: block not found using block hash + // assertions.go:387: + sysgo.SkipOnOpReth(t, "not supported") sys := newGapELP2PSystem(t) require := t.Require() logger := t.Logger() @@ -393,6 +405,14 @@ func TestSafeDoesNotAdvanceWhenUnsafeIsSyncing_NoELP2P(gt *testing.T) { // invalid payloads—whether rejected at the CL or EL—do not advance the chain. func TestInvalidPayloadThroughCLP2P(gt *testing.T) { t := devtest.ParallelT(gt) + // Example error with kona-node: + // + // assertions.go:387: ERROR[03-31|10:42:03.034] + // assertions.go:387: Error Trace: /Users/josh/repos/optimism/op-acceptance-tests/tests/sync/elsync/gap_elp2p/sync_test.go:436 + // assertions.go:387: Error: An error is expected but got nil. + // assertions.go:387: Test: TestInvalidPayloadThroughCLP2P + // assertions.go:387: + sysgo.SkipOnKonaNode(t, "not supported") sys := newGapELP2PSystem(t) logger := t.Logger() require := t.Require() diff --git a/op-acceptance-tests/tests/sync/elsync/reorg/sync_test.go b/op-acceptance-tests/tests/sync/elsync/reorg/sync_test.go index a9cf41af7cbd7..325e256c30de2 100644 --- a/op-acceptance-tests/tests/sync/elsync/reorg/sync_test.go +++ b/op-acceptance-tests/tests/sync/elsync/reorg/sync_test.go @@ -5,6 +5,7 @@ import ( "time" "github.com/ethereum-optimism/optimism/op-devstack/devtest" + "github.com/ethereum-optimism/optimism/op-devstack/sysgo" "github.com/ethereum-optimism/optimism/op-service/eth" "github.com/ethereum-optimism/optimism/op-supervisor/supervisor/types" "github.com/ethereum-optimism/optimism/op-test-sequencer/sequencer/seqtypes" @@ -18,6 +19,15 @@ import ( // 4. CLP2P is restored, the verifier backfills and the unsafe gap is closed. func TestUnsafeGapFillAfterSafeReorg(gt *testing.T) { t := devtest.ParallelT(gt) + // Example error with kona-node: + // + // assertions.go:387: ERROR[03-30|22:17:00.549] + // assertions.go:387: Error Trace: /op-devstack/dsl/l2_el.go:204 + // assertions.go:387: /op-acceptance-tests/tests/sync/elsync/reorg/sync_test.go:78 + // assertions.go:387: Error: Received unexpected error: + // assertions.go:387: operation failed permanently after 30 attempts: expected head to reorg 0xae5a516a6654d4ee6a2edfb9a8e2db12106991b1a29fbb3953dd5afb8a60914e:12, but got 0xae5a516a6654d4ee6a2edfb9a8e2db12106991b1a29fbb3953dd5afb8a60914e:12 + // assertions.go:387: Test: TestUnsafeGapFillAfterSafeReorg + sysgo.SkipOnKonaNode(t, "not supported (timeout)") sys := newReorgSystem(t) require := t.Require() logger := t.Logger() @@ -95,6 +105,15 @@ func TestUnsafeGapFillAfterSafeReorg(gt *testing.T) { // 4. Verifier then backfills and closes the unsafe gap once reconnected via CLP2P. func TestUnsafeGapFillAfterUnsafeReorg_RestartL2CL(gt *testing.T) { t := devtest.ParallelT(gt) + // Example error with kona-node: + // + // assertions.go:387: ERROR[03-30|22:17:07.231] + // assertions.go:387: Error Trace: /optimism/op-devstack/dsl/l2_el.go:204 + // assertions.go:387: /optimism/op-acceptance-tests/tests/sync/elsync/reorg/sync_test.go:211 + // assertions.go:387: Error: Received unexpected error: + // assertions.go:387: operation failed permanently after 30 attempts: expected head to reorg 0x893d77533b0ff9b37a92090679bf256d987b4535f06186ec71f29e68ddccd9a5:14, but got 0x893d77533b0ff9b37a92090679bf256d987b4535f06186ec71f29e68ddccd9a5:14 + // assertions.go:387: Test: TestUnsafeGapFillAfterUnsafeReorg_RestartL2CL + sysgo.SkipOnKonaNode(t, "not supported (timeout)") sys := newReorgSystem(t) require := t.Require() logger := t.Logger() @@ -217,6 +236,15 @@ func TestUnsafeGapFillAfterUnsafeReorg_RestartL2CL(gt *testing.T) { // 4. CLP2P is restored Verifier, the verifier backfills and the unsafe gap is closed. func TestUnsafeGapFillAfterUnsafeReorg_RestartCLP2P(gt *testing.T) { t := devtest.ParallelT(gt) + // Example error with kona-node: + // + // assertions.go:387: ERROR[03-31|11:15:39.398] + // assertions.go:387: Error Trace: /optimism/op-devstack/dsl/l2_el.go:204 + // assertions.go:387: /optimism/op-acceptance-tests/tests/sync/elsync/reorg/sync_test.go:356 + // assertions.go:387: Error: Received unexpected error: + // assertions.go:387: operation failed permanently after 30 attempts: expected head to reorg 0x166970054ad16ad090210e5d1045538eeccd2afd88ea991b010de026d0106870:18, but got 0x166970054ad16ad090210e5d1045538eeccd2afd88ea991b010de026d0106870:18 + // assertions.go:387: Test: TestUnsafeGapFillAfterUnsafeReorg_RestartCLP2P + sysgo.SkipOnKonaNode(t, "not supported (timeout)") sys := newReorgSystem(t) require := t.Require() logger := t.Logger() diff --git a/op-acceptance-tests/tests/sync/follow_l2/sync_test.go b/op-acceptance-tests/tests/sync/follow_l2/sync_test.go index f3e33c476d995..42ca10f62caa6 100644 --- a/op-acceptance-tests/tests/sync/follow_l2/sync_test.go +++ b/op-acceptance-tests/tests/sync/follow_l2/sync_test.go @@ -6,6 +6,7 @@ import ( "github.com/ethereum-optimism/optimism/op-devstack/devtest" "github.com/ethereum-optimism/optimism/op-devstack/dsl" + "github.com/ethereum-optimism/optimism/op-devstack/sysgo" "github.com/ethereum-optimism/optimism/op-service/eth" "github.com/ethereum-optimism/optimism/op-supervisor/supervisor/types" "github.com/ethereum-optimism/optimism/op-test-sequencer/sequencer/seqtypes" @@ -14,6 +15,19 @@ import ( func TestFollowL2_Safe_Finalized_CurrentL1(gt *testing.T) { t := devtest.ParallelT(gt) + // Example error with kona-node: + // + // assertions.go:387: ERROR[03-31|11:33:11.255] + // assertions.go:387: Error Trace: /optimism/op-devstack/sysgo/singlechain_variants.go:143 + // assertions.go:387: /optimism/op-devstack/sysgo/singlechain_variants.go:53 + // assertions.go:387: /optimism/op-devstack/presets/singlechain_twoverifiers.go:24 + // assertions.go:387: /optimism/op-acceptance-tests/tests/sync/follow_l2/setup_test.go:24 + // assertions.go:387: /optimism/op-acceptance-tests/tests/sync/follow_l2/sync_test.go:18 + // assertions.go:387: Error: Should be true + // assertions.go:387: Test: TestFollowL2_Safe_Finalized_CurrentL1 + // assertions.go:387: Messages: single-chain test sequencer requires an op-node CL node + sysgo.SkipOnKonaNode(t, "not supported") + sysgo.FlakyOnOpReth(t, "timeouts in merge queue but not locally") sys := newSingleChainTwoVerifiersFollowL2(t) logger := t.Logger() @@ -56,6 +70,18 @@ func TestFollowL2_Safe_Finalized_CurrentL1(gt *testing.T) { func TestFollowL2_ReorgRecovery(gt *testing.T) { t := devtest.ParallelT(gt) + // Example error with kona-node: + // + // assertions.go:387: ERROR[03-31|11:31:11.567] + // assertions.go:387: Error Trace: /optimism/op-devstack/sysgo/singlechain_variants.go:143 + // assertions.go:387: /optimism/op-devstack/sysgo/singlechain_variants.go:53 + // assertions.go:387: /optimism/op-devstack/presets/singlechain_twoverifiers.go:24 + // assertions.go:387: /optimism/op-acceptance-tests/tests/sync/follow_l2/setup_test.go:24 + // assertions.go:387: /optimism/op-acceptance-tests/tests/sync/follow_l2/sync_test.go:60 + // assertions.go:387: Error: Should be true + // assertions.go:387: Test: TestFollowL2_ReorgRecovery + // assertions.go:387: Messages: single-chain test sequencer requires an op-node CL node + sysgo.SkipOnKonaNode(t, "not supported") sys := newSingleChainTwoVerifiersFollowL2(t) require := t.Require() logger := t.Logger() @@ -133,6 +159,18 @@ func TestFollowL2_ReorgRecovery(gt *testing.T) { func TestFollowL2_WithoutCLP2P(gt *testing.T) { t := devtest.ParallelT(gt) + // Example error with kona-node: + // + // assertions.go:387: ERROR[03-31|11:27:57.797] + // assertions.go:387: Error Trace: /optimism/op-devstack/sysgo/singlechain_variants.go:143 + // assertions.go:387: /optimism/op-devstack/sysgo/singlechain_variants.go:53 + // assertions.go:387: /optimism/op-devstack/presets/singlechain_twoverifiers.go:24 + // assertions.go:387: /optimism/op-acceptance-tests/tests/sync/follow_l2/setup_test.go:24 + // assertions.go:387: /optimism/op-acceptance-tests/tests/sync/follow_l2/sync_test.go:136 + // assertions.go:387: Error: Should be true + // assertions.go:387: Test: TestFollowL2_WithoutCLP2P + // assertions.go:387: Messages: single-chain test sequencer requires an op-node CL nod + sysgo.SkipOnKonaNode(t, "not supported") sys := newSingleChainTwoVerifiersFollowL2(t) require := t.Require() logger := t.Logger() diff --git a/op-acceptance-tests/tests/sync/manual/sync_test.go b/op-acceptance-tests/tests/sync/manual/sync_test.go index 7b0bfdb9bf3f3..05c55ff73e920 100644 --- a/op-acceptance-tests/tests/sync/manual/sync_test.go +++ b/op-acceptance-tests/tests/sync/manual/sync_test.go @@ -14,6 +14,16 @@ import ( func TestVerifierManualSync(gt *testing.T) { t := devtest.ParallelT(gt) + // Example error using op-reth: + // + // assertions.go:387: ERROR[03-31|10:02:09.740] + // assertions.go:387: Error Trace: /optimism/op-devstack/dsl/l2_el.go:64 + // assertions.go:387: /optimism/op-acceptance-tests/tests/sync/manual/sync_test.go:53 + // assertions.go:387: Error: Received unexpected error: + // assertions.go:387: failed to determine block-hash of hash 0x2f6324eaff3942802eff5ddd47872544aa02182c222bf90042d5f715dfa72308, could not get payload: not found + // assertions.go:387: Test: TestVerifierManualSync + // assertions.go:387: Messages: block not found using block hash + sysgo.SkipOnOpReth(t, "not supported") // Disable ELP2P and Batcher sys := presets.NewSingleChainMultiNodeWithoutP2PWithoutCheck(t, diff --git a/op-acceptance-tests/tests/sync_tester/sync_tester_elsync_multi/sync_test.go b/op-acceptance-tests/tests/sync_tester/sync_tester_elsync_multi/sync_test.go index 8e9127f746772..bdf53ec3d4f63 100644 --- a/op-acceptance-tests/tests/sync_tester/sync_tester_elsync_multi/sync_test.go +++ b/op-acceptance-tests/tests/sync_tester/sync_tester_elsync_multi/sync_test.go @@ -29,6 +29,7 @@ func simpleWithSyncTesterOpts() []presets.Option { func TestMultiELSync(gt *testing.T) { t := devtest.ParallelT(gt) + sysgo.FlakyOnKonaNode(t, "fails in ci but passes locally") sys := presets.NewSimpleWithSyncTester(t, simpleWithSyncTesterOpts()...) require := t.Require() diff --git a/op-acceptance-tests/tests/sync_tester/sync_tester_hfs/sync_tester_hfs_test.go b/op-acceptance-tests/tests/sync_tester/sync_tester_hfs/sync_tester_hfs_test.go index 2264100c46c1d..547e9a557abd3 100644 --- a/op-acceptance-tests/tests/sync_tester/sync_tester_hfs/sync_tester_hfs_test.go +++ b/op-acceptance-tests/tests/sync_tester/sync_tester_hfs/sync_tester_hfs_test.go @@ -35,6 +35,15 @@ func simpleWithSyncTesterOpts() []presets.Option { func TestSyncTesterHardforks(gt *testing.T) { t := devtest.ParallelT(gt) + // Example error with op-reth: + // + // assertions.go:387: ERROR[03-31|10:08:04.055] + // assertions.go:387: Error Trace: /optimism/op-devstack/dsl/check.go:26 + // assertions.go:387: /optimism/op-acceptance-tests/tests/sync_tester/sync_tester_hfs/sync_tester_hfs_test.go:54 + // assertions.go:387: Error: Received unexpected error: + // assertions.go:387: operation failed permanently after 42 attempts: expected head to advance: unsafe + // assertions.go:387: Test: TestSyncTesterHardforks + sysgo.SkipOnOpReth(t, "not supported") sys := presets.NewSimpleWithSyncTester(t, simpleWithSyncTesterOpts()...) require := t.Require() diff --git a/op-devstack/sysgo/mixed_runtime.go b/op-devstack/sysgo/mixed_runtime.go index d3de1b786fb89..e90ef26a468ee 100644 --- a/op-devstack/sysgo/mixed_runtime.go +++ b/op-devstack/sysgo/mixed_runtime.go @@ -42,21 +42,11 @@ import ( type MixedL2ELKind string -const DevstackL2ELKindEnvVar = "DEVSTACK_L2EL_KIND" - const ( MixedL2ELOpGeth MixedL2ELKind = "op-geth" MixedL2ELOpReth MixedL2ELKind = "op-reth" ) -// SkipUnlessOpGeth skips the test when the L2 execution layer is op-reth -// (i.e. DEVSTACK_L2EL_KIND is not "op-geth"). -func SkipUnlessOpGeth(t devtest.T, reason string) { - if MixedL2ELKind(os.Getenv(DevstackL2ELKindEnvVar)) != MixedL2ELOpGeth { - t.Skipf("skipping on op-reth: %s", reason) - } -} - type MixedL2CLKind string const ( @@ -64,11 +54,37 @@ const ( MixedL2CLKona MixedL2CLKind = "kona-node" ) +// SkipOnOpReth skips the test when the L2 execution layer is op-reth +func SkipOnOpReth(t devtest.T, reason string) { + if devstackL2ELKind() == MixedL2ELOpReth { + t.Skipf("skipping on op-reth: %s", reason) + } +} + +// SkipOnKonaNode skips the test when the L2 consensus layer is kona-node +func SkipOnKonaNode(t devtest.T, reason string) { + if devstackL2CLKind() == MixedL2CLKona { + t.Skipf("skipping on kona-node: %s", reason) + } +} + +func FlakyOnOpReth(t devtest.T, reason string) { + if devstackL2ELKind() == MixedL2ELOpReth { + t.MarkFlaky(reason) + } +} + +func FlakyOnKonaNode(t devtest.T, reason string) { + if devstackL2CLKind() == MixedL2CLKona { + t.MarkFlaky(reason) + } +} + // devstackL2ELKind returns the L2 EL kind requested via the DEVSTACK_L2EL_KIND // environment variable. Returns the empty string when the variable is unset, // meaning "use the runtime's default". func devstackL2ELKind() MixedL2ELKind { - return MixedL2ELKind(os.Getenv(DevstackL2ELKindEnvVar)) + return MixedL2ELKind(os.Getenv("DEVSTACK_L2EL_KIND")) } // devstackL2CLKind returns the L2 CL kind requested via the DEVSTACK_L2CL_KIND diff --git a/rust/kona/tests/README.md b/rust/kona/tests/README.md index 77b7711123fcd..308ff4f21066d 100644 --- a/rust/kona/tests/README.md +++ b/rust/kona/tests/README.md @@ -46,13 +46,12 @@ just test-e2e-sysgo node node/common simple-kona To run acceptance tests for the Rust stack: ```bash -just acceptance-tests CL_TYPE EL_TYPE GATE +just acceptance-tests CL_TYPE EL_TYPE ``` Where: - `CL_TYPE`: Consensus layer type (`kona` or `op-node`) - `EL_TYPE`: Execution layer type (`op-reth` or `op-geth`) -- `GATE`: The gate to run (default: `jovian`) ### Other Recipes diff --git a/rust/kona/tests/justfile b/rust/kona/tests/justfile index e684d0f6f796a..495db0831ec97 100644 --- a/rust/kona/tests/justfile +++ b/rust/kona/tests/justfile @@ -21,7 +21,7 @@ build-kona PROFILE="release": #!/bin/bash cd {{SOURCE}}/.. && cargo build --bin kona-node --profile {{PROFILE}} -acceptance-tests CL_TYPE="kona-node" EL_TYPE="op-reth" GATE="jovian": +acceptance-tests CL_TYPE="kona-node" EL_TYPE="op-reth": #!/bin/bash if [ "{{CL_TYPE}}" = "kona-node" ] ; then echo "Building kona-node..." @@ -33,11 +33,10 @@ acceptance-tests CL_TYPE="kona-node" EL_TYPE="op-reth" GATE="jovian": just build-reth fi - just acceptance-tests-run {{CL_TYPE}} {{EL_TYPE}} {{GATE}} + just acceptance-tests-run {{CL_TYPE}} {{EL_TYPE}} # Run acceptance tests for the rust stack. By default runs the acceptance tests for kona-node with op-reth. -# Uses the jovian gate by default. -acceptance-tests-run CL_TYPE="kona-node" EL_TYPE="op-reth" GATE="jovian": +acceptance-tests-run CL_TYPE="kona-node" EL_TYPE="op-reth": #!/bin/bash if [ "{{CL_TYPE}}" = "kona-node" ] ; then echo "Running acceptance tests for kona-node" @@ -49,12 +48,12 @@ acceptance-tests-run CL_TYPE="kona-node" EL_TYPE="op-reth" GATE="jovian": export OP_RETH_EXEC_PATH="{{SOURCE}}/../../reth/target/debug/op-reth" fi - echo "Running acceptance tests for {{CL_TYPE}} with {{EL_TYPE}} on gate {{GATE}}..." + echo "Running acceptance tests for {{CL_TYPE}} with {{EL_TYPE}}..." export DEVSTACK_L2CL_KIND="{{CL_TYPE}}" export DEVSTACK_L2EL_KIND="{{EL_TYPE}}" export LOG_LEVEL="debug" - cd {{SOURCE}}/optimism/op-acceptance-tests && just acceptance-test "" "{{GATE}}" + cd {{SOURCE}}/optimism/op-acceptance-tests && just acceptance-test # Run the e2e tests for the sysgo orchestrator. # Builds the kona-node and op-reth binaries and runs the tests.