handle huge tx

.github/workflows/ci_zkevm.yml

@@ -291,6 +291,31 @@ jobs:
         run: sudo -E make test-realtime-e2e
         working-directory: test
 
+  test-hugetx-e2e:
+    strategy:
+      fail-fast: false
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install Foundry
+        uses: foundry-rs/foundry-toolchain@v1
+      - name: Make Foundry available to sudo
+        run: |
+          FOUNDRY_DIR=$(dirname $(which cast))
+          sudo ln -sf "$FOUNDRY_DIR/cast" /usr/local/bin/cast
+          sudo ln -sf "$FOUNDRY_DIR/forge" /usr/local/bin/forge
+          sudo ln -sf "$FOUNDRY_DIR/anvil" /usr/local/bin/anvil
+          sudo cast --version
+
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Build Docker
+        run: make build-docker
+
+      - name: Test
+        run: sudo -E make test-hugetx-e2e
+        working-directory: test
+
   # For X Layer
   check-chinese-chars:
     name: Check Chinese Characters

cmd/utils/flags_xlayer.go

@@ -108,6 +108,21 @@ var (
 		Usage: "EnableTimsort enable timsort to instead of built-in sorting",
 		Value: false,
 	}
+	TxPoolHugeTxThresholdRatio = cli.Uint64Flag{
+		Name:  "txpool.huge-tx-threshold-ratio",
+		Usage: "Percentage threshold of block gas limit to identify huge transactions (0-100)",
+		Value: 100,
+	}
+	TxPoolHugeTxQuotaRatio = cli.Uint64Flag{
+		Name:  "txpool.huge-tx-quota-ratio",
+		Usage: "Maximum percentage of block gas limit that can be consumed by huge transactions (0-100)",
+		Value: 100,
+	}
+	TxPoolIgnoreHugeTxQuotaInterval = cli.Uint64Flag{
+		Name:  "txpool.ignore-huge-tx-quota-interval",
+		Usage: "Block interval to ignore huge transaction quota restrictions. Set to 0 to disable huge tx quota entirely",
+		Value: 0,
+	}
 	// OkPay
 	OkPaySenderAccountsList = cli.StringFlag{
 		Name:  "okpay.sender-accounts-list",
@@ -602,6 +617,23 @@ func setTxPoolXLayer(ctx *cli.Context, cfg *ethconfig.DeprecatedTxPoolConfig) {
 	if ctx.IsSet(TxPoolEnableTimsort.Name) {
 		cfg.EnableTimsort = ctx.Bool(TxPoolEnableTimsort.Name)
 	}
+	if ctx.IsSet(TxPoolHugeTxThresholdRatio.Name) {
+		ratio := ctx.Uint64(TxPoolHugeTxThresholdRatio.Name)
+		if ratio > 100 {
+			panic(fmt.Sprintf("huge tx threshold ratio must be less than 100, but got %d", ratio))
+		}
+		cfg.HugeTxConfig.HugeTxThresholdRatio = ratio
+	}
+	if ctx.IsSet(TxPoolHugeTxQuotaRatio.Name) {
+		ratio := ctx.Uint64(TxPoolHugeTxQuotaRatio.Name)
+		if ratio > 100 {
+			panic(fmt.Sprintf("huge tx quota ratio must be less than 100, but got %d", ratio))
+		}
+		cfg.HugeTxConfig.HugeTxQuotaRatio = ratio
+	}
+	if ctx.IsSet(TxPoolIgnoreHugeTxQuotaInterval.Name) {
+		cfg.HugeTxConfig.IgnoreHugeTxQuotaInterval = ctx.Uint64(TxPoolIgnoreHugeTxQuotaInterval.Name)
+	}
 
 	// For OkPay
 	setOkPayXLayer(ctx, cfg)

eth/ethconfig/tx_pool.go

@@ -23,6 +23,15 @@ import (
 	"github.com/ledgerwatch/erigon-lib/txpool/txpoolcfg"
 )
 
+type HugeTxConfig struct {
+	HugeTxThresholdRatio      uint64
+	HugeTxQuotaRatio          uint64
+	IgnoreHugeTxQuotaInterval uint64
+
+	// for huge tx e2e test
+	HugeTxE2EYieldEnabled bool
+}
+
 // DeprecatedTxPoolConfig are the configuration parameters of the transaction pool.
 type DeprecatedTxPoolConfig struct {
 	Disable  bool
@@ -69,6 +78,8 @@ type DeprecatedTxPoolConfig struct {
 	FreeGasList []FreeGasInfo
 	// EnableTimsort is the switch to use timsort on the best slice of txpool
 	EnableTimsort bool // For X Layer, optimize the txpool
+	// HugeTxConfig is the config for huge tx
+	HugeTxConfig HugeTxConfig
 
 	// For X Layer, OkPay config
 	// OkPaySenderAccountsList is the list of OkPay sender accounts

eth/stagedsync/stage_mining_exec.go

@@ -206,7 +206,7 @@ func getNextTransactions(
 			remainingBlobGas = cfg.chainConfig.GetMaxBlobGasPerBlock() - *header.BlobGasUsed
 		}
 
-		if _, count, err = cfg.txPool2.YieldBest(amount, &txSlots, poolTx, executionAt, remainingGas, remainingBlobGas, alreadyYielded); err != nil {
+		if _, count, err = cfg.txPool2.YieldBest(amount, &txSlots, poolTx, executionAt, header.Number.Uint64(), remainingGas, remainingBlobGas, alreadyYielded); err != nil {
 			return err
 		}
 

test/Makefile

@@ -23,6 +23,9 @@ DOCKER_APOLLO_ADMINSERVICE := apollo-adminservice
 DOCKER_APOLLO_PORTAL := apollo-portal
 DOCKER_APOLLO_MYSQL := mysql
 
+# Configuration file for huge transaction tests
+HUGETX_CFG_FILE := config/apollo.seq.config.yaml
+
 RUN_DOCKER_BRIDGE_DB := $(DOCKER_COMPOSE) up -d $(DOCKER_BRIDGE_DB)
 RUN_DOCKER_BRIDGE_SERVICE := $(DOCKER_COMPOSE) up -d $(DOCKER_BRIDGE_SERVICE)
 RUN_DOCKER_BRIDGE_UI := $(DOCKER_COMPOSE) up -d $(DOCKER_BRIDGE_UI)
@@ -232,6 +235,36 @@ lint:
 	cd ../docs/endpoints && make check-doc
 	echo "Lint passed"
 
+PHONY: test-hugetx-e2e
+test-hugetx-e2e: stop
+	$(call set_seal_time,5s)
+	make apollo
+	trap '$(STOP) git checkout -- config/apollo.seq.config.yaml; rm -rf ./e2e/tmp' EXIT; MallocNanoZone=0 go test -tags hugetx -count=1 -failfast -race -v -p 1 -timeout 600s ./e2e/huge_tx_test.go
+	echo "HugeTx E2E tests passed"
+
+.PHONY: set_seal_time
+define set_seal_time
+	@file=$(HUGETX_CFG_FILE); \
+	key="zkevm.sequencer-block-seal-time"; \
+	key2="zkevm.sequencer-max-block-seal-time"; \
+	key3="zkevm.sequencer-batch-seal-time"; \
+	val="$(1)"; \
+	num=$${val%[a-zA-Z]*}; \
+	unit=$${val#$$num}; \
+	max="$$(($$num*2))$$unit"; \
+	batch="$$(($$num*3))$$unit"; \
+	if grep -q "^$$key:" $$file; then :; else echo "$$key: \"$$val\"" >> $$file; fi; \
+	if grep -q "^$$key2:" $$file; then :; else echo "$$key2: \"$$max\"" >> $$file; fi; \
+	if grep -q "^$$key3:" $$file; then :; else echo "$$key3: \"$$batch\"" >> $$file; fi; \
+	sed -i.bak \
+		-e "s/^$$key:.*/$$key: \"$$val\"/" \
+		-e "s/^$$key2:.*/$$key2: \"$$max\"/" \
+		-e "s/^$$key3:.*/$$key3: \"$$batch\"/" $$file; \
+	rm -f $$file.bak; \
+	echo "Set $$key=$$val, $$key2=$$max, $$key3=$$batch in $$file"; \
+	tail -n 15 $$file
+endef
+
 .PHONY: test-e2e
 test-e2e: stop all ## Runs group xlayer feature e2e tests checking race conditions
 	sleep 3

test/config/apollo.rpc.config.yaml

@@ -24,8 +24,8 @@ zkevm.datastream-version: 2
 
 log.console.verbosity: info
 
-zkevm.sequencer-block-seal-time: "400ms"
-zkevm.sequencer-batch-seal-time: "60s"
+zkevm.sequencer-block-seal-time: "6s"
+zkevm.sequencer-batch-seal-time: "12s"
 zkevm.sequencer-batch-sleep-duration: "0s"
 
 zkevm.data-stream-host: "localhost"
@@ -71,8 +71,6 @@ txpool.globalslots: 100000
 networkid: 195
 
 zkevm.pool-manager-url: http://xlayer-pool-manager:8545
-
-# http.methodratelimit: "{\"methods\":[\"eth_syncing\"],\"count\":10,\"bucket\":1}"
 #http.apikeys: |
 #  {"project":"project1","key":"944cd2a6939eb23053289d9b91d6c498","timeout":"2033-12-12","methods":["eth_syncing"],"count":5,"bucket":1}
 

test/config/apollo.seq.config.yaml

@@ -24,9 +24,9 @@ zkevm.datastream-version: 2
 
 log.console.verbosity: info
 
-zkevm.sequencer-block-seal-time: "400ms"
-zkevm.sequencer-max-block-seal-time: "400ms"
-zkevm.sequencer-batch-seal-time: "60s"
+zkevm.sequencer-block-seal-time: "1s"
+zkevm.sequencer-max-block-seal-time: "3s"
+zkevm.sequencer-batch-seal-time: "10s"
 zkevm.sequencer-batch-sleep-duration: "0s"
 
 zkevm.data-stream-host: "localhost"
@@ -69,7 +69,7 @@ db.read.concurrency: 20000
 txpool.globalslots: 1000000
 txpool.globalbasefeeslots: 1000000
 txpool.globalqueue: 1000000
-txpool.packbatchspeciallist : ["0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266", "0x8f8E2d6cF621f30e9a11309D6A56A876281Fd534", "0x3C44CdDdB6a900fa2b585dd299e03d12FA4293BC"]
+txpool.packbatchspeciallist : ["0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266", "0x3C44CdDdB6a900fa2b585dd299e03d12FA4293BC"]
 
 txpool.gaspricemultiple : 2
 txpool.blockedlist: ["0xdD2FD4581271e230360230F9337D5c0430Bf44C0"]
@@ -102,31 +102,33 @@ zkevm.seal-batch-immediately-on-overflow: true
 zkevm.sequencer-timeout-on-empty-tx-pool: 5ms
 zkevm.pre-run-address-list: [""]
 zkevm.block-info-concurrent: true
-# zkevm.standalone-smt-db: true  # true: enable split smt db, false: not split
-# zkevm.enable-async-commit: true
+zkevm.standalone-smt-db: true  # true: enable split smt db, false: not split
+zkevm.enable-async-commit: true
 
 # to reduce lock race in stress test
 txpool.enabletimsort: true
 zkevm.bulk-add-txs: true
 zkevm.enable-add-tx-notify: true
 zkevm.initial-batch.config: /usr/src/app/first-batch-config.json
 
-
 zkevm.verification-batch-delay: 2
 zkevm.skip-analysis-group-api: true
 # zkevm.analysis-group-nacos-urls: "url1:80,url2:80"
 # zkevm.analysis-group-nacos-namespace: "ns-global"
 # zkevm.analysis-group-service-name: "analysis-group"
 # zkevm.analysis-group-api-path: "api/v1/196/validHeight"
 
-# realtime.enable-flag: true
-# realtime.enable-subscribe-flag: false
+realtime.enable-flag: true
+realtime.enable-subscribe-flag: false
 realtime.cache-height-threshold: 10
 realtime.kafka-sync-bootstrap-servers: "xlayer-kafka:9092"
 realtime.kafka-sync-tx-topic: xlayer-tx
 realtime.kafka-sync-block-topic: xlayer-header
 realtime.kafka-sync-error-topic: xlayer-error
 realtime.kafka-sync-client-id: "xlayer-producer"
+
+bridgeIntercept.bridge-contract-address: "0x4B24266C13AFEf2bb60e2C69A4C08A482d81e3CA"
+bridgeIntercept.target-token-address: "0x5FbDB2315678afecb367f032d93F642f64180aa3"
 bridgeIntercept.max-bridge-amount: "1000000000000000000000"
-bridgeIntercept.whitelist-enabled: true
-bridgeIntercept.whitelist-addresses: "0x8f8E2d6cF621f30e9a11309D6A56A876281Fd534,0xDE282DC882bbB5100b8A24E30D38a2D5B3080c15"
+bridgeIntercept.whitelist-enabled: false
+bridgeIntercept.whitelist-addresses: "0x8f8E2d6cF621f30e9a11309D6A56A876281Fd534,0xDE282DC882bbB5100b8A24E30D38a2D5B3080c15"

test/docker-compose-apollo.yml

@@ -615,7 +615,6 @@ services:
 
   mysql:
     container_name: mysql
-    platform: linux/arm64
     image: mysql:8.0
     environment:
       - MYSQL_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD:-password}

test/docker-compose.yml

@@ -139,7 +139,7 @@ services:
       cdk-erigon --http.vhosts=* --http.corsdomain=* --ws --config=/usr/src/app/config.yaml
 
   xlayer-kafka:
-    image: bitnami/kafka:latest
+    image: bitnamilegacy/kafka:4.0.0-debian-12-r10
     container_name: xlayer-kafka
     ports:
       - 9095:9094