Update pruning logic: Exclude specific tables from moderate pruning

cmd/prune-mdbx-data/cmd/prune-chaindata/complete_deletion.go

@@ -370,12 +370,23 @@ func getPruneTables(allTables []string, level PruneLevel) []string {
 		"LogAddressIndex",          // Log address index
 	}
 
+	// Tables to exclude from moderate pruning
+	moderateExclusions := map[string]bool{
+		"BlockTransaction":       true,
+		"BlockTransactionLookup": true,
+		"LogTopicIndex":          true,
+		"LogAddressIndex":        true,
+		"CallFromIndex":          true,
+		"CallToIndex":            true,
+	}
+
 	switch level {
 	case PruneLevelModerate:
 		// Moderate: Only delete the tables that actually have data and are safe to remove
+		// Exclude specific tables that should be preserved in moderate mode
 		for _, table := range actualDataTables {
-			// Exclude tables that are critical or should be handled by partial pruning
-			if !critical[table] && contains(allTables, table) {
+			// Exclude tables that are critical, should be handled by partial pruning, or explicitly excluded
+			if !critical[table] && !moderateExclusions[table] && contains(allTables, table) {
 				toDelete = append(toDelete, table)
 			}
 		}

cmd/prune-mdbx-data/cmd/prune-chaindata/copy_truncate_restore.go

@@ -18,7 +18,7 @@ type KeyValueEntry struct {
 }
 
 // executeCopyTruncateRestore implements the optimized pruning strategy
-func executeCopyTruncateRestore(tx kv.RwTx, hermezDb *hermez_db.HermezDbReader, keepFromBatch, latestBatch, pruneBefore, genesisHeight uint64) (int, int, error) {
+func executeCopyTruncateRestore(tx kv.RwTx, hermezDb *hermez_db.HermezDbReader, keepFromBatch, latestBatch, pruneBefore uint64, config *MainConfig) (int, int, error) {
 	// Step 1: Identify all blocks in batches to keep
 	fmt.Printf("Step 1: Collecting blocks to preserve...\n")
 	var preserveBlocks []uint64
@@ -39,27 +39,27 @@ func executeCopyTruncateRestore(tx kv.RwTx, hermezDb *hermez_db.HermezDbReader,
 
 	if len(preserveBlocks) == 0 {
 		fmt.Printf("No blocks to preserve, will clear all tables\n")
-		return int(pruneBefore), 0, clearAllBatchTables(tx)
+		return int(pruneBefore), 0, clearAllBatchTables(tx, config.PruneLevel)
 	}
 
 	// Step 2: Copy data to preserve
 	fmt.Printf("Step 2: Copying data for %d blocks...\n", len(preserveBlocks))
-	preservedData, err := copyBlockData(tx, preserveBlocks, genesisHeight)
+	preservedData, err := copyBlockData(tx, preserveBlocks, config.GenesisBlockHeight, config.PruneLevel)
 	if err != nil {
 		fmt.Printf("Failed to copy data, falling back to old method\n")
 		return executeLegacyPruning(tx, hermezDb, pruneBefore)
 	}
 
 	// Step 3: Clear batch-related tables
 	fmt.Printf("Step 3: Clearing batch-related tables...\n")
-	err = clearBatchTables(tx)
+	err = clearBatchTables(tx, config.PruneLevel)
 	if err != nil {
 		return 0, 0, fmt.Errorf("failed to clear tables: %w", err)
 	}
 
 	// Step 4: Restore preserved data
 	fmt.Printf("Step 4: Restoring preserved data...\n")
-	err = restoreBlockData(tx, preservedData)
+	err = restoreBlockData(tx, preservedData, config.PruneLevel)
 	if err != nil {
 		return 0, 0, fmt.Errorf("failed to restore data: %w", err)
 	}
@@ -79,8 +79,36 @@ func executeCopyTruncateRestore(tx kv.RwTx, hermezDb *hermez_db.HermezDbReader,
 	return deletedBatches, deletedBlocks, nil
 }
 
+// getCopyTablesForPruneLevel returns the appropriate table lists based on pruning level
+func getCopyTablesForPruneLevel(pruneLevel PruneLevel) (simpleTables, compositeKeyTables []string) {
+	// Base tables for all pruning levels
+	simpleTables = []string{
+		"block_info_roots",
+		// Note: CanonicalHeader and hermez_blockBatches are EXCLUDED in Moderate mode
+		// These tables have dependencies with AccountChangeSet/StorageChangeSet
+		// and must remain intact to avoid MDBX_EKEYMISMATCH errors
+	}
+
+	compositeKeyTables = []string{
+		"Header",                            // block_num_u64 + hash
+		"BlockBody",                         // block_num_u64 + hash
+		"hermez_intermediate_tx_stateRoots", // l2blockno + txhash
+	}
+
+	// Add additional tables for aggressive pruning
+	if pruneLevel == PruneLevelAggressive {
+		simpleTables = append(simpleTables, "Receipt")
+		compositeKeyTables = append(compositeKeyTables,
+			"TxSender",       // block_num_u64 + blockHash
+			"TransactionLog", // block_num_u64 + txIndex + logIndex
+		)
+	}
+
+	return simpleTables, compositeKeyTables
+}
+
 // copyBlockData copies data for specified blocks from all relevant tables
-func copyBlockData(tx kv.RwTx, blockNos []uint64, genesisHeight uint64) ([]BlockData, error) {
+func copyBlockData(tx kv.RwTx, blockNos []uint64, genesisHeight uint64, pruneLevel PruneLevel) ([]BlockData, error) {
 	// Genesis protection: Always preserve genesis block to prevent genesis rewrite
 	hasGenesis := false
 	for _, blockNo := range blockNos {
@@ -96,23 +124,8 @@ func copyBlockData(tx kv.RwTx, blockNos []uint64, genesisHeight uint64) ([]Block
 
 	var preservedData []BlockData
 
-	// Tables with simple block_number key (key = block_num_u64)
-	simpleTables := []string{
-		"Receipt",
-		"block_info_roots",
-		// Note: CanonicalHeader and hermez_blockBatches are EXCLUDED in Moderate mode
-		// These tables have dependencies with AccountChangeSet/StorageChangeSet
-		// and must remain intact to avoid MDBX_EKEYMISMATCH errors
-	}
-
-	// Tables with composite keys starting with block_number (key = block_num_u64 + additional_data)
-	compositeKeyTables := []string{
-		"Header",                            // block_num_u64 + hash
-		"BlockBody",                         // block_num_u64 + hash
-		"TxSender",                          // block_num_u64 + blockHash
-		"TransactionLog",                    // block_num_u64 + txIndex + logIndex
-		"hermez_intermediate_tx_stateRoots", // l2blockno + txhash
-	}
+	// Get table lists based on pruning level
+	simpleTables, compositeKeyTables := getCopyTablesForPruneLevel(pruneLevel)
 
 	// Note: CanonicalHeader is excluded from moderate mode to maintain table consistency
 
@@ -176,19 +189,19 @@ func copyBlockData(tx kv.RwTx, blockNos []uint64, genesisHeight uint64) ([]Block
 	return preservedData, nil
 }
 
-// clearBatchTables clears all batch-related tables using optimized deletion strategy
-func clearBatchTables(tx kv.RwTx) error {
+// getClearTablesForPruneLevel returns the appropriate table lists based on pruning level
+func getClearTablesForPruneLevel(pruneLevel PruneLevel) (tablesToClear []string) {
 	// Tables to clear - all block-related tables that should be partially pruned
 	// Note: small tables (block_l1_info_tree_index, plain_state_version, smt_depths, MaxTxNum) excluded from cleanup
-	tablesToClear := []string{
+	tablesToClear = []string{
 		// Simple block tables (key = block_num_u64)
-		"Receipt",
+		//"Receipt",
 		"block_info_roots",
 		// Composite key tables (key = block_num_u64 + additional_data)
 		//"Header",
 		//"BlockBody",
-		"TxSender",
-		"TransactionLog",
+		//"TxSender",
+		//"TransactionLog",
 		"hermez_intermediate_tx_stateRoots",
 		// Special mapping table (key = header_hash -> block_num_u64)
 		//"HeaderNumber",
@@ -199,6 +212,21 @@ func clearBatchTables(tx kv.RwTx) error {
 		// Trade-off: this may cause some referential inconsistency but avoids database corruption
 	}
 
+	// Add additional tables for aggressive pruning
+	if pruneLevel == PruneLevelAggressive {
+		tablesToClear = append(tablesToClear,
+			"Receipt",
+			"TxSender",       // block_num_u64 + blockHash
+			"TransactionLog", // block_num_u64 + txIndex + logIndex
+		)
+	}
+
+	return tablesToClear
+}
+
+// clearBatchTables clears all batch-related tables using optimized deletion strategy
+func clearBatchTables(tx kv.RwTx, pruneLevel PruneLevel) error {
+	tablesToClear := getClearTablesForPruneLevel(pruneLevel)
 	fmt.Printf("🚀 Applying optimized deletion strategy to batch tables...\n")
 
 	// Apply layered optimization strategy to batch clearing operations
@@ -233,15 +261,17 @@ func clearTablesWithOptimization(tx kv.RwTx, tables []string, operationName stri
 }
 
 // restoreBlockData restores preserved block data to tables
-func restoreBlockData(tx kv.RwTx, preservedData []BlockData) error {
+func restoreBlockData(tx kv.RwTx, preservedData []BlockData, pruneLevel PruneLevel) error {
 	// Tables with simple block_number key (key = block_num_u64)
 	simpleTables := map[string]bool{
-		"Receipt":          true,
+		//"Receipt":          true,
 		"block_info_roots": true,
 		// Note: CanonicalHeader and hermez_blockBatches are EXCLUDED in Moderate mode
 		// These tables have dependencies with AccountChangeSet/StorageChangeSet
 		// and must remain intact to avoid MDBX_EKEYMISMATCH errors
 	}
+	if pruneLevel == PruneLevelAggressive {
+	}
 
 	for i, blockData := range preservedData {
 		if i%1000 == 0 {
@@ -288,9 +318,9 @@ func restoreBlockData(tx kv.RwTx, preservedData []BlockData) error {
 }
 
 // clearAllBatchTables clears all batch-related tables (used when no blocks to preserve)
-func clearAllBatchTables(tx kv.RwTx) error {
+func clearAllBatchTables(tx kv.RwTx, pruneLevel PruneLevel) error {
 	fmt.Printf("Clearing all batch-related tables...\n")
-	return clearBatchTables(tx)
+	return clearBatchTables(tx, pruneLevel)
 }
 
 // restoreBatchMetadata restores batch metadata for kept batches

cmd/prune-mdbx-data/cmd/prune-chaindata/main.go

@@ -389,47 +389,24 @@ func executeMainPruningOperations(paths *MainPaths, analysis *MainAnalysis, conf
 	if config.PruneLevel == PruneLevelModerate || config.PruneLevel == PruneLevelAggressive {
 		fmt.Printf("Closing database before batch operations...\n")
 		fmt.Printf("✓ Database closed\n")
-
-		stats.DeletedBatches, stats.DeletedBlocks = executeBatchOperationsWithCommit(paths.ChaindataPath, config.KeepRecentBatches, config.GenesisBlockHeight, log, ctx)
+		stats.DeletedBatches, stats.DeletedBlocks = executeBatchOperationsWithCommit(paths.ChaindataPath, config, log, ctx)
 	}
 
-	// Execute Aggressive mode operations with guaranteed commit (Aggressive mode only)
+	// Execute operations with guaranteed commit
 	if config.PruneLevel == PruneLevelAggressive {
 		// ✅ OPTIMAL ORDER: Clean from indexes/mappings to data (maintains consistency)
 
-		// Phase 1b: Header ecosystem cleanup - Clean indexes/mappings FIRST
+		// Header ecosystem cleanup - Clean indexes/mappings FIRST
 		// This ensures no dangling references to blocks that will have missing state data
 		headerRecords, headerSize := executeHeaderEcosystemCleanupWithCommit(paths.ChaindataPath, config.KeepRecentBatches, config.GenesisBlockHeight, log, ctx)
 		stats.DeletedCount += headerRecords
 		stats.ActualDeletedSize += headerSize
-
-		// Phase 1c: DupCursor data cleanup - Clean state change data AFTER header cleanup
-		// This prevents inconsistent state where headers exist but state changes are missing
-		dupCursorRecords, dupCursorSize := executeDupCursorOperationsWithCommit(paths.ChaindataPath, config.KeepRecentBatches, config.FastDupCursorMode, config.SafeFastMode, log, ctx)
-		stats.DeletedCount += dupCursorRecords
-		stats.ActualDeletedSize += dupCursorSize
-	}
-
-	// Filter out block tables from full deletion if we did partial pruning
-	partiallyPrunedTables := map[string]bool{
-		// Header-related tables (must use same strategy for data consistency)
-		"Header": true, "HeaderNumber": true,
-		// Other block data tables
-		"BlockBody": true, "Receipt": true, "TxSender": true, "TransactionLog": true,
-		// zkEVM intermediate data tables
-		"hermez_intermediate_tx_stateRoots": true,
-		// DupCursor tables - excluded from normal batch processing due to special cursor requirements
-		"CanonicalHeader":     true, // dupCursor table - needs special handling
-		"hermez_blockBatches": true, // dupCursor table - needs special handling
 	}
 
-	// In Aggressive mode, Header ecosystem tables are handled by Phase 1b
-	if config.PruneLevel == PruneLevelAggressive {
-		partiallyPrunedTables["CanonicalHeader"] = true // Processed in Phase 1b: Header Ecosystem Cleanup
-		partiallyPrunedTables["HeaderNumber"] = true    // Processed in Phase 1b: Header Ecosystem Cleanup
-		partiallyPrunedTables["Header"] = true          // Processed in Phase 1b: Header Ecosystem Cleanup
-		partiallyPrunedTables["BlockBody"] = true       // Processed in Phase 1b: Header Ecosystem Cleanup
-	}
+	// DupCursor data cleanup - Clean state change data
+	dupCursorRecords, dupCursorSize := executeDupCursorOperationsWithCommit(paths.ChaindataPath, config.KeepRecentBatches, config.FastDupCursorMode, config.SafeFastMode, log, ctx)
+	stats.DeletedCount += dupCursorRecords
+	stats.ActualDeletedSize += dupCursorSize
 
 	// Execute optimized table deletion using pre-collected statistics
 	fmt.Printf("\nStarting table deletion...\n")
@@ -439,7 +416,7 @@ func executeMainPruningOperations(paths *MainPaths, analysis *MainAnalysis, conf
 
 	// Execute table deletion operations with staged approach (user suggested optimization)
 	stats.DeletedCount, stats.ActuallyDeletedTables, stats.ActualDeletedSize = executeStagedTableDeletion(
-		paths.ChaindataPath, analysis.SortedTables, analysis.PreCollectedStats, partiallyPrunedTables, config.PruneLevel, log, ctx)
+		paths.ChaindataPath, analysis.SortedTables, analysis.PreCollectedStats, config.PruneLevel, log, ctx)
 
 	return stats, nil
 }

cmd/prune-mdbx-data/cmd/prune-chaindata/moderate_mode.go

@@ -10,7 +10,7 @@ import (
 )
 
 // executeBatchOperationsWithCommit executes batch operations with guaranteed commit for Moderate mode
-func executeBatchOperationsWithCommit(dbPath string, keepRecentBatches uint64, genesisHeight uint64, log logv3.Logger, ctx context.Context) (int, int) {
+func executeBatchOperationsWithCommit(dbPath string, config *MainConfig, log logv3.Logger, ctx context.Context) (int, int) {
 	fmt.Printf("\n=== Phase 1: Batch Operations ===\n")
 
 	// Open database for batch operations
@@ -44,7 +44,7 @@ func executeBatchOperationsWithCommit(dbPath string, keepRecentBatches uint64, g
 		}
 	}()
 
-	deletedBatches, deletedBlocks, err := partialPruneBatchTables(tx, keepRecentBatches, genesisHeight)
+	deletedBatches, deletedBlocks, err := partialPruneBatchTables(tx, config)
 	if err != nil {
 		logv3.Error("Failed to perform batch operations", "error", err)
 		return 0, 0

cmd/prune-mdbx-data/cmd/prune-chaindata/partial_cleanup.go

@@ -4,9 +4,9 @@ import (
 	"encoding/binary"
 	"fmt"
 
-	"github.com/ledgerwatch/erigon/core/types"
 	"github.com/ledgerwatch/erigon-lib/common"
 	"github.com/ledgerwatch/erigon-lib/kv"
+	"github.com/ledgerwatch/erigon/core/types"
 	"github.com/ledgerwatch/erigon/zk/hermez_db"
 )
 
@@ -72,7 +72,7 @@ func getLatestBatchNumber(tx kv.Tx) (uint64, error) {
 }
 
 // partialPruneBatchTables performs batch-based pruning on block-related tables
-func partialPruneBatchTables(tx kv.RwTx, keepRecentBatches uint64, genesisHeight uint64) (int, int, error) {
+func partialPruneBatchTables(tx kv.RwTx, config *MainConfig) (int, int, error) {
 	hermezDb := hermez_db.NewHermezDbReader(tx)
 
 	// 1. Get latest batch number
@@ -82,12 +82,12 @@ func partialPruneBatchTables(tx kv.RwTx, keepRecentBatches uint64, genesisHeight
 	}
 
 	fmt.Printf("Latest batch number: %d\n", latestBatch)
-	fmt.Printf("Keeping recent %d batches\n", keepRecentBatches)
+	fmt.Printf("Keeping recent %d batches\n", config.KeepRecentBatches)
 
 	// 2. Calculate batch range to keep
 	var pruneBefore uint64
-	if latestBatch > keepRecentBatches {
-		pruneBefore = latestBatch - keepRecentBatches + 1
+	if latestBatch > config.KeepRecentBatches {
+		pruneBefore = latestBatch - config.KeepRecentBatches + 1
 	} else {
 		fmt.Printf("No batches to prune (total batches <= keep recent batches)\n")
 		return 0, 0, nil
@@ -116,5 +116,5 @@ func partialPruneBatchTables(tx kv.RwTx, keepRecentBatches uint64, genesisHeight
 	fmt.Printf("Preserving batches %d to %d, deleting batches 0 to %d\n", keepFromBatch, latestBatch, pruneBefore-1)
 
 	// Use optimized strategy: copy recent data, truncate tables, restore data
-	return executeCopyTruncateRestore(tx, hermezDb, keepFromBatch, latestBatch, pruneBefore, genesisHeight)
+	return executeCopyTruncateRestore(tx, hermezDb, keepFromBatch, latestBatch, pruneBefore, config)
 }

cmd/prune-mdbx-data/cmd/prune-chaindata/staged_deletion.go

@@ -29,7 +29,6 @@ func executeStagedTableDeletion(
 		sizeBytes uint64
 		pages     uint64
 	},
-	partiallyPrunedTables map[string]bool,
 	pruneLevel PruneLevel,
 	log logv3.Logger,
 	ctx context.Context,
@@ -77,7 +76,7 @@ func executeStagedTableDeletion(
 	// === Stage 1: Delete small tables ===
 	if len(smallTables) > 0 {
 		deletedCount, actuallyDeletedTables, actualDeletedSize := executeSmallTableDeletion(
-			dbPath, smallTables, preCollectedStats, partiallyPrunedTables, pruneLevel, log, ctx)
+			dbPath, smallTables, preCollectedStats, pruneLevel, log, ctx)
 
 		totalDeletedCount += deletedCount
 		totalActuallyDeletedTables += actuallyDeletedTables
@@ -120,7 +119,6 @@ func executeSmallTableDeletion(
 		sizeBytes uint64
 		pages     uint64
 	},
-	partiallyPrunedTables map[string]bool,
 	pruneLevel PruneLevel,
 	log logv3.Logger,
 	ctx context.Context,

test/Makefile

@@ -616,19 +616,22 @@ vmtouch-check:
 .PHONY: prune
 prune:
 	@echo "Step 1: Pruning Sequencer Database..."
-	XLAYER_DATA_DIR=./data/seq $(DOCKER_COMPOSE) up xlayer-prune
+	XLAYER_DATA_DIR=./data/seq XLAYER_PRUNE_MODE=aggressive $(DOCKER_COMPOSE) up xlayer-prune
 	@echo "Step 2: Pruning RPC Database..."
-	XLAYER_DATA_DIR=./data/rpc $(DOCKER_COMPOSE) up xlayer-prune
+	XLAYER_DATA_DIR=./data/rpc XLAYER_PRUNE_MODE=moderate $(DOCKER_COMPOSE) up xlayer-prune
 	@echo "✅ All databases pruned successfully!"
 
 
+# Default values for database pruning
 XLAYER_DATA_DIR ?= ./data/seq
+XLAYER_PRUNE_MODE ?= aggressive
 
 .PHONY: prune-db
-prune-db:
+prune-db: ## Prune database with configurable mode (Usage: make prune-db XLAYER_DATA_DIR=./data/rpc XLAYER_PRUNE_MODE=moderate)
 	@echo "🗂️  Pruning Database:"
 	@echo "   Data Directory: $(XLAYER_DATA_DIR)"
-	XLAYER_DATA_DIR=$(XLAYER_DATA_DIR) $(DOCKER_COMPOSE) up xlayer-prune
+	@echo "   Prune Mode: $(XLAYER_PRUNE_MODE)"
+	XLAYER_DATA_DIR=$(XLAYER_DATA_DIR) XLAYER_PRUNE_MODE=$(XLAYER_PRUNE_MODE) $(DOCKER_COMPOSE) up xlayer-prune
 
 .PHONY: prune-list
 prune-list: