From 88a69be1cfd169aca062b3d4f08b58ad65078ee3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Diamond?= <32074058+Andre-Diamond@users.noreply.github.com> Date: Wed, 8 Oct 2025 11:34:16 +0200 Subject: [PATCH 1/2] refactor(batch-snapshot): enhance batch processing and error tracking - Increased delay between batches from 5 to 10 seconds for improved stability. - Updated batch processing logic to include network-specific data for mainnet and testnet. - Enhanced failure tracking with detailed error types and messages for better debugging. - Improved logging to provide clearer insights into wallet processing results and failures. --- .github/workflows/daily-balance-snapshots.yml | 2 +- scripts/batch-snapshot-orchestrator.ts | 120 ++++++++++++++++-- src/pages/api/v1/stats/run-snapshots-batch.ts | 92 ++++++++++++-- 3 files changed, 194 insertions(+), 20 deletions(-) diff --git a/.github/workflows/daily-balance-snapshots.yml b/.github/workflows/daily-balance-snapshots.yml index 0c1f9188..082c953f 100644 --- a/.github/workflows/daily-balance-snapshots.yml +++ b/.github/workflows/daily-balance-snapshots.yml @@ -41,7 +41,7 @@ jobs: API_BASE_URL: "https://multisig.meshjs.dev" SNAPSHOT_AUTH_TOKEN: ${{ secrets.SNAPSHOT_AUTH_TOKEN }} BATCH_SIZE: 10 - DELAY_BETWEEN_BATCHES: 5 + DELAY_BETWEEN_BATCHES: 10 MAX_RETRIES: 3 - name: Notify on failure diff --git a/scripts/batch-snapshot-orchestrator.ts b/scripts/batch-snapshot-orchestrator.ts index 6dc36609..665567ba 100644 --- a/scripts/batch-snapshot-orchestrator.ts +++ b/scripts/batch-snapshot-orchestrator.ts @@ -15,7 +15,7 @@ * - API_BASE_URL: Base URL for the API (default: http://localhost:3000) * - SNAPSHOT_AUTH_TOKEN: Authentication token for API requests * - BATCH_SIZE: Number of wallets per batch (default: 10) - * - DELAY_BETWEEN_BATCHES: Delay between batches in seconds (default: 5) + * - DELAY_BETWEEN_BATCHES: Delay between batches in seconds (default: 10) * - MAX_RETRIES: Maximum retries for failed batches (default: 3) */ @@ -24,8 +24,18 @@ interface BatchProgress { walletsInBatch: number; failedInBatch: number; snapshotsStored: number; - totalAdaBalance: number; totalBatches: number; + // Network-specific data + mainnetWallets: number; + testnetWallets: number; + mainnetAdaBalance: number; + testnetAdaBalance: number; + // Failure details + failures: Array<{ + walletId: string; + errorType: string; + errorMessage: string; + }>; } interface BatchResponse { @@ -40,9 +50,21 @@ interface BatchResults { failedBatches: number; totalWalletsProcessed: number; totalWalletsFailed: number; - totalAdaBalance: number; totalSnapshotsStored: number; executionTime: number; + // Network-specific data + totalMainnetWallets: number; + totalTestnetWallets: number; + totalMainnetAdaBalance: number; + totalTestnetAdaBalance: number; + // Failure tracking + allFailures: Array<{ + walletId: string; + errorType: string; + errorMessage: string; + batchNumber: number; + }>; + failureSummary: Record; } interface BatchConfig { @@ -70,9 +92,16 @@ class BatchSnapshotOrchestrator { failedBatches: 0, totalWalletsProcessed: 0, totalWalletsFailed: 0, - totalAdaBalance: 0, totalSnapshotsStored: 0, executionTime: 0, + // Network-specific data + totalMainnetWallets: 0, + totalTestnetWallets: 0, + totalMainnetAdaBalance: 0, + totalTestnetAdaBalance: 0, + // Failure tracking + allFailures: [], + failureSummary: {}, }; } @@ -88,7 +117,7 @@ class BatchSnapshotOrchestrator { apiBaseUrl, authToken, batchSize: parseInt(process.env.BATCH_SIZE || '10'), - delayBetweenBatches: parseInt(process.env.DELAY_BETWEEN_BATCHES || '5'), + delayBetweenBatches: parseInt(process.env.DELAY_BETWEEN_BATCHES || '10'), maxRetries: parseInt(process.env.MAX_RETRIES || '3'), }; } @@ -129,6 +158,17 @@ class BatchSnapshotOrchestrator { return new Promise(resolve => setTimeout(resolve, seconds * 1000)); } + private getFriendlyErrorName(errorType: string): string { + const errorMap: Record = { + 'wallet_build_failed': 'Wallet Build Failed', + 'utxo_fetch_failed': 'UTxO Fetch Failed', + 'address_generation_failed': 'Address Generation Failed', + 'balance_calculation_failed': 'Balance Calculation Failed', + 'processing_failed': 'General Processing Failed', + }; + return errorMap[errorType] || errorType; + } + private async processBatch(batchNumber: number, batchId: string): Promise { console.log(`šŸ“¦ Processing batch ${batchNumber}...`); @@ -148,7 +188,8 @@ class BatchSnapshotOrchestrator { console.log(` ā€¢ Processed: ${data.progress.processedInBatch}/${data.progress.walletsInBatch} wallets`); console.log(` ā€¢ Failed: ${data.progress.failedInBatch}`); console.log(` ā€¢ Snapshots stored: ${data.progress.snapshotsStored}`); - console.log(` ā€¢ Batch ADA balance: ${Math.round(data.progress.totalAdaBalance * 100) / 100} ADA`); + console.log(` ā€¢ Mainnet: ${data.progress.mainnetWallets} wallets, ${Math.round(data.progress.mainnetAdaBalance * 100) / 100} ADA`); + console.log(` ā€¢ Testnet: ${data.progress.testnetWallets} wallets, ${Math.round(data.progress.testnetAdaBalance * 100) / 100} ADA`); return data.progress; } else { @@ -193,8 +234,22 @@ class BatchSnapshotOrchestrator { this.results.completedBatches = 1; this.results.totalWalletsProcessed += firstBatch.processedInBatch; this.results.totalWalletsFailed += firstBatch.failedInBatch; - this.results.totalAdaBalance += firstBatch.totalAdaBalance; this.results.totalSnapshotsStored += firstBatch.snapshotsStored; + + // Accumulate network-specific data + this.results.totalMainnetWallets += firstBatch.mainnetWallets; + this.results.totalTestnetWallets += firstBatch.testnetWallets; + this.results.totalMainnetAdaBalance += firstBatch.mainnetAdaBalance; + this.results.totalTestnetAdaBalance += firstBatch.testnetAdaBalance; + + // Accumulate failures + firstBatch.failures.forEach(failure => { + this.results.allFailures.push({ + ...failure, + batchNumber: 1 + }); + this.results.failureSummary[failure.errorType] = (this.results.failureSummary[failure.errorType] || 0) + 1; + }); console.log(`šŸ“Š Total batches to process: ${this.results.totalBatches}`); @@ -210,8 +265,22 @@ class BatchSnapshotOrchestrator { this.results.completedBatches++; this.results.totalWalletsProcessed += batchProgress.processedInBatch; this.results.totalWalletsFailed += batchProgress.failedInBatch; - this.results.totalAdaBalance += batchProgress.totalAdaBalance; this.results.totalSnapshotsStored += batchProgress.snapshotsStored; + + // Accumulate network-specific data + this.results.totalMainnetWallets += batchProgress.mainnetWallets; + this.results.totalTestnetWallets += batchProgress.testnetWallets; + this.results.totalMainnetAdaBalance += batchProgress.mainnetAdaBalance; + this.results.totalTestnetAdaBalance += batchProgress.testnetAdaBalance; + + // Accumulate failures + batchProgress.failures.forEach(failure => { + this.results.allFailures.push({ + ...failure, + batchNumber + }); + this.results.failureSummary[failure.errorType] = (this.results.failureSummary[failure.errorType] || 0) + 1; + }); } else { this.results.failedBatches++; console.error(`āŒ Batch ${batchNumber} failed completely`); @@ -234,11 +303,42 @@ class BatchSnapshotOrchestrator { console.log(` ā€¢ Wallets processed: ${this.results.totalWalletsProcessed}`); console.log(` ā€¢ Wallets failed: ${this.results.totalWalletsFailed}`); console.log(` ā€¢ Snapshots stored: ${this.results.totalSnapshotsStored}`); - console.log(` ā€¢ Total TVL: ${Math.round(this.results.totalAdaBalance * 100) / 100} ADA`); console.log(` ā€¢ Execution time: ${this.results.executionTime}s`); + + // Network-specific breakdown + console.log(`\nšŸŒ Network Breakdown:`); + console.log(` šŸ“ˆ Mainnet:`); + console.log(` ā€¢ Wallets: ${this.results.totalMainnetWallets}`); + console.log(` ā€¢ TVL: ${Math.round(this.results.totalMainnetAdaBalance * 100) / 100} ADA`); + console.log(` šŸ§Ŗ Testnet:`); + console.log(` ā€¢ Wallets: ${this.results.totalTestnetWallets}`); + console.log(` ā€¢ TVL: ${Math.round(this.results.totalTestnetAdaBalance * 100) / 100} ADA`); + + // Failure analysis + if (this.results.totalWalletsFailed > 0) { + console.log(`\nāŒ Failure Analysis:`); + console.log(` ā€¢ Total failed wallets: ${this.results.totalWalletsFailed}`); + + // Show failure summary by type + Object.entries(this.results.failureSummary).forEach(([errorType, count]) => { + const friendlyName = this.getFriendlyErrorName(errorType); + console.log(` ā€¢ ${friendlyName}: ${count} wallets`); + }); + + // Show sample failures (first 3) + if (this.results.allFailures.length > 0) { + console.log(`\nšŸ“‹ Sample Failures:`); + this.results.allFailures.slice(0, 3).forEach((failure, index) => { + console.log(` ${index + 1}. ${failure.walletId.slice(0, 8)}... - ${failure.errorMessage}`); + }); + if (this.results.allFailures.length > 3) { + console.log(` ... and ${this.results.allFailures.length - 3} more`); + } + } + } if (this.results.failedBatches > 0) { - console.log(`āš ļø Warning: ${this.results.failedBatches} batches failed. You may need to retry those batches manually.`); + console.log(`\nāš ļø Warning: ${this.results.failedBatches} batches failed. You may need to retry those batches manually.`); } return this.results; diff --git a/src/pages/api/v1/stats/run-snapshots-batch.ts b/src/pages/api/v1/stats/run-snapshots-batch.ts index df7d2b68..71a5221f 100644 --- a/src/pages/api/v1/stats/run-snapshots-batch.ts +++ b/src/pages/api/v1/stats/run-snapshots-batch.ts @@ -16,6 +16,13 @@ interface WalletBalance { balance: Record; adaBalance: number; isArchived: boolean; + network: number; // 0 = testnet, 1 = mainnet +} + +interface WalletFailure { + walletId: string; + errorType: string; // e.g., "wallet_build_failed", "utxo_fetch_failed", "balance_calculation_failed" + errorMessage: string; // sanitized error message } interface BatchProgress { @@ -27,11 +34,17 @@ interface BatchProgress { failedInBatch: number; totalProcessed: number; totalFailed: number; - totalAdaBalance: number; snapshotsStored: number; isComplete: boolean; startedAt: string; lastUpdatedAt: string; + // Network-specific data + mainnetWallets: number; + testnetWallets: number; + mainnetAdaBalance: number; + testnetAdaBalance: number; + // Failure details + failures: WalletFailure[]; } interface BatchResponse { @@ -113,11 +126,17 @@ export default async function handler( failedInBatch: 0, totalProcessed: 0, totalFailed: 0, - totalAdaBalance: 0, snapshotsStored: 0, isComplete: true, startedAt: startTime, lastUpdatedAt: new Date().toISOString(), + // Network-specific data + mainnetWallets: 0, + testnetWallets: 0, + mainnetAdaBalance: 0, + testnetAdaBalance: 0, + // Failure details + failures: [], }, timestamp: new Date().toISOString(), }); @@ -125,9 +144,13 @@ export default async function handler( // Step 3: Process wallets in this batch const walletBalances: WalletBalance[] = []; + const failures: WalletFailure[] = []; let processedInBatch = 0; let failedInBatch = 0; - let totalAdaBalance = 0; + let mainnetWallets = 0; + let testnetWallets = 0; + let mainnetAdaBalance = 0; + let testnetAdaBalance = 0; for (const wallet of wallets) { try { @@ -163,6 +186,11 @@ export default async function handler( const mWallet = buildMultisigWallet(walletData, network); if (!mWallet) { console.error(`Failed to build multisig wallet for ${wallet.id.slice(0, 8)}...`); + failures.push({ + walletId: wallet.id, + errorType: "wallet_build_failed", + errorMessage: "Unable to build multisig wallet from provided data" + }); failedInBatch++; continue; } @@ -239,16 +267,49 @@ export default async function handler( balance, adaBalance: roundedAdaBalance, isArchived: wallet.isArchived, + network, }; walletBalances.push(walletBalance); - totalAdaBalance += roundedAdaBalance; + + // Track network-specific data + if (network === 1) { + mainnetWallets++; + mainnetAdaBalance += roundedAdaBalance; + } else { + testnetWallets++; + testnetAdaBalance += roundedAdaBalance; + } + processedInBatch++; - console.log(` āœ… Balance: ${roundedAdaBalance} ADA`); + console.log(` āœ… Balance: ${roundedAdaBalance} ADA (${network === 1 ? 'mainnet' : 'testnet'})`); } catch (error) { - console.error(`Error processing wallet ${wallet.id.slice(0, 8)}...:`, error); + const errorMessage = error instanceof Error ? error.message : 'Unknown error'; + console.error(`Error processing wallet ${wallet.id.slice(0, 8)}...:`, errorMessage); + + // Determine error type based on error message + let errorType = "processing_failed"; + let sanitizedMessage = "Wallet processing failed"; + + if (errorMessage.includes("fetchAddressUTxOs") || errorMessage.includes("UTxO")) { + errorType = "utxo_fetch_failed"; + sanitizedMessage = "Failed to fetch UTxOs from blockchain"; + } else if (errorMessage.includes("serializeNativeScript") || errorMessage.includes("address")) { + errorType = "address_generation_failed"; + sanitizedMessage = "Failed to generate wallet address"; + } else if (errorMessage.includes("balance") || errorMessage.includes("lovelace")) { + errorType = "balance_calculation_failed"; + sanitizedMessage = "Failed to calculate wallet balance"; + } + + failures.push({ + walletId: wallet.id, + errorType, + errorMessage: sanitizedMessage + }); + failedInBatch++; } } @@ -292,7 +353,8 @@ export default async function handler( console.log(` ā€¢ Processed: ${processedInBatch}/${wallets.length}`); console.log(` ā€¢ Failed: ${failedInBatch}`); console.log(` ā€¢ Snapshots stored: ${snapshotsStored}`); - console.log(` ā€¢ Batch ADA balance: ${Math.round(totalAdaBalance * 100) / 100} ADA`); + console.log(` ā€¢ Mainnet: ${mainnetWallets} wallets, ${Math.round(mainnetAdaBalance * 100) / 100} ADA`); + console.log(` ā€¢ Testnet: ${testnetWallets} wallets, ${Math.round(testnetAdaBalance * 100) / 100} ADA`); console.log(` ā€¢ Overall progress: ${totalProcessed}/${totalWallets} wallets`); const progress: BatchProgress = { @@ -304,11 +366,17 @@ export default async function handler( failedInBatch, totalProcessed, totalFailed, - totalAdaBalance, snapshotsStored, isComplete, startedAt: startTime, lastUpdatedAt: new Date().toISOString(), + // Network-specific data + mainnetWallets, + testnetWallets, + mainnetAdaBalance, + testnetAdaBalance, + // Failure details + failures, }; const response: BatchResponse = { @@ -338,11 +406,17 @@ export default async function handler( failedInBatch: 0, totalProcessed: 0, totalFailed: 0, - totalAdaBalance: 0, snapshotsStored: 0, isComplete: false, startedAt: startTime, lastUpdatedAt: new Date().toISOString(), + // Network-specific data + mainnetWallets: 0, + testnetWallets: 0, + mainnetAdaBalance: 0, + testnetAdaBalance: 0, + // Failure details + failures: [], }, timestamp: new Date().toISOString(), }); From 03fac37f79db7c4b445e6586510ab95283c927c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Diamond?= <32074058+Andre-Diamond@users.noreply.github.com> Date: Wed, 8 Oct 2025 12:03:54 +0200 Subject: [PATCH 2/2] refactor(workflow): enable daily balance snapshots scheduling - Re-enabled the daily balance snapshots workflow with a scheduled trigger at midnight UTC. - Improved logging in the batch snapshot orchestrator to display detailed failure information for better debugging. - Updated wallet ID logging to show a truncated version for clarity in error messages. --- .github/workflows/daily-balance-snapshots.yml | 6 +++--- scripts/batch-snapshot-orchestrator.ts | 21 ++++++++----------- src/pages/api/v1/stats/run-snapshots-batch.ts | 4 ++-- 3 files changed, 14 insertions(+), 17 deletions(-) diff --git a/.github/workflows/daily-balance-snapshots.yml b/.github/workflows/daily-balance-snapshots.yml index 082c953f..473637a4 100644 --- a/.github/workflows/daily-balance-snapshots.yml +++ b/.github/workflows/daily-balance-snapshots.yml @@ -4,9 +4,9 @@ name: Daily Balance Snapshots # API requests require SNAPSHOT_AUTH_TOKEN secret to be set in GitHub repository settings. on: - #schedule: - # Run at midnight UTC every day - #- cron: '0 0 * * *' + schedule: + # Run at midnight UTC every day + - cron: '0 0 * * *' # Allow manual triggering for testing workflow_dispatch: diff --git a/scripts/batch-snapshot-orchestrator.ts b/scripts/batch-snapshot-orchestrator.ts index 665567ba..7989b58c 100644 --- a/scripts/batch-snapshot-orchestrator.ts +++ b/scripts/batch-snapshot-orchestrator.ts @@ -191,6 +191,14 @@ class BatchSnapshotOrchestrator { console.log(` ā€¢ Mainnet: ${data.progress.mainnetWallets} wallets, ${Math.round(data.progress.mainnetAdaBalance * 100) / 100} ADA`); console.log(` ā€¢ Testnet: ${data.progress.testnetWallets} wallets, ${Math.round(data.progress.testnetAdaBalance * 100) / 100} ADA`); + // Show failures for this batch + if (data.progress.failures.length > 0) { + console.log(` āŒ Failures in this batch:`); + data.progress.failures.forEach((failure, index) => { + console.log(` ${index + 1}. ${failure.walletId}... - ${failure.errorMessage}`); + }); + } + return data.progress; } else { throw new Error(data.message || 'Batch processing failed'); @@ -316,7 +324,7 @@ class BatchSnapshotOrchestrator { // Failure analysis if (this.results.totalWalletsFailed > 0) { - console.log(`\nāŒ Failure Analysis:`); + console.log(`\nāŒ Failure Summary:`); console.log(` ā€¢ Total failed wallets: ${this.results.totalWalletsFailed}`); // Show failure summary by type @@ -324,17 +332,6 @@ class BatchSnapshotOrchestrator { const friendlyName = this.getFriendlyErrorName(errorType); console.log(` ā€¢ ${friendlyName}: ${count} wallets`); }); - - // Show sample failures (first 3) - if (this.results.allFailures.length > 0) { - console.log(`\nšŸ“‹ Sample Failures:`); - this.results.allFailures.slice(0, 3).forEach((failure, index) => { - console.log(` ${index + 1}. ${failure.walletId.slice(0, 8)}... - ${failure.errorMessage}`); - }); - if (this.results.allFailures.length > 3) { - console.log(` ... and ${this.results.allFailures.length - 3} more`); - } - } } if (this.results.failedBatches > 0) { diff --git a/src/pages/api/v1/stats/run-snapshots-batch.ts b/src/pages/api/v1/stats/run-snapshots-batch.ts index 71a5221f..d2b094e5 100644 --- a/src/pages/api/v1/stats/run-snapshots-batch.ts +++ b/src/pages/api/v1/stats/run-snapshots-batch.ts @@ -187,7 +187,7 @@ export default async function handler( if (!mWallet) { console.error(`Failed to build multisig wallet for ${wallet.id.slice(0, 8)}...`); failures.push({ - walletId: wallet.id, + walletId: wallet.id.slice(0, 8), errorType: "wallet_build_failed", errorMessage: "Unable to build multisig wallet from provided data" }); @@ -305,7 +305,7 @@ export default async function handler( } failures.push({ - walletId: wallet.id, + walletId: wallet.id.slice(0, 8), errorType, errorMessage: sanitizedMessage });