diff --git a/.github/workflows/daily-balance-snapshots.yml b/.github/workflows/daily-balance-snapshots.yml index 0c1f9188..473637a4 100644 --- a/.github/workflows/daily-balance-snapshots.yml +++ b/.github/workflows/daily-balance-snapshots.yml @@ -4,9 +4,9 @@ name: Daily Balance Snapshots # API requests require SNAPSHOT_AUTH_TOKEN secret to be set in GitHub repository settings. on: - #schedule: - # Run at midnight UTC every day - #- cron: '0 0 * * *' + schedule: + # Run at midnight UTC every day + - cron: '0 0 * * *' # Allow manual triggering for testing workflow_dispatch: @@ -41,7 +41,7 @@ jobs: API_BASE_URL: "https://multisig.meshjs.dev" SNAPSHOT_AUTH_TOKEN: ${{ secrets.SNAPSHOT_AUTH_TOKEN }} BATCH_SIZE: 10 - DELAY_BETWEEN_BATCHES: 5 + DELAY_BETWEEN_BATCHES: 10 MAX_RETRIES: 3 - name: Notify on failure diff --git a/scripts/batch-snapshot-orchestrator.ts b/scripts/batch-snapshot-orchestrator.ts index 6dc36609..7989b58c 100644 --- a/scripts/batch-snapshot-orchestrator.ts +++ b/scripts/batch-snapshot-orchestrator.ts @@ -15,7 +15,7 @@ * - API_BASE_URL: Base URL for the API (default: http://localhost:3000) * - SNAPSHOT_AUTH_TOKEN: Authentication token for API requests * - BATCH_SIZE: Number of wallets per batch (default: 10) - * - DELAY_BETWEEN_BATCHES: Delay between batches in seconds (default: 5) + * - DELAY_BETWEEN_BATCHES: Delay between batches in seconds (default: 10) * - MAX_RETRIES: Maximum retries for failed batches (default: 3) */ @@ -24,8 +24,18 @@ interface BatchProgress { walletsInBatch: number; failedInBatch: number; snapshotsStored: number; - totalAdaBalance: number; totalBatches: number; + // Network-specific data + mainnetWallets: number; + testnetWallets: number; + mainnetAdaBalance: number; + testnetAdaBalance: number; + // Failure details + failures: Array<{ + walletId: string; + errorType: string; + errorMessage: string; + }>; } interface BatchResponse { @@ -40,9 +50,21 @@ interface BatchResults { failedBatches: number; totalWalletsProcessed: number; totalWalletsFailed: number; - totalAdaBalance: number; totalSnapshotsStored: number; executionTime: number; + // Network-specific data + totalMainnetWallets: number; + totalTestnetWallets: number; + totalMainnetAdaBalance: number; + totalTestnetAdaBalance: number; + // Failure tracking + allFailures: Array<{ + walletId: string; + errorType: string; + errorMessage: string; + batchNumber: number; + }>; + failureSummary: Record; } interface BatchConfig { @@ -70,9 +92,16 @@ class BatchSnapshotOrchestrator { failedBatches: 0, totalWalletsProcessed: 0, totalWalletsFailed: 0, - totalAdaBalance: 0, totalSnapshotsStored: 0, executionTime: 0, + // Network-specific data + totalMainnetWallets: 0, + totalTestnetWallets: 0, + totalMainnetAdaBalance: 0, + totalTestnetAdaBalance: 0, + // Failure tracking + allFailures: [], + failureSummary: {}, }; } @@ -88,7 +117,7 @@ class BatchSnapshotOrchestrator { apiBaseUrl, authToken, batchSize: parseInt(process.env.BATCH_SIZE || '10'), - delayBetweenBatches: parseInt(process.env.DELAY_BETWEEN_BATCHES || '5'), + delayBetweenBatches: parseInt(process.env.DELAY_BETWEEN_BATCHES || '10'), maxRetries: parseInt(process.env.MAX_RETRIES || '3'), }; } @@ -129,6 +158,17 @@ class BatchSnapshotOrchestrator { return new Promise(resolve => setTimeout(resolve, seconds * 1000)); } + private getFriendlyErrorName(errorType: string): string { + const errorMap: Record = { + 'wallet_build_failed': 'Wallet Build Failed', + 'utxo_fetch_failed': 'UTxO Fetch Failed', + 'address_generation_failed': 'Address Generation Failed', + 'balance_calculation_failed': 'Balance Calculation Failed', + 'processing_failed': 'General Processing Failed', + }; + return errorMap[errorType] || errorType; + } + private async processBatch(batchNumber: number, batchId: string): Promise { console.log(`šŸ“¦ Processing batch ${batchNumber}...`); @@ -148,7 +188,16 @@ class BatchSnapshotOrchestrator { console.log(` • Processed: ${data.progress.processedInBatch}/${data.progress.walletsInBatch} wallets`); console.log(` • Failed: ${data.progress.failedInBatch}`); console.log(` • Snapshots stored: ${data.progress.snapshotsStored}`); - console.log(` • Batch ADA balance: ${Math.round(data.progress.totalAdaBalance * 100) / 100} ADA`); + console.log(` • Mainnet: ${data.progress.mainnetWallets} wallets, ${Math.round(data.progress.mainnetAdaBalance * 100) / 100} ADA`); + console.log(` • Testnet: ${data.progress.testnetWallets} wallets, ${Math.round(data.progress.testnetAdaBalance * 100) / 100} ADA`); + + // Show failures for this batch + if (data.progress.failures.length > 0) { + console.log(` āŒ Failures in this batch:`); + data.progress.failures.forEach((failure, index) => { + console.log(` ${index + 1}. ${failure.walletId}... - ${failure.errorMessage}`); + }); + } return data.progress; } else { @@ -193,8 +242,22 @@ class BatchSnapshotOrchestrator { this.results.completedBatches = 1; this.results.totalWalletsProcessed += firstBatch.processedInBatch; this.results.totalWalletsFailed += firstBatch.failedInBatch; - this.results.totalAdaBalance += firstBatch.totalAdaBalance; this.results.totalSnapshotsStored += firstBatch.snapshotsStored; + + // Accumulate network-specific data + this.results.totalMainnetWallets += firstBatch.mainnetWallets; + this.results.totalTestnetWallets += firstBatch.testnetWallets; + this.results.totalMainnetAdaBalance += firstBatch.mainnetAdaBalance; + this.results.totalTestnetAdaBalance += firstBatch.testnetAdaBalance; + + // Accumulate failures + firstBatch.failures.forEach(failure => { + this.results.allFailures.push({ + ...failure, + batchNumber: 1 + }); + this.results.failureSummary[failure.errorType] = (this.results.failureSummary[failure.errorType] || 0) + 1; + }); console.log(`šŸ“Š Total batches to process: ${this.results.totalBatches}`); @@ -210,8 +273,22 @@ class BatchSnapshotOrchestrator { this.results.completedBatches++; this.results.totalWalletsProcessed += batchProgress.processedInBatch; this.results.totalWalletsFailed += batchProgress.failedInBatch; - this.results.totalAdaBalance += batchProgress.totalAdaBalance; this.results.totalSnapshotsStored += batchProgress.snapshotsStored; + + // Accumulate network-specific data + this.results.totalMainnetWallets += batchProgress.mainnetWallets; + this.results.totalTestnetWallets += batchProgress.testnetWallets; + this.results.totalMainnetAdaBalance += batchProgress.mainnetAdaBalance; + this.results.totalTestnetAdaBalance += batchProgress.testnetAdaBalance; + + // Accumulate failures + batchProgress.failures.forEach(failure => { + this.results.allFailures.push({ + ...failure, + batchNumber + }); + this.results.failureSummary[failure.errorType] = (this.results.failureSummary[failure.errorType] || 0) + 1; + }); } else { this.results.failedBatches++; console.error(`āŒ Batch ${batchNumber} failed completely`); @@ -234,11 +311,31 @@ class BatchSnapshotOrchestrator { console.log(` • Wallets processed: ${this.results.totalWalletsProcessed}`); console.log(` • Wallets failed: ${this.results.totalWalletsFailed}`); console.log(` • Snapshots stored: ${this.results.totalSnapshotsStored}`); - console.log(` • Total TVL: ${Math.round(this.results.totalAdaBalance * 100) / 100} ADA`); console.log(` • Execution time: ${this.results.executionTime}s`); + + // Network-specific breakdown + console.log(`\n🌐 Network Breakdown:`); + console.log(` šŸ“ˆ Mainnet:`); + console.log(` • Wallets: ${this.results.totalMainnetWallets}`); + console.log(` • TVL: ${Math.round(this.results.totalMainnetAdaBalance * 100) / 100} ADA`); + console.log(` 🧪 Testnet:`); + console.log(` • Wallets: ${this.results.totalTestnetWallets}`); + console.log(` • TVL: ${Math.round(this.results.totalTestnetAdaBalance * 100) / 100} ADA`); + + // Failure analysis + if (this.results.totalWalletsFailed > 0) { + console.log(`\nāŒ Failure Summary:`); + console.log(` • Total failed wallets: ${this.results.totalWalletsFailed}`); + + // Show failure summary by type + Object.entries(this.results.failureSummary).forEach(([errorType, count]) => { + const friendlyName = this.getFriendlyErrorName(errorType); + console.log(` • ${friendlyName}: ${count} wallets`); + }); + } if (this.results.failedBatches > 0) { - console.log(`āš ļø Warning: ${this.results.failedBatches} batches failed. You may need to retry those batches manually.`); + console.log(`\nāš ļø Warning: ${this.results.failedBatches} batches failed. You may need to retry those batches manually.`); } return this.results; diff --git a/src/pages/api/v1/stats/run-snapshots-batch.ts b/src/pages/api/v1/stats/run-snapshots-batch.ts index df7d2b68..d2b094e5 100644 --- a/src/pages/api/v1/stats/run-snapshots-batch.ts +++ b/src/pages/api/v1/stats/run-snapshots-batch.ts @@ -16,6 +16,13 @@ interface WalletBalance { balance: Record; adaBalance: number; isArchived: boolean; + network: number; // 0 = testnet, 1 = mainnet +} + +interface WalletFailure { + walletId: string; + errorType: string; // e.g., "wallet_build_failed", "utxo_fetch_failed", "balance_calculation_failed" + errorMessage: string; // sanitized error message } interface BatchProgress { @@ -27,11 +34,17 @@ interface BatchProgress { failedInBatch: number; totalProcessed: number; totalFailed: number; - totalAdaBalance: number; snapshotsStored: number; isComplete: boolean; startedAt: string; lastUpdatedAt: string; + // Network-specific data + mainnetWallets: number; + testnetWallets: number; + mainnetAdaBalance: number; + testnetAdaBalance: number; + // Failure details + failures: WalletFailure[]; } interface BatchResponse { @@ -113,11 +126,17 @@ export default async function handler( failedInBatch: 0, totalProcessed: 0, totalFailed: 0, - totalAdaBalance: 0, snapshotsStored: 0, isComplete: true, startedAt: startTime, lastUpdatedAt: new Date().toISOString(), + // Network-specific data + mainnetWallets: 0, + testnetWallets: 0, + mainnetAdaBalance: 0, + testnetAdaBalance: 0, + // Failure details + failures: [], }, timestamp: new Date().toISOString(), }); @@ -125,9 +144,13 @@ export default async function handler( // Step 3: Process wallets in this batch const walletBalances: WalletBalance[] = []; + const failures: WalletFailure[] = []; let processedInBatch = 0; let failedInBatch = 0; - let totalAdaBalance = 0; + let mainnetWallets = 0; + let testnetWallets = 0; + let mainnetAdaBalance = 0; + let testnetAdaBalance = 0; for (const wallet of wallets) { try { @@ -163,6 +186,11 @@ export default async function handler( const mWallet = buildMultisigWallet(walletData, network); if (!mWallet) { console.error(`Failed to build multisig wallet for ${wallet.id.slice(0, 8)}...`); + failures.push({ + walletId: wallet.id.slice(0, 8), + errorType: "wallet_build_failed", + errorMessage: "Unable to build multisig wallet from provided data" + }); failedInBatch++; continue; } @@ -239,16 +267,49 @@ export default async function handler( balance, adaBalance: roundedAdaBalance, isArchived: wallet.isArchived, + network, }; walletBalances.push(walletBalance); - totalAdaBalance += roundedAdaBalance; + + // Track network-specific data + if (network === 1) { + mainnetWallets++; + mainnetAdaBalance += roundedAdaBalance; + } else { + testnetWallets++; + testnetAdaBalance += roundedAdaBalance; + } + processedInBatch++; - console.log(` āœ… Balance: ${roundedAdaBalance} ADA`); + console.log(` āœ… Balance: ${roundedAdaBalance} ADA (${network === 1 ? 'mainnet' : 'testnet'})`); } catch (error) { - console.error(`Error processing wallet ${wallet.id.slice(0, 8)}...:`, error); + const errorMessage = error instanceof Error ? error.message : 'Unknown error'; + console.error(`Error processing wallet ${wallet.id.slice(0, 8)}...:`, errorMessage); + + // Determine error type based on error message + let errorType = "processing_failed"; + let sanitizedMessage = "Wallet processing failed"; + + if (errorMessage.includes("fetchAddressUTxOs") || errorMessage.includes("UTxO")) { + errorType = "utxo_fetch_failed"; + sanitizedMessage = "Failed to fetch UTxOs from blockchain"; + } else if (errorMessage.includes("serializeNativeScript") || errorMessage.includes("address")) { + errorType = "address_generation_failed"; + sanitizedMessage = "Failed to generate wallet address"; + } else if (errorMessage.includes("balance") || errorMessage.includes("lovelace")) { + errorType = "balance_calculation_failed"; + sanitizedMessage = "Failed to calculate wallet balance"; + } + + failures.push({ + walletId: wallet.id.slice(0, 8), + errorType, + errorMessage: sanitizedMessage + }); + failedInBatch++; } } @@ -292,7 +353,8 @@ export default async function handler( console.log(` • Processed: ${processedInBatch}/${wallets.length}`); console.log(` • Failed: ${failedInBatch}`); console.log(` • Snapshots stored: ${snapshotsStored}`); - console.log(` • Batch ADA balance: ${Math.round(totalAdaBalance * 100) / 100} ADA`); + console.log(` • Mainnet: ${mainnetWallets} wallets, ${Math.round(mainnetAdaBalance * 100) / 100} ADA`); + console.log(` • Testnet: ${testnetWallets} wallets, ${Math.round(testnetAdaBalance * 100) / 100} ADA`); console.log(` • Overall progress: ${totalProcessed}/${totalWallets} wallets`); const progress: BatchProgress = { @@ -304,11 +366,17 @@ export default async function handler( failedInBatch, totalProcessed, totalFailed, - totalAdaBalance, snapshotsStored, isComplete, startedAt: startTime, lastUpdatedAt: new Date().toISOString(), + // Network-specific data + mainnetWallets, + testnetWallets, + mainnetAdaBalance, + testnetAdaBalance, + // Failure details + failures, }; const response: BatchResponse = { @@ -338,11 +406,17 @@ export default async function handler( failedInBatch: 0, totalProcessed: 0, totalFailed: 0, - totalAdaBalance: 0, snapshotsStored: 0, isComplete: false, startedAt: startTime, lastUpdatedAt: new Date().toISOString(), + // Network-specific data + mainnetWallets: 0, + testnetWallets: 0, + mainnetAdaBalance: 0, + testnetAdaBalance: 0, + // Failure details + failures: [], }, timestamp: new Date().toISOString(), });