diff --git a/.github/workflows/daily-balance-snapshots.yml b/.github/workflows/daily-balance-snapshots.yml index ba51aa15..0c1f9188 100644 --- a/.github/workflows/daily-balance-snapshots.yml +++ b/.github/workflows/daily-balance-snapshots.yml @@ -13,6 +13,7 @@ on: jobs: snapshot-balances: runs-on: ubuntu-latest + timeout-minutes: 300 # 5 hours timeout (GitHub Actions max is 6 hours) steps: - name: Checkout repository @@ -27,22 +28,31 @@ jobs: - name: Install dependencies run: npm ci - - name: Run balance snapshots - run: node scripts/balance-snapshots.js + - name: Install script dependencies + run: | + cd scripts + npm install + + - name: Run batch snapshot orchestration + run: | + cd scripts + npm start env: API_BASE_URL: "https://multisig.meshjs.dev" SNAPSHOT_AUTH_TOKEN: ${{ secrets.SNAPSHOT_AUTH_TOKEN }} - BATCH_SIZE: 3 - DELAY_BETWEEN_REQUESTS: 3 - DELAY_BETWEEN_BATCHES: 15 + BATCH_SIZE: 10 + DELAY_BETWEEN_BATCHES: 5 MAX_RETRIES: 3 - REQUEST_TIMEOUT: 30 - name: Notify on failure if: failure() + # Send failure notification run: | echo "āŒ Daily balance snapshot job failed" - # Optional: Send failure notification - # curl -X POST -H 'Content-type: application/json' \ - # --data "{\"text\":\"āŒ Daily balance snapshots failed. Check the GitHub Actions logs.\"}" \ - # ${{ secrets.DISCORD_WEBHOOK_URL }} + if [ -n "${{ secrets.SNAPSHOT_ERROR_DISCORD_WEBHOOK_URL }}" ]; then + curl -X POST -H 'Content-type: application/json' \ + --data "{\"text\":\"āŒ Daily balance snapshots failed. Check the GitHub Actions logs.\"}" \ + ${{ secrets.SNAPSHOT_ERROR_DISCORD_WEBHOOK_URL }} || echo "Failed to send Discord notification" + else + echo "SNAPSHOT_ERROR_DISCORD_WEBHOOK_URL not configured, skipping notification" + fi diff --git a/scripts/balance-snapshots.js b/scripts/balance-snapshots.js deleted file mode 100644 index 83dbdfaf..00000000 --- a/scripts/balance-snapshots.js +++ /dev/null @@ -1,272 +0,0 @@ -#!/usr/bin/env node - -/** - * Balance Snapshots Script (JavaScript version) - * - * This script fetches wallet balances and stores them as snapshots in the database. - * It can be run locally for testing or by GitHub Actions for automated snapshots. - * - * Usage: - * node scripts/balance-snapshots.js - * SNAPSHOT_AUTH_TOKEN=your_token node scripts/balance-snapshots.js - * - * Environment Variables: - * - API_BASE_URL: Base URL for the API (default: http://localhost:3000) - * - SNAPSHOT_AUTH_TOKEN: Authentication token for API requests - * - BATCH_SIZE: Number of wallets to process per batch (default: 3) - * - DELAY_BETWEEN_REQUESTS: Delay between requests in seconds (default: 3) - * - DELAY_BETWEEN_BATCHES: Delay between batches in seconds (default: 15) - * - MAX_RETRIES: Maximum retries for failed requests (default: 3) - * - REQUEST_TIMEOUT: Request timeout in seconds (default: 30) - */ - -class BalanceSnapshotService { - constructor() { - this.config = this.loadConfig(); - this.results = { - walletsFound: 0, - processedWallets: 0, - failedWallets: 0, - totalAdaBalance: 0, - snapshotsStored: 0, - executionTime: 0, - }; - } - - loadConfig() { - const apiBaseUrl = process.env.API_BASE_URL || 'http://localhost:3000'; - const authToken = process.env.SNAPSHOT_AUTH_TOKEN; - - if (!authToken) { - throw new Error('SNAPSHOT_AUTH_TOKEN environment variable is required'); - } - - return { - apiBaseUrl, - authToken, - batchSize: parseInt(process.env.BATCH_SIZE || '3'), - delayBetweenRequests: parseInt(process.env.DELAY_BETWEEN_REQUESTS || '3'), - delayBetweenBatches: parseInt(process.env.DELAY_BETWEEN_BATCHES || '15'), - maxRetries: parseInt(process.env.MAX_RETRIES || '3'), - requestTimeout: parseInt(process.env.REQUEST_TIMEOUT || '30'), - }; - } - - async makeRequest(/** @type {string} */ url, /** @type {RequestInit} */ options = {}) { - const controller = new AbortController(); - const timeoutId = setTimeout(() => controller.abort(), this.config.requestTimeout * 1000); - - try { - const response = await fetch(url, { - ...options, - headers: { - 'Authorization': `Bearer ${this.config.authToken}`, - 'Content-Type': 'application/json', - ...(options.headers || {}), - }, - signal: controller.signal, - }); - - clearTimeout(timeoutId); - - if (!response.ok) { - throw new Error(`HTTP ${response.status}: ${response.statusText}`); - } - - const data = await response.json(); - return { data, status: response.status }; - } catch (error) { - clearTimeout(timeoutId); - throw error; - } - } - - async delay(/** @type {number} */ seconds) { - return new Promise(resolve => setTimeout(resolve, seconds * 1000)); - } - - async fetchWallets() { - console.log('šŸ“‹ Fetching all wallets...'); - - const { data } = await this.makeRequest( - `${this.config.apiBaseUrl}/api/v1/aggregatedBalances/wallets` - ); - - console.log(`āœ… Found ${data.walletCount} wallets`); - this.results.walletsFound = data.walletCount; - - if (data.walletCount === 0) { - console.log('ā„¹ļø No wallets found, skipping snapshot process'); - return []; - } - - return data.wallets; - } - - async fetchWalletBalance(/** @type {any} */ wallet) { - const params = new URLSearchParams({ - walletId: wallet.walletId, - walletName: wallet.walletName, - signersAddresses: JSON.stringify(wallet.signersAddresses), - numRequiredSigners: wallet.numRequiredSigners.toString(), - type: wallet.type, - stakeCredentialHash: wallet.stakeCredentialHash || '', - isArchived: wallet.isArchived.toString(), - network: wallet.network.toString(), - }); - - const url = `${this.config.apiBaseUrl}/api/v1/aggregatedBalances/balance?${params}`; - - for (let attempt = 1; attempt <= this.config.maxRetries; attempt++) { - try { - const { data } = await this.makeRequest(url); - console.log(` āœ… Balance: ${data.walletBalance.adaBalance} ADA`); - return data.walletBalance; - } catch (error) { - const isLastAttempt = attempt === this.config.maxRetries; - const errorMessage = error instanceof Error ? error.message : 'Unknown error'; - - if (errorMessage.includes('429')) { - // Rate limited - wait longer before retry - const retryDelay = this.config.delayBetweenRequests * attempt * 2; - console.log(` āš ļø Rate limited (429). Waiting ${retryDelay}s before retry ${attempt}/${this.config.maxRetries}`); - await this.delay(retryDelay); - } else { - console.log(` āŒ Failed to fetch balance for wallet ${wallet.walletId}: ${errorMessage}`); - if (isLastAttempt) { - return null; - } - } - } - } - - return null; - } - - async processWalletsInBatches(/** @type {any[]} */ wallets) { - console.log(`šŸ’° Fetching balances for ${wallets.length} wallets with rate limiting...`); - console.log(`šŸ“Š Configuration: batch_size=${this.config.batchSize}, request_delay=${this.config.delayBetweenRequests}s, batch_delay=${this.config.delayBetweenBatches}s`); - - const walletBalances = []; - const totalBatches = Math.ceil(wallets.length / this.config.batchSize); - - for (let batchIndex = 0; batchIndex < totalBatches; batchIndex++) { - const batchStart = batchIndex * this.config.batchSize; - const batchEnd = Math.min(batchStart + this.config.batchSize, wallets.length); - const batchWallets = wallets.slice(batchStart, batchEnd); - - console.log(`šŸ“¦ Processing batch ${batchIndex + 1}/${totalBatches}: wallets ${batchStart + 1}-${batchEnd}`); - - for (let i = 0; i < batchWallets.length; i++) { - const wallet = batchWallets[i]; - if (!wallet) continue; - - console.log(` Processing wallet: ${wallet.walletName} (${wallet.walletId})`); - - const walletBalance = await this.fetchWalletBalance(wallet); - - if (walletBalance) { - walletBalances.push(walletBalance); - this.results.totalAdaBalance += walletBalance.adaBalance; - this.results.processedWallets++; - } else { - this.results.failedWallets++; - } - - // Delay between requests within a batch (except for the last request) - if (i < batchWallets.length - 1) { - await this.delay(this.config.delayBetweenRequests); - } - } - - // Delay between batches (except for the last batch) - if (batchIndex < totalBatches - 1) { - console.log(` ā³ Waiting ${this.config.delayBetweenBatches}s before next batch...`); - await this.delay(this.config.delayBetweenBatches); - } - } - - console.log(`šŸ“Š Balance fetching completed. Failed wallets: ${this.results.failedWallets}`); - console.log(`āœ… Successfully processed: ${walletBalances.length} wallets`); - - return walletBalances; - } - - async storeSnapshots(/** @type {any[]} */ walletBalances) { - console.log('šŸ’¾ Storing balance snapshots...'); - - const { data } = await this.makeRequest( - `${this.config.apiBaseUrl}/api/v1/aggregatedBalances/snapshots`, - { - method: 'POST', - body: JSON.stringify({ walletBalances }), - } - ); - - this.results.snapshotsStored = data.snapshotsStored; - console.log(`āœ… Successfully stored ${data.snapshotsStored} balance snapshots`); - } - - async run() { - const startTime = Date.now(); - - try { - console.log('šŸ”„ Starting daily balance snapshot process...'); - - // Step 1: Fetch all wallets - const wallets = await this.fetchWallets(); - - if (wallets.length === 0) { - console.log('ā„¹ļø No wallets to process'); - return this.results; - } - - // Step 2: Process wallets in batches - const walletBalances = await this.processWalletsInBatches(wallets); - - // Step 3: Store snapshots - if (walletBalances.length > 0) { - await this.storeSnapshots(walletBalances); - } - - // Calculate execution time - this.results.executionTime = Math.round((Date.now() - startTime) / 1000); - - // Final summary - console.log('\nšŸŽ‰ Balance snapshot process completed successfully!'); - console.log(`šŸ“Š Summary:`); - console.log(` • Wallets found: ${this.results.walletsFound}`); - console.log(` • Processed: ${this.results.processedWallets}`); - console.log(` • Failed: ${this.results.failedWallets}`); - console.log(` • Snapshots stored: ${this.results.snapshotsStored}`); - console.log(` • Total TVL: ${Math.round(this.results.totalAdaBalance * 100) / 100} ADA`); - console.log(` • Execution time: ${this.results.executionTime}s`); - - return this.results; - } catch (error) { - const errorMessage = error instanceof Error ? error.message : 'Unknown error'; - console.error('āŒ Balance snapshot process failed:', errorMessage); - throw error; - } - } -} - -// Main execution -async function main() { - try { - const service = new BalanceSnapshotService(); - await service.run(); - process.exit(0); - } catch (error) { - console.error('āŒ Script execution failed:', error); - process.exit(1); - } -} - -// Export for use in other modules -export { BalanceSnapshotService }; - -// Run if this file is executed directly -if (import.meta.url === `file://${process.argv[1]}`) { - main(); -} \ No newline at end of file diff --git a/scripts/batch-snapshot-orchestrator.ts b/scripts/batch-snapshot-orchestrator.ts new file mode 100644 index 00000000..6dc36609 --- /dev/null +++ b/scripts/batch-snapshot-orchestrator.ts @@ -0,0 +1,272 @@ +#!/usr/bin/env node + +/** + * Batch Snapshot Orchestrator + * + * This script orchestrates the batch processing of wallet snapshots by calling + * the server-side batch endpoint multiple times until all batches are complete. + * It handles timeout issues by processing wallets in small batches. + * + * Usage: + * npx tsx scripts/batch-snapshot-orchestrator.ts + * SNAPSHOT_AUTH_TOKEN=your_token npx tsx scripts/batch-snapshot-orchestrator.ts + * + * Environment Variables: + * - API_BASE_URL: Base URL for the API (default: http://localhost:3000) + * - SNAPSHOT_AUTH_TOKEN: Authentication token for API requests + * - BATCH_SIZE: Number of wallets per batch (default: 10) + * - DELAY_BETWEEN_BATCHES: Delay between batches in seconds (default: 5) + * - MAX_RETRIES: Maximum retries for failed batches (default: 3) + */ + +interface BatchProgress { + processedInBatch: number; + walletsInBatch: number; + failedInBatch: number; + snapshotsStored: number; + totalAdaBalance: number; + totalBatches: number; +} + +interface BatchResponse { + success: boolean; + message?: string; + progress: BatchProgress; +} + +interface BatchResults { + totalBatches: number; + completedBatches: number; + failedBatches: number; + totalWalletsProcessed: number; + totalWalletsFailed: number; + totalAdaBalance: number; + totalSnapshotsStored: number; + executionTime: number; +} + +interface BatchConfig { + apiBaseUrl: string; + authToken: string; + batchSize: number; + delayBetweenBatches: number; + maxRetries: number; +} + +interface ApiResponse { + data: T; + status: number; +} + +class BatchSnapshotOrchestrator { + private config: BatchConfig; + private results: BatchResults; + + constructor() { + this.config = this.loadConfig(); + this.results = { + totalBatches: 0, + completedBatches: 0, + failedBatches: 0, + totalWalletsProcessed: 0, + totalWalletsFailed: 0, + totalAdaBalance: 0, + totalSnapshotsStored: 0, + executionTime: 0, + }; + } + + private loadConfig(): BatchConfig { + const apiBaseUrl = process.env.API_BASE_URL || 'http://localhost:3000'; + const authToken = process.env.SNAPSHOT_AUTH_TOKEN; + + if (!authToken) { + throw new Error('SNAPSHOT_AUTH_TOKEN environment variable is required'); + } + + return { + apiBaseUrl, + authToken, + batchSize: parseInt(process.env.BATCH_SIZE || '10'), + delayBetweenBatches: parseInt(process.env.DELAY_BETWEEN_BATCHES || '5'), + maxRetries: parseInt(process.env.MAX_RETRIES || '3'), + }; + } + + private async makeRequest(url: string, options: RequestInit = {}): Promise> { + try { + // Add timeout to prevent hanging requests + const controller = new AbortController(); + const timeoutId = setTimeout(() => controller.abort(), 30000); // 30 second timeout + + const response = await fetch(url, { + ...options, + signal: controller.signal, + headers: { + 'Authorization': `Bearer ${this.config.authToken}`, + 'Content-Type': 'application/json', + ...(options.headers || {}), + }, + }); + + clearTimeout(timeoutId); + + if (!response.ok) { + throw new Error(`HTTP ${response.status}: ${response.statusText}`); + } + + const data = await response.json() as T; + return { data, status: response.status }; + } catch (error) { + if (error instanceof Error && error.name === 'AbortError') { + throw new Error('Request timeout after 30 seconds'); + } + throw error; + } + } + + private async delay(seconds: number): Promise { + return new Promise(resolve => setTimeout(resolve, seconds * 1000)); + } + + private async processBatch(batchNumber: number, batchId: string): Promise { + console.log(`šŸ“¦ Processing batch ${batchNumber}...`); + + for (let attempt = 1; attempt <= this.config.maxRetries; attempt++) { + try { + const url = new URL(`${this.config.apiBaseUrl}/api/v1/stats/run-snapshots-batch`); + url.searchParams.set('batchId', batchId); + url.searchParams.set('batchNumber', batchNumber.toString()); + url.searchParams.set('batchSize', this.config.batchSize.toString()); + + const { data } = await this.makeRequest(url.toString(), { + method: 'POST', + }); + + if (data.success) { + console.log(`āœ… Batch ${batchNumber} completed successfully`); + console.log(` • Processed: ${data.progress.processedInBatch}/${data.progress.walletsInBatch} wallets`); + console.log(` • Failed: ${data.progress.failedInBatch}`); + console.log(` • Snapshots stored: ${data.progress.snapshotsStored}`); + console.log(` • Batch ADA balance: ${Math.round(data.progress.totalAdaBalance * 100) / 100} ADA`); + + return data.progress; + } else { + throw new Error(data.message || 'Batch processing failed'); + } + } catch (error) { + const isLastAttempt = attempt === this.config.maxRetries; + const errorMessage = error instanceof Error ? error.message : 'Unknown error'; + + console.log(` āš ļø Batch ${batchNumber} attempt ${attempt}/${this.config.maxRetries} failed: ${errorMessage}`); + + if (isLastAttempt) { + console.error(`āŒ Batch ${batchNumber} failed after ${this.config.maxRetries} attempts`); + return null; + } + + // Wait before retry + await this.delay(this.config.delayBetweenBatches); + } + } + + return null; + } + + public async run(): Promise { + const startTime = Date.now(); + const batchId = `snapshot-${Date.now()}`; + + try { + console.log('šŸ”„ Starting batch snapshot orchestration...'); + console.log(`šŸ“Š Configuration: batch_size=${this.config.batchSize}, delay=${this.config.delayBetweenBatches}s`); + + // First, get the total number of batches by processing batch 1 + console.log('šŸ“‹ Determining total batches...'); + const firstBatch = await this.processBatch(1, batchId); + + if (!firstBatch) { + throw new Error('Failed to process first batch'); + } + + this.results.totalBatches = firstBatch.totalBatches; + this.results.completedBatches = 1; + this.results.totalWalletsProcessed += firstBatch.processedInBatch; + this.results.totalWalletsFailed += firstBatch.failedInBatch; + this.results.totalAdaBalance += firstBatch.totalAdaBalance; + this.results.totalSnapshotsStored += firstBatch.snapshotsStored; + + console.log(`šŸ“Š Total batches to process: ${this.results.totalBatches}`); + + // Process remaining batches + for (let batchNumber = 2; batchNumber <= this.results.totalBatches; batchNumber++) { + // Delay between batches to prevent overwhelming the server + console.log(`ā³ Waiting ${this.config.delayBetweenBatches}s before next batch...`); + await this.delay(this.config.delayBetweenBatches); + + const batchProgress = await this.processBatch(batchNumber, batchId); + + if (batchProgress) { + this.results.completedBatches++; + this.results.totalWalletsProcessed += batchProgress.processedInBatch; + this.results.totalWalletsFailed += batchProgress.failedInBatch; + this.results.totalAdaBalance += batchProgress.totalAdaBalance; + this.results.totalSnapshotsStored += batchProgress.snapshotsStored; + } else { + this.results.failedBatches++; + console.error(`āŒ Batch ${batchNumber} failed completely`); + } + + // Show progress + const progressPercent = Math.round((batchNumber / this.results.totalBatches) * 100); + console.log(`šŸ“ˆ Progress: ${batchNumber}/${this.results.totalBatches} batches (${progressPercent}%)`); + } + + // Calculate execution time + this.results.executionTime = Math.round((Date.now() - startTime) / 1000); + + // Final summary + console.log('\nšŸŽ‰ Batch snapshot orchestration completed!'); + console.log(`šŸ“Š Final Summary:`); + console.log(` • Total batches: ${this.results.totalBatches}`); + console.log(` • Completed: ${this.results.completedBatches}`); + console.log(` • Failed: ${this.results.failedBatches}`); + console.log(` • Wallets processed: ${this.results.totalWalletsProcessed}`); + console.log(` • Wallets failed: ${this.results.totalWalletsFailed}`); + console.log(` • Snapshots stored: ${this.results.totalSnapshotsStored}`); + console.log(` • Total TVL: ${Math.round(this.results.totalAdaBalance * 100) / 100} ADA`); + console.log(` • Execution time: ${this.results.executionTime}s`); + + if (this.results.failedBatches > 0) { + console.log(`āš ļø Warning: ${this.results.failedBatches} batches failed. You may need to retry those batches manually.`); + } + + return this.results; + } catch (error) { + const errorMessage = error instanceof Error ? error.message : 'Unknown error'; + console.error('āŒ Batch snapshot orchestration failed:', errorMessage); + throw error; + } + } +} + +// Main execution +async function main(): Promise { + try { + const orchestrator = new BatchSnapshotOrchestrator(); + await orchestrator.run(); + process.exit(0); + } catch (error) { + const errorMessage = error instanceof Error ? error.message : 'Unknown error'; + console.error('āŒ Orchestrator execution failed:', errorMessage); + process.exit(1); + } +} + +// Export for use in other modules +export { BatchSnapshotOrchestrator, type BatchResults, type BatchProgress, type BatchConfig }; + +// Run if this file is executed directly +if (import.meta.url === `file://${process.argv[1]}`) { + main(); +} diff --git a/scripts/package.json b/scripts/package.json new file mode 100644 index 00000000..473ff4bd --- /dev/null +++ b/scripts/package.json @@ -0,0 +1,15 @@ +{ + "name": "batch-snapshot-orchestrator", + "version": "1.0.0", + "description": "Batch snapshot orchestrator for wallet balance snapshots", + "type": "module", + "scripts": { + "start": "tsx batch-snapshot-orchestrator.ts" + }, + "dependencies": { + "tsx": "^4.7.0" + }, + "engines": { + "node": ">=18.0.0" + } +} diff --git a/src/pages/api/v1/aggregatedBalances/README.md b/src/pages/api/v1/aggregatedBalances/README.md deleted file mode 100644 index d7d037cb..00000000 --- a/src/pages/api/v1/aggregatedBalances/README.md +++ /dev/null @@ -1,179 +0,0 @@ -# AggregatedBalances API Routes - -This directory contains the modular API routes for handling wallet balance aggregation and snapshots. These endpoints are designed to work together to provide comprehensive wallet balance tracking with rate limiting and error handling capabilities. - -## Authentication - -All endpoints require authentication using the `SNAPSHOT_AUTH_TOKEN` environment variable: -- **Header**: `Authorization: Bearer ` -- **Environment Variable**: `SNAPSHOT_AUTH_TOKEN` - -## Routes - -### `/api/v1/aggregatedBalances/wallets` -- **Method**: GET -- **Purpose**: Returns all wallet information without fetching balances or building wallet objects -- **Authentication**: Required (Bearer token) -- **Response**: - ```json - { - "wallets": [ - { - "walletId": "string", - "walletName": "string", - "description": "string|null", - "signersAddresses": ["string"], - "signersStakeKeys": ["string"], - "signersDRepKeys": ["string"], - "signersDescriptions": ["string"], - "numRequiredSigners": number, - "verified": ["string"], - "scriptCbor": "string", - "stakeCredentialHash": "string|null", - "type": "string", - "isArchived": boolean, - "clarityApiKey": "string|null", - "network": number - } - ], - "walletCount": number, - "activeWalletCount": number, - "archivedWalletCount": number - } - ``` - -### `/api/v1/aggregatedBalances/balance` -- **Method**: GET -- **Purpose**: Fetches balance for a single wallet (builds wallet and generates addresses internally) -- **Authentication**: Required (Bearer token) -- **Query Parameters**: - - `walletId` (required) - Wallet ID - - `walletName` (required) - Wallet name - - `signersAddresses` (required) - JSON array of signer addresses - - `numRequiredSigners` (required) - Number of required signers - - `type` (required) - Wallet type - - `stakeCredentialHash` (optional) - Stake credential hash - - `isArchived` (required) - Whether wallet is archived - - `network` (required) - Network ID (0=testnet, 1=mainnet) -- **Response**: - ```json - { - "walletBalance": { - "walletId": "string", - "walletName": "string", - "address": "string", - "balance": { - "lovelace": "string", - "assetId": "quantity" - }, - "adaBalance": number, - "isArchived": boolean - } - } - ``` - -### `/api/v1/aggregatedBalances/snapshots` -- **Method**: POST -- **Purpose**: Stores balance snapshots in the database -- **Authentication**: Required (Bearer token) -- **Content-Type**: `application/json` -- **Body**: - ```json - { - "walletBalances": [ - { - "walletId": "string", - "walletName": "string", - "address": "string", - "balance": { - "lovelace": "string", - "assetId": "quantity" - }, - "adaBalance": number, - "isArchived": boolean - } - ] - } - ``` -- **Response**: - ```json - { - "snapshotsStored": number, - "totalWallets": number - } - ``` - -### `/api/v1/aggregatedBalances/test` -- **Method**: GET -- **Purpose**: Comprehensive test endpoint that validates all sub-routes with real data -- **Authentication**: Required (Bearer token) -- **Response**: - ```json - { - "message": "string", - "timestamp": "string", - "endpoints": { - "wallets": "string", - "balance": "string", - "snapshots": "string" - }, - "usage": { - "wallets": "string", - "balance": "string", - "snapshots": "string" - }, - "realData": { - "walletsFound": number, - "processedWallets": number, - "failedWallets": number, - "totalAdaBalance": number, - "sampleWallet": { - "id": "string", - "name": "string", - "adaBalance": number - }, - "snapshotsStored": number - } - } - ``` - -## Features - -The modular approach provides several advantages: - -1. **Rate Limit Mitigation**: Individual wallet balance requests can be spaced out to respect API limits -2. **Better Error Handling**: Failed wallet processing doesn't affect other wallets -3. **Modularity**: Each endpoint has a single responsibility -4. **Comprehensive Testing**: The test endpoint validates the entire workflow with real data -5. **Fallback Network Support**: Balance endpoint tries alternative networks if primary fails -6. **Batch Processing**: Snapshots endpoint handles multiple wallets efficiently - -## GitHub Actions Integration - -The daily balance snapshots workflow (`.github/workflows/daily-balance-snapshots.yml`) uses these endpoints in sequence: - -1. **Fetch Wallets**: Uses `/wallets` to get all wallet information -2. **Process Balances**: Uses `/balance` for each wallet with rate limiting: - - Batch size: 3 wallets per batch - - Delay between requests: 3 seconds - - Delay between batches: 15 seconds - - Max retries: 3 attempts -3. **Store Snapshots**: Uses `/snapshots` to persist all collected balances - -## Error Handling - -- **401 Unauthorized**: Invalid or missing authentication token -- **400 Bad Request**: Missing required parameters -- **405 Method Not Allowed**: Incorrect HTTP method -- **500 Internal Server Error**: Server-side processing errors - -## Database Schema - -The snapshots are stored in the `balanceSnapshot` table with the following structure: -- `walletId`: Wallet identifier -- `walletName`: Human-readable wallet name -- `address`: Wallet address used for balance calculation -- `adaBalance`: ADA balance in ADA units -- `assetBalances`: JSON object containing all asset balances -- `isArchived`: Whether the wallet is archived -- `createdAt`: Timestamp of snapshot creation diff --git a/src/pages/api/v1/aggregatedBalances/balance.ts b/src/pages/api/v1/aggregatedBalances/balance.ts deleted file mode 100644 index e9d94416..00000000 --- a/src/pages/api/v1/aggregatedBalances/balance.ts +++ /dev/null @@ -1,184 +0,0 @@ -import { cors, addCorsCacheBustingHeaders } from "@/lib/cors"; -import type { NextApiRequest, NextApiResponse } from "next"; -import { buildMultisigWallet } from "@/utils/common"; -import { getProvider } from "@/utils/get-provider"; -import { resolvePaymentKeyHash, serializeNativeScript } from "@meshsdk/core"; -import type { UTxO, NativeScript } from "@meshsdk/core"; -import { getBalance } from "@/utils/getBalance"; - -interface WalletBalance { - walletId: string; - walletName: string; - address: string; - balance: Record; - adaBalance: number; - isArchived: boolean; -} - -interface BalanceResponse { - walletBalance: WalletBalance; -} - -export default async function handler( - req: NextApiRequest, - res: NextApiResponse, -) { - // Add cache-busting headers for CORS - addCorsCacheBustingHeaders(res); - - await cors(req, res); - if (req.method === "OPTIONS") { - return res.status(200).end(); - } - if (req.method !== "GET") { - return res.status(405).json({ error: "Method Not Allowed" }); - } - - // Verify authentication for all requests - const authToken = req.headers.authorization?.replace('Bearer ', ''); - const expectedToken = process.env.SNAPSHOT_AUTH_TOKEN; - - if (!expectedToken) { - console.error('SNAPSHOT_AUTH_TOKEN environment variable not set'); - return res.status(500).json({ error: "Server configuration error" }); - } - - if (!authToken || authToken !== expectedToken) { - console.warn('Unauthorized request attempt', { - ip: req.headers['x-forwarded-for'] || req.connection.remoteAddress, - userAgent: req.headers['user-agent'], - authToken: authToken ? 'present' : 'missing', - query: req.query - }); - return res.status(401).json({ error: "Unauthorized" }); - } - - const { walletId, walletName, signersAddresses, numRequiredSigners, type, stakeCredentialHash, isArchived, network } = req.query; - - // Validate required parameters - if (!walletId || !walletName || !signersAddresses || !numRequiredSigners || !type || !network) { - return res.status(400).json({ error: "Missing required parameters" }); - } - - try { - const walletIdStr = walletId as string; - const walletNameStr = walletName as string; - const signersAddressesArray = JSON.parse(signersAddresses as string); - const numRequiredSignersNum = parseInt(numRequiredSigners as string); - const typeStr = type as string; - const stakeCredentialHashStr = stakeCredentialHash as string; - const isArchivedBool = isArchived === 'true'; - const networkNum = parseInt(network as string); - - // Build multisig wallet for address determination - const walletData = { - id: walletIdStr, - name: walletNameStr, - signersAddresses: signersAddressesArray, - numRequiredSigners: numRequiredSignersNum, - type: typeStr, - stakeCredentialHash: stakeCredentialHashStr, - isArchived: isArchivedBool, - description: null, - signersStakeKeys: [], - signersDRepKeys: [], - signersDescriptions: [], - clarityApiKey: null, - drepKey: null, - scriptType: null, - scriptCbor: "", // Required field for DbWallet type - verified: [], // Required field for DbWallet type - }; - - const mWallet = buildMultisigWallet(walletData, networkNum); - if (!mWallet) { - return res.status(400).json({ error: "Failed to build multisig wallet" }); - } - - // Generate addresses from the built wallet - const nativeScript = { - type: typeStr || "atLeast", - scripts: signersAddressesArray.map((addr: string) => ({ - type: "sig", - keyHash: resolvePaymentKeyHash(addr), - })), - }; - if (nativeScript.type == "atLeast") { - //@ts-ignore - nativeScript.required = numRequiredSignersNum; - } - - const paymentAddress = serializeNativeScript( - nativeScript as NativeScript, - stakeCredentialHashStr as undefined | string, - networkNum, - ).address; - - const stakeableAddress = mWallet.getScript().address; - - // Determine which address to use - const blockchainProvider = getProvider(networkNum); - - let paymentUtxos: UTxO[] = []; - let stakeableUtxos: UTxO[] = []; - - try { - paymentUtxos = await blockchainProvider.fetchAddressUTxOs(paymentAddress); - stakeableUtxos = await blockchainProvider.fetchAddressUTxOs(stakeableAddress); - } catch (utxoError) { - console.error(`Failed to fetch UTxOs for wallet ${walletIdStr}:`, utxoError); - // Continue with empty UTxOs - } - - const paymentAddrEmpty = paymentUtxos.length === 0; - let walletAddress = paymentAddress; - - if (paymentAddrEmpty && mWallet.stakingEnabled()) { - walletAddress = stakeableAddress; - } - - // Use the UTxOs from the selected address - let utxos: UTxO[] = walletAddress === stakeableAddress ? stakeableUtxos : paymentUtxos; - - // If we still have no UTxOs, try the other network as fallback - if (utxos.length === 0) { - const fallbackNetwork = networkNum === 0 ? 1 : 0; - try { - const fallbackProvider = getProvider(fallbackNetwork); - utxos = await fallbackProvider.fetchAddressUTxOs(walletAddress); - console.log(`Successfully fetched ${utxos.length} UTxOs for wallet ${walletIdStr} on fallback network ${fallbackNetwork}`); - } catch (fallbackError) { - console.error(`Failed to fetch UTxOs for wallet ${walletIdStr} on fallback network ${fallbackNetwork}:`, fallbackError); - // Continue with empty UTxOs - this wallet will show 0 balance - } - } - - // Get balance for this wallet - const balance = getBalance(utxos); - - // Calculate ADA balance - const adaBalance = balance.lovelace ? parseInt(balance.lovelace) / 1000000 : 0; - const roundedAdaBalance = Math.round(adaBalance * 100) / 100; - - const walletBalance: WalletBalance = { - walletId: walletIdStr, - walletName: walletNameStr, - address: walletAddress, - balance, - adaBalance: roundedAdaBalance, - isArchived: isArchivedBool, - }; - - const response: BalanceResponse = { - walletBalance, - }; - - res.status(200).json(response); - } catch (error) { - console.error("Error in balance handler", { - message: (error as Error)?.message, - stack: (error as Error)?.stack, - }); - res.status(500).json({ error: "Internal Server Error" }); - } -} diff --git a/src/pages/api/v1/aggregatedBalances/snapshots.ts b/src/pages/api/v1/aggregatedBalances/snapshots.ts deleted file mode 100644 index bb322a7a..00000000 --- a/src/pages/api/v1/aggregatedBalances/snapshots.ts +++ /dev/null @@ -1,99 +0,0 @@ -import { cors, addCorsCacheBustingHeaders } from "@/lib/cors"; -import type { NextApiRequest, NextApiResponse } from "next"; -import { db } from "@/server/db"; - -interface WalletBalance { - walletId: string; - walletName: string; - address: string; - balance: Record; - adaBalance: number; - isArchived: boolean; -} - -interface SnapshotsResponse { - snapshotsStored: number; - totalWallets: number; -} - -export default async function handler( - req: NextApiRequest, - res: NextApiResponse, -) { - // Add cache-busting headers for CORS - addCorsCacheBustingHeaders(res); - - await cors(req, res); - if (req.method === "OPTIONS") { - return res.status(200).end(); - } - if (req.method !== "POST") { - return res.status(405).json({ error: "Method Not Allowed" }); - } - - // Verify authentication for all requests - const authToken = req.headers.authorization?.replace('Bearer ', ''); - const expectedToken = process.env.SNAPSHOT_AUTH_TOKEN; - - if (!expectedToken) { - console.error('SNAPSHOT_AUTH_TOKEN environment variable not set'); - return res.status(500).json({ error: "Server configuration error" }); - } - - if (!authToken || authToken !== expectedToken) { - console.warn('Unauthorized request attempt', { - ip: req.headers['x-forwarded-for'] || req.connection.remoteAddress, - userAgent: req.headers['user-agent'], - authToken: authToken ? 'present' : 'missing', - body: req.body - }); - return res.status(401).json({ error: "Unauthorized" }); - } - - const { walletBalances } = req.body; - - // Validate required parameters - if (!walletBalances || !Array.isArray(walletBalances)) { - return res.status(400).json({ error: "Missing or invalid walletBalances array" }); - } - - try { - // Store individual wallet snapshots - const snapshotPromises = walletBalances.map(async (walletBalance: WalletBalance) => { - try { - await (db as any).balanceSnapshot.create({ - data: { - walletId: walletBalance.walletId, - walletName: walletBalance.walletName, - address: walletBalance.address, - adaBalance: walletBalance.adaBalance, - assetBalances: walletBalance.balance, - isArchived: walletBalance.isArchived, - }, - }); - return 1; - } catch (error) { - console.error('Failed to store snapshot for wallet %s:', walletBalance.walletId, error); - return 0; - } - }); - - const results = await Promise.all(snapshotPromises); - const snapshotsStored = results.reduce((sum: number, result: number) => sum + result, 0); - - console.log(`Stored ${snapshotsStored} balance snapshots out of ${walletBalances.length} wallets`); - - const response: SnapshotsResponse = { - snapshotsStored, - totalWallets: walletBalances.length, - }; - - res.status(200).json(response); - } catch (error) { - console.error("Error in snapshots handler", { - message: (error as Error)?.message, - stack: (error as Error)?.stack, - }); - res.status(500).json({ error: "Internal Server Error" }); - } -} diff --git a/src/pages/api/v1/aggregatedBalances/test.ts b/src/pages/api/v1/aggregatedBalances/test.ts deleted file mode 100644 index 3625de38..00000000 --- a/src/pages/api/v1/aggregatedBalances/test.ts +++ /dev/null @@ -1,103 +0,0 @@ -import { cors, addCorsCacheBustingHeaders } from "@/lib/cors"; -import type { NextApiRequest, NextApiResponse } from "next"; -import { BalanceSnapshotService } from "../../../../../scripts/balance-snapshots.js"; - -interface TestResponse { - message: string; - timestamp: string; - endpoints: { - wallets: string; - balance: string; - snapshots: string; - }; - usage: { - wallets: string; - balance: string; - snapshots: string; - }; - realData?: { - walletsFound: number; - processedWallets: number; - failedWallets: number; - totalAdaBalance: number; - snapshotsStored: number; - executionTime: number; - }; -} - -export default async function handler( - req: NextApiRequest, - res: NextApiResponse, -) { - // Add cache-busting headers for CORS - addCorsCacheBustingHeaders(res); - - await cors(req, res); - if (req.method === "OPTIONS") { - return res.status(200).end(); - } - if (req.method !== "GET") { - return res.status(405).json({ error: "Method Not Allowed" }); - } - - // Verify authentication for all requests - const authToken = req.headers.authorization?.replace('Bearer ', ''); - const expectedToken = process.env.SNAPSHOT_AUTH_TOKEN; - - if (!expectedToken) { - console.error('SNAPSHOT_AUTH_TOKEN environment variable not set'); - return res.status(500).json({ error: "Server configuration error" }); - } - - if (!authToken || authToken !== expectedToken) { - console.warn('Unauthorized request attempt', { - ip: req.headers['x-forwarded-for'] || req.connection.remoteAddress, - userAgent: req.headers['user-agent'], - authToken: authToken ? 'present' : 'missing', - query: req.query - }); - return res.status(401).json({ error: "Unauthorized" }); - } - - try { - // Set up environment for the script - const baseUrl = process.env.INTERNAL_BASE_URL || 'http://localhost:3000'; - process.env.API_BASE_URL = baseUrl; - process.env.SNAPSHOT_AUTH_TOKEN = authToken; - - // Run the balance snapshot service - const service = new BalanceSnapshotService(); - const results = await service.run(); - - const response: TestResponse = { - message: "AggregatedBalances API Test Endpoint - Real Data Test using BalanceSnapshotService", - timestamp: new Date().toISOString(), - endpoints: { - wallets: `${baseUrl}/api/v1/aggregatedBalances/wallets - Status: success`, - balance: `${baseUrl}/api/v1/aggregatedBalances/balance - Status: success`, - snapshots: `${baseUrl}/api/v1/aggregatedBalances/snapshots - Status: success`, - }, - usage: { - wallets: "GET - Returns all wallet information without balances", - balance: "GET - Fetches balance for a single wallet (requires query params)", - snapshots: "POST - Stores balance snapshots in database (requires body with walletBalances)", - }, - realData: { - walletsFound: results.walletsFound, - processedWallets: results.processedWallets, - failedWallets: results.failedWallets, - totalAdaBalance: Math.round(results.totalAdaBalance * 100) / 100, - snapshotsStored: results.snapshotsStored, - executionTime: results.executionTime, - }, - }; - - res.status(200).json(response); - } catch (error) { - console.error("Error in test handler", { - message: (error as Error)?.message, - stack: (error as Error)?.stack, - }); - res.status(500).json({ error: "Internal Server Error" }); - } -} diff --git a/src/pages/api/v1/aggregatedBalances/wallets.ts b/src/pages/api/v1/aggregatedBalances/wallets.ts deleted file mode 100644 index 615575f8..00000000 --- a/src/pages/api/v1/aggregatedBalances/wallets.ts +++ /dev/null @@ -1,139 +0,0 @@ -import { cors, addCorsCacheBustingHeaders } from "@/lib/cors"; -import type { Wallet as DbWallet } from "@prisma/client"; -import type { NextApiRequest, NextApiResponse } from "next"; -import { addressToNetwork } from "@/utils/multisigSDK"; -import { db } from "@/server/db"; - -interface WalletInfo { - walletId: string; - walletName: string; - description: string | null; - signersAddresses: string[]; - signersStakeKeys: string[]; - signersDRepKeys: string[]; - signersDescriptions: string[]; - numRequiredSigners: number; - verified: string[]; - scriptCbor: string; - stakeCredentialHash: string | null; - type: string; - isArchived: boolean; - clarityApiKey: string | null; - network: number; -} - -interface WalletsResponse { - wallets: WalletInfo[]; - walletCount: number; - activeWalletCount: number; - archivedWalletCount: number; -} - -export default async function handler( - req: NextApiRequest, - res: NextApiResponse, -) { - // Add cache-busting headers for CORS - addCorsCacheBustingHeaders(res); - - await cors(req, res); - if (req.method === "OPTIONS") { - return res.status(200).end(); - } - if (req.method !== "GET") { - return res.status(405).json({ error: "Method Not Allowed" }); - } - - // Verify authentication for all requests - const authToken = req.headers.authorization?.replace('Bearer ', ''); - const expectedToken = process.env.SNAPSHOT_AUTH_TOKEN; - - if (!expectedToken) { - console.error('SNAPSHOT_AUTH_TOKEN environment variable not set'); - return res.status(500).json({ error: "Server configuration error" }); - } - - if (!authToken || authToken !== expectedToken) { - console.warn('Unauthorized request attempt', { - ip: req.headers['x-forwarded-for'] || req.connection.remoteAddress, - userAgent: req.headers['user-agent'], - authToken: authToken ? 'present' : 'missing', - query: req.query - }); - return res.status(401).json({ error: "Unauthorized" }); - } - - try { - // Get ALL wallets from the database - const allWallets: DbWallet[] = await db.wallet.findMany(); - - if (!allWallets || allWallets.length === 0) { - return res.status(200).json({ - wallets: [], - walletCount: 0, - activeWalletCount: 0, - archivedWalletCount: 0, - }); - } - - const wallets: WalletInfo[] = []; - let activeWalletCount = 0; - let archivedWalletCount = 0; - - // Process each wallet to extract basic wallet information - for (const wallet of allWallets) { - try { - // Determine network from signer addresses - let network = 1; // Default to mainnet - if (wallet.signersAddresses.length > 0) { - const signerAddr = wallet.signersAddresses[0]!; - network = addressToNetwork(signerAddr); - } - - // Count wallet types - if (wallet.isArchived) { - archivedWalletCount++; - } else { - activeWalletCount++; - } - - wallets.push({ - walletId: wallet.id, - walletName: wallet.name, - description: wallet.description, - signersAddresses: wallet.signersAddresses, - signersStakeKeys: wallet.signersStakeKeys, - signersDRepKeys: wallet.signersDRepKeys, - signersDescriptions: wallet.signersDescriptions, - numRequiredSigners: wallet.numRequiredSigners!, - verified: wallet.verified, - scriptCbor: wallet.scriptCbor, - stakeCredentialHash: wallet.stakeCredentialHash, - type: wallet.type || "atLeast", - isArchived: wallet.isArchived, - clarityApiKey: wallet.clarityApiKey, - network, - }); - - } catch (error) { - console.error(`Error processing wallet ${wallet.id}:`, error); - // Continue with other wallets even if one fails - } - } - - const response: WalletsResponse = { - wallets, - walletCount: allWallets.length, - activeWalletCount, - archivedWalletCount, - }; - - res.status(200).json(response); - } catch (error) { - console.error("Error in wallets handler", { - message: (error as Error)?.message, - stack: (error as Error)?.stack, - }); - res.status(500).json({ error: "Internal Server Error" }); - } -} diff --git a/src/pages/api/v1/stats/README.md b/src/pages/api/v1/stats/README.md new file mode 100644 index 00000000..7d06ed8e --- /dev/null +++ b/src/pages/api/v1/stats/README.md @@ -0,0 +1,137 @@ +# Stats API Routes + +This directory contains the API route for handling wallet balance snapshots using a batch processing system to avoid timeout issues. + +## Overview + +The batch processing system addresses timeout issues when processing large numbers of wallets by: + +- **Solves**: Timeout issues with large wallet counts +- **Improves**: Reliability, monitoring, and error handling +- **Adds**: Comprehensive progress tracking and TVL reporting +- **Consolidates**: All snapshot functionality into a single, efficient endpoint + +## Authentication + +The endpoint requires authentication using the `SNAPSHOT_AUTH_TOKEN` environment variable: +- **Header**: `Authorization: Bearer ` +- **Environment Variable**: `SNAPSHOT_AUTH_TOKEN` + +## Route + +### `/api/v1/stats/run-snapshots-batch` +- **Method**: POST +- **Purpose**: Processes a batch of wallets for balance snapshots (main endpoint) +- **Authentication**: Required (Bearer token) +- **Content-Type**: `application/json` +- **Body**: + ```json + { + "batchId": "string", + "batchNumber": number, + "batchSize": number + } + ``` +- **Response**: + ```json + { + "success": boolean, + "message": "string", + "progress": { + "batchId": "string", + "totalBatches": number, + "currentBatch": number, + "walletsInBatch": number, + "processedInBatch": number, + "failedInBatch": number, + "totalProcessed": number, + "totalFailed": number, + "totalAdaBalance": number, + "snapshotsStored": number, + "isComplete": boolean, + "startedAt": "string", + "lastUpdatedAt": "string" + }, + "timestamp": "string" + } + ``` + +## Batch Processing System + +The new system processes wallets in small batches to avoid timeout issues: + +### How It Works +1. **Batch Processing**: Wallets are processed in configurable batches (default: 10 wallets per batch) +2. **Progress Tracking**: Each batch returns detailed progress information +3. **Resumable**: Can restart from any batch number if needed +4. **Fault Tolerant**: Failed batches can be retried individually + +### Orchestrator Script +The `scripts/batch-snapshot-orchestrator.js` script manages the entire process: +- Automatically processes all batches sequentially +- Handles retries for failed batches with exponential backoff +- Provides comprehensive progress reporting with emojis and detailed statistics +- Configurable batch size, delays, and retry attempts +- Calculates and reports total TVL (Total Value Locked) across all wallets +- Tracks execution time and provides final summary +- Exports the `BatchSnapshotOrchestrator` class for programmatic use + +### Configuration +- **`API_BASE_URL`**: Base URL for the API (default: http://localhost:3000) +- **`SNAPSHOT_AUTH_TOKEN`**: Authentication token for API requests (required) +- **`BATCH_SIZE`**: Wallets per batch (default: 10) +- **`DELAY_BETWEEN_BATCHES`**: Seconds between batches (default: 5) +- **`MAX_RETRIES`**: Retry attempts for failed batches (default: 3) + +## GitHub Actions Integration + +The daily balance snapshots workflow (`.github/workflows/daily-balance-snapshots.yml`) uses: +1. **Batch Orchestrator**: Runs `scripts/batch-snapshot-orchestrator.js` +2. **No Timeout Issues**: Each batch completes in under 30 seconds +3. **Comprehensive Reporting**: Detailed progress and final statistics +4. **Manual Trigger**: Currently configured for manual triggering only (schedule disabled for testing) + +**Note**: The workflow is currently set to manual trigger only. To enable daily automatic snapshots, uncomment the schedule section in the workflow file. + +## Error Handling + +- **401 Unauthorized**: Invalid or missing authentication token +- **400 Bad Request**: Missing required parameters +- **405 Method Not Allowed**: Incorrect HTTP method +- **500 Internal Server Error**: Server-side processing errors + +## Database Schema + +The snapshots are stored in the `balanceSnapshot` table with the following structure: +- `walletId`: Wallet identifier +- `walletName`: Human-readable wallet name +- `address`: Wallet address used for balance calculation +- `adaBalance`: ADA balance in ADA units +- `assetBalances`: JSON object containing all asset balances +- `isArchived`: Whether the wallet is archived +- `createdAt`: Timestamp of snapshot creation + +## Testing + +You can test the batch processing system by running the orchestrator script directly: + +```bash +# Set your authentication token +export SNAPSHOT_AUTH_TOKEN=your_token_here + +# Run the orchestrator (uses localhost by default) +node scripts/batch-snapshot-orchestrator.js + +# Or with custom configuration +API_BASE_URL=https://your-api-url.com \ +BATCH_SIZE=5 \ +DELAY_BETWEEN_BATCHES=10 \ +MAX_RETRIES=5 \ +node scripts/batch-snapshot-orchestrator.js +``` + +The orchestrator will: +- Process all wallets in configurable batches +- Provide detailed progress reporting +- Handle retries for failed batches +- Show comprehensive final statistics including total TVL \ No newline at end of file diff --git a/src/pages/api/v1/stats/run-snapshots-batch.ts b/src/pages/api/v1/stats/run-snapshots-batch.ts new file mode 100644 index 00000000..df7d2b68 --- /dev/null +++ b/src/pages/api/v1/stats/run-snapshots-batch.ts @@ -0,0 +1,350 @@ +import { cors, addCorsCacheBustingHeaders } from "@/lib/cors"; +import type { NextApiRequest, NextApiResponse } from "next"; +import { db } from "@/server/db"; +import { buildMultisigWallet } from "@/utils/common"; +import { getProvider } from "@/utils/get-provider"; +import { resolvePaymentKeyHash, serializeNativeScript } from "@meshsdk/core"; +import type { UTxO, NativeScript } from "@meshsdk/core"; +import { getBalance } from "@/utils/getBalance"; +import { addressToNetwork } from "@/utils/multisigSDK"; +import type { Wallet as DbWallet } from "@prisma/client"; + +interface WalletBalance { + walletId: string; + walletName: string; + address: string; + balance: Record; + adaBalance: number; + isArchived: boolean; +} + +interface BatchProgress { + batchId: string; + totalBatches: number; + currentBatch: number; + walletsInBatch: number; + processedInBatch: number; + failedInBatch: number; + totalProcessed: number; + totalFailed: number; + totalAdaBalance: number; + snapshotsStored: number; + isComplete: boolean; + startedAt: string; + lastUpdatedAt: string; +} + +interface BatchResponse { + success: boolean; + message: string; + progress: BatchProgress; + timestamp: string; +} + +export default async function handler( + req: NextApiRequest, + res: NextApiResponse, +) { + // Add cache-busting headers for CORS + addCorsCacheBustingHeaders(res); + + await cors(req, res); + if (req.method === "OPTIONS") { + return res.status(200).end(); + } + if (req.method !== "POST") { + return res.status(405).json({ error: "Method Not Allowed" }); + } + + // Verify authentication for all requests + const authToken = req.headers.authorization?.replace('Bearer ', ''); + const expectedToken = process.env.SNAPSHOT_AUTH_TOKEN; + + if (!expectedToken) { + console.error('SNAPSHOT_AUTH_TOKEN environment variable not set'); + return res.status(500).json({ error: "Server configuration error" }); + } + + if (!authToken || authToken !== expectedToken) { + console.warn('Unauthorized request attempt', { + ip: req.headers['x-forwarded-for'] || req.connection.remoteAddress, + userAgent: req.headers['user-agent'], + authTokenProvided: !!authToken, + timestamp: new Date().toISOString() + }); + return res.status(401).json({ error: "Unauthorized" }); + } + + const { batchId, batchNumber, batchSize } = req.query; + const startTime = new Date().toISOString(); + + // Convert string parameters to numbers + const parsedBatchNumber = batchNumber ? parseInt(batchNumber as string, 10) : 1; + const parsedBatchSize = batchSize ? parseInt(batchSize as string, 10) : 10; + + try { + console.log(`šŸ”„ Starting batch ${parsedBatchNumber} of balance snapshots...`); + + // Step 1: Get total wallet count and calculate batches + const totalWallets = await db.wallet.count(); + const totalBatches = Math.ceil(totalWallets / parsedBatchSize); + const currentBatch = parsedBatchNumber; + const offset = (currentBatch - 1) * parsedBatchSize; + + console.log(`šŸ“Š Processing batch ${currentBatch}/${totalBatches} (${parsedBatchSize} wallets per batch)`); + + // Step 2: Fetch wallets for this batch + const wallets: DbWallet[] = await db.wallet.findMany({ + skip: offset, + take: parsedBatchSize, + orderBy: { id: 'asc' }, // Consistent ordering + }); + + if (wallets.length === 0) { + return res.status(200).json({ + success: true, + message: "No wallets found in this batch", + progress: { + batchId: (batchId as string) || `batch-${currentBatch}`, + totalBatches, + currentBatch, + walletsInBatch: 0, + processedInBatch: 0, + failedInBatch: 0, + totalProcessed: 0, + totalFailed: 0, + totalAdaBalance: 0, + snapshotsStored: 0, + isComplete: true, + startedAt: startTime, + lastUpdatedAt: new Date().toISOString(), + }, + timestamp: new Date().toISOString(), + }); + } + + // Step 3: Process wallets in this batch + const walletBalances: WalletBalance[] = []; + let processedInBatch = 0; + let failedInBatch = 0; + let totalAdaBalance = 0; + + for (const wallet of wallets) { + try { + console.log(` Processing wallet: (${wallet.id.slice(0, 8)}...)`); + + // Determine network from signer addresses + let network = 1; // Default to mainnet + if (wallet.signersAddresses.length > 0) { + const signerAddr = wallet.signersAddresses[0]!; + network = addressToNetwork(signerAddr); + } + + // Build multisig wallet for address determination + const walletData = { + id: wallet.id, + name: wallet.name, + signersAddresses: wallet.signersAddresses, + numRequiredSigners: wallet.numRequiredSigners!, + type: wallet.type || "atLeast", + stakeCredentialHash: wallet.stakeCredentialHash, + isArchived: wallet.isArchived, + description: wallet.description, + signersStakeKeys: wallet.signersStakeKeys, + signersDRepKeys: wallet.signersDRepKeys, + signersDescriptions: wallet.signersDescriptions, + clarityApiKey: wallet.clarityApiKey, + drepKey: null, + scriptType: null, + scriptCbor: wallet.scriptCbor, + verified: wallet.verified, + }; + + const mWallet = buildMultisigWallet(walletData, network); + if (!mWallet) { + console.error(`Failed to build multisig wallet for ${wallet.id.slice(0, 8)}...`); + failedInBatch++; + continue; + } + + // Generate addresses from the built wallet + const nativeScript = { + type: wallet.type || "atLeast", + scripts: wallet.signersAddresses.map((addr: string) => ({ + type: "sig", + keyHash: resolvePaymentKeyHash(addr), + })), + }; + if (nativeScript.type == "atLeast") { + //@ts-ignore + nativeScript.required = wallet.numRequiredSigners!; + } + + const paymentAddress = serializeNativeScript( + nativeScript as NativeScript, + wallet.stakeCredentialHash as undefined | string, + network, + ).address; + + const stakeableAddress = mWallet.getScript().address; + + // Determine which address to use + const blockchainProvider = getProvider(network); + + let paymentUtxos: UTxO[] = []; + let stakeableUtxos: UTxO[] = []; + + try { + paymentUtxos = await blockchainProvider.fetchAddressUTxOs(paymentAddress); + stakeableUtxos = await blockchainProvider.fetchAddressUTxOs(stakeableAddress); + } catch (utxoError) { + console.error(`Failed to fetch UTxOs for wallet ${wallet.id.slice(0, 8)}...:`, utxoError); + // Continue with empty UTxOs + } + + const paymentAddrEmpty = paymentUtxos.length === 0; + let walletAddress = paymentAddress; + + if (paymentAddrEmpty && mWallet.stakingEnabled()) { + walletAddress = stakeableAddress; + } + + // Use the UTxOs from the selected address + let utxos: UTxO[] = walletAddress === stakeableAddress ? stakeableUtxos : paymentUtxos; + + // If we still have no UTxOs, try the other network as fallback + if (utxos.length === 0) { + const fallbackNetwork = network === 0 ? 1 : 0; + try { + const fallbackProvider = getProvider(fallbackNetwork); + utxos = await fallbackProvider.fetchAddressUTxOs(walletAddress); + console.log(`Successfully fetched ${utxos.length} UTxOs for wallet ${wallet.id.slice(0, 8)}... on fallback network ${fallbackNetwork}`); + } catch (fallbackError) { + console.error(`Failed to fetch UTxOs for wallet ${wallet.id.slice(0, 8)}... on fallback network ${fallbackNetwork}:`, fallbackError); + // Continue with empty UTxOs - this wallet will show 0 balance + } + } + + // Get balance for this wallet + const balance = getBalance(utxos); + + // Calculate ADA balance + const adaBalance = balance.lovelace ? parseInt(balance.lovelace) / 1000000 : 0; + const roundedAdaBalance = Math.round(adaBalance * 100) / 100; + + const walletBalance: WalletBalance = { + walletId: wallet.id, + walletName: wallet.name, + address: walletAddress, + balance, + adaBalance: roundedAdaBalance, + isArchived: wallet.isArchived, + }; + + walletBalances.push(walletBalance); + totalAdaBalance += roundedAdaBalance; + processedInBatch++; + + console.log(` āœ… Balance: ${roundedAdaBalance} ADA`); + + } catch (error) { + console.error(`Error processing wallet ${wallet.id.slice(0, 8)}...:`, error); + failedInBatch++; + } + } + + // Step 4: Store snapshots for this batch + let snapshotsStored = 0; + if (walletBalances.length > 0) { + console.log(`šŸ’¾ Storing ${walletBalances.length} balance snapshots...`); + + const snapshotPromises = walletBalances.map(async (walletBalance: WalletBalance) => { + try { + await (db as any).balanceSnapshot.create({ + data: { + walletId: walletBalance.walletId, + walletName: walletBalance.walletName, + address: walletBalance.address, + adaBalance: walletBalance.adaBalance, + assetBalances: walletBalance.balance, + isArchived: walletBalance.isArchived, + }, + }); + return 1; + } catch (error) { + console.error('Failed to store snapshot for wallet %s:', walletBalance.walletId.slice(0, 8) + '...', error); + return 0; + } + }); + + const snapshotResults = await Promise.all(snapshotPromises); + snapshotsStored = snapshotResults.reduce((sum: number, result: number) => sum + result, 0); + + console.log(`āœ… Successfully stored ${snapshotsStored} balance snapshots`); + } + + // Step 5: Calculate progress + const isComplete = currentBatch >= totalBatches; + const totalProcessed = (currentBatch - 1) * parsedBatchSize + processedInBatch; + const totalFailed = (currentBatch - 1) * parsedBatchSize + failedInBatch; + + console.log(`šŸ“Š Batch ${currentBatch}/${totalBatches} completed:`); + console.log(` • Processed: ${processedInBatch}/${wallets.length}`); + console.log(` • Failed: ${failedInBatch}`); + console.log(` • Snapshots stored: ${snapshotsStored}`); + console.log(` • Batch ADA balance: ${Math.round(totalAdaBalance * 100) / 100} ADA`); + console.log(` • Overall progress: ${totalProcessed}/${totalWallets} wallets`); + + const progress: BatchProgress = { + batchId: (batchId as string) || `batch-${currentBatch}`, + totalBatches, + currentBatch, + walletsInBatch: wallets.length, + processedInBatch, + failedInBatch, + totalProcessed, + totalFailed, + totalAdaBalance, + snapshotsStored, + isComplete, + startedAt: startTime, + lastUpdatedAt: new Date().toISOString(), + }; + + const response: BatchResponse = { + success: true, + message: isComplete + ? `All ${totalBatches} batches completed successfully` + : `Batch ${currentBatch}/${totalBatches} completed. Call next batch with batchNumber: ${currentBatch + 1}`, + progress, + timestamp: new Date().toISOString(), + }; + + res.status(200).json(response); + + } catch (error) { + const errorMessage = error instanceof Error ? error.message : 'Unknown error'; + console.error('āŒ Batch snapshot process failed:', errorMessage); + + res.status(500).json({ + success: false, + message: `Batch snapshot process failed: ${errorMessage}`, + progress: { + batchId: (batchId as string) || `batch-${parsedBatchNumber}`, + totalBatches: 0, + currentBatch: parsedBatchNumber, + walletsInBatch: 0, + processedInBatch: 0, + failedInBatch: 0, + totalProcessed: 0, + totalFailed: 0, + totalAdaBalance: 0, + snapshotsStored: 0, + isComplete: false, + startedAt: startTime, + lastUpdatedAt: new Date().toISOString(), + }, + timestamp: new Date().toISOString(), + }); + } +}