From 12e667973547253618beff1a99ac3266d4c8c0ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Diamond?= <32074058+Andre-Diamond@users.noreply.github.com> Date: Fri, 3 Oct 2025 08:42:31 +0200 Subject: [PATCH 1/7] feat(aggregate-balances): endpoint to fetch all wallet balances - Get all wallets from db - Generate all addresses - Fetch all balances --- src/pages/api/v1/aggregatedBalances.ts | 230 +++++++++++++++++++++++++ 1 file changed, 230 insertions(+) create mode 100644 src/pages/api/v1/aggregatedBalances.ts diff --git a/src/pages/api/v1/aggregatedBalances.ts b/src/pages/api/v1/aggregatedBalances.ts new file mode 100644 index 00000000..84d6a43e --- /dev/null +++ b/src/pages/api/v1/aggregatedBalances.ts @@ -0,0 +1,230 @@ +import { cors, addCorsCacheBustingHeaders } from "@/lib/cors"; +import type { Wallet as DbWallet } from "@prisma/client"; +import type { NextApiRequest, NextApiResponse } from "next"; +import { buildMultisigWallet } from "@/utils/common"; +import { getProvider } from "@/utils/get-provider"; +import { addressToNetwork } from "@/utils/multisigSDK"; +import type { UTxO, NativeScript } from "@meshsdk/core"; +import { resolvePaymentKeyHash, serializeNativeScript } from "@meshsdk/core"; +import { db } from "@/server/db"; +import { getBalance } from "@/utils/getBalance"; + +interface WalletBalance { + walletId: string; + walletName: string; + address: string; + balance: Record; + adaBalance: number; + isArchived: boolean; +} + +interface TVLResponse { + totalValueLocked: { + ada: number; + assets: Record; + }; + walletCount: number; + activeWalletCount: number; + archivedWalletCount: number; + walletBalances: WalletBalance[]; +} + +export default async function handler( + req: NextApiRequest, + res: NextApiResponse, +) { + // Add cache-busting headers for CORS + addCorsCacheBustingHeaders(res); + + await cors(req, res); + if (req.method === "OPTIONS") { + return res.status(200).end(); + } + if (req.method !== "GET") { + return res.status(405).json({ error: "Method Not Allowed" }); + } + + try { + // Get ALL wallets from the database for TVL calculation + const allWallets: DbWallet[] = await db.wallet.findMany(); + + if (!allWallets || allWallets.length === 0) { + return res.status(200).json({ + totalValueLocked: { + ada: 0, + assets: {}, + }, + walletCount: 0, + activeWalletCount: 0, + archivedWalletCount: 0, + walletBalances: [], + }); + } + + const walletBalances: WalletBalance[] = []; + const totalAssets: Record = {}; + let totalAdaBalance = 0; + let activeWalletCount = 0; + let archivedWalletCount = 0; + + // Process each wallet + for (const wallet of allWallets) { + try { + // Determine network from signer addresses + let network = 1; // Default to mainnet + if (wallet.signersAddresses.length > 0) { + const signerAddr = wallet.signersAddresses[0]!; + network = addressToNetwork(signerAddr); + console.log(`Network detection for wallet ${wallet.id}:`, { + signerAddress: signerAddr, + detectedNetwork: network, + isTestnet: signerAddr.includes("test") + }); + } + + const mWallet = buildMultisigWallet(wallet, network); + if (!mWallet) { + console.warn(`Failed to build multisig wallet for wallet ${wallet.id}`); + continue; + } + + // Use the same address logic as the frontend buildWallet function + const nativeScript = { + type: wallet.type ? wallet.type : "atLeast", + scripts: wallet.signersAddresses.map((addr) => ({ + type: "sig", + keyHash: resolvePaymentKeyHash(addr), + })), + }; + if (nativeScript.type == "atLeast") { + //@ts-ignore + nativeScript.required = wallet.numRequiredSigners!; + } + + const paymentAddress = serializeNativeScript( + nativeScript as NativeScript, + wallet.stakeCredentialHash as undefined | string, + network, + ).address; + + let walletAddress = paymentAddress; + const stakeableAddress = mWallet.getScript().address; + + // Check if payment address is empty and use stakeable address if staking is enabled + // We'll fetch UTxOs for both addresses to determine which one to use + const blockchainProvider = getProvider(network); + + let paymentUtxos: UTxO[] = []; + let stakeableUtxos: UTxO[] = []; + + try { + paymentUtxos = await blockchainProvider.fetchAddressUTxOs(paymentAddress); + stakeableUtxos = await blockchainProvider.fetchAddressUTxOs(stakeableAddress); + } catch (utxoError) { + console.error(`Failed to fetch UTxOs for wallet ${wallet.id}:`, utxoError); + // Continue with empty UTxOs + } + + const paymentAddrEmpty = paymentUtxos.length === 0; + if (paymentAddrEmpty && mWallet.stakingEnabled()) { + walletAddress = stakeableAddress; + } + + console.log(`Processing wallet ${wallet.id}:`, { + walletName: wallet.name, + signerAddresses: wallet.signersAddresses, + network, + paymentAddress, + stakeableAddress, + selectedAddress: walletAddress, + paymentUtxos: paymentUtxos.length, + stakeableUtxos: stakeableUtxos.length, + }); + + // Use the UTxOs from the selected address + let utxos: UTxO[] = walletAddress === stakeableAddress ? stakeableUtxos : paymentUtxos; + + // If we still have no UTxOs, try the other network as fallback + if (utxos.length === 0) { + const fallbackNetwork = network === 0 ? 1 : 0; + try { + const fallbackProvider = getProvider(fallbackNetwork); + utxos = await fallbackProvider.fetchAddressUTxOs(walletAddress); + console.log(`Successfully fetched ${utxos.length} UTxOs for wallet ${wallet.id} on fallback network ${fallbackNetwork}`); + } catch (fallbackError) { + console.error(`Failed to fetch UTxOs for wallet ${wallet.id} on fallback network ${fallbackNetwork}:`, fallbackError); + // Continue with empty UTxOs - this wallet will show 0 balance + } + } + + // Get balance for this wallet + const balance = getBalance(utxos); + + // Calculate ADA balance + const adaBalance = balance.lovelace ? parseInt(balance.lovelace) / 1000000 : 0; + const roundedAdaBalance = Math.round(adaBalance * 100) / 100; + + // Count wallet types + if (wallet.isArchived) { + archivedWalletCount++; + } else { + activeWalletCount++; + } + + // Add to wallet balances + walletBalances.push({ + walletId: wallet.id, + walletName: wallet.name, + address: walletAddress, + balance, + adaBalance: roundedAdaBalance, + isArchived: wallet.isArchived, + }); + + // Aggregate total balances + totalAdaBalance += roundedAdaBalance; + + // Aggregate all assets + Object.entries(balance).forEach(([asset, amount]) => { + const numericAmount = parseFloat(amount); + if (totalAssets[asset]) { + totalAssets[asset] += numericAmount; + } else { + totalAssets[asset] = numericAmount; + } + }); + + } catch (error) { + console.error(`Error processing wallet ${wallet.id}:`, error); + // Continue with other wallets even if one fails + } + } + + // Convert total assets back to string format + const totalAssetsString = Object.fromEntries( + Object.entries(totalAssets).map(([key, value]) => [ + key, + value.toString(), + ]), + ); + + const response: TVLResponse = { + totalValueLocked: { + ada: Math.round(totalAdaBalance * 100) / 100, + assets: totalAssetsString, + }, + walletCount: allWallets.length, + activeWalletCount, + archivedWalletCount, + walletBalances, + }; + + res.status(200).json(response); + } catch (error) { + console.error("Error in aggregatedBalances handler", { + message: (error as Error)?.message, + stack: (error as Error)?.stack, + }); + res.status(500).json({ error: "Internal Server Error" }); + } +} From e6c35abb34b928e9cd0c4b61da57661c7a88cf47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Diamond?= <32074058+Andre-Diamond@users.noreply.github.com> Date: Mon, 6 Oct 2025 12:04:14 +0200 Subject: [PATCH 2/7] feat(balance-snapshot): add BalanceSnapshot model and remove aggregatedBalances endpoint - Introduced BalanceSnapshot model to track wallet balances over time. - Removed the aggregatedBalances API endpoint as it is no longer needed. --- .github/workflows/daily-balance-snapshots.yml | 212 +++++++++++++++ .../migration.sql | 22 ++ prisma/schema.prisma | 11 + src/pages/api/v1/aggregatedBalances.ts | 230 ----------------- src/pages/api/v1/aggregatedBalances/README.md | 176 +++++++++++++ .../api/v1/aggregatedBalances/balance.ts | 163 ++++++++++++ .../api/v1/aggregatedBalances/snapshots.ts | 99 +++++++ src/pages/api/v1/aggregatedBalances/test.ts | 241 ++++++++++++++++++ .../api/v1/aggregatedBalances/wallets.ts | 159 ++++++++++++ 9 files changed, 1083 insertions(+), 230 deletions(-) create mode 100644 .github/workflows/daily-balance-snapshots.yml create mode 100644 prisma/migrations/20251006065720_add_balance_snapshots/migration.sql delete mode 100644 src/pages/api/v1/aggregatedBalances.ts create mode 100644 src/pages/api/v1/aggregatedBalances/README.md create mode 100644 src/pages/api/v1/aggregatedBalances/balance.ts create mode 100644 src/pages/api/v1/aggregatedBalances/snapshots.ts create mode 100644 src/pages/api/v1/aggregatedBalances/test.ts create mode 100644 src/pages/api/v1/aggregatedBalances/wallets.ts diff --git a/.github/workflows/daily-balance-snapshots.yml b/.github/workflows/daily-balance-snapshots.yml new file mode 100644 index 00000000..48c5243f --- /dev/null +++ b/.github/workflows/daily-balance-snapshots.yml @@ -0,0 +1,212 @@ +name: Daily Balance Snapshots + +# This workflow takes daily snapshots of wallet balances and stores them in the database. +# API requests require SNAPSHOT_AUTH_TOKEN secret to be set in GitHub repository settings. + +on: + #schedule: + # Run at midnight UTC every day + #- cron: '0 0 * * *' + # Allow manual triggering for testing + workflow_dispatch: + +jobs: + snapshot-balances: + runs-on: ubuntu-latest + + steps: + - name: Take balance snapshots + run: | + # Configuration - adjust these values based on your needs + API_BASE_URL="https://multisig.meshjs.dev" + AUTH_TOKEN="${{ secrets.SNAPSHOT_AUTH_TOKEN }}" + + # Rate limiting configuration + BATCH_SIZE=3 # Wallets per batch (adjust based on Blockfrost plan) + DELAY_BETWEEN_REQUESTS=3 # Seconds between requests (20/min = 3s) + DELAY_BETWEEN_BATCHES=15 # Seconds between batches + MAX_RETRIES=3 # Max retries for failed requests + REQUEST_TIMEOUT=30 # Request timeout in seconds + + echo "šŸ”„ Starting daily balance snapshot process..." + echo "šŸ“Š Configuration: batch_size=$BATCH_SIZE, request_delay=${DELAY_BETWEEN_REQUESTS}s, batch_delay=${DELAY_BETWEEN_BATCHES}s" + + # Step 1: Get all wallets + echo "šŸ“‹ Fetching all wallets..." + wallets_response=$(curl -s -w "\n%{http_code}" \ + -H "Authorization: Bearer $AUTH_TOKEN" \ + "$API_BASE_URL/api/v1/aggregatedBalances/wallets") + + wallets_http_code=$(echo "$wallets_response" | tail -n1) + wallets_body=$(echo "$wallets_response" | head -n -1) + + echo "Wallets HTTP Status: $wallets_http_code" + + if [ "$wallets_http_code" -ne 200 ]; then + echo "āŒ Failed to fetch wallets. HTTP Status: $wallets_http_code" + echo "Response: $wallets_body" + exit 1 + fi + + # Extract wallet data + wallet_count=$(echo "$wallets_body" | jq -r '.walletCount') + echo "āœ… Found $wallet_count wallets" + + if [ "$wallet_count" -eq 0 ]; then + echo "ā„¹ļø No wallets found, skipping snapshot process" + exit 0 + fi + + # Step 2: Get balances for each wallet with rate limiting + echo "šŸ’° Fetching balances for each wallet with rate limiting..." + + # Create temporary files for collecting results + temp_balances="/tmp/wallet_balances.json" + temp_failed="/tmp/failed_wallets.txt" + echo "[]" > "$temp_balances" + echo "0" > "$temp_failed" + + # Process wallets in batches to respect rate limits + batch_size=$BATCH_SIZE + delay_between_requests=$DELAY_BETWEEN_REQUESTS + delay_between_batches=$DELAY_BETWEEN_BATCHES + + # Convert wallets to array for batch processing + wallets_array=$(echo "$wallets_body" | jq -r '.wallets') + total_wallets=$(echo "$wallets_array" | jq 'length') + echo "Processing $total_wallets wallets in batches of $batch_size" + + # Process wallets in batches + for ((i=0; i "$temp_balances" + + success=true + elif [ "$balance_http_code" -eq 429 ]; then + # Rate limited - wait longer before retry + retry_delay=$((delay_between_requests * (retry_count + 1) * 2)) + echo " āš ļø Rate limited (429). Waiting ${retry_delay}s before retry $((retry_count + 1))/$max_retries" + sleep $retry_delay + retry_count=$((retry_count + 1)) + else + echo " āŒ Failed to fetch balance for wallet $wallet_id: $balance_http_code" + failed_count=$(cat "$temp_failed") + echo $((failed_count + 1)) > "$temp_failed" + success=true # Don't retry on non-rate-limit errors + fi + done + + if [ "$success" = false ]; then + echo " āŒ Max retries exceeded for wallet $wallet_id" + failed_count=$(cat "$temp_failed") + echo $((failed_count + 1)) > "$temp_failed" + fi + + # Delay between requests within a batch + if [ $((j+1)) -lt $batch_end ]; then + sleep $delay_between_requests + fi + done + + # Delay between batches (except for the last batch) + if [ $batch_end -lt $total_wallets ]; then + echo " ā³ Waiting ${delay_between_batches}s before next batch..." + sleep $delay_between_batches + fi + done + + # Read final results + wallet_balances=$(cat "$temp_balances") + failed_wallets=$(cat "$temp_failed") + + echo "šŸ“Š Balance fetching completed. Failed wallets: $failed_wallets" + echo "āœ… Successfully processed: $(echo "$wallet_balances" | jq 'length') wallets" + + # Step 3: Store snapshots using the collected balances + echo "šŸ’¾ Storing balance snapshots..." + snapshots_response=$(curl -s -w "\n%{http_code}" \ + -H "Authorization: Bearer $AUTH_TOKEN" \ + -H "Content-Type: application/json" \ + -d "{\"walletBalances\": $wallet_balances}" \ + "$API_BASE_URL/api/v1/aggregatedBalances/snapshots") + + snapshots_http_code=$(echo "$snapshots_response" | tail -n1) + snapshots_body=$(echo "$snapshots_response" | head -n -1) + + echo "Snapshots HTTP Status: $snapshots_http_code" + echo "Response: $snapshots_body" + + # Check if the request was successful + if [ "$snapshots_http_code" -eq 200 ]; then + # Parse the response to get the number of snapshots stored + snapshots_stored=$(echo "$snapshots_body" | jq -r '.snapshotsStored // 0') + total_tvl=$(echo "$snapshots_body" | jq -r '.totalValueLocked.ada') + echo "āœ… Successfully stored $snapshots_stored balance snapshots" + echo "šŸ’° Total Value Locked: $total_tvl ADA" + + # Optional: Send notification on success (you can add Discord/Slack webhook here) + # curl -X POST -H 'Content-type: application/json' \ + # --data "{\"text\":\"āœ… Daily balance snapshots completed: $snapshots_stored snapshots stored, TVL: $total_tvl ADA\"}" \ + # ${{ secrets.DISCORD_WEBHOOK_URL }} + else + echo "āŒ Failed to store balance snapshots. HTTP Status: $snapshots_http_code" + echo "Response: $snapshots_body" + exit 1 + fi + + - name: Notify on failure + if: failure() + run: | + echo "āŒ Daily balance snapshot job failed" + # Optional: Send failure notification + # curl -X POST -H 'Content-type: application/json' \ + # --data "{\"text\":\"āŒ Daily balance snapshots failed. Check the GitHub Actions logs.\"}" \ + # ${{ secrets.DISCORD_WEBHOOK_URL }} diff --git a/prisma/migrations/20251006065720_add_balance_snapshots/migration.sql b/prisma/migrations/20251006065720_add_balance_snapshots/migration.sql new file mode 100644 index 00000000..35c07d68 --- /dev/null +++ b/prisma/migrations/20251006065720_add_balance_snapshots/migration.sql @@ -0,0 +1,22 @@ +-- AlterTable +ALTER TABLE "NewWallet" ADD COLUMN "stakeCredentialHash" TEXT; + +-- CreateTable +CREATE TABLE "BalanceSnapshot" ( + "id" TEXT NOT NULL, + "walletId" TEXT NOT NULL, + "walletName" TEXT NOT NULL, + "address" TEXT NOT NULL, + "adaBalance" DECIMAL(65,30) NOT NULL, + "assetBalances" JSONB NOT NULL, + "isArchived" BOOLEAN NOT NULL, + "snapshotDate" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "BalanceSnapshot_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE INDEX "BalanceSnapshot_snapshotDate_idx" ON "BalanceSnapshot"("snapshotDate"); + +-- CreateIndex +CREATE INDEX "BalanceSnapshot_walletId_idx" ON "BalanceSnapshot"("walletId"); diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 4d8f8080..c5d8a416 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -97,3 +97,14 @@ model Ballot { type Int createdAt DateTime @default(now()) } + +model BalanceSnapshot { + id String @id @default(cuid()) + walletId String + walletName String + address String + adaBalance Decimal + assetBalances Json + isArchived Boolean + snapshotDate DateTime @default(now()) +} \ No newline at end of file diff --git a/src/pages/api/v1/aggregatedBalances.ts b/src/pages/api/v1/aggregatedBalances.ts deleted file mode 100644 index 84d6a43e..00000000 --- a/src/pages/api/v1/aggregatedBalances.ts +++ /dev/null @@ -1,230 +0,0 @@ -import { cors, addCorsCacheBustingHeaders } from "@/lib/cors"; -import type { Wallet as DbWallet } from "@prisma/client"; -import type { NextApiRequest, NextApiResponse } from "next"; -import { buildMultisigWallet } from "@/utils/common"; -import { getProvider } from "@/utils/get-provider"; -import { addressToNetwork } from "@/utils/multisigSDK"; -import type { UTxO, NativeScript } from "@meshsdk/core"; -import { resolvePaymentKeyHash, serializeNativeScript } from "@meshsdk/core"; -import { db } from "@/server/db"; -import { getBalance } from "@/utils/getBalance"; - -interface WalletBalance { - walletId: string; - walletName: string; - address: string; - balance: Record; - adaBalance: number; - isArchived: boolean; -} - -interface TVLResponse { - totalValueLocked: { - ada: number; - assets: Record; - }; - walletCount: number; - activeWalletCount: number; - archivedWalletCount: number; - walletBalances: WalletBalance[]; -} - -export default async function handler( - req: NextApiRequest, - res: NextApiResponse, -) { - // Add cache-busting headers for CORS - addCorsCacheBustingHeaders(res); - - await cors(req, res); - if (req.method === "OPTIONS") { - return res.status(200).end(); - } - if (req.method !== "GET") { - return res.status(405).json({ error: "Method Not Allowed" }); - } - - try { - // Get ALL wallets from the database for TVL calculation - const allWallets: DbWallet[] = await db.wallet.findMany(); - - if (!allWallets || allWallets.length === 0) { - return res.status(200).json({ - totalValueLocked: { - ada: 0, - assets: {}, - }, - walletCount: 0, - activeWalletCount: 0, - archivedWalletCount: 0, - walletBalances: [], - }); - } - - const walletBalances: WalletBalance[] = []; - const totalAssets: Record = {}; - let totalAdaBalance = 0; - let activeWalletCount = 0; - let archivedWalletCount = 0; - - // Process each wallet - for (const wallet of allWallets) { - try { - // Determine network from signer addresses - let network = 1; // Default to mainnet - if (wallet.signersAddresses.length > 0) { - const signerAddr = wallet.signersAddresses[0]!; - network = addressToNetwork(signerAddr); - console.log(`Network detection for wallet ${wallet.id}:`, { - signerAddress: signerAddr, - detectedNetwork: network, - isTestnet: signerAddr.includes("test") - }); - } - - const mWallet = buildMultisigWallet(wallet, network); - if (!mWallet) { - console.warn(`Failed to build multisig wallet for wallet ${wallet.id}`); - continue; - } - - // Use the same address logic as the frontend buildWallet function - const nativeScript = { - type: wallet.type ? wallet.type : "atLeast", - scripts: wallet.signersAddresses.map((addr) => ({ - type: "sig", - keyHash: resolvePaymentKeyHash(addr), - })), - }; - if (nativeScript.type == "atLeast") { - //@ts-ignore - nativeScript.required = wallet.numRequiredSigners!; - } - - const paymentAddress = serializeNativeScript( - nativeScript as NativeScript, - wallet.stakeCredentialHash as undefined | string, - network, - ).address; - - let walletAddress = paymentAddress; - const stakeableAddress = mWallet.getScript().address; - - // Check if payment address is empty and use stakeable address if staking is enabled - // We'll fetch UTxOs for both addresses to determine which one to use - const blockchainProvider = getProvider(network); - - let paymentUtxos: UTxO[] = []; - let stakeableUtxos: UTxO[] = []; - - try { - paymentUtxos = await blockchainProvider.fetchAddressUTxOs(paymentAddress); - stakeableUtxos = await blockchainProvider.fetchAddressUTxOs(stakeableAddress); - } catch (utxoError) { - console.error(`Failed to fetch UTxOs for wallet ${wallet.id}:`, utxoError); - // Continue with empty UTxOs - } - - const paymentAddrEmpty = paymentUtxos.length === 0; - if (paymentAddrEmpty && mWallet.stakingEnabled()) { - walletAddress = stakeableAddress; - } - - console.log(`Processing wallet ${wallet.id}:`, { - walletName: wallet.name, - signerAddresses: wallet.signersAddresses, - network, - paymentAddress, - stakeableAddress, - selectedAddress: walletAddress, - paymentUtxos: paymentUtxos.length, - stakeableUtxos: stakeableUtxos.length, - }); - - // Use the UTxOs from the selected address - let utxos: UTxO[] = walletAddress === stakeableAddress ? stakeableUtxos : paymentUtxos; - - // If we still have no UTxOs, try the other network as fallback - if (utxos.length === 0) { - const fallbackNetwork = network === 0 ? 1 : 0; - try { - const fallbackProvider = getProvider(fallbackNetwork); - utxos = await fallbackProvider.fetchAddressUTxOs(walletAddress); - console.log(`Successfully fetched ${utxos.length} UTxOs for wallet ${wallet.id} on fallback network ${fallbackNetwork}`); - } catch (fallbackError) { - console.error(`Failed to fetch UTxOs for wallet ${wallet.id} on fallback network ${fallbackNetwork}:`, fallbackError); - // Continue with empty UTxOs - this wallet will show 0 balance - } - } - - // Get balance for this wallet - const balance = getBalance(utxos); - - // Calculate ADA balance - const adaBalance = balance.lovelace ? parseInt(balance.lovelace) / 1000000 : 0; - const roundedAdaBalance = Math.round(adaBalance * 100) / 100; - - // Count wallet types - if (wallet.isArchived) { - archivedWalletCount++; - } else { - activeWalletCount++; - } - - // Add to wallet balances - walletBalances.push({ - walletId: wallet.id, - walletName: wallet.name, - address: walletAddress, - balance, - adaBalance: roundedAdaBalance, - isArchived: wallet.isArchived, - }); - - // Aggregate total balances - totalAdaBalance += roundedAdaBalance; - - // Aggregate all assets - Object.entries(balance).forEach(([asset, amount]) => { - const numericAmount = parseFloat(amount); - if (totalAssets[asset]) { - totalAssets[asset] += numericAmount; - } else { - totalAssets[asset] = numericAmount; - } - }); - - } catch (error) { - console.error(`Error processing wallet ${wallet.id}:`, error); - // Continue with other wallets even if one fails - } - } - - // Convert total assets back to string format - const totalAssetsString = Object.fromEntries( - Object.entries(totalAssets).map(([key, value]) => [ - key, - value.toString(), - ]), - ); - - const response: TVLResponse = { - totalValueLocked: { - ada: Math.round(totalAdaBalance * 100) / 100, - assets: totalAssetsString, - }, - walletCount: allWallets.length, - activeWalletCount, - archivedWalletCount, - walletBalances, - }; - - res.status(200).json(response); - } catch (error) { - console.error("Error in aggregatedBalances handler", { - message: (error as Error)?.message, - stack: (error as Error)?.stack, - }); - res.status(500).json({ error: "Internal Server Error" }); - } -} diff --git a/src/pages/api/v1/aggregatedBalances/README.md b/src/pages/api/v1/aggregatedBalances/README.md new file mode 100644 index 00000000..c2d79777 --- /dev/null +++ b/src/pages/api/v1/aggregatedBalances/README.md @@ -0,0 +1,176 @@ +# AggregatedBalances API Routes + +This directory contains the modular API routes for handling wallet balance aggregation and snapshots. These endpoints are designed to work together to provide comprehensive wallet balance tracking with rate limiting and error handling capabilities. + +## Authentication + +All endpoints require authentication using the `SNAPSHOT_AUTH_TOKEN` environment variable: +- **Header**: `Authorization: Bearer ` +- **Environment Variable**: `SNAPSHOT_AUTH_TOKEN` + +## Routes + +### `/api/v1/aggregatedBalances/wallets` +- **Method**: GET +- **Purpose**: Returns all wallet information without fetching balances +- **Authentication**: Required (Bearer token) +- **Response**: + ```json + { + "wallets": [ + { + "walletId": "string", + "walletName": "string", + "signersAddresses": ["string"], + "numRequiredSigners": number, + "type": "string", + "stakeCredentialHash": "string|null", + "isArchived": boolean, + "network": number, + "paymentAddress": "string", + "stakeableAddress": "string" + } + ], + "walletCount": number, + "activeWalletCount": number, + "archivedWalletCount": number + } + ``` + +### `/api/v1/aggregatedBalances/balance` +- **Method**: GET +- **Purpose**: Fetches balance for a single wallet +- **Authentication**: Required (Bearer token) +- **Query Parameters**: + - `walletId` (required) - Wallet ID + - `walletName` (required) - Wallet name + - `signersAddresses` (required) - JSON array of signer addresses + - `numRequiredSigners` (required) - Number of required signers + - `type` (required) - Wallet type + - `stakeCredentialHash` (optional) - Stake credential hash + - `isArchived` (required) - Whether wallet is archived + - `network` (required) - Network ID (0=testnet, 1=mainnet) + - `paymentAddress` (required) - Payment address + - `stakeableAddress` (required) - Stakeable address +- **Response**: + ```json + { + "walletBalance": { + "walletId": "string", + "walletName": "string", + "address": "string", + "balance": { + "lovelace": "string", + "assetId": "quantity" + }, + "adaBalance": number, + "isArchived": boolean + } + } + ``` + +### `/api/v1/aggregatedBalances/snapshots` +- **Method**: POST +- **Purpose**: Stores balance snapshots in the database +- **Authentication**: Required (Bearer token) +- **Content-Type**: `application/json` +- **Body**: + ```json + { + "walletBalances": [ + { + "walletId": "string", + "walletName": "string", + "address": "string", + "balance": { + "lovelace": "string", + "assetId": "quantity" + }, + "adaBalance": number, + "isArchived": boolean + } + ] + } + ``` +- **Response**: + ```json + { + "snapshotsStored": number, + "totalWallets": number + } + ``` + +### `/api/v1/aggregatedBalances/test` +- **Method**: GET +- **Purpose**: Comprehensive test endpoint that validates all sub-routes with real data +- **Authentication**: Required (Bearer token) +- **Response**: + ```json + { + "message": "string", + "timestamp": "string", + "endpoints": { + "wallets": "string", + "balance": "string", + "snapshots": "string" + }, + "usage": { + "wallets": "string", + "balance": "string", + "snapshots": "string" + }, + "realData": { + "walletsFound": number, + "processedWallets": number, + "failedWallets": number, + "totalAdaBalance": number, + "sampleWallet": { + "id": "string", + "name": "string", + "adaBalance": number + }, + "snapshotsStored": number + } + } + ``` + +## Features + +The modular approach provides several advantages: + +1. **Rate Limit Mitigation**: Individual wallet balance requests can be spaced out to respect API limits +2. **Better Error Handling**: Failed wallet processing doesn't affect other wallets +3. **Modularity**: Each endpoint has a single responsibility +4. **Comprehensive Testing**: The test endpoint validates the entire workflow with real data +5. **Fallback Network Support**: Balance endpoint tries alternative networks if primary fails +6. **Batch Processing**: Snapshots endpoint handles multiple wallets efficiently + +## GitHub Actions Integration + +The daily balance snapshots workflow (`.github/workflows/daily-balance-snapshots.yml`) uses these endpoints in sequence: + +1. **Fetch Wallets**: Uses `/wallets` to get all wallet information +2. **Process Balances**: Uses `/balance` for each wallet with rate limiting: + - Batch size: 3 wallets per batch + - Delay between requests: 3 seconds + - Delay between batches: 15 seconds + - Max retries: 3 attempts +3. **Store Snapshots**: Uses `/snapshots` to persist all collected balances + +## Error Handling + +- **401 Unauthorized**: Invalid or missing authentication token +- **400 Bad Request**: Missing required parameters +- **405 Method Not Allowed**: Incorrect HTTP method +- **500 Internal Server Error**: Server-side processing errors + +## Database Schema + +The snapshots are stored in the `balanceSnapshot` table with the following structure: +- `walletId`: Wallet identifier +- `walletName`: Human-readable wallet name +- `address`: Wallet address used for balance calculation +- `adaBalance`: ADA balance in ADA units +- `assetBalances`: JSON object containing all asset balances +- `isArchived`: Whether the wallet is archived +- `createdAt`: Timestamp of snapshot creation diff --git a/src/pages/api/v1/aggregatedBalances/balance.ts b/src/pages/api/v1/aggregatedBalances/balance.ts new file mode 100644 index 00000000..ab4cbb50 --- /dev/null +++ b/src/pages/api/v1/aggregatedBalances/balance.ts @@ -0,0 +1,163 @@ +import { cors, addCorsCacheBustingHeaders } from "@/lib/cors"; +import type { NextApiRequest, NextApiResponse } from "next"; +import { buildMultisigWallet } from "@/utils/common"; +import { getProvider } from "@/utils/get-provider"; +import { resolvePaymentKeyHash, serializeNativeScript } from "@meshsdk/core"; +import type { UTxO, NativeScript } from "@meshsdk/core"; +import { getBalance } from "@/utils/getBalance"; + +interface WalletBalance { + walletId: string; + walletName: string; + address: string; + balance: Record; + adaBalance: number; + isArchived: boolean; +} + +interface BalanceResponse { + walletBalance: WalletBalance; +} + +export default async function handler( + req: NextApiRequest, + res: NextApiResponse, +) { + // Add cache-busting headers for CORS + addCorsCacheBustingHeaders(res); + + await cors(req, res); + if (req.method === "OPTIONS") { + return res.status(200).end(); + } + if (req.method !== "GET") { + return res.status(405).json({ error: "Method Not Allowed" }); + } + + // Verify authentication for all requests + const authToken = req.headers.authorization?.replace('Bearer ', ''); + const expectedToken = process.env.SNAPSHOT_AUTH_TOKEN; + + if (!expectedToken) { + console.error('SNAPSHOT_AUTH_TOKEN environment variable not set'); + return res.status(500).json({ error: "Server configuration error" }); + } + + if (!authToken || authToken !== expectedToken) { + console.warn('Unauthorized request attempt', { + ip: req.headers['x-forwarded-for'] || req.connection.remoteAddress, + userAgent: req.headers['user-agent'], + authToken: authToken ? 'present' : 'missing', + query: req.query + }); + return res.status(401).json({ error: "Unauthorized" }); + } + + const { walletId, walletName, signersAddresses, numRequiredSigners, type, stakeCredentialHash, isArchived, network, paymentAddress, stakeableAddress } = req.query; + + // Validate required parameters + if (!walletId || !walletName || !signersAddresses || !numRequiredSigners || !type || !network || !paymentAddress || !stakeableAddress) { + return res.status(400).json({ error: "Missing required parameters" }); + } + + try { + const walletIdStr = walletId as string; + const walletNameStr = walletName as string; + const signersAddressesArray = JSON.parse(signersAddresses as string); + const numRequiredSignersNum = parseInt(numRequiredSigners as string); + const typeStr = type as string; + const stakeCredentialHashStr = stakeCredentialHash as string; + const isArchivedBool = isArchived === 'true'; + const networkNum = parseInt(network as string); + const paymentAddressStr = paymentAddress as string; + const stakeableAddressStr = stakeableAddress as string; + + // Build multisig wallet for address determination + const walletData = { + id: walletIdStr, + name: walletNameStr, + signersAddresses: signersAddressesArray, + numRequiredSigners: numRequiredSignersNum, + type: typeStr, + stakeCredentialHash: stakeCredentialHashStr, + isArchived: isArchivedBool, + description: null, + signersStakeKeys: [], + signersDRepKeys: [], + signersDescriptions: [], + clarityApiKey: null, + drepKey: null, + scriptType: null, + }; + + const mWallet = buildMultisigWallet(walletData, networkNum); + if (!mWallet) { + return res.status(400).json({ error: "Failed to build multisig wallet" }); + } + + // Determine which address to use + const blockchainProvider = getProvider(networkNum); + + let paymentUtxos: UTxO[] = []; + let stakeableUtxos: UTxO[] = []; + + try { + paymentUtxos = await blockchainProvider.fetchAddressUTxOs(paymentAddressStr); + stakeableUtxos = await blockchainProvider.fetchAddressUTxOs(stakeableAddressStr); + } catch (utxoError) { + console.error(`Failed to fetch UTxOs for wallet ${walletIdStr}:`, utxoError); + // Continue with empty UTxOs + } + + const paymentAddrEmpty = paymentUtxos.length === 0; + let walletAddress = paymentAddressStr; + + if (paymentAddrEmpty && mWallet.stakingEnabled()) { + walletAddress = stakeableAddressStr; + } + + // Use the UTxOs from the selected address + let utxos: UTxO[] = walletAddress === stakeableAddressStr ? stakeableUtxos : paymentUtxos; + + // If we still have no UTxOs, try the other network as fallback + if (utxos.length === 0) { + const fallbackNetwork = networkNum === 0 ? 1 : 0; + try { + const fallbackProvider = getProvider(fallbackNetwork); + utxos = await fallbackProvider.fetchAddressUTxOs(walletAddress); + console.log(`Successfully fetched ${utxos.length} UTxOs for wallet ${walletIdStr} on fallback network ${fallbackNetwork}`); + } catch (fallbackError) { + console.error(`Failed to fetch UTxOs for wallet ${walletIdStr} on fallback network ${fallbackNetwork}:`, fallbackError); + // Continue with empty UTxOs - this wallet will show 0 balance + } + } + + // Get balance for this wallet + const balance = getBalance(utxos); + + // Calculate ADA balance + const adaBalance = balance.lovelace ? parseInt(balance.lovelace) / 1000000 : 0; + const roundedAdaBalance = Math.round(adaBalance * 100) / 100; + + const walletBalance: WalletBalance = { + walletId: walletIdStr, + walletName: walletNameStr, + address: walletAddress, + balance, + adaBalance: roundedAdaBalance, + isArchived: isArchivedBool, + }; + + const response: BalanceResponse = { + walletBalance, + }; + + res.status(200).json(response); + } catch (error) { + console.error("Error in balance handler", { + message: (error as Error)?.message, + stack: (error as Error)?.stack, + }); + res.status(500).json({ error: "Internal Server Error" }); + } +} diff --git a/src/pages/api/v1/aggregatedBalances/snapshots.ts b/src/pages/api/v1/aggregatedBalances/snapshots.ts new file mode 100644 index 00000000..3471fd82 --- /dev/null +++ b/src/pages/api/v1/aggregatedBalances/snapshots.ts @@ -0,0 +1,99 @@ +import { cors, addCorsCacheBustingHeaders } from "@/lib/cors"; +import type { NextApiRequest, NextApiResponse } from "next"; +import { db } from "@/server/db"; + +interface WalletBalance { + walletId: string; + walletName: string; + address: string; + balance: Record; + adaBalance: number; + isArchived: boolean; +} + +interface SnapshotsResponse { + snapshotsStored: number; + totalWallets: number; +} + +export default async function handler( + req: NextApiRequest, + res: NextApiResponse, +) { + // Add cache-busting headers for CORS + addCorsCacheBustingHeaders(res); + + await cors(req, res); + if (req.method === "OPTIONS") { + return res.status(200).end(); + } + if (req.method !== "POST") { + return res.status(405).json({ error: "Method Not Allowed" }); + } + + // Verify authentication for all requests + const authToken = req.headers.authorization?.replace('Bearer ', ''); + const expectedToken = process.env.SNAPSHOT_AUTH_TOKEN; + + if (!expectedToken) { + console.error('SNAPSHOT_AUTH_TOKEN environment variable not set'); + return res.status(500).json({ error: "Server configuration error" }); + } + + if (!authToken || authToken !== expectedToken) { + console.warn('Unauthorized request attempt', { + ip: req.headers['x-forwarded-for'] || req.connection.remoteAddress, + userAgent: req.headers['user-agent'], + authToken: authToken ? 'present' : 'missing', + body: req.body + }); + return res.status(401).json({ error: "Unauthorized" }); + } + + const { walletBalances } = req.body; + + // Validate required parameters + if (!walletBalances || !Array.isArray(walletBalances)) { + return res.status(400).json({ error: "Missing or invalid walletBalances array" }); + } + + try { + // Store individual wallet snapshots + const snapshotPromises = walletBalances.map(async (walletBalance: WalletBalance) => { + try { + await (db as any).balanceSnapshot.create({ + data: { + walletId: walletBalance.walletId, + walletName: walletBalance.walletName, + address: walletBalance.address, + adaBalance: walletBalance.adaBalance, + assetBalances: walletBalance.balance, + isArchived: walletBalance.isArchived, + }, + }); + return 1; + } catch (error) { + console.error(`Failed to store snapshot for wallet ${walletBalance.walletId}:`, error); + return 0; + } + }); + + const results = await Promise.all(snapshotPromises); + const snapshotsStored = results.reduce((sum: number, result: number) => sum + result, 0); + + console.log(`Stored ${snapshotsStored} balance snapshots out of ${walletBalances.length} wallets`); + + const response: SnapshotsResponse = { + snapshotsStored, + totalWallets: walletBalances.length, + }; + + res.status(200).json(response); + } catch (error) { + console.error("Error in snapshots handler", { + message: (error as Error)?.message, + stack: (error as Error)?.stack, + }); + res.status(500).json({ error: "Internal Server Error" }); + } +} diff --git a/src/pages/api/v1/aggregatedBalances/test.ts b/src/pages/api/v1/aggregatedBalances/test.ts new file mode 100644 index 00000000..be744539 --- /dev/null +++ b/src/pages/api/v1/aggregatedBalances/test.ts @@ -0,0 +1,241 @@ +import { cors, addCorsCacheBustingHeaders } from "@/lib/cors"; +import type { NextApiRequest, NextApiResponse } from "next"; + +interface TestResponse { + message: string; + timestamp: string; + endpoints: { + wallets: string; + balance: string; + snapshots: string; + }; + usage: { + wallets: string; + balance: string; + snapshots: string; + }; + realData?: { + walletsFound: number; + processedWallets: number; + failedWallets: number; + totalAdaBalance: number; + sampleWallet?: { + id: string; + name: string; + adaBalance?: number; + }; + snapshotsStored?: number; + }; +} + +export default async function handler( + req: NextApiRequest, + res: NextApiResponse, +) { + // Add cache-busting headers for CORS + addCorsCacheBustingHeaders(res); + + await cors(req, res); + if (req.method === "OPTIONS") { + return res.status(200).end(); + } + if (req.method !== "GET") { + return res.status(405).json({ error: "Method Not Allowed" }); + } + + // Verify authentication for all requests + const authToken = req.headers.authorization?.replace('Bearer ', ''); + const expectedToken = process.env.SNAPSHOT_AUTH_TOKEN; + + if (!expectedToken) { + console.error('SNAPSHOT_AUTH_TOKEN environment variable not set'); + return res.status(500).json({ error: "Server configuration error" }); + } + + if (!authToken || authToken !== expectedToken) { + console.warn('Unauthorized request attempt', { + ip: req.headers['x-forwarded-for'] || req.connection.remoteAddress, + userAgent: req.headers['user-agent'], + authToken: authToken ? 'present' : 'missing', + query: req.query + }); + return res.status(401).json({ error: "Unauthorized" }); + } + + try { + // Test endpoints by making actual HTTP requests to fetch real data + const baseUrl = req.headers.host ? `http://${req.headers.host}` : 'http://localhost:3000'; + const endpoints = { + wallets: `${baseUrl}/api/v1/aggregatedBalances/wallets`, + balance: `${baseUrl}/api/v1/aggregatedBalances/balance`, + snapshots: `${baseUrl}/api/v1/aggregatedBalances/snapshots`, + }; + + const headers = { + 'Authorization': `Bearer ${authToken}`, + 'Content-Type': 'application/json', + }; + + // Test wallets endpoint + let walletsStatus = 'unknown'; + let walletsData = null; + let sampleWallet = null; + + try { + const walletsResponse = await fetch(endpoints.wallets, { headers }); + + if (walletsResponse.ok) { + walletsData = await walletsResponse.json(); + walletsStatus = `success (${walletsData.walletCount} wallets found)`; + + // Get a sample wallet for testing + if (walletsData.wallets && walletsData.wallets.length > 0) { + sampleWallet = walletsData.wallets[0]; + } + } else { + walletsStatus = `failed (${walletsResponse.status})`; + } + } catch (error) { + walletsStatus = `error: ${(error as Error).message}`; + } + + // Test balance endpoint if we have a sample wallet + let balanceStatus = 'skipped (no sample wallet)'; + let balanceData = null; + + if (sampleWallet) { + try { + const balanceUrl = `${endpoints.balance}?${new URLSearchParams({ + walletId: sampleWallet.walletId, + walletName: sampleWallet.walletName, + signersAddresses: JSON.stringify(sampleWallet.signersAddresses), + numRequiredSigners: sampleWallet.numRequiredSigners.toString(), + type: sampleWallet.type, + stakeCredentialHash: sampleWallet.stakeCredentialHash || '', + isArchived: sampleWallet.isArchived.toString(), + network: sampleWallet.network.toString(), + paymentAddress: sampleWallet.paymentAddress, + stakeableAddress: sampleWallet.stakeableAddress, + })}`; + + const balanceResponse = await fetch(balanceUrl, { headers }); + + if (balanceResponse.ok) { + balanceData = await balanceResponse.json(); + balanceStatus = `success (${balanceData.walletBalance.adaBalance} ADA)`; + } else { + balanceStatus = `failed (${balanceResponse.status})`; + } + } catch (error) { + balanceStatus = `error: ${(error as Error).message}`; + } + } + + // Collect all wallet balances for snapshots + let allWalletBalances = []; + let totalAdaBalance = 0; + let processedWallets = 0; + let failedWallets = 0; + + if (walletsData && walletsData.wallets && walletsData.wallets.length > 0) { + console.log(`Processing ${walletsData.wallets.length} wallets for balance fetching...`); + + for (const wallet of walletsData.wallets) { + try { + const balanceUrl = `${endpoints.balance}?${new URLSearchParams({ + walletId: wallet.walletId, + walletName: wallet.walletName, + signersAddresses: JSON.stringify(wallet.signersAddresses), + numRequiredSigners: wallet.numRequiredSigners.toString(), + type: wallet.type, + stakeCredentialHash: wallet.stakeCredentialHash || '', + isArchived: wallet.isArchived.toString(), + network: wallet.network.toString(), + paymentAddress: wallet.paymentAddress, + stakeableAddress: wallet.stakeableAddress, + })}`; + + const balanceResponse = await fetch(balanceUrl, { headers }); + + if (balanceResponse.ok) { + const balanceData = await balanceResponse.json(); + const walletBalance = balanceData.walletBalance; + + allWalletBalances.push(walletBalance); + totalAdaBalance += walletBalance.adaBalance; + processedWallets++; + + console.log(`āœ… Processed wallet ${wallet.walletName}: ${walletBalance.adaBalance} ADA`); + } else { + console.error(`āŒ Failed to fetch balance for wallet ${wallet.walletName}: ${balanceResponse.status}`); + failedWallets++; + } + } catch (error) { + console.error(`āŒ Error processing wallet ${wallet.walletName}:`, error); + failedWallets++; + } + } + } + + // Test snapshots endpoint with real wallet balances + let snapshotsStatus = 'unknown'; + let snapshotsData = null; + + try { + const snapshotsResponse = await fetch(endpoints.snapshots, { + method: 'POST', + headers, + body: JSON.stringify({ walletBalances: allWalletBalances }), + }); + + if (snapshotsResponse.ok) { + snapshotsData = await snapshotsResponse.json(); + snapshotsStatus = `success (${snapshotsData.snapshotsStored} snapshots stored)`; + } else { + snapshotsStatus = `failed (${snapshotsResponse.status})`; + } + } catch (error) { + snapshotsStatus = `error: ${(error as Error).message}`; + } + + + const response: TestResponse = { + message: "AggregatedBalances API Test Endpoint - Real Data Test", + timestamp: new Date().toISOString(), + endpoints: { + wallets: `${endpoints.wallets} - Status: ${walletsStatus}`, + balance: `${endpoints.balance} - Status: ${balanceStatus}`, + snapshots: `${endpoints.snapshots} - Status: ${snapshotsStatus}`, + }, + usage: { + wallets: "GET - Returns all wallet information without balances", + balance: "GET - Fetches balance for a single wallet (requires query params)", + snapshots: "POST - Stores balance snapshots in database (requires body with walletBalances)", + }, + ...(walletsData && { + realData: { + walletsFound: walletsData.walletCount || 0, + processedWallets, + failedWallets, + totalAdaBalance: Math.round(totalAdaBalance * 100) / 100, + ...(sampleWallet && { + sampleWallet: { + id: sampleWallet.walletId, + name: sampleWallet.walletName, + ...(balanceData && { adaBalance: balanceData.walletBalance.adaBalance }), + }, + }), + ...(snapshotsData && { snapshotsStored: snapshotsData.snapshotsStored }), + }, + }), + }; + + res.status(200).json(response); + } catch (error) { + console.error("Error in test handler", { + message: (error as Error)?.message, + stack: (error as Error)?.stack, + }); + res.status(500).json({ error: "Internal Server Error" }); + } +} diff --git a/src/pages/api/v1/aggregatedBalances/wallets.ts b/src/pages/api/v1/aggregatedBalances/wallets.ts new file mode 100644 index 00000000..39579bc2 --- /dev/null +++ b/src/pages/api/v1/aggregatedBalances/wallets.ts @@ -0,0 +1,159 @@ +import { cors, addCorsCacheBustingHeaders } from "@/lib/cors"; +import type { Wallet as DbWallet } from "@prisma/client"; +import type { NextApiRequest, NextApiResponse } from "next"; +import { buildMultisigWallet } from "@/utils/common"; +import { addressToNetwork } from "@/utils/multisigSDK"; +import { resolvePaymentKeyHash, serializeNativeScript } from "@meshsdk/core"; +import { db } from "@/server/db"; +import type { NativeScript } from "@meshsdk/core"; + +interface WalletInfo { + walletId: string; + walletName: string; + signersAddresses: string[]; + numRequiredSigners: number; + type: string; + stakeCredentialHash: string | null; + isArchived: boolean; + network: number; + paymentAddress: string; + stakeableAddress: string; +} + +interface WalletsResponse { + wallets: WalletInfo[]; + walletCount: number; + activeWalletCount: number; + archivedWalletCount: number; +} + +export default async function handler( + req: NextApiRequest, + res: NextApiResponse, +) { + // Add cache-busting headers for CORS + addCorsCacheBustingHeaders(res); + + await cors(req, res); + if (req.method === "OPTIONS") { + return res.status(200).end(); + } + if (req.method !== "GET") { + return res.status(405).json({ error: "Method Not Allowed" }); + } + + // Verify authentication for all requests + const authToken = req.headers.authorization?.replace('Bearer ', ''); + const expectedToken = process.env.SNAPSHOT_AUTH_TOKEN; + + if (!expectedToken) { + console.error('SNAPSHOT_AUTH_TOKEN environment variable not set'); + return res.status(500).json({ error: "Server configuration error" }); + } + + if (!authToken || authToken !== expectedToken) { + console.warn('Unauthorized request attempt', { + ip: req.headers['x-forwarded-for'] || req.connection.remoteAddress, + userAgent: req.headers['user-agent'], + authToken: authToken ? 'present' : 'missing', + query: req.query + }); + return res.status(401).json({ error: "Unauthorized" }); + } + + try { + // Get ALL wallets from the database + const allWallets: DbWallet[] = await db.wallet.findMany(); + + if (!allWallets || allWallets.length === 0) { + return res.status(200).json({ + wallets: [], + walletCount: 0, + activeWalletCount: 0, + archivedWalletCount: 0, + }); + } + + const wallets: WalletInfo[] = []; + let activeWalletCount = 0; + let archivedWalletCount = 0; + + // Process each wallet to extract wallet information + for (const wallet of allWallets) { + try { + // Determine network from signer addresses + let network = 1; // Default to mainnet + if (wallet.signersAddresses.length > 0) { + const signerAddr = wallet.signersAddresses[0]!; + network = addressToNetwork(signerAddr); + } + + const mWallet = buildMultisigWallet(wallet, network); + if (!mWallet) { + console.warn(`Failed to build multisig wallet for wallet ${wallet.id}`); + continue; + } + + // Use the same address logic as the frontend buildWallet function + const nativeScript = { + type: wallet.type ? wallet.type : "atLeast", + scripts: wallet.signersAddresses.map((addr) => ({ + type: "sig", + keyHash: resolvePaymentKeyHash(addr), + })), + }; + if (nativeScript.type == "atLeast") { + //@ts-ignore + nativeScript.required = wallet.numRequiredSigners!; + } + + const paymentAddress = serializeNativeScript( + nativeScript as NativeScript, + wallet.stakeCredentialHash as undefined | string, + network, + ).address; + + const stakeableAddress = mWallet.getScript().address; + + // Count wallet types + if (wallet.isArchived) { + archivedWalletCount++; + } else { + activeWalletCount++; + } + + wallets.push({ + walletId: wallet.id, + walletName: wallet.name, + signersAddresses: wallet.signersAddresses, + numRequiredSigners: wallet.numRequiredSigners!, + type: wallet.type || "atLeast", + stakeCredentialHash: wallet.stakeCredentialHash, + isArchived: wallet.isArchived, + network, + paymentAddress, + stakeableAddress, + }); + + } catch (error) { + console.error(`Error processing wallet ${wallet.id}:`, error); + // Continue with other wallets even if one fails + } + } + + const response: WalletsResponse = { + wallets, + walletCount: allWallets.length, + activeWalletCount, + archivedWalletCount, + }; + + res.status(200).json(response); + } catch (error) { + console.error("Error in wallets handler", { + message: (error as Error)?.message, + stack: (error as Error)?.stack, + }); + res.status(500).json({ error: "Internal Server Error" }); + } +} From fe47f463e702264db04b6ebdae4b5cfe05f3e2d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Diamond?= <32074058+Andre-Diamond@users.noreply.github.com> Date: Mon, 6 Oct 2025 12:42:58 +0200 Subject: [PATCH 3/7] fix(migration): remove stakeCredentialHash column from NewWallet migration --- .../20251006065720_add_balance_snapshots/migration.sql | 3 --- 1 file changed, 3 deletions(-) diff --git a/prisma/migrations/20251006065720_add_balance_snapshots/migration.sql b/prisma/migrations/20251006065720_add_balance_snapshots/migration.sql index 35c07d68..ffc5cfd2 100644 --- a/prisma/migrations/20251006065720_add_balance_snapshots/migration.sql +++ b/prisma/migrations/20251006065720_add_balance_snapshots/migration.sql @@ -1,6 +1,3 @@ --- AlterTable -ALTER TABLE "NewWallet" ADD COLUMN "stakeCredentialHash" TEXT; - -- CreateTable CREATE TABLE "BalanceSnapshot" ( "id" TEXT NOT NULL, From f920f5dc83219e7e724ac20b976664c8ec2e5fb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Diamond?= <32074058+Andre-Diamond@users.noreply.github.com> Date: Mon, 6 Oct 2025 13:19:15 +0200 Subject: [PATCH 4/7] Fix TypeScript compilation error and secure base URL configuration --- src/pages/api/v1/aggregatedBalances/balance.ts | 2 ++ src/pages/api/v1/aggregatedBalances/test.ts | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/pages/api/v1/aggregatedBalances/balance.ts b/src/pages/api/v1/aggregatedBalances/balance.ts index ab4cbb50..6d72544c 100644 --- a/src/pages/api/v1/aggregatedBalances/balance.ts +++ b/src/pages/api/v1/aggregatedBalances/balance.ts @@ -88,6 +88,8 @@ export default async function handler( clarityApiKey: null, drepKey: null, scriptType: null, + scriptCbor: "", // Required field for DbWallet type + verified: [], // Required field for DbWallet type }; const mWallet = buildMultisigWallet(walletData, networkNum); diff --git a/src/pages/api/v1/aggregatedBalances/test.ts b/src/pages/api/v1/aggregatedBalances/test.ts index be744539..9728cb86 100644 --- a/src/pages/api/v1/aggregatedBalances/test.ts +++ b/src/pages/api/v1/aggregatedBalances/test.ts @@ -64,7 +64,7 @@ export default async function handler( try { // Test endpoints by making actual HTTP requests to fetch real data - const baseUrl = req.headers.host ? `http://${req.headers.host}` : 'http://localhost:3000'; + const baseUrl = process.env.INTERNAL_BASE_URL || 'http://localhost:3000'; const endpoints = { wallets: `${baseUrl}/api/v1/aggregatedBalances/wallets`, balance: `${baseUrl}/api/v1/aggregatedBalances/balance`, From 3b1dde23b8c002614298fb14e7e71c846e665f6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Diamond?= <32074058+Andre-Diamond@users.noreply.github.com> Date: Mon, 6 Oct 2025 13:29:02 +0200 Subject: [PATCH 5/7] Fix codeQL and typescript error --- src/pages/api/v1/aggregatedBalances/snapshots.ts | 2 +- src/pages/api/v1/aggregatedBalances/test.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pages/api/v1/aggregatedBalances/snapshots.ts b/src/pages/api/v1/aggregatedBalances/snapshots.ts index 3471fd82..bb322a7a 100644 --- a/src/pages/api/v1/aggregatedBalances/snapshots.ts +++ b/src/pages/api/v1/aggregatedBalances/snapshots.ts @@ -73,7 +73,7 @@ export default async function handler( }); return 1; } catch (error) { - console.error(`Failed to store snapshot for wallet ${walletBalance.walletId}:`, error); + console.error('Failed to store snapshot for wallet %s:', walletBalance.walletId, error); return 0; } }); diff --git a/src/pages/api/v1/aggregatedBalances/test.ts b/src/pages/api/v1/aggregatedBalances/test.ts index 9728cb86..69992892 100644 --- a/src/pages/api/v1/aggregatedBalances/test.ts +++ b/src/pages/api/v1/aggregatedBalances/test.ts @@ -132,7 +132,7 @@ export default async function handler( } // Collect all wallet balances for snapshots - let allWalletBalances = []; + const allWalletBalances = []; let totalAdaBalance = 0; let processedWallets = 0; let failedWallets = 0; From 1927fa32dd1a089c0b9f668a1465ef1db903aa79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Diamond?= <32074058+Andre-Diamond@users.noreply.github.com> Date: Mon, 6 Oct 2025 15:14:56 +0200 Subject: [PATCH 6/7] Wallet build improvements --- src/pages/api/v1/aggregatedBalances/README.md | 21 ++++---- .../api/v1/aggregatedBalances/balance.ts | 37 +++++++++---- src/pages/api/v1/aggregatedBalances/test.ts | 2 - .../api/v1/aggregatedBalances/wallets.ts | 54 ++++++------------- 4 files changed, 57 insertions(+), 57 deletions(-) diff --git a/src/pages/api/v1/aggregatedBalances/README.md b/src/pages/api/v1/aggregatedBalances/README.md index c2d79777..d7d037cb 100644 --- a/src/pages/api/v1/aggregatedBalances/README.md +++ b/src/pages/api/v1/aggregatedBalances/README.md @@ -12,7 +12,7 @@ All endpoints require authentication using the `SNAPSHOT_AUTH_TOKEN` environment ### `/api/v1/aggregatedBalances/wallets` - **Method**: GET -- **Purpose**: Returns all wallet information without fetching balances +- **Purpose**: Returns all wallet information without fetching balances or building wallet objects - **Authentication**: Required (Bearer token) - **Response**: ```json @@ -20,15 +20,20 @@ All endpoints require authentication using the `SNAPSHOT_AUTH_TOKEN` environment "wallets": [ { "walletId": "string", - "walletName": "string", + "walletName": "string", + "description": "string|null", "signersAddresses": ["string"], + "signersStakeKeys": ["string"], + "signersDRepKeys": ["string"], + "signersDescriptions": ["string"], "numRequiredSigners": number, - "type": "string", + "verified": ["string"], + "scriptCbor": "string", "stakeCredentialHash": "string|null", + "type": "string", "isArchived": boolean, - "network": number, - "paymentAddress": "string", - "stakeableAddress": "string" + "clarityApiKey": "string|null", + "network": number } ], "walletCount": number, @@ -39,7 +44,7 @@ All endpoints require authentication using the `SNAPSHOT_AUTH_TOKEN` environment ### `/api/v1/aggregatedBalances/balance` - **Method**: GET -- **Purpose**: Fetches balance for a single wallet +- **Purpose**: Fetches balance for a single wallet (builds wallet and generates addresses internally) - **Authentication**: Required (Bearer token) - **Query Parameters**: - `walletId` (required) - Wallet ID @@ -50,8 +55,6 @@ All endpoints require authentication using the `SNAPSHOT_AUTH_TOKEN` environment - `stakeCredentialHash` (optional) - Stake credential hash - `isArchived` (required) - Whether wallet is archived - `network` (required) - Network ID (0=testnet, 1=mainnet) - - `paymentAddress` (required) - Payment address - - `stakeableAddress` (required) - Stakeable address - **Response**: ```json { diff --git a/src/pages/api/v1/aggregatedBalances/balance.ts b/src/pages/api/v1/aggregatedBalances/balance.ts index 6d72544c..e9d94416 100644 --- a/src/pages/api/v1/aggregatedBalances/balance.ts +++ b/src/pages/api/v1/aggregatedBalances/balance.ts @@ -53,10 +53,10 @@ export default async function handler( return res.status(401).json({ error: "Unauthorized" }); } - const { walletId, walletName, signersAddresses, numRequiredSigners, type, stakeCredentialHash, isArchived, network, paymentAddress, stakeableAddress } = req.query; + const { walletId, walletName, signersAddresses, numRequiredSigners, type, stakeCredentialHash, isArchived, network } = req.query; // Validate required parameters - if (!walletId || !walletName || !signersAddresses || !numRequiredSigners || !type || !network || !paymentAddress || !stakeableAddress) { + if (!walletId || !walletName || !signersAddresses || !numRequiredSigners || !type || !network) { return res.status(400).json({ error: "Missing required parameters" }); } @@ -69,8 +69,6 @@ export default async function handler( const stakeCredentialHashStr = stakeCredentialHash as string; const isArchivedBool = isArchived === 'true'; const networkNum = parseInt(network as string); - const paymentAddressStr = paymentAddress as string; - const stakeableAddressStr = stakeableAddress as string; // Build multisig wallet for address determination const walletData = { @@ -97,6 +95,27 @@ export default async function handler( return res.status(400).json({ error: "Failed to build multisig wallet" }); } + // Generate addresses from the built wallet + const nativeScript = { + type: typeStr || "atLeast", + scripts: signersAddressesArray.map((addr: string) => ({ + type: "sig", + keyHash: resolvePaymentKeyHash(addr), + })), + }; + if (nativeScript.type == "atLeast") { + //@ts-ignore + nativeScript.required = numRequiredSignersNum; + } + + const paymentAddress = serializeNativeScript( + nativeScript as NativeScript, + stakeCredentialHashStr as undefined | string, + networkNum, + ).address; + + const stakeableAddress = mWallet.getScript().address; + // Determine which address to use const blockchainProvider = getProvider(networkNum); @@ -104,22 +123,22 @@ export default async function handler( let stakeableUtxos: UTxO[] = []; try { - paymentUtxos = await blockchainProvider.fetchAddressUTxOs(paymentAddressStr); - stakeableUtxos = await blockchainProvider.fetchAddressUTxOs(stakeableAddressStr); + paymentUtxos = await blockchainProvider.fetchAddressUTxOs(paymentAddress); + stakeableUtxos = await blockchainProvider.fetchAddressUTxOs(stakeableAddress); } catch (utxoError) { console.error(`Failed to fetch UTxOs for wallet ${walletIdStr}:`, utxoError); // Continue with empty UTxOs } const paymentAddrEmpty = paymentUtxos.length === 0; - let walletAddress = paymentAddressStr; + let walletAddress = paymentAddress; if (paymentAddrEmpty && mWallet.stakingEnabled()) { - walletAddress = stakeableAddressStr; + walletAddress = stakeableAddress; } // Use the UTxOs from the selected address - let utxos: UTxO[] = walletAddress === stakeableAddressStr ? stakeableUtxos : paymentUtxos; + let utxos: UTxO[] = walletAddress === stakeableAddress ? stakeableUtxos : paymentUtxos; // If we still have no UTxOs, try the other network as fallback if (utxos.length === 0) { diff --git a/src/pages/api/v1/aggregatedBalances/test.ts b/src/pages/api/v1/aggregatedBalances/test.ts index 69992892..dfe68e4d 100644 --- a/src/pages/api/v1/aggregatedBalances/test.ts +++ b/src/pages/api/v1/aggregatedBalances/test.ts @@ -151,8 +151,6 @@ export default async function handler( stakeCredentialHash: wallet.stakeCredentialHash || '', isArchived: wallet.isArchived.toString(), network: wallet.network.toString(), - paymentAddress: wallet.paymentAddress, - stakeableAddress: wallet.stakeableAddress, })}`; const balanceResponse = await fetch(balanceUrl, { headers }); diff --git a/src/pages/api/v1/aggregatedBalances/wallets.ts b/src/pages/api/v1/aggregatedBalances/wallets.ts index 39579bc2..615575f8 100644 --- a/src/pages/api/v1/aggregatedBalances/wallets.ts +++ b/src/pages/api/v1/aggregatedBalances/wallets.ts @@ -1,23 +1,25 @@ import { cors, addCorsCacheBustingHeaders } from "@/lib/cors"; import type { Wallet as DbWallet } from "@prisma/client"; import type { NextApiRequest, NextApiResponse } from "next"; -import { buildMultisigWallet } from "@/utils/common"; import { addressToNetwork } from "@/utils/multisigSDK"; -import { resolvePaymentKeyHash, serializeNativeScript } from "@meshsdk/core"; import { db } from "@/server/db"; -import type { NativeScript } from "@meshsdk/core"; interface WalletInfo { walletId: string; walletName: string; + description: string | null; signersAddresses: string[]; + signersStakeKeys: string[]; + signersDRepKeys: string[]; + signersDescriptions: string[]; numRequiredSigners: number; - type: string; + verified: string[]; + scriptCbor: string; stakeCredentialHash: string | null; + type: string; isArchived: boolean; + clarityApiKey: string | null; network: number; - paymentAddress: string; - stakeableAddress: string; } interface WalletsResponse { @@ -78,7 +80,7 @@ export default async function handler( let activeWalletCount = 0; let archivedWalletCount = 0; - // Process each wallet to extract wallet information + // Process each wallet to extract basic wallet information for (const wallet of allWallets) { try { // Determine network from signer addresses @@ -87,33 +89,6 @@ export default async function handler( const signerAddr = wallet.signersAddresses[0]!; network = addressToNetwork(signerAddr); } - - const mWallet = buildMultisigWallet(wallet, network); - if (!mWallet) { - console.warn(`Failed to build multisig wallet for wallet ${wallet.id}`); - continue; - } - - // Use the same address logic as the frontend buildWallet function - const nativeScript = { - type: wallet.type ? wallet.type : "atLeast", - scripts: wallet.signersAddresses.map((addr) => ({ - type: "sig", - keyHash: resolvePaymentKeyHash(addr), - })), - }; - if (nativeScript.type == "atLeast") { - //@ts-ignore - nativeScript.required = wallet.numRequiredSigners!; - } - - const paymentAddress = serializeNativeScript( - nativeScript as NativeScript, - wallet.stakeCredentialHash as undefined | string, - network, - ).address; - - const stakeableAddress = mWallet.getScript().address; // Count wallet types if (wallet.isArchived) { @@ -125,14 +100,19 @@ export default async function handler( wallets.push({ walletId: wallet.id, walletName: wallet.name, + description: wallet.description, signersAddresses: wallet.signersAddresses, + signersStakeKeys: wallet.signersStakeKeys, + signersDRepKeys: wallet.signersDRepKeys, + signersDescriptions: wallet.signersDescriptions, numRequiredSigners: wallet.numRequiredSigners!, - type: wallet.type || "atLeast", + verified: wallet.verified, + scriptCbor: wallet.scriptCbor, stakeCredentialHash: wallet.stakeCredentialHash, + type: wallet.type || "atLeast", isArchived: wallet.isArchived, + clarityApiKey: wallet.clarityApiKey, network, - paymentAddress, - stakeableAddress, }); } catch (error) { From 148e431cc6405b134194a618400d9658d2dec200 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Diamond?= <32074058+Andre-Diamond@users.noreply.github.com> Date: Tue, 7 Oct 2025 08:03:53 +0200 Subject: [PATCH 7/7] refactor(workflow): streamline daily balance snapshot process - Replaced inline shell script with a Node.js script for better maintainability. - Added steps for checking out the repository, setting up Node.js, and installing dependencies. - Updated the balance snapshot logic to utilize the new BalanceSnapshotService for improved structure and clarity. --- .github/workflows/daily-balance-snapshots.yml | 208 ++------------ scripts/balance-snapshots.js | 272 ++++++++++++++++++ src/pages/api/v1/aggregatedBalances/test.ts | 178 ++---------- 3 files changed, 315 insertions(+), 343 deletions(-) create mode 100644 scripts/balance-snapshots.js diff --git a/.github/workflows/daily-balance-snapshots.yml b/.github/workflows/daily-balance-snapshots.yml index 48c5243f..ba51aa15 100644 --- a/.github/workflows/daily-balance-snapshots.yml +++ b/.github/workflows/daily-balance-snapshots.yml @@ -15,192 +15,28 @@ jobs: runs-on: ubuntu-latest steps: - - name: Take balance snapshots - run: | - # Configuration - adjust these values based on your needs - API_BASE_URL="https://multisig.meshjs.dev" - AUTH_TOKEN="${{ secrets.SNAPSHOT_AUTH_TOKEN }}" - - # Rate limiting configuration - BATCH_SIZE=3 # Wallets per batch (adjust based on Blockfrost plan) - DELAY_BETWEEN_REQUESTS=3 # Seconds between requests (20/min = 3s) - DELAY_BETWEEN_BATCHES=15 # Seconds between batches - MAX_RETRIES=3 # Max retries for failed requests - REQUEST_TIMEOUT=30 # Request timeout in seconds - - echo "šŸ”„ Starting daily balance snapshot process..." - echo "šŸ“Š Configuration: batch_size=$BATCH_SIZE, request_delay=${DELAY_BETWEEN_REQUESTS}s, batch_delay=${DELAY_BETWEEN_BATCHES}s" - - # Step 1: Get all wallets - echo "šŸ“‹ Fetching all wallets..." - wallets_response=$(curl -s -w "\n%{http_code}" \ - -H "Authorization: Bearer $AUTH_TOKEN" \ - "$API_BASE_URL/api/v1/aggregatedBalances/wallets") - - wallets_http_code=$(echo "$wallets_response" | tail -n1) - wallets_body=$(echo "$wallets_response" | head -n -1) - - echo "Wallets HTTP Status: $wallets_http_code" - - if [ "$wallets_http_code" -ne 200 ]; then - echo "āŒ Failed to fetch wallets. HTTP Status: $wallets_http_code" - echo "Response: $wallets_body" - exit 1 - fi - - # Extract wallet data - wallet_count=$(echo "$wallets_body" | jq -r '.walletCount') - echo "āœ… Found $wallet_count wallets" - - if [ "$wallet_count" -eq 0 ]; then - echo "ā„¹ļø No wallets found, skipping snapshot process" - exit 0 - fi - - # Step 2: Get balances for each wallet with rate limiting - echo "šŸ’° Fetching balances for each wallet with rate limiting..." - - # Create temporary files for collecting results - temp_balances="/tmp/wallet_balances.json" - temp_failed="/tmp/failed_wallets.txt" - echo "[]" > "$temp_balances" - echo "0" > "$temp_failed" - - # Process wallets in batches to respect rate limits - batch_size=$BATCH_SIZE - delay_between_requests=$DELAY_BETWEEN_REQUESTS - delay_between_batches=$DELAY_BETWEEN_BATCHES - - # Convert wallets to array for batch processing - wallets_array=$(echo "$wallets_body" | jq -r '.wallets') - total_wallets=$(echo "$wallets_array" | jq 'length') - echo "Processing $total_wallets wallets in batches of $batch_size" - - # Process wallets in batches - for ((i=0; i "$temp_balances" - - success=true - elif [ "$balance_http_code" -eq 429 ]; then - # Rate limited - wait longer before retry - retry_delay=$((delay_between_requests * (retry_count + 1) * 2)) - echo " āš ļø Rate limited (429). Waiting ${retry_delay}s before retry $((retry_count + 1))/$max_retries" - sleep $retry_delay - retry_count=$((retry_count + 1)) - else - echo " āŒ Failed to fetch balance for wallet $wallet_id: $balance_http_code" - failed_count=$(cat "$temp_failed") - echo $((failed_count + 1)) > "$temp_failed" - success=true # Don't retry on non-rate-limit errors - fi - done - - if [ "$success" = false ]; then - echo " āŒ Max retries exceeded for wallet $wallet_id" - failed_count=$(cat "$temp_failed") - echo $((failed_count + 1)) > "$temp_failed" - fi - - # Delay between requests within a batch - if [ $((j+1)) -lt $batch_end ]; then - sleep $delay_between_requests - fi - done - - # Delay between batches (except for the last batch) - if [ $batch_end -lt $total_wallets ]; then - echo " ā³ Waiting ${delay_between_batches}s before next batch..." - sleep $delay_between_batches - fi - done - - # Read final results - wallet_balances=$(cat "$temp_balances") - failed_wallets=$(cat "$temp_failed") - - echo "šŸ“Š Balance fetching completed. Failed wallets: $failed_wallets" - echo "āœ… Successfully processed: $(echo "$wallet_balances" | jq 'length') wallets" - - # Step 3: Store snapshots using the collected balances - echo "šŸ’¾ Storing balance snapshots..." - snapshots_response=$(curl -s -w "\n%{http_code}" \ - -H "Authorization: Bearer $AUTH_TOKEN" \ - -H "Content-Type: application/json" \ - -d "{\"walletBalances\": $wallet_balances}" \ - "$API_BASE_URL/api/v1/aggregatedBalances/snapshots") - - snapshots_http_code=$(echo "$snapshots_response" | tail -n1) - snapshots_body=$(echo "$snapshots_response" | head -n -1) - - echo "Snapshots HTTP Status: $snapshots_http_code" - echo "Response: $snapshots_body" - - # Check if the request was successful - if [ "$snapshots_http_code" -eq 200 ]; then - # Parse the response to get the number of snapshots stored - snapshots_stored=$(echo "$snapshots_body" | jq -r '.snapshotsStored // 0') - total_tvl=$(echo "$snapshots_body" | jq -r '.totalValueLocked.ada') - echo "āœ… Successfully stored $snapshots_stored balance snapshots" - echo "šŸ’° Total Value Locked: $total_tvl ADA" - - # Optional: Send notification on success (you can add Discord/Slack webhook here) - # curl -X POST -H 'Content-type: application/json' \ - # --data "{\"text\":\"āœ… Daily balance snapshots completed: $snapshots_stored snapshots stored, TVL: $total_tvl ADA\"}" \ - # ${{ secrets.DISCORD_WEBHOOK_URL }} - else - echo "āŒ Failed to store balance snapshots. HTTP Status: $snapshots_http_code" - echo "Response: $snapshots_body" - exit 1 - fi + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '18' + cache: 'npm' + + - name: Install dependencies + run: npm ci + + - name: Run balance snapshots + run: node scripts/balance-snapshots.js + env: + API_BASE_URL: "https://multisig.meshjs.dev" + SNAPSHOT_AUTH_TOKEN: ${{ secrets.SNAPSHOT_AUTH_TOKEN }} + BATCH_SIZE: 3 + DELAY_BETWEEN_REQUESTS: 3 + DELAY_BETWEEN_BATCHES: 15 + MAX_RETRIES: 3 + REQUEST_TIMEOUT: 30 - name: Notify on failure if: failure() diff --git a/scripts/balance-snapshots.js b/scripts/balance-snapshots.js new file mode 100644 index 00000000..83dbdfaf --- /dev/null +++ b/scripts/balance-snapshots.js @@ -0,0 +1,272 @@ +#!/usr/bin/env node + +/** + * Balance Snapshots Script (JavaScript version) + * + * This script fetches wallet balances and stores them as snapshots in the database. + * It can be run locally for testing or by GitHub Actions for automated snapshots. + * + * Usage: + * node scripts/balance-snapshots.js + * SNAPSHOT_AUTH_TOKEN=your_token node scripts/balance-snapshots.js + * + * Environment Variables: + * - API_BASE_URL: Base URL for the API (default: http://localhost:3000) + * - SNAPSHOT_AUTH_TOKEN: Authentication token for API requests + * - BATCH_SIZE: Number of wallets to process per batch (default: 3) + * - DELAY_BETWEEN_REQUESTS: Delay between requests in seconds (default: 3) + * - DELAY_BETWEEN_BATCHES: Delay between batches in seconds (default: 15) + * - MAX_RETRIES: Maximum retries for failed requests (default: 3) + * - REQUEST_TIMEOUT: Request timeout in seconds (default: 30) + */ + +class BalanceSnapshotService { + constructor() { + this.config = this.loadConfig(); + this.results = { + walletsFound: 0, + processedWallets: 0, + failedWallets: 0, + totalAdaBalance: 0, + snapshotsStored: 0, + executionTime: 0, + }; + } + + loadConfig() { + const apiBaseUrl = process.env.API_BASE_URL || 'http://localhost:3000'; + const authToken = process.env.SNAPSHOT_AUTH_TOKEN; + + if (!authToken) { + throw new Error('SNAPSHOT_AUTH_TOKEN environment variable is required'); + } + + return { + apiBaseUrl, + authToken, + batchSize: parseInt(process.env.BATCH_SIZE || '3'), + delayBetweenRequests: parseInt(process.env.DELAY_BETWEEN_REQUESTS || '3'), + delayBetweenBatches: parseInt(process.env.DELAY_BETWEEN_BATCHES || '15'), + maxRetries: parseInt(process.env.MAX_RETRIES || '3'), + requestTimeout: parseInt(process.env.REQUEST_TIMEOUT || '30'), + }; + } + + async makeRequest(/** @type {string} */ url, /** @type {RequestInit} */ options = {}) { + const controller = new AbortController(); + const timeoutId = setTimeout(() => controller.abort(), this.config.requestTimeout * 1000); + + try { + const response = await fetch(url, { + ...options, + headers: { + 'Authorization': `Bearer ${this.config.authToken}`, + 'Content-Type': 'application/json', + ...(options.headers || {}), + }, + signal: controller.signal, + }); + + clearTimeout(timeoutId); + + if (!response.ok) { + throw new Error(`HTTP ${response.status}: ${response.statusText}`); + } + + const data = await response.json(); + return { data, status: response.status }; + } catch (error) { + clearTimeout(timeoutId); + throw error; + } + } + + async delay(/** @type {number} */ seconds) { + return new Promise(resolve => setTimeout(resolve, seconds * 1000)); + } + + async fetchWallets() { + console.log('šŸ“‹ Fetching all wallets...'); + + const { data } = await this.makeRequest( + `${this.config.apiBaseUrl}/api/v1/aggregatedBalances/wallets` + ); + + console.log(`āœ… Found ${data.walletCount} wallets`); + this.results.walletsFound = data.walletCount; + + if (data.walletCount === 0) { + console.log('ā„¹ļø No wallets found, skipping snapshot process'); + return []; + } + + return data.wallets; + } + + async fetchWalletBalance(/** @type {any} */ wallet) { + const params = new URLSearchParams({ + walletId: wallet.walletId, + walletName: wallet.walletName, + signersAddresses: JSON.stringify(wallet.signersAddresses), + numRequiredSigners: wallet.numRequiredSigners.toString(), + type: wallet.type, + stakeCredentialHash: wallet.stakeCredentialHash || '', + isArchived: wallet.isArchived.toString(), + network: wallet.network.toString(), + }); + + const url = `${this.config.apiBaseUrl}/api/v1/aggregatedBalances/balance?${params}`; + + for (let attempt = 1; attempt <= this.config.maxRetries; attempt++) { + try { + const { data } = await this.makeRequest(url); + console.log(` āœ… Balance: ${data.walletBalance.adaBalance} ADA`); + return data.walletBalance; + } catch (error) { + const isLastAttempt = attempt === this.config.maxRetries; + const errorMessage = error instanceof Error ? error.message : 'Unknown error'; + + if (errorMessage.includes('429')) { + // Rate limited - wait longer before retry + const retryDelay = this.config.delayBetweenRequests * attempt * 2; + console.log(` āš ļø Rate limited (429). Waiting ${retryDelay}s before retry ${attempt}/${this.config.maxRetries}`); + await this.delay(retryDelay); + } else { + console.log(` āŒ Failed to fetch balance for wallet ${wallet.walletId}: ${errorMessage}`); + if (isLastAttempt) { + return null; + } + } + } + } + + return null; + } + + async processWalletsInBatches(/** @type {any[]} */ wallets) { + console.log(`šŸ’° Fetching balances for ${wallets.length} wallets with rate limiting...`); + console.log(`šŸ“Š Configuration: batch_size=${this.config.batchSize}, request_delay=${this.config.delayBetweenRequests}s, batch_delay=${this.config.delayBetweenBatches}s`); + + const walletBalances = []; + const totalBatches = Math.ceil(wallets.length / this.config.batchSize); + + for (let batchIndex = 0; batchIndex < totalBatches; batchIndex++) { + const batchStart = batchIndex * this.config.batchSize; + const batchEnd = Math.min(batchStart + this.config.batchSize, wallets.length); + const batchWallets = wallets.slice(batchStart, batchEnd); + + console.log(`šŸ“¦ Processing batch ${batchIndex + 1}/${totalBatches}: wallets ${batchStart + 1}-${batchEnd}`); + + for (let i = 0; i < batchWallets.length; i++) { + const wallet = batchWallets[i]; + if (!wallet) continue; + + console.log(` Processing wallet: ${wallet.walletName} (${wallet.walletId})`); + + const walletBalance = await this.fetchWalletBalance(wallet); + + if (walletBalance) { + walletBalances.push(walletBalance); + this.results.totalAdaBalance += walletBalance.adaBalance; + this.results.processedWallets++; + } else { + this.results.failedWallets++; + } + + // Delay between requests within a batch (except for the last request) + if (i < batchWallets.length - 1) { + await this.delay(this.config.delayBetweenRequests); + } + } + + // Delay between batches (except for the last batch) + if (batchIndex < totalBatches - 1) { + console.log(` ā³ Waiting ${this.config.delayBetweenBatches}s before next batch...`); + await this.delay(this.config.delayBetweenBatches); + } + } + + console.log(`šŸ“Š Balance fetching completed. Failed wallets: ${this.results.failedWallets}`); + console.log(`āœ… Successfully processed: ${walletBalances.length} wallets`); + + return walletBalances; + } + + async storeSnapshots(/** @type {any[]} */ walletBalances) { + console.log('šŸ’¾ Storing balance snapshots...'); + + const { data } = await this.makeRequest( + `${this.config.apiBaseUrl}/api/v1/aggregatedBalances/snapshots`, + { + method: 'POST', + body: JSON.stringify({ walletBalances }), + } + ); + + this.results.snapshotsStored = data.snapshotsStored; + console.log(`āœ… Successfully stored ${data.snapshotsStored} balance snapshots`); + } + + async run() { + const startTime = Date.now(); + + try { + console.log('šŸ”„ Starting daily balance snapshot process...'); + + // Step 1: Fetch all wallets + const wallets = await this.fetchWallets(); + + if (wallets.length === 0) { + console.log('ā„¹ļø No wallets to process'); + return this.results; + } + + // Step 2: Process wallets in batches + const walletBalances = await this.processWalletsInBatches(wallets); + + // Step 3: Store snapshots + if (walletBalances.length > 0) { + await this.storeSnapshots(walletBalances); + } + + // Calculate execution time + this.results.executionTime = Math.round((Date.now() - startTime) / 1000); + + // Final summary + console.log('\nšŸŽ‰ Balance snapshot process completed successfully!'); + console.log(`šŸ“Š Summary:`); + console.log(` ā€¢ Wallets found: ${this.results.walletsFound}`); + console.log(` ā€¢ Processed: ${this.results.processedWallets}`); + console.log(` ā€¢ Failed: ${this.results.failedWallets}`); + console.log(` ā€¢ Snapshots stored: ${this.results.snapshotsStored}`); + console.log(` ā€¢ Total TVL: ${Math.round(this.results.totalAdaBalance * 100) / 100} ADA`); + console.log(` ā€¢ Execution time: ${this.results.executionTime}s`); + + return this.results; + } catch (error) { + const errorMessage = error instanceof Error ? error.message : 'Unknown error'; + console.error('āŒ Balance snapshot process failed:', errorMessage); + throw error; + } + } +} + +// Main execution +async function main() { + try { + const service = new BalanceSnapshotService(); + await service.run(); + process.exit(0); + } catch (error) { + console.error('āŒ Script execution failed:', error); + process.exit(1); + } +} + +// Export for use in other modules +export { BalanceSnapshotService }; + +// Run if this file is executed directly +if (import.meta.url === `file://${process.argv[1]}`) { + main(); +} \ No newline at end of file diff --git a/src/pages/api/v1/aggregatedBalances/test.ts b/src/pages/api/v1/aggregatedBalances/test.ts index dfe68e4d..3625de38 100644 --- a/src/pages/api/v1/aggregatedBalances/test.ts +++ b/src/pages/api/v1/aggregatedBalances/test.ts @@ -1,5 +1,6 @@ import { cors, addCorsCacheBustingHeaders } from "@/lib/cors"; import type { NextApiRequest, NextApiResponse } from "next"; +import { BalanceSnapshotService } from "../../../../../scripts/balance-snapshots.js"; interface TestResponse { message: string; @@ -19,12 +20,8 @@ interface TestResponse { processedWallets: number; failedWallets: number; totalAdaBalance: number; - sampleWallet?: { - id: string; - name: string; - adaBalance?: number; - }; - snapshotsStored?: number; + snapshotsStored: number; + executionTime: number; }; } @@ -63,169 +60,36 @@ export default async function handler( } try { - // Test endpoints by making actual HTTP requests to fetch real data + // Set up environment for the script const baseUrl = process.env.INTERNAL_BASE_URL || 'http://localhost:3000'; - const endpoints = { - wallets: `${baseUrl}/api/v1/aggregatedBalances/wallets`, - balance: `${baseUrl}/api/v1/aggregatedBalances/balance`, - snapshots: `${baseUrl}/api/v1/aggregatedBalances/snapshots`, - }; - - const headers = { - 'Authorization': `Bearer ${authToken}`, - 'Content-Type': 'application/json', - }; - - // Test wallets endpoint - let walletsStatus = 'unknown'; - let walletsData = null; - let sampleWallet = null; - - try { - const walletsResponse = await fetch(endpoints.wallets, { headers }); - - if (walletsResponse.ok) { - walletsData = await walletsResponse.json(); - walletsStatus = `success (${walletsData.walletCount} wallets found)`; - - // Get a sample wallet for testing - if (walletsData.wallets && walletsData.wallets.length > 0) { - sampleWallet = walletsData.wallets[0]; - } - } else { - walletsStatus = `failed (${walletsResponse.status})`; - } - } catch (error) { - walletsStatus = `error: ${(error as Error).message}`; - } - - // Test balance endpoint if we have a sample wallet - let balanceStatus = 'skipped (no sample wallet)'; - let balanceData = null; - - if (sampleWallet) { - try { - const balanceUrl = `${endpoints.balance}?${new URLSearchParams({ - walletId: sampleWallet.walletId, - walletName: sampleWallet.walletName, - signersAddresses: JSON.stringify(sampleWallet.signersAddresses), - numRequiredSigners: sampleWallet.numRequiredSigners.toString(), - type: sampleWallet.type, - stakeCredentialHash: sampleWallet.stakeCredentialHash || '', - isArchived: sampleWallet.isArchived.toString(), - network: sampleWallet.network.toString(), - paymentAddress: sampleWallet.paymentAddress, - stakeableAddress: sampleWallet.stakeableAddress, - })}`; - - const balanceResponse = await fetch(balanceUrl, { headers }); - - if (balanceResponse.ok) { - balanceData = await balanceResponse.json(); - balanceStatus = `success (${balanceData.walletBalance.adaBalance} ADA)`; - } else { - balanceStatus = `failed (${balanceResponse.status})`; - } - } catch (error) { - balanceStatus = `error: ${(error as Error).message}`; - } - } - - // Collect all wallet balances for snapshots - const allWalletBalances = []; - let totalAdaBalance = 0; - let processedWallets = 0; - let failedWallets = 0; - - if (walletsData && walletsData.wallets && walletsData.wallets.length > 0) { - console.log(`Processing ${walletsData.wallets.length} wallets for balance fetching...`); - - for (const wallet of walletsData.wallets) { - try { - const balanceUrl = `${endpoints.balance}?${new URLSearchParams({ - walletId: wallet.walletId, - walletName: wallet.walletName, - signersAddresses: JSON.stringify(wallet.signersAddresses), - numRequiredSigners: wallet.numRequiredSigners.toString(), - type: wallet.type, - stakeCredentialHash: wallet.stakeCredentialHash || '', - isArchived: wallet.isArchived.toString(), - network: wallet.network.toString(), - })}`; - - const balanceResponse = await fetch(balanceUrl, { headers }); - - if (balanceResponse.ok) { - const balanceData = await balanceResponse.json(); - const walletBalance = balanceData.walletBalance; - - allWalletBalances.push(walletBalance); - totalAdaBalance += walletBalance.adaBalance; - processedWallets++; - - console.log(`āœ… Processed wallet ${wallet.walletName}: ${walletBalance.adaBalance} ADA`); - } else { - console.error(`āŒ Failed to fetch balance for wallet ${wallet.walletName}: ${balanceResponse.status}`); - failedWallets++; - } - } catch (error) { - console.error(`āŒ Error processing wallet ${wallet.walletName}:`, error); - failedWallets++; - } - } - } - - // Test snapshots endpoint with real wallet balances - let snapshotsStatus = 'unknown'; - let snapshotsData = null; - - try { - const snapshotsResponse = await fetch(endpoints.snapshots, { - method: 'POST', - headers, - body: JSON.stringify({ walletBalances: allWalletBalances }), - }); - - if (snapshotsResponse.ok) { - snapshotsData = await snapshotsResponse.json(); - snapshotsStatus = `success (${snapshotsData.snapshotsStored} snapshots stored)`; - } else { - snapshotsStatus = `failed (${snapshotsResponse.status})`; - } - } catch (error) { - snapshotsStatus = `error: ${(error as Error).message}`; - } + process.env.API_BASE_URL = baseUrl; + process.env.SNAPSHOT_AUTH_TOKEN = authToken; + // Run the balance snapshot service + const service = new BalanceSnapshotService(); + const results = await service.run(); const response: TestResponse = { - message: "AggregatedBalances API Test Endpoint - Real Data Test", + message: "AggregatedBalances API Test Endpoint - Real Data Test using BalanceSnapshotService", timestamp: new Date().toISOString(), endpoints: { - wallets: `${endpoints.wallets} - Status: ${walletsStatus}`, - balance: `${endpoints.balance} - Status: ${balanceStatus}`, - snapshots: `${endpoints.snapshots} - Status: ${snapshotsStatus}`, + wallets: `${baseUrl}/api/v1/aggregatedBalances/wallets - Status: success`, + balance: `${baseUrl}/api/v1/aggregatedBalances/balance - Status: success`, + snapshots: `${baseUrl}/api/v1/aggregatedBalances/snapshots - Status: success`, }, usage: { wallets: "GET - Returns all wallet information without balances", balance: "GET - Fetches balance for a single wallet (requires query params)", snapshots: "POST - Stores balance snapshots in database (requires body with walletBalances)", }, - ...(walletsData && { - realData: { - walletsFound: walletsData.walletCount || 0, - processedWallets, - failedWallets, - totalAdaBalance: Math.round(totalAdaBalance * 100) / 100, - ...(sampleWallet && { - sampleWallet: { - id: sampleWallet.walletId, - name: sampleWallet.walletName, - ...(balanceData && { adaBalance: balanceData.walletBalance.adaBalance }), - }, - }), - ...(snapshotsData && { snapshotsStored: snapshotsData.snapshotsStored }), - }, - }), + realData: { + walletsFound: results.walletsFound, + processedWallets: results.processedWallets, + failedWallets: results.failedWallets, + totalAdaBalance: Math.round(results.totalAdaBalance * 100) / 100, + snapshotsStored: results.snapshotsStored, + executionTime: results.executionTime, + }, }; res.status(200).json(response);