From d360ec59ea6cf33afb7acc02105d5b6993b66775 Mon Sep 17 00:00:00 2001 From: crthpl Date: Sun, 8 Mar 2026 12:18:19 -0400 Subject: [PATCH 1/2] Include traded_market_ids in Portfolio for reliable history loading The performance page needs to know which markets an account has ever traded/redeemed in order to request full trade history. Previously it only discovered markets from current portfolio exposure or already-loaded trades, missing closed positions and historical redemptions. Add a `traded_market_ids` field to the Portfolio proto, populated from a UNION query over trade and redemption tables during initial data send. The frontend stores these per-account and uses them to drive GetFullTradeHistory requests. Also fixes redeem fee double-counting in per-market filtered PnL views. --- ...30f2b37f3d17211d002fbfcb811439f1c933f.json | 20 +++++++ backend/src/convert.rs | 2 + backend/src/db.rs | 23 ++++++++ backend/src/handle_socket.rs | 3 +- frontend/src/lib/api.svelte.ts | 8 +++ frontend/src/lib/pnlMetrics.ts | 23 ++++++-- frontend/src/routes/performance/+page.svelte | 8 ++- schema-js/index.d.ts | 6 ++ schema-js/index.js | 57 +++++++++++++++++++ schema/portfolio.proto | 1 + 10 files changed, 145 insertions(+), 6 deletions(-) create mode 100644 backend/.sqlx/query-1d70ff3817d7d94273082a874db30f2b37f3d17211d002fbfcb811439f1c933f.json diff --git a/backend/.sqlx/query-1d70ff3817d7d94273082a874db30f2b37f3d17211d002fbfcb811439f1c933f.json b/backend/.sqlx/query-1d70ff3817d7d94273082a874db30f2b37f3d17211d002fbfcb811439f1c933f.json new file mode 100644 index 00000000..bf47a76f --- /dev/null +++ b/backend/.sqlx/query-1d70ff3817d7d94273082a874db30f2b37f3d17211d002fbfcb811439f1c933f.json @@ -0,0 +1,20 @@ +{ + "db_name": "SQLite", + "query": "\n SELECT DISTINCT market_id as \"market_id!\"\n FROM (\n SELECT market_id FROM trade WHERE buyer_id = ?1 OR seller_id = ?1\n UNION\n SELECT fund_id AS market_id FROM redemption WHERE redeemer_id = ?1\n )\n ", + "describe": { + "columns": [ + { + "name": "market_id!", + "ordinal": 0, + "type_info": "Int64" + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + true + ] + }, + "hash": "1d70ff3817d7d94273082a874db30f2b37f3d17211d002fbfcb811439f1c933f" +} diff --git a/backend/src/convert.rs b/backend/src/convert.rs index ad7cee1b..9836d990 100644 --- a/backend/src/convert.rs +++ b/backend/src/convert.rs @@ -15,6 +15,7 @@ impl From for websocket_api::Portfolio { available_balance, market_exposures, owner_credits, + traded_market_ids, }: db::Portfolio, ) -> Self { Self { @@ -39,6 +40,7 @@ impl From for websocket_api::Portfolio { credit: credit.credit.0.try_into().unwrap(), }) .collect(), + traded_market_ids, } } } diff --git a/backend/src/db.rs b/backend/src/db.rs index 57037c9e..5801fb6f 100644 --- a/backend/src/db.rs +++ b/backend/src/db.rs @@ -862,6 +862,27 @@ impl DB { } } + /// Get all market IDs an account has ever traded in or redeemed from. + /// + /// # Errors + /// Returns an error if the database query fails. + pub async fn get_traded_market_ids(&self, account_id: i64) -> SqlxResult> { + let rows = sqlx::query_scalar!( + r#" + SELECT DISTINCT market_id as "market_id!" + FROM ( + SELECT market_id FROM trade WHERE buyer_id = ?1 OR seller_id = ?1 + UNION + SELECT fund_id AS market_id FROM redemption WHERE redeemer_id = ?1 + ) + "#, + account_id + ) + .fetch_all(&self.pool) + .await?; + Ok(rows) + } + #[must_use] pub fn get_all_accounts(&self) -> BoxStream<'_, SqlxResult> { sqlx::query_as!( @@ -3633,6 +3654,7 @@ async fn get_portfolio( available_balance, market_exposures, owner_credits: vec![], + traded_market_ids: vec![], })) } @@ -4061,6 +4083,7 @@ pub struct Portfolio { pub available_balance: Decimal, pub market_exposures: Vec, pub owner_credits: Vec, + pub traded_market_ids: Vec, } #[derive(Debug)] diff --git a/backend/src/handle_socket.rs b/backend/src/handle_socket.rs index 2364784d..f1f14903 100644 --- a/backend/src/handle_socket.rs +++ b/backend/src/handle_socket.rs @@ -278,10 +278,11 @@ async fn send_initial_private_data( let mut transfers = Vec::new(); let mut portfolios = Vec::new(); for &account_id in accounts { - let Some(portfolio) = db.get_portfolio(account_id).await? else { + let Some(mut portfolio) = db.get_portfolio(account_id).await? else { tracing::warn!("Account {account_id} not found"); continue; }; + portfolio.traded_market_ids = db.get_traded_market_ids(account_id).await?; portfolios.push(Portfolio::from(portfolio)); transfers.extend( db.get_transfers(account_id) diff --git a/frontend/src/lib/api.svelte.ts b/frontend/src/lib/api.svelte.ts index b022f2f7..36e0df1f 100644 --- a/frontend/src/lib/api.svelte.ts +++ b/frontend/src/lib/api.svelte.ts @@ -63,6 +63,7 @@ export const serverState = $state({ marketGroups: new SvelteMap(), auctions: new SvelteMap(), universes: new SvelteMap(), + tradedMarketIds: new SvelteMap>(), lastKnownTransactionId: 0, arborPixieAccountId: undefined as number | undefined }); @@ -258,9 +259,16 @@ socket.onmessage = (event: MessageEvent) => { if (msg.portfolios) { if (!msg.portfolios.areNewOwnerships) { serverState.portfolios.clear(); + serverState.tradedMarketIds.clear(); } for (const p of msg.portfolios.portfolios || []) { serverState.portfolios.set(p.accountId, p); + if (p.tradedMarketIds?.length) { + serverState.tradedMarketIds.set( + p.accountId as number, + new Set(p.tradedMarketIds.map(Number)) + ); + } if (p.accountId == serverState.actingAs) { serverState.portfolio = p; } diff --git a/frontend/src/lib/pnlMetrics.ts b/frontend/src/lib/pnlMetrics.ts index bcd24e63..2161ce2b 100644 --- a/frontend/src/lib/pnlMetrics.ts +++ b/frontend/src/lib/pnlMetrics.ts @@ -357,8 +357,10 @@ export function computePnLOverTime( if (!cashFlowByMarket.has(cid)) cashFlowByMarket.set(cid, 0); } - // Track redeem fee - totalRedeemFees += event.redeemFee * amount; + // Track redeem fee (attributed to the fund market) + if (!filterMarketIds || filterMarketIds.has(fundId)) { + totalRedeemFees += event.redeemFee * amount; + } const { totalCash, totalMtM } = computePnLAtPoint(); dataPoints.push({ @@ -466,12 +468,25 @@ export function computePnLOverTime( export function getMarketsNeedingHistory( accountId: number, markets: Map, - portfolio: websocket_api.IPortfolio | undefined + portfolio: websocket_api.IPortfolio | undefined, + tradedMarketIds: Set | undefined ): number[] { const needed: number[] = []; const seen = new Set(); - // Markets with current exposure + // Markets the backend told us this account has traded/redeemed + if (tradedMarketIds) { + for (const marketId of tradedMarketIds) { + const marketData = markets.get(marketId); + if (marketData && !marketData.hasFullTradeHistory && !seen.has(marketId)) { + seen.add(marketId); + needed.push(marketId); + } + } + } + + // Markets with current exposure (covers markets not yet in traded_market_ids, + // e.g. from redemption constituents the account never directly traded) for (const exposure of portfolio?.marketExposures || []) { const marketId = Number(exposure.marketId ?? 0); const marketData = markets.get(marketId); diff --git a/frontend/src/routes/performance/+page.svelte b/frontend/src/routes/performance/+page.svelte index db1aba9c..e511a66f 100644 --- a/frontend/src/routes/performance/+page.svelte +++ b/frontend/src/routes/performance/+page.svelte @@ -145,7 +145,13 @@ } const portfolio = serverState.portfolios.get(accountId); - const needed = getMarketsNeedingHistory(accountId, serverState.markets, portfolio); + const tradedMarketIds = serverState.tradedMarketIds.get(accountId); + const needed = getMarketsNeedingHistory( + accountId, + serverState.markets, + portfolio, + tradedMarketIds + ); let delay = 0; for (const marketId of needed) { diff --git a/schema-js/index.d.ts b/schema-js/index.d.ts index 3800558b..104c0579 100644 --- a/schema-js/index.d.ts +++ b/schema-js/index.d.ts @@ -1567,6 +1567,9 @@ export namespace websocket_api { /** Portfolio ownerCredits */ ownerCredits?: (websocket_api.Portfolio.IOwnerCredit[]|null); + + /** Portfolio tradedMarketIds */ + tradedMarketIds?: ((number|Long)[]|null); } /** Represents a Portfolio. */ @@ -1593,6 +1596,9 @@ export namespace websocket_api { /** Portfolio ownerCredits. */ public ownerCredits: websocket_api.Portfolio.IOwnerCredit[]; + /** Portfolio tradedMarketIds. */ + public tradedMarketIds: (number|Long)[]; + /** * Creates a new Portfolio instance using the specified properties. * @param [properties] Properties to set diff --git a/schema-js/index.js b/schema-js/index.js index a15f8022..10e7cdd5 100644 --- a/schema-js/index.js +++ b/schema-js/index.js @@ -4159,6 +4159,7 @@ $root.websocket_api = (function() { * @property {number|null} [availableBalance] Portfolio availableBalance * @property {Array.|null} [marketExposures] Portfolio marketExposures * @property {Array.|null} [ownerCredits] Portfolio ownerCredits + * @property {Array.|null} [tradedMarketIds] Portfolio tradedMarketIds */ /** @@ -4172,6 +4173,7 @@ $root.websocket_api = (function() { function Portfolio(properties) { this.marketExposures = []; this.ownerCredits = []; + this.tradedMarketIds = []; if (properties) for (var keys = Object.keys(properties), i = 0; i < keys.length; ++i) if (properties[keys[i]] != null) @@ -4218,6 +4220,14 @@ $root.websocket_api = (function() { */ Portfolio.prototype.ownerCredits = $util.emptyArray; + /** + * Portfolio tradedMarketIds. + * @member {Array.} tradedMarketIds + * @memberof websocket_api.Portfolio + * @instance + */ + Portfolio.prototype.tradedMarketIds = $util.emptyArray; + /** * Creates a new Portfolio instance using the specified properties. * @function create @@ -4254,6 +4264,12 @@ $root.websocket_api = (function() { if (message.ownerCredits != null && message.ownerCredits.length) for (var i = 0; i < message.ownerCredits.length; ++i) $root.websocket_api.Portfolio.OwnerCredit.encode(message.ownerCredits[i], writer.uint32(/* id 5, wireType 2 =*/42).fork()).ldelim(); + if (message.tradedMarketIds != null && message.tradedMarketIds.length) { + writer.uint32(/* id 6, wireType 2 =*/50).fork(); + for (var i = 0; i < message.tradedMarketIds.length; ++i) + writer.int64(message.tradedMarketIds[i]); + writer.ldelim(); + } return writer; }; @@ -4312,6 +4328,17 @@ $root.websocket_api = (function() { message.ownerCredits.push($root.websocket_api.Portfolio.OwnerCredit.decode(reader, reader.uint32())); break; } + case 6: { + if (!(message.tradedMarketIds && message.tradedMarketIds.length)) + message.tradedMarketIds = []; + if ((tag & 7) === 2) { + var end2 = reader.uint32() + reader.pos; + while (reader.pos < end2) + message.tradedMarketIds.push(reader.int64()); + } else + message.tradedMarketIds.push(reader.int64()); + break; + } default: reader.skipType(tag & 7); break; @@ -4374,6 +4401,13 @@ $root.websocket_api = (function() { return "ownerCredits." + error; } } + if (message.tradedMarketIds != null && message.hasOwnProperty("tradedMarketIds")) { + if (!Array.isArray(message.tradedMarketIds)) + return "tradedMarketIds: array expected"; + for (var i = 0; i < message.tradedMarketIds.length; ++i) + if (!$util.isInteger(message.tradedMarketIds[i]) && !(message.tradedMarketIds[i] && $util.isInteger(message.tradedMarketIds[i].low) && $util.isInteger(message.tradedMarketIds[i].high))) + return "tradedMarketIds: integer|Long[] expected"; + } return null; }; @@ -4422,6 +4456,20 @@ $root.websocket_api = (function() { message.ownerCredits[i] = $root.websocket_api.Portfolio.OwnerCredit.fromObject(object.ownerCredits[i]); } } + if (object.tradedMarketIds) { + if (!Array.isArray(object.tradedMarketIds)) + throw TypeError(".websocket_api.Portfolio.tradedMarketIds: array expected"); + message.tradedMarketIds = []; + for (var i = 0; i < object.tradedMarketIds.length; ++i) + if ($util.Long) + (message.tradedMarketIds[i] = $util.Long.fromValue(object.tradedMarketIds[i])).unsigned = false; + else if (typeof object.tradedMarketIds[i] === "string") + message.tradedMarketIds[i] = parseInt(object.tradedMarketIds[i], 10); + else if (typeof object.tradedMarketIds[i] === "number") + message.tradedMarketIds[i] = object.tradedMarketIds[i]; + else if (typeof object.tradedMarketIds[i] === "object") + message.tradedMarketIds[i] = new $util.LongBits(object.tradedMarketIds[i].low >>> 0, object.tradedMarketIds[i].high >>> 0).toNumber(); + } return message; }; @@ -4441,6 +4489,7 @@ $root.websocket_api = (function() { if (options.arrays || options.defaults) { object.marketExposures = []; object.ownerCredits = []; + object.tradedMarketIds = []; } if (options.defaults) { if ($util.Long) { @@ -4470,6 +4519,14 @@ $root.websocket_api = (function() { for (var j = 0; j < message.ownerCredits.length; ++j) object.ownerCredits[j] = $root.websocket_api.Portfolio.OwnerCredit.toObject(message.ownerCredits[j], options); } + if (message.tradedMarketIds && message.tradedMarketIds.length) { + object.tradedMarketIds = []; + for (var j = 0; j < message.tradedMarketIds.length; ++j) + if (typeof message.tradedMarketIds[j] === "number") + object.tradedMarketIds[j] = options.longs === String ? String(message.tradedMarketIds[j]) : message.tradedMarketIds[j]; + else + object.tradedMarketIds[j] = options.longs === String ? $util.Long.prototype.toString.call(message.tradedMarketIds[j]) : options.longs === Number ? new $util.LongBits(message.tradedMarketIds[j].low >>> 0, message.tradedMarketIds[j].high >>> 0).toNumber() : message.tradedMarketIds[j]; + } return object; }; diff --git a/schema/portfolio.proto b/schema/portfolio.proto index 343244ce..55ef5efc 100644 --- a/schema/portfolio.proto +++ b/schema/portfolio.proto @@ -7,6 +7,7 @@ message Portfolio { double available_balance = 3; repeated MarketExposure market_exposures = 4; repeated OwnerCredit owner_credits = 5; + repeated int64 traded_market_ids = 6; message MarketExposure { int64 market_id = 1; From 5c24f2d61411a6cc3281d45d57ce9fbe8b112066 Mon Sep 17 00:00:00 2001 From: crthpl Date: Sun, 8 Mar 2026 12:38:00 -0400 Subject: [PATCH 2/2] Split OR into separate UNIONs so SQLite uses both indexes --- ...6068969577978efcb1df79079b8dd5c3dab459a3cf79387787f4.json} | 4 ++-- backend/src/db.rs | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) rename backend/.sqlx/{query-1d70ff3817d7d94273082a874db30f2b37f3d17211d002fbfcb811439f1c933f.json => query-f56b5ce821696068969577978efcb1df79079b8dd5c3dab459a3cf79387787f4.json} (50%) diff --git a/backend/.sqlx/query-1d70ff3817d7d94273082a874db30f2b37f3d17211d002fbfcb811439f1c933f.json b/backend/.sqlx/query-f56b5ce821696068969577978efcb1df79079b8dd5c3dab459a3cf79387787f4.json similarity index 50% rename from backend/.sqlx/query-1d70ff3817d7d94273082a874db30f2b37f3d17211d002fbfcb811439f1c933f.json rename to backend/.sqlx/query-f56b5ce821696068969577978efcb1df79079b8dd5c3dab459a3cf79387787f4.json index bf47a76f..24891c50 100644 --- a/backend/.sqlx/query-1d70ff3817d7d94273082a874db30f2b37f3d17211d002fbfcb811439f1c933f.json +++ b/backend/.sqlx/query-f56b5ce821696068969577978efcb1df79079b8dd5c3dab459a3cf79387787f4.json @@ -1,6 +1,6 @@ { "db_name": "SQLite", - "query": "\n SELECT DISTINCT market_id as \"market_id!\"\n FROM (\n SELECT market_id FROM trade WHERE buyer_id = ?1 OR seller_id = ?1\n UNION\n SELECT fund_id AS market_id FROM redemption WHERE redeemer_id = ?1\n )\n ", + "query": "\n SELECT DISTINCT market_id as \"market_id!\"\n FROM (\n SELECT market_id FROM trade WHERE buyer_id = ?1\n UNION\n SELECT market_id FROM trade WHERE seller_id = ?1\n UNION\n SELECT fund_id AS market_id FROM redemption WHERE redeemer_id = ?1\n )\n ", "describe": { "columns": [ { @@ -16,5 +16,5 @@ true ] }, - "hash": "1d70ff3817d7d94273082a874db30f2b37f3d17211d002fbfcb811439f1c933f" + "hash": "f56b5ce821696068969577978efcb1df79079b8dd5c3dab459a3cf79387787f4" } diff --git a/backend/src/db.rs b/backend/src/db.rs index 5801fb6f..9a44d88a 100644 --- a/backend/src/db.rs +++ b/backend/src/db.rs @@ -871,7 +871,9 @@ impl DB { r#" SELECT DISTINCT market_id as "market_id!" FROM ( - SELECT market_id FROM trade WHERE buyer_id = ?1 OR seller_id = ?1 + SELECT market_id FROM trade WHERE buyer_id = ?1 + UNION + SELECT market_id FROM trade WHERE seller_id = ?1 UNION SELECT fund_id AS market_id FROM redemption WHERE redeemer_id = ?1 )