From 4324151cd51c527b7c8ad9443bacd0a3deb91af2 Mon Sep 17 00:00:00 2001 From: elcharitas Date: Tue, 18 Nov 2025 11:11:57 +0100 Subject: [PATCH 1/2] feat: infinite data load in dune tables --- .../src/api/services/custom-data/types.ts | 1 + .../src/api/utils/customDataUtils.tsx | 24 ++++ .../src/containers/dune/DuneContainer.tsx | 90 +++++++++++--- .../containers/dune/DuneTableContainer.tsx | 110 ++++++++++++++---- 4 files changed, 186 insertions(+), 39 deletions(-) diff --git a/packages/frontend/src/api/services/custom-data/types.ts b/packages/frontend/src/api/services/custom-data/types.ts index b0de333aa..21f3a3598 100644 --- a/packages/frontend/src/api/services/custom-data/types.ts +++ b/packages/frontend/src/api/services/custom-data/types.ts @@ -4,6 +4,7 @@ import { TPagination } from "../baseTypes"; export type TGetCustomItemsRequest = { page?: number; tags?: string; + limit?: number; endpointUrl: string; }; diff --git a/packages/frontend/src/api/utils/customDataUtils.tsx b/packages/frontend/src/api/utils/customDataUtils.tsx index 0a17dd836..d2cdddd1a 100644 --- a/packages/frontend/src/api/utils/customDataUtils.tsx +++ b/packages/frontend/src/api/utils/customDataUtils.tsx @@ -289,12 +289,28 @@ export const formatCustomDataField: ( }; } if (format === "decimal" || format === "number") { + // Handle null or invalid numeric values + if (rawField === null || rawField === undefined || rawField === "" || + (typeof rawField === "string" && isNaN(parseFloat(rawField)))) { + return { + field: "-", + error: undefined, + }; + } return { field: formatNumber({ value: rawField }).value, error: undefined, }; } if (format === "currency") { + // Handle null or invalid numeric values + if (rawField === null || rawField === undefined || rawField === "" || + (typeof rawField === "string" && isNaN(parseFloat(rawField)))) { + return { + field: "-", + error: undefined, + }; + } return { field: formatNumber({ value: rawField, @@ -305,6 +321,14 @@ export const formatCustomDataField: ( }; } if (format === "percentage") { + // Handle null or invalid numeric values + if (rawField === null || rawField === undefined || rawField === "" || + (typeof rawField === "string" && isNaN(parseFloat(rawField)))) { + return { + field: "-", + error: undefined, + }; + } return { field: formatNumber({ value: rawField, diff --git a/packages/frontend/src/containers/dune/DuneContainer.tsx b/packages/frontend/src/containers/dune/DuneContainer.tsx index 255743058..1207d1a17 100644 --- a/packages/frontend/src/containers/dune/DuneContainer.tsx +++ b/packages/frontend/src/containers/dune/DuneContainer.tsx @@ -1,5 +1,5 @@ -import { FC, useMemo } from "react"; -import { useAuth, useWidgetHeight } from "src/api/hooks"; +import { FC, useMemo, useState, useEffect, useCallback } from "react"; +import { useAuth, usePagination, useWidgetHeight } from "src/api/hooks"; import { useView } from "src/api/hooks/useView"; import { useImportDuneMutation, @@ -8,11 +8,16 @@ import { } from "src/api/services"; import { setWidgetHeight, updateWidgetCustomDataMeta } from "src/api/store"; import { useAppDispatch } from "src/api/store/hooks"; +import { TCustomItem } from "src/api/types"; import { extractDuneQueryId } from "src/api/utils/duneUtils"; +import { buildUniqueItemList } from "src/api/utils/itemUtils"; import { Logger } from "src/api/utils/logging"; import DuneModule from "src/components/dune/DuneModule"; +import CONFIG from "src/config"; import { IModuleContainer } from "src/types"; +const { MAX_PAGE_NUMBER } = CONFIG.WIDGETS.DUNE; + const DuneContainer: FC = ({ moduleData }) => { const dispatch = useAppDispatch(); const { isAuthenticated } = useAuth(); @@ -27,15 +32,73 @@ const DuneContainer: FC = ({ moduleData }) => { const widgetHeight = useWidgetHeight(moduleData); - const { data: apiData, isLoading: isLoadingApi } = useGetCustomItemsQuery( + const [currentPage, setCurrentPage] = useState( + undefined + ); + const [items, setItems] = useState(); + + const { + data: apiData, + isLoading: isLoadingApi, + isSuccess, + } = useGetCustomItemsQuery( { endpointUrl: endpoint_url || "", + page: currentPage, }, { skip: !endpoint_url, } ); + const { + nextPage, + handleNextPage, + reset: resetPagination, + } = usePagination(apiData?.links, MAX_PAGE_NUMBER, isSuccess); + + const handlePaginate = useCallback(() => { + handleNextPage("next"); + }, [handleNextPage]); + + // Set current page after next page is determined + useEffect(() => { + if (nextPage === undefined) { + return () => null; + } + const timeout = setTimeout(() => { + setCurrentPage(nextPage); + }, 350); + return () => { + clearTimeout(timeout); + }; + }, [nextPage]); + + // Reset pagination when endpoint changes + useEffect(() => { + if (endpoint_url) { + setItems(undefined); + setCurrentPage(undefined); + resetPagination(); + } + }, [endpoint_url, resetPagination]); + + // Build unique items list when new data arrives + useEffect(() => { + const newItems = apiData?.results; + if (newItems) { + setItems((prevItems) => { + if (prevItems) { + return buildUniqueItemList([ + ...prevItems, + ...newItems, + ]); + } + return newItems; + }); + } + }, [apiData?.results]); + const handleSetWidgetHeight = (height: number) => { dispatch( setWidgetHeight({ @@ -58,18 +121,11 @@ const DuneContainer: FC = ({ moduleData }) => { }; }, [custom_meta]); - const items = useMemo(() => { - // Use API data when endpoint_url is set and data_type is not Static + const displayItems = useMemo(() => { + // Use accumulated items from API when endpoint_url is set if (endpoint_url) { - const apiResults = apiData?.results; - if (apiResults && !Array.isArray(apiResults)) { - Logger.error( - "DuneContainer: API results is not an array", - apiResults - ); - return undefined; - } - return apiResults ?? []; + // Use accumulated items if available, otherwise fall back to current API data + return items || apiData?.results || []; } // Use static custom_data @@ -89,7 +145,7 @@ const DuneContainer: FC = ({ moduleData }) => { return undefined; } return custom_data; - }, [custom_data, custom_meta, apiData?.results, endpoint_url]); + }, [custom_data, custom_meta, items, endpoint_url, apiData?.results]); const isLoading = isImporting || isLoadingApi; @@ -151,11 +207,11 @@ const DuneContainer: FC = ({ moduleData }) => { return ( ({})} + handlePaginate={handlePaginate} widgetHeight={widgetHeight} setWidgetHeight={handleSetWidgetHeight} onSetEndpointUrl={handleSetEndpointUrl} diff --git a/packages/frontend/src/containers/dune/DuneTableContainer.tsx b/packages/frontend/src/containers/dune/DuneTableContainer.tsx index 722dc0a9a..5b4ec3d82 100644 --- a/packages/frontend/src/containers/dune/DuneTableContainer.tsx +++ b/packages/frontend/src/containers/dune/DuneTableContainer.tsx @@ -1,20 +1,82 @@ -import { FC, useMemo } from "react"; -import { useWidgetHeight } from "src/api/hooks"; -import { EWidgetData } from "src/api/services"; +import { FC, useMemo, useState, useEffect, useCallback } from "react"; +import { usePagination, useWidgetHeight } from "src/api/hooks"; +import { useGetCustomItemsQuery } from "src/api/services"; import { setWidgetHeight } from "src/api/store"; import { useAppDispatch } from "src/api/store/hooks"; +import { TCustomItem } from "src/api/types"; +import { buildUniqueItemList } from "src/api/utils/itemUtils"; import { Logger } from "src/api/utils/logging"; import DuneTableModule from "src/components/dune/DuneTableModule"; +import CONFIG from "src/config"; import { IModuleContainer } from "src/types"; +const { MAX_PAGE_NUMBER } = CONFIG.WIDGETS.DUNE_TABLE; + const DuneTableContainer: FC = ({ moduleData }) => { const dispatch = useAppDispatch(); /* eslint-disable @typescript-eslint/naming-convention */ - const { custom_data, custom_meta, data_type } = moduleData.widget; + const { custom_data, custom_meta, endpoint_url } = moduleData.widget; /* eslint-enable @typescript-eslint/naming-convention */ const widgetHeight = useWidgetHeight(moduleData); + + const [currentPage, setCurrentPage] = useState(1); + const [items, setItems] = useState(); + + const { + data: apiData, + isLoading: isLoadingApi, + isSuccess, + } = useGetCustomItemsQuery( + { + endpointUrl: endpoint_url || "", + page: currentPage, + limit: 20, + }, + { + skip: !endpoint_url, + } + ); + + const { + nextPage, + handleNextPage, + } = usePagination(apiData?.links, MAX_PAGE_NUMBER, isSuccess); + + const handlePaginate = useCallback(() => { + handleNextPage("next"); + }, [handleNextPage]); + + // Set current page after next page is determined + useEffect(() => { + if (nextPage === undefined) { + return; + } + const timeout = setTimeout(() => { + setCurrentPage(nextPage); + }, 350); + return () => { + clearTimeout(timeout); + }; + }, [nextPage]); + + // Build unique items list when new data arrives + useEffect(() => { + const newItems = apiData?.results; + if (newItems) { + setItems((prevItems) => { + if (prevItems) { + return buildUniqueItemList([ + ...prevItems, + ...newItems, + ]); + } + return newItems; + }); + } + }, [apiData?.results]); + const handleSetWidgetHeight = (height: number) => { dispatch( setWidgetHeight({ @@ -24,22 +86,26 @@ const DuneTableContainer: FC = ({ moduleData }) => { ); }; - const items = useMemo(() => { - if (data_type === EWidgetData.Static) { - if (!custom_data || !custom_meta) { - Logger.error("DuneTableContainer: missing data or meta"); - return []; - } - if (custom_meta.layout_type !== "table") { - Logger.error( - "DuneTableContainer: invalid layout type, expected table" - ); - return []; - } - return custom_data; + const displayItems = useMemo(() => { + // Use accumulated items from API when endpoint_url is set + if (endpoint_url) { + // Use accumulated items if available, otherwise fall back to current API data + return items || apiData?.results || []; + } + + // Use static custom_data + if (!custom_data || !custom_meta) { + Logger.error("DuneTableContainer: missing data or meta"); + return []; + } + if (custom_meta.layout_type !== "table") { + Logger.error( + "DuneTableContainer: invalid layout type, expected table" + ); + return []; } - return []; - }, [custom_data, custom_meta, data_type]); + return custom_data; + }, [custom_data, custom_meta, items, endpoint_url, apiData?.results]); const meta = useMemo(() => { if (custom_meta?.layout_type === "table") { @@ -56,11 +122,11 @@ const DuneTableContainer: FC = ({ moduleData }) => { return ( ({})} + isLoadingItems={isLoadingApi} + handlePaginate={handlePaginate} widgetHeight={widgetHeight} setWidgetHeight={handleSetWidgetHeight} /> From c75c16984107b25c8593d9120206ec929321b428 Mon Sep 17 00:00:00 2001 From: elcharitas Date: Tue, 18 Nov 2025 12:01:28 +0100 Subject: [PATCH 2/2] chore: lint fixes --- .../src/api/utils/customDataUtils.tsx | 24 ++++++++++++++----- .../containers/dune/DuneTableContainer.tsx | 11 +++++---- 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/packages/frontend/src/api/utils/customDataUtils.tsx b/packages/frontend/src/api/utils/customDataUtils.tsx index d2cdddd1a..d910c64ba 100644 --- a/packages/frontend/src/api/utils/customDataUtils.tsx +++ b/packages/frontend/src/api/utils/customDataUtils.tsx @@ -290,8 +290,12 @@ export const formatCustomDataField: ( } if (format === "decimal" || format === "number") { // Handle null or invalid numeric values - if (rawField === null || rawField === undefined || rawField === "" || - (typeof rawField === "string" && isNaN(parseFloat(rawField)))) { + if ( + rawField === null || + rawField === undefined || + rawField === "" || + (typeof rawField === "string" && isNaN(parseFloat(rawField))) + ) { return { field: "-", error: undefined, @@ -304,8 +308,12 @@ export const formatCustomDataField: ( } if (format === "currency") { // Handle null or invalid numeric values - if (rawField === null || rawField === undefined || rawField === "" || - (typeof rawField === "string" && isNaN(parseFloat(rawField)))) { + if ( + rawField === null || + rawField === undefined || + rawField === "" || + (typeof rawField === "string" && isNaN(parseFloat(rawField))) + ) { return { field: "-", error: undefined, @@ -322,8 +330,12 @@ export const formatCustomDataField: ( } if (format === "percentage") { // Handle null or invalid numeric values - if (rawField === null || rawField === undefined || rawField === "" || - (typeof rawField === "string" && isNaN(parseFloat(rawField)))) { + if ( + rawField === null || + rawField === undefined || + rawField === "" || + (typeof rawField === "string" && isNaN(parseFloat(rawField))) + ) { return { field: "-", error: undefined, diff --git a/packages/frontend/src/containers/dune/DuneTableContainer.tsx b/packages/frontend/src/containers/dune/DuneTableContainer.tsx index 5b4ec3d82..1ad3eeac0 100644 --- a/packages/frontend/src/containers/dune/DuneTableContainer.tsx +++ b/packages/frontend/src/containers/dune/DuneTableContainer.tsx @@ -39,10 +39,11 @@ const DuneTableContainer: FC = ({ moduleData }) => { } ); - const { - nextPage, - handleNextPage, - } = usePagination(apiData?.links, MAX_PAGE_NUMBER, isSuccess); + const { nextPage, handleNextPage } = usePagination( + apiData?.links, + MAX_PAGE_NUMBER, + isSuccess + ); const handlePaginate = useCallback(() => { handleNextPage("next"); @@ -51,7 +52,7 @@ const DuneTableContainer: FC = ({ moduleData }) => { // Set current page after next page is determined useEffect(() => { if (nextPage === undefined) { - return; + return undefined; } const timeout = setTimeout(() => { setCurrentPage(nextPage);