From 1abd469c47ce9a0f465cc12d8693e1c91a3d53f3 Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Wed, 2 Apr 2025 14:16:32 -0700 Subject: [PATCH 01/86] mark the top bar location Signed-off-by: Jialiang Liang --- .../data/public/ui/query_string_input/query_bar_top_row.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/src/plugins/data/public/ui/query_string_input/query_bar_top_row.tsx b/src/plugins/data/public/ui/query_string_input/query_bar_top_row.tsx index 63cd290273bb..1ec3cda6c388 100644 --- a/src/plugins/data/public/ui/query_string_input/query_bar_top_row.tsx +++ b/src/plugins/data/public/ui/query_string_input/query_bar_top_row.tsx @@ -408,6 +408,7 @@ export default function QueryBarTopRow(props: QueryBarTopRowProps) { ? createPortal(renderUpdateButton(), props.datePickerRef!.current!) : renderUpdateButton()} + {/* Here is the place if we consider to add the button at top bar Sync button should be here? */} ); From f929d31f4aeba09713c57ea797f95445ed3b2cdf Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Thu, 3 Apr 2025 12:09:51 -0700 Subject: [PATCH 02/86] draft - trying to find index name through panels Signed-off-by: Jialiang Liang --- .../public/lib/panel/embeddable_panel.tsx | 80 +++++++++++++------ 1 file changed, 57 insertions(+), 23 deletions(-) diff --git a/src/plugins/embeddable/public/lib/panel/embeddable_panel.tsx b/src/plugins/embeddable/public/lib/panel/embeddable_panel.tsx index 02ebf8b7f567..cc1de0c87fe2 100644 --- a/src/plugins/embeddable/public/lib/panel/embeddable_panel.tsx +++ b/src/plugins/embeddable/public/lib/panel/embeddable_panel.tsx @@ -87,6 +87,8 @@ interface Props { // https://github.com/opensearch-project/OpenSearch-Dashboards/issues/4483 hasBorder?: boolean; hasShadow?: boolean; + http: CoreStart['http']; + savedObjects?: CoreStart['savedObjects']; } interface State { @@ -101,6 +103,7 @@ interface State { loading?: boolean; error?: EmbeddableError; errorEmbeddable?: ErrorEmbeddable; + isSpecificType?: boolean; // Added for button logic } export class EmbeddablePanel extends React.Component { @@ -129,6 +132,7 @@ export class EmbeddablePanel extends React.Component { closeContextMenu: false, badges: [], notifications: [], + isSpecificType: true, // Added }; this.embeddableRoot = React.createRef(); @@ -217,6 +221,54 @@ export class EmbeddablePanel extends React.Component { this.props.embeddable.destroy(); } + public componentDidMount() { + if (this.embeddableRoot.current) { + this.subscription.add( + this.props.embeddable.getOutput$().subscribe( + (output: EmbeddableOutput) => { + this.setState({ + error: output.error, + loading: output.loading, + }); + }, + (error) => { + if (this.embeddableRoot.current) { + const errorEmbeddable = new ErrorEmbeddable(error, { id: this.props.embeddable.id }); + errorEmbeddable.render(this.embeddableRoot.current); + this.setState({ errorEmbeddable }); + } + } + ) + ); + this.props.embeddable.render(this.embeddableRoot.current); + } + this.checkIndexData(); // Added to check index type + } + + private async checkIndexData() { + const input = this.props.embeddable.getInput() as any; // Type assertion for now + console.log('Embeddable input:', input); // Debug input + + const indexName = input.indexPattern || input.index || 'unknown'; + if (indexName !== 'unknown') { + try { + const mappingResponse = await this.props.http.get(`/${indexName}/_mapping`); + const mapping = mappingResponse[indexName]?.mappings || {}; + const isSpecificType = this.isSpecificIndexType(mapping); + if (this.mounted) { + this.setState({ isSpecificType }); + } + } catch (error) { + console.error(`Error fetching mapping for ${indexName}:`, error); + } + } + } + + private isSpecificIndexType(mapping: any): boolean { + // TODO: Replace with your specific index type logic + return true; // Placeholder + } + public onFocus = (focusedPanelIndex: string) => { this.setState({ focusedPanelIndex }); }; @@ -266,34 +318,16 @@ export class EmbeddablePanel extends React.Component { /> )} + {this.state.isSpecificType && ( + + )}
); } - public componentDidMount() { - if (this.embeddableRoot.current) { - this.subscription.add( - this.props.embeddable.getOutput$().subscribe( - (output: EmbeddableOutput) => { - this.setState({ - error: output.error, - loading: output.loading, - }); - }, - (error) => { - if (this.embeddableRoot.current) { - const errorEmbeddable = new ErrorEmbeddable(error, { id: this.props.embeddable.id }); - errorEmbeddable.render(this.embeddableRoot.current); - this.setState({ errorEmbeddable }); - } - } - ) - ); - this.props.embeddable.render(this.embeddableRoot.current); - } - } - closeMyContextMenuPanel = () => { if (this.mounted) { this.setState({ closeContextMenu: true }, () => { From d73087e0bd1cb8875ca4d8f66530d680dae10987 Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Fri, 4 Apr 2025 00:41:30 -0700 Subject: [PATCH 03/86] demo experiment 1 for button on every panels Signed-off-by: Jialiang Liang --- .../public/lib/panel/embeddable_panel.tsx | 39 ++++++++++--------- src/plugins/embeddable/public/plugin.tsx | 2 + 2 files changed, 23 insertions(+), 18 deletions(-) diff --git a/src/plugins/embeddable/public/lib/panel/embeddable_panel.tsx b/src/plugins/embeddable/public/lib/panel/embeddable_panel.tsx index cc1de0c87fe2..aab7576427c2 100644 --- a/src/plugins/embeddable/public/lib/panel/embeddable_panel.tsx +++ b/src/plugins/embeddable/public/lib/panel/embeddable_panel.tsx @@ -103,7 +103,8 @@ interface State { loading?: boolean; error?: EmbeddableError; errorEmbeddable?: ErrorEmbeddable; - isSpecificType?: boolean; // Added for button logic + isSpecificType?: boolean; + panelSavedObjectIds: string[]; // Added to collect savedObjectIds } export class EmbeddablePanel extends React.Component { @@ -132,7 +133,8 @@ export class EmbeddablePanel extends React.Component { closeContextMenu: false, badges: [], notifications: [], - isSpecificType: true, // Added + isSpecificType: true, + panelSavedObjectIds: [], // Initialize empty array }; this.embeddableRoot = React.createRef(); @@ -242,31 +244,32 @@ export class EmbeddablePanel extends React.Component { ); this.props.embeddable.render(this.embeddableRoot.current); } - this.checkIndexData(); // Added to check index type + this.collectPanelSavedObjectId(); } - private async checkIndexData() { + private collectPanelSavedObjectId() { const input = this.props.embeddable.getInput() as any; // Type assertion for now - console.log('Embeddable input:', input); // Debug input - - const indexName = input.indexPattern || input.index || 'unknown'; - if (indexName !== 'unknown') { - try { - const mappingResponse = await this.props.http.get(`/${indexName}/_mapping`); - const mapping = mappingResponse[indexName]?.mappings || {}; - const isSpecificType = this.isSpecificIndexType(mapping); - if (this.mounted) { - this.setState({ isSpecificType }); + const savedObjectId = input.savedObjectId; + + if (savedObjectId) { + // Add this panel's savedObjectId to the array + this.setState( + (prevState) => ({ + panelSavedObjectIds: [...prevState.panelSavedObjectIds, savedObjectId], + }), + () => { + // Log the updated array after state is set + console.log('Panel Saved Object IDs:', this.state.panelSavedObjectIds); } - } catch (error) { - console.error(`Error fetching mapping for ${indexName}:`, error); - } + ); + } else { + console.log('No savedObjectId found for this panel'); } } private isSpecificIndexType(mapping: any): boolean { // TODO: Replace with your specific index type logic - return true; // Placeholder + return true; // Your placeholder } public onFocus = (focusedPanelIndex: string) => { diff --git a/src/plugins/embeddable/public/plugin.tsx b/src/plugins/embeddable/public/plugin.tsx index 4ff348a512fc..9747c764b0e4 100644 --- a/src/plugins/embeddable/public/plugin.tsx +++ b/src/plugins/embeddable/public/plugin.tsx @@ -200,6 +200,8 @@ export class EmbeddablePublicPlugin implements Plugin ); From 52b8e5a13158ce26c5aa7f883f1313fac78de9e4 Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Fri, 4 Apr 2025 00:45:46 -0700 Subject: [PATCH 04/86] revert to clean start for embeddable_panel class Signed-off-by: Jialiang Liang --- .../public/lib/panel/embeddable_panel.tsx | 83 +++++-------------- 1 file changed, 23 insertions(+), 60 deletions(-) diff --git a/src/plugins/embeddable/public/lib/panel/embeddable_panel.tsx b/src/plugins/embeddable/public/lib/panel/embeddable_panel.tsx index aab7576427c2..02ebf8b7f567 100644 --- a/src/plugins/embeddable/public/lib/panel/embeddable_panel.tsx +++ b/src/plugins/embeddable/public/lib/panel/embeddable_panel.tsx @@ -87,8 +87,6 @@ interface Props { // https://github.com/opensearch-project/OpenSearch-Dashboards/issues/4483 hasBorder?: boolean; hasShadow?: boolean; - http: CoreStart['http']; - savedObjects?: CoreStart['savedObjects']; } interface State { @@ -103,8 +101,6 @@ interface State { loading?: boolean; error?: EmbeddableError; errorEmbeddable?: ErrorEmbeddable; - isSpecificType?: boolean; - panelSavedObjectIds: string[]; // Added to collect savedObjectIds } export class EmbeddablePanel extends React.Component { @@ -133,8 +129,6 @@ export class EmbeddablePanel extends React.Component { closeContextMenu: false, badges: [], notifications: [], - isSpecificType: true, - panelSavedObjectIds: [], // Initialize empty array }; this.embeddableRoot = React.createRef(); @@ -223,55 +217,6 @@ export class EmbeddablePanel extends React.Component { this.props.embeddable.destroy(); } - public componentDidMount() { - if (this.embeddableRoot.current) { - this.subscription.add( - this.props.embeddable.getOutput$().subscribe( - (output: EmbeddableOutput) => { - this.setState({ - error: output.error, - loading: output.loading, - }); - }, - (error) => { - if (this.embeddableRoot.current) { - const errorEmbeddable = new ErrorEmbeddable(error, { id: this.props.embeddable.id }); - errorEmbeddable.render(this.embeddableRoot.current); - this.setState({ errorEmbeddable }); - } - } - ) - ); - this.props.embeddable.render(this.embeddableRoot.current); - } - this.collectPanelSavedObjectId(); - } - - private collectPanelSavedObjectId() { - const input = this.props.embeddable.getInput() as any; // Type assertion for now - const savedObjectId = input.savedObjectId; - - if (savedObjectId) { - // Add this panel's savedObjectId to the array - this.setState( - (prevState) => ({ - panelSavedObjectIds: [...prevState.panelSavedObjectIds, savedObjectId], - }), - () => { - // Log the updated array after state is set - console.log('Panel Saved Object IDs:', this.state.panelSavedObjectIds); - } - ); - } else { - console.log('No savedObjectId found for this panel'); - } - } - - private isSpecificIndexType(mapping: any): boolean { - // TODO: Replace with your specific index type logic - return true; // Your placeholder - } - public onFocus = (focusedPanelIndex: string) => { this.setState({ focusedPanelIndex }); }; @@ -321,16 +266,34 @@ export class EmbeddablePanel extends React.Component { /> )} - {this.state.isSpecificType && ( - - )}
); } + public componentDidMount() { + if (this.embeddableRoot.current) { + this.subscription.add( + this.props.embeddable.getOutput$().subscribe( + (output: EmbeddableOutput) => { + this.setState({ + error: output.error, + loading: output.loading, + }); + }, + (error) => { + if (this.embeddableRoot.current) { + const errorEmbeddable = new ErrorEmbeddable(error, { id: this.props.embeddable.id }); + errorEmbeddable.render(this.embeddableRoot.current); + this.setState({ errorEmbeddable }); + } + } + ) + ); + this.props.embeddable.render(this.embeddableRoot.current); + } + } + closeMyContextMenuPanel = () => { if (this.mounted) { this.setState({ closeContextMenu: true }, () => { From fc874c6face3e96f9bceb847be7fabbeab2f9f89 Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Fri, 4 Apr 2025 03:23:57 -0700 Subject: [PATCH 05/86] revert plugin.tsx for the internal embeddable plugin Signed-off-by: Jialiang Liang --- src/plugins/embeddable/public/plugin.tsx | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/plugins/embeddable/public/plugin.tsx b/src/plugins/embeddable/public/plugin.tsx index 9747c764b0e4..4ff348a512fc 100644 --- a/src/plugins/embeddable/public/plugin.tsx +++ b/src/plugins/embeddable/public/plugin.tsx @@ -200,8 +200,6 @@ export class EmbeddablePublicPlugin implements Plugin ); From 2b8230b8a3790fc6259fef0fb2eb1b83da10958c Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Mon, 7 Apr 2025 15:19:35 -0700 Subject: [PATCH 06/86] poc phase 0 - setup the collector function for collecting panel meta-data within dashboards Signed-off-by: Jialiang Liang --- .../embeddable/dashboard_container.tsx | 2 + .../embeddable/grid/dashboard_grid.tsx | 50 ++++++++++++++++++- .../viewport/dashboard_viewport.tsx | 9 +++- src/plugins/dashboard/public/plugin.tsx | 1 + 4 files changed, 58 insertions(+), 4 deletions(-) diff --git a/src/plugins/dashboard/public/application/embeddable/dashboard_container.tsx b/src/plugins/dashboard/public/application/embeddable/dashboard_container.tsx index d994e98142db..8bb9d43c97b3 100644 --- a/src/plugins/dashboard/public/application/embeddable/dashboard_container.tsx +++ b/src/plugins/dashboard/public/application/embeddable/dashboard_container.tsx @@ -104,6 +104,7 @@ export interface DashboardContainerOptions { SavedObjectFinder: React.ComponentType; ExitFullScreenButton: React.ComponentType; uiActions: UiActionsStart; + savedObjectsClient: CoreStart['savedObjects']['client']; } export type DashboardReactContextValue = OpenSearchDashboardsReactContextValue< @@ -244,6 +245,7 @@ export class DashboardContainer extends Container , diff --git a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx index 374e20e715d4..50b89fbd3426 100644 --- a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx +++ b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx @@ -39,6 +39,7 @@ import _ from 'lodash'; import React from 'react'; import { Subscription } from 'rxjs'; import ReactGridLayout, { Layout, ReactGridLayoutProps } from 'react-grid-layout'; +import type { SavedObjectsClientContract } from 'src/core/public'; import { ViewMode, EmbeddableChildPanel, EmbeddableStart } from '../../../../../embeddable/public'; import { GridData } from '../../../../common'; import { DASHBOARD_GRID_COLUMN_COUNT, DASHBOARD_GRID_HEIGHT } from '../dashboard_constants'; @@ -127,6 +128,7 @@ export interface DashboardGridProps extends ReactIntl.InjectedIntlProps { opensearchDashboards: DashboardReactContextValue; PanelComponent: EmbeddableStart['EmbeddablePanel']; container: DashboardContainer; + savedObjectsClient: SavedObjectsClientContract; } interface State { @@ -137,6 +139,7 @@ interface State { viewMode: ViewMode; useMargins: boolean; expandedPanelId?: string; + panelMetadata: Array<{ panelId: string; savedObjectId: string; type: string }>; } interface PanelLayout extends Layout { @@ -161,6 +164,7 @@ class DashboardGridUi extends React.Component { viewMode: this.props.container.getInput().viewMode, useMargins: this.props.container.getInput().useMargins, expandedPanelId: this.props.container.getInput().expandedPanelId, + panelMetadata: [], }; } @@ -171,8 +175,7 @@ class DashboardGridUi extends React.Component { try { layout = this.buildLayoutFromPanels(); } catch (error: any) { - console.error(error); // eslint-disable-line no-console - + console.error(error); isLayoutInvalid = true; this.props.opensearchDashboards.notifications.toasts.danger({ title: this.props.intl.formatMessage({ @@ -198,8 +201,11 @@ class DashboardGridUi extends React.Component { useMargins: input.useMargins, expandedPanelId: input.expandedPanelId, }); + this.collectAllPanelMetadata(); } }); + + this.collectAllPanelMetadata(); } public componentWillUnmount() { @@ -246,6 +252,46 @@ class DashboardGridUi extends React.Component { } }; + /** + * Collects metadata (panelId, savedObjectId, type) for all panels in the dashboard. + * Runs on mount and when the container input (panels) changes. + */ + private async collectAllPanelMetadata() { + const panels = this.state.panels; + + const panelDataPromises = Object.keys(panels).map(async (panelId) => { + const panel = panels[panelId]; + const panelEmbeddable = await this.props.container.untilEmbeddableLoaded(panelId); + const embeddableInput = panelEmbeddable.getInput() as any; + console.log(`Embeddable input for panel ${panelId}:`, embeddableInput); + const savedObjectId = embeddableInput.savedObjectId || 'unknown'; + + if (!savedObjectId || savedObjectId === 'unknown') { + console.log(`No valid savedObjectId for panel ${panelId}`); + return { panelId, savedObjectId: 'unknown', type: 'unknown' }; + } + + try { + const savedObject = await this.props.savedObjectsClient.get(panel.type, savedObjectId); + + console.log(`Saved object for ${savedObjectId}:`, savedObject); + const visState = savedObject.attributes.visState + ? JSON.parse(savedObject.attributes.visState) + : {}; + const visType = visState.type || savedObject.attributes.type || 'unknown'; + return { panelId, savedObjectId, type: visType }; + } catch (error) { + console.error(`Error fetching saved object for ${savedObjectId}:`, error); + return { panelId, savedObjectId, type: 'unknown' }; + } + }); + + const panelMetadata = await Promise.all(panelDataPromises); + this.setState({ panelMetadata }, () => { + console.log('All Panel Metadata:', this.state.panelMetadata); + }); + } + public renderPanels() { const { focusedPanelIndex, panels, expandedPanelId } = this.state; diff --git a/src/plugins/dashboard/public/application/embeddable/viewport/dashboard_viewport.tsx b/src/plugins/dashboard/public/application/embeddable/viewport/dashboard_viewport.tsx index 60a3dba7384b..ff75a9278fa8 100644 --- a/src/plugins/dashboard/public/application/embeddable/viewport/dashboard_viewport.tsx +++ b/src/plugins/dashboard/public/application/embeddable/viewport/dashboard_viewport.tsx @@ -30,7 +30,7 @@ import React from 'react'; import { Subscription } from 'rxjs'; -import { Logos } from 'opensearch-dashboards/public'; +import { Logos, SavedObjectsClientContract } from 'opensearch-dashboards/public'; import { PanelState, EmbeddableStart } from '../../../../../embeddable/public'; import { DashboardContainer, DashboardReactContextValue } from '../dashboard_container'; import { DashboardGrid } from '../grid'; @@ -41,6 +41,7 @@ export interface DashboardViewportProps { PanelComponent: EmbeddableStart['EmbeddablePanel']; renderEmpty?: () => React.ReactNode; logos: Logos; + savedObjectsClient: SavedObjectsClientContract; } interface State { @@ -159,7 +160,11 @@ export class DashboardViewport extends React.Component )} - +
); } diff --git a/src/plugins/dashboard/public/plugin.tsx b/src/plugins/dashboard/public/plugin.tsx index 55ed9a02d24d..8e28c987e390 100644 --- a/src/plugins/dashboard/public/plugin.tsx +++ b/src/plugins/dashboard/public/plugin.tsx @@ -276,6 +276,7 @@ export class DashboardPlugin SavedObjectFinder: getSavedObjectFinder(coreStart.savedObjects, coreStart.uiSettings), ExitFullScreenButton, uiActions: deps.uiActions, + savedObjectsClient: coreStart.savedObjects.client, // HERE TO ADD SAVED OBJECTS CLIENT }; }; From fcf4a8b6edb80a23d9e77263cc195caa018f0280 Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Mon, 7 Apr 2025 16:22:14 -0700 Subject: [PATCH 07/86] poc phase 1 - implement hte fetch for getting the index pattern id of first pie chart Signed-off-by: Jialiang Liang --- .../embeddable/grid/dashboard_grid.tsx | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx index 50b89fbd3426..39f2b6e34c23 100644 --- a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx +++ b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx @@ -287,8 +287,30 @@ class DashboardGridUi extends React.Component { }); const panelMetadata = await Promise.all(panelDataPromises); - this.setState({ panelMetadata }, () => { + this.setState({ panelMetadata }, async () => { console.log('All Panel Metadata:', this.state.panelMetadata); + + // POC ONLY: Extract first pie chart savedObjectId + const firstPie = this.state.panelMetadata.find((meta) => meta.type === 'pie'); + if (firstPie) { + console.log('First pie visualization savedObjectId:', firstPie.savedObjectId); + + try { + // POC ONLY: Fetch the full saved object for the pie visualization + const pieSavedObject = await this.props.savedObjectsClient.get( + 'visualization', + firstPie.savedObjectId + ); + console.log('First pie visualization saved object metadata:', pieSavedObject); + } catch (error) { + console.error( + `Error fetching metadata for pie saved object ID ${firstPie.savedObjectId}:`, + error + ); + } + } else { + console.log('No pie visualizations found.'); + } }); } From b70e6f0a82a22849c5cd58508b4f82c7e9209320 Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Mon, 7 Apr 2025 17:22:29 -0700 Subject: [PATCH 08/86] poc phase 2 - get the name of mv index Signed-off-by: Jialiang Liang --- .../embeddable/grid/dashboard_grid.tsx | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx index 39f2b6e34c23..fc2993c0f324 100644 --- a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx +++ b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx @@ -302,6 +302,25 @@ class DashboardGridUi extends React.Component { firstPie.savedObjectId ); console.log('First pie visualization saved object metadata:', pieSavedObject); + + const indexPatternRef = pieSavedObject.references.find( + (ref: any) => ref.type === 'index-pattern' + ); + + if (indexPatternRef) { + try { + const indexPattern = await this.props.savedObjectsClient.get( + 'index-pattern', + indexPatternRef.id + ); + const indexTitle = indexPattern.attributes.title; + console.log('Index pattern title (index name):', indexTitle); + } catch (err) { + console.error(`Failed to fetch index pattern ${indexPatternRef.id}:`, err); + } + } else { + console.warn('No index-pattern reference found in the pie visualization saved object.'); + } } catch (error) { console.error( `Error fetching metadata for pie saved object ID ${firstPie.savedObjectId}:`, From 9ecfa9b6cb77e700381ba66e32c8776ca933bb66 Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Tue, 8 Apr 2025 11:14:39 -0700 Subject: [PATCH 09/86] poc phase 2 - get the datasource/database/given index name info Signed-off-by: Jialiang Liang --- .../application/embeddable/grid/dashboard_grid.tsx | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx index fc2993c0f324..3219f97d43dd 100644 --- a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx +++ b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx @@ -315,6 +315,18 @@ class DashboardGridUi extends React.Component { ); const indexTitle = indexPattern.attributes.title; console.log('Index pattern title (index name):', indexTitle); + // Extract datasource, database, and index name from index title + const trimmedTitle = indexTitle.replace(/^flint_/, ''); + const parts = trimmedTitle.split('_'); + + const datasource = parts[0] || 'unknown'; + const database = parts[1] || 'unknown'; + const index = parts.slice(2).join('_') || 'unknown'; + + console.log('Extracted Info:'); + console.log('Datasource:', datasource); + console.log('Database:', database); + console.log('Index:', index); } catch (err) { console.error(`Failed to fetch index pattern ${indexPatternRef.id}:`, err); } From 083ad414c47a5a544ac651419d7f378b2ea9be39 Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Tue, 8 Apr 2025 15:02:28 -0700 Subject: [PATCH 10/86] poc phase 3 - implement a dummy refresh button Signed-off-by: Jialiang Liang --- .../embeddable/grid/dashboard_grid.tsx | 34 ++++++++++++++----- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx index 3219f97d43dd..67a506717cdc 100644 --- a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx +++ b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx @@ -40,6 +40,7 @@ import React from 'react'; import { Subscription } from 'rxjs'; import ReactGridLayout, { Layout, ReactGridLayoutProps } from 'react-grid-layout'; import type { SavedObjectsClientContract } from 'src/core/public'; +import { EuiButton } from '@elastic/eui'; import { ViewMode, EmbeddableChildPanel, EmbeddableStart } from '../../../../../embeddable/public'; import { GridData } from '../../../../common'; import { DASHBOARD_GRID_COLUMN_COUNT, DASHBOARD_GRID_HEIGHT } from '../dashboard_constants'; @@ -345,6 +346,11 @@ class DashboardGridUi extends React.Component { }); } + private synchronizeNow = () => { + console.log('Synchronize Now clicked!'); + // Add any sync logic here + }; + public renderPanels() { const { focusedPanelIndex, panels, expandedPanelId } = this.state; @@ -399,15 +405,25 @@ class DashboardGridUi extends React.Component { const { viewMode } = this.state; const isViewMode = viewMode === ViewMode.VIEW; return ( - - {this.renderPanels()} - +
+ {/* ✅ Top-left corner "Synchronize Now" button */} +
+ + Synchronize Now + +
+ + {/* ✅ The grid itself */} + + {this.renderPanels()} + +
); } } From 6d912c3fea68b7c4977df933742869d84eafb53e Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Tue, 8 Apr 2025 20:44:06 -0700 Subject: [PATCH 11/86] poc phase 4 - introduce http into the grid class but no server setup yet Signed-off-by: Jialiang Liang --- .../dashboard/opensearch_dashboards.json | 5 +++-- .../embeddable/dashboard_container.tsx | 2 ++ .../embeddable/grid/dashboard_grid.tsx | 17 ++++++++++++++--- .../embeddable/viewport/dashboard_viewport.tsx | 4 +++- src/plugins/dashboard/public/plugin.tsx | 1 + 5 files changed, 23 insertions(+), 6 deletions(-) diff --git a/src/plugins/dashboard/opensearch_dashboards.json b/src/plugins/dashboard/opensearch_dashboards.json index 348a0c9fe9dc..79a9eb3b297a 100644 --- a/src/plugins/dashboard/opensearch_dashboards.json +++ b/src/plugins/dashboard/opensearch_dashboards.json @@ -9,9 +9,10 @@ "urlForwarding", "navigation", "uiActions", - "savedObjects" + "savedObjects", + "dataSourceManagement" ], - "optionalPlugins": ["home", "share", "usageCollection"], + "optionalPlugins": ["home", "share", "usageCollection","dataSouce"], "server": true, "ui": true, "requiredBundles": ["opensearchDashboardsUtils", "opensearchDashboardsReact", "home"] diff --git a/src/plugins/dashboard/public/application/embeddable/dashboard_container.tsx b/src/plugins/dashboard/public/application/embeddable/dashboard_container.tsx index 8bb9d43c97b3..fd822291caa0 100644 --- a/src/plugins/dashboard/public/application/embeddable/dashboard_container.tsx +++ b/src/plugins/dashboard/public/application/embeddable/dashboard_container.tsx @@ -105,6 +105,7 @@ export interface DashboardContainerOptions { ExitFullScreenButton: React.ComponentType; uiActions: UiActionsStart; savedObjectsClient: CoreStart['savedObjects']['client']; + http: CoreStart['http']; } export type DashboardReactContextValue = OpenSearchDashboardsReactContextValue< @@ -246,6 +247,7 @@ export class DashboardContainer extends Container , diff --git a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx index 67a506717cdc..7977f9c8b625 100644 --- a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx +++ b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx @@ -40,6 +40,7 @@ import React from 'react'; import { Subscription } from 'rxjs'; import ReactGridLayout, { Layout, ReactGridLayoutProps } from 'react-grid-layout'; import type { SavedObjectsClientContract } from 'src/core/public'; +import { HttpStart } from 'src/core/public'; import { EuiButton } from '@elastic/eui'; import { ViewMode, EmbeddableChildPanel, EmbeddableStart } from '../../../../../embeddable/public'; import { GridData } from '../../../../common'; @@ -130,6 +131,7 @@ export interface DashboardGridProps extends ReactIntl.InjectedIntlProps { PanelComponent: EmbeddableStart['EmbeddablePanel']; container: DashboardContainer; savedObjectsClient: SavedObjectsClientContract; + http: HttpStart; } interface State { @@ -346,9 +348,18 @@ class DashboardGridUi extends React.Component { }); } - private synchronizeNow = () => { - console.log('Synchronize Now clicked!'); - // Add any sync logic here + synchronizeNow = async () => { + try { + console.log('http:', this.props.http); + const response = await this.props.http.get(`api/observability/dsl/indices.getFieldMapping`, { + query: { + index: 'flint_flinttest1_default_vpc_mv_1106', + }, + }); + console.log('Index Mapping:', response); + } catch (error) { + console.error('Error fetching index mapping:', error); + } }; public renderPanels() { diff --git a/src/plugins/dashboard/public/application/embeddable/viewport/dashboard_viewport.tsx b/src/plugins/dashboard/public/application/embeddable/viewport/dashboard_viewport.tsx index ff75a9278fa8..faaacb6bd37e 100644 --- a/src/plugins/dashboard/public/application/embeddable/viewport/dashboard_viewport.tsx +++ b/src/plugins/dashboard/public/application/embeddable/viewport/dashboard_viewport.tsx @@ -30,7 +30,7 @@ import React from 'react'; import { Subscription } from 'rxjs'; -import { Logos, SavedObjectsClientContract } from 'opensearch-dashboards/public'; +import { Logos, SavedObjectsClientContract, HttpStart } from 'opensearch-dashboards/public'; import { PanelState, EmbeddableStart } from '../../../../../embeddable/public'; import { DashboardContainer, DashboardReactContextValue } from '../dashboard_container'; import { DashboardGrid } from '../grid'; @@ -42,6 +42,7 @@ export interface DashboardViewportProps { renderEmpty?: () => React.ReactNode; logos: Logos; savedObjectsClient: SavedObjectsClientContract; + http: HttpStart; } interface State { @@ -164,6 +165,7 @@ export class DashboardViewport extends React.Component
); diff --git a/src/plugins/dashboard/public/plugin.tsx b/src/plugins/dashboard/public/plugin.tsx index 8e28c987e390..1b6c213a3cb7 100644 --- a/src/plugins/dashboard/public/plugin.tsx +++ b/src/plugins/dashboard/public/plugin.tsx @@ -277,6 +277,7 @@ export class DashboardPlugin ExitFullScreenButton, uiActions: deps.uiActions, savedObjectsClient: coreStart.savedObjects.client, // HERE TO ADD SAVED OBJECTS CLIENT + http: coreStart.http, // HERE TO ADD HTTP }; }; From 26e40d390f71c85ff90683d5601347a93c64f189 Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Tue, 8 Apr 2025 21:28:46 -0700 Subject: [PATCH 12/86] poc phase 5 - FIX THE BUTTON FOR TESTING INDEX MAPPING CALL AND SET UP SERVER (UNNECESSARY) Signed-off-by: Jialiang Liang --- .../dashboard/framework/utils/shared.ts | 166 +++++++++ .../embeddable/grid/dashboard_grid.tsx | 2 +- ...pensearch_data_source_management_plugin.ts | 50 +++ .../server/adaptors/ppl_datasource.ts | 84 +++++ .../dashboard/server/adaptors/ppl_plugin.ts | 93 +++++ .../dashboard/server/common/types/index.ts | 23 ++ src/plugins/dashboard/server/plugin.ts | 11 +- .../server/routes/data_connections_router.ts | 344 ++++++++++++++++++ .../server/routes/datasources_router.ts | 122 +++++++ src/plugins/dashboard/server/routes/dsl.ts | 241 ++++++++++++ src/plugins/dashboard/server/routes/index.ts | 54 +++ src/plugins/dashboard/server/routes/ppl.ts | 38 ++ .../server/services/facets/dsl_facet.ts | 37 ++ .../server/services/facets/ppl_facet.ts | 43 +++ 14 files changed, 1305 insertions(+), 3 deletions(-) create mode 100644 src/plugins/dashboard/framework/utils/shared.ts create mode 100644 src/plugins/dashboard/server/adaptors/opensearch_data_source_management_plugin.ts create mode 100644 src/plugins/dashboard/server/adaptors/ppl_datasource.ts create mode 100644 src/plugins/dashboard/server/adaptors/ppl_plugin.ts create mode 100644 src/plugins/dashboard/server/common/types/index.ts create mode 100644 src/plugins/dashboard/server/routes/data_connections_router.ts create mode 100644 src/plugins/dashboard/server/routes/datasources_router.ts create mode 100644 src/plugins/dashboard/server/routes/dsl.ts create mode 100644 src/plugins/dashboard/server/routes/index.ts create mode 100644 src/plugins/dashboard/server/routes/ppl.ts create mode 100644 src/plugins/dashboard/server/services/facets/dsl_facet.ts create mode 100644 src/plugins/dashboard/server/services/facets/ppl_facet.ts diff --git a/src/plugins/dashboard/framework/utils/shared.ts b/src/plugins/dashboard/framework/utils/shared.ts new file mode 100644 index 000000000000..49f5f0d85390 --- /dev/null +++ b/src/plugins/dashboard/framework/utils/shared.ts @@ -0,0 +1,166 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export function get(obj: Record, path: string, defaultValue?: T): T { + return path.split('.').reduce((acc: any, part: string) => acc && acc[part], obj) || defaultValue; +} + +export function addBackticksIfNeeded(input: string): string { + if (input === undefined) { + return ''; + } + // Check if the string already has backticks + if (input.startsWith('`') && input.endsWith('`')) { + return input; // Return the string as it is + } + // Add backticks to the string + return '`' + input + '`'; +} + +export function combineSchemaAndDatarows( + schema: Array<{ name: string; type: string }>, + datarows: Array> +): object[] { + const combinedData: object[] = []; + + datarows.forEach((row) => { + const rowData: { [key: string]: string | number | boolean } = {}; + schema.forEach((field, index) => { + rowData[field.name] = row[index]; + }); + combinedData.push(rowData); + }); + + return combinedData; +} + +export const formatError = (name: string, message: string, details: string) => { + return { + name, + message, + body: { + attributes: { + error: { + caused_by: { + type: '', + reason: details, + }, + }, + }, + }, + }; +}; + +// Client route +export const DIRECT_QUERY_BASE = '/api/dashboard'; +export const PPL_BASE = `${DIRECT_QUERY_BASE}/ppl`; +export const PPL_SEARCH = '/search'; +export const DSL_BASE = `${DIRECT_QUERY_BASE}/dsl`; +export const DSL_SEARCH = '/search'; +export const DSL_CAT = '/cat.indices'; +export const DSL_MAPPING = '/indices.getFieldMapping'; +export const DSL_SETTINGS = '/indices.getFieldSettings'; +export const DSM_BASE = '/api/dashboard'; +export const INTEGRATIONS_BASE = '/api/integrations'; +export const JOBS_BASE = '/query/jobs'; +export const DATACONNECTIONS_BASE = `${DIRECT_QUERY_BASE}/dataconnections`; +export const EDIT = '/edit'; +export const DATACONNECTIONS_UPDATE_STATUS = '/status'; +export const SECURITY_ROLES = '/api/v1/configuration/roles'; +export const EVENT_ANALYTICS = '/event_analytics'; +export const SAVED_OBJECTS = '/saved_objects'; +export const SAVED_QUERY = '/query'; +export const SAVED_VISUALIZATION = '/vis'; +export const CONSOLE_PROXY = '/api/console/proxy'; +export const SECURITY_PLUGIN_ACCOUNT_API = '/api/v1/configuration/account'; + +// Server route +export const PPL_ENDPOINT = '/_plugins/_ppl'; +export const SQL_ENDPOINT = '/_plugins/_sql'; +export const DSL_ENDPOINT = '/_plugins/_dsl'; +export const DATACONNECTIONS_ENDPOINT = '/_plugins/_query/_datasources'; +export const JOBS_ENDPOINT_BASE = '/_plugins/_async_query'; +export const JOB_RESULT_ENDPOINT = '/result'; + +export const observabilityID = 'observability-logs'; +export const observabilityTitle = 'Observability'; +export const observabilityPluginOrder = 1500; + +export const observabilityApplicationsID = 'observability-applications'; +export const observabilityApplicationsTitle = 'Applications'; +export const observabilityApplicationsPluginOrder = 5090; + +export const observabilityLogsID = 'observability-logs'; +export const observabilityLogsTitle = 'Logs'; +export const observabilityLogsPluginOrder = 5091; + +export const observabilityMetricsID = 'observability-metrics'; +export const observabilityMetricsTitle = 'Metrics'; +export const observabilityMetricsPluginOrder = 5092; + +export const observabilityTracesID = 'observability-traces'; +export const observabilityTracesTitle = 'Traces'; +export const observabilityTracesPluginOrder = 5093; + +export const observabilityNotebookID = 'observability-notebooks'; +export const observabilityNotebookTitle = 'Notebooks'; +export const observabilityNotebookPluginOrder = 5094; + +export const observabilityPanelsID = 'observability-dashboards'; +export const observabilityPanelsTitle = 'Dashboards'; +export const observabilityPanelsPluginOrder = 5095; + +export const observabilityIntegrationsID = 'integrations'; +export const observabilityIntegrationsTitle = 'Integrations'; +export const observabilityIntegrationsPluginOrder = 9020; + +export const observabilityDataConnectionsID = 'datasources'; +export const observabilityDataConnectionsTitle = 'Data sources'; +export const observabilityDataConnectionsPluginOrder = 9030; + +export const queryWorkbenchPluginID = 'opensearch-query-workbench'; +export const queryWorkbenchPluginCheck = 'plugin:queryWorkbenchDashboards'; + +// Observability plugin URI +const BASE_OBSERVABILITY_URI = '/_plugins/_observability'; +const BASE_DATACONNECTIONS_URI = '/_plugins/_query/_datasources'; +export const OPENSEARCH_PANELS_API = { + OBJECT: `${BASE_OBSERVABILITY_URI}/object`, +}; +export const OPENSEARCH_DATACONNECTIONS_API = { + DATACONNECTION: `${BASE_DATACONNECTIONS_URI}`, +}; + +// Saved Objects +export const SAVED_OBJECT = '/object'; + +export const S3_DATA_SOURCE_TYPE = 's3glue'; + +export const ASYNC_QUERY_SESSION_ID = 'async-query-session-id'; +export const ASYNC_QUERY_DATASOURCE_CACHE = 'async-query-catalog-cache'; +export const ASYNC_QUERY_ACCELERATIONS_CACHE = 'async-query-acclerations-cache'; + +export const DIRECT_DUMMY_QUERY = 'select 1'; + +export enum DirectQueryLoadingStatus { + SUCCESS = 'success', + FAILED = 'failed', + RUNNING = 'running', + SCHEDULED = 'scheduled', + CANCELED = 'canceled', + WAITING = 'waiting', + INITIAL = 'initial', +} +const catalogCacheFetchingStatus = [ + DirectQueryLoadingStatus.RUNNING, + DirectQueryLoadingStatus.WAITING, + DirectQueryLoadingStatus.SCHEDULED, +]; + +export const isCatalogCacheFetching = (...statuses: DirectQueryLoadingStatus[]) => { + return statuses.some((status: DirectQueryLoadingStatus) => + catalogCacheFetchingStatus.includes(status) + ); +}; diff --git a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx index 7977f9c8b625..58080a678ef9 100644 --- a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx +++ b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx @@ -351,7 +351,7 @@ class DashboardGridUi extends React.Component { synchronizeNow = async () => { try { console.log('http:', this.props.http); - const response = await this.props.http.get(`api/observability/dsl/indices.getFieldMapping`, { + const response = await this.props.http.get(`/api/directquery/dsl/indices.getFieldMapping`, { query: { index: 'flint_flinttest1_default_vpc_mv_1106', }, diff --git a/src/plugins/dashboard/server/adaptors/opensearch_data_source_management_plugin.ts b/src/plugins/dashboard/server/adaptors/opensearch_data_source_management_plugin.ts new file mode 100644 index 000000000000..4d694a296d5a --- /dev/null +++ b/src/plugins/dashboard/server/adaptors/opensearch_data_source_management_plugin.ts @@ -0,0 +1,50 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { JOBS_ENDPOINT_BASE } from '../../framework/utils/shared'; + +export function OpenSearchDataSourceManagementPlugin(Client: any, config: any, components: any) { + const clientAction = components.clientAction.factory; + + Client.prototype.datasourcemanagement = components.clientAction.namespaceFactory(); + const datasourcemanagement = Client.prototype.datasourcemanagement.prototype; + + // Get async job status + datasourcemanagement.getJobStatus = clientAction({ + url: { + fmt: `${JOBS_ENDPOINT_BASE}/<%=queryId%>`, + req: { + queryId: { + type: 'string', + required: true, + }, + }, + }, + method: 'GET', + }); + + // Delete async job + datasourcemanagement.deleteJob = clientAction({ + url: { + fmt: `${JOBS_ENDPOINT_BASE}/<%=queryId%>`, + req: { + queryId: { + type: 'string', + required: true, + }, + }, + }, + method: 'DELETE', + }); + + // Run async job + datasourcemanagement.runDirectQuery = clientAction({ + url: { + fmt: `${JOBS_ENDPOINT_BASE}`, + }, + method: 'POST', + needBody: true, + }); +} diff --git a/src/plugins/dashboard/server/adaptors/ppl_datasource.ts b/src/plugins/dashboard/server/adaptors/ppl_datasource.ts new file mode 100644 index 000000000000..38cacbbe0ac7 --- /dev/null +++ b/src/plugins/dashboard/server/adaptors/ppl_datasource.ts @@ -0,0 +1,84 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import _ from 'lodash'; +import { IPPLEventsDataSource, IPPLVisualizationDataSource } from '../common/types'; + +type PPLResponse = IPPLEventsDataSource & IPPLVisualizationDataSource; + +export class PPLDataSource { + constructor(private pplDataSource: PPLResponse, private dataType: string) { + if (this.dataType === 'jdbc') { + this.addSchemaRowMapping(); + } else if (this.dataType === 'viz') { + this.addStatsMapping(); + } + } + + private addStatsMapping = () => { + const visData = this.pplDataSource; + + /** + * Add vis mapping for runtime fields + * json data structure added to response will be + * [{ + * agent: "mozilla", + * avg(bytes): 5756 + * ... + * }, { + * agent: "MSIE", + * avg(bytes): 5605 + * ... + * }, { + * agent: "chrome", + * avg(bytes): 5648 + * ... + * }] + */ + const res = []; + if (visData?.metadata?.fields) { + const queriedFields = visData.metadata.fields; + for (let i = 0; i < visData.size; i++) { + const entry: any = {}; + queriedFields.map((field: any) => { + const statsDataSet = visData?.data; + entry[field.name] = statsDataSet[field.name][i]; + }); + res.push(entry); + } + visData.jsonData = res; + } + }; + + /** + * Add 'schemaName: data' entries for UI rendering + */ + private addSchemaRowMapping = () => { + const pplRes = this.pplDataSource; + + const data: any[] = []; + + _.forEach(pplRes.datarows, (row) => { + const record: any = {}; + + for (let i = 0; i < pplRes.schema.length; i++) { + const cur = pplRes.schema[i]; + + if (typeof row[i] === 'object') { + record[cur.name] = JSON.stringify(row[i]); + } else if (typeof row[i] === 'boolean') { + record[cur.name] = row[i].toString(); + } else { + record[cur.name] = row[i]; + } + } + + data.push(record); + }); + pplRes.jsonData = data; + }; + + public getDataSource = (): PPLResponse => this.pplDataSource; +} diff --git a/src/plugins/dashboard/server/adaptors/ppl_plugin.ts b/src/plugins/dashboard/server/adaptors/ppl_plugin.ts new file mode 100644 index 000000000000..7ef02f89d2ba --- /dev/null +++ b/src/plugins/dashboard/server/adaptors/ppl_plugin.ts @@ -0,0 +1,93 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { + OPENSEARCH_DATACONNECTIONS_API, + PPL_ENDPOINT, + SQL_ENDPOINT, +} from '../../framework/utils/shared'; + +export const PPLPlugin = function (Client, config, components) { + const ca = components.clientAction.factory; + Client.prototype.ppl = components.clientAction.namespaceFactory(); + const ppl = Client.prototype.ppl.prototype; + + ppl.pplQuery = ca({ + url: { + fmt: `${PPL_ENDPOINT}`, + params: { + format: { + type: 'string', + required: true, + }, + }, + }, + needBody: true, + method: 'POST', + }); + + ppl.sqlQuery = ca({ + url: { + fmt: `${SQL_ENDPOINT}`, + params: { + format: { + type: 'string', + required: true, + }, + }, + }, + needBody: true, + method: 'POST', + }); + + ppl.getDataConnectionById = ca({ + url: { + fmt: `${OPENSEARCH_DATACONNECTIONS_API.DATACONNECTION}/<%=dataconnection%>`, + req: { + dataconnection: { + type: 'string', + required: true, + }, + }, + }, + method: 'GET', + }); + + ppl.deleteDataConnection = ca({ + url: { + fmt: `${OPENSEARCH_DATACONNECTIONS_API.DATACONNECTION}/<%=dataconnection%>`, + req: { + dataconnection: { + type: 'string', + required: true, + }, + }, + }, + method: 'DELETE', + }); + + ppl.createDataSource = ca({ + url: { + fmt: `${OPENSEARCH_DATACONNECTIONS_API.DATACONNECTION}`, + }, + needBody: true, + method: 'POST', + }); + + ppl.modifyDataConnection = ca({ + url: { + fmt: `${OPENSEARCH_DATACONNECTIONS_API.DATACONNECTION}`, + }, + needBody: true, + method: 'PATCH', + }); + + ppl.getDataConnections = ca({ + url: { + fmt: `${OPENSEARCH_DATACONNECTIONS_API.DATACONNECTION}`, + }, + method: 'GET', + }); +}; diff --git a/src/plugins/dashboard/server/common/types/index.ts b/src/plugins/dashboard/server/common/types/index.ts new file mode 100644 index 000000000000..3d6525d8f5f1 --- /dev/null +++ b/src/plugins/dashboard/server/common/types/index.ts @@ -0,0 +1,23 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export interface ISchema { + name: string; + type: string; +} + +export interface IPPLVisualizationDataSource { + data: any; + metadata: any; + jsonData?: any[]; + size: number; + status: number; +} + +export interface IPPLEventsDataSource { + schema: ISchema[]; + datarows: any[]; + jsonData?: any[]; +} diff --git a/src/plugins/dashboard/server/plugin.ts b/src/plugins/dashboard/server/plugin.ts index 4e377e24bbce..087c528d9342 100644 --- a/src/plugins/dashboard/server/plugin.ts +++ b/src/plugins/dashboard/server/plugin.ts @@ -34,12 +34,13 @@ import { CoreStart, Plugin, Logger, + ILegacyClusterClient, } from '../../../core/server'; import { dashboardSavedObjectType } from './saved_objects'; import { capabilitiesProvider } from './capabilities_provider'; - import { DashboardPluginSetup, DashboardPluginStart } from './types'; +import { setupRoutes } from './routes'; // <-- We'll define this next export class DashboardPlugin implements Plugin { private readonly logger: Logger; @@ -48,7 +49,7 @@ export class DashboardPlugin implements Plugin => { + try { + const dataConnectionsresponse = await context.opensearch_dashboard_plugin.dashboardPluginClient + .asScoped(request) + .callAsCurrentUser('ppl.getDataConnectionById', { + dataconnection: request.params.name, + }); + return response.ok({ + body: dataConnectionsresponse, + }); + } catch (error: any) { + console.error('Issue in fetching data connection:', error); + return response.custom({ + statusCode: error.statusCode || 500, + body: error.message, + }); + } + } + ); + + router.delete( + { + path: `${DATACONNECTIONS_BASE}/{name}`, + validate: { + params: schema.object({ + name: schema.string(), + }), + }, + }, + async (context, request, response): Promise => { + try { + const dataConnectionsresponse = await context.opensearch_dashboard_plugin.dashboardPluginClient + .asScoped(request) + .callAsCurrentUser('ppl.deleteDataConnection', { + dataconnection: request.params.name, + }); + return response.ok({ + body: dataConnectionsresponse, + }); + } catch (error: any) { + console.error('Issue in deleting data connection:', error); + return response.custom({ + statusCode: error.statusCode || 500, + body: error.message, + }); + } + } + ); + + router.post( + { + path: `${DATACONNECTIONS_BASE}${EDIT}`, + validate: { + body: schema.object({ + name: schema.string(), + allowedRoles: schema.arrayOf(schema.string()), + }), + }, + }, + async (context, request, response): Promise => { + try { + const dataConnectionsresponse = await context.opensearch_dashboard_plugin.dashboardPluginClient + .asScoped(request) + .callAsCurrentUser('ppl.modifyDataConnection', { + body: { + name: request.body.name, + allowedRoles: request.body.allowedRoles, + }, + }); + return response.ok({ + body: dataConnectionsresponse, + }); + } catch (error: any) { + console.error('Issue in modifying data connection:', error); + return response.custom({ + statusCode: error.statusCode || 500, + body: error.message, + }); + } + } + ); + + router.post( + { + path: `${DATACONNECTIONS_BASE}${EDIT}${DATACONNECTIONS_UPDATE_STATUS}`, + validate: { + body: schema.object({ + name: schema.string(), + status: schema.string(), + }), + }, + }, + async (context, request, response): Promise => { + try { + const dataConnectionsresponse = await context.opensearch_dashboard_plugin.dashboardPluginClient + .asScoped(request) + .callAsCurrentUser('ppl.modifyDataConnection', { + body: { + name: request.body.name, + status: request.body.status, + }, + }); + return response.ok({ + body: dataConnectionsresponse, + }); + } catch (error: any) { + console.error('Issue in modifying data connection:', error); + return response.custom({ + statusCode: error.statusCode || 500, + body: error.message, + }); + } + } + ); + + router.post( + { + path: `${DATACONNECTIONS_BASE}`, + validate: { + body: schema.object({ + name: schema.string(), + connector: schema.string(), + allowedRoles: schema.arrayOf(schema.string()), + properties: schema.any(), + }), + }, + }, + async ( + context, + request, + response + ): Promise> => { + try { + const dataConnectionsresponse = await context.opensearch_dashboard_plugin.dashboardPluginClient + .asScoped(request) + .callAsCurrentUser('ppl.createDataSource', { + body: { + name: request.body.name, + connector: request.body.connector, + allowedRoles: request.body.allowedRoles, + properties: request.body.properties, + }, + }); + return response.ok({ + body: dataConnectionsresponse, + }); + } catch (error: any) { + console.error('Issue in creating data source:', error); + return response.custom({ + statusCode: error.statusCode || 500, + body: error.response, + }); + } + } + ); + + router.get( + { + path: `${DATACONNECTIONS_BASE}`, + validate: false, + }, + async (context, request, response): Promise => { + try { + const dataConnectionsresponse = await context.opensearch_dashboard_plugin.dashboardPluginClient + .asScoped(request) + .callAsCurrentUser('ppl.getDataConnections'); + return response.ok({ + body: dataConnectionsresponse, + }); + } catch (error: any) { + console.error('Issue in fetching data sources:', error); + return response.custom({ + statusCode: error.statusCode || 500, + body: error.response, + }); + } + } + ); +} + +export function registerDataConnectionsRoute(router: IRouter, dataSourceEnabled: boolean) { + router.get( + { + path: `${DATACONNECTIONS_BASE}/dataSourceMDSId={dataSourceMDSId?}`, + validate: { + params: schema.object({ + dataSourceMDSId: schema.maybe(schema.string({ defaultValue: '' })), + }), + }, + }, + async (context, request, response): Promise => { + const dataSourceMDSId = request.params.dataSourceMDSId; + try { + let dataConnectionsresponse; + if (dataSourceEnabled && dataSourceMDSId) { + const client = await context.dataSource.opensearch.legacy.getClient(dataSourceMDSId); + dataConnectionsresponse = await client.callAPI('ppl.getDataConnections', { + requestTimeout: 5000, // Enforce timeout to avoid hanging requests + }); + } else { + dataConnectionsresponse = await context.opensearch_dashboard_plugin.dashboardPluginClient + .asScoped(request) + .callAsCurrentUser('ppl.getDataConnections'); + } + return response.ok({ + body: dataConnectionsresponse, + }); + } catch (error: any) { + console.error('Issue in fetching data sources:', error); + const statusCode = error.statusCode || error.body?.statusCode || 500; + const errorBody = error.body || + error.response || { message: error.message || 'Unknown error occurred' }; + + return response.custom({ + statusCode, + body: { + error: errorBody, + message: errorBody.message || error.message, + }, + }); + } + } + ); + + router.get( + { + path: `${DATACONNECTIONS_BASE}/{name}/dataSourceMDSId={dataSourceMDSId?}`, + validate: { + params: schema.object({ + name: schema.string(), + dataSourceMDSId: schema.maybe(schema.string({ defaultValue: '' })), + }), + }, + }, + async (context, request, response): Promise => { + const dataSourceMDSId = request.params.dataSourceMDSId; + try { + let dataConnectionsresponse; + if (dataSourceEnabled && dataSourceMDSId) { + const client = await context.dataSource.opensearch.legacy.getClient(dataSourceMDSId); + dataConnectionsresponse = await client.callAPI('ppl.getDataConnectionById', { + dataconnection: request.params.name, + }); + } else { + dataConnectionsresponse = await context.opensearch_dashboard_plugin.dashboardPluginClient + .asScoped(request) + .callAsCurrentUser('ppl.getDataConnectionById', { + dataconnection: request.params.name, + }); + } + return response.ok({ + body: dataConnectionsresponse, + }); + } catch (error: any) { + console.error('Issue in fetching data sources:', error); + const statusCode = error.statusCode || error.body?.statusCode || 500; + const errorBody = error.body || + error.response || { message: error.message || 'Unknown error occurred' }; + + return response.custom({ + statusCode, + body: { + error: errorBody, + message: errorBody.message || error.message, + }, + }); + } + } + ); + + router.delete( + { + path: `${DATACONNECTIONS_BASE}/{name}/dataSourceMDSId={dataSourceMDSId?}`, + validate: { + params: schema.object({ + name: schema.string(), + dataSourceMDSId: schema.maybe(schema.string({ defaultValue: '' })), + }), + }, + }, + async (context, request, response): Promise => { + const dataSourceMDSId = request.params.dataSourceMDSId; + try { + let dataConnectionsresponse; + if (dataSourceEnabled && dataSourceMDSId) { + const client = await context.dataSource.opensearch.legacy.getClient(dataSourceMDSId); + dataConnectionsresponse = await client.callAPI('ppl.deleteDataConnection', { + dataconnection: request.params.name, + }); + } else { + dataConnectionsresponse = await context.opensearch_dashboard_plugin.dashboardPluginClient + .asScoped(request) + .callAsCurrentUser('ppl.deleteDataConnection', { + dataconnection: request.params.name, + }); + } + return response.ok({ + body: dataConnectionsresponse, + }); + } catch (error: any) { + console.error('Issue in deleting data sources:', error); + const statusCode = error.statusCode || error.body?.statusCode || 500; + const errorBody = error.body || + error.response || { message: error.message || 'Unknown error occurred' }; + + return response.custom({ + statusCode, + body: { + error: errorBody, + message: errorBody.message || error.message, + }, + }); + } + } + ); +} diff --git a/src/plugins/dashboard/server/routes/datasources_router.ts b/src/plugins/dashboard/server/routes/datasources_router.ts new file mode 100644 index 000000000000..be6682d3faa0 --- /dev/null +++ b/src/plugins/dashboard/server/routes/datasources_router.ts @@ -0,0 +1,122 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +/* eslint-disable no-console*/ +import { schema } from '@osd/config-schema'; +import { IRouter } from '../../../../../src/core/server'; +import { JOBS_BASE, DSM_BASE } from '../../framework/utils/shared'; + +export function registerDatasourcesRoute(router: IRouter, dataSourceEnabled: boolean) { + router.post( + { + path: `${DSM_BASE}${JOBS_BASE}`, + validate: { + body: schema.object({ + query: schema.string(), + lang: schema.string(), + datasource: schema.string(), + sessionId: schema.maybe(schema.string()), + }), + }, + }, + async (context, request, response): Promise => { + const dataSourceMDSId = request.url.searchParams.get('dataSourceMDSId'); + const params = { + body: { + ...request.body, + }, + }; + try { + let res; + if (dataSourceEnabled && dataSourceMDSId) { + const client = await context.dataSource.opensearch.legacy.getClient(dataSourceMDSId); + res = await client.callAPI('datasourcemanagement.runDirectQuery', params); + } else { + res = await context.opensearch_dashboard_plugin.dashboardPluginClient + .asScoped(request) + .callAsCurrentUser('datasourcemanagement.runDirectQuery', params); + } + return response.ok({ + body: res, + }); + } catch (error: any) { + console.error('Error in running direct query:', error); + return response.custom({ + statusCode: error.statusCode || 500, + body: error.body, + }); + } + } + ); + + router.get( + { + path: `${DSM_BASE}${JOBS_BASE}/{queryId}/{dataSourceMDSId?}`, + validate: { + params: schema.object({ + queryId: schema.string(), + dataSourceMDSId: schema.maybe(schema.string({ defaultValue: '' })), + }), + }, + }, + async (context, request, response): Promise => { + try { + let res; + if (dataSourceEnabled && request.params.dataSourceMDSId) { + const client = await context.dataSource.opensearch.legacy.getClient( + request.params.dataSourceMDSId + ); + res = await client.callAPI('datasourcemanagement.getJobStatus', { + queryId: request.params.queryId, + }); + } else { + res = await context.opensearch_dashboard_plugin.dashboardPluginClient + .asScoped(request) + .callAsCurrentUser('datasourcemanagement.getJobStatus', { + queryId: request.params.queryId, + }); + } + return response.ok({ + body: res, + }); + } catch (error: any) { + console.error('Error in fetching job status:', error); + return response.custom({ + statusCode: error.statusCode || 500, + body: error.message, + }); + } + } + ); + + router.delete( + { + path: `${DSM_BASE}${JOBS_BASE}/{queryId}`, + validate: { + params: schema.object({ + queryId: schema.string(), + }), + }, + }, + async (context, request, response): Promise => { + try { + const res = await context.opensearch_dashboard_plugin.dashboardPluginClient + .asScoped(request) + .callAsCurrentUser('datasourcemanagement.deleteJob', { + queryId: request.params.queryId, + }); + return response.ok({ + body: res, + }); + } catch (error: any) { + console.error('Error in deleting job:', error); + return response.custom({ + statusCode: error.statusCode || 500, + body: error.message, + }); + } + } + ); +} diff --git a/src/plugins/dashboard/server/routes/dsl.ts b/src/plugins/dashboard/server/routes/dsl.ts new file mode 100644 index 000000000000..43ef3f3de157 --- /dev/null +++ b/src/plugins/dashboard/server/routes/dsl.ts @@ -0,0 +1,241 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +/* eslint-disable no-console*/ +import { schema } from '@osd/config-schema'; +import { RequestParams } from '@elastic/elasticsearch'; +import { IRouter } from '../../../../core/server'; +import { DSLFacet } from '../services/facets/dsl_facet'; +import { + DSL_BASE, + DSL_SEARCH, + DSL_CAT, + DSL_MAPPING, + DSL_SETTINGS, +} from '../../framework/utils/shared'; + +export function registerDslRoute( + { router }: { router: IRouter; facet: DSLFacet }, + dataSourceEnabled: boolean +) { + router.post( + { + path: `${DSL_BASE}${DSL_SEARCH}`, + validate: { body: schema.any() }, + }, + async (context, request, response) => { + const { index, size, ...rest } = request.body; + const params: RequestParams.Search = { + index, + size, + body: rest, + }; + try { + const resp = await context.core.opensearch.legacy.client.callAsCurrentUser( + 'search', + params + ); + return response.ok({ + body: resp, + }); + } catch (error) { + if (error.statusCode !== 404) console.error(error); + return response.custom({ + statusCode: error.statusCode || 500, + body: error.message, + }); + } + } + ); + + router.get( + { + path: `${DSL_BASE}${DSL_CAT}`, + validate: { + query: schema.object({ + format: schema.string(), + index: schema.maybe(schema.string()), + }), + }, + }, + async (context, request, response) => { + try { + const resp = await context.core.opensearch.legacy.client.callAsCurrentUser( + 'cat.indices', + request.query + ); + return response.ok({ + body: resp, + }); + } catch (error) { + if (error.statusCode !== 404) console.error(error); + return response.custom({ + statusCode: error.statusCode || 500, + body: error.message, + }); + } + } + ); + + router.get( + { + path: `${DSL_BASE}${DSL_MAPPING}`, + validate: { query: schema.any() }, + }, + async (context, request, response) => { + try { + const resp = await context.core.opensearch.legacy.client.callAsCurrentUser( + 'indices.getMapping', + { index: request.query.index } + ); + return response.ok({ + body: resp, + }); + } catch (error) { + if (error.statusCode !== 404) console.error(error); + return response.custom({ + statusCode: error.statusCode || 500, + body: error.message, + }); + } + } + ); + + router.get( + { + path: `${DSL_BASE}${DSL_SETTINGS}`, + validate: { query: schema.any() }, + }, + async (context, request, response) => { + try { + const resp = await context.core.opensearch.legacy.client.callAsCurrentUser( + 'indices.getSettings', + { index: request.query.index } + ); + return response.ok({ + body: resp, + }); + } catch (error) { + if (error.statusCode !== 404) console.error(error); + return response.custom({ + statusCode: error.statusCode || 500, + body: error.message, + }); + } + } + ); + + // New routes for mds enabled + router.get( + { + path: `${DSL_BASE}${DSL_CAT}/dataSourceMDSId={dataSourceMDSId?}`, + validate: { + query: schema.object({ + format: schema.string(), + index: schema.maybe(schema.string()), + }), + params: schema.object({ + dataSourceMDSId: schema.maybe(schema.string({ defaultValue: '' })), + }), + }, + }, + async (context, request, response) => { + const dataSourceMDSId = request.params.dataSourceMDSId; + try { + let resp; + if (dataSourceEnabled && dataSourceMDSId) { + const client = await context.dataSource.opensearch.legacy.getClient(dataSourceMDSId); + resp = await client.callAPI('cat.indices', request.query); + } else { + resp = await context.core.opensearch.legacy.client.callAsCurrentUser( + 'cat.indices', + request.query + ); + } + return response.ok({ + body: resp, + }); + } catch (error) { + if (error.statusCode !== 404) console.error(error); + return response.custom({ + statusCode: error.statusCode || 500, + body: error.message, + }); + } + } + ); + + router.get( + { + path: `${DSL_BASE}${DSL_MAPPING}/dataSourceMDSId={dataSourceMDSId?}`, + validate: { + query: schema.any(), + params: schema.object({ + dataSourceMDSId: schema.maybe(schema.string({ defaultValue: '' })), + }), + }, + }, + async (context, request, response) => { + const dataSourceMDSId = request.params.dataSourceMDSId; + try { + let resp; + if (dataSourceEnabled && dataSourceMDSId) { + const client = await context.dataSource.opensearch.legacy.getClient(dataSourceMDSId); + resp = await client.callAPI('indices.getMapping', { index: request.query.index }); + } else { + resp = await context.core.opensearch.legacy.client.callAsCurrentUser( + 'indices.getMapping', + { index: request.query.index } + ); + } + return response.ok({ + body: resp, + }); + } catch (error) { + if (error.statusCode !== 404) console.error(error); + return response.custom({ + statusCode: error.statusCode || 500, + body: error.message, + }); + } + } + ); + + router.get( + { + path: `${DSL_BASE}${DSL_SETTINGS}/dataSourceMDSId={dataSourceMDSId?}`, + validate: { + query: schema.any(), + params: schema.object({ + dataSourceMDSId: schema.maybe(schema.string({ defaultValue: '' })), + }), + }, + }, + async (context, request, response) => { + const dataSourceMDSId = request.params.dataSourceMDSId; + try { + let resp; + if (dataSourceEnabled && dataSourceMDSId) { + const client = await context.dataSource.opensearch.legacy.getClient(dataSourceMDSId); + resp = await client.callAPI('indices.getSettings', { index: request.query.index }); + } else { + resp = await context.core.opensearch.legacy.client.callAsCurrentUser( + 'indices.getSettings', + { index: request.query.index } + ); + } + return response.ok({ + body: resp, + }); + } catch (error) { + if (error.statusCode !== 404) console.error(error); + return response.custom({ + statusCode: error.statusCode || 500, + body: error.message, + }); + } + } + ); +} diff --git a/src/plugins/dashboard/server/routes/index.ts b/src/plugins/dashboard/server/routes/index.ts new file mode 100644 index 000000000000..2bab16798262 --- /dev/null +++ b/src/plugins/dashboard/server/routes/index.ts @@ -0,0 +1,54 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { IRouter, ILegacyClusterClient } from '../../../../core/server'; +import { registerDslRoute } from './dsl'; +import { + registerDataConnectionsRoute, + registerNonMdsDataConnectionsRoute, +} from './data_connections_router'; +import { registerDatasourcesRoute } from './datasources_router'; +import { registerPplRoute } from './ppl'; +import { DSLFacet } from '../services/facets/dsl_facet'; +import { PPLFacet } from '../services/facets/ppl_facet'; + +export function defineRoutes(router: IRouter) { + router.get( + { + path: '/api/data_source_management/example', + validate: false, + }, + async (context, request, response) => { + return response.ok({ + body: { + time: new Date().toISOString(), + }, + }); + } + ); +} + +export function setupRoutes({ + router, + client, + dataSourceEnabled, +}: { + router: IRouter; + client: ILegacyClusterClient; + dataSourceEnabled: boolean; +}) { + registerPplRoute({ router, facet: new PPLFacet(client) }); + registerDslRoute({ router, facet: new DSLFacet(client) }, dataSourceEnabled); + + // notebooks routes + // const queryService = new QueryService(client); + // registerSqlRoute(router, queryService); + + if (!dataSourceEnabled) { + registerNonMdsDataConnectionsRoute(router); + } + registerDataConnectionsRoute(router, dataSourceEnabled); + registerDatasourcesRoute(router, dataSourceEnabled); +} diff --git a/src/plugins/dashboard/server/routes/ppl.ts b/src/plugins/dashboard/server/routes/ppl.ts new file mode 100644 index 000000000000..d94f1379c3b0 --- /dev/null +++ b/src/plugins/dashboard/server/routes/ppl.ts @@ -0,0 +1,38 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { schema } from '@osd/config-schema'; +import { IRouter, IOpenSearchDashboardsResponse, ResponseError } from './../../../../core/server'; +import { PPLFacet } from '../services/facets/ppl_facet'; +import { PPL_BASE, PPL_SEARCH } from '../../framework/utils/shared'; + +export function registerPplRoute({ router, facet }: { router: IRouter; facet: PPLFacet }) { + router.post( + { + path: `${PPL_BASE}${PPL_SEARCH}`, + validate: { + body: schema.object({ + query: schema.string(), + format: schema.string(), + }), + }, + }, + async (context, req, res): Promise> => { + const queryRes: any = await facet.describeQuery(req); + if (queryRes.success) { + const result: any = { + body: { + ...queryRes.data, + }, + }; + return res.ok(result); + } + return res.custom({ + statusCode: queryRes.data.statusCode || queryRes.data.status || 500, + body: queryRes.data.body || queryRes.data.message || '', + }); + } + ); +} diff --git a/src/plugins/dashboard/server/services/facets/dsl_facet.ts b/src/plugins/dashboard/server/services/facets/dsl_facet.ts new file mode 100644 index 000000000000..61e779402861 --- /dev/null +++ b/src/plugins/dashboard/server/services/facets/dsl_facet.ts @@ -0,0 +1,37 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +/* eslint-disable no-console*/ +import _ from 'lodash'; + +export class DSLFacet { + constructor(private client: any) { + this.client = client; + } + + private fetch = async (request: any, format: string, responseFormat: string) => { + const res = { + success: false, + data: {}, + }; + try { + const params = { + query: JSON.stringify(request.body), + }; + const queryRes = await this.client.asScoped(request).callAsCurrentUser(format, params); + const dslDataSource = queryRes; + res.success = true; + res.data = dslDataSource; + } catch (err: any) { + console.error(err); + res.data = err.body; + } + return res; + }; + + describeQuery = async (request: any) => { + return this.fetch(request, 'dsl.dslQuery', 'json'); + }; +} diff --git a/src/plugins/dashboard/server/services/facets/ppl_facet.ts b/src/plugins/dashboard/server/services/facets/ppl_facet.ts new file mode 100644 index 000000000000..f662d0f25684 --- /dev/null +++ b/src/plugins/dashboard/server/services/facets/ppl_facet.ts @@ -0,0 +1,43 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +/* eslint-disable no-console */ +import _ from 'lodash'; +import { PPLDataSource } from '../../adaptors/ppl_datasource'; + +export class PPLFacet { + constructor(private client: any) { + this.client = client; + } + + private fetch = async (request: any, format: string, responseFormat: string) => { + const res = { + success: false, + data: {}, + }; + try { + const params = { + body: { + query: request.body.query, + }, + }; + if (request.body.format !== 'jdbc') { + params.format = request.body.format; + } + const queryRes = await this.client.asScoped(request).callAsCurrentUser(format, params); + const pplDataSource = new PPLDataSource(queryRes, request.body.format); + res.success = true; + res.data = pplDataSource.getDataSource(); + } catch (err: any) { + console.error('PPL query fetch err: ', err); + res.data = err; + } + return res; + }; + + describeQuery = async (request: any) => { + return this.fetch(request, 'ppl.pplQuery', 'json'); + }; +} From fd5d56bf24afbd4e29d899f0599a3fd8f3a4bb74 Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Tue, 8 Apr 2025 21:50:26 -0700 Subject: [PATCH 13/86] poc related - remove unnecessary comment Signed-off-by: Jialiang Liang --- .../public/application/embeddable/grid/dashboard_grid.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx index 58080a678ef9..a5dd87afd10c 100644 --- a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx +++ b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx @@ -417,14 +417,14 @@ class DashboardGridUi extends React.Component { const isViewMode = viewMode === ViewMode.VIEW; return (
- {/* ✅ Top-left corner "Synchronize Now" button */} + {/* Top-left corner "Synchronize Now" button */}
Synchronize Now
- {/* ✅ The grid itself */} + {/* The grid itself */} Date: Tue, 8 Apr 2025 22:31:54 -0700 Subject: [PATCH 14/86] poc phase 6 - implement refresh query call but no polling Signed-off-by: Jialiang Liang --- .../embeddable/grid/dashboard_grid.tsx | 44 ++++++++++++++++--- 1 file changed, 37 insertions(+), 7 deletions(-) diff --git a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx index a5dd87afd10c..6f6ba2c3046b 100644 --- a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx +++ b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx @@ -156,6 +156,10 @@ class DashboardGridUi extends React.Component { // item. private gridItems = {} as { [key: string]: HTMLDivElement | null }; + private extractedDatasource?: string; + private extractedDatabase?: string; + private extractedIndex?: string; + constructor(props: DashboardGridProps) { super(props); @@ -326,6 +330,10 @@ class DashboardGridUi extends React.Component { const database = parts[1] || 'unknown'; const index = parts.slice(2).join('_') || 'unknown'; + this.extractedDatasource = datasource; + this.extractedDatabase = database; + this.extractedIndex = index; + console.log('Extracted Info:'); console.log('Datasource:', datasource); console.log('Database:', database); @@ -350,15 +358,37 @@ class DashboardGridUi extends React.Component { synchronizeNow = async () => { try { - console.log('http:', this.props.http); - const response = await this.props.http.get(`/api/directquery/dsl/indices.getFieldMapping`, { - query: { - index: 'flint_flinttest1_default_vpc_mv_1106', - }, + const { extractedDatasource, extractedDatabase, extractedIndex } = this; + + if ( + !extractedDatasource || + extractedDatasource === 'unknown' || + !extractedDatabase || + extractedDatabase === 'unknown' || + !extractedIndex || + extractedIndex === 'unknown' + ) { + console.error( + 'Datasource, database, or index not properly set. Cannot run REFRESH command.' + ); + return; + } + + const query = `REFRESH MATERIALIZED VIEW \`${extractedDatasource}\`.\`${extractedDatabase}\`.\`${extractedIndex}\``; + + const queryParams = { + query, + lang: 'sql', + datasource: extractedDatasource, + }; + + const response = await this.props.http.post('/api/observability/query/jobs', { + body: JSON.stringify(queryParams), }); - console.log('Index Mapping:', response); + + console.log('Materialized view refresh response:', response); } catch (error) { - console.error('Error fetching index mapping:', error); + console.error('Error executing REFRESH MATERIALIZED VIEW:', error); } }; From c41b2ddd4080c1940d735e08f3a0096d22f84fac Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Tue, 15 Apr 2025 15:33:52 -0700 Subject: [PATCH 15/86] poc phase 7 - implement direct query hook in dashboard plugin Signed-off-by: Jialiang Liang --- src/plugins/dashboard/framework/constants.tsx | 99 +++++ .../framework/hooks/direct_query_hook.tsx | 99 +++++ .../dashboard/framework/requests/sql.ts | 70 ++++ src/plugins/dashboard/framework/types.tsx | 388 ++++++++++++++++++ .../framework/utils/query_session_utils.ts | 16 + .../dashboard/framework/utils/use_polling.tsx | 137 +++++++ .../embeddable/dashboard_container.tsx | 5 +- .../embeddable/grid/dashboard_grid.tsx | 73 ++-- .../viewport/dashboard_viewport.tsx | 38 +- 9 files changed, 887 insertions(+), 38 deletions(-) create mode 100644 src/plugins/dashboard/framework/constants.tsx create mode 100644 src/plugins/dashboard/framework/hooks/direct_query_hook.tsx create mode 100644 src/plugins/dashboard/framework/requests/sql.ts create mode 100644 src/plugins/dashboard/framework/types.tsx create mode 100644 src/plugins/dashboard/framework/utils/query_session_utils.ts create mode 100644 src/plugins/dashboard/framework/utils/use_polling.tsx diff --git a/src/plugins/dashboard/framework/constants.tsx b/src/plugins/dashboard/framework/constants.tsx new file mode 100644 index 000000000000..785e17bbaa95 --- /dev/null +++ b/src/plugins/dashboard/framework/constants.tsx @@ -0,0 +1,99 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export const ASYNC_QUERY_SESSION_ID = 'async-query-session-id'; + +export const DATA_SOURCE_NAME_URL_PARAM_KEY = 'datasourceName'; +export const DATA_SOURCE_TYPE_URL_PARAM_KEY = 'datasourceType'; +export const OLLY_QUESTION_URL_PARAM_KEY = 'olly_q'; +export const INDEX_URL_PARAM_KEY = 'indexPattern'; +export const DEFAULT_DATA_SOURCE_TYPE = 'DEFAULT_INDEX_PATTERNS'; +export const DEFAULT_DATA_SOURCE_NAME = 'Default cluster'; +export const DEFAULT_DATA_SOURCE_OBSERVABILITY_DISPLAY_NAME = 'OpenSearch'; +export const DEFAULT_DATA_SOURCE_TYPE_NAME = 'Default Group'; +export const enum QUERY_LANGUAGE { + PPL = 'PPL', + SQL = 'SQL', + DQL = 'DQL', +} +export enum DATA_SOURCE_TYPES { + DEFAULT_CLUSTER_TYPE = DEFAULT_DATA_SOURCE_TYPE, + SPARK = 'spark', + S3Glue = 's3glue', +} +export const ASYNC_POLLING_INTERVAL = 2000; + +export const CATALOG_CACHE_VERSION = '1.0'; +export const ACCELERATION_DEFUALT_SKIPPING_INDEX_NAME = 'skipping'; +export const ACCELERATION_TIME_INTERVAL = [ + { text: 'millisecond(s)', value: 'millisecond' }, + { text: 'second(s)', value: 'second' }, + { text: 'minutes(s)', value: 'minute' }, + { text: 'hour(s)', value: 'hour' }, + { text: 'day(s)', value: 'day' }, + { text: 'week(s)', value: 'week' }, +]; +export const ACCELERATION_REFRESH_TIME_INTERVAL = [ + { text: 'minutes(s)', value: 'minute' }, + { text: 'hour(s)', value: 'hour' }, + { text: 'day(s)', value: 'day' }, + { text: 'week(s)', value: 'week' }, +]; + +export const ACCELERATION_ADD_FIELDS_TEXT = '(add fields here)'; +export const ACCELERATION_INDEX_NAME_REGEX = /^[a-z0-9_]+$/; +export const ACCELERATION_S3_URL_REGEX = /^(s3|s3a):\/\/[a-zA-Z0-9.\-]+/; +export const SPARK_HIVE_TABLE_REGEX = /Provider:\s*hive/; +export const SANITIZE_QUERY_REGEX = /\s+/g; +export const SPARK_TIMESTAMP_DATATYPE = 'timestamp'; +export const SPARK_STRING_DATATYPE = 'string'; + +export const ACCELERATION_INDEX_TYPES = [ + { label: 'Skipping Index', value: 'skipping' }, + { label: 'Covering Index', value: 'covering' }, + { label: 'Materialized View', value: 'materialized' }, +]; + +export const ACC_INDEX_TYPE_DOCUMENTATION_URL = + 'https://github.com/opensearch-project/opensearch-spark/blob/main/docs/index.md'; +export const ACC_CHECKPOINT_DOCUMENTATION_URL = + 'https://github.com/opensearch-project/opensearch-spark/blob/main/docs/index.md#create-index-options'; + +export const ACCELERATION_INDEX_NAME_INFO = `All OpenSearch acceleration indices have a naming format of pattern: \`prefix__suffix\`. They share a common prefix structure, which is \`flint____\`. Additionally, they may have a suffix that varies based on the index type. +##### Skipping Index +- For 'Skipping' indices, a fixed index name 'skipping' is used, and this name cannot be modified by the user. The suffix added to this type is \`_index\`. + - An example of a 'Skipping' index name would be: \`flint_mydatasource_mydb_mytable_skipping_index\`. +##### Covering Index +- 'Covering' indices allow users to specify their index name. The suffix added to this type is \`_index\`. + - For instance, a 'Covering' index name could be: \`flint_mydatasource_mydb_mytable_myindexname_index\`. +##### Materialized View Index +- 'Materialized View' indices also enable users to define their index name, but they do not have a suffix. + - An example of a 'Materialized View' index name might look like: \`flint_mydatasource_mydb_mytable_myindexname\`. +##### Note: +- All user given index names must be in lowercase letters, numbers and underscore. Spaces, commas, and characters -, :, ", *, +, /, \, |, ?, #, >, or < are not allowed. + `; + +export const SKIPPING_INDEX_ACCELERATION_METHODS = [ + { value: 'PARTITION', text: 'Partition' }, + { value: 'VALUE_SET', text: 'Value Set' }, + { value: 'MIN_MAX', text: 'Min Max' }, + { value: 'BLOOM_FILTER', text: 'Bloom Filter' }, +]; + +export const ACCELERATION_AGGREGRATION_FUNCTIONS = [ + { label: 'window.start' }, + { label: 'count' }, + { label: 'sum' }, + { label: 'avg' }, + { label: 'max' }, + { label: 'min' }, +]; + +export const SPARK_PARTITION_INFO = `# Partition Information`; +export const OBS_DEFAULT_CLUSTER = 'observability-default'; // prefix key for generating data source id for default cluster in data selector +export const OBS_S3_DATA_SOURCE = 'observability-s3'; // prefix key for generating data source id for s3 data sources in data selector +export const S3_DATA_SOURCE_GROUP_DISPLAY_NAME = 'Amazon S3'; // display group name for Amazon-managed-s3 data sources in data selector +export const S3_DATA_SOURCE_GROUP_SPARK_DISPLAY_NAME = 'Spark'; // display group name for OpenSearch-spark-s3 data sources in data selector +export const SECURITY_DASHBOARDS_LOGOUT_URL = '/logout'; diff --git a/src/plugins/dashboard/framework/hooks/direct_query_hook.tsx b/src/plugins/dashboard/framework/hooks/direct_query_hook.tsx new file mode 100644 index 000000000000..a2b05f47e9ee --- /dev/null +++ b/src/plugins/dashboard/framework/hooks/direct_query_hook.tsx @@ -0,0 +1,99 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { useEffect, useState } from 'react'; +import { HttpStart, NotificationsStart } from 'opensearch-dashboards/public'; +import { ASYNC_POLLING_INTERVAL } from '../constants'; +import { DirectQueryLoadingStatus, DirectQueryRequest } from '../types'; +import { getAsyncSessionId, setAsyncSessionId } from '../utils/query_session_utils'; +import { get as getObjValue, formatError } from '../utils/shared'; +import { usePolling } from '../utils/use_polling'; +import { SQLService } from '../requests/sql'; + +export const useDirectQuery = ( + http: HttpStart, + notifications: NotificationsStart, + dataSourceMDSId?: string +) => { + const sqlService = new SQLService(http); + const [loadStatus, setLoadStatus] = useState( + DirectQueryLoadingStatus.SCHEDULED + ); + + const { + data: pollingResult, + loading: _pollingLoading, + error: pollingError, + startPolling, + stopPolling: stopLoading, + } = usePolling((params) => { + return sqlService.fetchWithJobId(params, dataSourceMDSId || ''); + }, ASYNC_POLLING_INTERVAL); + + const startLoading = (requestPayload: DirectQueryRequest) => { + setLoadStatus(DirectQueryLoadingStatus.SCHEDULED); + + const sessionId = getAsyncSessionId(requestPayload.datasource); + if (sessionId) { + requestPayload = { ...requestPayload, sessionId }; + } + + sqlService + .fetch(requestPayload, dataSourceMDSId) + .then((result) => { + setAsyncSessionId(requestPayload.datasource, getObjValue(result, 'sessionId', null)); + if (result.queryId) { + startPolling({ + queryId: result.queryId, + }); + } else { + // eslint-disable-next-line no-console + console.error('No query id found in response'); + setLoadStatus(DirectQueryLoadingStatus.FAILED); + } + }) + .catch((e) => { + setLoadStatus(DirectQueryLoadingStatus.FAILED); + const formattedError = formatError( + '', + 'The query failed to execute and the operation could not be complete.', + e.body?.message + ); + notifications.toasts.addError(formattedError, { + title: 'Query Failed', + }); + // eslint-disable-next-line no-console + console.error(e); + }); + }; + + useEffect(() => { + // cancel direct query + if (!pollingResult) return; + const { status: anyCaseStatus, datarows, error } = pollingResult; + const status = anyCaseStatus?.toLowerCase(); + + if (status === DirectQueryLoadingStatus.SUCCESS || datarows) { + setLoadStatus(status); + stopLoading(); + } else if (status === DirectQueryLoadingStatus.FAILED) { + setLoadStatus(status); + stopLoading(); + const formattedError = formatError( + '', + 'The query failed to execute and the operation could not be complete.', + error + ); + notifications.toasts.addError(formattedError, { + title: 'Query Failed', + }); + } else { + setLoadStatus(status); + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [pollingResult, pollingError, stopLoading]); + + return { loadStatus, startLoading, stopLoading, pollingResult }; +}; diff --git a/src/plugins/dashboard/framework/requests/sql.ts b/src/plugins/dashboard/framework/requests/sql.ts new file mode 100644 index 000000000000..274289af4543 --- /dev/null +++ b/src/plugins/dashboard/framework/requests/sql.ts @@ -0,0 +1,70 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +/* eslint-disable no-console */ + +import { HttpStart } from 'opensearch-dashboards/public'; +import { DirectQueryRequest } from '../types'; + +interface FetchError { + body: string; + message?: string; + [key: string]: any; +} + +export class SQLService { + private http: HttpStart; + + constructor(http: HttpStart) { + this.http = http; + } + + fetch = async ( + params: DirectQueryRequest, + dataSourceMDSId?: string, + errorHandler?: (error: FetchError) => void + ) => { + const query = { + dataSourceMDSId, + }; + return this.http + .post('/api/observability/query/jobs', { + body: JSON.stringify(params), + query, + }) + .catch((error: FetchError) => { + console.error('fetch error: ', error.body); + if (errorHandler) errorHandler(error); + throw error; + }); + }; + + fetchWithJobId = async ( + params: { queryId: string }, + dataSourceMDSId?: string, + errorHandler?: (error: FetchError) => void + ) => { + return this.http + .get(`/api/observability/query/jobs/${params.queryId}/${dataSourceMDSId ?? ''}`) + .catch((error: FetchError) => { + console.error('fetch error: ', error.body); + if (errorHandler) errorHandler(error); + throw error; + }); + }; + + deleteWithJobId = async ( + params: { queryId: string }, + errorHandler?: (error: FetchError) => void + ) => { + return this.http + .delete(`/api/observability/query/jobs/${params.queryId}`) + .catch((error: FetchError) => { + console.error('delete error: ', error.body); + if (errorHandler) errorHandler(error); + throw error; + }); + }; +} diff --git a/src/plugins/dashboard/framework/types.tsx b/src/plugins/dashboard/framework/types.tsx new file mode 100644 index 000000000000..34174a6c4b0e --- /dev/null +++ b/src/plugins/dashboard/framework/types.tsx @@ -0,0 +1,388 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { EuiComboBoxOptionOption } from '@elastic/eui'; + +export enum DirectQueryLoadingStatus { + SUCCESS = 'success', + FAILED = 'failed', + RUNNING = 'running', + SCHEDULED = 'scheduled', + CANCELED = 'canceled', + WAITING = 'waiting', + INITIAL = 'initial', +} + +export interface DirectQueryRequest { + query: string; + lang: string; + datasource: string; + sessionId?: string; +} + +export type AccelerationStatus = 'ACTIVE' | 'INACTIVE'; + +export interface PermissionsConfigurationProps { + roles: Role[]; + selectedRoles: Role[]; + setSelectedRoles: React.Dispatch>; + layout: 'horizontal' | 'vertical'; + hasSecurityAccess: boolean; +} + +export interface TableColumn { + name: string; + dataType: string; +} + +export interface Acceleration { + name: string; + status: AccelerationStatus; + type: string; + database: string; + table: string; + destination: string; + dateCreated: number; + dateUpdated: number; + index: string; + sql: string; +} + +export interface AssociatedObject { + tableName: string; + datasource: string; + id: string; + name: string; + database: string; + type: AssociatedObjectIndexType; + accelerations: CachedAcceleration[] | AssociatedObject; + columns?: CachedColumn[]; +} + +export type Role = EuiComboBoxOptionOption; + +export type DatasourceType = 'S3GLUE' | 'PROMETHEUS'; + +export interface S3GlueProperties { + 'glue.indexstore.opensearch.uri': string; + 'glue.indexstore.opensearch.region': string; +} + +export interface PrometheusProperties { + 'prometheus.uri': string; +} + +export type DatasourceStatus = 'ACTIVE' | 'DISABLED'; + +export interface DatasourceDetails { + allowedRoles: string[]; + name: string; + connector: DatasourceType; + description: string; + properties: S3GlueProperties | PrometheusProperties; + status: DatasourceStatus; +} + +interface AsyncApiDataResponse { + status: string; + schema?: Array<{ name: string; type: string }>; + datarows?: any; + total?: number; + size?: number; + error?: string; +} + +export interface AsyncApiResponse { + data: { + ok: boolean; + resp: AsyncApiDataResponse; + }; +} + +export type PollingCallback = (statusObj: AsyncApiResponse) => void; + +export type AssociatedObjectIndexType = AccelerationIndexType | 'table'; + +export type AccelerationIndexType = 'skipping' | 'covering' | 'materialized'; + +export type LoadCacheType = 'databases' | 'tables' | 'accelerations' | 'tableColumns'; + +export enum CachedDataSourceStatus { + Updated = 'Updated', + Failed = 'Failed', + Empty = 'Empty', +} + +export interface CachedColumn { + fieldName: string; + dataType: string; +} + +export interface CachedTable { + name: string; + columns?: CachedColumn[]; +} + +export interface CachedDatabase { + name: string; + tables: CachedTable[]; + lastUpdated: string; // date string in UTC format + status: CachedDataSourceStatus; +} + +export interface CachedDataSource { + name: string; + lastUpdated: string; // date string in UTC format + status: CachedDataSourceStatus; + databases: CachedDatabase[]; + dataSourceMDSId?: string; +} + +export interface DataSourceCacheData { + version: string; + dataSources: CachedDataSource[]; +} + +export interface CachedAcceleration { + flintIndexName: string; + type: AccelerationIndexType; + database: string; + table: string; + indexName: string; + autoRefresh: boolean; + status: string; +} + +export interface CachedAccelerationByDataSource { + name: string; + accelerations: CachedAcceleration[]; + lastUpdated: string; // date string in UTC format + status: CachedDataSourceStatus; + dataSourceMDSId?: string; +} + +export interface AccelerationsCacheData { + version: string; + dataSources: CachedAccelerationByDataSource[]; +} + +export interface PollingSuccessResult { + schema: Array<{ name: string; type: string }>; + datarows: Array>; +} + +export type AsyncPollingResult = PollingSuccessResult | null; + +export type AggregationFunctionType = 'count' | 'sum' | 'avg' | 'max' | 'min' | 'window.start'; + +export interface MaterializedViewColumn { + id: string; + functionName: AggregationFunctionType; + functionParam?: string; + fieldAlias?: string; +} + +export type SkippingIndexAccMethodType = 'PARTITION' | 'VALUE_SET' | 'MIN_MAX' | 'BLOOM_FILTER'; + +export interface SkippingIndexRowType { + id: string; + fieldName: string; + dataType: string; + accelerationMethod: SkippingIndexAccMethodType; +} + +export interface DataTableFieldsType { + id: string; + fieldName: string; + dataType: string; +} + +export interface RefreshIntervalType { + refreshWindow: number; + refreshInterval: string; +} + +export interface WatermarkDelayType { + delayWindow: number; + delayInterval: string; +} + +export interface GroupByTumbleType { + timeField: string; + tumbleWindow: number; + tumbleInterval: string; +} + +export interface MaterializedViewQueryType { + columnsValues: MaterializedViewColumn[]; + groupByTumbleValue: GroupByTumbleType; +} + +export interface FormErrorsType { + dataSourceError: string[]; + databaseError: string[]; + dataTableError: string[]; + skippingIndexError: string[]; + coveringIndexError: string[]; + materializedViewError: string[]; + indexNameError: string[]; + primaryShardsError: string[]; + replicaShardsError: string[]; + refreshIntervalError: string[]; + checkpointLocationError: string[]; + watermarkDelayError: string[]; +} + +export type AccelerationRefreshType = 'autoInterval' | 'manual' | 'manualIncrement'; + +export interface CreateAccelerationForm { + dataSource: string; + database: string; + dataTable: string; + dataTableFields: DataTableFieldsType[]; + accelerationIndexType: AccelerationIndexType; + skippingIndexQueryData: SkippingIndexRowType[]; + coveringIndexQueryData: string[]; + materializedViewQueryData: MaterializedViewQueryType; + accelerationIndexName: string; + primaryShardsCount: number; + replicaShardsCount: number; + refreshType: AccelerationRefreshType; + checkpointLocation: string | undefined; + watermarkDelay: WatermarkDelayType; + refreshIntervalOptions: RefreshIntervalType; + formErrors: FormErrorsType; +} + +export interface LoadCachehookOutput { + loadStatus: DirectQueryLoadingStatus; + startLoading: (params: StartLoadingParams) => void; + stopLoading: () => void; +} + +export interface StartLoadingParams { + dataSourceName: string; + dataSourceMDSId?: string; + databaseName?: string; + tableName?: string; +} + +export interface RenderAccelerationFlyoutParams { + dataSourceName: string; + dataSourceMDSId?: string; + databaseName?: string; + tableName?: string; + handleRefresh?: () => void; +} + +export interface RenderAssociatedObjectsDetailsFlyoutParams { + tableDetail: AssociatedObject; + dataSourceName: string; + handleRefresh?: () => void; + dataSourceMDSId?: string; +} + +export interface RenderAccelerationDetailsFlyoutParams { + acceleration: CachedAcceleration; + dataSourceName: string; + handleRefresh?: () => void; + dataSourceMDSId?: string; +} + +// Integration types + +export interface StaticAsset { + annotation?: string; + path: string; +} + +export interface IntegrationWorkflow { + name: string; + label: string; + description: string; + enabled_by_default: boolean; +} + +export interface IntegrationStatics { + logo?: StaticAsset; + gallery?: StaticAsset[]; + darkModeLogo?: StaticAsset; + darkModeGallery?: StaticAsset[]; +} + +export interface IntegrationComponent { + name: string; + version: string; +} + +export type SupportedAssetType = 'savedObjectBundle' | 'query'; + +export interface IntegrationAsset { + name: string; + version: string; + extension: string; + type: SupportedAssetType; + workflows?: string[]; +} + +export interface IntegrationConfig { + name: string; + version: string; + displayName?: string; + license: string; + type: string; + labels?: string[]; + author?: string; + description?: string; + sourceUrl?: string; + workflows?: IntegrationWorkflow[]; + statics?: IntegrationStatics; + components: IntegrationComponent[]; + assets: IntegrationAsset[]; + sampleData?: { + path: string; + }; +} + +export interface AvailableIntegrationsList { + hits: IntegrationConfig[]; +} + +export interface AssetReference { + assetType: string; + assetId: string; + isDefaultAsset: boolean; + description: string; + status?: string; +} + +export interface IntegrationInstanceResult extends IntegrationInstance { + id: string; + status: string; +} + +export interface IntegrationInstance { + name: string; + templateName: string; + dataSource: string; + creationDate: string; + assets: AssetReference[]; +} + +export interface AvailableIntegrationsList { + hits: IntegrationConfig[]; +} + +export interface IntegrationInstancesSearchResult { + hits: IntegrationInstanceResult[]; +} + +export type ParsedIntegrationAsset = + | { type: 'savedObjectBundle'; workflows?: string[]; data: object[] } + | { type: 'query'; workflows?: string[]; query: string; language: string }; + +export type Result = + | { ok: true; value: T; error?: undefined } + | { ok: false; error: E; value?: undefined }; diff --git a/src/plugins/dashboard/framework/utils/query_session_utils.ts b/src/plugins/dashboard/framework/utils/query_session_utils.ts new file mode 100644 index 000000000000..beabcb48c197 --- /dev/null +++ b/src/plugins/dashboard/framework/utils/query_session_utils.ts @@ -0,0 +1,16 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { ASYNC_QUERY_SESSION_ID } from '../constants'; + +export const setAsyncSessionId = (dataSource: string, value: string | null) => { + if (value !== null) { + sessionStorage.setItem(`${ASYNC_QUERY_SESSION_ID}_${dataSource}`, value); + } +}; + +export const getAsyncSessionId = (dataSource: string) => { + return sessionStorage.getItem(`${ASYNC_QUERY_SESSION_ID}_${dataSource}`); +}; diff --git a/src/plugins/dashboard/framework/utils/use_polling.tsx b/src/plugins/dashboard/framework/utils/use_polling.tsx new file mode 100644 index 000000000000..f2af5979ab59 --- /dev/null +++ b/src/plugins/dashboard/framework/utils/use_polling.tsx @@ -0,0 +1,137 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { useEffect, useRef, useState } from 'react'; + +type FetchFunction = (params?: P) => Promise; + +export interface PollingConfigurations { + tabId: string; +} + +export class UsePolling { + public data: T | null = null; + public error: Error | null = null; + public loading: boolean = true; + private shouldPoll: boolean = false; + private intervalRef?: NodeJS.Timeout; + + constructor( + private fetchFunction: FetchFunction, + private interval: number = 5000, + private onPollingSuccess?: (data: T, configurations: PollingConfigurations) => boolean, + private onPollingError?: (error: Error) => boolean, + private configurations?: PollingConfigurations + ) {} + + async fetchData(params?: P) { + this.loading = true; + try { + const result = await this.fetchFunction(params); + this.data = result; + this.loading = false; + + if (this.onPollingSuccess && this.onPollingSuccess(result, this.configurations!)) { + this.stopPolling(); + } + } catch (err) { + this.error = err as Error; + this.loading = false; + + if (this.onPollingError && this.onPollingError(this.error)) { + this.stopPolling(); + } + } + } + + startPolling(params?: P) { + this.shouldPoll = true; + if (!this.intervalRef) { + this.intervalRef = setInterval(() => { + if (this.shouldPoll) { + this.fetchData(params); + } + }, this.interval); + } + } + + stopPolling() { + this.shouldPoll = false; + if (this.intervalRef) { + clearInterval((this.intervalRef as unknown) as NodeJS.Timeout); + this.intervalRef = undefined; + } + } +} + +interface UsePollingReturn { + data: T | null; + loading: boolean; + error: Error | null; + startPolling: (params?: any) => void; + stopPolling: () => void; +} + +export function usePolling( + fetchFunction: FetchFunction, + interval: number = 5000, + onPollingSuccess?: (data: T, configurations: PollingConfigurations) => boolean, + onPollingError?: (error: Error) => boolean, + configurations?: PollingConfigurations +): UsePollingReturn { + const [data, setData] = useState(null); + const [error, setError] = useState(null); + const [loading, setLoading] = useState(true); + const intervalRef = useRef(undefined); + const unmounted = useRef(false); + + const shouldPoll = useRef(false); + + const startPolling = (params?: P) => { + shouldPoll.current = true; + const intervalId = setInterval(() => { + if (shouldPoll.current) { + fetchData(params); + } + }, interval); + intervalRef.current = intervalId; + if (unmounted.current) { + clearInterval((intervalId as unknown) as NodeJS.Timeout); + } + }; + + const stopPolling = () => { + shouldPoll.current = false; + clearInterval((intervalRef.current as unknown) as NodeJS.Timeout); + }; + + const fetchData = async (params?: P) => { + try { + const result = await fetchFunction(params); + setData(result); + // Check the success condition and stop polling if it's met + if (onPollingSuccess && onPollingSuccess(result, configurations)) { + stopPolling(); + } + } catch (err: unknown) { + setError(err as Error); + + // Check the error condition and stop polling if it's met + if (onPollingError && onPollingError(err as Error)) { + stopPolling(); + } + } finally { + setLoading(false); + } + }; + + useEffect(() => { + return () => { + unmounted.current = true; + }; + }, []); + + return { data, loading, error, startPolling, stopPolling }; +} diff --git a/src/plugins/dashboard/public/application/embeddable/dashboard_container.tsx b/src/plugins/dashboard/public/application/embeddable/dashboard_container.tsx index fd822291caa0..1c4654d32c36 100644 --- a/src/plugins/dashboard/public/application/embeddable/dashboard_container.tsx +++ b/src/plugins/dashboard/public/application/embeddable/dashboard_container.tsx @@ -53,7 +53,7 @@ import { UiActionsStart } from '../../../../ui_actions/public'; import { DASHBOARD_CONTAINER_TYPE } from './dashboard_constants'; import { createPanelState } from './panel'; import { DashboardPanelState } from './types'; -import { DashboardViewport } from './viewport/dashboard_viewport'; +import { DashboardViewportWithQuery } from './viewport/dashboard_viewport'; import { OpenSearchDashboardsContextProvider, OpenSearchDashboardsReactContext, @@ -240,7 +240,7 @@ export class DashboardContainer extends Container - , diff --git a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx index 6f6ba2c3046b..0709ba1ed89d 100644 --- a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx +++ b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx @@ -40,7 +40,7 @@ import React from 'react'; import { Subscription } from 'rxjs'; import ReactGridLayout, { Layout, ReactGridLayoutProps } from 'react-grid-layout'; import type { SavedObjectsClientContract } from 'src/core/public'; -import { HttpStart } from 'src/core/public'; +import { HttpStart, NotificationsStart } from 'src/core/public'; import { EuiButton } from '@elastic/eui'; import { ViewMode, EmbeddableChildPanel, EmbeddableStart } from '../../../../../embeddable/public'; import { GridData } from '../../../../common'; @@ -49,6 +49,8 @@ import { DashboardPanelState } from '../types'; import { withOpenSearchDashboards } from '../../../../../opensearch_dashboards_react/public'; import { DashboardContainerInput } from '../dashboard_container'; import { DashboardContainer, DashboardReactContextValue } from '../dashboard_container'; +import { DirectQueryLoadingStatus } from '../../../../framework/types'; +import { DirectQueryRequest } from '../../../../framework/types'; let lastValidGridSize = 0; @@ -132,6 +134,12 @@ export interface DashboardGridProps extends ReactIntl.InjectedIntlProps { container: DashboardContainer; savedObjectsClient: SavedObjectsClientContract; http: HttpStart; + notifications: NotificationsStart; + + // direct query loading + startLoading: (payload: DirectQueryRequest) => void; + loadStatus: DirectQueryLoadingStatus; + pollingResult: any; } interface State { @@ -356,40 +364,32 @@ class DashboardGridUi extends React.Component { }); } - synchronizeNow = async () => { - try { - const { extractedDatasource, extractedDatabase, extractedIndex } = this; - - if ( - !extractedDatasource || - extractedDatasource === 'unknown' || - !extractedDatabase || - extractedDatabase === 'unknown' || - !extractedIndex || - extractedIndex === 'unknown' - ) { - console.error( - 'Datasource, database, or index not properly set. Cannot run REFRESH command.' - ); - return; - } - - const query = `REFRESH MATERIALIZED VIEW \`${extractedDatasource}\`.\`${extractedDatabase}\`.\`${extractedIndex}\``; + synchronizeNow = () => { + const { extractedDatasource, extractedDatabase, extractedIndex } = this; + + if ( + !extractedDatasource || + extractedDatasource === 'unknown' || + !extractedDatabase || + extractedDatabase === 'unknown' || + !extractedIndex || + extractedIndex === 'unknown' + ) { + console.error('Datasource, database, or index not properly set. Cannot run REFRESH command.'); + return; + } - const queryParams = { - query, - lang: 'sql', - datasource: extractedDatasource, - }; + // POC ONLY: Run REFRESH MATERIALIZED VIEW command + // TODO: Add the logic to check the index type + const query = `REFRESH MATERIALIZED VIEW \`${extractedDatasource}\`.\`${extractedDatabase}\`.\`${extractedIndex}\``; - const response = await this.props.http.post('/api/observability/query/jobs', { - body: JSON.stringify(queryParams), - }); + const queryPayload = { + query, + lang: 'sql', + datasource: extractedDatasource, + }; - console.log('Materialized view refresh response:', response); - } catch (error) { - console.error('Error executing REFRESH MATERIALIZED VIEW:', error); - } + this.props.startLoading(queryPayload); }; public renderPanels() { @@ -449,12 +449,17 @@ class DashboardGridUi extends React.Component {
{/* Top-left corner "Synchronize Now" button */}
- + Synchronize Now
- {/* The grid itself */} void; + loadStatus: DirectQueryLoadingStatus; + pollingResult: any; } interface State { @@ -137,7 +148,7 @@ export class DashboardViewport extends React.Component
); @@ -180,3 +195,22 @@ export class DashboardViewport extends React.Component +) => { + const { http, notifications, ...restProps } = props; + const { startLoading, loadStatus, pollingResult } = useDirectQuery(http, notifications); + + return ( + + ); +}; +// From 9998922c3e8c62dd4a1fc2505263c141cc1a27c7 Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Tue, 15 Apr 2025 16:55:35 -0700 Subject: [PATCH 16/86] poc phase 7 - using internal resolve index api for getting index name from index pattern Signed-off-by: Jialiang Liang --- .../embeddable/grid/dashboard_grid.tsx | 59 +++++++++++++------ 1 file changed, 42 insertions(+), 17 deletions(-) diff --git a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx index 0709ba1ed89d..0224ced3f613 100644 --- a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx +++ b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx @@ -329,23 +329,48 @@ class DashboardGridUi extends React.Component { indexPatternRef.id ); const indexTitle = indexPattern.attributes.title; - console.log('Index pattern title (index name):', indexTitle); - // Extract datasource, database, and index name from index title - const trimmedTitle = indexTitle.replace(/^flint_/, ''); - const parts = trimmedTitle.split('_'); - - const datasource = parts[0] || 'unknown'; - const database = parts[1] || 'unknown'; - const index = parts.slice(2).join('_') || 'unknown'; - - this.extractedDatasource = datasource; - this.extractedDatabase = database; - this.extractedIndex = index; - - console.log('Extracted Info:'); - console.log('Datasource:', datasource); - console.log('Database:', database); - console.log('Index:', index); + console.log('Index pattern title (raw):', indexTitle); + + // If indexTitle contains a wildcard, try to resolve to real indices + if (indexTitle.includes('*')) { + try { + const resolved = await this.props.http.get( + `/internal/index-pattern-management/resolve_index/${encodeURIComponent( + indexTitle + )}` + ); + const matchedIndices = resolved?.indices || []; + console.log('Resolved index pattern to concrete indices:', matchedIndices); + + if (matchedIndices.length > 0) { + // TODO: Handle multiple matches if needed + const firstMatch = matchedIndices[0].name; + const trimmedTitle = firstMatch.replace(/^flint_/, ''); + const parts = trimmedTitle.split('_'); + + this.extractedDatasource = parts[0] || 'unknown'; + this.extractedDatabase = parts[1] || 'unknown'; + this.extractedIndex = parts.slice(2).join('_') || 'unknown'; + + console.log('Resolved from concrete index name:'); + console.log('Datasource:', this.extractedDatasource); + console.log('Database:', this.extractedDatabase); + console.log('Index:', this.extractedIndex); + } else { + console.warn('No concrete indices matched the wildcard pattern.'); + } + } catch (err) { + console.error('Failed to resolve concrete index for pattern:', indexTitle, err); + } + } else { + // Original fallback logic (non-wildcard) + const trimmedTitle = indexTitle.replace(/^flint_/, ''); + const parts = trimmedTitle.split('_'); + + this.extractedDatasource = parts[0] || 'unknown'; + this.extractedDatabase = parts[1] || 'unknown'; + this.extractedIndex = parts.slice(2).join('_') || 'unknown'; + } } catch (err) { console.error(`Failed to fetch index pattern ${indexPatternRef.id}:`, err); } From 62698c2348f14e3a9dda666b90004ad4d36f8028 Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Wed, 16 Apr 2025 14:40:59 -0700 Subject: [PATCH 17/86] Progress proto Signed-off-by: Simeon Widdis --- .../framework/hooks/direct_query_hook.tsx | 2 +- src/plugins/dashboard/framework/types.tsx | 3 +- .../embeddable/grid/dashboard_grid.tsx | 35 ++++++++++++++++--- 3 files changed, 34 insertions(+), 6 deletions(-) diff --git a/src/plugins/dashboard/framework/hooks/direct_query_hook.tsx b/src/plugins/dashboard/framework/hooks/direct_query_hook.tsx index a2b05f47e9ee..fa65e0da1ffc 100644 --- a/src/plugins/dashboard/framework/hooks/direct_query_hook.tsx +++ b/src/plugins/dashboard/framework/hooks/direct_query_hook.tsx @@ -19,7 +19,7 @@ export const useDirectQuery = ( ) => { const sqlService = new SQLService(http); const [loadStatus, setLoadStatus] = useState( - DirectQueryLoadingStatus.SCHEDULED + DirectQueryLoadingStatus.SUCCESS ); const { diff --git a/src/plugins/dashboard/framework/types.tsx b/src/plugins/dashboard/framework/types.tsx index 34174a6c4b0e..7a0d822191da 100644 --- a/src/plugins/dashboard/framework/types.tsx +++ b/src/plugins/dashboard/framework/types.tsx @@ -9,8 +9,9 @@ export enum DirectQueryLoadingStatus { SUCCESS = 'success', FAILED = 'failed', RUNNING = 'running', + SUBMITTED = 'submitted', SCHEDULED = 'scheduled', - CANCELED = 'canceled', + CANCELLED = 'cancelled', WAITING = 'waiting', INITIAL = 'initial', } diff --git a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx index 0224ced3f613..be68bc3848f1 100644 --- a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx +++ b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx @@ -41,7 +41,7 @@ import { Subscription } from 'rxjs'; import ReactGridLayout, { Layout, ReactGridLayoutProps } from 'react-grid-layout'; import type { SavedObjectsClientContract } from 'src/core/public'; import { HttpStart, NotificationsStart } from 'src/core/public'; -import { EuiButton } from '@elastic/eui'; +import { EuiButton, EuiFlexGroup, EuiFlexItem, EuiProgress, EuiText } from '@elastic/eui'; import { ViewMode, EmbeddableChildPanel, EmbeddableStart } from '../../../../../embeddable/public'; import { GridData } from '../../../../common'; import { DASHBOARD_GRID_COLUMN_COUNT, DASHBOARD_GRID_HEIGHT } from '../dashboard_constants'; @@ -463,6 +463,22 @@ class DashboardGridUi extends React.Component { }); } + // TODO find a home for this + EMR_STATES: Map = new Map( + Object.entries({ + submitted: { ord: 10, terminal: false, color: 'vis0' }, + queued: { ord: 20, terminal: false, color: 'vis0' }, + pending: { ord: 30, terminal: false, color: 'vis0' }, + scheduled: { ord: 40, terminal: false, color: 'vis0' }, + running: { ord: 50, terminal: false, color: 'vis0' }, + cancelling: { ord: 50, terminal: false, color: 'vis0' }, + success: { ord: 100, terminal: true, color: 'success' }, + failed: { ord: 100, terminal: true, color: 'danger' }, + cancelled: { ord: 100, terminal: true, color: 'subdued' }, + }) + ); + MAX_ORD: number = 100; + public render() { if (this.state.isLayoutInvalid) { return null; @@ -470,19 +486,30 @@ class DashboardGridUi extends React.Component { const { viewMode } = this.state; const isViewMode = viewMode === ViewMode.VIEW; + const state = this.EMR_STATES.get(this.props.loadStatus as string)!; + return (
{/* Top-left corner "Synchronize Now" button */} -
+
Synchronize Now +
Date: Wed, 16 Apr 2025 10:56:51 -0700 Subject: [PATCH 18/86] CLEAN UP THE UNNECESSARY SERVER SETUP Signed-off-by: Jialiang Liang --- ...pensearch_data_source_management_plugin.ts | 50 --- .../server/adaptors/ppl_datasource.ts | 84 ----- .../dashboard/server/adaptors/ppl_plugin.ts | 93 ----- .../dashboard/server/common/types/index.ts | 23 -- src/plugins/dashboard/server/plugin.ts | 11 +- .../server/routes/data_connections_router.ts | 344 ------------------ .../server/routes/datasources_router.ts | 122 ------- src/plugins/dashboard/server/routes/dsl.ts | 241 ------------ src/plugins/dashboard/server/routes/index.ts | 54 --- src/plugins/dashboard/server/routes/ppl.ts | 38 -- .../server/services/facets/dsl_facet.ts | 37 -- .../server/services/facets/ppl_facet.ts | 43 --- 12 files changed, 2 insertions(+), 1138 deletions(-) delete mode 100644 src/plugins/dashboard/server/adaptors/opensearch_data_source_management_plugin.ts delete mode 100644 src/plugins/dashboard/server/adaptors/ppl_datasource.ts delete mode 100644 src/plugins/dashboard/server/adaptors/ppl_plugin.ts delete mode 100644 src/plugins/dashboard/server/common/types/index.ts delete mode 100644 src/plugins/dashboard/server/routes/data_connections_router.ts delete mode 100644 src/plugins/dashboard/server/routes/datasources_router.ts delete mode 100644 src/plugins/dashboard/server/routes/dsl.ts delete mode 100644 src/plugins/dashboard/server/routes/index.ts delete mode 100644 src/plugins/dashboard/server/routes/ppl.ts delete mode 100644 src/plugins/dashboard/server/services/facets/dsl_facet.ts delete mode 100644 src/plugins/dashboard/server/services/facets/ppl_facet.ts diff --git a/src/plugins/dashboard/server/adaptors/opensearch_data_source_management_plugin.ts b/src/plugins/dashboard/server/adaptors/opensearch_data_source_management_plugin.ts deleted file mode 100644 index 4d694a296d5a..000000000000 --- a/src/plugins/dashboard/server/adaptors/opensearch_data_source_management_plugin.ts +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import { JOBS_ENDPOINT_BASE } from '../../framework/utils/shared'; - -export function OpenSearchDataSourceManagementPlugin(Client: any, config: any, components: any) { - const clientAction = components.clientAction.factory; - - Client.prototype.datasourcemanagement = components.clientAction.namespaceFactory(); - const datasourcemanagement = Client.prototype.datasourcemanagement.prototype; - - // Get async job status - datasourcemanagement.getJobStatus = clientAction({ - url: { - fmt: `${JOBS_ENDPOINT_BASE}/<%=queryId%>`, - req: { - queryId: { - type: 'string', - required: true, - }, - }, - }, - method: 'GET', - }); - - // Delete async job - datasourcemanagement.deleteJob = clientAction({ - url: { - fmt: `${JOBS_ENDPOINT_BASE}/<%=queryId%>`, - req: { - queryId: { - type: 'string', - required: true, - }, - }, - }, - method: 'DELETE', - }); - - // Run async job - datasourcemanagement.runDirectQuery = clientAction({ - url: { - fmt: `${JOBS_ENDPOINT_BASE}`, - }, - method: 'POST', - needBody: true, - }); -} diff --git a/src/plugins/dashboard/server/adaptors/ppl_datasource.ts b/src/plugins/dashboard/server/adaptors/ppl_datasource.ts deleted file mode 100644 index 38cacbbe0ac7..000000000000 --- a/src/plugins/dashboard/server/adaptors/ppl_datasource.ts +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import _ from 'lodash'; -import { IPPLEventsDataSource, IPPLVisualizationDataSource } from '../common/types'; - -type PPLResponse = IPPLEventsDataSource & IPPLVisualizationDataSource; - -export class PPLDataSource { - constructor(private pplDataSource: PPLResponse, private dataType: string) { - if (this.dataType === 'jdbc') { - this.addSchemaRowMapping(); - } else if (this.dataType === 'viz') { - this.addStatsMapping(); - } - } - - private addStatsMapping = () => { - const visData = this.pplDataSource; - - /** - * Add vis mapping for runtime fields - * json data structure added to response will be - * [{ - * agent: "mozilla", - * avg(bytes): 5756 - * ... - * }, { - * agent: "MSIE", - * avg(bytes): 5605 - * ... - * }, { - * agent: "chrome", - * avg(bytes): 5648 - * ... - * }] - */ - const res = []; - if (visData?.metadata?.fields) { - const queriedFields = visData.metadata.fields; - for (let i = 0; i < visData.size; i++) { - const entry: any = {}; - queriedFields.map((field: any) => { - const statsDataSet = visData?.data; - entry[field.name] = statsDataSet[field.name][i]; - }); - res.push(entry); - } - visData.jsonData = res; - } - }; - - /** - * Add 'schemaName: data' entries for UI rendering - */ - private addSchemaRowMapping = () => { - const pplRes = this.pplDataSource; - - const data: any[] = []; - - _.forEach(pplRes.datarows, (row) => { - const record: any = {}; - - for (let i = 0; i < pplRes.schema.length; i++) { - const cur = pplRes.schema[i]; - - if (typeof row[i] === 'object') { - record[cur.name] = JSON.stringify(row[i]); - } else if (typeof row[i] === 'boolean') { - record[cur.name] = row[i].toString(); - } else { - record[cur.name] = row[i]; - } - } - - data.push(record); - }); - pplRes.jsonData = data; - }; - - public getDataSource = (): PPLResponse => this.pplDataSource; -} diff --git a/src/plugins/dashboard/server/adaptors/ppl_plugin.ts b/src/plugins/dashboard/server/adaptors/ppl_plugin.ts deleted file mode 100644 index 7ef02f89d2ba..000000000000 --- a/src/plugins/dashboard/server/adaptors/ppl_plugin.ts +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import { - OPENSEARCH_DATACONNECTIONS_API, - PPL_ENDPOINT, - SQL_ENDPOINT, -} from '../../framework/utils/shared'; - -export const PPLPlugin = function (Client, config, components) { - const ca = components.clientAction.factory; - Client.prototype.ppl = components.clientAction.namespaceFactory(); - const ppl = Client.prototype.ppl.prototype; - - ppl.pplQuery = ca({ - url: { - fmt: `${PPL_ENDPOINT}`, - params: { - format: { - type: 'string', - required: true, - }, - }, - }, - needBody: true, - method: 'POST', - }); - - ppl.sqlQuery = ca({ - url: { - fmt: `${SQL_ENDPOINT}`, - params: { - format: { - type: 'string', - required: true, - }, - }, - }, - needBody: true, - method: 'POST', - }); - - ppl.getDataConnectionById = ca({ - url: { - fmt: `${OPENSEARCH_DATACONNECTIONS_API.DATACONNECTION}/<%=dataconnection%>`, - req: { - dataconnection: { - type: 'string', - required: true, - }, - }, - }, - method: 'GET', - }); - - ppl.deleteDataConnection = ca({ - url: { - fmt: `${OPENSEARCH_DATACONNECTIONS_API.DATACONNECTION}/<%=dataconnection%>`, - req: { - dataconnection: { - type: 'string', - required: true, - }, - }, - }, - method: 'DELETE', - }); - - ppl.createDataSource = ca({ - url: { - fmt: `${OPENSEARCH_DATACONNECTIONS_API.DATACONNECTION}`, - }, - needBody: true, - method: 'POST', - }); - - ppl.modifyDataConnection = ca({ - url: { - fmt: `${OPENSEARCH_DATACONNECTIONS_API.DATACONNECTION}`, - }, - needBody: true, - method: 'PATCH', - }); - - ppl.getDataConnections = ca({ - url: { - fmt: `${OPENSEARCH_DATACONNECTIONS_API.DATACONNECTION}`, - }, - method: 'GET', - }); -}; diff --git a/src/plugins/dashboard/server/common/types/index.ts b/src/plugins/dashboard/server/common/types/index.ts deleted file mode 100644 index 3d6525d8f5f1..000000000000 --- a/src/plugins/dashboard/server/common/types/index.ts +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -export interface ISchema { - name: string; - type: string; -} - -export interface IPPLVisualizationDataSource { - data: any; - metadata: any; - jsonData?: any[]; - size: number; - status: number; -} - -export interface IPPLEventsDataSource { - schema: ISchema[]; - datarows: any[]; - jsonData?: any[]; -} diff --git a/src/plugins/dashboard/server/plugin.ts b/src/plugins/dashboard/server/plugin.ts index 087c528d9342..4e377e24bbce 100644 --- a/src/plugins/dashboard/server/plugin.ts +++ b/src/plugins/dashboard/server/plugin.ts @@ -34,13 +34,12 @@ import { CoreStart, Plugin, Logger, - ILegacyClusterClient, } from '../../../core/server'; import { dashboardSavedObjectType } from './saved_objects'; import { capabilitiesProvider } from './capabilities_provider'; + import { DashboardPluginSetup, DashboardPluginStart } from './types'; -import { setupRoutes } from './routes'; // <-- We'll define this next export class DashboardPlugin implements Plugin { private readonly logger: Logger; @@ -49,7 +48,7 @@ export class DashboardPlugin implements Plugin => { - try { - const dataConnectionsresponse = await context.opensearch_dashboard_plugin.dashboardPluginClient - .asScoped(request) - .callAsCurrentUser('ppl.getDataConnectionById', { - dataconnection: request.params.name, - }); - return response.ok({ - body: dataConnectionsresponse, - }); - } catch (error: any) { - console.error('Issue in fetching data connection:', error); - return response.custom({ - statusCode: error.statusCode || 500, - body: error.message, - }); - } - } - ); - - router.delete( - { - path: `${DATACONNECTIONS_BASE}/{name}`, - validate: { - params: schema.object({ - name: schema.string(), - }), - }, - }, - async (context, request, response): Promise => { - try { - const dataConnectionsresponse = await context.opensearch_dashboard_plugin.dashboardPluginClient - .asScoped(request) - .callAsCurrentUser('ppl.deleteDataConnection', { - dataconnection: request.params.name, - }); - return response.ok({ - body: dataConnectionsresponse, - }); - } catch (error: any) { - console.error('Issue in deleting data connection:', error); - return response.custom({ - statusCode: error.statusCode || 500, - body: error.message, - }); - } - } - ); - - router.post( - { - path: `${DATACONNECTIONS_BASE}${EDIT}`, - validate: { - body: schema.object({ - name: schema.string(), - allowedRoles: schema.arrayOf(schema.string()), - }), - }, - }, - async (context, request, response): Promise => { - try { - const dataConnectionsresponse = await context.opensearch_dashboard_plugin.dashboardPluginClient - .asScoped(request) - .callAsCurrentUser('ppl.modifyDataConnection', { - body: { - name: request.body.name, - allowedRoles: request.body.allowedRoles, - }, - }); - return response.ok({ - body: dataConnectionsresponse, - }); - } catch (error: any) { - console.error('Issue in modifying data connection:', error); - return response.custom({ - statusCode: error.statusCode || 500, - body: error.message, - }); - } - } - ); - - router.post( - { - path: `${DATACONNECTIONS_BASE}${EDIT}${DATACONNECTIONS_UPDATE_STATUS}`, - validate: { - body: schema.object({ - name: schema.string(), - status: schema.string(), - }), - }, - }, - async (context, request, response): Promise => { - try { - const dataConnectionsresponse = await context.opensearch_dashboard_plugin.dashboardPluginClient - .asScoped(request) - .callAsCurrentUser('ppl.modifyDataConnection', { - body: { - name: request.body.name, - status: request.body.status, - }, - }); - return response.ok({ - body: dataConnectionsresponse, - }); - } catch (error: any) { - console.error('Issue in modifying data connection:', error); - return response.custom({ - statusCode: error.statusCode || 500, - body: error.message, - }); - } - } - ); - - router.post( - { - path: `${DATACONNECTIONS_BASE}`, - validate: { - body: schema.object({ - name: schema.string(), - connector: schema.string(), - allowedRoles: schema.arrayOf(schema.string()), - properties: schema.any(), - }), - }, - }, - async ( - context, - request, - response - ): Promise> => { - try { - const dataConnectionsresponse = await context.opensearch_dashboard_plugin.dashboardPluginClient - .asScoped(request) - .callAsCurrentUser('ppl.createDataSource', { - body: { - name: request.body.name, - connector: request.body.connector, - allowedRoles: request.body.allowedRoles, - properties: request.body.properties, - }, - }); - return response.ok({ - body: dataConnectionsresponse, - }); - } catch (error: any) { - console.error('Issue in creating data source:', error); - return response.custom({ - statusCode: error.statusCode || 500, - body: error.response, - }); - } - } - ); - - router.get( - { - path: `${DATACONNECTIONS_BASE}`, - validate: false, - }, - async (context, request, response): Promise => { - try { - const dataConnectionsresponse = await context.opensearch_dashboard_plugin.dashboardPluginClient - .asScoped(request) - .callAsCurrentUser('ppl.getDataConnections'); - return response.ok({ - body: dataConnectionsresponse, - }); - } catch (error: any) { - console.error('Issue in fetching data sources:', error); - return response.custom({ - statusCode: error.statusCode || 500, - body: error.response, - }); - } - } - ); -} - -export function registerDataConnectionsRoute(router: IRouter, dataSourceEnabled: boolean) { - router.get( - { - path: `${DATACONNECTIONS_BASE}/dataSourceMDSId={dataSourceMDSId?}`, - validate: { - params: schema.object({ - dataSourceMDSId: schema.maybe(schema.string({ defaultValue: '' })), - }), - }, - }, - async (context, request, response): Promise => { - const dataSourceMDSId = request.params.dataSourceMDSId; - try { - let dataConnectionsresponse; - if (dataSourceEnabled && dataSourceMDSId) { - const client = await context.dataSource.opensearch.legacy.getClient(dataSourceMDSId); - dataConnectionsresponse = await client.callAPI('ppl.getDataConnections', { - requestTimeout: 5000, // Enforce timeout to avoid hanging requests - }); - } else { - dataConnectionsresponse = await context.opensearch_dashboard_plugin.dashboardPluginClient - .asScoped(request) - .callAsCurrentUser('ppl.getDataConnections'); - } - return response.ok({ - body: dataConnectionsresponse, - }); - } catch (error: any) { - console.error('Issue in fetching data sources:', error); - const statusCode = error.statusCode || error.body?.statusCode || 500; - const errorBody = error.body || - error.response || { message: error.message || 'Unknown error occurred' }; - - return response.custom({ - statusCode, - body: { - error: errorBody, - message: errorBody.message || error.message, - }, - }); - } - } - ); - - router.get( - { - path: `${DATACONNECTIONS_BASE}/{name}/dataSourceMDSId={dataSourceMDSId?}`, - validate: { - params: schema.object({ - name: schema.string(), - dataSourceMDSId: schema.maybe(schema.string({ defaultValue: '' })), - }), - }, - }, - async (context, request, response): Promise => { - const dataSourceMDSId = request.params.dataSourceMDSId; - try { - let dataConnectionsresponse; - if (dataSourceEnabled && dataSourceMDSId) { - const client = await context.dataSource.opensearch.legacy.getClient(dataSourceMDSId); - dataConnectionsresponse = await client.callAPI('ppl.getDataConnectionById', { - dataconnection: request.params.name, - }); - } else { - dataConnectionsresponse = await context.opensearch_dashboard_plugin.dashboardPluginClient - .asScoped(request) - .callAsCurrentUser('ppl.getDataConnectionById', { - dataconnection: request.params.name, - }); - } - return response.ok({ - body: dataConnectionsresponse, - }); - } catch (error: any) { - console.error('Issue in fetching data sources:', error); - const statusCode = error.statusCode || error.body?.statusCode || 500; - const errorBody = error.body || - error.response || { message: error.message || 'Unknown error occurred' }; - - return response.custom({ - statusCode, - body: { - error: errorBody, - message: errorBody.message || error.message, - }, - }); - } - } - ); - - router.delete( - { - path: `${DATACONNECTIONS_BASE}/{name}/dataSourceMDSId={dataSourceMDSId?}`, - validate: { - params: schema.object({ - name: schema.string(), - dataSourceMDSId: schema.maybe(schema.string({ defaultValue: '' })), - }), - }, - }, - async (context, request, response): Promise => { - const dataSourceMDSId = request.params.dataSourceMDSId; - try { - let dataConnectionsresponse; - if (dataSourceEnabled && dataSourceMDSId) { - const client = await context.dataSource.opensearch.legacy.getClient(dataSourceMDSId); - dataConnectionsresponse = await client.callAPI('ppl.deleteDataConnection', { - dataconnection: request.params.name, - }); - } else { - dataConnectionsresponse = await context.opensearch_dashboard_plugin.dashboardPluginClient - .asScoped(request) - .callAsCurrentUser('ppl.deleteDataConnection', { - dataconnection: request.params.name, - }); - } - return response.ok({ - body: dataConnectionsresponse, - }); - } catch (error: any) { - console.error('Issue in deleting data sources:', error); - const statusCode = error.statusCode || error.body?.statusCode || 500; - const errorBody = error.body || - error.response || { message: error.message || 'Unknown error occurred' }; - - return response.custom({ - statusCode, - body: { - error: errorBody, - message: errorBody.message || error.message, - }, - }); - } - } - ); -} diff --git a/src/plugins/dashboard/server/routes/datasources_router.ts b/src/plugins/dashboard/server/routes/datasources_router.ts deleted file mode 100644 index be6682d3faa0..000000000000 --- a/src/plugins/dashboard/server/routes/datasources_router.ts +++ /dev/null @@ -1,122 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -/* eslint-disable no-console*/ -import { schema } from '@osd/config-schema'; -import { IRouter } from '../../../../../src/core/server'; -import { JOBS_BASE, DSM_BASE } from '../../framework/utils/shared'; - -export function registerDatasourcesRoute(router: IRouter, dataSourceEnabled: boolean) { - router.post( - { - path: `${DSM_BASE}${JOBS_BASE}`, - validate: { - body: schema.object({ - query: schema.string(), - lang: schema.string(), - datasource: schema.string(), - sessionId: schema.maybe(schema.string()), - }), - }, - }, - async (context, request, response): Promise => { - const dataSourceMDSId = request.url.searchParams.get('dataSourceMDSId'); - const params = { - body: { - ...request.body, - }, - }; - try { - let res; - if (dataSourceEnabled && dataSourceMDSId) { - const client = await context.dataSource.opensearch.legacy.getClient(dataSourceMDSId); - res = await client.callAPI('datasourcemanagement.runDirectQuery', params); - } else { - res = await context.opensearch_dashboard_plugin.dashboardPluginClient - .asScoped(request) - .callAsCurrentUser('datasourcemanagement.runDirectQuery', params); - } - return response.ok({ - body: res, - }); - } catch (error: any) { - console.error('Error in running direct query:', error); - return response.custom({ - statusCode: error.statusCode || 500, - body: error.body, - }); - } - } - ); - - router.get( - { - path: `${DSM_BASE}${JOBS_BASE}/{queryId}/{dataSourceMDSId?}`, - validate: { - params: schema.object({ - queryId: schema.string(), - dataSourceMDSId: schema.maybe(schema.string({ defaultValue: '' })), - }), - }, - }, - async (context, request, response): Promise => { - try { - let res; - if (dataSourceEnabled && request.params.dataSourceMDSId) { - const client = await context.dataSource.opensearch.legacy.getClient( - request.params.dataSourceMDSId - ); - res = await client.callAPI('datasourcemanagement.getJobStatus', { - queryId: request.params.queryId, - }); - } else { - res = await context.opensearch_dashboard_plugin.dashboardPluginClient - .asScoped(request) - .callAsCurrentUser('datasourcemanagement.getJobStatus', { - queryId: request.params.queryId, - }); - } - return response.ok({ - body: res, - }); - } catch (error: any) { - console.error('Error in fetching job status:', error); - return response.custom({ - statusCode: error.statusCode || 500, - body: error.message, - }); - } - } - ); - - router.delete( - { - path: `${DSM_BASE}${JOBS_BASE}/{queryId}`, - validate: { - params: schema.object({ - queryId: schema.string(), - }), - }, - }, - async (context, request, response): Promise => { - try { - const res = await context.opensearch_dashboard_plugin.dashboardPluginClient - .asScoped(request) - .callAsCurrentUser('datasourcemanagement.deleteJob', { - queryId: request.params.queryId, - }); - return response.ok({ - body: res, - }); - } catch (error: any) { - console.error('Error in deleting job:', error); - return response.custom({ - statusCode: error.statusCode || 500, - body: error.message, - }); - } - } - ); -} diff --git a/src/plugins/dashboard/server/routes/dsl.ts b/src/plugins/dashboard/server/routes/dsl.ts deleted file mode 100644 index 43ef3f3de157..000000000000 --- a/src/plugins/dashboard/server/routes/dsl.ts +++ /dev/null @@ -1,241 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -/* eslint-disable no-console*/ -import { schema } from '@osd/config-schema'; -import { RequestParams } from '@elastic/elasticsearch'; -import { IRouter } from '../../../../core/server'; -import { DSLFacet } from '../services/facets/dsl_facet'; -import { - DSL_BASE, - DSL_SEARCH, - DSL_CAT, - DSL_MAPPING, - DSL_SETTINGS, -} from '../../framework/utils/shared'; - -export function registerDslRoute( - { router }: { router: IRouter; facet: DSLFacet }, - dataSourceEnabled: boolean -) { - router.post( - { - path: `${DSL_BASE}${DSL_SEARCH}`, - validate: { body: schema.any() }, - }, - async (context, request, response) => { - const { index, size, ...rest } = request.body; - const params: RequestParams.Search = { - index, - size, - body: rest, - }; - try { - const resp = await context.core.opensearch.legacy.client.callAsCurrentUser( - 'search', - params - ); - return response.ok({ - body: resp, - }); - } catch (error) { - if (error.statusCode !== 404) console.error(error); - return response.custom({ - statusCode: error.statusCode || 500, - body: error.message, - }); - } - } - ); - - router.get( - { - path: `${DSL_BASE}${DSL_CAT}`, - validate: { - query: schema.object({ - format: schema.string(), - index: schema.maybe(schema.string()), - }), - }, - }, - async (context, request, response) => { - try { - const resp = await context.core.opensearch.legacy.client.callAsCurrentUser( - 'cat.indices', - request.query - ); - return response.ok({ - body: resp, - }); - } catch (error) { - if (error.statusCode !== 404) console.error(error); - return response.custom({ - statusCode: error.statusCode || 500, - body: error.message, - }); - } - } - ); - - router.get( - { - path: `${DSL_BASE}${DSL_MAPPING}`, - validate: { query: schema.any() }, - }, - async (context, request, response) => { - try { - const resp = await context.core.opensearch.legacy.client.callAsCurrentUser( - 'indices.getMapping', - { index: request.query.index } - ); - return response.ok({ - body: resp, - }); - } catch (error) { - if (error.statusCode !== 404) console.error(error); - return response.custom({ - statusCode: error.statusCode || 500, - body: error.message, - }); - } - } - ); - - router.get( - { - path: `${DSL_BASE}${DSL_SETTINGS}`, - validate: { query: schema.any() }, - }, - async (context, request, response) => { - try { - const resp = await context.core.opensearch.legacy.client.callAsCurrentUser( - 'indices.getSettings', - { index: request.query.index } - ); - return response.ok({ - body: resp, - }); - } catch (error) { - if (error.statusCode !== 404) console.error(error); - return response.custom({ - statusCode: error.statusCode || 500, - body: error.message, - }); - } - } - ); - - // New routes for mds enabled - router.get( - { - path: `${DSL_BASE}${DSL_CAT}/dataSourceMDSId={dataSourceMDSId?}`, - validate: { - query: schema.object({ - format: schema.string(), - index: schema.maybe(schema.string()), - }), - params: schema.object({ - dataSourceMDSId: schema.maybe(schema.string({ defaultValue: '' })), - }), - }, - }, - async (context, request, response) => { - const dataSourceMDSId = request.params.dataSourceMDSId; - try { - let resp; - if (dataSourceEnabled && dataSourceMDSId) { - const client = await context.dataSource.opensearch.legacy.getClient(dataSourceMDSId); - resp = await client.callAPI('cat.indices', request.query); - } else { - resp = await context.core.opensearch.legacy.client.callAsCurrentUser( - 'cat.indices', - request.query - ); - } - return response.ok({ - body: resp, - }); - } catch (error) { - if (error.statusCode !== 404) console.error(error); - return response.custom({ - statusCode: error.statusCode || 500, - body: error.message, - }); - } - } - ); - - router.get( - { - path: `${DSL_BASE}${DSL_MAPPING}/dataSourceMDSId={dataSourceMDSId?}`, - validate: { - query: schema.any(), - params: schema.object({ - dataSourceMDSId: schema.maybe(schema.string({ defaultValue: '' })), - }), - }, - }, - async (context, request, response) => { - const dataSourceMDSId = request.params.dataSourceMDSId; - try { - let resp; - if (dataSourceEnabled && dataSourceMDSId) { - const client = await context.dataSource.opensearch.legacy.getClient(dataSourceMDSId); - resp = await client.callAPI('indices.getMapping', { index: request.query.index }); - } else { - resp = await context.core.opensearch.legacy.client.callAsCurrentUser( - 'indices.getMapping', - { index: request.query.index } - ); - } - return response.ok({ - body: resp, - }); - } catch (error) { - if (error.statusCode !== 404) console.error(error); - return response.custom({ - statusCode: error.statusCode || 500, - body: error.message, - }); - } - } - ); - - router.get( - { - path: `${DSL_BASE}${DSL_SETTINGS}/dataSourceMDSId={dataSourceMDSId?}`, - validate: { - query: schema.any(), - params: schema.object({ - dataSourceMDSId: schema.maybe(schema.string({ defaultValue: '' })), - }), - }, - }, - async (context, request, response) => { - const dataSourceMDSId = request.params.dataSourceMDSId; - try { - let resp; - if (dataSourceEnabled && dataSourceMDSId) { - const client = await context.dataSource.opensearch.legacy.getClient(dataSourceMDSId); - resp = await client.callAPI('indices.getSettings', { index: request.query.index }); - } else { - resp = await context.core.opensearch.legacy.client.callAsCurrentUser( - 'indices.getSettings', - { index: request.query.index } - ); - } - return response.ok({ - body: resp, - }); - } catch (error) { - if (error.statusCode !== 404) console.error(error); - return response.custom({ - statusCode: error.statusCode || 500, - body: error.message, - }); - } - } - ); -} diff --git a/src/plugins/dashboard/server/routes/index.ts b/src/plugins/dashboard/server/routes/index.ts deleted file mode 100644 index 2bab16798262..000000000000 --- a/src/plugins/dashboard/server/routes/index.ts +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import { IRouter, ILegacyClusterClient } from '../../../../core/server'; -import { registerDslRoute } from './dsl'; -import { - registerDataConnectionsRoute, - registerNonMdsDataConnectionsRoute, -} from './data_connections_router'; -import { registerDatasourcesRoute } from './datasources_router'; -import { registerPplRoute } from './ppl'; -import { DSLFacet } from '../services/facets/dsl_facet'; -import { PPLFacet } from '../services/facets/ppl_facet'; - -export function defineRoutes(router: IRouter) { - router.get( - { - path: '/api/data_source_management/example', - validate: false, - }, - async (context, request, response) => { - return response.ok({ - body: { - time: new Date().toISOString(), - }, - }); - } - ); -} - -export function setupRoutes({ - router, - client, - dataSourceEnabled, -}: { - router: IRouter; - client: ILegacyClusterClient; - dataSourceEnabled: boolean; -}) { - registerPplRoute({ router, facet: new PPLFacet(client) }); - registerDslRoute({ router, facet: new DSLFacet(client) }, dataSourceEnabled); - - // notebooks routes - // const queryService = new QueryService(client); - // registerSqlRoute(router, queryService); - - if (!dataSourceEnabled) { - registerNonMdsDataConnectionsRoute(router); - } - registerDataConnectionsRoute(router, dataSourceEnabled); - registerDatasourcesRoute(router, dataSourceEnabled); -} diff --git a/src/plugins/dashboard/server/routes/ppl.ts b/src/plugins/dashboard/server/routes/ppl.ts deleted file mode 100644 index d94f1379c3b0..000000000000 --- a/src/plugins/dashboard/server/routes/ppl.ts +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import { schema } from '@osd/config-schema'; -import { IRouter, IOpenSearchDashboardsResponse, ResponseError } from './../../../../core/server'; -import { PPLFacet } from '../services/facets/ppl_facet'; -import { PPL_BASE, PPL_SEARCH } from '../../framework/utils/shared'; - -export function registerPplRoute({ router, facet }: { router: IRouter; facet: PPLFacet }) { - router.post( - { - path: `${PPL_BASE}${PPL_SEARCH}`, - validate: { - body: schema.object({ - query: schema.string(), - format: schema.string(), - }), - }, - }, - async (context, req, res): Promise> => { - const queryRes: any = await facet.describeQuery(req); - if (queryRes.success) { - const result: any = { - body: { - ...queryRes.data, - }, - }; - return res.ok(result); - } - return res.custom({ - statusCode: queryRes.data.statusCode || queryRes.data.status || 500, - body: queryRes.data.body || queryRes.data.message || '', - }); - } - ); -} diff --git a/src/plugins/dashboard/server/services/facets/dsl_facet.ts b/src/plugins/dashboard/server/services/facets/dsl_facet.ts deleted file mode 100644 index 61e779402861..000000000000 --- a/src/plugins/dashboard/server/services/facets/dsl_facet.ts +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -/* eslint-disable no-console*/ -import _ from 'lodash'; - -export class DSLFacet { - constructor(private client: any) { - this.client = client; - } - - private fetch = async (request: any, format: string, responseFormat: string) => { - const res = { - success: false, - data: {}, - }; - try { - const params = { - query: JSON.stringify(request.body), - }; - const queryRes = await this.client.asScoped(request).callAsCurrentUser(format, params); - const dslDataSource = queryRes; - res.success = true; - res.data = dslDataSource; - } catch (err: any) { - console.error(err); - res.data = err.body; - } - return res; - }; - - describeQuery = async (request: any) => { - return this.fetch(request, 'dsl.dslQuery', 'json'); - }; -} diff --git a/src/plugins/dashboard/server/services/facets/ppl_facet.ts b/src/plugins/dashboard/server/services/facets/ppl_facet.ts deleted file mode 100644 index f662d0f25684..000000000000 --- a/src/plugins/dashboard/server/services/facets/ppl_facet.ts +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -/* eslint-disable no-console */ -import _ from 'lodash'; -import { PPLDataSource } from '../../adaptors/ppl_datasource'; - -export class PPLFacet { - constructor(private client: any) { - this.client = client; - } - - private fetch = async (request: any, format: string, responseFormat: string) => { - const res = { - success: false, - data: {}, - }; - try { - const params = { - body: { - query: request.body.query, - }, - }; - if (request.body.format !== 'jdbc') { - params.format = request.body.format; - } - const queryRes = await this.client.asScoped(request).callAsCurrentUser(format, params); - const pplDataSource = new PPLDataSource(queryRes, request.body.format); - res.success = true; - res.data = pplDataSource.getDataSource(); - } catch (err: any) { - console.error('PPL query fetch err: ', err); - res.data = err; - } - return res; - }; - - describeQuery = async (request: any) => { - return this.fetch(request, 'ppl.pplQuery', 'json'); - }; -} From 85c7a841c51836b87bb49d30acd045533bce1385 Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Wed, 16 Apr 2025 14:37:22 -0700 Subject: [PATCH 19/86] refactor - extract the logic for sync out of grid class Signed-off-by: Jialiang Liang --- .../embeddable/grid/dashboard_grid.tsx | 151 ++++-------------- .../viewport/dashboard_viewport.tsx | 1 - .../direct_query_sync/direct_query_sync.ts | 82 ++++++++++ 3 files changed, 109 insertions(+), 125 deletions(-) create mode 100644 src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.ts diff --git a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx index be68bc3848f1..5c1cbf49ad6a 100644 --- a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx +++ b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx @@ -51,6 +51,10 @@ import { DashboardContainerInput } from '../dashboard_container'; import { DashboardContainer, DashboardReactContextValue } from '../dashboard_container'; import { DirectQueryLoadingStatus } from '../../../../framework/types'; import { DirectQueryRequest } from '../../../../framework/types'; +import { + extractIndexInfoFromDashboard, + generateRefreshQuery, +} from '../../utils/direct_query_sync/direct_query_sync'; let lastValidGridSize = 0; @@ -104,9 +108,9 @@ function ResponsiveGrid({ width={lastValidGridSize} className={classes} isDraggable={true} - isResizable={true} // There is a bug with d3 + firefox + elements using transforms. // See https://github.com/elastic/kibana/issues/16870 for more context. + isResizable={true} useCSSTransforms={false} margin={[MARGINS, MARGINS]} cols={DASHBOARD_GRID_COLUMN_COUNT} @@ -135,8 +139,6 @@ export interface DashboardGridProps extends ReactIntl.InjectedIntlProps { savedObjectsClient: SavedObjectsClientContract; http: HttpStart; notifications: NotificationsStart; - - // direct query loading startLoading: (payload: DirectQueryRequest) => void; loadStatus: DirectQueryLoadingStatus; pollingResult: any; @@ -190,7 +192,8 @@ class DashboardGridUi extends React.Component { try { layout = this.buildLayoutFromPanels(); } catch (error: any) { - console.error(error); + console.error(error); // eslint-disable-line no-console + isLayoutInvalid = true; this.props.opensearchDashboards.notifications.toasts.danger({ title: this.props.intl.formatMessage({ @@ -272,141 +275,41 @@ class DashboardGridUi extends React.Component { * Runs on mount and when the container input (panels) changes. */ private async collectAllPanelMetadata() { - const panels = this.state.panels; - - const panelDataPromises = Object.keys(panels).map(async (panelId) => { - const panel = panels[panelId]; - const panelEmbeddable = await this.props.container.untilEmbeddableLoaded(panelId); - const embeddableInput = panelEmbeddable.getInput() as any; - console.log(`Embeddable input for panel ${panelId}:`, embeddableInput); - const savedObjectId = embeddableInput.savedObjectId || 'unknown'; - - if (!savedObjectId || savedObjectId === 'unknown') { - console.log(`No valid savedObjectId for panel ${panelId}`); - return { panelId, savedObjectId: 'unknown', type: 'unknown' }; - } - - try { - const savedObject = await this.props.savedObjectsClient.get(panel.type, savedObjectId); - - console.log(`Saved object for ${savedObjectId}:`, savedObject); - const visState = savedObject.attributes.visState - ? JSON.parse(savedObject.attributes.visState) - : {}; - const visType = visState.type || savedObject.attributes.type || 'unknown'; - return { panelId, savedObjectId, type: visType }; - } catch (error) { - console.error(`Error fetching saved object for ${savedObjectId}:`, error); - return { panelId, savedObjectId, type: 'unknown' }; - } - }); + const indexInfo = await extractIndexInfoFromDashboard( + this.state.panels, + this.props.savedObjectsClient, + this.props.http + ); - const panelMetadata = await Promise.all(panelDataPromises); - this.setState({ panelMetadata }, async () => { - console.log('All Panel Metadata:', this.state.panelMetadata); - - // POC ONLY: Extract first pie chart savedObjectId - const firstPie = this.state.panelMetadata.find((meta) => meta.type === 'pie'); - if (firstPie) { - console.log('First pie visualization savedObjectId:', firstPie.savedObjectId); - - try { - // POC ONLY: Fetch the full saved object for the pie visualization - const pieSavedObject = await this.props.savedObjectsClient.get( - 'visualization', - firstPie.savedObjectId - ); - console.log('First pie visualization saved object metadata:', pieSavedObject); - - const indexPatternRef = pieSavedObject.references.find( - (ref: any) => ref.type === 'index-pattern' - ); - - if (indexPatternRef) { - try { - const indexPattern = await this.props.savedObjectsClient.get( - 'index-pattern', - indexPatternRef.id - ); - const indexTitle = indexPattern.attributes.title; - console.log('Index pattern title (raw):', indexTitle); - - // If indexTitle contains a wildcard, try to resolve to real indices - if (indexTitle.includes('*')) { - try { - const resolved = await this.props.http.get( - `/internal/index-pattern-management/resolve_index/${encodeURIComponent( - indexTitle - )}` - ); - const matchedIndices = resolved?.indices || []; - console.log('Resolved index pattern to concrete indices:', matchedIndices); - - if (matchedIndices.length > 0) { - // TODO: Handle multiple matches if needed - const firstMatch = matchedIndices[0].name; - const trimmedTitle = firstMatch.replace(/^flint_/, ''); - const parts = trimmedTitle.split('_'); - - this.extractedDatasource = parts[0] || 'unknown'; - this.extractedDatabase = parts[1] || 'unknown'; - this.extractedIndex = parts.slice(2).join('_') || 'unknown'; - - console.log('Resolved from concrete index name:'); - console.log('Datasource:', this.extractedDatasource); - console.log('Database:', this.extractedDatabase); - console.log('Index:', this.extractedIndex); - } else { - console.warn('No concrete indices matched the wildcard pattern.'); - } - } catch (err) { - console.error('Failed to resolve concrete index for pattern:', indexTitle, err); - } - } else { - // Original fallback logic (non-wildcard) - const trimmedTitle = indexTitle.replace(/^flint_/, ''); - const parts = trimmedTitle.split('_'); - - this.extractedDatasource = parts[0] || 'unknown'; - this.extractedDatabase = parts[1] || 'unknown'; - this.extractedIndex = parts.slice(2).join('_') || 'unknown'; - } - } catch (err) { - console.error(`Failed to fetch index pattern ${indexPatternRef.id}:`, err); - } - } else { - console.warn('No index-pattern reference found in the pie visualization saved object.'); - } - } catch (error) { - console.error( - `Error fetching metadata for pie saved object ID ${firstPie.savedObjectId}:`, - error - ); - } - } else { - console.log('No pie visualizations found.'); - } - }); + if (indexInfo) { + this.extractedDatasource = indexInfo.datasource; + this.extractedDatabase = indexInfo.database; + this.extractedIndex = indexInfo.index; + console.log('Resolved index info:', indexInfo); + } else { + console.warn('Could not extract index info from pie visualization.'); + } } synchronizeNow = () => { const { extractedDatasource, extractedDatabase, extractedIndex } = this; - if ( !extractedDatasource || - extractedDatasource === 'unknown' || !extractedDatabase || - extractedDatabase === 'unknown' || !extractedIndex || + extractedDatasource === 'unknown' || + extractedDatabase === 'unknown' || extractedIndex === 'unknown' ) { console.error('Datasource, database, or index not properly set. Cannot run REFRESH command.'); return; } - // POC ONLY: Run REFRESH MATERIALIZED VIEW command - // TODO: Add the logic to check the index type - const query = `REFRESH MATERIALIZED VIEW \`${extractedDatasource}\`.\`${extractedDatabase}\`.\`${extractedIndex}\``; + const query = generateRefreshQuery({ + datasource: extractedDatasource, + database: extractedDatabase, + index: extractedIndex, + }); const queryPayload = { query, diff --git a/src/plugins/dashboard/public/application/embeddable/viewport/dashboard_viewport.tsx b/src/plugins/dashboard/public/application/embeddable/viewport/dashboard_viewport.tsx index 168418b3d7f3..0db4eb77c50c 100644 --- a/src/plugins/dashboard/public/application/embeddable/viewport/dashboard_viewport.tsx +++ b/src/plugins/dashboard/public/application/embeddable/viewport/dashboard_viewport.tsx @@ -213,4 +213,3 @@ export const DashboardViewportWithQuery = ( /> ); }; -// diff --git a/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.ts b/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.ts new file mode 100644 index 000000000000..af285d63acdb --- /dev/null +++ b/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.ts @@ -0,0 +1,82 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { HttpStart, SavedObjectsClientContract } from 'src/core/public'; + +interface IndexExtractionResult { + datasource: string; + database: string; + index: string; +} + +export async function resolveConcreteIndex( + indexTitle: string, + http: HttpStart +): Promise { + if (!indexTitle.includes('*')) return indexTitle; + + try { + const resolved = await http.get( + `/internal/index-pattern-management/resolve_index/${encodeURIComponent(indexTitle)}` + ); + const matchedIndices = resolved?.indices || []; + return matchedIndices.length > 0 ? matchedIndices[0].name : null; + } catch (err) { + console.error(`Failed to resolve index pattern "${indexTitle}"`, err); + return null; + } +} + +export function extractIndexParts(indexTitle: string): IndexExtractionResult { + const trimmed = indexTitle.replace(/^flint_/, ''); + const parts = trimmed.split('_'); + return { + datasource: parts[0] || 'unknown', + database: parts[1] || 'unknown', + index: parts.slice(2).join('_') || 'unknown', + }; +} + +export function generateRefreshQuery(info: IndexExtractionResult): string { + return `REFRESH MATERIALIZED VIEW \`${info.datasource}\`.\`${info.database}\`.\`${info.index}\``; +} + +export async function extractIndexInfoFromDashboard( + panels: { [key: string]: any }, + savedObjectsClient: SavedObjectsClientContract, + http: HttpStart +): Promise { + for (const panelId of Object.keys(panels)) { + try { + const panel = panels[panelId]; + const savedObjectId = panel.explicitInput?.savedObjectId; + const type = panel.type; + + if (!savedObjectId || type !== 'visualization') continue; + + const savedObject = await savedObjectsClient.get(type, savedObjectId); + const visState = JSON.parse(savedObject.attributes.visState || '{}'); + + if (visState.type !== 'pie') continue; + + const indexPatternRef = savedObject.references.find( + (ref: any) => ref.type === 'index-pattern' + ); + if (!indexPatternRef) continue; + + const indexPattern = await savedObjectsClient.get('index-pattern', indexPatternRef.id); + const indexTitleRaw = indexPattern.attributes.title; + + const concreteTitle = await resolveConcreteIndex(indexTitleRaw, http); + if (!concreteTitle) return null; + + return extractIndexParts(concreteTitle); + } catch (err) { + console.warn(`Skipping panel ${panelId} due to error:`, err); + } + } + + return null; +} From abd4512b962f61b02bed9c39b5abf32c68deecb2 Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Wed, 16 Apr 2025 17:00:43 -0700 Subject: [PATCH 20/86] poc phase 8 - add index mapping fetch Signed-off-by: Jialiang Liang --- .../embeddable/grid/dashboard_grid.tsx | 1 + .../direct_query_sync/direct_query_sync.ts | 35 +++++++++++++++++-- 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx index 5c1cbf49ad6a..75927083ae5c 100644 --- a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx +++ b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx @@ -54,6 +54,7 @@ import { DirectQueryRequest } from '../../../../framework/types'; import { extractIndexInfoFromDashboard, generateRefreshQuery, + fetchIndexMapping, } from '../../utils/direct_query_sync/direct_query_sync'; let lastValidGridSize = 0; diff --git a/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.ts b/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.ts index af285d63acdb..99b574a47936 100644 --- a/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.ts +++ b/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.ts @@ -11,6 +11,10 @@ interface IndexExtractionResult { index: string; } +export const DIRECT_QUERY_BASE = '/api/directquery'; +export const DSL_MAPPING = '/indices.getFieldMapping'; +export const DSL_BASE = `${DIRECT_QUERY_BASE}/dsl`; + export async function resolveConcreteIndex( indexTitle: string, http: HttpStart @@ -29,8 +33,8 @@ export async function resolveConcreteIndex( } } -export function extractIndexParts(indexTitle: string): IndexExtractionResult { - const trimmed = indexTitle.replace(/^flint_/, ''); +export function extractIndexParts(fullIndexName: string): IndexExtractionResult { + const trimmed = fullIndexName.replace(/^flint_/, ''); const parts = trimmed.split('_'); return { datasource: parts[0] || 'unknown', @@ -46,7 +50,8 @@ export function generateRefreshQuery(info: IndexExtractionResult): string { export async function extractIndexInfoFromDashboard( panels: { [key: string]: any }, savedObjectsClient: SavedObjectsClientContract, - http: HttpStart + http: HttpStart, + mdsId?: string ): Promise { for (const panelId of Object.keys(panels)) { try { @@ -72,6 +77,10 @@ export async function extractIndexInfoFromDashboard( const concreteTitle = await resolveConcreteIndex(indexTitleRaw, http); if (!concreteTitle) return null; + // Fetch mapping immediately after resolving index + const mapping = await fetchIndexMapping(concreteTitle, http, mdsId); + console.log('Index Mapping Result:', mapping); + return extractIndexParts(concreteTitle); } catch (err) { console.warn(`Skipping panel ${panelId} due to error:`, err); @@ -80,3 +89,23 @@ export async function extractIndexInfoFromDashboard( return null; } + +export async function fetchIndexMapping( + index: string, + http: HttpStart, + mdsId?: string +): Promise | null> { + try { + const baseUrl = `${DSL_BASE}${DSL_MAPPING}`; + const url = mdsId ? `${baseUrl}/dataSourceMDSId=${encodeURIComponent(mdsId)}` : baseUrl; + console.log('url', url); + const response = await http.get(url, { + query: { index }, + }); + + return response; + } catch (err) { + console.error(`Failed to fetch mapping for index "${index}"`, err); + return null; + } +} From 6d0a9282d6a36da71c90cfa18c9b2c7f752e62e9 Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Thu, 17 Apr 2025 13:36:54 -0700 Subject: [PATCH 21/86] Progress proto v2 Signed-off-by: Simeon Widdis --- .../framework/hooks/direct_query_hook.tsx | 2 +- src/plugins/dashboard/framework/types.tsx | 1 + .../embeddable/grid/dashboard_grid.tsx | 83 ++++++++++++++----- .../direct_query_sync/direct_query_sync.ts | 8 +- 4 files changed, 69 insertions(+), 25 deletions(-) diff --git a/src/plugins/dashboard/framework/hooks/direct_query_hook.tsx b/src/plugins/dashboard/framework/hooks/direct_query_hook.tsx index fa65e0da1ffc..e6250f784c48 100644 --- a/src/plugins/dashboard/framework/hooks/direct_query_hook.tsx +++ b/src/plugins/dashboard/framework/hooks/direct_query_hook.tsx @@ -19,7 +19,7 @@ export const useDirectQuery = ( ) => { const sqlService = new SQLService(http); const [loadStatus, setLoadStatus] = useState( - DirectQueryLoadingStatus.SUCCESS + DirectQueryLoadingStatus.FRESH ); const { diff --git a/src/plugins/dashboard/framework/types.tsx b/src/plugins/dashboard/framework/types.tsx index 7a0d822191da..73c83d85ad0b 100644 --- a/src/plugins/dashboard/framework/types.tsx +++ b/src/plugins/dashboard/framework/types.tsx @@ -14,6 +14,7 @@ export enum DirectQueryLoadingStatus { CANCELLED = 'cancelled', WAITING = 'waiting', INITIAL = 'initial', + FRESH = 'fresh', } export interface DirectQueryRequest { diff --git a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx index 75927083ae5c..dac4054848a5 100644 --- a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx +++ b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx @@ -41,7 +41,14 @@ import { Subscription } from 'rxjs'; import ReactGridLayout, { Layout, ReactGridLayoutProps } from 'react-grid-layout'; import type { SavedObjectsClientContract } from 'src/core/public'; import { HttpStart, NotificationsStart } from 'src/core/public'; -import { EuiButton, EuiFlexGroup, EuiFlexItem, EuiProgress, EuiText } from '@elastic/eui'; +import { + EuiButton, + EuiFlexGroup, + EuiFlexItem, + EuiLoadingSpinner, + EuiProgress, + EuiText, +} from '@elastic/eui'; import { ViewMode, EmbeddableChildPanel, EmbeddableStart } from '../../../../../embeddable/public'; import { GridData } from '../../../../common'; import { DASHBOARD_GRID_COLUMN_COUNT, DASHBOARD_GRID_HEIGHT } from '../dashboard_constants'; @@ -56,6 +63,7 @@ import { generateRefreshQuery, fetchIndexMapping, } from '../../utils/direct_query_sync/direct_query_sync'; +import { loadStatus } from 'src/core/public/core_app/status/lib'; let lastValidGridSize = 0; @@ -154,6 +162,8 @@ interface State { useMargins: boolean; expandedPanelId?: string; panelMetadata: Array<{ panelId: string; savedObjectId: string; type: string }>; + extractedProps: { lastRefreshTime: number } | null; + prevStatus?: string; } interface PanelLayout extends Layout { @@ -183,6 +193,7 @@ class DashboardGridUi extends React.Component { useMargins: this.props.container.getInput().useMargins, expandedPanelId: this.props.container.getInput().expandedPanelId, panelMetadata: [], + extractedProps: null, }; } @@ -283,9 +294,10 @@ class DashboardGridUi extends React.Component { ); if (indexInfo) { - this.extractedDatasource = indexInfo.datasource; - this.extractedDatabase = indexInfo.database; - this.extractedIndex = indexInfo.index; + this.extractedDatasource = indexInfo.parts.datasource; + this.extractedDatabase = indexInfo.parts.database; + this.extractedIndex = indexInfo.parts.index; + this.setState({ extractedProps: indexInfo.mapping }); console.log('Resolved index info:', indexInfo); } else { console.warn('Could not extract index info from pie visualization.'); @@ -368,21 +380,33 @@ class DashboardGridUi extends React.Component { } // TODO find a home for this - EMR_STATES: Map = new Map( + EMR_STATES: Map = new Map( Object.entries({ - submitted: { ord: 10, terminal: false, color: 'vis0' }, - queued: { ord: 20, terminal: false, color: 'vis0' }, - pending: { ord: 30, terminal: false, color: 'vis0' }, - scheduled: { ord: 40, terminal: false, color: 'vis0' }, - running: { ord: 50, terminal: false, color: 'vis0' }, - cancelling: { ord: 50, terminal: false, color: 'vis0' }, - success: { ord: 100, terminal: true, color: 'success' }, - failed: { ord: 100, terminal: true, color: 'danger' }, - cancelled: { ord: 100, terminal: true, color: 'subdued' }, + submitted: { ord: 0, terminal: false }, + queued: { ord: 8, terminal: false }, + pending: { ord: 16, terminal: false }, + scheduled: { ord: 33, terminal: false }, + running: { ord: 67, terminal: false }, + cancelling: { ord: 50, terminal: false }, + success: { ord: 100, terminal: true }, + failed: { ord: 100, terminal: true }, + cancelled: { ord: 100, terminal: true }, + fresh: { ord: 100, terminal: true }, }) ); MAX_ORD: number = 100; + private timeSince(date: number): string { + const seconds = Math.floor((new Date().getTime() - date) / 1000); + + const interval: number = seconds / 60; + + if (interval > 1) { + return Math.floor(interval) + ' minutes'; + } + return Math.floor(seconds) + ' seconds'; + } + public render() { if (this.state.isLayoutInvalid) { return null; @@ -391,6 +415,9 @@ class DashboardGridUi extends React.Component { const { viewMode } = this.state; const isViewMode = viewMode === ViewMode.VIEW; const state = this.EMR_STATES.get(this.props.loadStatus as string)!; + if (state.terminal && this.props.loadStatus !== 'fresh') { + window.location.reload(); + } return (
@@ -407,13 +434,27 @@ class DashboardGridUi extends React.Component { > Synchronize Now - + {state.terminal ? ( + + Last Refresh:{' '} + {this.state.extractedProps ? ( + this.timeSince(this.state.extractedProps.lastRefreshTime) + ' ago' + ) : ( + <> +    + + + )} + + ) : ( + + )}
{ +): Promise<{ parts: IndexExtractionResult; mapping: { lastRefreshTime: number } } | null> { for (const panelId of Object.keys(panels)) { try { const panel = panels[panelId]; @@ -78,10 +78,12 @@ export async function extractIndexInfoFromDashboard( if (!concreteTitle) return null; // Fetch mapping immediately after resolving index - const mapping = await fetchIndexMapping(concreteTitle, http, mdsId); + const mapping = (await fetchIndexMapping(concreteTitle, http, mdsId))!; console.log('Index Mapping Result:', mapping); - return extractIndexParts(concreteTitle); + for (const val of Object.values(mapping)) { + return { mapping: val.mappings._meta.properties!, parts: extractIndexParts(concreteTitle) }; + } } catch (err) { console.warn(`Skipping panel ${panelId} due to error:`, err); } From 46c8ed0cb6cba42266f8e17453ab8335e076d347 Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Fri, 18 Apr 2025 11:36:30 -0700 Subject: [PATCH 22/86] FIX SOME LINTING ERROR Signed-off-by: Jialiang Liang --- .../application/embeddable/grid/dashboard_grid.tsx | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx index dac4054848a5..7a5d84867e34 100644 --- a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx +++ b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx @@ -41,14 +41,7 @@ import { Subscription } from 'rxjs'; import ReactGridLayout, { Layout, ReactGridLayoutProps } from 'react-grid-layout'; import type { SavedObjectsClientContract } from 'src/core/public'; import { HttpStart, NotificationsStart } from 'src/core/public'; -import { - EuiButton, - EuiFlexGroup, - EuiFlexItem, - EuiLoadingSpinner, - EuiProgress, - EuiText, -} from '@elastic/eui'; +import { EuiButton, EuiLoadingSpinner, EuiProgress, EuiText } from '@elastic/eui'; import { ViewMode, EmbeddableChildPanel, EmbeddableStart } from '../../../../../embeddable/public'; import { GridData } from '../../../../common'; import { DASHBOARD_GRID_COLUMN_COUNT, DASHBOARD_GRID_HEIGHT } from '../dashboard_constants'; @@ -61,9 +54,7 @@ import { DirectQueryRequest } from '../../../../framework/types'; import { extractIndexInfoFromDashboard, generateRefreshQuery, - fetchIndexMapping, } from '../../utils/direct_query_sync/direct_query_sync'; -import { loadStatus } from 'src/core/public/core_app/status/lib'; let lastValidGridSize = 0; From 01a6b6ddae25b98063ca880785bbd65f5b7736d6 Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Fri, 18 Apr 2025 13:21:01 -0700 Subject: [PATCH 23/86] Refactor - moving the EMR status utils into dq sync class Signed-off-by: Jialiang Liang --- .../embeddable/grid/dashboard_grid.tsx | 52 ++++++------------- .../direct_query_sync/direct_query_sync.ts | 24 +++++++++ 2 files changed, 39 insertions(+), 37 deletions(-) diff --git a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx index 7a5d84867e34..fffb2e1a7437 100644 --- a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx +++ b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx @@ -54,6 +54,9 @@ import { DirectQueryRequest } from '../../../../framework/types'; import { extractIndexInfoFromDashboard, generateRefreshQuery, + EMR_STATES, + MAX_ORD, + timeSince, } from '../../utils/direct_query_sync/direct_query_sync'; let lastValidGridSize = 0; @@ -315,13 +318,11 @@ class DashboardGridUi extends React.Component { index: extractedIndex, }); - const queryPayload = { + this.props.startLoading({ query, lang: 'sql', datasource: extractedDatasource, - }; - - this.props.startLoading(queryPayload); + }); }; public renderPanels() { @@ -370,34 +371,6 @@ class DashboardGridUi extends React.Component { }); } - // TODO find a home for this - EMR_STATES: Map = new Map( - Object.entries({ - submitted: { ord: 0, terminal: false }, - queued: { ord: 8, terminal: false }, - pending: { ord: 16, terminal: false }, - scheduled: { ord: 33, terminal: false }, - running: { ord: 67, terminal: false }, - cancelling: { ord: 50, terminal: false }, - success: { ord: 100, terminal: true }, - failed: { ord: 100, terminal: true }, - cancelled: { ord: 100, terminal: true }, - fresh: { ord: 100, terminal: true }, - }) - ); - MAX_ORD: number = 100; - - private timeSince(date: number): string { - const seconds = Math.floor((new Date().getTime() - date) / 1000); - - const interval: number = seconds / 60; - - if (interval > 1) { - return Math.floor(interval) + ' minutes'; - } - return Math.floor(seconds) + ' seconds'; - } - public render() { if (this.state.isLayoutInvalid) { return null; @@ -405,16 +378,21 @@ class DashboardGridUi extends React.Component { const { viewMode } = this.state; const isViewMode = viewMode === ViewMode.VIEW; - const state = this.EMR_STATES.get(this.props.loadStatus as string)!; + const state = EMR_STATES.get(this.props.loadStatus as string)!; + if (state.terminal && this.props.loadStatus !== 'fresh') { window.location.reload(); } return (
- {/* Top-left corner "Synchronize Now" button */}
{ Last Refresh:{' '} {this.state.extractedProps ? ( - this.timeSince(this.state.extractedProps.lastRefreshTime) + ' ago' + timeSince(this.state.extractedProps.lastRefreshTime) + ' ago' ) : ( <>    @@ -440,7 +418,7 @@ class DashboardGridUi extends React.Component { ) : ( ( + Object.entries({ + submitted: { ord: 0, terminal: false }, + queued: { ord: 8, terminal: false }, + pending: { ord: 16, terminal: false }, + scheduled: { ord: 33, terminal: false }, + running: { ord: 67, terminal: false }, + cancelling: { ord: 50, terminal: false }, + success: { ord: 100, terminal: true }, + failed: { ord: 100, terminal: true }, + cancelled: { ord: 100, terminal: true }, + fresh: { ord: 100, terminal: true }, + }) +); + +export const MAX_ORD = 100; + +export function timeSince(date: number): string { + const seconds = Math.floor((new Date().getTime() - date) / 1000); + const interval: number = seconds / 60; + return interval > 1 ? Math.floor(interval) + ' minutes' : Math.floor(seconds) + ' seconds'; +} + export async function resolveConcreteIndex( indexTitle: string, http: HttpStart From 40c2f799b270a4227d3f74c4febc366cf4a67643 Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Fri, 18 Apr 2025 13:45:01 -0700 Subject: [PATCH 24/86] Refactor - moving the sync related UI component into a seprate class Signed-off-by: Jialiang Liang --- .../embeddable/grid/dashboard_flint_sync.tsx | 76 +++++++++++++++++++ .../embeddable/grid/dashboard_grid.tsx | 47 ++---------- 2 files changed, 83 insertions(+), 40 deletions(-) create mode 100644 src/plugins/dashboard/public/application/embeddable/grid/dashboard_flint_sync.tsx diff --git a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_flint_sync.tsx b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_flint_sync.tsx new file mode 100644 index 000000000000..236f62470709 --- /dev/null +++ b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_flint_sync.tsx @@ -0,0 +1,76 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Any modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import React from 'react'; +import { EuiButton, EuiLoadingSpinner, EuiProgress, EuiText } from '@elastic/eui'; +import { DirectQueryLoadingStatus } from '../../../../framework/types'; +import { EMR_STATES, MAX_ORD, timeSince } from '../../utils/direct_query_sync/direct_query_sync'; + +interface DashboardFlintSyncProps { + loadStatus: DirectQueryLoadingStatus; + lastRefreshTime?: number; + onSynchronize: () => void; +} + +export const DashboardFlintSync: React.FC = ({ + loadStatus, + lastRefreshTime, + onSynchronize, +}) => { + const state = EMR_STATES.get(loadStatus)!; + + return ( +
+ + Synchronize Now + + {state.terminal ? ( + + Last Refresh:{' '} + {typeof lastRefreshTime === 'number' ? timeSince(lastRefreshTime) + ' ago' : '--'} + + ) : ( + + )} +
+ ); +}; diff --git a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx index fffb2e1a7437..e789d3701341 100644 --- a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx +++ b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx @@ -58,6 +58,7 @@ import { MAX_ORD, timeSince, } from '../../utils/direct_query_sync/direct_query_sync'; +import { DashboardFlintSync } from './dashboard_flint_sync'; let lastValidGridSize = 0; @@ -286,6 +287,7 @@ class DashboardGridUi extends React.Component { this.props.savedObjectsClient, this.props.http ); + console.log('Extracted metadata123:', indexInfo?.mapping); if (indexInfo) { this.extractedDatasource = indexInfo.parts.datasource; @@ -386,46 +388,11 @@ class DashboardGridUi extends React.Component { return (
-
- - Synchronize Now - - {state.terminal ? ( - - Last Refresh:{' '} - {this.state.extractedProps ? ( - timeSince(this.state.extractedProps.lastRefreshTime) + ' ago' - ) : ( - <> -    - - - )} - - ) : ( - - )} -
- + Date: Fri, 18 Apr 2025 15:05:02 -0700 Subject: [PATCH 25/86] Lint - remove some unused imports Signed-off-by: Jialiang Liang --- .../public/application/embeddable/grid/dashboard_grid.tsx | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx index e789d3701341..a1f10e1b80b5 100644 --- a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx +++ b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx @@ -41,7 +41,6 @@ import { Subscription } from 'rxjs'; import ReactGridLayout, { Layout, ReactGridLayoutProps } from 'react-grid-layout'; import type { SavedObjectsClientContract } from 'src/core/public'; import { HttpStart, NotificationsStart } from 'src/core/public'; -import { EuiButton, EuiLoadingSpinner, EuiProgress, EuiText } from '@elastic/eui'; import { ViewMode, EmbeddableChildPanel, EmbeddableStart } from '../../../../../embeddable/public'; import { GridData } from '../../../../common'; import { DASHBOARD_GRID_COLUMN_COUNT, DASHBOARD_GRID_HEIGHT } from '../dashboard_constants'; @@ -55,8 +54,6 @@ import { extractIndexInfoFromDashboard, generateRefreshQuery, EMR_STATES, - MAX_ORD, - timeSince, } from '../../utils/direct_query_sync/direct_query_sync'; import { DashboardFlintSync } from './dashboard_flint_sync'; From fca592811cf7919be4dc760793f4f9692dbcc069 Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Fri, 18 Apr 2025 16:10:09 -0700 Subject: [PATCH 26/86] production phase 0 - add the implementation of feature flag dashboard.directQueryConnectionSync Signed-off-by: Jialiang Liang --- src/plugins/dashboard/config.ts | 1 + .../application/embeddable/dashboard_container.tsx | 5 +++++ .../application/embeddable/grid/dashboard_grid.tsx | 14 +++++++++----- .../embeddable/viewport/dashboard_viewport.tsx | 2 ++ src/plugins/dashboard/public/plugin.tsx | 3 +++ src/plugins/dashboard/public/types.ts | 2 ++ 6 files changed, 22 insertions(+), 5 deletions(-) diff --git a/src/plugins/dashboard/config.ts b/src/plugins/dashboard/config.ts index 14125fe69333..46fc45c7ea56 100644 --- a/src/plugins/dashboard/config.ts +++ b/src/plugins/dashboard/config.ts @@ -32,6 +32,7 @@ import { schema, TypeOf } from '@osd/config-schema'; export const configSchema = schema.object({ allowByValueEmbeddables: schema.boolean({ defaultValue: false }), + directQueryConnectionSync: schema.boolean({ defaultValue: false }), }); export type ConfigSchema = TypeOf; diff --git a/src/plugins/dashboard/public/application/embeddable/dashboard_container.tsx b/src/plugins/dashboard/public/application/embeddable/dashboard_container.tsx index 1c4654d32c36..f1b6f3ea7e2a 100644 --- a/src/plugins/dashboard/public/application/embeddable/dashboard_container.tsx +++ b/src/plugins/dashboard/public/application/embeddable/dashboard_container.tsx @@ -61,6 +61,7 @@ import { } from '../../../../opensearch_dashboards_react/public'; import { PLACEHOLDER_EMBEDDABLE } from './placeholder'; import { PanelPlacementMethod, IPanelPlacementArgs } from './panel/dashboard_panel_placement'; +import { DashboardFeatureFlagConfig } from '../../plugin'; export interface DashboardContainerInput extends ContainerInput { viewMode: ViewMode; @@ -106,6 +107,7 @@ export interface DashboardContainerOptions { uiActions: UiActionsStart; savedObjectsClient: CoreStart['savedObjects']['client']; http: CoreStart['http']; + dashboardFeatureFlagConfig: DashboardFeatureFlagConfig; } export type DashboardReactContextValue = OpenSearchDashboardsReactContextValue< @@ -249,6 +251,9 @@ export class DashboardContainer extends Container , diff --git a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx index a1f10e1b80b5..0a3b0993657d 100644 --- a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx +++ b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx @@ -143,6 +143,7 @@ export interface DashboardGridProps extends ReactIntl.InjectedIntlProps { startLoading: (payload: DirectQueryRequest) => void; loadStatus: DirectQueryLoadingStatus; pollingResult: any; + isDirectQuerySyncEnabled: boolean; } interface State { @@ -385,11 +386,14 @@ class DashboardGridUi extends React.Component { return (
- + {this.props.isDirectQuerySyncEnabled && ( + + )} + void; loadStatus: DirectQueryLoadingStatus; pollingResult: any; + isDirectQuerySyncEnabled: boolean; } interface State { @@ -181,6 +182,7 @@ export class DashboardViewport extends React.Component
); diff --git a/src/plugins/dashboard/public/plugin.tsx b/src/plugins/dashboard/public/plugin.tsx index 1b6c213a3cb7..f7f2ba72947c 100644 --- a/src/plugins/dashboard/public/plugin.tsx +++ b/src/plugins/dashboard/public/plugin.tsx @@ -138,6 +138,7 @@ export type DashboardUrlGenerator = UrlGeneratorContract void; addBasePath?: (url: string) => string; toastNotifications: ToastsStart; + dashboardFeatureFlagConfig: DashboardFeatureFlagConfig; } From f1205ca458314060f658b550ce098602bac77744 Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Fri, 18 Apr 2025 16:02:14 -0700 Subject: [PATCH 27/86] Add some EMR states comments Signed-off-by: Simeon Widdis --- .../utils/direct_query_sync/direct_query_sync.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.ts b/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.ts index 837d91c1443e..21c28d6e0a80 100644 --- a/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.ts +++ b/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.ts @@ -15,7 +15,9 @@ export const DIRECT_QUERY_BASE = '/api/directquery'; export const DSL_MAPPING = '/indices.getFieldMapping'; export const DSL_BASE = `${DIRECT_QUERY_BASE}/dsl`; -// Module for handling EMR states for Dashboards Progress Bar +// Module for handling EMR states for Dashboards Progress Bar. All of these except "fresh" are +// directly from the EMR job run states. "ord" is used to approximate progress (eyeballed relative +// stage times), and "terminal" indicates whether a job is in progress at all. export const EMR_STATES = new Map( Object.entries({ submitted: { ord: 0, terminal: false }, @@ -23,10 +25,11 @@ export const EMR_STATES = new Map( pending: { ord: 16, terminal: false }, scheduled: { ord: 33, terminal: false }, running: { ord: 67, terminal: false }, - cancelling: { ord: 50, terminal: false }, + cancelling: { ord: 90, terminal: false }, success: { ord: 100, terminal: true }, failed: { ord: 100, terminal: true }, cancelled: { ord: 100, terminal: true }, + // The "null state" for a fresh page load, which components conditionally use on load. fresh: { ord: 100, terminal: true }, }) ); From df3718da0892bbdbb4b01f40ac817768b9127f67 Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Wed, 23 Apr 2025 16:12:42 -0700 Subject: [PATCH 28/86] production phase 1 - add the mds support Signed-off-by: Jialiang Liang --- .../embeddable/grid/dashboard_grid.tsx | 4 ++++ .../viewport/dashboard_viewport.tsx | 6 +++++- .../direct_query_sync/direct_query_sync.ts | 19 +++++++++++++++---- 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx index 0a3b0993657d..4488cb9f9afe 100644 --- a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx +++ b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx @@ -144,6 +144,7 @@ export interface DashboardGridProps extends ReactIntl.InjectedIntlProps { loadStatus: DirectQueryLoadingStatus; pollingResult: any; isDirectQuerySyncEnabled: boolean; + setMdsId?: (mdsId?: string) => void; } interface State { @@ -293,6 +294,9 @@ class DashboardGridUi extends React.Component { this.extractedIndex = indexInfo.parts.index; this.setState({ extractedProps: indexInfo.mapping }); console.log('Resolved index info:', indexInfo); + if (this.props.setMdsId) { + this.props.setMdsId(indexInfo.mdsId); + } } else { console.warn('Could not extract index info from pie visualization.'); } diff --git a/src/plugins/dashboard/public/application/embeddable/viewport/dashboard_viewport.tsx b/src/plugins/dashboard/public/application/embeddable/viewport/dashboard_viewport.tsx index f1ef1708541c..446df7ca311f 100644 --- a/src/plugins/dashboard/public/application/embeddable/viewport/dashboard_viewport.tsx +++ b/src/plugins/dashboard/public/application/embeddable/viewport/dashboard_viewport.tsx @@ -55,6 +55,7 @@ export interface DashboardViewportProps { loadStatus: DirectQueryLoadingStatus; pollingResult: any; isDirectQuerySyncEnabled: boolean; + setMdsId?: (mdsId?: string) => void; } interface State { @@ -183,6 +184,7 @@ export class DashboardViewport extends React.Component
); @@ -201,8 +203,9 @@ export class DashboardViewport extends React.Component ) => { + const [mdsId, setMdsId] = React.useState(undefined); const { http, notifications, ...restProps } = props; - const { startLoading, loadStatus, pollingResult } = useDirectQuery(http, notifications); + const { startLoading, loadStatus, pollingResult } = useDirectQuery(http, notifications, mdsId); return ( ); }; diff --git a/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.ts b/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.ts index 21c28d6e0a80..ef1c328a4a21 100644 --- a/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.ts +++ b/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.ts @@ -77,9 +77,12 @@ export function generateRefreshQuery(info: IndexExtractionResult): string { export async function extractIndexInfoFromDashboard( panels: { [key: string]: any }, savedObjectsClient: SavedObjectsClientContract, - http: HttpStart, - mdsId?: string -): Promise<{ parts: IndexExtractionResult; mapping: { lastRefreshTime: number } } | null> { + http: HttpStart +): Promise<{ + parts: IndexExtractionResult; + mapping: { lastRefreshTime: number }; + mdsId?: string; +} | null> { for (const panelId of Object.keys(panels)) { try { const panel = panels[panelId]; @@ -99,17 +102,25 @@ export async function extractIndexInfoFromDashboard( if (!indexPatternRef) continue; const indexPattern = await savedObjectsClient.get('index-pattern', indexPatternRef.id); + console.log('Index Pattern:', indexPattern); const indexTitleRaw = indexPattern.attributes.title; const concreteTitle = await resolveConcreteIndex(indexTitleRaw, http); if (!concreteTitle) return null; + const mdsId = + indexPattern.references?.find((ref: any) => ref.type === 'data-source')?.id || undefined; + // Fetch mapping immediately after resolving index const mapping = (await fetchIndexMapping(concreteTitle, http, mdsId))!; console.log('Index Mapping Result:', mapping); for (const val of Object.values(mapping)) { - return { mapping: val.mappings._meta.properties!, parts: extractIndexParts(concreteTitle) }; + return { + mapping: val.mappings._meta.properties!, + parts: extractIndexParts(concreteTitle), + mdsId, + }; } } catch (err) { console.warn(`Skipping panel ${panelId} due to error:`, err); From 5e6c46b6ad2215fac9a9bfa0cc6717c412b20f81 Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Thu, 24 Apr 2025 17:08:03 -0700 Subject: [PATCH 29/86] Update to new sync UI Signed-off-by: Simeon Widdis --- .../embeddable/grid/dashboard_flint_sync.tsx | 60 +++++++++++-------- .../embeddable/grid/dashboard_grid.tsx | 3 +- .../direct_query_sync/direct_query_sync.ts | 15 +++-- 3 files changed, 43 insertions(+), 35 deletions(-) diff --git a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_flint_sync.tsx b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_flint_sync.tsx index 236f62470709..d37deab8597d 100644 --- a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_flint_sync.tsx +++ b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_flint_sync.tsx @@ -29,48 +29,56 @@ */ import React from 'react'; -import { EuiButton, EuiLoadingSpinner, EuiProgress, EuiText } from '@elastic/eui'; +import { + EuiButton, + EuiCallOut, + EuiLink, + EuiLoadingSpinner, + EuiProgress, + EuiText, +} from '@elastic/eui'; import { DirectQueryLoadingStatus } from '../../../../framework/types'; -import { EMR_STATES, MAX_ORD, timeSince } from '../../utils/direct_query_sync/direct_query_sync'; +import { + EMR_STATES, + MAX_ORD, + intervalAsMinutes, +} from '../../utils/direct_query_sync/direct_query_sync'; interface DashboardFlintSyncProps { loadStatus: DirectQueryLoadingStatus; lastRefreshTime?: number; + refreshInterval?: number; onSynchronize: () => void; } export const DashboardFlintSync: React.FC = ({ loadStatus, lastRefreshTime, + refreshInterval, onSynchronize, }) => { const state = EMR_STATES.get(loadStatus)!; - return ( -
- - Synchronize Now - - {state.terminal ? ( - - Last Refresh:{' '} - {typeof lastRefreshTime === 'number' ? timeSince(lastRefreshTime) + ' ago' : '--'} - + return state.terminal ? ( + + Data scheduled to sync every{' '} + {refreshInterval ? intervalAsMinutes(1000 * refreshInterval) : '--'}. Last sync:{' '} + {lastRefreshTime ? ( + <> + {new Date(lastRefreshTime).toLocaleString()} ( + {intervalAsMinutes(new Date().getTime() - lastRefreshTime)} ago) + ) : ( - + '--' )} -
+ .    + Sync data + + ) : ( + + +    Data sync is in progress ({state.ord}% complete). The dashboard will + reload on completion. + ); }; diff --git a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx index 4488cb9f9afe..d3f19cdd51b2 100644 --- a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx +++ b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx @@ -156,7 +156,7 @@ interface State { useMargins: boolean; expandedPanelId?: string; panelMetadata: Array<{ panelId: string; savedObjectId: string; type: string }>; - extractedProps: { lastRefreshTime: number } | null; + extractedProps: { lastRefreshTime?: number; refreshInterval?: number } | null; prevStatus?: string; } @@ -394,6 +394,7 @@ class DashboardGridUi extends React.Component { )} diff --git a/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.ts b/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.ts index ef1c328a4a21..f97141c86bd8 100644 --- a/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.ts +++ b/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.ts @@ -21,10 +21,10 @@ export const DSL_BASE = `${DIRECT_QUERY_BASE}/dsl`; export const EMR_STATES = new Map( Object.entries({ submitted: { ord: 0, terminal: false }, - queued: { ord: 8, terminal: false }, - pending: { ord: 16, terminal: false }, - scheduled: { ord: 33, terminal: false }, - running: { ord: 67, terminal: false }, + queued: { ord: 10, terminal: false }, + pending: { ord: 20, terminal: false }, + scheduled: { ord: 30, terminal: false }, + running: { ord: 70, terminal: false }, cancelling: { ord: 90, terminal: false }, success: { ord: 100, terminal: true }, failed: { ord: 100, terminal: true }, @@ -36,10 +36,9 @@ export const EMR_STATES = new Map( export const MAX_ORD = 100; -export function timeSince(date: number): string { - const seconds = Math.floor((new Date().getTime() - date) / 1000); - const interval: number = seconds / 60; - return interval > 1 ? Math.floor(interval) + ' minutes' : Math.floor(seconds) + ' seconds'; +export function intervalAsMinutes(interval: number): string { + const minutes = Math.floor(interval / 60000); + return minutes === 1 ? '1 minute' : minutes + ' minutes'; } export async function resolveConcreteIndex( From 14e563c58540b7d69dd7dc067265eb1b9f06bcd1 Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Thu, 24 Apr 2025 18:10:55 -0700 Subject: [PATCH 30/86] production phase 3 - implement the URL feature flag Signed-off-by: Jialiang Liang --- .../embeddable/grid/dashboard_grid.tsx | 25 +++++++++++++------ .../direct_query_sync_url_flag.ts | 17 +++++++++++++ 2 files changed, 34 insertions(+), 8 deletions(-) create mode 100644 src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync_url_flag.ts diff --git a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx index d3f19cdd51b2..b757c378bbb3 100644 --- a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx +++ b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx @@ -56,6 +56,7 @@ import { EMR_STATES, } from '../../utils/direct_query_sync/direct_query_sync'; import { DashboardFlintSync } from './dashboard_flint_sync'; +import { isDirectQuerySyncEnabledByUrl } from '../../utils/direct_query_sync/direct_query_sync_url_flag'; let lastValidGridSize = 0; @@ -390,14 +391,22 @@ class DashboardGridUi extends React.Component { return (
- {this.props.isDirectQuerySyncEnabled && ( - - )} + {(() => { + const urlOverride = isDirectQuerySyncEnabledByUrl(); + const shouldRender = + urlOverride !== undefined + ? urlOverride + : this.props.isDirectQuerySyncEnabled; + + return shouldRender ? ( + + ) : null; + })()} Date: Fri, 25 Apr 2025 19:12:19 -0700 Subject: [PATCH 31/86] production phase 4 - fix the resolve index for mds support Signed-off-by: Jialiang Liang --- .../direct_query_sync/direct_query_sync.ts | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.ts b/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.ts index f97141c86bd8..c2e4278097b0 100644 --- a/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.ts +++ b/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.ts @@ -43,13 +43,16 @@ export function intervalAsMinutes(interval: number): string { export async function resolveConcreteIndex( indexTitle: string, - http: HttpStart + http: HttpStart, + mdsId?: string ): Promise { if (!indexTitle.includes('*')) return indexTitle; try { + const query: any = mdsId ? { data_source: mdsId } : {}; const resolved = await http.get( - `/internal/index-pattern-management/resolve_index/${encodeURIComponent(indexTitle)}` + `/internal/index-pattern-management/resolve_index/${encodeURIComponent(indexTitle)}`, + { query } ); const matchedIndices = resolved?.indices || []; return matchedIndices.length > 0 ? matchedIndices[0].name : null; @@ -93,6 +96,7 @@ export async function extractIndexInfoFromDashboard( const savedObject = await savedObjectsClient.get(type, savedObjectId); const visState = JSON.parse(savedObject.attributes.visState || '{}'); + // Check if the visualization is a pie chart TODO: add more types if (visState.type !== 'pie') continue; const indexPatternRef = savedObject.references.find( @@ -102,14 +106,15 @@ export async function extractIndexInfoFromDashboard( const indexPattern = await savedObjectsClient.get('index-pattern', indexPatternRef.id); console.log('Index Pattern:', indexPattern); - const indexTitleRaw = indexPattern.attributes.title; - - const concreteTitle = await resolveConcreteIndex(indexTitleRaw, http); - if (!concreteTitle) return null; const mdsId = indexPattern.references?.find((ref: any) => ref.type === 'data-source')?.id || undefined; + const indexTitleRaw = indexPattern.attributes.title; + + const concreteTitle = await resolveConcreteIndex(indexTitleRaw, http, mdsId); + if (!concreteTitle) return null; + // Fetch mapping immediately after resolving index const mapping = (await fetchIndexMapping(concreteTitle, http, mdsId))!; console.log('Index Mapping Result:', mapping); From fac529e62935c56ed4adb831659eb8f0326675ec Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Mon, 28 Apr 2025 18:32:26 -0700 Subject: [PATCH 32/86] production phase 5 - enhance the souce checking logic (handle more than pie visualizations) Signed-off-by: Jialiang Liang --- .../embeddable/grid/dashboard_grid.tsx | 24 +++-- .../direct_query_sync/direct_query_sync.ts | 92 +++++++++++++------ 2 files changed, 81 insertions(+), 35 deletions(-) diff --git a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx index b757c378bbb3..97524d5772a5 100644 --- a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx +++ b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx @@ -287,7 +287,7 @@ class DashboardGridUi extends React.Component { this.props.savedObjectsClient, this.props.http ); - console.log('Extracted metadata123:', indexInfo?.mapping); + console.log('Extracted metadata:', indexInfo?.mapping); if (indexInfo) { this.extractedDatasource = indexInfo.parts.datasource; @@ -299,7 +299,13 @@ class DashboardGridUi extends React.Component { this.props.setMdsId(indexInfo.mdsId); } } else { - console.warn('Could not extract index info from pie visualization.'); + console.warn( + 'Dashboard does not qualify for synchronization: inconsistent or unsupported visualization sources.' + ); + this.setState({ extractedProps: null }); + if (this.props.setMdsId) { + this.props.setMdsId(undefined); + } } } @@ -393,12 +399,14 @@ class DashboardGridUi extends React.Component {
{(() => { const urlOverride = isDirectQuerySyncEnabledByUrl(); - const shouldRender = - urlOverride !== undefined - ? urlOverride - : this.props.isDirectQuerySyncEnabled; - - return shouldRender ? ( + const featureFlagEnabled = + urlOverride !== undefined ? urlOverride : this.props.isDirectQuerySyncEnabled; + + const metadataAvailable = this.state.extractedProps !== null; + + const shouldRenderSyncUI = featureFlagEnabled && metadataAvailable; + + return shouldRenderSyncUI ? ( { + const indexPatternIds: string[] = []; + const mdsIds: Array = []; + for (const panelId of Object.keys(panels)) { try { const panel = panels[panelId]; const savedObjectId = panel.explicitInput?.savedObjectId; + if (!savedObjectId) continue; + const type = panel.type; + const savedObject = await savedObjectsClient.get(type, savedObjectId); - if (!savedObjectId || type !== 'visualization') continue; + const references = savedObject.references || []; - const savedObject = await savedObjectsClient.get(type, savedObjectId); - const visState = JSON.parse(savedObject.attributes.visState || '{}'); + if (references.length === 0) { + continue; // No references, skip (acceptable) + } - // Check if the visualization is a pie chart TODO: add more types - if (visState.type !== 'pie') continue; + // Check if there is any non-index-pattern reference + if (references.some((ref: any) => ref.type !== 'index-pattern')) { + console.warn( + `Visualization ${panelId} references a non-index-pattern object. Disabling sync.` + ); + return null; + } - const indexPatternRef = savedObject.references.find( - (ref: any) => ref.type === 'index-pattern' - ); - if (!indexPatternRef) continue; + const indexPatternRef = references.find((ref: any) => ref.type === 'index-pattern'); + if (!indexPatternRef) { + console.warn( + `Visualization ${panelId} does not reference an index-pattern. Disabling sync.` + ); + return null; + } const indexPattern = await savedObjectsClient.get('index-pattern', indexPatternRef.id); - console.log('Index Pattern:', indexPattern); - const mdsId = indexPattern.references?.find((ref: any) => ref.type === 'data-source')?.id || undefined; - const indexTitleRaw = indexPattern.attributes.title; - - const concreteTitle = await resolveConcreteIndex(indexTitleRaw, http, mdsId); - if (!concreteTitle) return null; - - // Fetch mapping immediately after resolving index - const mapping = (await fetchIndexMapping(concreteTitle, http, mdsId))!; - console.log('Index Mapping Result:', mapping); - - for (const val of Object.values(mapping)) { - return { - mapping: val.mappings._meta.properties!, - parts: extractIndexParts(concreteTitle), - mdsId, - }; - } + indexPatternIds.push(indexPatternRef.id); + mdsIds.push(mdsId); } catch (err) { console.warn(`Skipping panel ${panelId} due to error:`, err); } } + if (!sourceCheck(indexPatternIds, mdsIds)) { + return null; + } + + const selectedIndexPatternId = indexPatternIds[0]; + const selectedMdsId = mdsIds[0]; + + const indexPattern = await savedObjectsClient.get('index-pattern', selectedIndexPatternId); + const indexTitleRaw = indexPattern.attributes.title; + const concreteTitle = await resolveConcreteIndex(indexTitleRaw, http, selectedMdsId); + + if (!concreteTitle) return null; + + const mapping = await fetchIndexMapping(concreteTitle, http, selectedMdsId); + if (!mapping) return null; + + for (const val of Object.values(mapping)) { + return { + mapping: val.mappings._meta.properties!, + parts: extractIndexParts(concreteTitle), + mdsId: selectedMdsId, + }; + } + return null; } @@ -153,3 +176,18 @@ export async function fetchIndexMapping( return null; } } + +export function sourceCheck(indexPatternIds: string[], mdsIds: Array): boolean { + const uniqueIndexPatternIds = Array.from(new Set(indexPatternIds)); + const uniqueMdsIds = Array.from(new Set(mdsIds)); + + const isConsistent = uniqueIndexPatternIds.length === 1 && uniqueMdsIds.length === 1; + + if (!isConsistent) { + console.warn( + 'Dashboard uses multiple data sources or multiple index patterns. Sync feature disabled.' + ); + } + + return isConsistent; +} From 83d773498d052b67e54c257c1baaf7743563fe61 Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Mon, 28 Apr 2025 18:51:55 -0700 Subject: [PATCH 33/86] production phase 5 - enhance the source checking logic a bit to handle the case of not an index pattern type Signed-off-by: Jialiang Liang --- .../application/utils/direct_query_sync/direct_query_sync.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.ts b/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.ts index 1db66c97327f..60e1e458a007 100644 --- a/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.ts +++ b/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.ts @@ -178,6 +178,11 @@ export async function fetchIndexMapping( } export function sourceCheck(indexPatternIds: string[], mdsIds: Array): boolean { + // If no visualizations reference an index pattern, treat as acceptable (no sync, but no conflict). + if (indexPatternIds.length === 0 && mdsIds.length === 0) { + return true; + } + const uniqueIndexPatternIds = Array.from(new Set(indexPatternIds)); const uniqueMdsIds = Array.from(new Set(mdsIds)); From d9b7c0cd02b83f443192c62f1f71c9ae9e8b0051 Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Mon, 28 Apr 2025 18:53:08 -0700 Subject: [PATCH 34/86] production phase 6 - add some basic test cases for util classes first Signed-off-by: Jialiang Liang --- .../direct_query_sync.test.ts | 44 +++++++++++++++++ .../direct_query_sync_url_flag.test.ts | 48 +++++++++++++++++++ 2 files changed, 92 insertions(+) create mode 100644 src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.test.ts create mode 100644 src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync_url_flag.test.ts diff --git a/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.test.ts b/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.test.ts new file mode 100644 index 000000000000..5b041db400d2 --- /dev/null +++ b/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.test.ts @@ -0,0 +1,44 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { sourceCheck } from './direct_query_sync'; + +describe('sourceCheck', () => { + it('returns true if all indexPatternIds and mdsIds are the same', () => { + const indexPatternIds = ['pattern-1', 'pattern-1', 'pattern-1']; + const mdsIds = ['mds-1', 'mds-1', 'mds-1']; + + expect(sourceCheck(indexPatternIds, mdsIds)).toBe(true); + }); + + it('returns false if indexPatternIds are different', () => { + const indexPatternIds = ['pattern-1', 'pattern-2', 'pattern-1']; + const mdsIds = ['mds-1', 'mds-1', 'mds-1']; + + expect(sourceCheck(indexPatternIds, mdsIds)).toBe(false); + }); + + it('returns false if mdsIds are different', () => { + const indexPatternIds = ['pattern-1', 'pattern-1', 'pattern-1']; + const mdsIds = ['mds-1', 'mds-2', 'mds-1']; + + expect(sourceCheck(indexPatternIds, mdsIds)).toBe(false); + }); + + it('returns false if both indexPatternIds and mdsIds are different', () => { + const indexPatternIds = ['pattern-1', 'pattern-2']; + const mdsIds = ['mds-1', 'mds-2']; + + expect(sourceCheck(indexPatternIds, mdsIds)).toBe(false); + }); + + it('returns true if empty arrays (edge case)', () => { + expect(sourceCheck([], [])).toBe(true); + }); + + it('returns true if single entry arrays', () => { + expect(sourceCheck(['pattern-1'], ['mds-1'])).toBe(true); + }); +}); diff --git a/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync_url_flag.test.ts b/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync_url_flag.test.ts new file mode 100644 index 000000000000..7b1a5d826b1d --- /dev/null +++ b/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync_url_flag.test.ts @@ -0,0 +1,48 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { isDirectQuerySyncEnabledByUrl } from './direct_query_sync_url_flag'; + +describe('isDirectQuerySyncEnabledByUrl', () => { + const originalLocation = window.location; + + beforeEach(() => { + // @ts-ignore + delete window.location; + window.location = { + hash: '', + } as any; + }); + + afterEach(() => { + window.location = originalLocation; + }); + + it('returns true when URL param is true', () => { + window.location.hash = '#/view/someid?dashboard.directQueryConnectionSync=true'; + expect(isDirectQuerySyncEnabledByUrl()).toBe(true); + }); + + it('returns false when URL param is false', () => { + window.location.hash = '#/view/someid?dashboard.directQueryConnectionSync=false'; + expect(isDirectQuerySyncEnabledByUrl()).toBe(false); + }); + + it('returns undefined when URL param is missing', () => { + window.location.hash = '#/view/someid'; + expect(isDirectQuerySyncEnabledByUrl()).toBe(undefined); + }); + + it('returns undefined when URL param is not true/false', () => { + window.location.hash = '#/view/someid?dashboard.directQueryConnectionSync=maybe'; + expect(isDirectQuerySyncEnabledByUrl()).toBe(undefined); + }); + + it('handles multiple URL params correctly', () => { + window.location.hash = + '#/view/someid?otherparam=abc&dashboard.directQueryConnectionSync=true&anotherparam=xyz'; + expect(isDirectQuerySyncEnabledByUrl()).toBe(true); + }); +}); From 71ceeafa3bc5642a713cce3b4626f39f548e4943 Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Tue, 29 Apr 2025 12:07:21 -0700 Subject: [PATCH 35/86] production phase 7 - clean up the hook Signed-off-by: Jialiang Liang --- src/plugins/dashboard/framework/constants.tsx | 92 ----- src/plugins/dashboard/framework/types.tsx | 367 ------------------ .../dashboard/framework/utils/shared.ts | 141 ------- 3 files changed, 600 deletions(-) diff --git a/src/plugins/dashboard/framework/constants.tsx b/src/plugins/dashboard/framework/constants.tsx index 785e17bbaa95..4878352d8355 100644 --- a/src/plugins/dashboard/framework/constants.tsx +++ b/src/plugins/dashboard/framework/constants.tsx @@ -4,96 +4,4 @@ */ export const ASYNC_QUERY_SESSION_ID = 'async-query-session-id'; - -export const DATA_SOURCE_NAME_URL_PARAM_KEY = 'datasourceName'; -export const DATA_SOURCE_TYPE_URL_PARAM_KEY = 'datasourceType'; -export const OLLY_QUESTION_URL_PARAM_KEY = 'olly_q'; -export const INDEX_URL_PARAM_KEY = 'indexPattern'; -export const DEFAULT_DATA_SOURCE_TYPE = 'DEFAULT_INDEX_PATTERNS'; -export const DEFAULT_DATA_SOURCE_NAME = 'Default cluster'; -export const DEFAULT_DATA_SOURCE_OBSERVABILITY_DISPLAY_NAME = 'OpenSearch'; -export const DEFAULT_DATA_SOURCE_TYPE_NAME = 'Default Group'; -export const enum QUERY_LANGUAGE { - PPL = 'PPL', - SQL = 'SQL', - DQL = 'DQL', -} -export enum DATA_SOURCE_TYPES { - DEFAULT_CLUSTER_TYPE = DEFAULT_DATA_SOURCE_TYPE, - SPARK = 'spark', - S3Glue = 's3glue', -} export const ASYNC_POLLING_INTERVAL = 2000; - -export const CATALOG_CACHE_VERSION = '1.0'; -export const ACCELERATION_DEFUALT_SKIPPING_INDEX_NAME = 'skipping'; -export const ACCELERATION_TIME_INTERVAL = [ - { text: 'millisecond(s)', value: 'millisecond' }, - { text: 'second(s)', value: 'second' }, - { text: 'minutes(s)', value: 'minute' }, - { text: 'hour(s)', value: 'hour' }, - { text: 'day(s)', value: 'day' }, - { text: 'week(s)', value: 'week' }, -]; -export const ACCELERATION_REFRESH_TIME_INTERVAL = [ - { text: 'minutes(s)', value: 'minute' }, - { text: 'hour(s)', value: 'hour' }, - { text: 'day(s)', value: 'day' }, - { text: 'week(s)', value: 'week' }, -]; - -export const ACCELERATION_ADD_FIELDS_TEXT = '(add fields here)'; -export const ACCELERATION_INDEX_NAME_REGEX = /^[a-z0-9_]+$/; -export const ACCELERATION_S3_URL_REGEX = /^(s3|s3a):\/\/[a-zA-Z0-9.\-]+/; -export const SPARK_HIVE_TABLE_REGEX = /Provider:\s*hive/; -export const SANITIZE_QUERY_REGEX = /\s+/g; -export const SPARK_TIMESTAMP_DATATYPE = 'timestamp'; -export const SPARK_STRING_DATATYPE = 'string'; - -export const ACCELERATION_INDEX_TYPES = [ - { label: 'Skipping Index', value: 'skipping' }, - { label: 'Covering Index', value: 'covering' }, - { label: 'Materialized View', value: 'materialized' }, -]; - -export const ACC_INDEX_TYPE_DOCUMENTATION_URL = - 'https://github.com/opensearch-project/opensearch-spark/blob/main/docs/index.md'; -export const ACC_CHECKPOINT_DOCUMENTATION_URL = - 'https://github.com/opensearch-project/opensearch-spark/blob/main/docs/index.md#create-index-options'; - -export const ACCELERATION_INDEX_NAME_INFO = `All OpenSearch acceleration indices have a naming format of pattern: \`prefix__suffix\`. They share a common prefix structure, which is \`flint___
_\`. Additionally, they may have a suffix that varies based on the index type. -##### Skipping Index -- For 'Skipping' indices, a fixed index name 'skipping' is used, and this name cannot be modified by the user. The suffix added to this type is \`_index\`. - - An example of a 'Skipping' index name would be: \`flint_mydatasource_mydb_mytable_skipping_index\`. -##### Covering Index -- 'Covering' indices allow users to specify their index name. The suffix added to this type is \`_index\`. - - For instance, a 'Covering' index name could be: \`flint_mydatasource_mydb_mytable_myindexname_index\`. -##### Materialized View Index -- 'Materialized View' indices also enable users to define their index name, but they do not have a suffix. - - An example of a 'Materialized View' index name might look like: \`flint_mydatasource_mydb_mytable_myindexname\`. -##### Note: -- All user given index names must be in lowercase letters, numbers and underscore. Spaces, commas, and characters -, :, ", *, +, /, \, |, ?, #, >, or < are not allowed. - `; - -export const SKIPPING_INDEX_ACCELERATION_METHODS = [ - { value: 'PARTITION', text: 'Partition' }, - { value: 'VALUE_SET', text: 'Value Set' }, - { value: 'MIN_MAX', text: 'Min Max' }, - { value: 'BLOOM_FILTER', text: 'Bloom Filter' }, -]; - -export const ACCELERATION_AGGREGRATION_FUNCTIONS = [ - { label: 'window.start' }, - { label: 'count' }, - { label: 'sum' }, - { label: 'avg' }, - { label: 'max' }, - { label: 'min' }, -]; - -export const SPARK_PARTITION_INFO = `# Partition Information`; -export const OBS_DEFAULT_CLUSTER = 'observability-default'; // prefix key for generating data source id for default cluster in data selector -export const OBS_S3_DATA_SOURCE = 'observability-s3'; // prefix key for generating data source id for s3 data sources in data selector -export const S3_DATA_SOURCE_GROUP_DISPLAY_NAME = 'Amazon S3'; // display group name for Amazon-managed-s3 data sources in data selector -export const S3_DATA_SOURCE_GROUP_SPARK_DISPLAY_NAME = 'Spark'; // display group name for OpenSearch-spark-s3 data sources in data selector -export const SECURITY_DASHBOARDS_LOGOUT_URL = '/logout'; diff --git a/src/plugins/dashboard/framework/types.tsx b/src/plugins/dashboard/framework/types.tsx index 73c83d85ad0b..e8010cc44005 100644 --- a/src/plugins/dashboard/framework/types.tsx +++ b/src/plugins/dashboard/framework/types.tsx @@ -3,8 +3,6 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { EuiComboBoxOptionOption } from '@elastic/eui'; - export enum DirectQueryLoadingStatus { SUCCESS = 'success', FAILED = 'failed', @@ -23,368 +21,3 @@ export interface DirectQueryRequest { datasource: string; sessionId?: string; } - -export type AccelerationStatus = 'ACTIVE' | 'INACTIVE'; - -export interface PermissionsConfigurationProps { - roles: Role[]; - selectedRoles: Role[]; - setSelectedRoles: React.Dispatch>; - layout: 'horizontal' | 'vertical'; - hasSecurityAccess: boolean; -} - -export interface TableColumn { - name: string; - dataType: string; -} - -export interface Acceleration { - name: string; - status: AccelerationStatus; - type: string; - database: string; - table: string; - destination: string; - dateCreated: number; - dateUpdated: number; - index: string; - sql: string; -} - -export interface AssociatedObject { - tableName: string; - datasource: string; - id: string; - name: string; - database: string; - type: AssociatedObjectIndexType; - accelerations: CachedAcceleration[] | AssociatedObject; - columns?: CachedColumn[]; -} - -export type Role = EuiComboBoxOptionOption; - -export type DatasourceType = 'S3GLUE' | 'PROMETHEUS'; - -export interface S3GlueProperties { - 'glue.indexstore.opensearch.uri': string; - 'glue.indexstore.opensearch.region': string; -} - -export interface PrometheusProperties { - 'prometheus.uri': string; -} - -export type DatasourceStatus = 'ACTIVE' | 'DISABLED'; - -export interface DatasourceDetails { - allowedRoles: string[]; - name: string; - connector: DatasourceType; - description: string; - properties: S3GlueProperties | PrometheusProperties; - status: DatasourceStatus; -} - -interface AsyncApiDataResponse { - status: string; - schema?: Array<{ name: string; type: string }>; - datarows?: any; - total?: number; - size?: number; - error?: string; -} - -export interface AsyncApiResponse { - data: { - ok: boolean; - resp: AsyncApiDataResponse; - }; -} - -export type PollingCallback = (statusObj: AsyncApiResponse) => void; - -export type AssociatedObjectIndexType = AccelerationIndexType | 'table'; - -export type AccelerationIndexType = 'skipping' | 'covering' | 'materialized'; - -export type LoadCacheType = 'databases' | 'tables' | 'accelerations' | 'tableColumns'; - -export enum CachedDataSourceStatus { - Updated = 'Updated', - Failed = 'Failed', - Empty = 'Empty', -} - -export interface CachedColumn { - fieldName: string; - dataType: string; -} - -export interface CachedTable { - name: string; - columns?: CachedColumn[]; -} - -export interface CachedDatabase { - name: string; - tables: CachedTable[]; - lastUpdated: string; // date string in UTC format - status: CachedDataSourceStatus; -} - -export interface CachedDataSource { - name: string; - lastUpdated: string; // date string in UTC format - status: CachedDataSourceStatus; - databases: CachedDatabase[]; - dataSourceMDSId?: string; -} - -export interface DataSourceCacheData { - version: string; - dataSources: CachedDataSource[]; -} - -export interface CachedAcceleration { - flintIndexName: string; - type: AccelerationIndexType; - database: string; - table: string; - indexName: string; - autoRefresh: boolean; - status: string; -} - -export interface CachedAccelerationByDataSource { - name: string; - accelerations: CachedAcceleration[]; - lastUpdated: string; // date string in UTC format - status: CachedDataSourceStatus; - dataSourceMDSId?: string; -} - -export interface AccelerationsCacheData { - version: string; - dataSources: CachedAccelerationByDataSource[]; -} - -export interface PollingSuccessResult { - schema: Array<{ name: string; type: string }>; - datarows: Array>; -} - -export type AsyncPollingResult = PollingSuccessResult | null; - -export type AggregationFunctionType = 'count' | 'sum' | 'avg' | 'max' | 'min' | 'window.start'; - -export interface MaterializedViewColumn { - id: string; - functionName: AggregationFunctionType; - functionParam?: string; - fieldAlias?: string; -} - -export type SkippingIndexAccMethodType = 'PARTITION' | 'VALUE_SET' | 'MIN_MAX' | 'BLOOM_FILTER'; - -export interface SkippingIndexRowType { - id: string; - fieldName: string; - dataType: string; - accelerationMethod: SkippingIndexAccMethodType; -} - -export interface DataTableFieldsType { - id: string; - fieldName: string; - dataType: string; -} - -export interface RefreshIntervalType { - refreshWindow: number; - refreshInterval: string; -} - -export interface WatermarkDelayType { - delayWindow: number; - delayInterval: string; -} - -export interface GroupByTumbleType { - timeField: string; - tumbleWindow: number; - tumbleInterval: string; -} - -export interface MaterializedViewQueryType { - columnsValues: MaterializedViewColumn[]; - groupByTumbleValue: GroupByTumbleType; -} - -export interface FormErrorsType { - dataSourceError: string[]; - databaseError: string[]; - dataTableError: string[]; - skippingIndexError: string[]; - coveringIndexError: string[]; - materializedViewError: string[]; - indexNameError: string[]; - primaryShardsError: string[]; - replicaShardsError: string[]; - refreshIntervalError: string[]; - checkpointLocationError: string[]; - watermarkDelayError: string[]; -} - -export type AccelerationRefreshType = 'autoInterval' | 'manual' | 'manualIncrement'; - -export interface CreateAccelerationForm { - dataSource: string; - database: string; - dataTable: string; - dataTableFields: DataTableFieldsType[]; - accelerationIndexType: AccelerationIndexType; - skippingIndexQueryData: SkippingIndexRowType[]; - coveringIndexQueryData: string[]; - materializedViewQueryData: MaterializedViewQueryType; - accelerationIndexName: string; - primaryShardsCount: number; - replicaShardsCount: number; - refreshType: AccelerationRefreshType; - checkpointLocation: string | undefined; - watermarkDelay: WatermarkDelayType; - refreshIntervalOptions: RefreshIntervalType; - formErrors: FormErrorsType; -} - -export interface LoadCachehookOutput { - loadStatus: DirectQueryLoadingStatus; - startLoading: (params: StartLoadingParams) => void; - stopLoading: () => void; -} - -export interface StartLoadingParams { - dataSourceName: string; - dataSourceMDSId?: string; - databaseName?: string; - tableName?: string; -} - -export interface RenderAccelerationFlyoutParams { - dataSourceName: string; - dataSourceMDSId?: string; - databaseName?: string; - tableName?: string; - handleRefresh?: () => void; -} - -export interface RenderAssociatedObjectsDetailsFlyoutParams { - tableDetail: AssociatedObject; - dataSourceName: string; - handleRefresh?: () => void; - dataSourceMDSId?: string; -} - -export interface RenderAccelerationDetailsFlyoutParams { - acceleration: CachedAcceleration; - dataSourceName: string; - handleRefresh?: () => void; - dataSourceMDSId?: string; -} - -// Integration types - -export interface StaticAsset { - annotation?: string; - path: string; -} - -export interface IntegrationWorkflow { - name: string; - label: string; - description: string; - enabled_by_default: boolean; -} - -export interface IntegrationStatics { - logo?: StaticAsset; - gallery?: StaticAsset[]; - darkModeLogo?: StaticAsset; - darkModeGallery?: StaticAsset[]; -} - -export interface IntegrationComponent { - name: string; - version: string; -} - -export type SupportedAssetType = 'savedObjectBundle' | 'query'; - -export interface IntegrationAsset { - name: string; - version: string; - extension: string; - type: SupportedAssetType; - workflows?: string[]; -} - -export interface IntegrationConfig { - name: string; - version: string; - displayName?: string; - license: string; - type: string; - labels?: string[]; - author?: string; - description?: string; - sourceUrl?: string; - workflows?: IntegrationWorkflow[]; - statics?: IntegrationStatics; - components: IntegrationComponent[]; - assets: IntegrationAsset[]; - sampleData?: { - path: string; - }; -} - -export interface AvailableIntegrationsList { - hits: IntegrationConfig[]; -} - -export interface AssetReference { - assetType: string; - assetId: string; - isDefaultAsset: boolean; - description: string; - status?: string; -} - -export interface IntegrationInstanceResult extends IntegrationInstance { - id: string; - status: string; -} - -export interface IntegrationInstance { - name: string; - templateName: string; - dataSource: string; - creationDate: string; - assets: AssetReference[]; -} - -export interface AvailableIntegrationsList { - hits: IntegrationConfig[]; -} - -export interface IntegrationInstancesSearchResult { - hits: IntegrationInstanceResult[]; -} - -export type ParsedIntegrationAsset = - | { type: 'savedObjectBundle'; workflows?: string[]; data: object[] } - | { type: 'query'; workflows?: string[]; query: string; language: string }; - -export type Result = - | { ok: true; value: T; error?: undefined } - | { ok: false; error: E; value?: undefined }; diff --git a/src/plugins/dashboard/framework/utils/shared.ts b/src/plugins/dashboard/framework/utils/shared.ts index 49f5f0d85390..e575767218f3 100644 --- a/src/plugins/dashboard/framework/utils/shared.ts +++ b/src/plugins/dashboard/framework/utils/shared.ts @@ -7,35 +7,6 @@ export function get(obj: Record, path: string, default return path.split('.').reduce((acc: any, part: string) => acc && acc[part], obj) || defaultValue; } -export function addBackticksIfNeeded(input: string): string { - if (input === undefined) { - return ''; - } - // Check if the string already has backticks - if (input.startsWith('`') && input.endsWith('`')) { - return input; // Return the string as it is - } - // Add backticks to the string - return '`' + input + '`'; -} - -export function combineSchemaAndDatarows( - schema: Array<{ name: string; type: string }>, - datarows: Array> -): object[] { - const combinedData: object[] = []; - - datarows.forEach((row) => { - const rowData: { [key: string]: string | number | boolean } = {}; - schema.forEach((field, index) => { - rowData[field.name] = row[index]; - }); - combinedData.push(rowData); - }); - - return combinedData; -} - export const formatError = (name: string, message: string, details: string) => { return { name, @@ -52,115 +23,3 @@ export const formatError = (name: string, message: string, details: string) => { }, }; }; - -// Client route -export const DIRECT_QUERY_BASE = '/api/dashboard'; -export const PPL_BASE = `${DIRECT_QUERY_BASE}/ppl`; -export const PPL_SEARCH = '/search'; -export const DSL_BASE = `${DIRECT_QUERY_BASE}/dsl`; -export const DSL_SEARCH = '/search'; -export const DSL_CAT = '/cat.indices'; -export const DSL_MAPPING = '/indices.getFieldMapping'; -export const DSL_SETTINGS = '/indices.getFieldSettings'; -export const DSM_BASE = '/api/dashboard'; -export const INTEGRATIONS_BASE = '/api/integrations'; -export const JOBS_BASE = '/query/jobs'; -export const DATACONNECTIONS_BASE = `${DIRECT_QUERY_BASE}/dataconnections`; -export const EDIT = '/edit'; -export const DATACONNECTIONS_UPDATE_STATUS = '/status'; -export const SECURITY_ROLES = '/api/v1/configuration/roles'; -export const EVENT_ANALYTICS = '/event_analytics'; -export const SAVED_OBJECTS = '/saved_objects'; -export const SAVED_QUERY = '/query'; -export const SAVED_VISUALIZATION = '/vis'; -export const CONSOLE_PROXY = '/api/console/proxy'; -export const SECURITY_PLUGIN_ACCOUNT_API = '/api/v1/configuration/account'; - -// Server route -export const PPL_ENDPOINT = '/_plugins/_ppl'; -export const SQL_ENDPOINT = '/_plugins/_sql'; -export const DSL_ENDPOINT = '/_plugins/_dsl'; -export const DATACONNECTIONS_ENDPOINT = '/_plugins/_query/_datasources'; -export const JOBS_ENDPOINT_BASE = '/_plugins/_async_query'; -export const JOB_RESULT_ENDPOINT = '/result'; - -export const observabilityID = 'observability-logs'; -export const observabilityTitle = 'Observability'; -export const observabilityPluginOrder = 1500; - -export const observabilityApplicationsID = 'observability-applications'; -export const observabilityApplicationsTitle = 'Applications'; -export const observabilityApplicationsPluginOrder = 5090; - -export const observabilityLogsID = 'observability-logs'; -export const observabilityLogsTitle = 'Logs'; -export const observabilityLogsPluginOrder = 5091; - -export const observabilityMetricsID = 'observability-metrics'; -export const observabilityMetricsTitle = 'Metrics'; -export const observabilityMetricsPluginOrder = 5092; - -export const observabilityTracesID = 'observability-traces'; -export const observabilityTracesTitle = 'Traces'; -export const observabilityTracesPluginOrder = 5093; - -export const observabilityNotebookID = 'observability-notebooks'; -export const observabilityNotebookTitle = 'Notebooks'; -export const observabilityNotebookPluginOrder = 5094; - -export const observabilityPanelsID = 'observability-dashboards'; -export const observabilityPanelsTitle = 'Dashboards'; -export const observabilityPanelsPluginOrder = 5095; - -export const observabilityIntegrationsID = 'integrations'; -export const observabilityIntegrationsTitle = 'Integrations'; -export const observabilityIntegrationsPluginOrder = 9020; - -export const observabilityDataConnectionsID = 'datasources'; -export const observabilityDataConnectionsTitle = 'Data sources'; -export const observabilityDataConnectionsPluginOrder = 9030; - -export const queryWorkbenchPluginID = 'opensearch-query-workbench'; -export const queryWorkbenchPluginCheck = 'plugin:queryWorkbenchDashboards'; - -// Observability plugin URI -const BASE_OBSERVABILITY_URI = '/_plugins/_observability'; -const BASE_DATACONNECTIONS_URI = '/_plugins/_query/_datasources'; -export const OPENSEARCH_PANELS_API = { - OBJECT: `${BASE_OBSERVABILITY_URI}/object`, -}; -export const OPENSEARCH_DATACONNECTIONS_API = { - DATACONNECTION: `${BASE_DATACONNECTIONS_URI}`, -}; - -// Saved Objects -export const SAVED_OBJECT = '/object'; - -export const S3_DATA_SOURCE_TYPE = 's3glue'; - -export const ASYNC_QUERY_SESSION_ID = 'async-query-session-id'; -export const ASYNC_QUERY_DATASOURCE_CACHE = 'async-query-catalog-cache'; -export const ASYNC_QUERY_ACCELERATIONS_CACHE = 'async-query-acclerations-cache'; - -export const DIRECT_DUMMY_QUERY = 'select 1'; - -export enum DirectQueryLoadingStatus { - SUCCESS = 'success', - FAILED = 'failed', - RUNNING = 'running', - SCHEDULED = 'scheduled', - CANCELED = 'canceled', - WAITING = 'waiting', - INITIAL = 'initial', -} -const catalogCacheFetchingStatus = [ - DirectQueryLoadingStatus.RUNNING, - DirectQueryLoadingStatus.WAITING, - DirectQueryLoadingStatus.SCHEDULED, -]; - -export const isCatalogCacheFetching = (...statuses: DirectQueryLoadingStatus[]) => { - return statuses.some((status: DirectQueryLoadingStatus) => - catalogCacheFetchingStatus.includes(status) - ); -}; From 05ab96f1d5d57745e165c3bb4c037277cd434a06 Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Tue, 29 Apr 2025 15:56:06 -0700 Subject: [PATCH 36/86] production phase 8 - rename the sync component as direct query instead, and not calling it flint and add tests Signed-off-by: Jialiang Liang --- .../grid/dashboard_direct_query_sync.test.tsx | 88 +++++++++++++++++++ ...nc.tsx => dashboard_direct_query_sync.tsx} | 19 +--- .../embeddable/grid/dashboard_grid.tsx | 4 +- 3 files changed, 94 insertions(+), 17 deletions(-) create mode 100644 src/plugins/dashboard/public/application/embeddable/grid/dashboard_direct_query_sync.test.tsx rename src/plugins/dashboard/public/application/embeddable/grid/{dashboard_flint_sync.tsx => dashboard_direct_query_sync.tsx} (86%) diff --git a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_direct_query_sync.test.tsx b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_direct_query_sync.test.tsx new file mode 100644 index 000000000000..c5858c62c9fa --- /dev/null +++ b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_direct_query_sync.test.tsx @@ -0,0 +1,88 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Any modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import React from 'react'; +import { render, screen, fireEvent } from '@testing-library/react'; +import '@testing-library/jest-dom'; +import { DashboardDirectQuerySync } from './dashboard_direct_query_sync'; + +describe('DashboardDirectQuerySync', () => { + const mockSynchronize = jest.fn(); + + it('renders sync info with refresh link when state is terminal', () => { + render( + + ); + + expect(screen.getByText(/Data scheduled to sync every/i)).toBeInTheDocument(); + expect(screen.getByText(/Sync data/i)).toBeInTheDocument(); + + fireEvent.click(screen.getByText(/Sync data/i)); + expect(mockSynchronize).toHaveBeenCalled(); + }); + + it('renders loading spinner when state is not terminal', () => { + render( + + ); + + expect(screen.getByText(/Data sync is in progress/i)).toBeInTheDocument(); + expect(screen.queryByText(/Sync data/i)).toBeNull(); + }); + + it('handles missing lastRefreshTime and refreshInterval gracefully', () => { + render( + + ); + + expect( + screen.getByText( + (content) => content.includes('Data scheduled to sync every') && content.includes('--') + ) + ).toBeInTheDocument(); + + expect(screen.getByText(/Sync data/i)).toBeInTheDocument(); + }); +}); diff --git a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_flint_sync.tsx b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_direct_query_sync.tsx similarity index 86% rename from src/plugins/dashboard/public/application/embeddable/grid/dashboard_flint_sync.tsx rename to src/plugins/dashboard/public/application/embeddable/grid/dashboard_direct_query_sync.tsx index d37deab8597d..e57539fb7068 100644 --- a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_flint_sync.tsx +++ b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_direct_query_sync.tsx @@ -29,29 +29,18 @@ */ import React from 'react'; -import { - EuiButton, - EuiCallOut, - EuiLink, - EuiLoadingSpinner, - EuiProgress, - EuiText, -} from '@elastic/eui'; +import { EuiCallOut, EuiLink, EuiLoadingSpinner, EuiText } from '@elastic/eui'; import { DirectQueryLoadingStatus } from '../../../../framework/types'; -import { - EMR_STATES, - MAX_ORD, - intervalAsMinutes, -} from '../../utils/direct_query_sync/direct_query_sync'; +import { EMR_STATES, intervalAsMinutes } from '../../utils/direct_query_sync/direct_query_sync'; -interface DashboardFlintSyncProps { +interface DashboardDirectQuerySyncProps { loadStatus: DirectQueryLoadingStatus; lastRefreshTime?: number; refreshInterval?: number; onSynchronize: () => void; } -export const DashboardFlintSync: React.FC = ({ +export const DashboardDirectQuerySync: React.FC = ({ loadStatus, lastRefreshTime, refreshInterval, diff --git a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx index 97524d5772a5..ef094afe56f3 100644 --- a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx +++ b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx @@ -55,7 +55,7 @@ import { generateRefreshQuery, EMR_STATES, } from '../../utils/direct_query_sync/direct_query_sync'; -import { DashboardFlintSync } from './dashboard_flint_sync'; +import { DashboardDirectQuerySync } from './dashboard_direct_query_sync'; import { isDirectQuerySyncEnabledByUrl } from '../../utils/direct_query_sync/direct_query_sync_url_flag'; let lastValidGridSize = 0; @@ -407,7 +407,7 @@ class DashboardGridUi extends React.Component { const shouldRenderSyncUI = featureFlagEnabled && metadataAvailable; return shouldRenderSyncUI ? ( - Date: Tue, 29 Apr 2025 16:00:44 -0700 Subject: [PATCH 37/86] minor cleanup Signed-off-by: Jialiang Liang --- src/plugins/dashboard/public/plugin.tsx | 6 +++--- .../data/public/ui/query_string_input/query_bar_top_row.tsx | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/plugins/dashboard/public/plugin.tsx b/src/plugins/dashboard/public/plugin.tsx index f7f2ba72947c..ef8fcb5be91d 100644 --- a/src/plugins/dashboard/public/plugin.tsx +++ b/src/plugins/dashboard/public/plugin.tsx @@ -277,9 +277,9 @@ export class DashboardPlugin SavedObjectFinder: getSavedObjectFinder(coreStart.savedObjects, coreStart.uiSettings), ExitFullScreenButton, uiActions: deps.uiActions, - savedObjectsClient: coreStart.savedObjects.client, // HERE TO ADD SAVED OBJECTS CLIENT - http: coreStart.http, // HERE TO ADD HTTP - dashboardFeatureFlagConfig: this.dashboardFeatureFlagConfig!, // HERE TO PASS THE DASHBOARD FEATURE FLAG CONFIG + savedObjectsClient: coreStart.savedObjects.client, + http: coreStart.http, + dashboardFeatureFlagConfig: this.dashboardFeatureFlagConfig!, }; }; diff --git a/src/plugins/data/public/ui/query_string_input/query_bar_top_row.tsx b/src/plugins/data/public/ui/query_string_input/query_bar_top_row.tsx index 1ec3cda6c388..63cd290273bb 100644 --- a/src/plugins/data/public/ui/query_string_input/query_bar_top_row.tsx +++ b/src/plugins/data/public/ui/query_string_input/query_bar_top_row.tsx @@ -408,7 +408,6 @@ export default function QueryBarTopRow(props: QueryBarTopRowProps) { ? createPortal(renderUpdateButton(), props.datePickerRef!.current!) : renderUpdateButton()} - {/* Here is the place if we consider to add the button at top bar Sync button should be here? */} ); From 499f4c28d971f1114d0f553132884ec126da570a Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Tue, 29 Apr 2025 17:07:50 -0700 Subject: [PATCH 38/86] production phase 9 - fix existing grid test and some broken snapshots Signed-off-by: Jialiang Liang --- .../__snapshots__/dashboard_top_nav.test.tsx.snap | 6 ++++++ .../embeddable/grid/dashboard_grid.test.tsx | 15 ++++++++++++++- .../embeddable/grid/dashboard_grid.tsx | 2 +- .../dashboard/public/application/utils/mocks.ts | 4 ++++ 4 files changed, 25 insertions(+), 2 deletions(-) diff --git a/src/plugins/dashboard/public/application/components/dashboard_top_nav/__snapshots__/dashboard_top_nav.test.tsx.snap b/src/plugins/dashboard/public/application/components/dashboard_top_nav/__snapshots__/dashboard_top_nav.test.tsx.snap index 6f6c59afe8d5..76b01f5bf83b 100644 --- a/src/plugins/dashboard/public/application/components/dashboard_top_nav/__snapshots__/dashboard_top_nav.test.tsx.snap +++ b/src/plugins/dashboard/public/application/components/dashboard_top_nav/__snapshots__/dashboard_top_nav.test.tsx.snap @@ -921,6 +921,7 @@ exports[`Dashboard top nav render in embed mode 1`] = ` }, "savedObjectsClient": Object { "find": [MockFunction], + "get": [MockFunction], }, "savedObjectsPublic": Object { "settings": Object { @@ -2048,6 +2049,7 @@ exports[`Dashboard top nav render in embed mode, and force hide filter bar 1`] = }, "savedObjectsClient": Object { "find": [MockFunction], + "get": [MockFunction], }, "savedObjectsPublic": Object { "settings": Object { @@ -3175,6 +3177,7 @@ exports[`Dashboard top nav render in embed mode, components can be forced show b }, "savedObjectsClient": Object { "find": [MockFunction], + "get": [MockFunction], }, "savedObjectsPublic": Object { "settings": Object { @@ -4302,6 +4305,7 @@ exports[`Dashboard top nav render in full screen mode with appended URL param bu }, "savedObjectsClient": Object { "find": [MockFunction], + "get": [MockFunction], }, "savedObjectsPublic": Object { "settings": Object { @@ -5429,6 +5433,7 @@ exports[`Dashboard top nav render in full screen mode, no componenets should be }, "savedObjectsClient": Object { "find": [MockFunction], + "get": [MockFunction], }, "savedObjectsPublic": Object { "settings": Object { @@ -6556,6 +6561,7 @@ exports[`Dashboard top nav render with all components 1`] = ` }, "savedObjectsClient": Object { "find": [MockFunction], + "get": [MockFunction], }, "savedObjectsPublic": Object { "settings": Object { diff --git a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.test.tsx b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.test.tsx index f143b18b7c48..91e1e8354363 100644 --- a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.test.tsx +++ b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.test.tsx @@ -42,6 +42,7 @@ import { ContactCardEmbeddableFactory, } from '../../../../../embeddable/public/lib/test_samples'; import { embeddablePluginMock } from '../../../../../embeddable/public/mocks'; +import { createDashboardServicesMock } from '../../utils/mocks'; import { OpenSearchDashboardsContextProvider } from '../../../../../opensearch_dashboards_react/public'; let dashboardContainer: DashboardContainer | undefined; @@ -90,11 +91,23 @@ function prepare(props?: Partial) { } as any, }; dashboardContainer = new DashboardContainer(initialInput, options); + + const services = createDashboardServicesMock(); + const defaultTestProps: DashboardGridProps = { container: dashboardContainer, PanelComponent: () =>
, - opensearchDashboards: null as any, + opensearchDashboards: { + services, + }, intl: null as any, + savedObjectsClient: services.savedObjectsClient, + http: services.http, + notifications: services.notifications, + startLoading: jest.fn(), + loadStatus: 'fresh', + pollingResult: {}, + isDirectQuerySyncEnabled: false, }; return { diff --git a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx index ef094afe56f3..0bb80808074a 100644 --- a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx +++ b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx @@ -391,7 +391,7 @@ class DashboardGridUi extends React.Component { const isViewMode = viewMode === ViewMode.VIEW; const state = EMR_STATES.get(this.props.loadStatus as string)!; - if (state.terminal && this.props.loadStatus !== 'fresh') { + if (state?.terminal && this.props.loadStatus !== 'fresh') { window.location.reload(); } diff --git a/src/plugins/dashboard/public/application/utils/mocks.ts b/src/plugins/dashboard/public/application/utils/mocks.ts index 529808eb6258..213bf8f879eb 100644 --- a/src/plugins/dashboard/public/application/utils/mocks.ts +++ b/src/plugins/dashboard/public/application/utils/mocks.ts @@ -40,6 +40,10 @@ export const createDashboardServicesMock = () => { embeddable, savedObjectsClient: { find: jest.fn(), + get: jest.fn().mockResolvedValue({ + attributes: { title: 'flint_ds1_db1_index1' }, + references: [], + }), }, savedObjectsPublic: { settings: { From 614da99fa075c7b7676f8fbef55e79ece0593a72 Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Tue, 29 Apr 2025 17:39:55 -0700 Subject: [PATCH 39/86] production phase 9 - fix existing viewport tests Signed-off-by: Jialiang Liang --- .../viewport/dashboard_viewport.test.tsx | 17 ++++++++++++++++- .../dashboard/public/application/utils/mocks.ts | 2 +- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/src/plugins/dashboard/public/application/embeddable/viewport/dashboard_viewport.test.tsx b/src/plugins/dashboard/public/application/embeddable/viewport/dashboard_viewport.test.tsx index 7d286f6fbf49..7569dd9b6e25 100644 --- a/src/plugins/dashboard/public/application/embeddable/viewport/dashboard_viewport.test.tsx +++ b/src/plugins/dashboard/public/application/embeddable/viewport/dashboard_viewport.test.tsx @@ -74,7 +74,9 @@ function getProps( getEmbeddableFactory: start.getEmbeddableFactory, } as any, chrome: {} as any, - notifications: {} as any, + notifications: { + toasts: { addDanger: jest.fn(), addSuccess: jest.fn(), addWarning: jest.fn() }, + } as any, overlays: {} as any, inspector: { isAvailable: jest.fn(), @@ -106,6 +108,19 @@ function getProps( container: dashboardContainer, logos: options.chrome.logos, PanelComponent: () =>
, + savedObjectsClient: { + get: jest.fn().mockResolvedValue({ + attributes: { title: 'test-pattern' }, + references: [], + }), + find: jest.fn(), + } as any, + http: { get: jest.fn() } as any, + notifications: options.notifications, + startLoading: jest.fn(), + loadStatus: 'fresh', + pollingResult: {}, + isDirectQuerySyncEnabled: false, }; return { diff --git a/src/plugins/dashboard/public/application/utils/mocks.ts b/src/plugins/dashboard/public/application/utils/mocks.ts index 213bf8f879eb..52731f3b3e10 100644 --- a/src/plugins/dashboard/public/application/utils/mocks.ts +++ b/src/plugins/dashboard/public/application/utils/mocks.ts @@ -42,7 +42,7 @@ export const createDashboardServicesMock = () => { find: jest.fn(), get: jest.fn().mockResolvedValue({ attributes: { title: 'flint_ds1_db1_index1' }, - references: [], + references: [{ type: 'data-source', id: 'test-mds' }], }), }, savedObjectsPublic: { From d6c3ffdab097090f9ea3ee31f0928ca9f648a6d8 Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Tue, 29 Apr 2025 18:45:36 -0700 Subject: [PATCH 40/86] production phase 10 - add test cases in dashboards grid test Signed-off-by: Jialiang Liang --- .../grid/dashboard_direct_query_sync.tsx | 2 +- .../embeddable/grid/dashboard_grid.test.tsx | 104 ++++++++++++++++++ 2 files changed, 105 insertions(+), 1 deletion(-) diff --git a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_direct_query_sync.tsx b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_direct_query_sync.tsx index e57539fb7068..f35364b07e35 100644 --- a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_direct_query_sync.tsx +++ b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_direct_query_sync.tsx @@ -33,7 +33,7 @@ import { EuiCallOut, EuiLink, EuiLoadingSpinner, EuiText } from '@elastic/eui'; import { DirectQueryLoadingStatus } from '../../../../framework/types'; import { EMR_STATES, intervalAsMinutes } from '../../utils/direct_query_sync/direct_query_sync'; -interface DashboardDirectQuerySyncProps { +export interface DashboardDirectQuerySyncProps { loadStatus: DirectQueryLoadingStatus; lastRefreshTime?: number; refreshInterval?: number; diff --git a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.test.tsx b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.test.tsx index 91e1e8354363..afb5a472f41d 100644 --- a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.test.tsx +++ b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.test.tsx @@ -44,6 +44,32 @@ import { import { embeddablePluginMock } from '../../../../../embeddable/public/mocks'; import { createDashboardServicesMock } from '../../utils/mocks'; import { OpenSearchDashboardsContextProvider } from '../../../../../opensearch_dashboards_react/public'; +import { DashboardDirectQuerySyncProps } from './dashboard_direct_query_sync'; +import { + extractIndexInfoFromDashboard, + generateRefreshQuery, +} from '../../utils/direct_query_sync/direct_query_sync'; + +jest.mock('../../utils/direct_query_sync/direct_query_sync', () => { + const actual = jest.requireActual('../../utils/direct_query_sync/direct_query_sync'); + return { + ...actual, + extractIndexInfoFromDashboard: jest.fn(), + generateRefreshQuery: jest.fn(), + EMR_STATES: new Map([ + ['submitted', { ord: 0, terminal: false }], + ['queued', { ord: 10, terminal: false }], + ['pending', { ord: 20, terminal: false }], + ['scheduled', { ord: 30, terminal: false }], + ['running', { ord: 70, terminal: false }], + ['cancelling', { ord: 90, terminal: false }], + ['success', { ord: 100, terminal: true }], + ['failed', { ord: 100, terminal: true }], + ['cancelled', { ord: 100, terminal: true }], + ['fresh', { ord: 100, terminal: true }], + ]), + }; +}); let dashboardContainer: DashboardContainer | undefined; @@ -212,3 +238,81 @@ test('DashboardGrid unmount unsubscribes', (done) => { props.container.updateInput({ expandedPanelId: '1' }); }); + +test('renders sync UI when feature flag is enabled and metadata is present', async () => { + const { props, options } = prepare({ isDirectQuerySyncEnabled: true }); + + (extractIndexInfoFromDashboard as jest.Mock).mockResolvedValue({ + parts: { datasource: 'ds', database: 'db', index: 'idx' }, + mapping: { lastRefreshTime: 123456, refreshInterval: 30000 }, + mdsId: '', + }); + + const component = mountWithIntl( + + + + ); + + // Wait for async metadata collection + await new Promise((resolve) => setTimeout(resolve, 0)); + component.update(); + + expect(component.find('DashboardDirectQuerySync').exists()).toBe(true); +}); + +test('does not render sync UI when feature flag is off', async () => { + const { props, options } = prepare({ isDirectQuerySyncEnabled: false }); + + (extractIndexInfoFromDashboard as jest.Mock).mockResolvedValue({ + parts: { datasource: 'ds', database: 'db', index: 'idx' }, + mapping: { lastRefreshTime: 123456, refreshInterval: 30000 }, + mdsId: '', + }); + + const component = mountWithIntl( + + + + ); + + await new Promise((resolve) => setTimeout(resolve, 0)); + component.update(); + + expect(component.find('DashboardDirectQuerySync').exists()).toBe(false); +}); + +test('synchronizeNow triggers REFRESH query generation and startLoading', async () => { + const { props, options } = prepare({ isDirectQuerySyncEnabled: true }); + + const mockRefreshQuery = 'REFRESH MATERIALIZED VIEW ds.db.idx'; + (extractIndexInfoFromDashboard as jest.Mock).mockResolvedValue({ + parts: { datasource: 'ds', database: 'db', index: 'idx' }, + mapping: { lastRefreshTime: 123456, refreshInterval: 30000 }, + mdsId: '', + }); + + (generateRefreshQuery as jest.Mock).mockReturnValue(mockRefreshQuery); + + const startLoadingSpy = jest.fn(); + props.startLoading = startLoadingSpy; + + const component = mountWithIntl( + + + + ); + + await new Promise((resolve) => setTimeout(resolve, 0)); + component.update(); + + (component + .find('DashboardDirectQuerySync') + .props() as DashboardDirectQuerySyncProps).onSynchronize(); + + expect(startLoadingSpy).toHaveBeenCalledWith({ + query: mockRefreshQuery, + lang: 'sql', + datasource: 'ds', + }); +}); From d5fe7180bbd7ef016e6fc8bc52452f7a70d67c09 Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Thu, 1 May 2025 11:18:03 -0700 Subject: [PATCH 41/86] fix lint Signed-off-by: Jialiang Liang --- .../embeddable/grid/dashboard_grid.tsx | 6 ------ .../direct_query_sync/direct_query_sync.ts | 19 ++----------------- .../direct_query_sync_url_flag.ts | 1 - 3 files changed, 2 insertions(+), 24 deletions(-) diff --git a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx index 0bb80808074a..c41f32e643c8 100644 --- a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx +++ b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx @@ -287,21 +287,16 @@ class DashboardGridUi extends React.Component { this.props.savedObjectsClient, this.props.http ); - console.log('Extracted metadata:', indexInfo?.mapping); if (indexInfo) { this.extractedDatasource = indexInfo.parts.datasource; this.extractedDatabase = indexInfo.parts.database; this.extractedIndex = indexInfo.parts.index; this.setState({ extractedProps: indexInfo.mapping }); - console.log('Resolved index info:', indexInfo); if (this.props.setMdsId) { this.props.setMdsId(indexInfo.mdsId); } } else { - console.warn( - 'Dashboard does not qualify for synchronization: inconsistent or unsupported visualization sources.' - ); this.setState({ extractedProps: null }); if (this.props.setMdsId) { this.props.setMdsId(undefined); @@ -319,7 +314,6 @@ class DashboardGridUi extends React.Component { extractedDatabase === 'unknown' || extractedIndex === 'unknown' ) { - console.error('Datasource, database, or index not properly set. Cannot run REFRESH command.'); return; } diff --git a/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.ts b/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.ts index 60e1e458a007..cef1bf278343 100644 --- a/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.ts +++ b/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.ts @@ -57,7 +57,6 @@ export async function resolveConcreteIndex( const matchedIndices = resolved?.indices || []; return matchedIndices.length > 0 ? matchedIndices[0].name : null; } catch (err) { - console.error(`Failed to resolve index pattern "${indexTitle}"`, err); return null; } } @@ -100,22 +99,16 @@ export async function extractIndexInfoFromDashboard( const references = savedObject.references || []; if (references.length === 0) { - continue; // No references, skip (acceptable) + continue; } // Check if there is any non-index-pattern reference if (references.some((ref: any) => ref.type !== 'index-pattern')) { - console.warn( - `Visualization ${panelId} references a non-index-pattern object. Disabling sync.` - ); return null; } const indexPatternRef = references.find((ref: any) => ref.type === 'index-pattern'); if (!indexPatternRef) { - console.warn( - `Visualization ${panelId} does not reference an index-pattern. Disabling sync.` - ); return null; } @@ -126,7 +119,7 @@ export async function extractIndexInfoFromDashboard( indexPatternIds.push(indexPatternRef.id); mdsIds.push(mdsId); } catch (err) { - console.warn(`Skipping panel ${panelId} due to error:`, err); + // Ignoring error: saved object might be missing or invalid } } @@ -165,14 +158,12 @@ export async function fetchIndexMapping( try { const baseUrl = `${DSL_BASE}${DSL_MAPPING}`; const url = mdsId ? `${baseUrl}/dataSourceMDSId=${encodeURIComponent(mdsId)}` : baseUrl; - console.log('url', url); const response = await http.get(url, { query: { index }, }); return response; } catch (err) { - console.error(`Failed to fetch mapping for index "${index}"`, err); return null; } } @@ -188,11 +179,5 @@ export function sourceCheck(indexPatternIds: string[], mdsIds: Array Date: Mon, 5 May 2025 13:32:23 -0700 Subject: [PATCH 42/86] fix ci - 0 Signed-off-by: Jialiang Liang --- .../embeddable/grid/dashboard_grid.test.tsx | 34 ++++++++++++++++--- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.test.tsx b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.test.tsx index afb5a472f41d..536f84279fe3 100644 --- a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.test.tsx +++ b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.test.tsx @@ -75,6 +75,7 @@ let dashboardContainer: DashboardContainer | undefined; function prepare(props?: Partial) { const { setup, doStart } = embeddablePluginMock.createInstance(); + setup.registerEmbeddableFactory( CONTACT_CARD_EMBEDDABLE, new ContactCardEmbeddableFactory((() => null) as any, {} as any) @@ -87,15 +88,41 @@ function prepare(props?: Partial) { '1': { gridData: { x: 0, y: 0, w: 6, h: 6, i: '1' }, type: CONTACT_CARD_EMBEDDABLE, - explicitInput: { id: '1' }, + explicitInput: { id: '1', savedObjectId: 'vis-1' }, // <-- added }, '2': { gridData: { x: 6, y: 6, w: 6, h: 6, i: '2' }, type: CONTACT_CARD_EMBEDDABLE, - explicitInput: { id: '2' }, + explicitInput: { id: '2', savedObjectId: 'vis-2' }, // <-- added }, }, }); + + const services = createDashboardServicesMock(); + + // Mock savedObjectsClient.get to handle both visualization and index-pattern types + jest.spyOn(services.savedObjectsClient, 'get').mockImplementation((type: string, id: string) => { + if (!type || !id) throw new Error('requires type and id'); + + if (type === 'visualization' || type === CONTACT_CARD_EMBEDDABLE) { + return Promise.resolve({ + id, + attributes: {}, + references: [{ type: 'index-pattern', id: 'index-pattern-1' }], + }); + } + + if (type === 'index-pattern') { + return Promise.resolve({ + id, + attributes: {}, + references: [{ type: 'data-source', id: 'ds-id' }], + }); + } + + throw new Error(`Unknown saved object type: ${type}`); + }); + const options: DashboardContainerOptions = { application: {} as any, embeddable: { @@ -116,9 +143,8 @@ function prepare(props?: Partial) { getTriggerCompatibleActions: (() => []) as any, } as any, }; - dashboardContainer = new DashboardContainer(initialInput, options); - const services = createDashboardServicesMock(); + dashboardContainer = new DashboardContainer(initialInput, options); const defaultTestProps: DashboardGridProps = { container: dashboardContainer, From c554c1e02db590f3d7726a0403396268599b0060 Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Mon, 5 May 2025 14:39:08 -0700 Subject: [PATCH 43/86] fix ci - 1 remove inline style Signed-off-by: Jialiang Liang --- .../embeddable/grid/dashboard_grid.tsx | 47 +++++++++---------- 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx index c41f32e643c8..69a585567a4b 100644 --- a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx +++ b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx @@ -389,38 +389,37 @@ class DashboardGridUi extends React.Component { window.location.reload(); } - return ( -
- {(() => { - const urlOverride = isDirectQuerySyncEnabledByUrl(); - const featureFlagEnabled = - urlOverride !== undefined ? urlOverride : this.props.isDirectQuerySyncEnabled; + return (() => { + const urlOverride = isDirectQuerySyncEnabledByUrl(); + const featureFlagEnabled = + urlOverride !== undefined ? urlOverride : this.props.isDirectQuerySyncEnabled; - const metadataAvailable = this.state.extractedProps !== null; + const metadataAvailable = this.state.extractedProps !== null; + const shouldRenderSyncUI = featureFlagEnabled && metadataAvailable; - const shouldRenderSyncUI = featureFlagEnabled && metadataAvailable; - - return shouldRenderSyncUI ? ( + return ( + <> + {shouldRenderSyncUI && ( - ) : null; - })()} - - - {this.renderPanels()} - -
- ); + )} + + + {this.renderPanels()} + + + ); + })(); } } From e05b57498c9803cb55efe3e1d2918c71c0358e37 Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Mon, 5 May 2025 15:03:34 -0700 Subject: [PATCH 44/86] production phase 11 - setup scss instead of inline style Signed-off-by: Jialiang Liang --- .../grid/_dashboard_direct_query_sync.scss | 4 ++ .../grid/dashboard_direct_query_sync.tsx | 43 +++++++++++-------- 2 files changed, 28 insertions(+), 19 deletions(-) create mode 100644 src/plugins/dashboard/public/application/embeddable/grid/_dashboard_direct_query_sync.scss diff --git a/src/plugins/dashboard/public/application/embeddable/grid/_dashboard_direct_query_sync.scss b/src/plugins/dashboard/public/application/embeddable/grid/_dashboard_direct_query_sync.scss new file mode 100644 index 000000000000..91997b284eb5 --- /dev/null +++ b/src/plugins/dashboard/public/application/embeddable/grid/_dashboard_direct_query_sync.scss @@ -0,0 +1,4 @@ +.dshDashboardGrid__syncBar { + margin-bottom: $euiSizeS; + margin-left: $euiSizeS; +} diff --git a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_direct_query_sync.tsx b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_direct_query_sync.tsx index f35364b07e35..fe9a4b0252be 100644 --- a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_direct_query_sync.tsx +++ b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_direct_query_sync.tsx @@ -32,6 +32,7 @@ import React from 'react'; import { EuiCallOut, EuiLink, EuiLoadingSpinner, EuiText } from '@elastic/eui'; import { DirectQueryLoadingStatus } from '../../../../framework/types'; import { EMR_STATES, intervalAsMinutes } from '../../utils/direct_query_sync/direct_query_sync'; +import './_dashboard_direct_query_sync.scss'; export interface DashboardDirectQuerySyncProps { loadStatus: DirectQueryLoadingStatus; @@ -48,26 +49,30 @@ export const DashboardDirectQuerySync: React.FC = }) => { const state = EMR_STATES.get(loadStatus)!; - return state.terminal ? ( - - Data scheduled to sync every{' '} - {refreshInterval ? intervalAsMinutes(1000 * refreshInterval) : '--'}. Last sync:{' '} - {lastRefreshTime ? ( - <> - {new Date(lastRefreshTime).toLocaleString()} ( - {intervalAsMinutes(new Date().getTime() - lastRefreshTime)} ago) - + return ( +
+ {state.terminal ? ( + + Data scheduled to sync every{' '} + {refreshInterval ? intervalAsMinutes(1000 * refreshInterval) : '--'}. Last sync:{' '} + {lastRefreshTime ? ( + <> + {new Date(lastRefreshTime).toLocaleString()} ( + {intervalAsMinutes(new Date().getTime() - lastRefreshTime)} ago) + + ) : ( + '--' + )} + .    + Sync data + ) : ( - '--' + + +    Data sync is in progress ({state.ord}% complete). The dashboard + will reload on completion. + )} - .    - Sync data - - ) : ( - - -    Data sync is in progress ({state.ord}% complete). The dashboard will - reload on completion. - +
); }; From 37a8b83ee7e170cefaf5cdceb7f94869500913f7 Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Mon, 5 May 2025 16:43:14 -0700 Subject: [PATCH 45/86] production phase 12 - do not go into the logic for check when feature is disable Signed-off-by: Jialiang Liang --- .../embeddable/grid/dashboard_grid.tsx | 29 +++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx index 69a585567a4b..0c910241ea55 100644 --- a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx +++ b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx @@ -226,11 +226,24 @@ class DashboardGridUi extends React.Component { useMargins: input.useMargins, expandedPanelId: input.expandedPanelId, }); - this.collectAllPanelMetadata(); + + const urlOverride = isDirectQuerySyncEnabledByUrl(); + const featureFlagEnabled = + urlOverride !== undefined ? urlOverride : this.props.isDirectQuerySyncEnabled; + + if (featureFlagEnabled) { + this.collectAllPanelMetadata(); + } } }); - this.collectAllPanelMetadata(); + const urlOverride = isDirectQuerySyncEnabledByUrl(); + const featureFlagEnabled = + urlOverride !== undefined ? urlOverride : this.props.isDirectQuerySyncEnabled; + + if (featureFlagEnabled) { + this.collectAllPanelMetadata(); + } } public componentWillUnmount() { @@ -282,6 +295,12 @@ class DashboardGridUi extends React.Component { * Runs on mount and when the container input (panels) changes. */ private async collectAllPanelMetadata() { + const urlOverride = isDirectQuerySyncEnabledByUrl(); + const featureFlagEnabled = + urlOverride !== undefined ? urlOverride : this.props.isDirectQuerySyncEnabled; + + if (!featureFlagEnabled) return; + const indexInfo = await extractIndexInfoFromDashboard( this.state.panels, this.props.savedObjectsClient, @@ -305,6 +324,12 @@ class DashboardGridUi extends React.Component { } synchronizeNow = () => { + const urlOverride = isDirectQuerySyncEnabledByUrl(); + const featureFlagEnabled = + urlOverride !== undefined ? urlOverride : this.props.isDirectQuerySyncEnabled; + + if (!featureFlagEnabled) return; + const { extractedDatasource, extractedDatabase, extractedIndex } = this; if ( !extractedDatasource || From 3556596f1dbade85c5da5a9fc43b0064bca43fc9 Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Tue, 6 May 2025 12:26:06 -0700 Subject: [PATCH 46/86] Add more test cases Signed-off-by: Jialiang Liang --- .../direct_query_sync.test.ts | 210 +++++++++++++++++- .../direct_query_sync/direct_query_sync.ts | 4 + 2 files changed, 213 insertions(+), 1 deletion(-) diff --git a/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.test.ts b/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.test.ts index 5b041db400d2..9e4310d453ad 100644 --- a/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.test.ts +++ b/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.test.ts @@ -3,7 +3,15 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { sourceCheck } from './direct_query_sync'; +import { + sourceCheck, + resolveConcreteIndex, + extractIndexParts, + generateRefreshQuery, + fetchIndexMapping, + extractIndexInfoFromDashboard, +} from './direct_query_sync'; +import { HttpStart, SavedObjectsClientContract } from 'opensearch-dashboards/public'; describe('sourceCheck', () => { it('returns true if all indexPatternIds and mdsIds are the same', () => { @@ -41,4 +49,204 @@ describe('sourceCheck', () => { it('returns true if single entry arrays', () => { expect(sourceCheck(['pattern-1'], ['mds-1'])).toBe(true); }); + + it('returns false if mdsIds contains undefined and other values', () => { + const indexPatternIds = ['pattern-1', 'pattern-1']; + const mdsIds = [undefined, 'mds-1']; + expect(sourceCheck(indexPatternIds, mdsIds)).toBe(false); + }); + + it('returns true if all mdsIds are undefined', () => { + const indexPatternIds = ['pattern-1', 'pattern-1']; + const mdsIds = [undefined, undefined]; + expect(sourceCheck(indexPatternIds, mdsIds)).toBe(true); + }); +}); + +describe('resolveConcreteIndex', () => { + let mockHttp: jest.Mocked; + + beforeEach(() => { + mockHttp = ({ + get: jest.fn(), + } as unknown) as jest.Mocked; + }); + + it('returns the input index if it does not contain wildcards', async () => { + const result = await resolveConcreteIndex('my_index', mockHttp); + expect(result).toBe('my_index'); + expect(mockHttp.get).not.toHaveBeenCalled(); + }); + + it('resolves wildcard index with mdsId', async () => { + mockHttp.get.mockResolvedValue({ indices: [{ name: 'resolved_index' }] }); + const result = await resolveConcreteIndex('my_index*', mockHttp, 'mds-1'); + expect(mockHttp.get).toHaveBeenCalledWith( + '/internal/index-pattern-management/resolve_index/my_index*', + { + query: { data_source: 'mds-1' }, + } + ); + expect(result).toBe('resolved_index'); + }); + + it('returns null if no matching indices are found', async () => { + mockHttp.get.mockResolvedValue({ indices: [] }); + const result = await resolveConcreteIndex('my_index*', mockHttp); + expect(result).toBe(null); + }); + + it('returns null on HTTP error', async () => { + mockHttp.get.mockRejectedValue(new Error('Network error')); + const result = await resolveConcreteIndex('my_index*', mockHttp); + expect(result).toBe(null); + }); +}); + +describe('extractIndexParts', () => { + it('correctly extracts parts from a full index name', () => { + const result = extractIndexParts('flint_datasource1_database1_my_index'); + expect(result).toEqual({ + datasource: 'datasource1', + database: 'database1', + index: 'my_index', + }); + }); + + it('handles missing parts with unknown values', () => { + const result = extractIndexParts('flint_datasource1'); + expect(result).toEqual({ + datasource: 'datasource1', + database: 'unknown', + index: 'unknown', + }); + }); + + it('handles empty index name', () => { + const result = extractIndexParts(''); + expect(result).toEqual({ + datasource: 'unknown', + database: 'unknown', + index: 'unknown', + }); + }); +}); + +describe('generateRefreshQuery', () => { + it('generates correct refresh query', () => { + const info = { + datasource: 'datasource1', + database: 'database1', + index: 'my_index', + }; + const result = generateRefreshQuery(info); + expect(result).toBe('REFRESH MATERIALIZED VIEW `datasource1`.`database1`.`my_index`'); + }); +}); + +describe('fetchIndexMapping', () => { + let mockHttp: jest.Mocked; + + beforeEach(() => { + mockHttp = ({ + get: jest.fn(), + } as unknown) as jest.Mocked; + }); + + it('fetches mapping without mdsId', async () => { + const mockResponse = { mapping: { _meta: { properties: { lastRefreshTime: 12345 } } } }; + mockHttp.get.mockResolvedValue(mockResponse); + const result = await fetchIndexMapping('my_index', mockHttp); + expect(mockHttp.get).toHaveBeenCalledWith('/api/directquery/dsl/indices.getFieldMapping', { + query: { index: 'my_index' }, + }); + expect(result).toBe(mockResponse); + }); + + it('fetches mapping with mdsId', async () => { + const mockResponse = { mapping: { _meta: { properties: { lastRefreshTime: 12345 } } } }; + mockHttp.get.mockResolvedValue(mockResponse); + const result = await fetchIndexMapping('my_index', mockHttp, 'mds-1'); + expect(mockHttp.get).toHaveBeenCalledWith( + '/api/directquery/dsl/indices.getFieldMapping/dataSourceMDSId=mds-1', + { + query: { index: 'my_index' }, + } + ); + expect(result).toBe(mockResponse); + }); + + it('returns null on error', async () => { + mockHttp.get.mockRejectedValue(new Error('Network error')); + const result = await fetchIndexMapping('my_index', mockHttp); + expect(result).toBe(null); + }); +}); + +describe('extractIndexInfoFromDashboard', () => { + let mockSavedObjectsClient: jest.Mocked; + let mockHttp: jest.Mocked; + + beforeEach(() => { + mockSavedObjectsClient = ({ + get: jest.fn(), + } as unknown) as jest.Mocked; + mockHttp = ({ + get: jest.fn(), + } as unknown) as jest.Mocked; + }); + + it('returns null if panels have no savedObjectId', async () => { + const panels = { panel1: { explicitInput: {} } }; + const result = await extractIndexInfoFromDashboard(panels, mockSavedObjectsClient, mockHttp); + expect(result).toBe(null); + expect(mockSavedObjectsClient.get).not.toHaveBeenCalled(); + }); + + it('returns null if references include non-index-pattern types', async () => { + mockSavedObjectsClient.get.mockResolvedValueOnce({ + references: [{ type: 'data-source', id: 'ds-1', name: 'Data Source 1' }], + attributes: {}, + }); + const panels = { panel1: { explicitInput: { savedObjectId: 'so-1' }, type: 'visualization' } }; + const result = await extractIndexInfoFromDashboard(panels, mockSavedObjectsClient, mockHttp); + expect(result).toBe(null); + }); + + it('returns null if no index-pattern reference is found', async () => { + mockSavedObjectsClient.get.mockResolvedValueOnce({ + references: [{ type: 'other', id: 'other-1', name: 'Other 1' }], + attributes: {}, + }); + const panels = { panel1: { explicitInput: { savedObjectId: 'so-1' }, type: 'visualization' } }; + const result = await extractIndexInfoFromDashboard(panels, mockSavedObjectsClient, mockHttp); + expect(result).toBe(null); + }); + + it('returns null if concrete index cannot be resolved', async () => { + mockSavedObjectsClient.get + .mockResolvedValueOnce({ + references: [{ type: 'index-pattern', id: 'ip-1', name: 'Index Pattern 1' }], + attributes: {}, + }) + .mockResolvedValueOnce({ + attributes: { title: 'my_index*' }, + references: [], + }) + .mockResolvedValueOnce({ + attributes: { title: 'my_index*' }, + references: [], + }); + mockHttp.get.mockResolvedValueOnce({ indices: [] }); + const panels = { panel1: { explicitInput: { savedObjectId: 'so-1' }, type: 'visualization' } }; + const result = await extractIndexInfoFromDashboard(panels, mockSavedObjectsClient, mockHttp); + expect(result).toBe(null); + }); + + it('handles saved object errors gracefully', async () => { + mockSavedObjectsClient.get.mockRejectedValueOnce(new Error('Saved object not found')); + const panels = { panel1: { explicitInput: { savedObjectId: 'so-1' }, type: 'visualization' } }; + const result = await extractIndexInfoFromDashboard(panels, mockSavedObjectsClient, mockHttp); + expect(result).toBe(null); + }); }); diff --git a/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.ts b/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.ts index cef1bf278343..1a47304ddb29 100644 --- a/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.ts +++ b/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.ts @@ -123,6 +123,10 @@ export async function extractIndexInfoFromDashboard( } } + if (indexPatternIds.length === 0) { + return null; + } + if (!sourceCheck(indexPatternIds, mdsIds)) { return null; } From eed6a457371a65f4da883b1bdf829266bf33519d Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Tue, 6 May 2025 12:27:45 -0700 Subject: [PATCH 47/86] modify the header Signed-off-by: Jialiang Liang --- .../grid/dashboard_direct_query_sync.tsx | 27 +------------------ 1 file changed, 1 insertion(+), 26 deletions(-) diff --git a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_direct_query_sync.tsx b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_direct_query_sync.tsx index fe9a4b0252be..6befc8cb066a 100644 --- a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_direct_query_sync.tsx +++ b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_direct_query_sync.tsx @@ -1,31 +1,6 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Any modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. */ import React from 'react'; From 6517e60aa27900f84092243b74c7f496691d818f Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Tue, 6 May 2025 12:58:38 -0700 Subject: [PATCH 48/86] remove the unhandled error in extract info function and update the test Signed-off-by: Jialiang Liang --- .../direct_query_sync/direct_query_sync.test.ts | 12 ++++++++++-- .../utils/direct_query_sync/direct_query_sync.ts | 8 +++++--- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.test.ts b/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.test.ts index 9e4310d453ad..8d8d82c9a008 100644 --- a/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.test.ts +++ b/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.test.ts @@ -243,10 +243,18 @@ describe('extractIndexInfoFromDashboard', () => { expect(result).toBe(null); }); - it('handles saved object errors gracefully', async () => { - mockSavedObjectsClient.get.mockRejectedValueOnce(new Error('Saved object not found')); + it('handles 404 saved object errors gracefully', async () => { + mockSavedObjectsClient.get.mockRejectedValueOnce({ response: { status: 404 } }); const panels = { panel1: { explicitInput: { savedObjectId: 'so-1' }, type: 'visualization' } }; const result = await extractIndexInfoFromDashboard(panels, mockSavedObjectsClient, mockHttp); expect(result).toBe(null); }); + + it('throws non-404 saved object errors', async () => { + mockSavedObjectsClient.get.mockRejectedValueOnce({ response: { status: 500 } }); + const panels = { panel1: { explicitInput: { savedObjectId: 'so-1' }, type: 'visualization' } }; + await expect( + extractIndexInfoFromDashboard(panels, mockSavedObjectsClient, mockHttp) + ).rejects.toMatchObject({ response: { status: 500 } }); + }); }); diff --git a/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.ts b/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.ts index 1a47304ddb29..6b8dd62e6b29 100644 --- a/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.ts +++ b/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.ts @@ -118,8 +118,10 @@ export async function extractIndexInfoFromDashboard( indexPatternIds.push(indexPatternRef.id); mdsIds.push(mdsId); - } catch (err) { - // Ignoring error: saved object might be missing or invalid + } catch (err: any) { + if (err?.response?.status !== 404) { + throw err; + } } } @@ -181,7 +183,7 @@ export function sourceCheck(indexPatternIds: string[], mdsIds: Array Date: Tue, 6 May 2025 13:14:28 -0700 Subject: [PATCH 49/86] remove unnecessary comments Signed-off-by: Jialiang Liang --- .../application/embeddable/grid/dashboard_grid.test.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.test.tsx b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.test.tsx index 536f84279fe3..b90eb80bd947 100644 --- a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.test.tsx +++ b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.test.tsx @@ -88,12 +88,12 @@ function prepare(props?: Partial) { '1': { gridData: { x: 0, y: 0, w: 6, h: 6, i: '1' }, type: CONTACT_CARD_EMBEDDABLE, - explicitInput: { id: '1', savedObjectId: 'vis-1' }, // <-- added + explicitInput: { id: '1', savedObjectId: 'vis-1' }, }, '2': { gridData: { x: 6, y: 6, w: 6, h: 6, i: '2' }, type: CONTACT_CARD_EMBEDDABLE, - explicitInput: { id: '2', savedObjectId: 'vis-2' }, // <-- added + explicitInput: { id: '2', savedObjectId: 'vis-2' }, }, }, }); From 074f50be5b5cccb0fd7c5e4eacd22c6019080e4f Mon Sep 17 00:00:00 2001 From: "opensearch-changeset-bot[bot]" <154024398+opensearch-changeset-bot[bot]@users.noreply.github.com> Date: Tue, 6 May 2025 21:17:33 +0000 Subject: [PATCH 50/86] Changeset file for PR #9745 created/updated --- changelogs/fragments/9745.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 changelogs/fragments/9745.yml diff --git a/changelogs/fragments/9745.yml b/changelogs/fragments/9745.yml new file mode 100644 index 000000000000..8eb37ee53f29 --- /dev/null +++ b/changelogs/fragments/9745.yml @@ -0,0 +1,2 @@ +feat: +- [Integration] Vended Dashboards Refresh #9745 ([#9745](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/9745)) \ No newline at end of file From e747e4ea9adf72421868de92459ba807eb374ba0 Mon Sep 17 00:00:00 2001 From: "opensearch-changeset-bot[bot]" <154024398+opensearch-changeset-bot[bot]@users.noreply.github.com> Date: Tue, 6 May 2025 21:20:03 +0000 Subject: [PATCH 51/86] Changeset file for PR #9745 created/updated --- changelogs/fragments/9745.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelogs/fragments/9745.yml b/changelogs/fragments/9745.yml index 8eb37ee53f29..85c5c074fd5f 100644 --- a/changelogs/fragments/9745.yml +++ b/changelogs/fragments/9745.yml @@ -1,2 +1,2 @@ feat: -- [Integration] Vended Dashboards Refresh #9745 ([#9745](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/9745)) \ No newline at end of file +- [Integration] Vended Dashboards Synchronization #9745 ([#9745](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/9745)) \ No newline at end of file From 945b8bbb4c7f96ab073b603ca86d0816ecc7e8a6 Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Tue, 6 May 2025 14:02:08 -0700 Subject: [PATCH 52/86] add more test coverage for grid class Signed-off-by: Jialiang Liang --- .../embeddable/grid/dashboard_grid.test.tsx | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.test.tsx b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.test.tsx index b90eb80bd947..9c5c848a1eff 100644 --- a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.test.tsx +++ b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.test.tsx @@ -342,3 +342,50 @@ test('synchronizeNow triggers REFRESH query generation and startLoading', async datasource: 'ds', }); }); + +test('synchronizeNow does nothing when feature flag is disabled', async () => { + const { props, options } = prepare({ isDirectQuerySyncEnabled: false }); + + const startLoadingSpy = jest.fn(); + props.startLoading = startLoadingSpy; + + const component = mountWithIntl( + + + + ); + + await new Promise((resolve) => setTimeout(resolve, 0)); + component.update(); + + // Simulate calling synchronizeNow directly + (component.find('DashboardGridUi').instance() as any).synchronizeNow(); + + expect(startLoadingSpy).not.toHaveBeenCalled(); +}); + +test('synchronizeNow does nothing when metadata contains unknown values', async () => { + const { props, options } = prepare({ isDirectQuerySyncEnabled: true }); + + (extractIndexInfoFromDashboard as jest.Mock).mockResolvedValue({ + parts: { datasource: 'ds', database: 'db', index: 'unknown' }, + mapping: { lastRefreshTime: 123456, refreshInterval: 30000 }, + mdsId: '', + }); + + const startLoadingSpy = jest.fn(); + props.startLoading = startLoadingSpy; + + const component = mountWithIntl( + + + + ); + + await new Promise((resolve) => setTimeout(resolve, 0)); + component.update(); + + (component.find('DashboardGridUi').instance() as any).synchronizeNow(); + + expect(startLoadingSpy).not.toHaveBeenCalled(); +}); From 7e393d0a550690d178b32f516658e0cdf8e8450b Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Tue, 6 May 2025 15:29:56 -0700 Subject: [PATCH 53/86] add more test coverage for polling Signed-off-by: Jialiang Liang --- .../framework/utils/use_polling.test.tsx | 151 ++++++++++++++++++ 1 file changed, 151 insertions(+) create mode 100644 src/plugins/dashboard/framework/utils/use_polling.test.tsx diff --git a/src/plugins/dashboard/framework/utils/use_polling.test.tsx b/src/plugins/dashboard/framework/utils/use_polling.test.tsx new file mode 100644 index 000000000000..ad2e695e1bf4 --- /dev/null +++ b/src/plugins/dashboard/framework/utils/use_polling.test.tsx @@ -0,0 +1,151 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { act, renderHook } from '@testing-library/react-hooks'; +import { usePolling, PollingConfigurations } from './use_polling'; + +describe('usePolling', () => { + const mockFetchFunction = jest.fn(); + const mockOnPollingSuccess = jest.fn(); + const mockOnPollingError = jest.fn(); + const mockConfigurations: PollingConfigurations = { tabId: 'test-tab' }; + + beforeEach(() => { + jest.useFakeTimers(); + jest.clearAllMocks(); + jest.clearAllTimers(); + }); + + afterEach(() => { + jest.useRealTimers(); + }); + + it('returns correct initial state', () => { + const { result } = renderHook(() => usePolling(mockFetchFunction)); + + expect(result.current.data).toBe(null); + expect(result.current.loading).toBe(true); + expect(result.current.error).toBe(null); + expect(typeof result.current.startPolling).toBe('function'); + expect(typeof result.current.stopPolling).toBe('function'); + }); + + it('fetches data successfully and updates state', async () => { + const mockData = { result: 'success' }; + mockFetchFunction.mockResolvedValue(mockData); + + const { result, waitForNextUpdate } = renderHook(() => usePolling(mockFetchFunction, 5000)); + + await act(async () => { + result.current.startPolling(); + jest.advanceTimersByTime(5000); + await waitForNextUpdate(); + }); + + expect(mockFetchFunction).toHaveBeenCalledTimes(1); + expect(result.current.data).toEqual(mockData); + expect(result.current.loading).toBe(false); + expect(result.current.error).toBe(null); + }); + + it('stops polling on successful fetch when onPollingSuccess returns true', async () => { + const mockData = { result: 'success' }; + mockFetchFunction.mockResolvedValue(mockData); + mockOnPollingSuccess.mockReturnValue(true); + + const { result, waitForNextUpdate } = renderHook(() => + usePolling(mockFetchFunction, 5000, mockOnPollingSuccess, undefined, mockConfigurations) + ); + + await act(async () => { + result.current.startPolling(); + jest.advanceTimersByTime(5000); + await waitForNextUpdate(); + }); + + expect(mockFetchFunction).toHaveBeenCalledTimes(1); + expect(mockOnPollingSuccess).toHaveBeenCalledWith(mockData, mockConfigurations); + expect(result.current.data).toEqual(mockData); + expect(result.current.loading).toBe(false); + expect(result.current.error).toBe(null); + + // Advance timers to check if polling continues + await act(async () => { + jest.advanceTimersByTime(5000); + }); + + // Should not fetch again + expect(mockFetchFunction).toHaveBeenCalledTimes(1); + }); + + it('handles fetch error and updates state', async () => { + const mockError = new Error('Fetch failed'); + mockFetchFunction.mockRejectedValue(mockError); + + const { result, waitForNextUpdate } = renderHook(() => usePolling(mockFetchFunction, 5000)); + + await act(async () => { + result.current.startPolling(); + jest.advanceTimersByTime(5000); + await waitForNextUpdate(); + }); + + expect(mockFetchFunction).toHaveBeenCalledTimes(1); + expect(result.current.data).toBe(null); + expect(result.current.loading).toBe(false); + expect(result.current.error).toEqual(mockError); + }); + + it('stops polling on error when onPollingError returns true', async () => { + const mockError = new Error('Fetch failed'); + mockFetchFunction.mockRejectedValue(mockError); + mockOnPollingError.mockReturnValue(true); + + const { result, waitForNextUpdate } = renderHook(() => + usePolling(mockFetchFunction, 5000, undefined, mockOnPollingError) + ); + + await act(async () => { + result.current.startPolling(); + jest.advanceTimersByTime(5000); + await waitForNextUpdate(); + }); + + expect(mockFetchFunction).toHaveBeenCalledTimes(1); + expect(mockOnPollingError).toHaveBeenCalledWith(mockError); + expect(result.current.data).toBe(null); + expect(result.current.loading).toBe(false); + expect(result.current.error).toEqual(mockError); + + // Advance timers to check if polling continues + await act(async () => { + jest.advanceTimersByTime(5000); + }); + + expect(mockFetchFunction).toHaveBeenCalledTimes(1); // Should not fetch again + }); + + it('starts and stops polling correctly', async () => { + const mockData = { result: 'success' }; + mockFetchFunction.mockResolvedValue(mockData); + + const { result, waitForNextUpdate } = renderHook(() => usePolling(mockFetchFunction, 5000)); + + await act(async () => { + result.current.startPolling(); + jest.advanceTimersByTime(5000); + await waitForNextUpdate(); + }); + + expect(mockFetchFunction).toHaveBeenCalledTimes(1); + + await act(async () => { + result.current.stopPolling(); + jest.advanceTimersByTime(5000); + }); + + expect(mockFetchFunction).toHaveBeenCalledTimes(1); // No additional calls after stopping + }); +}); From 9e62b5a9d1863c5815b24582918e9a15ff5e7691 Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Tue, 6 May 2025 17:30:11 -0700 Subject: [PATCH 54/86] enhance the flint info fetching parsing logic Signed-off-by: Jialiang Liang --- .../direct_query_sync/direct_query_sync.ts | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.ts b/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.ts index 6b8dd62e6b29..19db798dfec1 100644 --- a/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.ts +++ b/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.ts @@ -61,7 +61,21 @@ export async function resolveConcreteIndex( } } -export function extractIndexParts(fullIndexName: string): IndexExtractionResult { +export function extractIndexParts( + fullIndexName: string, + mappingName?: string +): IndexExtractionResult { + // Prefer parsing the mapping name if provided + if (mappingName) { + const parts = mappingName.split('.'); + return { + datasource: parts[0] || 'unknown', + database: parts[1] || 'unknown', + index: parts.slice(2).join('.') || 'unknown', + }; + } + + // Fallback to original regex-based parsing const trimmed = fullIndexName.replace(/^flint_/, ''); const parts = trimmed.split('_'); return { @@ -119,6 +133,7 @@ export async function extractIndexInfoFromDashboard( indexPatternIds.push(indexPatternRef.id); mdsIds.push(mdsId); } catch (err: any) { + // Ignore only 404 errors (missing saved object) if (err?.response?.status !== 404) { throw err; } @@ -146,9 +161,10 @@ export async function extractIndexInfoFromDashboard( if (!mapping) return null; for (const val of Object.values(mapping)) { + const mappingName = val.mappings?._meta?.name; return { mapping: val.mappings._meta.properties!, - parts: extractIndexParts(concreteTitle), + parts: extractIndexParts(concreteTitle, mappingName), mdsId: selectedMdsId, }; } From 68ada456be2f2378fb8ef33094a115e0bbf96577 Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Tue, 6 May 2025 20:37:26 -0700 Subject: [PATCH 55/86] REUSE FRAMEWORKS IN DSM Signed-off-by: Jialiang Liang --- src/plugins/dashboard/framework/constants.tsx | 7 - .../framework/hooks/direct_query_hook.tsx | 99 ------------ .../dashboard/framework/requests/sql.ts | 70 -------- src/plugins/dashboard/framework/types.tsx | 23 --- .../framework/utils/query_session_utils.ts | 16 -- .../dashboard/framework/utils/shared.ts | 25 --- .../framework/utils/use_polling.test.tsx | 151 ------------------ .../dashboard/framework/utils/use_polling.tsx | 137 ---------------- .../grid/dashboard_direct_query_sync.tsx | 2 +- .../embeddable/grid/dashboard_grid.tsx | 3 +- .../viewport/dashboard_viewport.tsx | 8 +- .../framework/hooks/direct_query_hook.tsx | 2 +- .../framework/types.tsx | 4 +- .../data_source_management/public/index.ts | 9 ++ tsconfig.base.json | 2 + 15 files changed, 22 insertions(+), 536 deletions(-) delete mode 100644 src/plugins/dashboard/framework/constants.tsx delete mode 100644 src/plugins/dashboard/framework/hooks/direct_query_hook.tsx delete mode 100644 src/plugins/dashboard/framework/requests/sql.ts delete mode 100644 src/plugins/dashboard/framework/types.tsx delete mode 100644 src/plugins/dashboard/framework/utils/query_session_utils.ts delete mode 100644 src/plugins/dashboard/framework/utils/shared.ts delete mode 100644 src/plugins/dashboard/framework/utils/use_polling.test.tsx delete mode 100644 src/plugins/dashboard/framework/utils/use_polling.tsx diff --git a/src/plugins/dashboard/framework/constants.tsx b/src/plugins/dashboard/framework/constants.tsx deleted file mode 100644 index 4878352d8355..000000000000 --- a/src/plugins/dashboard/framework/constants.tsx +++ /dev/null @@ -1,7 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -export const ASYNC_QUERY_SESSION_ID = 'async-query-session-id'; -export const ASYNC_POLLING_INTERVAL = 2000; diff --git a/src/plugins/dashboard/framework/hooks/direct_query_hook.tsx b/src/plugins/dashboard/framework/hooks/direct_query_hook.tsx deleted file mode 100644 index e6250f784c48..000000000000 --- a/src/plugins/dashboard/framework/hooks/direct_query_hook.tsx +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import { useEffect, useState } from 'react'; -import { HttpStart, NotificationsStart } from 'opensearch-dashboards/public'; -import { ASYNC_POLLING_INTERVAL } from '../constants'; -import { DirectQueryLoadingStatus, DirectQueryRequest } from '../types'; -import { getAsyncSessionId, setAsyncSessionId } from '../utils/query_session_utils'; -import { get as getObjValue, formatError } from '../utils/shared'; -import { usePolling } from '../utils/use_polling'; -import { SQLService } from '../requests/sql'; - -export const useDirectQuery = ( - http: HttpStart, - notifications: NotificationsStart, - dataSourceMDSId?: string -) => { - const sqlService = new SQLService(http); - const [loadStatus, setLoadStatus] = useState( - DirectQueryLoadingStatus.FRESH - ); - - const { - data: pollingResult, - loading: _pollingLoading, - error: pollingError, - startPolling, - stopPolling: stopLoading, - } = usePolling((params) => { - return sqlService.fetchWithJobId(params, dataSourceMDSId || ''); - }, ASYNC_POLLING_INTERVAL); - - const startLoading = (requestPayload: DirectQueryRequest) => { - setLoadStatus(DirectQueryLoadingStatus.SCHEDULED); - - const sessionId = getAsyncSessionId(requestPayload.datasource); - if (sessionId) { - requestPayload = { ...requestPayload, sessionId }; - } - - sqlService - .fetch(requestPayload, dataSourceMDSId) - .then((result) => { - setAsyncSessionId(requestPayload.datasource, getObjValue(result, 'sessionId', null)); - if (result.queryId) { - startPolling({ - queryId: result.queryId, - }); - } else { - // eslint-disable-next-line no-console - console.error('No query id found in response'); - setLoadStatus(DirectQueryLoadingStatus.FAILED); - } - }) - .catch((e) => { - setLoadStatus(DirectQueryLoadingStatus.FAILED); - const formattedError = formatError( - '', - 'The query failed to execute and the operation could not be complete.', - e.body?.message - ); - notifications.toasts.addError(formattedError, { - title: 'Query Failed', - }); - // eslint-disable-next-line no-console - console.error(e); - }); - }; - - useEffect(() => { - // cancel direct query - if (!pollingResult) return; - const { status: anyCaseStatus, datarows, error } = pollingResult; - const status = anyCaseStatus?.toLowerCase(); - - if (status === DirectQueryLoadingStatus.SUCCESS || datarows) { - setLoadStatus(status); - stopLoading(); - } else if (status === DirectQueryLoadingStatus.FAILED) { - setLoadStatus(status); - stopLoading(); - const formattedError = formatError( - '', - 'The query failed to execute and the operation could not be complete.', - error - ); - notifications.toasts.addError(formattedError, { - title: 'Query Failed', - }); - } else { - setLoadStatus(status); - } - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [pollingResult, pollingError, stopLoading]); - - return { loadStatus, startLoading, stopLoading, pollingResult }; -}; diff --git a/src/plugins/dashboard/framework/requests/sql.ts b/src/plugins/dashboard/framework/requests/sql.ts deleted file mode 100644 index 274289af4543..000000000000 --- a/src/plugins/dashboard/framework/requests/sql.ts +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -/* eslint-disable no-console */ - -import { HttpStart } from 'opensearch-dashboards/public'; -import { DirectQueryRequest } from '../types'; - -interface FetchError { - body: string; - message?: string; - [key: string]: any; -} - -export class SQLService { - private http: HttpStart; - - constructor(http: HttpStart) { - this.http = http; - } - - fetch = async ( - params: DirectQueryRequest, - dataSourceMDSId?: string, - errorHandler?: (error: FetchError) => void - ) => { - const query = { - dataSourceMDSId, - }; - return this.http - .post('/api/observability/query/jobs', { - body: JSON.stringify(params), - query, - }) - .catch((error: FetchError) => { - console.error('fetch error: ', error.body); - if (errorHandler) errorHandler(error); - throw error; - }); - }; - - fetchWithJobId = async ( - params: { queryId: string }, - dataSourceMDSId?: string, - errorHandler?: (error: FetchError) => void - ) => { - return this.http - .get(`/api/observability/query/jobs/${params.queryId}/${dataSourceMDSId ?? ''}`) - .catch((error: FetchError) => { - console.error('fetch error: ', error.body); - if (errorHandler) errorHandler(error); - throw error; - }); - }; - - deleteWithJobId = async ( - params: { queryId: string }, - errorHandler?: (error: FetchError) => void - ) => { - return this.http - .delete(`/api/observability/query/jobs/${params.queryId}`) - .catch((error: FetchError) => { - console.error('delete error: ', error.body); - if (errorHandler) errorHandler(error); - throw error; - }); - }; -} diff --git a/src/plugins/dashboard/framework/types.tsx b/src/plugins/dashboard/framework/types.tsx deleted file mode 100644 index e8010cc44005..000000000000 --- a/src/plugins/dashboard/framework/types.tsx +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -export enum DirectQueryLoadingStatus { - SUCCESS = 'success', - FAILED = 'failed', - RUNNING = 'running', - SUBMITTED = 'submitted', - SCHEDULED = 'scheduled', - CANCELLED = 'cancelled', - WAITING = 'waiting', - INITIAL = 'initial', - FRESH = 'fresh', -} - -export interface DirectQueryRequest { - query: string; - lang: string; - datasource: string; - sessionId?: string; -} diff --git a/src/plugins/dashboard/framework/utils/query_session_utils.ts b/src/plugins/dashboard/framework/utils/query_session_utils.ts deleted file mode 100644 index beabcb48c197..000000000000 --- a/src/plugins/dashboard/framework/utils/query_session_utils.ts +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import { ASYNC_QUERY_SESSION_ID } from '../constants'; - -export const setAsyncSessionId = (dataSource: string, value: string | null) => { - if (value !== null) { - sessionStorage.setItem(`${ASYNC_QUERY_SESSION_ID}_${dataSource}`, value); - } -}; - -export const getAsyncSessionId = (dataSource: string) => { - return sessionStorage.getItem(`${ASYNC_QUERY_SESSION_ID}_${dataSource}`); -}; diff --git a/src/plugins/dashboard/framework/utils/shared.ts b/src/plugins/dashboard/framework/utils/shared.ts deleted file mode 100644 index e575767218f3..000000000000 --- a/src/plugins/dashboard/framework/utils/shared.ts +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -export function get(obj: Record, path: string, defaultValue?: T): T { - return path.split('.').reduce((acc: any, part: string) => acc && acc[part], obj) || defaultValue; -} - -export const formatError = (name: string, message: string, details: string) => { - return { - name, - message, - body: { - attributes: { - error: { - caused_by: { - type: '', - reason: details, - }, - }, - }, - }, - }; -}; diff --git a/src/plugins/dashboard/framework/utils/use_polling.test.tsx b/src/plugins/dashboard/framework/utils/use_polling.test.tsx deleted file mode 100644 index ad2e695e1bf4..000000000000 --- a/src/plugins/dashboard/framework/utils/use_polling.test.tsx +++ /dev/null @@ -1,151 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import { act, renderHook } from '@testing-library/react-hooks'; -import { usePolling, PollingConfigurations } from './use_polling'; - -describe('usePolling', () => { - const mockFetchFunction = jest.fn(); - const mockOnPollingSuccess = jest.fn(); - const mockOnPollingError = jest.fn(); - const mockConfigurations: PollingConfigurations = { tabId: 'test-tab' }; - - beforeEach(() => { - jest.useFakeTimers(); - jest.clearAllMocks(); - jest.clearAllTimers(); - }); - - afterEach(() => { - jest.useRealTimers(); - }); - - it('returns correct initial state', () => { - const { result } = renderHook(() => usePolling(mockFetchFunction)); - - expect(result.current.data).toBe(null); - expect(result.current.loading).toBe(true); - expect(result.current.error).toBe(null); - expect(typeof result.current.startPolling).toBe('function'); - expect(typeof result.current.stopPolling).toBe('function'); - }); - - it('fetches data successfully and updates state', async () => { - const mockData = { result: 'success' }; - mockFetchFunction.mockResolvedValue(mockData); - - const { result, waitForNextUpdate } = renderHook(() => usePolling(mockFetchFunction, 5000)); - - await act(async () => { - result.current.startPolling(); - jest.advanceTimersByTime(5000); - await waitForNextUpdate(); - }); - - expect(mockFetchFunction).toHaveBeenCalledTimes(1); - expect(result.current.data).toEqual(mockData); - expect(result.current.loading).toBe(false); - expect(result.current.error).toBe(null); - }); - - it('stops polling on successful fetch when onPollingSuccess returns true', async () => { - const mockData = { result: 'success' }; - mockFetchFunction.mockResolvedValue(mockData); - mockOnPollingSuccess.mockReturnValue(true); - - const { result, waitForNextUpdate } = renderHook(() => - usePolling(mockFetchFunction, 5000, mockOnPollingSuccess, undefined, mockConfigurations) - ); - - await act(async () => { - result.current.startPolling(); - jest.advanceTimersByTime(5000); - await waitForNextUpdate(); - }); - - expect(mockFetchFunction).toHaveBeenCalledTimes(1); - expect(mockOnPollingSuccess).toHaveBeenCalledWith(mockData, mockConfigurations); - expect(result.current.data).toEqual(mockData); - expect(result.current.loading).toBe(false); - expect(result.current.error).toBe(null); - - // Advance timers to check if polling continues - await act(async () => { - jest.advanceTimersByTime(5000); - }); - - // Should not fetch again - expect(mockFetchFunction).toHaveBeenCalledTimes(1); - }); - - it('handles fetch error and updates state', async () => { - const mockError = new Error('Fetch failed'); - mockFetchFunction.mockRejectedValue(mockError); - - const { result, waitForNextUpdate } = renderHook(() => usePolling(mockFetchFunction, 5000)); - - await act(async () => { - result.current.startPolling(); - jest.advanceTimersByTime(5000); - await waitForNextUpdate(); - }); - - expect(mockFetchFunction).toHaveBeenCalledTimes(1); - expect(result.current.data).toBe(null); - expect(result.current.loading).toBe(false); - expect(result.current.error).toEqual(mockError); - }); - - it('stops polling on error when onPollingError returns true', async () => { - const mockError = new Error('Fetch failed'); - mockFetchFunction.mockRejectedValue(mockError); - mockOnPollingError.mockReturnValue(true); - - const { result, waitForNextUpdate } = renderHook(() => - usePolling(mockFetchFunction, 5000, undefined, mockOnPollingError) - ); - - await act(async () => { - result.current.startPolling(); - jest.advanceTimersByTime(5000); - await waitForNextUpdate(); - }); - - expect(mockFetchFunction).toHaveBeenCalledTimes(1); - expect(mockOnPollingError).toHaveBeenCalledWith(mockError); - expect(result.current.data).toBe(null); - expect(result.current.loading).toBe(false); - expect(result.current.error).toEqual(mockError); - - // Advance timers to check if polling continues - await act(async () => { - jest.advanceTimersByTime(5000); - }); - - expect(mockFetchFunction).toHaveBeenCalledTimes(1); // Should not fetch again - }); - - it('starts and stops polling correctly', async () => { - const mockData = { result: 'success' }; - mockFetchFunction.mockResolvedValue(mockData); - - const { result, waitForNextUpdate } = renderHook(() => usePolling(mockFetchFunction, 5000)); - - await act(async () => { - result.current.startPolling(); - jest.advanceTimersByTime(5000); - await waitForNextUpdate(); - }); - - expect(mockFetchFunction).toHaveBeenCalledTimes(1); - - await act(async () => { - result.current.stopPolling(); - jest.advanceTimersByTime(5000); - }); - - expect(mockFetchFunction).toHaveBeenCalledTimes(1); // No additional calls after stopping - }); -}); diff --git a/src/plugins/dashboard/framework/utils/use_polling.tsx b/src/plugins/dashboard/framework/utils/use_polling.tsx deleted file mode 100644 index f2af5979ab59..000000000000 --- a/src/plugins/dashboard/framework/utils/use_polling.tsx +++ /dev/null @@ -1,137 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import { useEffect, useRef, useState } from 'react'; - -type FetchFunction = (params?: P) => Promise; - -export interface PollingConfigurations { - tabId: string; -} - -export class UsePolling { - public data: T | null = null; - public error: Error | null = null; - public loading: boolean = true; - private shouldPoll: boolean = false; - private intervalRef?: NodeJS.Timeout; - - constructor( - private fetchFunction: FetchFunction, - private interval: number = 5000, - private onPollingSuccess?: (data: T, configurations: PollingConfigurations) => boolean, - private onPollingError?: (error: Error) => boolean, - private configurations?: PollingConfigurations - ) {} - - async fetchData(params?: P) { - this.loading = true; - try { - const result = await this.fetchFunction(params); - this.data = result; - this.loading = false; - - if (this.onPollingSuccess && this.onPollingSuccess(result, this.configurations!)) { - this.stopPolling(); - } - } catch (err) { - this.error = err as Error; - this.loading = false; - - if (this.onPollingError && this.onPollingError(this.error)) { - this.stopPolling(); - } - } - } - - startPolling(params?: P) { - this.shouldPoll = true; - if (!this.intervalRef) { - this.intervalRef = setInterval(() => { - if (this.shouldPoll) { - this.fetchData(params); - } - }, this.interval); - } - } - - stopPolling() { - this.shouldPoll = false; - if (this.intervalRef) { - clearInterval((this.intervalRef as unknown) as NodeJS.Timeout); - this.intervalRef = undefined; - } - } -} - -interface UsePollingReturn { - data: T | null; - loading: boolean; - error: Error | null; - startPolling: (params?: any) => void; - stopPolling: () => void; -} - -export function usePolling( - fetchFunction: FetchFunction, - interval: number = 5000, - onPollingSuccess?: (data: T, configurations: PollingConfigurations) => boolean, - onPollingError?: (error: Error) => boolean, - configurations?: PollingConfigurations -): UsePollingReturn { - const [data, setData] = useState(null); - const [error, setError] = useState(null); - const [loading, setLoading] = useState(true); - const intervalRef = useRef(undefined); - const unmounted = useRef(false); - - const shouldPoll = useRef(false); - - const startPolling = (params?: P) => { - shouldPoll.current = true; - const intervalId = setInterval(() => { - if (shouldPoll.current) { - fetchData(params); - } - }, interval); - intervalRef.current = intervalId; - if (unmounted.current) { - clearInterval((intervalId as unknown) as NodeJS.Timeout); - } - }; - - const stopPolling = () => { - shouldPoll.current = false; - clearInterval((intervalRef.current as unknown) as NodeJS.Timeout); - }; - - const fetchData = async (params?: P) => { - try { - const result = await fetchFunction(params); - setData(result); - // Check the success condition and stop polling if it's met - if (onPollingSuccess && onPollingSuccess(result, configurations)) { - stopPolling(); - } - } catch (err: unknown) { - setError(err as Error); - - // Check the error condition and stop polling if it's met - if (onPollingError && onPollingError(err as Error)) { - stopPolling(); - } - } finally { - setLoading(false); - } - }; - - useEffect(() => { - return () => { - unmounted.current = true; - }; - }, []); - - return { data, loading, error, startPolling, stopPolling }; -} diff --git a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_direct_query_sync.tsx b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_direct_query_sync.tsx index 6befc8cb066a..385a9471a85b 100644 --- a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_direct_query_sync.tsx +++ b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_direct_query_sync.tsx @@ -5,7 +5,7 @@ import React from 'react'; import { EuiCallOut, EuiLink, EuiLoadingSpinner, EuiText } from '@elastic/eui'; -import { DirectQueryLoadingStatus } from '../../../../framework/types'; +import { DirectQueryLoadingStatus } from '../../../../../data_source_management/public'; import { EMR_STATES, intervalAsMinutes } from '../../utils/direct_query_sync/direct_query_sync'; import './_dashboard_direct_query_sync.scss'; diff --git a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx index 0c910241ea55..c6a648aaa56f 100644 --- a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx +++ b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx @@ -41,6 +41,7 @@ import { Subscription } from 'rxjs'; import ReactGridLayout, { Layout, ReactGridLayoutProps } from 'react-grid-layout'; import type { SavedObjectsClientContract } from 'src/core/public'; import { HttpStart, NotificationsStart } from 'src/core/public'; +import { DirectQueryLoadingStatus, DirectQueryRequest } from 'data_source_management/public'; import { ViewMode, EmbeddableChildPanel, EmbeddableStart } from '../../../../../embeddable/public'; import { GridData } from '../../../../common'; import { DASHBOARD_GRID_COLUMN_COUNT, DASHBOARD_GRID_HEIGHT } from '../dashboard_constants'; @@ -48,8 +49,6 @@ import { DashboardPanelState } from '../types'; import { withOpenSearchDashboards } from '../../../../../opensearch_dashboards_react/public'; import { DashboardContainerInput } from '../dashboard_container'; import { DashboardContainer, DashboardReactContextValue } from '../dashboard_container'; -import { DirectQueryLoadingStatus } from '../../../../framework/types'; -import { DirectQueryRequest } from '../../../../framework/types'; import { extractIndexInfoFromDashboard, generateRefreshQuery, diff --git a/src/plugins/dashboard/public/application/embeddable/viewport/dashboard_viewport.tsx b/src/plugins/dashboard/public/application/embeddable/viewport/dashboard_viewport.tsx index 446df7ca311f..f322469e77f2 100644 --- a/src/plugins/dashboard/public/application/embeddable/viewport/dashboard_viewport.tsx +++ b/src/plugins/dashboard/public/application/embeddable/viewport/dashboard_viewport.tsx @@ -40,9 +40,11 @@ import { PanelState, EmbeddableStart } from '../../../../../embeddable/public'; import { DashboardContainer, DashboardReactContextValue } from '../dashboard_container'; import { DashboardGrid } from '../grid'; import { context } from '../../../../../opensearch_dashboards_react/public'; -import { useDirectQuery } from '../../../../framework/hooks/direct_query_hook'; -import { DirectQueryRequest, DirectQueryLoadingStatus } from '../../../../framework/types'; - +import { + useDirectQuery, + DirectQueryRequest, + DirectQueryLoadingStatus, +} from '../../../../../data_source_management/public'; export interface DashboardViewportProps { container: DashboardContainer; PanelComponent: EmbeddableStart['EmbeddablePanel']; diff --git a/src/plugins/data_source_management/framework/hooks/direct_query_hook.tsx b/src/plugins/data_source_management/framework/hooks/direct_query_hook.tsx index a2b05f47e9ee..e6250f784c48 100644 --- a/src/plugins/data_source_management/framework/hooks/direct_query_hook.tsx +++ b/src/plugins/data_source_management/framework/hooks/direct_query_hook.tsx @@ -19,7 +19,7 @@ export const useDirectQuery = ( ) => { const sqlService = new SQLService(http); const [loadStatus, setLoadStatus] = useState( - DirectQueryLoadingStatus.SCHEDULED + DirectQueryLoadingStatus.FRESH ); const { diff --git a/src/plugins/data_source_management/framework/types.tsx b/src/plugins/data_source_management/framework/types.tsx index 34174a6c4b0e..73c83d85ad0b 100644 --- a/src/plugins/data_source_management/framework/types.tsx +++ b/src/plugins/data_source_management/framework/types.tsx @@ -9,10 +9,12 @@ export enum DirectQueryLoadingStatus { SUCCESS = 'success', FAILED = 'failed', RUNNING = 'running', + SUBMITTED = 'submitted', SCHEDULED = 'scheduled', - CANCELED = 'canceled', + CANCELLED = 'cancelled', WAITING = 'waiting', INITIAL = 'initial', + FRESH = 'fresh', } export interface DirectQueryRequest { diff --git a/src/plugins/data_source_management/public/index.ts b/src/plugins/data_source_management/public/index.ts index 11c2864ec86f..757eeb13b44c 100644 --- a/src/plugins/data_source_management/public/index.ts +++ b/src/plugins/data_source_management/public/index.ts @@ -27,3 +27,12 @@ export { export { DataSourceSelectionService } from './service/data_source_selection_service'; export { getDefaultDataSourceId, getDefaultDataSourceId$ } from './components/utils'; export { DATACONNECTIONS_BASE, DatasourceTypeToDisplayName } from './constants'; + +// Export framework utilities +export { usePolling, UsePolling, PollingConfigurations } from '../framework/utils/use_polling'; +export { SQLService } from '../framework/requests/sql'; +export { useDirectQuery } from '../framework/hooks/direct_query_hook'; +export { DirectQueryRequest, DirectQueryLoadingStatus } from '../framework/types'; +export { getAsyncSessionId, setAsyncSessionId } from '../framework/utils/query_session_utils'; +export { formatError } from '../framework/utils/shared'; +export { ASYNC_POLLING_INTERVAL } from '../framework/constants'; diff --git a/tsconfig.base.json b/tsconfig.base.json index d122bd79a5de..ddde899cf4c0 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -11,6 +11,8 @@ "plugins/*": ["src/legacy/core_plugins/*/public/"], "test_utils/*": ["src/test_utils/public/*"], "fixtures/*": ["src/fixtures/*"], + "data_source_management/public": ["src/plugins/data_source_management/public"], + "data_source_management/public/*": ["src/plugins/data_source_management/public/*"], "@opensearch-project/opensearch": ["node_modules/@opensearch-project/opensearch/api/new"], "@opensearch-project/opensearch/lib/*": ["node_modules/@opensearch-project/opensearch/lib/*"], }, From 6ff52a36e6327618a9485cdf5065b54d84ec8f2e Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Tue, 6 May 2025 20:44:38 -0700 Subject: [PATCH 56/86] remove unnecessary bundle Signed-off-by: Jialiang Liang --- src/plugins/dashboard/opensearch_dashboards.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/dashboard/opensearch_dashboards.json b/src/plugins/dashboard/opensearch_dashboards.json index 79a9eb3b297a..e00dcbd4255c 100644 --- a/src/plugins/dashboard/opensearch_dashboards.json +++ b/src/plugins/dashboard/opensearch_dashboards.json @@ -12,7 +12,7 @@ "savedObjects", "dataSourceManagement" ], - "optionalPlugins": ["home", "share", "usageCollection","dataSouce"], + "optionalPlugins": ["home", "share", "usageCollection"], "server": true, "ui": true, "requiredBundles": ["opensearchDashboardsUtils", "opensearchDashboardsReact", "home"] From a08af10e9fe4a2f7cb5c51c9c9d74f291bd4d0b8 Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Tue, 6 May 2025 21:00:12 -0700 Subject: [PATCH 57/86] UPDATE THE DIRECT QUERY HOOK TESTS Signed-off-by: Jialiang Liang --- .../frame_work_test/hooks/drect_query_hook.test.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/plugins/data_source_management/public/components/frame_work_test/hooks/drect_query_hook.test.tsx b/src/plugins/data_source_management/public/components/frame_work_test/hooks/drect_query_hook.test.tsx index c68906511bbb..7d088ee54b0f 100644 --- a/src/plugins/data_source_management/public/components/frame_work_test/hooks/drect_query_hook.test.tsx +++ b/src/plugins/data_source_management/public/components/frame_work_test/hooks/drect_query_hook.test.tsx @@ -68,9 +68,9 @@ describe('useDirectQuery', () => { })); }); - it('should initialize with scheduled status', () => { + it('should initialize with fresh status', () => { const { result } = renderHook(() => useDirectQuery(httpMock, notificationsMock)); - expect(result.current.loadStatus).toBe(DirectQueryLoadingStatus.SCHEDULED); + expect(result.current.loadStatus).toBe(DirectQueryLoadingStatus.FRESH); }); it('should handle successful query execution and start polling', async () => { From aa84467f927ead15826701e0b3579e64b7d5c980 Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Tue, 6 May 2025 23:50:46 -0700 Subject: [PATCH 58/86] fix the acc creation status check due to the outdated loading status Signed-off-by: Jialiang Liang --- .../create/create_acceleration_button.test.tsx | 2 +- .../create/create_acceleration_button.tsx | 15 ++++++++++++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/plugins/data_source_management/public/components/direct_query_data_sources_components/acceleration_creation/create/create_acceleration_button.test.tsx b/src/plugins/data_source_management/public/components/direct_query_data_sources_components/acceleration_creation/create/create_acceleration_button.test.tsx index 135d6fa0e1d8..e836ea9b1c8d 100644 --- a/src/plugins/data_source_management/public/components/direct_query_data_sources_components/acceleration_creation/create/create_acceleration_button.test.tsx +++ b/src/plugins/data_source_management/public/components/direct_query_data_sources_components/acceleration_creation/create/create_acceleration_button.test.tsx @@ -185,7 +185,7 @@ describe('CreateAccelerationButton', () => { test('handles other statuses correctly', async () => { const statuses = [ DirectQueryLoadingStatus.FAILED, - DirectQueryLoadingStatus.CANCELED, + DirectQueryLoadingStatus.CANCELLED, DirectQueryLoadingStatus.RUNNING, ]; diff --git a/src/plugins/data_source_management/public/components/direct_query_data_sources_components/acceleration_creation/create/create_acceleration_button.tsx b/src/plugins/data_source_management/public/components/direct_query_data_sources_components/acceleration_creation/create/create_acceleration_button.tsx index e7f021af2399..a6780d47fe89 100644 --- a/src/plugins/data_source_management/public/components/direct_query_data_sources_components/acceleration_creation/create/create_acceleration_button.tsx +++ b/src/plugins/data_source_management/public/components/direct_query_data_sources_components/acceleration_creation/create/create_acceleration_button.tsx @@ -61,16 +61,25 @@ export const CreateAccelerationButton = ({ useEffect(() => { const status = directqueryLoadStatus.toLowerCase(); - if (status === DirectQueryLoadingStatus.SUCCESS) { + if (status === DirectQueryLoadingStatus.SUCCESS.toLowerCase()) { setIsLoading(false); notifications.toasts.addSuccess('Create acceleration query submitted successfully!'); if (refreshHandler) refreshHandler(); resetFlyout(); } else if ( - status === DirectQueryLoadingStatus.FAILED || - status === DirectQueryLoadingStatus.CANCELED + status === DirectQueryLoadingStatus.FAILED.toLowerCase() || + status === DirectQueryLoadingStatus.CANCELLED.toLowerCase() || + status === DirectQueryLoadingStatus.INITIAL.toLowerCase() || + status === DirectQueryLoadingStatus.FRESH.toLowerCase() ) { setIsLoading(false); + } else if ( + status === DirectQueryLoadingStatus.SUBMITTED.toLowerCase() || + status === DirectQueryLoadingStatus.WAITING.toLowerCase() || + status === DirectQueryLoadingStatus.RUNNING.toLowerCase() || + status === DirectQueryLoadingStatus.SCHEDULED.toLowerCase() + ) { + setIsLoading(true); } // eslint-disable-next-line react-hooks/exhaustive-deps }, [directqueryLoadStatus]); From 8c831f9ad6af8705d0cb61cad947b0598166465c Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Wed, 7 May 2025 02:15:27 -0700 Subject: [PATCH 59/86] enhance the window reload logic Signed-off-by: Jialiang Liang --- .../application/embeddable/grid/dashboard_grid.tsx | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx index c6a648aaa56f..c45f8fb934ec 100644 --- a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx +++ b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx @@ -41,7 +41,10 @@ import { Subscription } from 'rxjs'; import ReactGridLayout, { Layout, ReactGridLayoutProps } from 'react-grid-layout'; import type { SavedObjectsClientContract } from 'src/core/public'; import { HttpStart, NotificationsStart } from 'src/core/public'; -import { DirectQueryLoadingStatus, DirectQueryRequest } from 'data_source_management/public'; +import { + DirectQueryLoadingStatus, + DirectQueryRequest, +} from '../../../../../data_source_management/public'; import { ViewMode, EmbeddableChildPanel, EmbeddableStart } from '../../../../../embeddable/public'; import { GridData } from '../../../../common'; import { DASHBOARD_GRID_COLUMN_COUNT, DASHBOARD_GRID_HEIGHT } from '../dashboard_constants'; @@ -409,7 +412,12 @@ class DashboardGridUi extends React.Component { const isViewMode = viewMode === ViewMode.VIEW; const state = EMR_STATES.get(this.props.loadStatus as string)!; - if (state?.terminal && this.props.loadStatus !== 'fresh') { + if ( + state?.terminal && + this.props.loadStatus !== DirectQueryLoadingStatus.FRESH && + this.props.loadStatus !== DirectQueryLoadingStatus.FAILED && + this.props.loadStatus !== DirectQueryLoadingStatus.CANCELLED + ) { window.location.reload(); } From a68483ae1527a9b02008c203c6264a78c8d5fdf5 Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Wed, 7 May 2025 02:19:40 -0700 Subject: [PATCH 60/86] fix header of test 2 Signed-off-by: Jialiang Liang --- .../grid/dashboard_direct_query_sync.test.tsx | 27 +------------------ 1 file changed, 1 insertion(+), 26 deletions(-) diff --git a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_direct_query_sync.test.tsx b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_direct_query_sync.test.tsx index c5858c62c9fa..2f8f64e53681 100644 --- a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_direct_query_sync.test.tsx +++ b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_direct_query_sync.test.tsx @@ -1,31 +1,6 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Any modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. */ import React from 'react'; From 5ed1d3a17b841fbfbe8dbad02dabf2363395b2a8 Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Wed, 7 May 2025 11:07:06 -0700 Subject: [PATCH 61/86] josh - address i18n-0 Signed-off-by: Jialiang Liang --- .../grid/dashboard_direct_query_sync.tsx | 40 ++++++++++++------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_direct_query_sync.tsx b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_direct_query_sync.tsx index 385a9471a85b..e30bdfefb307 100644 --- a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_direct_query_sync.tsx +++ b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_direct_query_sync.tsx @@ -5,6 +5,7 @@ import React from 'react'; import { EuiCallOut, EuiLink, EuiLoadingSpinner, EuiText } from '@elastic/eui'; +import { i18n } from '@osd/i18n'; import { DirectQueryLoadingStatus } from '../../../../../data_source_management/public'; import { EMR_STATES, intervalAsMinutes } from '../../utils/direct_query_sync/direct_query_sync'; import './_dashboard_direct_query_sync.scss'; @@ -28,24 +29,35 @@ export const DashboardDirectQuerySync: React.FC =
{state.terminal ? ( - Data scheduled to sync every{' '} - {refreshInterval ? intervalAsMinutes(1000 * refreshInterval) : '--'}. Last sync:{' '} - {lastRefreshTime ? ( - <> - {new Date(lastRefreshTime).toLocaleString()} ( - {intervalAsMinutes(new Date().getTime() - lastRefreshTime)} ago) - - ) : ( - '--' - )} - .    - Sync data + {i18n.translate('dashboard.directQuerySync.dataScheduledToSync', { + defaultMessage: 'Data scheduled to sync every {interval}. Last sync: {lastSyncTime}.', + values: { + interval: refreshInterval ? intervalAsMinutes(1000 * refreshInterval) : '--', + lastSyncTime: lastRefreshTime + ? `${new Date(lastRefreshTime).toLocaleString()} (${intervalAsMinutes( + new Date().getTime() - lastRefreshTime + )} ago)` + : '--', + }, + })} + + + {i18n.translate('dashboard.directQuerySync.syncDataLink', { + defaultMessage: 'Sync data', + })} + ) : ( -    Data sync is in progress ({state.ord}% complete). The dashboard - will reload on completion. + + {i18n.translate('dashboard.directQuerySync.dataSyncInProgress', { + defaultMessage: + 'Data sync is in progress ({progress}% complete). The dashboard will reload on completion.', + values: { + progress: state.ord, + }, + })} )}
From fc7383c10ebc5902fbc7cc5d13d4f27c1fbc0451 Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Wed, 7 May 2025 11:15:30 -0700 Subject: [PATCH 62/86] josh - address feature flag var renaming Signed-off-by: Jialiang Liang --- .../embeddable/grid/dashboard_grid.tsx | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx index c45f8fb934ec..c60763ce6318 100644 --- a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx +++ b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx @@ -230,20 +230,20 @@ class DashboardGridUi extends React.Component { }); const urlOverride = isDirectQuerySyncEnabledByUrl(); - const featureFlagEnabled = + const directQuerySyncEnabled = urlOverride !== undefined ? urlOverride : this.props.isDirectQuerySyncEnabled; - if (featureFlagEnabled) { + if (directQuerySyncEnabled) { this.collectAllPanelMetadata(); } } }); const urlOverride = isDirectQuerySyncEnabledByUrl(); - const featureFlagEnabled = + const directQuerySyncEnabled = urlOverride !== undefined ? urlOverride : this.props.isDirectQuerySyncEnabled; - if (featureFlagEnabled) { + if (directQuerySyncEnabled) { this.collectAllPanelMetadata(); } } @@ -298,10 +298,10 @@ class DashboardGridUi extends React.Component { */ private async collectAllPanelMetadata() { const urlOverride = isDirectQuerySyncEnabledByUrl(); - const featureFlagEnabled = + const directQuerySyncEnabled = urlOverride !== undefined ? urlOverride : this.props.isDirectQuerySyncEnabled; - if (!featureFlagEnabled) return; + if (!directQuerySyncEnabled) return; const indexInfo = await extractIndexInfoFromDashboard( this.state.panels, @@ -327,10 +327,10 @@ class DashboardGridUi extends React.Component { synchronizeNow = () => { const urlOverride = isDirectQuerySyncEnabledByUrl(); - const featureFlagEnabled = + const directQuerySyncEnabled = urlOverride !== undefined ? urlOverride : this.props.isDirectQuerySyncEnabled; - if (!featureFlagEnabled) return; + if (!directQuerySyncEnabled) return; const { extractedDatasource, extractedDatabase, extractedIndex } = this; if ( @@ -423,11 +423,11 @@ class DashboardGridUi extends React.Component { return (() => { const urlOverride = isDirectQuerySyncEnabledByUrl(); - const featureFlagEnabled = + const directQuerySyncEnabled = urlOverride !== undefined ? urlOverride : this.props.isDirectQuerySyncEnabled; const metadataAvailable = this.state.extractedProps !== null; - const shouldRenderSyncUI = featureFlagEnabled && metadataAvailable; + const shouldRenderSyncUI = directQuerySyncEnabled && metadataAvailable; return ( <> From 583a607bb0e382602cb3042ee04109d419fb8d10 Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Wed, 7 May 2025 11:23:11 -0700 Subject: [PATCH 63/86] josh - extract feature flag check into a function Signed-off-by: Jialiang Liang --- .../embeddable/grid/dashboard_grid.tsx | 34 ++++++------------- 1 file changed, 10 insertions(+), 24 deletions(-) diff --git a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx index c60763ce6318..d3884137f6ef 100644 --- a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx +++ b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx @@ -194,6 +194,11 @@ class DashboardGridUi extends React.Component { }; } + private isDirectQuerySyncEnabled(): boolean { + const urlOverride = isDirectQuerySyncEnabledByUrl(); + return urlOverride !== undefined ? urlOverride : this.props.isDirectQuerySyncEnabled; + } + public componentDidMount() { this.mounted = true; let isLayoutInvalid = false; @@ -229,21 +234,13 @@ class DashboardGridUi extends React.Component { expandedPanelId: input.expandedPanelId, }); - const urlOverride = isDirectQuerySyncEnabledByUrl(); - const directQuerySyncEnabled = - urlOverride !== undefined ? urlOverride : this.props.isDirectQuerySyncEnabled; - - if (directQuerySyncEnabled) { + if (this.isDirectQuerySyncEnabled()) { this.collectAllPanelMetadata(); } } }); - const urlOverride = isDirectQuerySyncEnabledByUrl(); - const directQuerySyncEnabled = - urlOverride !== undefined ? urlOverride : this.props.isDirectQuerySyncEnabled; - - if (directQuerySyncEnabled) { + if (this.isDirectQuerySyncEnabled()) { this.collectAllPanelMetadata(); } } @@ -297,11 +294,7 @@ class DashboardGridUi extends React.Component { * Runs on mount and when the container input (panels) changes. */ private async collectAllPanelMetadata() { - const urlOverride = isDirectQuerySyncEnabledByUrl(); - const directQuerySyncEnabled = - urlOverride !== undefined ? urlOverride : this.props.isDirectQuerySyncEnabled; - - if (!directQuerySyncEnabled) return; + if (!this.isDirectQuerySyncEnabled()) return; const indexInfo = await extractIndexInfoFromDashboard( this.state.panels, @@ -326,11 +319,7 @@ class DashboardGridUi extends React.Component { } synchronizeNow = () => { - const urlOverride = isDirectQuerySyncEnabledByUrl(); - const directQuerySyncEnabled = - urlOverride !== undefined ? urlOverride : this.props.isDirectQuerySyncEnabled; - - if (!directQuerySyncEnabled) return; + if (!this.isDirectQuerySyncEnabled()) return; const { extractedDatasource, extractedDatabase, extractedIndex } = this; if ( @@ -422,10 +411,7 @@ class DashboardGridUi extends React.Component { } return (() => { - const urlOverride = isDirectQuerySyncEnabledByUrl(); - const directQuerySyncEnabled = - urlOverride !== undefined ? urlOverride : this.props.isDirectQuerySyncEnabled; - + const directQuerySyncEnabled = this.isDirectQuerySyncEnabled(); const metadataAvailable = this.state.extractedProps !== null; const shouldRenderSyncUI = directQuerySyncEnabled && metadataAvailable; From 36c11c7807afe7bbd36361ad9c4e248363cbbcf1 Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Wed, 7 May 2025 11:43:15 -0700 Subject: [PATCH 64/86] josh - emr state renaming Signed-off-by: Jialiang Liang --- .../public/application/embeddable/grid/dashboard_grid.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx index d3884137f6ef..fa1dce38d8be 100644 --- a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx +++ b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx @@ -399,10 +399,10 @@ class DashboardGridUi extends React.Component { const { viewMode } = this.state; const isViewMode = viewMode === ViewMode.VIEW; - const state = EMR_STATES.get(this.props.loadStatus as string)!; + const emrState = EMR_STATES.get(this.props.loadStatus as string); if ( - state?.terminal && + emrState?.terminal && this.props.loadStatus !== DirectQueryLoadingStatus.FRESH && this.props.loadStatus !== DirectQueryLoadingStatus.FAILED && this.props.loadStatus !== DirectQueryLoadingStatus.CANCELLED From 8118a6f5d93e5c85f5dd1185e9c5d92196c34057 Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Wed, 7 May 2025 11:52:16 -0700 Subject: [PATCH 65/86] josh - apply suggestion - remove any for query type Signed-off-by: Jialiang Liang --- .../application/utils/direct_query_sync/direct_query_sync.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.ts b/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.ts index 19db798dfec1..db1a9ae35cb5 100644 --- a/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.ts +++ b/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.ts @@ -49,7 +49,7 @@ export async function resolveConcreteIndex( if (!indexTitle.includes('*')) return indexTitle; try { - const query: any = mdsId ? { data_source: mdsId } : {}; + const query = mdsId ? { data_source: mdsId } : {}; const resolved = await http.get( `/internal/index-pattern-management/resolve_index/${encodeURIComponent(indexTitle)}`, { query } From a12ea475e835ea38d0c94bc09571b2c06f76f727 Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Wed, 7 May 2025 12:10:33 -0700 Subject: [PATCH 66/86] josh - apply suggestion - private sync function and add some comments Signed-off-by: Jialiang Liang --- .../public/application/embeddable/grid/dashboard_grid.tsx | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx index fa1dce38d8be..1247e1f536ac 100644 --- a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx +++ b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx @@ -318,7 +318,12 @@ class DashboardGridUi extends React.Component { } } - synchronizeNow = () => { + /** + * Initiates a direct query sync to refresh the dashboard data. + * Uses the extracted datasource, database, and index to construct a refresh query, + * and triggers the sync process if direct query sync is enabled. + */ + private synchronizeNow = () => { if (!this.isDirectQuerySyncEnabled()) return; const { extractedDatasource, extractedDatabase, extractedIndex } = this; From 092471414d6ca845b5980907d2c7b7aeb0d616c4 Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Wed, 7 May 2025 12:31:26 -0700 Subject: [PATCH 67/86] josh - add lang parameter to accept more language types Signed-off-by: Jialiang Liang --- .../public/application/embeddable/grid/dashboard_grid.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx index 1247e1f536ac..0494ea38a8b2 100644 --- a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx +++ b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx @@ -147,6 +147,7 @@ export interface DashboardGridProps extends ReactIntl.InjectedIntlProps { loadStatus: DirectQueryLoadingStatus; pollingResult: any; isDirectQuerySyncEnabled: boolean; + queryLang?: string; setMdsId?: (mdsId?: string) => void; } @@ -346,7 +347,7 @@ class DashboardGridUi extends React.Component { this.props.startLoading({ query, - lang: 'sql', + lang: this.props.queryLang || 'sql', datasource: extractedDatasource, }); }; From d0836dc75c31c6fa2a8da7b4d92a1ca3ba011287 Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Wed, 7 May 2025 12:59:24 -0700 Subject: [PATCH 68/86] josh - add lang parameter to accept more language types and add some tests Signed-off-by: Jialiang Liang --- .../embeddable/grid/dashboard_grid.test.tsx | 218 ++++++++++++++++++ .../embeddable/grid/dashboard_grid.tsx | 51 ++-- 2 files changed, 253 insertions(+), 16 deletions(-) diff --git a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.test.tsx b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.test.tsx index 9c5c848a1eff..183040f0162e 100644 --- a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.test.tsx +++ b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.test.tsx @@ -389,3 +389,221 @@ test('synchronizeNow does nothing when metadata contains unknown values', async expect(startLoadingSpy).not.toHaveBeenCalled(); }); + +test('synchronizeNow exits early when feature flag is disabled and datasource is invalid', async () => { + const { props, options } = prepare({ isDirectQuerySyncEnabled: false }); + + (extractIndexInfoFromDashboard as jest.Mock).mockResolvedValue({ + parts: { datasource: 'ds', database: 'db', index: 'unknown' }, + mapping: { lastRefreshTime: 123456, refreshInterval: 30000 }, + mdsId: '', + }); + + const startLoadingSpy = jest.fn(); + props.startLoading = startLoadingSpy; + + const component = mountWithIntl( + + + + ); + + await new Promise((resolve) => setTimeout(resolve, 0)); + component.update(); + + (component.find('DashboardGridUi').instance() as any).synchronizeNow(); + + expect(startLoadingSpy).not.toHaveBeenCalled(); +}); + +test('synchronizeNow exits early when feature flag is enabled but datasource is invalid', async () => { + const { props, options } = prepare({ isDirectQuerySyncEnabled: true }); + + (extractIndexInfoFromDashboard as jest.Mock).mockResolvedValue({ + parts: { datasource: 'ds', database: '', index: 'idx' }, + mapping: { lastRefreshTime: 123456, refreshInterval: 30000 }, + mdsId: '', + }); + + const startLoadingSpy = jest.fn(); + props.startLoading = startLoadingSpy; + + const component = mountWithIntl( + + + + ); + + await new Promise((resolve) => setTimeout(resolve, 0)); + component.update(); + + (component.find('DashboardGridUi').instance() as any).synchronizeNow(); + + expect(startLoadingSpy).not.toHaveBeenCalled(); +}); + +test('areDataSourceParamsValid returns true for valid datasource params', async () => { + const { props, options } = prepare({ isDirectQuerySyncEnabled: true }); + + (extractIndexInfoFromDashboard as jest.Mock).mockResolvedValue({ + parts: { datasource: 'ds', database: 'db', index: 'idx' }, + mapping: { lastRefreshTime: 123456, refreshInterval: 30000 }, + mdsId: '', + }); + + const component = mountWithIntl( + + + + ); + + await new Promise((resolve) => setTimeout(resolve, 0)); + component.update(); + + const instance = component.find('DashboardGridUi').instance() as any; + expect(instance.areDataSourceParamsValid()).toBe(true); +}); + +test('areDataSourceParamsValid returns false for invalid datasource params', async () => { + const { props, options } = prepare({ isDirectQuerySyncEnabled: true }); + + (extractIndexInfoFromDashboard as jest.Mock).mockResolvedValue({ + parts: { datasource: 'ds', database: 'unknown', index: 'idx' }, + mapping: { lastRefreshTime: 123456, refreshInterval: 30000 }, + mdsId: '', + }); + + const component = mountWithIntl( + + + + ); + + await new Promise((resolve) => setTimeout(resolve, 0)); + component.update(); + + const instance = component.find('DashboardGridUi').instance() as any; + expect(instance.areDataSourceParamsValid()).toBe(false); +}); + +test('getQueryLanguage returns sql when feature is enabled and queryLang is not provided', async () => { + const { props, options } = prepare({ isDirectQuerySyncEnabled: true }); + + const component = mountWithIntl( + + + + ); + + await new Promise((resolve) => setTimeout(resolve, 0)); + component.update(); + + const instance = component.find('DashboardGridUi').instance() as any; + expect(instance.getQueryLanguage()).toBe('sql'); +}); + +test('getQueryLanguage returns empty string when feature is disabled and queryLang is not provided', async () => { + const { props, options } = prepare({ isDirectQuerySyncEnabled: false }); + + const component = mountWithIntl( + + + + ); + + await new Promise((resolve) => setTimeout(resolve, 0)); + component.update(); + + const instance = component.find('DashboardGridUi').instance() as any; + expect(instance.getQueryLanguage()).toBe(''); +}); + +test('getQueryLanguage returns provided queryLang when specified, regardless of feature flag', async () => { + const { props, options } = prepare({ isDirectQuerySyncEnabled: false, queryLang: 'ppl' }); + + const component = mountWithIntl( + + + + ); + + await new Promise((resolve) => setTimeout(resolve, 0)); + component.update(); + + const instance = component.find('DashboardGridUi').instance() as any; + expect(instance.getQueryLanguage()).toBe('ppl'); + + // Test with feature flag enabled + component.setProps({ isDirectQuerySyncEnabled: true }); + expect(instance.getQueryLanguage()).toBe('ppl'); +}); + +test('synchronizeNow uses sql when feature is enabled and queryLang is not provided', async () => { + const { props, options } = prepare({ isDirectQuerySyncEnabled: true }); + + const mockRefreshQuery = 'REFRESH MATERIALIZED VIEW ds.db.idx'; + (extractIndexInfoFromDashboard as jest.Mock).mockResolvedValue({ + parts: { datasource: 'ds', database: 'db', index: 'idx' }, + mapping: { lastRefreshTime: 123456, refreshInterval: 30000 }, + mdsId: '', + }); + + (generateRefreshQuery as jest.Mock).mockReturnValue(mockRefreshQuery); + + const startLoadingSpy = jest.fn(); + props.startLoading = startLoadingSpy; + + const component = mountWithIntl( + + + + ); + + await new Promise((resolve) => setTimeout(resolve, 0)); + component.update(); + + (component + .find('DashboardDirectQuerySync') + .props() as DashboardDirectQuerySyncProps).onSynchronize(); + + expect(startLoadingSpy).toHaveBeenCalledWith({ + query: mockRefreshQuery, + lang: 'sql', + datasource: 'ds', + }); +}); + +test('synchronizeNow uses provided queryLang when specified', async () => { + const { props, options } = prepare({ isDirectQuerySyncEnabled: true, queryLang: 'ppl' }); + + const mockRefreshQuery = 'REFRESH MATERIALIZED VIEW ds.db.idx'; + (extractIndexInfoFromDashboard as jest.Mock).mockResolvedValue({ + parts: { datasource: 'ds', database: 'db', index: 'idx' }, + mapping: { lastRefreshTime: 123456, refreshInterval: 30000 }, + mdsId: '', + }); + + (generateRefreshQuery as jest.Mock).mockReturnValue(mockRefreshQuery); + + const startLoadingSpy = jest.fn(); + props.startLoading = startLoadingSpy; + + const component = mountWithIntl( + + + + ); + + await new Promise((resolve) => setTimeout(resolve, 0)); + component.update(); + + (component + .find('DashboardDirectQuerySync') + .props() as DashboardDirectQuerySyncProps).onSynchronize(); + + expect(startLoadingSpy).toHaveBeenCalledWith({ + query: mockRefreshQuery, + lang: 'ppl', + datasource: 'ds', + }); +}); diff --git a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx index 0494ea38a8b2..bd2aa4c8b7f6 100644 --- a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx +++ b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx @@ -200,6 +200,17 @@ class DashboardGridUi extends React.Component { return urlOverride !== undefined ? urlOverride : this.props.isDirectQuerySyncEnabled; } + /** + * Determines the query language to use for direct query sync. + * Returns the provided queryLang if specified; otherwise, defaults to 'sql' if the feature is enabled. + */ + private getQueryLanguage(): string { + if (this.props.queryLang) { + return this.props.queryLang; + } + return this.isDirectQuerySyncEnabled() ? 'sql' : ''; + } + public componentDidMount() { this.mounted = true; let isLayoutInvalid = false; @@ -290,6 +301,22 @@ class DashboardGridUi extends React.Component { } }; + /** + * Validates if the extracted datasource, database, and index are present and valid. + * Returns true if all values are non-empty and not 'unknown', false otherwise. + */ + private areDataSourceParamsValid(): boolean { + const { extractedDatasource, extractedDatabase, extractedIndex } = this; + return ( + !!extractedDatasource && + !!extractedDatabase && + !!extractedIndex && + extractedDatasource !== 'unknown' && + extractedDatabase !== 'unknown' && + extractedIndex !== 'unknown' + ); + } + /** * Collects metadata (panelId, savedObjectId, type) for all panels in the dashboard. * Runs on mount and when the container input (panels) changes. @@ -325,30 +352,22 @@ class DashboardGridUi extends React.Component { * and triggers the sync process if direct query sync is enabled. */ private synchronizeNow = () => { - if (!this.isDirectQuerySyncEnabled()) return; - - const { extractedDatasource, extractedDatabase, extractedIndex } = this; - if ( - !extractedDatasource || - !extractedDatabase || - !extractedIndex || - extractedDatasource === 'unknown' || - extractedDatabase === 'unknown' || - extractedIndex === 'unknown' - ) { + if (!this.isDirectQuerySyncEnabled() || !this.areDataSourceParamsValid()) { return; } + const { extractedDatasource, extractedDatabase, extractedIndex } = this; + const query = generateRefreshQuery({ - datasource: extractedDatasource, - database: extractedDatabase, - index: extractedIndex, + datasource: extractedDatasource!, + database: extractedDatabase!, + index: extractedIndex!, }); this.props.startLoading({ query, - lang: this.props.queryLang || 'sql', - datasource: extractedDatasource, + lang: this.getQueryLanguage(), + datasource: extractedDatasource!, }); }; From d7f5b5c80a6925616ba1c8d2fa0d22f45cbba296 Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Wed, 7 May 2025 13:07:05 -0700 Subject: [PATCH 69/86] josh - add some comment for extractIndexInfoFromDashboard Signed-off-by: Jialiang Liang --- .../utils/direct_query_sync/direct_query_sync.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.ts b/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.ts index db1a9ae35cb5..4aee89c28f74 100644 --- a/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.ts +++ b/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.ts @@ -89,6 +89,12 @@ export function generateRefreshQuery(info: IndexExtractionResult): string { return `REFRESH MATERIALIZED VIEW \`${info.datasource}\`.\`${info.database}\`.\`${info.index}\``; } +/** + * Extracts index-related information from a dashboard's panels for direct query sync. + * Analyzes saved objects in the panels to identify a consistent index pattern use case of Integration Vended Dashboards, resolves it to a concrete index, + * fetches its mapping, and extracts datasource, database, and index details along with metadata like last refresh time. + * Returns null if the panels reference inconsistent index patterns, lack references, or if the index cannot be resolved. + */ export async function extractIndexInfoFromDashboard( panels: { [key: string]: any }, savedObjectsClient: SavedObjectsClientContract, From 06ca32d3ab40ef5f2202599e37d03138113afb20 Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Wed, 7 May 2025 13:18:11 -0700 Subject: [PATCH 70/86] josh - add i18n to min on ui Signed-off-by: Jialiang Liang --- .../utils/direct_query_sync/direct_query_sync.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.ts b/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.ts index 4aee89c28f74..693042254a75 100644 --- a/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.ts +++ b/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.ts @@ -4,6 +4,7 @@ */ import { HttpStart, SavedObjectsClientContract } from 'src/core/public'; +import { i18n } from '@osd/i18n'; interface IndexExtractionResult { datasource: string; @@ -38,7 +39,14 @@ export const MAX_ORD = 100; export function intervalAsMinutes(interval: number): string { const minutes = Math.floor(interval / 60000); - return minutes === 1 ? '1 minute' : minutes + ' minutes'; + return minutes === 1 + ? i18n.translate('dashboard.directQuerySync.intervalAsMinutes.oneMinute', { + defaultMessage: '1 minute', + }) + : i18n.translate('dashboard.directQuerySync.intervalAsMinutes.multipleMinutes', { + defaultMessage: '{minutes} minutes', + values: { minutes }, + }); } export async function resolveConcreteIndex( From 135af2c96c610eb35127064e14a3d4cdd2914133 Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Wed, 7 May 2025 13:20:27 -0700 Subject: [PATCH 71/86] josh - remove any in ref param Signed-off-by: Jialiang Liang --- .../application/utils/direct_query_sync/direct_query_sync.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.ts b/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.ts index 693042254a75..3c63707cf7f6 100644 --- a/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.ts +++ b/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.ts @@ -142,7 +142,7 @@ export async function extractIndexInfoFromDashboard( const indexPattern = await savedObjectsClient.get('index-pattern', indexPatternRef.id); const mdsId = - indexPattern.references?.find((ref: any) => ref.type === 'data-source')?.id || undefined; + indexPattern.references?.find((ref) => ref.type === 'data-source')?.id || undefined; indexPatternIds.push(indexPatternRef.id); mdsIds.push(mdsId); From 8ffc464e940b4b7b78d3987000d0bcaad13f0bc7 Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Wed, 7 May 2025 13:26:32 -0700 Subject: [PATCH 72/86] josh - refactor emr status map Signed-off-by: Jialiang Liang --- .../direct_query_sync/direct_query_sync.ts | 28 +++++++++---------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.ts b/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.ts index 3c63707cf7f6..1bdd8a782949 100644 --- a/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.ts +++ b/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.ts @@ -19,21 +19,19 @@ export const DSL_BASE = `${DIRECT_QUERY_BASE}/dsl`; // Module for handling EMR states for Dashboards Progress Bar. All of these except "fresh" are // directly from the EMR job run states. "ord" is used to approximate progress (eyeballed relative // stage times), and "terminal" indicates whether a job is in progress at all. -export const EMR_STATES = new Map( - Object.entries({ - submitted: { ord: 0, terminal: false }, - queued: { ord: 10, terminal: false }, - pending: { ord: 20, terminal: false }, - scheduled: { ord: 30, terminal: false }, - running: { ord: 70, terminal: false }, - cancelling: { ord: 90, terminal: false }, - success: { ord: 100, terminal: true }, - failed: { ord: 100, terminal: true }, - cancelled: { ord: 100, terminal: true }, - // The "null state" for a fresh page load, which components conditionally use on load. - fresh: { ord: 100, terminal: true }, - }) -); +export const EMR_STATES = new Map([ + ['submitted', { ord: 0, terminal: false }], + ['queued', { ord: 10, terminal: false }], + ['pending', { ord: 20, terminal: false }], + ['scheduled', { ord: 30, terminal: false }], + ['running', { ord: 70, terminal: false }], + ['cancelling', { ord: 90, terminal: false }], + ['success', { ord: 100, terminal: true }], + ['failed', { ord: 100, terminal: true }], + ['cancelled', { ord: 100, terminal: true }], + // The "null state" for a fresh page load, which components conditionally use on load. + ['fresh', { ord: 100, terminal: true }], +]); export const MAX_ORD = 100; From e0fd3efbf8d072396851cb74e7e8bc2a4e474daa Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Wed, 7 May 2025 13:32:29 -0700 Subject: [PATCH 73/86] josh - add commment for wrapper DashboardViewportWithQuery Signed-off-by: Jialiang Liang --- .../application/embeddable/viewport/dashboard_viewport.tsx | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/plugins/dashboard/public/application/embeddable/viewport/dashboard_viewport.tsx b/src/plugins/dashboard/public/application/embeddable/viewport/dashboard_viewport.tsx index f322469e77f2..2240169245ce 100644 --- a/src/plugins/dashboard/public/application/embeddable/viewport/dashboard_viewport.tsx +++ b/src/plugins/dashboard/public/application/embeddable/viewport/dashboard_viewport.tsx @@ -202,6 +202,12 @@ export class DashboardViewport extends React.Component ) => { From be93742a38a077e6514511e901d1b4c8cdb7783c Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Wed, 7 May 2025 14:32:39 -0700 Subject: [PATCH 74/86] josh - chaneg the return to null instead of unknown Signed-off-by: Jialiang Liang --- .../embeddable/grid/dashboard_grid.test.tsx | 11 ++-- .../embeddable/grid/dashboard_grid.tsx | 21 ++----- .../direct_query_sync.test.ts | 62 ++++++++++++++++--- .../direct_query_sync/direct_query_sync.ts | 36 +++++------ 4 files changed, 82 insertions(+), 48 deletions(-) diff --git a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.test.tsx b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.test.tsx index 183040f0162e..c840bed6fcf7 100644 --- a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.test.tsx +++ b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.test.tsx @@ -364,11 +364,11 @@ test('synchronizeNow does nothing when feature flag is disabled', async () => { expect(startLoadingSpy).not.toHaveBeenCalled(); }); -test('synchronizeNow does nothing when metadata contains unknown values', async () => { +test('synchronizeNow does nothing when metadata contains invalid values', async () => { const { props, options } = prepare({ isDirectQuerySyncEnabled: true }); (extractIndexInfoFromDashboard as jest.Mock).mockResolvedValue({ - parts: { datasource: 'ds', database: 'db', index: 'unknown' }, + parts: { datasource: 'ds', database: 'db', index: null }, mapping: { lastRefreshTime: 123456, refreshInterval: 30000 }, mdsId: '', }); @@ -390,11 +390,12 @@ test('synchronizeNow does nothing when metadata contains unknown values', async expect(startLoadingSpy).not.toHaveBeenCalled(); }); +// New tests for recent changes test('synchronizeNow exits early when feature flag is disabled and datasource is invalid', async () => { const { props, options } = prepare({ isDirectQuerySyncEnabled: false }); (extractIndexInfoFromDashboard as jest.Mock).mockResolvedValue({ - parts: { datasource: 'ds', database: 'db', index: 'unknown' }, + parts: { datasource: 'ds', database: 'db', index: null }, mapping: { lastRefreshTime: 123456, refreshInterval: 30000 }, mdsId: '', }); @@ -420,7 +421,7 @@ test('synchronizeNow exits early when feature flag is enabled but datasource is const { props, options } = prepare({ isDirectQuerySyncEnabled: true }); (extractIndexInfoFromDashboard as jest.Mock).mockResolvedValue({ - parts: { datasource: 'ds', database: '', index: 'idx' }, + parts: { datasource: 'ds', database: null, index: 'idx' }, mapping: { lastRefreshTime: 123456, refreshInterval: 30000 }, mdsId: '', }); @@ -468,7 +469,7 @@ test('areDataSourceParamsValid returns false for invalid datasource params', asy const { props, options } = prepare({ isDirectQuerySyncEnabled: true }); (extractIndexInfoFromDashboard as jest.Mock).mockResolvedValue({ - parts: { datasource: 'ds', database: 'unknown', index: 'idx' }, + parts: { datasource: 'ds', database: null, index: 'idx' }, mapping: { lastRefreshTime: 123456, refreshInterval: 30000 }, mdsId: '', }); diff --git a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx index bd2aa4c8b7f6..9fea61717bf7 100644 --- a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx +++ b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx @@ -175,9 +175,9 @@ class DashboardGridUi extends React.Component { // item. private gridItems = {} as { [key: string]: HTMLDivElement | null }; - private extractedDatasource?: string; - private extractedDatabase?: string; - private extractedIndex?: string; + private extractedDatasource?: string | null; + private extractedDatabase?: string | null; + private extractedIndex?: string | null; constructor(props: DashboardGridProps) { super(props); @@ -303,18 +303,11 @@ class DashboardGridUi extends React.Component { /** * Validates if the extracted datasource, database, and index are present and valid. - * Returns true if all values are non-empty and not 'unknown', false otherwise. + * Returns true if all values are non-null, false otherwise. */ private areDataSourceParamsValid(): boolean { const { extractedDatasource, extractedDatabase, extractedIndex } = this; - return ( - !!extractedDatasource && - !!extractedDatabase && - !!extractedIndex && - extractedDatasource !== 'unknown' && - extractedDatabase !== 'unknown' && - extractedIndex !== 'unknown' - ); + return !!extractedDatasource && !!extractedDatabase && !!extractedIndex; } /** @@ -352,9 +345,7 @@ class DashboardGridUi extends React.Component { * and triggers the sync process if direct query sync is enabled. */ private synchronizeNow = () => { - if (!this.isDirectQuerySyncEnabled() || !this.areDataSourceParamsValid()) { - return; - } + if (!this.isDirectQuerySyncEnabled() || !this.areDataSourceParamsValid()) return; const { extractedDatasource, extractedDatabase, extractedIndex } = this; diff --git a/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.test.ts b/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.test.ts index 8d8d82c9a008..b39b3ceba9ca 100644 --- a/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.test.ts +++ b/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.test.ts @@ -104,8 +104,8 @@ describe('resolveConcreteIndex', () => { }); describe('extractIndexParts', () => { - it('correctly extracts parts from a full index name', () => { - const result = extractIndexParts('flint_datasource1_database1_my_index'); + it('correctly extracts parts from a mapping name', () => { + const result = extractIndexParts('datasource1.database1.my_index'); expect(result).toEqual({ datasource: 'datasource1', database: 'database1', @@ -113,21 +113,30 @@ describe('extractIndexParts', () => { }); }); - it('handles missing parts with unknown values', () => { - const result = extractIndexParts('flint_datasource1'); + it('handles missing parts with null values', () => { + const result = extractIndexParts('datasource1'); expect(result).toEqual({ datasource: 'datasource1', - database: 'unknown', - index: 'unknown', + database: null, + index: null, }); }); - it('handles empty index name', () => { + it('handles empty mapping name with null values', () => { const result = extractIndexParts(''); expect(result).toEqual({ - datasource: 'unknown', - database: 'unknown', - index: 'unknown', + datasource: null, + database: null, + index: null, + }); + }); + + it('returns null values when mappingName is undefined', () => { + const result = extractIndexParts(undefined); + expect(result).toEqual({ + datasource: null, + database: null, + index: null, }); }); }); @@ -142,6 +151,39 @@ describe('generateRefreshQuery', () => { const result = generateRefreshQuery(info); expect(result).toBe('REFRESH MATERIALIZED VIEW `datasource1`.`database1`.`my_index`'); }); + + it('throws an error if datasource is null', () => { + const info = { + datasource: null, + database: 'database1', + index: 'my_index', + }; + expect(() => generateRefreshQuery(info)).toThrow( + 'Cannot generate refresh query: missing required datasource, database, or index' + ); + }); + + it('throws an error if database is null', () => { + const info = { + datasource: 'datasource1', + database: null, + index: 'my_index', + }; + expect(() => generateRefreshQuery(info)).toThrow( + 'Cannot generate refresh query: missing required datasource, database, or index' + ); + }); + + it('throws an error if index is null', () => { + const info = { + datasource: 'datasource1', + database: 'database1', + index: null, + }; + expect(() => generateRefreshQuery(info)).toThrow( + 'Cannot generate refresh query: missing required datasource, database, or index' + ); + }); }); describe('fetchIndexMapping', () => { diff --git a/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.ts b/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.ts index 1bdd8a782949..0134a5199f07 100644 --- a/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.ts +++ b/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.ts @@ -7,9 +7,9 @@ import { HttpStart, SavedObjectsClientContract } from 'src/core/public'; import { i18n } from '@osd/i18n'; interface IndexExtractionResult { - datasource: string; - database: string; - index: string; + datasource: string | null; + database: string | null; + index: string | null; } export const DIRECT_QUERY_BASE = '/api/directquery'; @@ -67,31 +67,31 @@ export async function resolveConcreteIndex( } } -export function extractIndexParts( - fullIndexName: string, - mappingName?: string -): IndexExtractionResult { - // Prefer parsing the mapping name if provided +export function extractIndexParts(mappingName?: string): IndexExtractionResult { + // Use mapping name if provided; otherwise, return null values if (mappingName) { const parts = mappingName.split('.'); return { - datasource: parts[0] || 'unknown', - database: parts[1] || 'unknown', - index: parts.slice(2).join('.') || 'unknown', + datasource: parts[0] || null, + database: parts[1] || null, + index: parts.slice(2).join('.') || null, }; } - // Fallback to original regex-based parsing - const trimmed = fullIndexName.replace(/^flint_/, ''); - const parts = trimmed.split('_'); return { - datasource: parts[0] || 'unknown', - database: parts[1] || 'unknown', - index: parts.slice(2).join('_') || 'unknown', + datasource: null, + database: null, + index: null, }; } export function generateRefreshQuery(info: IndexExtractionResult): string { + // Ensure all required fields are non-null before constructing the query + if (!info.datasource || !info.database || !info.index) { + throw new Error( + 'Cannot generate refresh query: missing required datasource, database, or index' + ); + } return `REFRESH MATERIALIZED VIEW \`${info.datasource}\`.\`${info.database}\`.\`${info.index}\``; } @@ -176,7 +176,7 @@ export async function extractIndexInfoFromDashboard( const mappingName = val.mappings?._meta?.name; return { mapping: val.mappings._meta.properties!, - parts: extractIndexParts(concreteTitle, mappingName), + parts: extractIndexParts(mappingName), mdsId: selectedMdsId, }; } From 51a5d3fadb2b006f2b7fbb9ea64131b13bc0e3ab Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Wed, 7 May 2025 15:37:54 -0700 Subject: [PATCH 75/86] josh - EXTRACT AS THE LOGIC IN GRID AS A SERVICE Signed-off-by: Jialiang Liang --- .../grid/dashboard_direct_query_sync.tsx | 5 +- .../embeddable/grid/dashboard_grid.tsx | 180 +++--------------- .../services/direct_query_sync_services.ts | 156 +++++++++++++++ .../viewport/dashboard_viewport.tsx | 74 +++++-- 4 files changed, 247 insertions(+), 168 deletions(-) create mode 100644 src/plugins/dashboard/public/application/embeddable/services/direct_query_sync_services.ts diff --git a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_direct_query_sync.tsx b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_direct_query_sync.tsx index e30bdfefb307..c349b26a6666 100644 --- a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_direct_query_sync.tsx +++ b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_direct_query_sync.tsx @@ -11,7 +11,7 @@ import { EMR_STATES, intervalAsMinutes } from '../../utils/direct_query_sync/dir import './_dashboard_direct_query_sync.scss'; export interface DashboardDirectQuerySyncProps { - loadStatus: DirectQueryLoadingStatus; + loadStatus?: DirectQueryLoadingStatus; lastRefreshTime?: number; refreshInterval?: number; onSynchronize: () => void; @@ -23,7 +23,8 @@ export const DashboardDirectQuerySync: React.FC = refreshInterval, onSynchronize, }) => { - const state = EMR_STATES.get(loadStatus)!; + // If loadStatus is undefined, default to a non-terminal state to avoid errors + const state = loadStatus ? EMR_STATES.get(loadStatus)! : { ord: 0, terminal: false }; return (
diff --git a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx index 9fea61717bf7..6d2b54c287a9 100644 --- a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx +++ b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx @@ -39,12 +39,6 @@ import _ from 'lodash'; import React from 'react'; import { Subscription } from 'rxjs'; import ReactGridLayout, { Layout, ReactGridLayoutProps } from 'react-grid-layout'; -import type { SavedObjectsClientContract } from 'src/core/public'; -import { HttpStart, NotificationsStart } from 'src/core/public'; -import { - DirectQueryLoadingStatus, - DirectQueryRequest, -} from '../../../../../data_source_management/public'; import { ViewMode, EmbeddableChildPanel, EmbeddableStart } from '../../../../../embeddable/public'; import { GridData } from '../../../../common'; import { DASHBOARD_GRID_COLUMN_COUNT, DASHBOARD_GRID_HEIGHT } from '../dashboard_constants'; @@ -52,13 +46,8 @@ import { DashboardPanelState } from '../types'; import { withOpenSearchDashboards } from '../../../../../opensearch_dashboards_react/public'; import { DashboardContainerInput } from '../dashboard_container'; import { DashboardContainer, DashboardReactContextValue } from '../dashboard_container'; -import { - extractIndexInfoFromDashboard, - generateRefreshQuery, - EMR_STATES, -} from '../../utils/direct_query_sync/direct_query_sync'; import { DashboardDirectQuerySync } from './dashboard_direct_query_sync'; -import { isDirectQuerySyncEnabledByUrl } from '../../utils/direct_query_sync/direct_query_sync_url_flag'; +import { DirectQueryLoadingStatus } from '../../../../../data_source_management/public'; let lastValidGridSize = 0; @@ -140,15 +129,11 @@ export interface DashboardGridProps extends ReactIntl.InjectedIntlProps { opensearchDashboards: DashboardReactContextValue; PanelComponent: EmbeddableStart['EmbeddablePanel']; container: DashboardContainer; - savedObjectsClient: SavedObjectsClientContract; - http: HttpStart; - notifications: NotificationsStart; - startLoading: (payload: DirectQueryRequest) => void; - loadStatus: DirectQueryLoadingStatus; - pollingResult: any; - isDirectQuerySyncEnabled: boolean; - queryLang?: string; - setMdsId?: (mdsId?: string) => void; + loadStatus?: DirectQueryLoadingStatus; + lastRefreshTime?: number; + refreshInterval?: number; + shouldRenderSyncUI?: boolean; + onSynchronize?: () => void; } interface State { @@ -159,9 +144,6 @@ interface State { viewMode: ViewMode; useMargins: boolean; expandedPanelId?: string; - panelMetadata: Array<{ panelId: string; savedObjectId: string; type: string }>; - extractedProps: { lastRefreshTime?: number; refreshInterval?: number } | null; - prevStatus?: string; } interface PanelLayout extends Layout { @@ -175,10 +157,6 @@ class DashboardGridUi extends React.Component { // item. private gridItems = {} as { [key: string]: HTMLDivElement | null }; - private extractedDatasource?: string | null; - private extractedDatabase?: string | null; - private extractedIndex?: string | null; - constructor(props: DashboardGridProps) { super(props); @@ -190,27 +168,9 @@ class DashboardGridUi extends React.Component { viewMode: this.props.container.getInput().viewMode, useMargins: this.props.container.getInput().useMargins, expandedPanelId: this.props.container.getInput().expandedPanelId, - panelMetadata: [], - extractedProps: null, }; } - private isDirectQuerySyncEnabled(): boolean { - const urlOverride = isDirectQuerySyncEnabledByUrl(); - return urlOverride !== undefined ? urlOverride : this.props.isDirectQuerySyncEnabled; - } - - /** - * Determines the query language to use for direct query sync. - * Returns the provided queryLang if specified; otherwise, defaults to 'sql' if the feature is enabled. - */ - private getQueryLanguage(): string { - if (this.props.queryLang) { - return this.props.queryLang; - } - return this.isDirectQuerySyncEnabled() ? 'sql' : ''; - } - public componentDidMount() { this.mounted = true; let isLayoutInvalid = false; @@ -245,16 +205,8 @@ class DashboardGridUi extends React.Component { useMargins: input.useMargins, expandedPanelId: input.expandedPanelId, }); - - if (this.isDirectQuerySyncEnabled()) { - this.collectAllPanelMetadata(); - } } }); - - if (this.isDirectQuerySyncEnabled()) { - this.collectAllPanelMetadata(); - } } public componentWillUnmount() { @@ -301,67 +253,6 @@ class DashboardGridUi extends React.Component { } }; - /** - * Validates if the extracted datasource, database, and index are present and valid. - * Returns true if all values are non-null, false otherwise. - */ - private areDataSourceParamsValid(): boolean { - const { extractedDatasource, extractedDatabase, extractedIndex } = this; - return !!extractedDatasource && !!extractedDatabase && !!extractedIndex; - } - - /** - * Collects metadata (panelId, savedObjectId, type) for all panels in the dashboard. - * Runs on mount and when the container input (panels) changes. - */ - private async collectAllPanelMetadata() { - if (!this.isDirectQuerySyncEnabled()) return; - - const indexInfo = await extractIndexInfoFromDashboard( - this.state.panels, - this.props.savedObjectsClient, - this.props.http - ); - - if (indexInfo) { - this.extractedDatasource = indexInfo.parts.datasource; - this.extractedDatabase = indexInfo.parts.database; - this.extractedIndex = indexInfo.parts.index; - this.setState({ extractedProps: indexInfo.mapping }); - if (this.props.setMdsId) { - this.props.setMdsId(indexInfo.mdsId); - } - } else { - this.setState({ extractedProps: null }); - if (this.props.setMdsId) { - this.props.setMdsId(undefined); - } - } - } - - /** - * Initiates a direct query sync to refresh the dashboard data. - * Uses the extracted datasource, database, and index to construct a refresh query, - * and triggers the sync process if direct query sync is enabled. - */ - private synchronizeNow = () => { - if (!this.isDirectQuerySyncEnabled() || !this.areDataSourceParamsValid()) return; - - const { extractedDatasource, extractedDatabase, extractedIndex } = this; - - const query = generateRefreshQuery({ - datasource: extractedDatasource!, - database: extractedDatabase!, - index: extractedIndex!, - }); - - this.props.startLoading({ - query, - lang: this.getQueryLanguage(), - datasource: extractedDatasource!, - }); - }; - public renderPanels() { const { focusedPanelIndex, panels, expandedPanelId } = this.state; @@ -415,45 +306,28 @@ class DashboardGridUi extends React.Component { const { viewMode } = this.state; const isViewMode = viewMode === ViewMode.VIEW; - const emrState = EMR_STATES.get(this.props.loadStatus as string); - if ( - emrState?.terminal && - this.props.loadStatus !== DirectQueryLoadingStatus.FRESH && - this.props.loadStatus !== DirectQueryLoadingStatus.FAILED && - this.props.loadStatus !== DirectQueryLoadingStatus.CANCELLED - ) { - window.location.reload(); - } - - return (() => { - const directQuerySyncEnabled = this.isDirectQuerySyncEnabled(); - const metadataAvailable = this.state.extractedProps !== null; - const shouldRenderSyncUI = directQuerySyncEnabled && metadataAvailable; - - return ( - <> - {shouldRenderSyncUI && ( - - )} - - - {this.renderPanels()} - - - ); - })(); + return ( + <> + {this.props.shouldRenderSyncUI && ( + + )} + + {this.renderPanels()} + + + ); } } diff --git a/src/plugins/dashboard/public/application/embeddable/services/direct_query_sync_services.ts b/src/plugins/dashboard/public/application/embeddable/services/direct_query_sync_services.ts new file mode 100644 index 000000000000..c010de9c7a94 --- /dev/null +++ b/src/plugins/dashboard/public/application/embeddable/services/direct_query_sync_services.ts @@ -0,0 +1,156 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { SavedObjectsClientContract, HttpStart } from 'src/core/public'; +import { DirectQueryRequest } from '../../../../../data_source_management/public'; +import { + extractIndexInfoFromDashboard, + generateRefreshQuery, +} from '../../utils/direct_query_sync/direct_query_sync'; +import { isDirectQuerySyncEnabledByUrl } from '../../utils/direct_query_sync/direct_query_sync_url_flag'; +import { DashboardPanelState } from '../'; + +interface DirectQuerySyncServiceProps { + savedObjectsClient: SavedObjectsClientContract; + http: HttpStart; + startLoading: (payload: DirectQueryRequest) => void; + setMdsId?: (mdsId?: string) => void; + isDirectQuerySyncEnabled: boolean; + queryLang?: string; +} + +interface DirectQuerySyncState { + extractedProps: { lastRefreshTime?: number; refreshInterval?: number } | null; + panelMetadata: Array<{ panelId: string; savedObjectId: string; type: string }>; +} + +export class DirectQuerySyncService { + private savedObjectsClient: SavedObjectsClientContract; + private http: HttpStart; + private startLoading: (payload: DirectQueryRequest) => void; + private setMdsId?: (mdsId?: string) => void; + private isDirectQuerySyncEnabledProp: boolean; + private queryLang?: string; + + private extractedDatasource: string | null = null; + private extractedDatabase: string | null = null; + private extractedIndex: string | null = null; + private state: DirectQuerySyncState = { + extractedProps: null, + panelMetadata: [], + }; + + constructor(props: DirectQuerySyncServiceProps) { + this.savedObjectsClient = props.savedObjectsClient; + this.http = props.http; + this.startLoading = props.startLoading; + this.setMdsId = props.setMdsId; + this.isDirectQuerySyncEnabledProp = props.isDirectQuerySyncEnabled; + this.queryLang = props.queryLang; + } + + /** + * Determines if direct query sync is enabled, considering URL overrides. + */ + public isDirectQuerySyncEnabled(): boolean { + const urlOverride = isDirectQuerySyncEnabledByUrl(); + return urlOverride !== undefined ? urlOverride : this.isDirectQuerySyncEnabledProp; + } + + /** + * Determines the query language to use for direct query sync. + * Returns the provided queryLang if specified; otherwise, defaults to 'sql' if the feature is enabled. + */ + public getQueryLanguage(): string { + if (this.queryLang) { + return this.queryLang; + } + return this.isDirectQuerySyncEnabled() ? 'sql' : ''; + } + + /** + * Validates if the extracted datasource, database, and index are present and valid. + * Returns true if all values are non-null, false otherwise. + */ + private areDataSourceParamsValid(): boolean { + return !!this.extractedDatasource && !!this.extractedDatabase && !!this.extractedIndex; + } + + /** + * Collects metadata (panelId, savedObjectId, type) for all panels in the dashboard. + * Updates the service state with extracted props and sets MDS ID if applicable. + */ + public async collectAllPanelMetadata(panels: { [key: string]: DashboardPanelState }) { + if (!this.isDirectQuerySyncEnabled()) return; + + const indexInfo = await extractIndexInfoFromDashboard( + panels, + this.savedObjectsClient, + this.http + ); + + if (indexInfo) { + this.extractedDatasource = indexInfo.parts.datasource; + this.extractedDatabase = indexInfo.parts.database; + this.extractedIndex = indexInfo.parts.index; + this.state.extractedProps = indexInfo.mapping; + if (this.setMdsId) { + this.setMdsId(indexInfo.mdsId); + } + } else { + this.extractedDatasource = null; + this.extractedDatabase = null; + this.extractedIndex = null; + this.state.extractedProps = null; + if (this.setMdsId) { + this.setMdsId(undefined); + } + } + } + + /** + * Initiates a direct query sync to refresh the dashboard data. + * Uses the extracted datasource, database, and index to construct a refresh query, + * and triggers the sync process if direct query sync is enabled. + */ + public synchronizeNow = () => { + if (!this.isDirectQuerySyncEnabled() || !this.areDataSourceParamsValid()) return; + + const query = generateRefreshQuery({ + datasource: this.extractedDatasource!, + database: this.extractedDatabase!, + index: this.extractedIndex!, + }); + + this.startLoading({ + query, + lang: this.getQueryLanguage(), + datasource: this.extractedDatasource!, + }); + }; + + /** + * Returns the current state of extracted properties (lastRefreshTime, refreshInterval). + */ + public getExtractedProps(): { lastRefreshTime?: number; refreshInterval?: number } | null { + return this.state.extractedProps; + } + + /** + * Updates the service with new panels, triggering metadata collection if needed. + */ + public updatePanels(panels: { [key: string]: DashboardPanelState }) { + if (this.isDirectQuerySyncEnabled()) { + this.collectAllPanelMetadata(panels); + } + } + + /** + * Cleans up any resources or subscriptions (if added in the future). + */ + public destroy() { + // Currently, no subscriptions to clean up, but this method can be extended later. + } +} diff --git a/src/plugins/dashboard/public/application/embeddable/viewport/dashboard_viewport.tsx b/src/plugins/dashboard/public/application/embeddable/viewport/dashboard_viewport.tsx index 2240169245ce..4eeefd415448 100644 --- a/src/plugins/dashboard/public/application/embeddable/viewport/dashboard_viewport.tsx +++ b/src/plugins/dashboard/public/application/embeddable/viewport/dashboard_viewport.tsx @@ -41,10 +41,13 @@ import { DashboardContainer, DashboardReactContextValue } from '../dashboard_con import { DashboardGrid } from '../grid'; import { context } from '../../../../../opensearch_dashboards_react/public'; import { - useDirectQuery, DirectQueryRequest, DirectQueryLoadingStatus, } from '../../../../../data_source_management/public'; +import { useDirectQuery } from '../../../../../data_source_management/public'; +import { DirectQuerySyncService } from '../../../application/embeddable/services/direct_query_sync_services'; +import { EMR_STATES } from '../../utils/direct_query_sync/direct_query_sync'; + export interface DashboardViewportProps { container: DashboardContainer; PanelComponent: EmbeddableStart['EmbeddablePanel']; @@ -53,10 +56,11 @@ export interface DashboardViewportProps { savedObjectsClient: SavedObjectsClientContract; http: HttpStart; notifications: NotificationsStart; - startLoading: (payload: DirectQueryRequest) => void; - loadStatus: DirectQueryLoadingStatus; - pollingResult: any; + startLoading?: (payload: DirectQueryRequest) => void; + loadStatus?: DirectQueryLoadingStatus; + pollingResult?: any; isDirectQuerySyncEnabled: boolean; + queryLang?: string; setMdsId?: (mdsId?: string) => void; } @@ -76,6 +80,8 @@ export class DashboardViewport extends React.Component { + // This will be set by DashboardViewportWithQuery + if (this.props.startLoading) { + this.props.startLoading(payload); + } + }, + setMdsId: (mdsId?: string) => { + // This will be set by DashboardViewportWithQuery + if (this.props.setMdsId) { + this.props.setMdsId(mdsId); + } + }, + isDirectQuerySyncEnabled: this.props.isDirectQuerySyncEnabled, + queryLang: this.props.queryLang, + }); + + // Initial metadata collection + if (this.syncService.isDirectQuerySyncEnabled()) { + this.syncService.collectAllPanelMetadata(panels); + } } public componentDidMount() { @@ -107,6 +138,7 @@ export class DashboardViewport extends React.Component { @@ -152,7 +187,7 @@ export class DashboardViewport extends React.Component
); } public render() { + const emrState = EMR_STATES.get(this.props.loadStatus as string); + + if ( + emrState?.terminal && + this.props.loadStatus !== DirectQueryLoadingStatus.FRESH && + this.props.loadStatus !== DirectQueryLoadingStatus.FAILED && + this.props.loadStatus !== DirectQueryLoadingStatus.CANCELLED + ) { + window.location.reload(); + } + return ( {this.state.isEmptyState ? this.renderEmptyScreen() : null} From 50d193266a513b5194292d6f899c39bf1506fee4 Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Wed, 7 May 2025 15:51:36 -0700 Subject: [PATCH 76/86] josh - REVERT GRID TEST AND JUST ADD A CASE FOR SYNC Signed-off-by: Jialiang Liang --- .../embeddable/grid/dashboard_grid.test.tsx | 402 ++---------------- 1 file changed, 45 insertions(+), 357 deletions(-) diff --git a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.test.tsx b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.test.tsx index c840bed6fcf7..407c08508f96 100644 --- a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.test.tsx +++ b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.test.tsx @@ -45,31 +45,7 @@ import { embeddablePluginMock } from '../../../../../embeddable/public/mocks'; import { createDashboardServicesMock } from '../../utils/mocks'; import { OpenSearchDashboardsContextProvider } from '../../../../../opensearch_dashboards_react/public'; import { DashboardDirectQuerySyncProps } from './dashboard_direct_query_sync'; -import { - extractIndexInfoFromDashboard, - generateRefreshQuery, -} from '../../utils/direct_query_sync/direct_query_sync'; - -jest.mock('../../utils/direct_query_sync/direct_query_sync', () => { - const actual = jest.requireActual('../../utils/direct_query_sync/direct_query_sync'); - return { - ...actual, - extractIndexInfoFromDashboard: jest.fn(), - generateRefreshQuery: jest.fn(), - EMR_STATES: new Map([ - ['submitted', { ord: 0, terminal: false }], - ['queued', { ord: 10, terminal: false }], - ['pending', { ord: 20, terminal: false }], - ['scheduled', { ord: 30, terminal: false }], - ['running', { ord: 70, terminal: false }], - ['cancelling', { ord: 90, terminal: false }], - ['success', { ord: 100, terminal: true }], - ['failed', { ord: 100, terminal: true }], - ['cancelled', { ord: 100, terminal: true }], - ['fresh', { ord: 100, terminal: true }], - ]), - }; -}); +import { DirectQueryLoadingStatus } from '../../../../../data_source_management/public'; let dashboardContainer: DashboardContainer | undefined; @@ -100,29 +76,6 @@ function prepare(props?: Partial) { const services = createDashboardServicesMock(); - // Mock savedObjectsClient.get to handle both visualization and index-pattern types - jest.spyOn(services.savedObjectsClient, 'get').mockImplementation((type: string, id: string) => { - if (!type || !id) throw new Error('requires type and id'); - - if (type === 'visualization' || type === CONTACT_CARD_EMBEDDABLE) { - return Promise.resolve({ - id, - attributes: {}, - references: [{ type: 'index-pattern', id: 'index-pattern-1' }], - }); - } - - if (type === 'index-pattern') { - return Promise.resolve({ - id, - attributes: {}, - references: [{ type: 'data-source', id: 'ds-id' }], - }); - } - - throw new Error(`Unknown saved object type: ${type}`); - }); - const options: DashboardContainerOptions = { application: {} as any, embeddable: { @@ -142,6 +95,11 @@ function prepare(props?: Partial) { uiActions: { getTriggerCompatibleActions: (() => []) as any, } as any, + savedObjectsClient: {} as any, + http: {} as any, + dashboardFeatureFlagConfig: { + directQueryConnectionSync: true, + } as any, }; dashboardContainer = new DashboardContainer(initialInput, options); @@ -153,13 +111,6 @@ function prepare(props?: Partial) { services, }, intl: null as any, - savedObjectsClient: services.savedObjectsClient, - http: services.http, - notifications: services.notifications, - startLoading: jest.fn(), - loadStatus: 'fresh', - pollingResult: {}, - isDirectQuerySyncEnabled: false, }; return { @@ -265,13 +216,14 @@ test('DashboardGrid unmount unsubscribes', (done) => { props.container.updateInput({ expandedPanelId: '1' }); }); -test('renders sync UI when feature flag is enabled and metadata is present', async () => { - const { props, options } = prepare({ isDirectQuerySyncEnabled: true }); - - (extractIndexInfoFromDashboard as jest.Mock).mockResolvedValue({ - parts: { datasource: 'ds', database: 'db', index: 'idx' }, - mapping: { lastRefreshTime: 123456, refreshInterval: 30000 }, - mdsId: '', +test('renders sync UI when shouldRenderSyncUI is true', () => { + const onSynchronizeMock = jest.fn(); + const { props, options } = prepare({ + shouldRenderSyncUI: true, + loadStatus: DirectQueryLoadingStatus.FRESH, + lastRefreshTime: 123456, + refreshInterval: 30000, + onSynchronize: onSynchronizeMock, }); const component = mountWithIntl( @@ -280,279 +232,36 @@ test('renders sync UI when feature flag is enabled and metadata is present', asy ); - // Wait for async metadata collection - await new Promise((resolve) => setTimeout(resolve, 0)); - component.update(); - expect(component.find('DashboardDirectQuerySync').exists()).toBe(true); -}); - -test('does not render sync UI when feature flag is off', async () => { - const { props, options } = prepare({ isDirectQuerySyncEnabled: false }); - - (extractIndexInfoFromDashboard as jest.Mock).mockResolvedValue({ - parts: { datasource: 'ds', database: 'db', index: 'idx' }, - mapping: { lastRefreshTime: 123456, refreshInterval: 30000 }, - mdsId: '', - }); - - const component = mountWithIntl( - - - - ); - - await new Promise((resolve) => setTimeout(resolve, 0)); - component.update(); - - expect(component.find('DashboardDirectQuerySync').exists()).toBe(false); -}); - -test('synchronizeNow triggers REFRESH query generation and startLoading', async () => { - const { props, options } = prepare({ isDirectQuerySyncEnabled: true }); - - const mockRefreshQuery = 'REFRESH MATERIALIZED VIEW ds.db.idx'; - (extractIndexInfoFromDashboard as jest.Mock).mockResolvedValue({ - parts: { datasource: 'ds', database: 'db', index: 'idx' }, - mapping: { lastRefreshTime: 123456, refreshInterval: 30000 }, - mdsId: '', - }); - - (generateRefreshQuery as jest.Mock).mockReturnValue(mockRefreshQuery); - - const startLoadingSpy = jest.fn(); - props.startLoading = startLoadingSpy; - - const component = mountWithIntl( - - - - ); - - await new Promise((resolve) => setTimeout(resolve, 0)); - component.update(); - - (component + const syncComponentProps = component .find('DashboardDirectQuerySync') - .props() as DashboardDirectQuerySyncProps).onSynchronize(); - - expect(startLoadingSpy).toHaveBeenCalledWith({ - query: mockRefreshQuery, - lang: 'sql', - datasource: 'ds', - }); + .props() as DashboardDirectQuerySyncProps; + expect(syncComponentProps.loadStatus).toBe(DirectQueryLoadingStatus.FRESH); + expect(syncComponentProps.lastRefreshTime).toBe(123456); + expect(syncComponentProps.refreshInterval).toBe(30000); + expect(syncComponentProps.onSynchronize).toBe(onSynchronizeMock); }); -test('synchronizeNow does nothing when feature flag is disabled', async () => { - const { props, options } = prepare({ isDirectQuerySyncEnabled: false }); - - const startLoadingSpy = jest.fn(); - props.startLoading = startLoadingSpy; - - const component = mountWithIntl( - - - - ); - - await new Promise((resolve) => setTimeout(resolve, 0)); - component.update(); - - // Simulate calling synchronizeNow directly - (component.find('DashboardGridUi').instance() as any).synchronizeNow(); - - expect(startLoadingSpy).not.toHaveBeenCalled(); -}); - -test('synchronizeNow does nothing when metadata contains invalid values', async () => { - const { props, options } = prepare({ isDirectQuerySyncEnabled: true }); - - (extractIndexInfoFromDashboard as jest.Mock).mockResolvedValue({ - parts: { datasource: 'ds', database: 'db', index: null }, - mapping: { lastRefreshTime: 123456, refreshInterval: 30000 }, - mdsId: '', - }); - - const startLoadingSpy = jest.fn(); - props.startLoading = startLoadingSpy; - - const component = mountWithIntl( - - - - ); - - await new Promise((resolve) => setTimeout(resolve, 0)); - component.update(); - - (component.find('DashboardGridUi').instance() as any).synchronizeNow(); - - expect(startLoadingSpy).not.toHaveBeenCalled(); -}); - -// New tests for recent changes -test('synchronizeNow exits early when feature flag is disabled and datasource is invalid', async () => { - const { props, options } = prepare({ isDirectQuerySyncEnabled: false }); - - (extractIndexInfoFromDashboard as jest.Mock).mockResolvedValue({ - parts: { datasource: 'ds', database: 'db', index: null }, - mapping: { lastRefreshTime: 123456, refreshInterval: 30000 }, - mdsId: '', +test('does not render sync UI when shouldRenderSyncUI is false', () => { + const { props, options } = prepare({ + shouldRenderSyncUI: false, + loadStatus: DirectQueryLoadingStatus.FRESH, + lastRefreshTime: 123456, + refreshInterval: 30000, + onSynchronize: jest.fn(), }); - const startLoadingSpy = jest.fn(); - props.startLoading = startLoadingSpy; - const component = mountWithIntl( ); - await new Promise((resolve) => setTimeout(resolve, 0)); - component.update(); - - (component.find('DashboardGridUi').instance() as any).synchronizeNow(); - - expect(startLoadingSpy).not.toHaveBeenCalled(); -}); - -test('synchronizeNow exits early when feature flag is enabled but datasource is invalid', async () => { - const { props, options } = prepare({ isDirectQuerySyncEnabled: true }); - - (extractIndexInfoFromDashboard as jest.Mock).mockResolvedValue({ - parts: { datasource: 'ds', database: null, index: 'idx' }, - mapping: { lastRefreshTime: 123456, refreshInterval: 30000 }, - mdsId: '', - }); - - const startLoadingSpy = jest.fn(); - props.startLoading = startLoadingSpy; - - const component = mountWithIntl( - - - - ); - - await new Promise((resolve) => setTimeout(resolve, 0)); - component.update(); - - (component.find('DashboardGridUi').instance() as any).synchronizeNow(); - - expect(startLoadingSpy).not.toHaveBeenCalled(); -}); - -test('areDataSourceParamsValid returns true for valid datasource params', async () => { - const { props, options } = prepare({ isDirectQuerySyncEnabled: true }); - - (extractIndexInfoFromDashboard as jest.Mock).mockResolvedValue({ - parts: { datasource: 'ds', database: 'db', index: 'idx' }, - mapping: { lastRefreshTime: 123456, refreshInterval: 30000 }, - mdsId: '', - }); - - const component = mountWithIntl( - - - - ); - - await new Promise((resolve) => setTimeout(resolve, 0)); - component.update(); - - const instance = component.find('DashboardGridUi').instance() as any; - expect(instance.areDataSourceParamsValid()).toBe(true); -}); - -test('areDataSourceParamsValid returns false for invalid datasource params', async () => { - const { props, options } = prepare({ isDirectQuerySyncEnabled: true }); - - (extractIndexInfoFromDashboard as jest.Mock).mockResolvedValue({ - parts: { datasource: 'ds', database: null, index: 'idx' }, - mapping: { lastRefreshTime: 123456, refreshInterval: 30000 }, - mdsId: '', - }); - - const component = mountWithIntl( - - - - ); - - await new Promise((resolve) => setTimeout(resolve, 0)); - component.update(); - - const instance = component.find('DashboardGridUi').instance() as any; - expect(instance.areDataSourceParamsValid()).toBe(false); -}); - -test('getQueryLanguage returns sql when feature is enabled and queryLang is not provided', async () => { - const { props, options } = prepare({ isDirectQuerySyncEnabled: true }); - - const component = mountWithIntl( - - - - ); - - await new Promise((resolve) => setTimeout(resolve, 0)); - component.update(); - - const instance = component.find('DashboardGridUi').instance() as any; - expect(instance.getQueryLanguage()).toBe('sql'); -}); - -test('getQueryLanguage returns empty string when feature is disabled and queryLang is not provided', async () => { - const { props, options } = prepare({ isDirectQuerySyncEnabled: false }); - - const component = mountWithIntl( - - - - ); - - await new Promise((resolve) => setTimeout(resolve, 0)); - component.update(); - - const instance = component.find('DashboardGridUi').instance() as any; - expect(instance.getQueryLanguage()).toBe(''); -}); - -test('getQueryLanguage returns provided queryLang when specified, regardless of feature flag', async () => { - const { props, options } = prepare({ isDirectQuerySyncEnabled: false, queryLang: 'ppl' }); - - const component = mountWithIntl( - - - - ); - - await new Promise((resolve) => setTimeout(resolve, 0)); - component.update(); - - const instance = component.find('DashboardGridUi').instance() as any; - expect(instance.getQueryLanguage()).toBe('ppl'); - - // Test with feature flag enabled - component.setProps({ isDirectQuerySyncEnabled: true }); - expect(instance.getQueryLanguage()).toBe('ppl'); + expect(component.find('DashboardDirectQuerySync').exists()).toBe(false); }); -test('synchronizeNow uses sql when feature is enabled and queryLang is not provided', async () => { - const { props, options } = prepare({ isDirectQuerySyncEnabled: true }); - - const mockRefreshQuery = 'REFRESH MATERIALIZED VIEW ds.db.idx'; - (extractIndexInfoFromDashboard as jest.Mock).mockResolvedValue({ - parts: { datasource: 'ds', database: 'db', index: 'idx' }, - mapping: { lastRefreshTime: 123456, refreshInterval: 30000 }, - mdsId: '', - }); - - (generateRefreshQuery as jest.Mock).mockReturnValue(mockRefreshQuery); - - const startLoadingSpy = jest.fn(); - props.startLoading = startLoadingSpy; +test('does not render sync UI when shouldRenderSyncUI is undefined', () => { + const { props, options } = prepare(); const component = mountWithIntl( @@ -560,51 +269,30 @@ test('synchronizeNow uses sql when feature is enabled and queryLang is not provi ); - await new Promise((resolve) => setTimeout(resolve, 0)); - component.update(); - - (component - .find('DashboardDirectQuerySync') - .props() as DashboardDirectQuerySyncProps).onSynchronize(); - - expect(startLoadingSpy).toHaveBeenCalledWith({ - query: mockRefreshQuery, - lang: 'sql', - datasource: 'ds', - }); + expect(component.find('DashboardDirectQuerySync').exists()).toBe(false); }); -test('synchronizeNow uses provided queryLang when specified', async () => { - const { props, options } = prepare({ isDirectQuerySyncEnabled: true, queryLang: 'ppl' }); - - const mockRefreshQuery = 'REFRESH MATERIALIZED VIEW ds.db.idx'; - (extractIndexInfoFromDashboard as jest.Mock).mockResolvedValue({ - parts: { datasource: 'ds', database: 'db', index: 'idx' }, - mapping: { lastRefreshTime: 123456, refreshInterval: 30000 }, - mdsId: '', +test('calls onSynchronize when sync button is clicked', () => { + const onSynchronizeMock = jest.fn(); + const { props, options } = prepare({ + shouldRenderSyncUI: true, + loadStatus: DirectQueryLoadingStatus.FRESH, + lastRefreshTime: 123456, + refreshInterval: 30000, + onSynchronize: onSynchronizeMock, }); - (generateRefreshQuery as jest.Mock).mockReturnValue(mockRefreshQuery); - - const startLoadingSpy = jest.fn(); - props.startLoading = startLoadingSpy; - const component = mountWithIntl( ); - await new Promise((resolve) => setTimeout(resolve, 0)); - component.update(); + // Simulate clicking the "Sync data" link + component + .find('[data-test-subj="dashboardDirectQuerySyncBar"]') + .find('EuiLink') + .simulate('click'); - (component - .find('DashboardDirectQuerySync') - .props() as DashboardDirectQuerySyncProps).onSynchronize(); - - expect(startLoadingSpy).toHaveBeenCalledWith({ - query: mockRefreshQuery, - lang: 'ppl', - datasource: 'ds', - }); + expect(onSynchronizeMock).toHaveBeenCalled(); }); From 0600777001c8a490b38503578769b3d71f6bcb1d Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Wed, 7 May 2025 16:09:20 -0700 Subject: [PATCH 77/86] josh - revert unnecessary changes in grid class Signed-off-by: Jialiang Liang --- .../public/application/embeddable/grid/dashboard_grid.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx index 6d2b54c287a9..4518c8000429 100644 --- a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx +++ b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx @@ -101,9 +101,9 @@ function ResponsiveGrid({ width={lastValidGridSize} className={classes} isDraggable={true} + isResizable={true} // There is a bug with d3 + firefox + elements using transforms. // See https://github.com/elastic/kibana/issues/16870 for more context. - isResizable={true} useCSSTransforms={false} margin={[MARGINS, MARGINS]} cols={DASHBOARD_GRID_COLUMN_COUNT} From faa5f90f3514c76058b10d62f4a1298f58e2aef8 Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Wed, 7 May 2025 16:14:04 -0700 Subject: [PATCH 78/86] enhance service and add tests Signed-off-by: Jialiang Liang --- .../direct_query_sync_services.test.ts | 544 ++++++++++++++++++ .../services/direct_query_sync_services.ts | 17 +- 2 files changed, 554 insertions(+), 7 deletions(-) create mode 100644 src/plugins/dashboard/public/application/embeddable/services/direct_query_sync_services.test.ts diff --git a/src/plugins/dashboard/public/application/embeddable/services/direct_query_sync_services.test.ts b/src/plugins/dashboard/public/application/embeddable/services/direct_query_sync_services.test.ts new file mode 100644 index 000000000000..0664d386d4db --- /dev/null +++ b/src/plugins/dashboard/public/application/embeddable/services/direct_query_sync_services.test.ts @@ -0,0 +1,544 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { DirectQuerySyncService } from './direct_query_sync_services'; +import { + extractIndexInfoFromDashboard, + generateRefreshQuery, +} from '../../utils/direct_query_sync/direct_query_sync'; +import { isDirectQuerySyncEnabledByUrl } from '../../utils/direct_query_sync/direct_query_sync_url_flag'; + +// Mock dependencies +jest.mock('../../utils/direct_query_sync/direct_query_sync'); +jest.mock('../../utils/direct_query_sync/direct_query_sync_url_flag'); + +const mockRefreshQuery = 'REFRESH MATERIALIZED VIEW `datasource`.`database`.`index`'; + +describe('DirectQuerySyncService', () => { + let startLoadingSpy: jest.Mock; + let setMdsIdSpy: jest.Mock; + + beforeEach(() => { + startLoadingSpy = jest.fn(); + setMdsIdSpy = jest.fn(); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + describe('isDirectQuerySyncEnabled', () => { + let service: DirectQuerySyncService; + + beforeEach(() => { + // Reset mocks for each test to ensure isolation + (isDirectQuerySyncEnabledByUrl as jest.Mock).mockReset(); + service = new DirectQuerySyncService({ + savedObjectsClient: {} as any, + http: {} as any, + startLoading: startLoadingSpy, + setMdsId: setMdsIdSpy, + isDirectQuerySyncEnabled: true, + queryLang: undefined, + }); + }); + + test('returns true when feature flag is enabled and no URL override', () => { + (isDirectQuerySyncEnabledByUrl as jest.Mock).mockReturnValue(undefined); + expect(service.isDirectQuerySyncEnabled()).toBe(true); + }); + + test('returns false when feature flag is disabled and no URL override', () => { + (isDirectQuerySyncEnabledByUrl as jest.Mock).mockReturnValue(undefined); + service = new DirectQuerySyncService({ + savedObjectsClient: {} as any, + http: {} as any, + startLoading: startLoadingSpy, + setMdsId: setMdsIdSpy, + isDirectQuerySyncEnabled: false, + queryLang: undefined, + }); + expect(service.isDirectQuerySyncEnabled()).toBe(false); + }); + + test('returns true when URL override is true', () => { + (isDirectQuerySyncEnabledByUrl as jest.Mock).mockReturnValue(true); + expect(service.isDirectQuerySyncEnabled()).toBe(true); + }); + + test('returns false when URL override is false', () => { + (isDirectQuerySyncEnabledByUrl as jest.Mock).mockReturnValue(false); + expect(service.isDirectQuerySyncEnabled()).toBe(false); + }); + }); + + describe('getQueryLanguage', () => { + let service: DirectQuerySyncService; + + beforeEach(() => { + (isDirectQuerySyncEnabledByUrl as jest.Mock).mockReset(); + service = new DirectQuerySyncService({ + savedObjectsClient: {} as any, + http: {} as any, + startLoading: startLoadingSpy, + setMdsId: setMdsIdSpy, + isDirectQuerySyncEnabled: true, + queryLang: undefined, + }); + }); + + test('returns sql when feature is enabled and queryLang is not provided', () => { + (isDirectQuerySyncEnabledByUrl as jest.Mock).mockReturnValue(true); + expect(service.getQueryLanguage()).toBe('sql'); + }); + + test('returns empty string when feature is disabled and queryLang is not provided', () => { + (isDirectQuerySyncEnabledByUrl as jest.Mock).mockReturnValue(false); + service = new DirectQuerySyncService({ + savedObjectsClient: {} as any, + http: {} as any, + startLoading: startLoadingSpy, + setMdsId: setMdsIdSpy, + isDirectQuerySyncEnabled: false, + queryLang: undefined, + }); + expect(service.getQueryLanguage()).toBe(''); + }); + + test('returns provided queryLang when specified, regardless of feature flag', () => { + // Feature flag disabled + (isDirectQuerySyncEnabledByUrl as jest.Mock).mockReturnValue(false); + service = new DirectQuerySyncService({ + savedObjectsClient: {} as any, + http: {} as any, + startLoading: startLoadingSpy, + setMdsId: setMdsIdSpy, + isDirectQuerySyncEnabled: false, + queryLang: 'ppl', + }); + expect(service.getQueryLanguage()).toBe('ppl'); + + // Feature flag enabled + (isDirectQuerySyncEnabledByUrl as jest.Mock).mockReturnValue(true); + service = new DirectQuerySyncService({ + savedObjectsClient: {} as any, + http: {} as any, + startLoading: startLoadingSpy, + setMdsId: setMdsIdSpy, + isDirectQuerySyncEnabled: true, + queryLang: 'ppl', + }); + expect(service.getQueryLanguage()).toBe('ppl'); + }); + + test('returns sql when queryLang is an empty string and feature is enabled', () => { + service = new DirectQuerySyncService({ + savedObjectsClient: {} as any, + http: {} as any, + startLoading: startLoadingSpy, + setMdsId: setMdsIdSpy, + isDirectQuerySyncEnabled: true, + queryLang: '', + }); + expect(service.getQueryLanguage()).toBe('sql'); + }); + + test('returns empty string when queryLang is an empty string and feature is disabled', () => { + (isDirectQuerySyncEnabledByUrl as jest.Mock).mockReturnValue(false); + service = new DirectQuerySyncService({ + savedObjectsClient: {} as any, + http: {} as any, + startLoading: startLoadingSpy, + setMdsId: setMdsIdSpy, + isDirectQuerySyncEnabled: false, + queryLang: '', + }); + expect(service.getQueryLanguage()).toBe(''); + }); + }); + + describe('areDataSourceParamsValid', () => { + let service: DirectQuerySyncService; + + beforeEach(() => { + (extractIndexInfoFromDashboard as jest.Mock).mockReset(); + (isDirectQuerySyncEnabledByUrl as jest.Mock).mockReturnValue(true); + service = new DirectQuerySyncService({ + savedObjectsClient: {} as any, + http: {} as any, + startLoading: startLoadingSpy, + setMdsId: setMdsIdSpy, + isDirectQuerySyncEnabled: true, + queryLang: undefined, + }); + // Use jest.spyOn to mock private methods + jest.spyOn(DirectQuerySyncService.prototype as any, 'areDataSourceParamsValid'); + }); + + afterEach(() => { + jest.restoreAllMocks(); + }); + + test('returns true for valid datasource params', async () => { + (extractIndexInfoFromDashboard as jest.Mock).mockResolvedValue({ + parts: { + datasource: 'datasource', + database: 'database', + index: 'index', + }, + mapping: { lastRefreshTime: 12345, refreshInterval: 30000 }, + mdsId: 'mds-1', + }); + + await service.collectAllPanelMetadata({} as any); + expect((service as any).areDataSourceParamsValid()).toBe(true); + }); + + test('returns false for invalid datasource params', async () => { + (extractIndexInfoFromDashboard as jest.Mock).mockResolvedValue(null); + + await service.collectAllPanelMetadata({} as any); + expect((service as any).areDataSourceParamsValid()).toBe(false); + }); + + test('returns false when only some params are missing', async () => { + (extractIndexInfoFromDashboard as jest.Mock).mockResolvedValue({ + parts: { + datasource: 'datasource', + database: null, // Missing database + index: 'index', + }, + mapping: { lastRefreshTime: 12345, refreshInterval: 30000 }, + mdsId: 'mds-1', + }); + + await service.collectAllPanelMetadata({} as any); + expect((service as any).areDataSourceParamsValid()).toBe(false); + }); + }); + + describe('synchronizeNow', () => { + let service: DirectQuerySyncService; + + beforeEach(() => { + (generateRefreshQuery as jest.Mock).mockReturnValue(mockRefreshQuery); + (extractIndexInfoFromDashboard as jest.Mock).mockReset(); + (isDirectQuerySyncEnabledByUrl as jest.Mock).mockReturnValue(true); + service = new DirectQuerySyncService({ + savedObjectsClient: {} as any, + http: {} as any, + startLoading: startLoadingSpy, + setMdsId: setMdsIdSpy, + isDirectQuerySyncEnabled: true, + queryLang: undefined, + }); + // Set up valid datasource params for all tests + (extractIndexInfoFromDashboard as jest.Mock).mockResolvedValue({ + parts: { + datasource: 'datasource', + database: 'database', + index: 'index', + }, + mapping: { lastRefreshTime: 12345, refreshInterval: 30000 }, + mdsId: 'mds-1', + }); + jest.spyOn(DirectQuerySyncService.prototype as any, 'areDataSourceParamsValid'); + }); + + afterEach(() => { + jest.restoreAllMocks(); + }); + + test('triggers REFRESH query generation and startLoading', async () => { + await service.collectAllPanelMetadata({} as any); + service.synchronizeNow(); + + expect(generateRefreshQuery).toHaveBeenCalledWith({ + datasource: 'datasource', + database: 'database', + index: 'index', + }); + expect(startLoadingSpy).toHaveBeenCalledWith({ + query: mockRefreshQuery, + lang: 'sql', + datasource: 'datasource', + }); + }); + + test('does nothing when feature flag is disabled', async () => { + (isDirectQuerySyncEnabledByUrl as jest.Mock).mockReturnValue(false); + service = new DirectQuerySyncService({ + savedObjectsClient: {} as any, + http: {} as any, + startLoading: startLoadingSpy, + setMdsId: setMdsIdSpy, + isDirectQuerySyncEnabled: false, + queryLang: undefined, + }); + + await service.collectAllPanelMetadata({} as any); + service.synchronizeNow(); + expect(startLoadingSpy).not.toHaveBeenCalled(); + }); + + test('does nothing when datasource params are invalid', async () => { + (extractIndexInfoFromDashboard as jest.Mock).mockResolvedValue(null); + + await service.collectAllPanelMetadata({} as any); + service.synchronizeNow(); + + expect(startLoadingSpy).not.toHaveBeenCalled(); + }); + + test('uses sql when feature is enabled and queryLang is not provided', async () => { + await service.collectAllPanelMetadata({} as any); + service.synchronizeNow(); + + expect(startLoadingSpy).toHaveBeenCalledWith({ + query: mockRefreshQuery, + lang: 'sql', + datasource: 'datasource', + }); + }); + + test('uses provided queryLang when specified', async () => { + service = new DirectQuerySyncService({ + savedObjectsClient: {} as any, + http: {} as any, + startLoading: startLoadingSpy, + setMdsId: setMdsIdSpy, + isDirectQuerySyncEnabled: true, + queryLang: 'ppl', + }); + + await service.collectAllPanelMetadata({} as any); + service.synchronizeNow(); + + expect(startLoadingSpy).toHaveBeenCalledWith({ + query: mockRefreshQuery, + lang: 'ppl', + datasource: 'datasource', + }); + }); + + test('uses sql when queryLang is empty and feature is enabled', async () => { + service = new DirectQuerySyncService({ + savedObjectsClient: {} as any, + http: {} as any, + startLoading: startLoadingSpy, + setMdsId: setMdsIdSpy, + isDirectQuerySyncEnabled: true, + queryLang: '', + }); + + await service.collectAllPanelMetadata({} as any); + service.synchronizeNow(); + + expect(startLoadingSpy).toHaveBeenCalledWith({ + query: mockRefreshQuery, + lang: 'sql', + datasource: 'datasource', + }); + }); + + test('uses empty string when queryLang is empty and feature is disabled', async () => { + (isDirectQuerySyncEnabledByUrl as jest.Mock).mockReturnValue(false); + service = new DirectQuerySyncService({ + savedObjectsClient: {} as any, + http: {} as any, + startLoading: startLoadingSpy, + setMdsId: setMdsIdSpy, + isDirectQuerySyncEnabled: false, + queryLang: '', + }); + + await service.collectAllPanelMetadata({} as any); + service.synchronizeNow(); + + expect(startLoadingSpy).not.toHaveBeenCalled(); + }); + }); + + describe('collectAllPanelMetadata', () => { + let service: DirectQuerySyncService; + + beforeEach(() => { + (extractIndexInfoFromDashboard as jest.Mock).mockReset(); + (isDirectQuerySyncEnabledByUrl as jest.Mock).mockReset(); + service = new DirectQuerySyncService({ + savedObjectsClient: {} as any, + http: {} as any, + startLoading: startLoadingSpy, + setMdsId: setMdsIdSpy, + isDirectQuerySyncEnabled: true, + queryLang: undefined, + }); + }); + + test('sets extracted props and MDS ID when metadata is available', async () => { + (extractIndexInfoFromDashboard as jest.Mock).mockResolvedValue({ + parts: { + datasource: 'datasource', + database: 'database', + index: 'index', + }, + mapping: { lastRefreshTime: 12345, refreshInterval: 30000 }, + mdsId: 'mds-1', + }); + + await service.collectAllPanelMetadata({} as any); + + expect(service.getExtractedProps()).toEqual({ + lastRefreshTime: 12345, + refreshInterval: 30000, + }); + expect(setMdsIdSpy).toHaveBeenCalledWith('mds-1'); + }); + + test('clears extracted props and MDS ID when metadata is not available', async () => { + (extractIndexInfoFromDashboard as jest.Mock).mockResolvedValue(null); + + await service.collectAllPanelMetadata({} as any); + + expect(service.getExtractedProps()).toBeNull(); + expect(setMdsIdSpy).toHaveBeenCalledWith(undefined); + }); + + test('does not collect metadata when feature is disabled', async () => { + (isDirectQuerySyncEnabledByUrl as jest.Mock).mockReturnValue(false); + service = new DirectQuerySyncService({ + savedObjectsClient: {} as any, + http: {} as any, + startLoading: startLoadingSpy, + setMdsId: setMdsIdSpy, + isDirectQuerySyncEnabled: false, + queryLang: undefined, + }); + + await service.collectAllPanelMetadata({} as any); + + expect(extractIndexInfoFromDashboard).not.toHaveBeenCalled(); + expect(service.getExtractedProps()).toBeNull(); + expect(setMdsIdSpy).not.toHaveBeenCalled(); + }); + + test('handles errors from extractIndexInfoFromDashboard', async () => { + (extractIndexInfoFromDashboard as jest.Mock).mockRejectedValue( + new Error('Failed to extract index info') + ); + + await service.collectAllPanelMetadata({} as any); + + expect(service.getExtractedProps()).toBeNull(); + expect(setMdsIdSpy).toHaveBeenCalledWith(undefined); + }); + + test('updates extracted props on subsequent calls with different metadata', async () => { + (extractIndexInfoFromDashboard as jest.Mock).mockResolvedValueOnce({ + parts: { + datasource: 'datasource1', + database: 'database1', + index: 'index1', + }, + mapping: { lastRefreshTime: 12345, refreshInterval: 30000 }, + mdsId: 'mds-1', + }); + + await service.collectAllPanelMetadata({} as any); + expect(service.getExtractedProps()).toEqual({ + lastRefreshTime: 12345, + refreshInterval: 30000, + }); + expect(setMdsIdSpy).toHaveBeenCalledWith('mds-1'); + + (extractIndexInfoFromDashboard as jest.Mock).mockResolvedValueOnce({ + parts: { + datasource: 'datasource2', + database: 'database2', + index: 'index2', + }, + mapping: { lastRefreshTime: 67890, refreshInterval: 60000 }, + mdsId: 'mds-2', + }); + + await service.collectAllPanelMetadata({} as any); + expect(service.getExtractedProps()).toEqual({ + lastRefreshTime: 67890, + refreshInterval: 60000, + }); + expect(setMdsIdSpy).toHaveBeenCalledWith('mds-2'); + }); + }); + + describe('updatePanels', () => { + let service: DirectQuerySyncService; + + beforeEach(() => { + (isDirectQuerySyncEnabledByUrl as jest.Mock).mockReset(); + service = new DirectQuerySyncService({ + savedObjectsClient: {} as any, + http: {} as any, + startLoading: startLoadingSpy, + setMdsId: setMdsIdSpy, + isDirectQuerySyncEnabled: true, + queryLang: undefined, + }); + }); + + test('calls collectAllPanelMetadata when feature is enabled', async () => { + (isDirectQuerySyncEnabledByUrl as jest.Mock).mockReturnValue(true); + (extractIndexInfoFromDashboard as jest.Mock).mockResolvedValue({ + parts: { + datasource: 'datasource', + database: 'database', + index: 'index', + }, + mapping: { lastRefreshTime: 12345, refreshInterval: 30000 }, + mdsId: 'mds-1', + }); + + const spy = jest.spyOn(service, 'collectAllPanelMetadata'); + await service.updatePanels({} as any); + + expect(spy).toHaveBeenCalledWith({}); + }); + + test('does not call collectAllPanelMetadata when feature is disabled', async () => { + (isDirectQuerySyncEnabledByUrl as jest.Mock).mockReturnValue(false); + service = new DirectQuerySyncService({ + savedObjectsClient: {} as any, + http: {} as any, + startLoading: startLoadingSpy, + setMdsId: setMdsIdSpy, + isDirectQuerySyncEnabled: false, + queryLang: undefined, + }); + + const spy = jest.spyOn(service, 'collectAllPanelMetadata'); + await service.updatePanels({} as any); + + expect(spy).not.toHaveBeenCalled(); + }); + }); + + describe('destroy', () => { + let service: DirectQuerySyncService; + + beforeEach(() => { + service = new DirectQuerySyncService({ + savedObjectsClient: {} as any, + http: {} as any, + startLoading: startLoadingSpy, + setMdsId: setMdsIdSpy, + isDirectQuerySyncEnabled: true, + queryLang: undefined, + }); + }); + + test('does not throw errors', () => { + expect(() => service.destroy()).not.toThrow(); + }); + }); +}); diff --git a/src/plugins/dashboard/public/application/embeddable/services/direct_query_sync_services.ts b/src/plugins/dashboard/public/application/embeddable/services/direct_query_sync_services.ts index c010de9c7a94..68beb2f08e6f 100644 --- a/src/plugins/dashboard/public/application/embeddable/services/direct_query_sync_services.ts +++ b/src/plugins/dashboard/public/application/embeddable/services/direct_query_sync_services.ts @@ -61,10 +61,10 @@ export class DirectQuerySyncService { /** * Determines the query language to use for direct query sync. - * Returns the provided queryLang if specified; otherwise, defaults to 'sql' if the feature is enabled. + * Returns the provided queryLang if specified and non-empty; otherwise, defaults to 'sql' if the feature is enabled. */ public getQueryLanguage(): string { - if (this.queryLang) { + if (this.queryLang !== undefined && this.queryLang !== '') { return this.queryLang; } return this.isDirectQuerySyncEnabled() ? 'sql' : ''; @@ -85,11 +85,14 @@ export class DirectQuerySyncService { public async collectAllPanelMetadata(panels: { [key: string]: DashboardPanelState }) { if (!this.isDirectQuerySyncEnabled()) return; - const indexInfo = await extractIndexInfoFromDashboard( - panels, - this.savedObjectsClient, - this.http - ); + let indexInfo; + try { + indexInfo = await extractIndexInfoFromDashboard(panels, this.savedObjectsClient, this.http); + } catch (error) { + // eslint-disable-next-line no-console + console.error('Caught error in collectAllPanelMetadata:', error); + indexInfo = null; + } if (indexInfo) { this.extractedDatasource = indexInfo.parts.datasource; From a339e707d99110fe87ed0e12f79bb5431a8de8ce Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Wed, 7 May 2025 16:18:01 -0700 Subject: [PATCH 79/86] josh - relocate sync service under direct query sync dir Signed-off-by: Jialiang Liang --- .../embeddable/viewport/dashboard_viewport.tsx | 2 +- .../direct_query_sync_services.test.ts | 7 ++----- .../direct_query_sync}/direct_query_sync_services.ts | 9 +++------ 3 files changed, 6 insertions(+), 12 deletions(-) rename src/plugins/dashboard/public/application/{embeddable/services => utils/direct_query_sync}/direct_query_sync_services.test.ts (98%) rename src/plugins/dashboard/public/application/{embeddable/services => utils/direct_query_sync}/direct_query_sync_services.ts (94%) diff --git a/src/plugins/dashboard/public/application/embeddable/viewport/dashboard_viewport.tsx b/src/plugins/dashboard/public/application/embeddable/viewport/dashboard_viewport.tsx index 4eeefd415448..699fd3e88de4 100644 --- a/src/plugins/dashboard/public/application/embeddable/viewport/dashboard_viewport.tsx +++ b/src/plugins/dashboard/public/application/embeddable/viewport/dashboard_viewport.tsx @@ -45,7 +45,7 @@ import { DirectQueryLoadingStatus, } from '../../../../../data_source_management/public'; import { useDirectQuery } from '../../../../../data_source_management/public'; -import { DirectQuerySyncService } from '../../../application/embeddable/services/direct_query_sync_services'; +import { DirectQuerySyncService } from '../../utils/direct_query_sync/direct_query_sync_services'; import { EMR_STATES } from '../../utils/direct_query_sync/direct_query_sync'; export interface DashboardViewportProps { diff --git a/src/plugins/dashboard/public/application/embeddable/services/direct_query_sync_services.test.ts b/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync_services.test.ts similarity index 98% rename from src/plugins/dashboard/public/application/embeddable/services/direct_query_sync_services.test.ts rename to src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync_services.test.ts index 0664d386d4db..803144d431d4 100644 --- a/src/plugins/dashboard/public/application/embeddable/services/direct_query_sync_services.test.ts +++ b/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync_services.test.ts @@ -4,11 +4,8 @@ */ import { DirectQuerySyncService } from './direct_query_sync_services'; -import { - extractIndexInfoFromDashboard, - generateRefreshQuery, -} from '../../utils/direct_query_sync/direct_query_sync'; -import { isDirectQuerySyncEnabledByUrl } from '../../utils/direct_query_sync/direct_query_sync_url_flag'; +import { extractIndexInfoFromDashboard, generateRefreshQuery } from './direct_query_sync'; +import { isDirectQuerySyncEnabledByUrl } from './direct_query_sync_url_flag'; // Mock dependencies jest.mock('../../utils/direct_query_sync/direct_query_sync'); diff --git a/src/plugins/dashboard/public/application/embeddable/services/direct_query_sync_services.ts b/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync_services.ts similarity index 94% rename from src/plugins/dashboard/public/application/embeddable/services/direct_query_sync_services.ts rename to src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync_services.ts index 68beb2f08e6f..c52e2b815c1d 100644 --- a/src/plugins/dashboard/public/application/embeddable/services/direct_query_sync_services.ts +++ b/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync_services.ts @@ -5,12 +5,9 @@ import { SavedObjectsClientContract, HttpStart } from 'src/core/public'; import { DirectQueryRequest } from '../../../../../data_source_management/public'; -import { - extractIndexInfoFromDashboard, - generateRefreshQuery, -} from '../../utils/direct_query_sync/direct_query_sync'; -import { isDirectQuerySyncEnabledByUrl } from '../../utils/direct_query_sync/direct_query_sync_url_flag'; -import { DashboardPanelState } from '../'; +import { extractIndexInfoFromDashboard, generateRefreshQuery } from './direct_query_sync'; +import { isDirectQuerySyncEnabledByUrl } from './direct_query_sync_url_flag'; +import { DashboardPanelState } from '../../embeddable'; interface DirectQuerySyncServiceProps { savedObjectsClient: SavedObjectsClientContract; From 222a364081e215b4e2277beded934c86b18350b1 Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Wed, 7 May 2025 16:29:19 -0700 Subject: [PATCH 80/86] josh - move render logic in grid to upper level and fix tests Signed-off-by: Jialiang Liang --- .../embeddable/grid/dashboard_grid.test.tsx | 83 ------------------- .../embeddable/grid/dashboard_grid.tsx | 35 ++------ .../viewport/dashboard_viewport.tsx | 19 +++-- 3 files changed, 19 insertions(+), 118 deletions(-) diff --git a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.test.tsx b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.test.tsx index 407c08508f96..4408708f961f 100644 --- a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.test.tsx +++ b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.test.tsx @@ -44,8 +44,6 @@ import { import { embeddablePluginMock } from '../../../../../embeddable/public/mocks'; import { createDashboardServicesMock } from '../../utils/mocks'; import { OpenSearchDashboardsContextProvider } from '../../../../../opensearch_dashboards_react/public'; -import { DashboardDirectQuerySyncProps } from './dashboard_direct_query_sync'; -import { DirectQueryLoadingStatus } from '../../../../../data_source_management/public'; let dashboardContainer: DashboardContainer | undefined; @@ -215,84 +213,3 @@ test('DashboardGrid unmount unsubscribes', (done) => { props.container.updateInput({ expandedPanelId: '1' }); }); - -test('renders sync UI when shouldRenderSyncUI is true', () => { - const onSynchronizeMock = jest.fn(); - const { props, options } = prepare({ - shouldRenderSyncUI: true, - loadStatus: DirectQueryLoadingStatus.FRESH, - lastRefreshTime: 123456, - refreshInterval: 30000, - onSynchronize: onSynchronizeMock, - }); - - const component = mountWithIntl( - - - - ); - - expect(component.find('DashboardDirectQuerySync').exists()).toBe(true); - const syncComponentProps = component - .find('DashboardDirectQuerySync') - .props() as DashboardDirectQuerySyncProps; - expect(syncComponentProps.loadStatus).toBe(DirectQueryLoadingStatus.FRESH); - expect(syncComponentProps.lastRefreshTime).toBe(123456); - expect(syncComponentProps.refreshInterval).toBe(30000); - expect(syncComponentProps.onSynchronize).toBe(onSynchronizeMock); -}); - -test('does not render sync UI when shouldRenderSyncUI is false', () => { - const { props, options } = prepare({ - shouldRenderSyncUI: false, - loadStatus: DirectQueryLoadingStatus.FRESH, - lastRefreshTime: 123456, - refreshInterval: 30000, - onSynchronize: jest.fn(), - }); - - const component = mountWithIntl( - - - - ); - - expect(component.find('DashboardDirectQuerySync').exists()).toBe(false); -}); - -test('does not render sync UI when shouldRenderSyncUI is undefined', () => { - const { props, options } = prepare(); - - const component = mountWithIntl( - - - - ); - - expect(component.find('DashboardDirectQuerySync').exists()).toBe(false); -}); - -test('calls onSynchronize when sync button is clicked', () => { - const onSynchronizeMock = jest.fn(); - const { props, options } = prepare({ - shouldRenderSyncUI: true, - loadStatus: DirectQueryLoadingStatus.FRESH, - lastRefreshTime: 123456, - refreshInterval: 30000, - onSynchronize: onSynchronizeMock, - }); - - const component = mountWithIntl( - - - - ); - - // Simulate clicking the "Sync data" link - component - .find('[data-test-subj="dashboardDirectQuerySyncBar"]') - .find('EuiLink') - .simulate('click'); - - expect(onSynchronizeMock).toHaveBeenCalled(); -}); diff --git a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx index 4518c8000429..24878af997a0 100644 --- a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx +++ b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx @@ -46,8 +46,6 @@ import { DashboardPanelState } from '../types'; import { withOpenSearchDashboards } from '../../../../../opensearch_dashboards_react/public'; import { DashboardContainerInput } from '../dashboard_container'; import { DashboardContainer, DashboardReactContextValue } from '../dashboard_container'; -import { DashboardDirectQuerySync } from './dashboard_direct_query_sync'; -import { DirectQueryLoadingStatus } from '../../../../../data_source_management/public'; let lastValidGridSize = 0; @@ -129,11 +127,6 @@ export interface DashboardGridProps extends ReactIntl.InjectedIntlProps { opensearchDashboards: DashboardReactContextValue; PanelComponent: EmbeddableStart['EmbeddablePanel']; container: DashboardContainer; - loadStatus?: DirectQueryLoadingStatus; - lastRefreshTime?: number; - refreshInterval?: number; - shouldRenderSyncUI?: boolean; - onSynchronize?: () => void; } interface State { @@ -308,25 +301,15 @@ class DashboardGridUi extends React.Component { const isViewMode = viewMode === ViewMode.VIEW; return ( - <> - {this.props.shouldRenderSyncUI && ( - - )} - - {this.renderPanels()} - - + + {this.renderPanels()} + ); } } diff --git a/src/plugins/dashboard/public/application/embeddable/viewport/dashboard_viewport.tsx b/src/plugins/dashboard/public/application/embeddable/viewport/dashboard_viewport.tsx index 699fd3e88de4..04f8a9c29d69 100644 --- a/src/plugins/dashboard/public/application/embeddable/viewport/dashboard_viewport.tsx +++ b/src/plugins/dashboard/public/application/embeddable/viewport/dashboard_viewport.tsx @@ -47,6 +47,7 @@ import { import { useDirectQuery } from '../../../../../data_source_management/public'; import { DirectQuerySyncService } from '../../utils/direct_query_sync/direct_query_sync_services'; import { EMR_STATES } from '../../utils/direct_query_sync/direct_query_sync'; +import { DashboardDirectQuerySync } from '../grid/dashboard_direct_query_sync'; export interface DashboardViewportProps { container: DashboardContainer; @@ -216,15 +217,15 @@ export class DashboardViewport extends React.Component )} - + {shouldRenderSyncUI && ( + + )} +
); } From 7f30454c38a90255409ca5c00c925cdc94a15962 Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Wed, 7 May 2025 16:43:07 -0700 Subject: [PATCH 81/86] josh - move the feature flag logic into service Signed-off-by: Jialiang Liang --- .../viewport/dashboard_viewport.tsx | 12 +- .../direct_query_sync_services.test.ts | 123 +++++++++++++++++- .../direct_query_sync_services.ts | 27 ++++ 3 files changed, 151 insertions(+), 11 deletions(-) diff --git a/src/plugins/dashboard/public/application/embeddable/viewport/dashboard_viewport.tsx b/src/plugins/dashboard/public/application/embeddable/viewport/dashboard_viewport.tsx index 04f8a9c29d69..75897d858e28 100644 --- a/src/plugins/dashboard/public/application/embeddable/viewport/dashboard_viewport.tsx +++ b/src/plugins/dashboard/public/application/embeddable/viewport/dashboard_viewport.tsx @@ -198,9 +198,8 @@ export class DashboardViewport extends React.Component )} {shouldRenderSyncUI && ( - + )}
diff --git a/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync_services.test.ts b/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync_services.test.ts index 803144d431d4..084acb389136 100644 --- a/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync_services.test.ts +++ b/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync_services.test.ts @@ -4,8 +4,11 @@ */ import { DirectQuerySyncService } from './direct_query_sync_services'; -import { extractIndexInfoFromDashboard, generateRefreshQuery } from './direct_query_sync'; -import { isDirectQuerySyncEnabledByUrl } from './direct_query_sync_url_flag'; +import { + extractIndexInfoFromDashboard, + generateRefreshQuery, +} from '../../utils/direct_query_sync/direct_query_sync'; +import { isDirectQuerySyncEnabledByUrl } from '../../utils/direct_query_sync/direct_query_sync_url_flag'; // Mock dependencies jest.mock('../../utils/direct_query_sync/direct_query_sync'); @@ -216,6 +219,122 @@ describe('DirectQuerySyncService', () => { }); }); + describe('shouldRenderSyncUI', () => { + let service: DirectQuerySyncService; + + beforeEach(() => { + (extractIndexInfoFromDashboard as jest.Mock).mockReset(); + (isDirectQuerySyncEnabledByUrl as jest.Mock).mockReset(); + service = new DirectQuerySyncService({ + savedObjectsClient: {} as any, + http: {} as any, + startLoading: startLoadingSpy, + setMdsId: setMdsIdSpy, + isDirectQuerySyncEnabled: true, + queryLang: undefined, + }); + }); + + test('returns true when feature is enabled and extracted props are available', async () => { + (isDirectQuerySyncEnabledByUrl as jest.Mock).mockReturnValue(true); + (extractIndexInfoFromDashboard as jest.Mock).mockResolvedValue({ + parts: { + datasource: 'datasource', + database: 'database', + index: 'index', + }, + mapping: { lastRefreshTime: 12345, refreshInterval: 30000 }, + mdsId: 'mds-1', + }); + + await service.collectAllPanelMetadata({} as any); + expect(service.shouldRenderSyncUI()).toBe(true); + }); + + test('returns false when feature is disabled', () => { + (isDirectQuerySyncEnabledByUrl as jest.Mock).mockReturnValue(false); + service = new DirectQuerySyncService({ + savedObjectsClient: {} as any, + http: {} as any, + startLoading: startLoadingSpy, + setMdsId: setMdsIdSpy, + isDirectQuerySyncEnabled: false, + queryLang: undefined, + }); + + expect(service.shouldRenderSyncUI()).toBe(false); + }); + + test('returns false when extracted props are not available', async () => { + (isDirectQuerySyncEnabledByUrl as jest.Mock).mockReturnValue(true); + (extractIndexInfoFromDashboard as jest.Mock).mockResolvedValue(null); + + await service.collectAllPanelMetadata({} as any); + expect(service.shouldRenderSyncUI()).toBe(false); + }); + }); + + describe('getSyncUIProps', () => { + let service: DirectQuerySyncService; + + beforeEach(() => { + (extractIndexInfoFromDashboard as jest.Mock).mockReset(); + (isDirectQuerySyncEnabledByUrl as jest.Mock).mockReturnValue(true); + // Ensure generateRefreshQuery is mocked for synchronizeNow calls + (generateRefreshQuery as jest.Mock).mockReturnValue(mockRefreshQuery); + service = new DirectQuerySyncService({ + savedObjectsClient: {} as any, + http: {} as any, + startLoading: startLoadingSpy, + setMdsId: setMdsIdSpy, + isDirectQuerySyncEnabled: true, + queryLang: undefined, + }); + }); + + test('returns correct props when extracted props are available', async () => { + (extractIndexInfoFromDashboard as jest.Mock).mockResolvedValue({ + parts: { + datasource: 'datasource', + database: 'database', + index: 'index', + }, + mapping: { lastRefreshTime: 12345, refreshInterval: 30000 }, + mdsId: 'mds-1', + }); + + await service.collectAllPanelMetadata({} as any); + const props = service.getSyncUIProps(); + + expect(props).toEqual({ + lastRefreshTime: 12345, + refreshInterval: 30000, + onSynchronize: expect.any(Function), + }); + + // Verify onSynchronize calls synchronizeNow + props.onSynchronize(); + expect(startLoadingSpy).toHaveBeenCalledWith({ + query: mockRefreshQuery, + lang: 'sql', + datasource: 'datasource', + }); + }); + + test('returns undefined props when extracted props are not available', async () => { + (extractIndexInfoFromDashboard as jest.Mock).mockResolvedValue(null); + + await service.collectAllPanelMetadata({} as any); + const props = service.getSyncUIProps(); + + expect(props).toEqual({ + lastRefreshTime: undefined, + refreshInterval: undefined, + onSynchronize: expect.any(Function), + }); + }); + }); + describe('synchronizeNow', () => { let service: DirectQuerySyncService; diff --git a/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync_services.ts b/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync_services.ts index c52e2b815c1d..e7152f20dcb1 100644 --- a/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync_services.ts +++ b/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync_services.ts @@ -23,6 +23,12 @@ interface DirectQuerySyncState { panelMetadata: Array<{ panelId: string; savedObjectId: string; type: string }>; } +export interface DirectQuerySyncUIProps { + lastRefreshTime?: number; + refreshInterval?: number; + onSynchronize: () => void; +} + export class DirectQuerySyncService { private savedObjectsClient: SavedObjectsClientContract; private http: HttpStart; @@ -138,6 +144,27 @@ export class DirectQuerySyncService { return this.state.extractedProps; } + /** + * Determines if the direct query sync UI should be rendered. + * Returns true if the feature is enabled and extracted props are available. + */ + public shouldRenderSyncUI(): boolean { + const extractedProps = this.getExtractedProps(); + return this.isDirectQuerySyncEnabled() && extractedProps !== null; + } + + /** + * Returns the props needed to render the direct query sync UI (excluding loadStatus). + */ + public getSyncUIProps(): DirectQuerySyncUIProps { + const extractedProps = this.getExtractedProps(); + return { + lastRefreshTime: extractedProps?.lastRefreshTime, + refreshInterval: extractedProps?.refreshInterval, + onSynchronize: () => this.synchronizeNow(), + }; + } + /** * Updates the service with new panels, triggering metadata collection if needed. */ From 614edf74cc2079c810925bba8e5dfd326bf56ea4 Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Wed, 7 May 2025 16:51:31 -0700 Subject: [PATCH 82/86] josh - switch to add dsm as required bundle instead of plugin Signed-off-by: Jialiang Liang --- src/plugins/dashboard/opensearch_dashboards.json | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/plugins/dashboard/opensearch_dashboards.json b/src/plugins/dashboard/opensearch_dashboards.json index e00dcbd4255c..fc68b4c3ca2b 100644 --- a/src/plugins/dashboard/opensearch_dashboards.json +++ b/src/plugins/dashboard/opensearch_dashboards.json @@ -9,11 +9,10 @@ "urlForwarding", "navigation", "uiActions", - "savedObjects", - "dataSourceManagement" + "savedObjects" ], "optionalPlugins": ["home", "share", "usageCollection"], "server": true, "ui": true, - "requiredBundles": ["opensearchDashboardsUtils", "opensearchDashboardsReact", "home"] + "requiredBundles": ["opensearchDashboardsUtils", "opensearchDashboardsReact", "home", "dataSourceManagement"] } From 4054f9da9928940f6ab775ec7274c9a65b5b992e Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Wed, 7 May 2025 16:56:25 -0700 Subject: [PATCH 83/86] revert grid back Signed-off-by: Jialiang Liang --- .../embeddable/grid/dashboard_grid.test.tsx | 20 +++---------------- .../embeddable/grid/dashboard_grid.tsx | 1 - 2 files changed, 3 insertions(+), 18 deletions(-) diff --git a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.test.tsx b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.test.tsx index 4408708f961f..f143b18b7c48 100644 --- a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.test.tsx +++ b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.test.tsx @@ -42,14 +42,12 @@ import { ContactCardEmbeddableFactory, } from '../../../../../embeddable/public/lib/test_samples'; import { embeddablePluginMock } from '../../../../../embeddable/public/mocks'; -import { createDashboardServicesMock } from '../../utils/mocks'; import { OpenSearchDashboardsContextProvider } from '../../../../../opensearch_dashboards_react/public'; let dashboardContainer: DashboardContainer | undefined; function prepare(props?: Partial) { const { setup, doStart } = embeddablePluginMock.createInstance(); - setup.registerEmbeddableFactory( CONTACT_CARD_EMBEDDABLE, new ContactCardEmbeddableFactory((() => null) as any, {} as any) @@ -62,18 +60,15 @@ function prepare(props?: Partial) { '1': { gridData: { x: 0, y: 0, w: 6, h: 6, i: '1' }, type: CONTACT_CARD_EMBEDDABLE, - explicitInput: { id: '1', savedObjectId: 'vis-1' }, + explicitInput: { id: '1' }, }, '2': { gridData: { x: 6, y: 6, w: 6, h: 6, i: '2' }, type: CONTACT_CARD_EMBEDDABLE, - explicitInput: { id: '2', savedObjectId: 'vis-2' }, + explicitInput: { id: '2' }, }, }, }); - - const services = createDashboardServicesMock(); - const options: DashboardContainerOptions = { application: {} as any, embeddable: { @@ -93,21 +88,12 @@ function prepare(props?: Partial) { uiActions: { getTriggerCompatibleActions: (() => []) as any, } as any, - savedObjectsClient: {} as any, - http: {} as any, - dashboardFeatureFlagConfig: { - directQueryConnectionSync: true, - } as any, }; - dashboardContainer = new DashboardContainer(initialInput, options); - const defaultTestProps: DashboardGridProps = { container: dashboardContainer, PanelComponent: () =>
, - opensearchDashboards: { - services, - }, + opensearchDashboards: null as any, intl: null as any, }; diff --git a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx index 24878af997a0..374e20e715d4 100644 --- a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx +++ b/src/plugins/dashboard/public/application/embeddable/grid/dashboard_grid.tsx @@ -299,7 +299,6 @@ class DashboardGridUi extends React.Component { const { viewMode } = this.state; const isViewMode = viewMode === ViewMode.VIEW; - return ( Date: Wed, 7 May 2025 19:46:50 -0700 Subject: [PATCH 84/86] fix ft Signed-off-by: Jialiang Liang --- .../viewport/dashboard_viewport.tsx | 32 ++++++++++--------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/src/plugins/dashboard/public/application/embeddable/viewport/dashboard_viewport.tsx b/src/plugins/dashboard/public/application/embeddable/viewport/dashboard_viewport.tsx index 75897d858e28..73ca4c1ae73f 100644 --- a/src/plugins/dashboard/public/application/embeddable/viewport/dashboard_viewport.tsx +++ b/src/plugins/dashboard/public/application/embeddable/viewport/dashboard_viewport.tsx @@ -202,24 +202,26 @@ export class DashboardViewport extends React.Component - {isFullScreenMode && ( - - )} +
{shouldRenderSyncUI && ( )} - +
+ {isFullScreenMode && ( + + )} + +
); } From f6682de0c90ccd634227eaa1d778311a21b38e88 Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Thu, 8 May 2025 01:26:53 -0700 Subject: [PATCH 85/86] relocate the sync dir Signed-off-by: Jialiang Liang --- .../_dashboard_direct_query_sync.scss | 0 .../dashboard_direct_query_sync.test.tsx | 0 .../dashboard_direct_query_sync.tsx | 2 +- .../direct_query_sync/direct_query_sync.test.ts | 0 .../direct_query_sync/direct_query_sync.ts | 0 .../direct_query_sync_services.test.ts | 11 ++++------- .../direct_query_sync/direct_query_sync_services.ts | 2 +- .../direct_query_sync_url_flag.test.ts | 0 .../direct_query_sync/direct_query_sync_url_flag.ts | 0 .../embeddable/viewport/dashboard_viewport.tsx | 6 +++--- 10 files changed, 9 insertions(+), 12 deletions(-) rename src/plugins/dashboard/public/application/embeddable/{grid => direct_query_sync}/_dashboard_direct_query_sync.scss (100%) rename src/plugins/dashboard/public/application/embeddable/{grid => direct_query_sync}/dashboard_direct_query_sync.test.tsx (100%) rename src/plugins/dashboard/public/application/embeddable/{grid => direct_query_sync}/dashboard_direct_query_sync.tsx (95%) rename src/plugins/dashboard/public/application/{utils => embeddable}/direct_query_sync/direct_query_sync.test.ts (100%) rename src/plugins/dashboard/public/application/{utils => embeddable}/direct_query_sync/direct_query_sync.ts (100%) rename src/plugins/dashboard/public/application/{utils => embeddable}/direct_query_sync/direct_query_sync_services.test.ts (98%) rename src/plugins/dashboard/public/application/{utils => embeddable}/direct_query_sync/direct_query_sync_services.ts (99%) rename src/plugins/dashboard/public/application/{utils => embeddable}/direct_query_sync/direct_query_sync_url_flag.test.ts (100%) rename src/plugins/dashboard/public/application/{utils => embeddable}/direct_query_sync/direct_query_sync_url_flag.ts (100%) diff --git a/src/plugins/dashboard/public/application/embeddable/grid/_dashboard_direct_query_sync.scss b/src/plugins/dashboard/public/application/embeddable/direct_query_sync/_dashboard_direct_query_sync.scss similarity index 100% rename from src/plugins/dashboard/public/application/embeddable/grid/_dashboard_direct_query_sync.scss rename to src/plugins/dashboard/public/application/embeddable/direct_query_sync/_dashboard_direct_query_sync.scss diff --git a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_direct_query_sync.test.tsx b/src/plugins/dashboard/public/application/embeddable/direct_query_sync/dashboard_direct_query_sync.test.tsx similarity index 100% rename from src/plugins/dashboard/public/application/embeddable/grid/dashboard_direct_query_sync.test.tsx rename to src/plugins/dashboard/public/application/embeddable/direct_query_sync/dashboard_direct_query_sync.test.tsx diff --git a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_direct_query_sync.tsx b/src/plugins/dashboard/public/application/embeddable/direct_query_sync/dashboard_direct_query_sync.tsx similarity index 95% rename from src/plugins/dashboard/public/application/embeddable/grid/dashboard_direct_query_sync.tsx rename to src/plugins/dashboard/public/application/embeddable/direct_query_sync/dashboard_direct_query_sync.tsx index c349b26a6666..59dd0ce8626d 100644 --- a/src/plugins/dashboard/public/application/embeddable/grid/dashboard_direct_query_sync.tsx +++ b/src/plugins/dashboard/public/application/embeddable/direct_query_sync/dashboard_direct_query_sync.tsx @@ -7,7 +7,7 @@ import React from 'react'; import { EuiCallOut, EuiLink, EuiLoadingSpinner, EuiText } from '@elastic/eui'; import { i18n } from '@osd/i18n'; import { DirectQueryLoadingStatus } from '../../../../../data_source_management/public'; -import { EMR_STATES, intervalAsMinutes } from '../../utils/direct_query_sync/direct_query_sync'; +import { EMR_STATES, intervalAsMinutes } from './direct_query_sync'; import './_dashboard_direct_query_sync.scss'; export interface DashboardDirectQuerySyncProps { diff --git a/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.test.ts b/src/plugins/dashboard/public/application/embeddable/direct_query_sync/direct_query_sync.test.ts similarity index 100% rename from src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.test.ts rename to src/plugins/dashboard/public/application/embeddable/direct_query_sync/direct_query_sync.test.ts diff --git a/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.ts b/src/plugins/dashboard/public/application/embeddable/direct_query_sync/direct_query_sync.ts similarity index 100% rename from src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync.ts rename to src/plugins/dashboard/public/application/embeddable/direct_query_sync/direct_query_sync.ts diff --git a/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync_services.test.ts b/src/plugins/dashboard/public/application/embeddable/direct_query_sync/direct_query_sync_services.test.ts similarity index 98% rename from src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync_services.test.ts rename to src/plugins/dashboard/public/application/embeddable/direct_query_sync/direct_query_sync_services.test.ts index 084acb389136..b26a662798cd 100644 --- a/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync_services.test.ts +++ b/src/plugins/dashboard/public/application/embeddable/direct_query_sync/direct_query_sync_services.test.ts @@ -4,15 +4,12 @@ */ import { DirectQuerySyncService } from './direct_query_sync_services'; -import { - extractIndexInfoFromDashboard, - generateRefreshQuery, -} from '../../utils/direct_query_sync/direct_query_sync'; -import { isDirectQuerySyncEnabledByUrl } from '../../utils/direct_query_sync/direct_query_sync_url_flag'; +import { extractIndexInfoFromDashboard, generateRefreshQuery } from './direct_query_sync'; +import { isDirectQuerySyncEnabledByUrl } from './direct_query_sync_url_flag'; // Mock dependencies -jest.mock('../../utils/direct_query_sync/direct_query_sync'); -jest.mock('../../utils/direct_query_sync/direct_query_sync_url_flag'); +jest.mock('./direct_query_sync'); +jest.mock('./direct_query_sync_url_flag'); const mockRefreshQuery = 'REFRESH MATERIALIZED VIEW `datasource`.`database`.`index`'; diff --git a/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync_services.ts b/src/plugins/dashboard/public/application/embeddable/direct_query_sync/direct_query_sync_services.ts similarity index 99% rename from src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync_services.ts rename to src/plugins/dashboard/public/application/embeddable/direct_query_sync/direct_query_sync_services.ts index e7152f20dcb1..4ae810f306eb 100644 --- a/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync_services.ts +++ b/src/plugins/dashboard/public/application/embeddable/direct_query_sync/direct_query_sync_services.ts @@ -7,7 +7,7 @@ import { SavedObjectsClientContract, HttpStart } from 'src/core/public'; import { DirectQueryRequest } from '../../../../../data_source_management/public'; import { extractIndexInfoFromDashboard, generateRefreshQuery } from './direct_query_sync'; import { isDirectQuerySyncEnabledByUrl } from './direct_query_sync_url_flag'; -import { DashboardPanelState } from '../../embeddable'; +import { DashboardPanelState } from '..'; interface DirectQuerySyncServiceProps { savedObjectsClient: SavedObjectsClientContract; diff --git a/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync_url_flag.test.ts b/src/plugins/dashboard/public/application/embeddable/direct_query_sync/direct_query_sync_url_flag.test.ts similarity index 100% rename from src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync_url_flag.test.ts rename to src/plugins/dashboard/public/application/embeddable/direct_query_sync/direct_query_sync_url_flag.test.ts diff --git a/src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync_url_flag.ts b/src/plugins/dashboard/public/application/embeddable/direct_query_sync/direct_query_sync_url_flag.ts similarity index 100% rename from src/plugins/dashboard/public/application/utils/direct_query_sync/direct_query_sync_url_flag.ts rename to src/plugins/dashboard/public/application/embeddable/direct_query_sync/direct_query_sync_url_flag.ts diff --git a/src/plugins/dashboard/public/application/embeddable/viewport/dashboard_viewport.tsx b/src/plugins/dashboard/public/application/embeddable/viewport/dashboard_viewport.tsx index 73ca4c1ae73f..8d27c6314c6b 100644 --- a/src/plugins/dashboard/public/application/embeddable/viewport/dashboard_viewport.tsx +++ b/src/plugins/dashboard/public/application/embeddable/viewport/dashboard_viewport.tsx @@ -45,9 +45,9 @@ import { DirectQueryLoadingStatus, } from '../../../../../data_source_management/public'; import { useDirectQuery } from '../../../../../data_source_management/public'; -import { DirectQuerySyncService } from '../../utils/direct_query_sync/direct_query_sync_services'; -import { EMR_STATES } from '../../utils/direct_query_sync/direct_query_sync'; -import { DashboardDirectQuerySync } from '../grid/dashboard_direct_query_sync'; +import { DirectQuerySyncService } from '../direct_query_sync/direct_query_sync_services'; +import { EMR_STATES } from '../direct_query_sync/direct_query_sync'; +import { DashboardDirectQuerySync } from '../direct_query_sync/dashboard_direct_query_sync'; export interface DashboardViewportProps { container: DashboardContainer; From edd48d22ea45ba003eb77a71f0185b51d660d241 Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Fri, 9 May 2025 09:34:18 -0700 Subject: [PATCH 86/86] WRAP VIEWPORT IN DSHBOARD CAONTAINER Signed-off-by: Jialiang Liang --- .../embeddable/dashboard_container.tsx | 82 +++++-- .../dashboard_direct_query_sync.tsx | 75 +++--- .../direct_query_sync_context.tsx | 229 ++++++++++++++++++ .../viewport/dashboard_viewport.tsx | 129 ++-------- 4 files changed, 348 insertions(+), 167 deletions(-) create mode 100644 src/plugins/dashboard/public/application/embeddable/direct_query_sync/direct_query_sync_context.tsx diff --git a/src/plugins/dashboard/public/application/embeddable/dashboard_container.tsx b/src/plugins/dashboard/public/application/embeddable/dashboard_container.tsx index f1b6f3ea7e2a..5aa579b58437 100644 --- a/src/plugins/dashboard/public/application/embeddable/dashboard_container.tsx +++ b/src/plugins/dashboard/public/application/embeddable/dashboard_container.tsx @@ -53,12 +53,13 @@ import { UiActionsStart } from '../../../../ui_actions/public'; import { DASHBOARD_CONTAINER_TYPE } from './dashboard_constants'; import { createPanelState } from './panel'; import { DashboardPanelState } from './types'; -import { DashboardViewportWithQuery } from './viewport/dashboard_viewport'; +import { DashboardViewport } from './viewport/dashboard_viewport'; import { OpenSearchDashboardsContextProvider, OpenSearchDashboardsReactContext, OpenSearchDashboardsReactContextValue, } from '../../../../opensearch_dashboards_react/public'; +import { DirectQuerySyncProvider } from './direct_query_sync/direct_query_sync_context'; import { PLACEHOLDER_EMBEDDABLE } from './placeholder'; import { PanelPlacementMethod, IPanelPlacementArgs } from './panel/dashboard_panel_placement'; import { DashboardFeatureFlagConfig } from '../../plugin'; @@ -125,6 +126,9 @@ export class DashboardContainer extends Container @@ -194,10 +210,6 @@ export class DashboardContainer extends Container, newPanelState: Partial ) { - // TODO: In the current infrastructure, embeddables in a container do not react properly to - // changes. Removing the existing embeddable, and adding a new one is a temporary workaround - // until the container logic is fixed. - const finalPanels = { ...this.input.panels }; delete finalPanels[previousPanelState.explicitInput.id]; const newPanelId = newPanelState.explicitInput?.id ? newPanelState.explicitInput.id : uuid.v4(); @@ -239,26 +251,48 @@ export class DashboardContainer extends Container - - - - , - dom + this.renderCount++; + console.log( + `[DashboardContainer] Rendering (instanceId: ${this.instanceId}, count: ${this.renderCount}, hasRendered: ${this.hasRendered})` ); + + const DashboardContainerWrapper = () => { + return ( + + + + + + + + ); + }; + + if (!this.hasRendered) { + console.log( + `[DashboardContainer] Initial render with ReactDOM.render (instanceId: ${this.instanceId})` + ); + ReactDOM.render(, dom); + this.hasRendered = true; + } else { + console.log( + `[DashboardContainer] Skipping ReactDOM.render, already rendered (instanceId: ${this.instanceId})` + ); + } } protected getInheritedInput(id: string): InheritedChildInput { diff --git a/src/plugins/dashboard/public/application/embeddable/direct_query_sync/dashboard_direct_query_sync.tsx b/src/plugins/dashboard/public/application/embeddable/direct_query_sync/dashboard_direct_query_sync.tsx index 59dd0ce8626d..763be1329301 100644 --- a/src/plugins/dashboard/public/application/embeddable/direct_query_sync/dashboard_direct_query_sync.tsx +++ b/src/plugins/dashboard/public/application/embeddable/direct_query_sync/dashboard_direct_query_sync.tsx @@ -15,6 +15,7 @@ export interface DashboardDirectQuerySyncProps { lastRefreshTime?: number; refreshInterval?: number; onSynchronize: () => void; + className?: string; } export const DashboardDirectQuerySync: React.FC = ({ @@ -22,44 +23,60 @@ export const DashboardDirectQuerySync: React.FC = lastRefreshTime, refreshInterval, onSynchronize, + className, }) => { + console.log('DashboardDirectQuerySync: Rendering with props:', { + loadStatus, + lastRefreshTime, + refreshInterval, + className, + onSynchronize: '[Function]', // Simplified since onSynchronize is guaranteed to be a function + }); + // If loadStatus is undefined, default to a non-terminal state to avoid errors const state = loadStatus ? EMR_STATES.get(loadStatus)! : { ord: 0, terminal: false }; + console.log('DashboardDirectQuerySync: Computed state=', state); return ( -
+
{state.terminal ? ( - - {i18n.translate('dashboard.directQuerySync.dataScheduledToSync', { - defaultMessage: 'Data scheduled to sync every {interval}. Last sync: {lastSyncTime}.', - values: { - interval: refreshInterval ? intervalAsMinutes(1000 * refreshInterval) : '--', - lastSyncTime: lastRefreshTime - ? `${new Date(lastRefreshTime).toLocaleString()} (${intervalAsMinutes( - new Date().getTime() - lastRefreshTime - )} ago)` - : '--', - }, - })} - - - {i18n.translate('dashboard.directQuerySync.syncDataLink', { - defaultMessage: 'Sync data', + <> + {console.log('DashboardDirectQuerySync: Rendering terminal state UI')} + + {i18n.translate('dashboard.directQuerySync.dataScheduledToSync', { + defaultMessage: 'Data scheduled to sync every {interval}. Last sync: {lastSyncTime}.', + values: { + interval: refreshInterval ? intervalAsMinutes(1000 * refreshInterval) : '--', + lastSyncTime: lastRefreshTime + ? `${new Date(lastRefreshTime).toLocaleString()} (${intervalAsMinutes( + new Date().getTime() - lastRefreshTime + )} ago)` + : '--', + }, })} - - + + + {i18n.translate('dashboard.directQuerySync.syncDataLink', { + defaultMessage: 'Sync data', + })} + + + ) : ( - - + <> + {console.log('DashboardDirectQuerySync: Rendering in-progress state UI')} + + - {i18n.translate('dashboard.directQuerySync.dataSyncInProgress', { - defaultMessage: - 'Data sync is in progress ({progress}% complete). The dashboard will reload on completion.', - values: { - progress: state.ord, - }, - })} - + {i18n.translate('dashboard.directQuerySync.dataSyncInProgress', { + defaultMessage: + 'Data sync is in progress ({progress}% complete). The dashboard will reload on completion.', + values: { + progress: state.ord, + }, + })} + + )}
); diff --git a/src/plugins/dashboard/public/application/embeddable/direct_query_sync/direct_query_sync_context.tsx b/src/plugins/dashboard/public/application/embeddable/direct_query_sync/direct_query_sync_context.tsx new file mode 100644 index 000000000000..fa8c0704d77b --- /dev/null +++ b/src/plugins/dashboard/public/application/embeddable/direct_query_sync/direct_query_sync_context.tsx @@ -0,0 +1,229 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import React, { createContext, useState, useEffect, useContext, useMemo, Children } from 'react'; +import { + SavedObjectsClientContract, + HttpStart, + NotificationsStart, +} from 'opensearch-dashboards/public'; +import { Subscription } from 'rxjs'; +import isEqual from 'lodash/isEqual'; +import { + DirectQueryRequest, + DirectQueryLoadingStatus, +} from '../../../../../data_source_management/public'; +import { useDirectQuery } from '../../../../../data_source_management/public'; +import { DirectQuerySyncService, DirectQuerySyncUIProps } from './direct_query_sync_services'; +import { isDirectQuerySyncEnabledByUrl } from './direct_query_sync_url_flag'; +import { DashboardPanelState } from '../types'; +import { DashboardContainer } from '../dashboard_container'; +import { EMR_STATES } from './direct_query_sync'; +import { DashboardDirectQuerySync } from './dashboard_direct_query_sync'; + +interface DirectQuerySyncContextProps { + shouldRenderSyncUI: boolean; + syncUIProps: DirectQuerySyncUIProps; + loadStatus: DirectQueryLoadingStatus | undefined; + mdsId: string | undefined; +} + +export const DirectQuerySyncContext = createContext({ + shouldRenderSyncUI: false, + syncUIProps: { + lastRefreshTime: undefined, + refreshInterval: undefined, + onSynchronize: () => {}, + }, + loadStatus: undefined, + mdsId: undefined, +}); + +interface DirectQuerySyncProviderProps { + savedObjectsClient: SavedObjectsClientContract; + http: HttpStart; + notifications: NotificationsStart; + isDirectQuerySyncEnabled: boolean; + queryLang?: string; + container: DashboardContainer; + children: React.ReactNode; +} + +export const DirectQuerySyncProvider: React.FC = ({ + savedObjectsClient, + http, + notifications, + isDirectQuerySyncEnabled, + queryLang, + container, + children, +}) => { + console.log('DirectQuerySyncProvider: Mounting'); + console.log('DirectQuerySyncProvider: isDirectQuerySyncEnabled prop=', isDirectQuerySyncEnabled); + + // Check URL override + const urlOverride = isDirectQuerySyncEnabledByUrl(); + console.log('DirectQuerySyncProvider: URL override=', urlOverride); + + // Align feature enable/disable logic with syncService.isDirectQuerySyncEnabled() + const isFeatureEnabled = urlOverride !== undefined ? urlOverride : isDirectQuerySyncEnabled; + console.log('DirectQuerySyncProvider: isFeatureEnabled=', isFeatureEnabled); + + // Log the children with more detail + const childrenArray = Children.toArray(children); + console.log( + 'DirectQuerySyncProvider: children=', + childrenArray.map((child) => { + if (React.isValidElement(child)) { + return { + type: child.type?.displayName || child.type?.name || 'Unknown', + props: { + ...child.props, + // Avoid logging complex props that might cause circular reference issues + container: child.props.container ? '[DashboardContainer]' : undefined, + PanelComponent: child.props.PanelComponent ? '[Function]' : undefined, + onSynchronize: child.props.onSynchronize ? '[Function]' : undefined, + }, + }; + } + return child; + }) + ); + + const [mdsId, setMdsId] = useState(undefined); + const [panels, setPanels] = useState<{ [key: string]: DashboardPanelState }>( + container.getInput().panels + ); + + const { startLoading, loadStatus, pollingResult } = useDirectQuery(http, notifications, mdsId); + + const syncService = useMemo(() => { + console.log('Creating new DirectQuerySyncService'); + return new DirectQuerySyncService({ + savedObjectsClient, + http, + startLoading: (payload: DirectQueryRequest) => { + startLoading(payload); + }, + setMdsId: (newMdsId?: string) => { + setMdsId(newMdsId); + }, + isDirectQuerySyncEnabled, + queryLang, + }); + }, [savedObjectsClient, http, startLoading, setMdsId, isDirectQuerySyncEnabled, queryLang]); + + // Effect for subscribing to container input changes + useEffect(() => { + console.log('Subscription effect: container reference=', container); + const initialPanels = container.getInput().panels; + console.log('Subscription effect: Initial panels on mount=', Object.keys(initialPanels)); + setPanels(initialPanels); // Ensure initial panels are set + + const subscription: Subscription = container.getInput$().subscribe(() => { + const { panels: newPanels } = container.getInput(); + console.log('Subscription effect: newPanels=', Object.keys(newPanels)); + setPanels((prevPanels) => { + console.log('Subscription effect: prevPanels=', Object.keys(prevPanels)); + if (isEqual(prevPanels, newPanels)) { + console.log('Subscription effect: Panels unchanged, skipping update'); + return prevPanels; + } + console.log('Panels changed:', Object.keys(newPanels)); + return newPanels; + }); + }); + + return () => { + subscription.unsubscribe(); + }; + }, [container]); + + // Effect for fetching metadata when panels change + useEffect(() => { + console.log('Fetch effect: syncService reference=', syncService); + console.log('Fetch effect: panels=', Object.keys(panels)); + if (syncService.isDirectQuerySyncEnabled()) { + console.log('Fetching metadata for panels:', Object.keys(panels)); + syncService.collectAllPanelMetadata(panels); + } else { + console.log('Not fetching metadata: syncService.isDirectQuerySyncEnabled()=false'); + } + }, [syncService, panels]); + + // Effect for handling EMR state polling and page reload + useEffect(() => { + if (!loadStatus) return; + + console.log('Polling effect: loadStatus=', loadStatus); + const emrState = EMR_STATES.get(loadStatus as string); + + if ( + emrState?.terminal && + loadStatus !== DirectQueryLoadingStatus.FRESH && + loadStatus !== DirectQueryLoadingStatus.FAILED && + loadStatus !== DirectQueryLoadingStatus.CANCELLED + ) { + console.log('Reloading page due to EMR state:', emrState); + window.location.reload(); + } + }, [loadStatus]); + + // Cleanup on unmount + useEffect(() => { + return () => { + console.log('DirectQuerySyncProvider: Unmounting'); + syncService.destroy(); + }; + }, [syncService]); + + const shouldRenderSyncUI = isFeatureEnabled; + const syncUIProps = syncService?.getSyncUIProps() ?? { + lastRefreshTime: undefined, + refreshInterval: undefined, + onSynchronize: () => {}, + }; + + console.log('DirectQuerySyncProvider: shouldRenderSyncUI=', shouldRenderSyncUI); + + const contextValue: DirectQuerySyncContextProps = { + shouldRenderSyncUI, + syncUIProps, + loadStatus, + mdsId, + }; + + console.log('DirectQuerySyncProvider: Providing context value:', { + shouldRenderSyncUI, + loadStatus, + mdsId, + }); + + // Render the sync bar as a decoration above the children (DashboardViewport) + const decoratedChildren = shouldRenderSyncUI ? ( + <> + + {children} + + ) : ( + children + ); + + return ( + + {decoratedChildren} + + ); +}; + +export const useDirectQuerySync = (): DirectQuerySyncContextProps => { + const context = useContext(DirectQuerySyncContext); + console.log('useDirectQuerySync: Consuming context value:', context); + return context; +}; diff --git a/src/plugins/dashboard/public/application/embeddable/viewport/dashboard_viewport.tsx b/src/plugins/dashboard/public/application/embeddable/viewport/dashboard_viewport.tsx index 8d27c6314c6b..60a3dba7384b 100644 --- a/src/plugins/dashboard/public/application/embeddable/viewport/dashboard_viewport.tsx +++ b/src/plugins/dashboard/public/application/embeddable/viewport/dashboard_viewport.tsx @@ -30,39 +30,17 @@ import React from 'react'; import { Subscription } from 'rxjs'; -import { - Logos, - SavedObjectsClientContract, - HttpStart, - NotificationsStart, -} from 'opensearch-dashboards/public'; +import { Logos } from 'opensearch-dashboards/public'; import { PanelState, EmbeddableStart } from '../../../../../embeddable/public'; import { DashboardContainer, DashboardReactContextValue } from '../dashboard_container'; import { DashboardGrid } from '../grid'; import { context } from '../../../../../opensearch_dashboards_react/public'; -import { - DirectQueryRequest, - DirectQueryLoadingStatus, -} from '../../../../../data_source_management/public'; -import { useDirectQuery } from '../../../../../data_source_management/public'; -import { DirectQuerySyncService } from '../direct_query_sync/direct_query_sync_services'; -import { EMR_STATES } from '../direct_query_sync/direct_query_sync'; -import { DashboardDirectQuerySync } from '../direct_query_sync/dashboard_direct_query_sync'; export interface DashboardViewportProps { container: DashboardContainer; PanelComponent: EmbeddableStart['EmbeddablePanel']; renderEmpty?: () => React.ReactNode; logos: Logos; - savedObjectsClient: SavedObjectsClientContract; - http: HttpStart; - notifications: NotificationsStart; - startLoading?: (payload: DirectQueryRequest) => void; - loadStatus?: DirectQueryLoadingStatus; - pollingResult?: any; - isDirectQuerySyncEnabled: boolean; - queryLang?: string; - setMdsId?: (mdsId?: string) => void; } interface State { @@ -81,8 +59,6 @@ export class DashboardViewport extends React.Component { - // This will be set by DashboardViewportWithQuery - if (this.props.startLoading) { - this.props.startLoading(payload); - } - }, - setMdsId: (mdsId?: string) => { - // This will be set by DashboardViewportWithQuery - if (this.props.setMdsId) { - this.props.setMdsId(mdsId); - } - }, - isDirectQuerySyncEnabled: this.props.isDirectQuerySyncEnabled, - queryLang: this.props.queryLang, - }); - - // Initial metadata collection - if (this.syncService.isDirectQuerySyncEnabled()) { - this.syncService.collectAllPanelMetadata(panels); - } } public componentDidMount() { @@ -139,7 +90,6 @@ export class DashboardViewport extends React.Component { @@ -197,47 +144,27 @@ export class DashboardViewport extends React.Component - {shouldRenderSyncUI && ( - +
+ {isFullScreenMode && ( + )} -
- {isFullScreenMode && ( - - )} - -
+
); } public render() { - const emrState = EMR_STATES.get(this.props.loadStatus as string); - - if ( - emrState?.terminal && - this.props.loadStatus !== DirectQueryLoadingStatus.FRESH && - this.props.loadStatus !== DirectQueryLoadingStatus.FAILED && - this.props.loadStatus !== DirectQueryLoadingStatus.CANCELLED - ) { - window.location.reload(); - } - return ( {this.state.isEmptyState ? this.renderEmptyScreen() : null} @@ -246,29 +173,3 @@ export class DashboardViewport extends React.Component -) => { - const [mdsId, setMdsId] = React.useState(undefined); - const { http, notifications, ...restProps } = props; - const { startLoading, loadStatus, pollingResult } = useDirectQuery(http, notifications, mdsId); - - return ( - - ); -};