From 34b2a34514e89a7d91065dfcddba7740037ac480 Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Mon, 16 Mar 2026 16:46:20 -0700 Subject: [PATCH 01/32] chore: point fideslang to branch with data_purposes support Update fideslang dependency to use the feat/add-data-purposes-to-dataset-models branch which adds data_purposes at dataset, collection, field, and sub-field levels. Dependency: ethyca/fideslang#39 Co-Authored-By: Claude Opus 4.6 (1M context) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 5790328d99e..d6eaec0e93f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,7 +56,7 @@ dependencies = [ "fastapi-cli~=0.0.16", "fastapi-pagination[sqlalchemy]==0.15.0", "fastapi[all]==0.123.3", - "fideslang==3.1.4a1", + "fideslang @ git+https://github.com/ethyca/fideslang.git@feat/add-data-purposes-to-dataset-models", "fideslog==1.2.15", "firebase-admin==5.3.0", "flower==2.0.1", From 9c0240ebdb3d094b3e5b6cc9e9512535e67eb7f2 Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Mon, 16 Mar 2026 17:49:38 -0700 Subject: [PATCH 02/32] fix: allow direct references in hatch metadata for fideslang git dep Co-Authored-By: Claude Opus 4.6 (1M context) --- pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index d6eaec0e93f..3a752695e54 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -180,6 +180,9 @@ dev = [ fides = "fides.cli:cli" webserver = "fides.api.main:start_webserver" +[tool.hatch.metadata] +allow-direct-references = true + [tool.hatch.version] source = "vcs" fallback-version = "0.0.0+dev" From 5662d04f236591a63a589eb38e740d3f5bf45cee Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Mon, 16 Mar 2026 18:01:16 -0700 Subject: [PATCH 03/32] fix: move fideslang git ref to requirements.txt to avoid hatchling direct-reference error Co-Authored-By: Claude Opus 4.6 (1M context) --- pyproject.toml | 5 +---- requirements.txt | 1 + 2 files changed, 2 insertions(+), 4 deletions(-) create mode 100644 requirements.txt diff --git a/pyproject.toml b/pyproject.toml index 3a752695e54..5790328d99e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,7 +56,7 @@ dependencies = [ "fastapi-cli~=0.0.16", "fastapi-pagination[sqlalchemy]==0.15.0", "fastapi[all]==0.123.3", - "fideslang @ git+https://github.com/ethyca/fideslang.git@feat/add-data-purposes-to-dataset-models", + "fideslang==3.1.4a1", "fideslog==1.2.15", "firebase-admin==5.3.0", "flower==2.0.1", @@ -180,9 +180,6 @@ dev = [ fides = "fides.cli:cli" webserver = "fides.api.main:start_webserver" -[tool.hatch.metadata] -allow-direct-references = true - [tool.hatch.version] source = "vcs" fallback-version = "0.0.0+dev" diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000000..72766131c14 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +fideslang @ git+https://github.com/ethyca/fideslang.git@feat/add-data-purposes-to-dataset-models From 9dace64b2ffe73146ceff5f56b154293f703a42a Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Wed, 18 Mar 2026 22:16:36 -0700 Subject: [PATCH 04/32] =?UTF-8?q?feat:=20PBAC=20UI=20=E2=80=94=20data=20pu?= =?UTF-8?q?rposes,=20data=20consumers,=20and=20query=20log=20config?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add admin UI for managing purpose-based access control (PBAC) entities: Data purposes & data consumers management pages: - List pages with Ant Table, search, pagination under Core Configuration - Add/edit pages using Ant Design v5 Form (Form.useForm, Form.Item) - Data purpose form mirrors privacy declaration fields (data use, categories, subjects, legal basis, retention, special category, features) - Data consumer form includes purpose assignment via multi-select - Delete confirmation modals with scope-based access control - RTK Query slices with cache invalidation (DataPurpose, DataConsumer tags) - Nav registration gated behind alphaPurposeBasedAccessControl flag Query log config integration tab: - Settings toggle panel on integration detail page (not CRUD table) - Enable/disable switch with poll interval selector - Inline Test connection and Poll now action buttons - RTK Query slice for query log config CRUD + test + poll endpoints - Tab registered for BigQuery and test_datastore (mock) integration types - test_datastore connections always pass connection test (no secrets needed) Infrastructure: - 12 new OAuth scope enums (DATA_PURPOSE_*, DATA_CONSUMER_*, QUERY_LOG_SOURCE_*) - 3 new cache tags (DataPurpose, DataConsumer, QueryLogConfig) - IntegrationFeature.QUERY_LOGGING enum value - test_datastore integration type info with QUERY_LOGGING feature - Connection test always succeeds for test_datastore/test_website types Co-Authored-By: Claude Opus 4.6 (1M context) --- .../admin-ui/src/features/common/api.slice.ts | 3 + .../src/features/common/nav/nav-config.tsx | 14 + .../src/features/common/nav/routes.ts | 10 + .../DataConsumerActionsCell.tsx | 40 +++ .../data-consumers/DataConsumerForm.tsx | 169 ++++++++++ .../data-consumers/DataConsumersTable.tsx | 43 +++ .../DeleteDataConsumerModal.tsx | 56 ++++ .../src/features/data-consumers/constants.ts | 19 ++ .../data-consumers/data-consumer.slice.ts | 113 +++++++ .../src/features/data-consumers/index.ts | 7 + .../data-consumers/useDataConsumersTable.tsx | 155 +++++++++ .../data-purposes/DataPurposeActionsCell.tsx | 48 +++ .../data-purposes/DataPurposeForm.tsx | 293 ++++++++++++++++++ .../data-purposes/DataPurposesTable.tsx | 42 +++ .../data-purposes/DeleteDataPurposeModal.tsx | 62 ++++ .../src/features/data-purposes/constants.ts | 1 + .../data-purposes/data-purpose.slice.ts | 90 ++++++ .../src/features/data-purposes/index.ts | 1 + .../data-purposes/useDataPurposesTable.tsx | 133 ++++++++ .../add-integration/allIntegrationTypes.tsx | 12 + .../ConfigureQueryLogModal.tsx | 215 +++++++++++++ .../QueryLogConfigActionsCell.tsx | 128 ++++++++ .../configure-query-log/QueryLogConfigTab.tsx | 232 ++++++++++++++ .../configure-query-log/constants.ts | 21 ++ .../query-log-config.slice.ts | 135 ++++++++ .../useQueryLogConfigTable.tsx | 109 +++++++ .../hooks/useFeatureBasedTabs.tsx | 9 + .../integration-type-info/bigqueryInfo.tsx | 5 +- .../src/pages/data-consumers/[id].tsx | 113 +++++++ .../src/pages/data-consumers/index.tsx | 36 +++ .../admin-ui/src/pages/data-consumers/new.tsx | 71 +++++ .../src/pages/data-purposes/[fidesKey].tsx | 78 +++++ .../src/pages/data-purposes/index.tsx | 35 +++ .../admin-ui/src/pages/data-purposes/new.tsx | 51 +++ .../types/api/models/IntegrationFeature.ts | 1 + .../src/types/api/models/ScopeRegistryEnum.ts | 12 + .../service/connection/connection_service.py | 13 + 37 files changed, 2574 insertions(+), 1 deletion(-) create mode 100644 clients/admin-ui/src/features/data-consumers/DataConsumerActionsCell.tsx create mode 100644 clients/admin-ui/src/features/data-consumers/DataConsumerForm.tsx create mode 100644 clients/admin-ui/src/features/data-consumers/DataConsumersTable.tsx create mode 100644 clients/admin-ui/src/features/data-consumers/DeleteDataConsumerModal.tsx create mode 100644 clients/admin-ui/src/features/data-consumers/constants.ts create mode 100644 clients/admin-ui/src/features/data-consumers/data-consumer.slice.ts create mode 100644 clients/admin-ui/src/features/data-consumers/index.ts create mode 100644 clients/admin-ui/src/features/data-consumers/useDataConsumersTable.tsx create mode 100644 clients/admin-ui/src/features/data-purposes/DataPurposeActionsCell.tsx create mode 100644 clients/admin-ui/src/features/data-purposes/DataPurposeForm.tsx create mode 100644 clients/admin-ui/src/features/data-purposes/DataPurposesTable.tsx create mode 100644 clients/admin-ui/src/features/data-purposes/DeleteDataPurposeModal.tsx create mode 100644 clients/admin-ui/src/features/data-purposes/constants.ts create mode 100644 clients/admin-ui/src/features/data-purposes/data-purpose.slice.ts create mode 100644 clients/admin-ui/src/features/data-purposes/index.ts create mode 100644 clients/admin-ui/src/features/data-purposes/useDataPurposesTable.tsx create mode 100644 clients/admin-ui/src/features/integrations/configure-query-log/ConfigureQueryLogModal.tsx create mode 100644 clients/admin-ui/src/features/integrations/configure-query-log/QueryLogConfigActionsCell.tsx create mode 100644 clients/admin-ui/src/features/integrations/configure-query-log/QueryLogConfigTab.tsx create mode 100644 clients/admin-ui/src/features/integrations/configure-query-log/constants.ts create mode 100644 clients/admin-ui/src/features/integrations/configure-query-log/query-log-config.slice.ts create mode 100644 clients/admin-ui/src/features/integrations/configure-query-log/useQueryLogConfigTable.tsx create mode 100644 clients/admin-ui/src/pages/data-consumers/[id].tsx create mode 100644 clients/admin-ui/src/pages/data-consumers/index.tsx create mode 100644 clients/admin-ui/src/pages/data-consumers/new.tsx create mode 100644 clients/admin-ui/src/pages/data-purposes/[fidesKey].tsx create mode 100644 clients/admin-ui/src/pages/data-purposes/index.tsx create mode 100644 clients/admin-ui/src/pages/data-purposes/new.tsx diff --git a/clients/admin-ui/src/features/common/api.slice.ts b/clients/admin-ui/src/features/common/api.slice.ts index 318782c289d..16e8f33c28c 100644 --- a/clients/admin-ui/src/features/common/api.slice.ts +++ b/clients/admin-ui/src/features/common/api.slice.ts @@ -71,6 +71,9 @@ export const baseApi = createApi({ "Property", "Property-Specific Messaging Templates", "Purpose", + "DataPurpose", + "DataConsumer", + "QueryLogConfig", "Shared Monitor Configs", "System", "System Assets", diff --git a/clients/admin-ui/src/features/common/nav/nav-config.tsx b/clients/admin-ui/src/features/common/nav/nav-config.tsx index 87c229459f7..aae71d7d7b1 100644 --- a/clients/admin-ui/src/features/common/nav/nav-config.tsx +++ b/clients/admin-ui/src/features/common/nav/nav-config.tsx @@ -241,6 +241,20 @@ export const NAV_CONFIG: NavConfigGroup[] = [ ScopeRegistryEnum.CONFIG_UPDATE, ], }, + { + title: "Data purposes", + path: routes.DATA_PURPOSES_ROUTE, + requiresPlus: true, + requiresFlag: "alphaPurposeBasedAccessControl", + scopes: [ScopeRegistryEnum.DATA_PURPOSE_READ], + }, + { + title: "Data consumers", + path: routes.DATA_CONSUMERS_ROUTE, + requiresPlus: true, + requiresFlag: "alphaPurposeBasedAccessControl", + scopes: [ScopeRegistryEnum.DATA_CONSUMER_READ], + }, { title: "Access policies", path: routes.ACCESS_POLICIES_ROUTE, diff --git a/clients/admin-ui/src/features/common/nav/routes.ts b/clients/admin-ui/src/features/common/nav/routes.ts index 0bf2857a749..bd663541aa8 100644 --- a/clients/admin-ui/src/features/common/nav/routes.ts +++ b/clients/admin-ui/src/features/common/nav/routes.ts @@ -114,6 +114,16 @@ export const SANDBOX_PRIVACY_NOTICES_ROUTE = "/sandbox/privacy-notices"; export const PRIVACY_ASSESSMENTS_ROUTE = "/privacy-assessments"; export const PRIVACY_ASSESSMENTS_DETAIL_ROUTE = "/privacy-assessments/[id]"; +// Data Purposes (Core Configuration) +export const DATA_PURPOSES_ROUTE = "/data-purposes"; +export const DATA_PURPOSES_NEW_ROUTE = "/data-purposes/new"; +export const DATA_PURPOSES_EDIT_ROUTE = "/data-purposes/[fidesKey]"; + +// Data Consumers (Core Configuration) +export const DATA_CONSUMERS_ROUTE = "/data-consumers"; +export const DATA_CONSUMERS_NEW_ROUTE = "/data-consumers/new"; +export const DATA_CONSUMERS_EDIT_ROUTE = "/data-consumers/[id]"; + // Access Policies (Core Configuration) export const ACCESS_POLICIES_ROUTE = "/access-policies"; export const ACCESS_POLICIES_ONBOARDING_ROUTE = "/access-policies/onboarding"; diff --git a/clients/admin-ui/src/features/data-consumers/DataConsumerActionsCell.tsx b/clients/admin-ui/src/features/data-consumers/DataConsumerActionsCell.tsx new file mode 100644 index 00000000000..4e1de1678cf --- /dev/null +++ b/clients/admin-ui/src/features/data-consumers/DataConsumerActionsCell.tsx @@ -0,0 +1,40 @@ +import { Button, Flex, Icons } from "fidesui"; +import { useRouter } from "next/router"; + +import { DATA_CONSUMERS_ROUTE } from "~/features/common/nav/routes"; +import Restrict from "~/features/common/Restrict"; +import { ScopeRegistryEnum } from "~/types/api"; + +import { DataConsumer } from "./data-consumer.slice"; +import DeleteDataConsumerModal from "./DeleteDataConsumerModal"; + +interface Props { + consumer: DataConsumer; +} + +const DataConsumerActionsCell = ({ consumer }: Props) => { + const router = useRouter(); + + const handleEdit = () => { + router.push(`${DATA_CONSUMERS_ROUTE}/${consumer.id}`); + }; + + return ( + + + + + + + ); +}; + +export default DataConsumerForm; diff --git a/clients/admin-ui/src/features/data-consumers/DataConsumersTable.tsx b/clients/admin-ui/src/features/data-consumers/DataConsumersTable.tsx new file mode 100644 index 00000000000..5e515915540 --- /dev/null +++ b/clients/admin-ui/src/features/data-consumers/DataConsumersTable.tsx @@ -0,0 +1,43 @@ +import { Button, Flex, Table } from "fidesui"; +import { useRouter } from "next/router"; + +import { DebouncedSearchInput } from "~/features/common/DebouncedSearchInput"; +import { DATA_CONSUMERS_NEW_ROUTE } from "~/features/common/nav/routes"; +import Restrict from "~/features/common/Restrict"; +import { ScopeRegistryEnum } from "~/types/api"; + +import useDataConsumersTable from "./useDataConsumersTable"; + +const DataConsumersTable = () => { + const router = useRouter(); + const { tableProps, columns, searchQuery, updateSearch } = + useDataConsumersTable(); + + return ( + + + + + + + + + + ); +}; + +export default DataConsumersTable; diff --git a/clients/admin-ui/src/features/data-consumers/DeleteDataConsumerModal.tsx b/clients/admin-ui/src/features/data-consumers/DeleteDataConsumerModal.tsx new file mode 100644 index 00000000000..f99ce506e01 --- /dev/null +++ b/clients/admin-ui/src/features/data-consumers/DeleteDataConsumerModal.tsx @@ -0,0 +1,56 @@ +import { Button, useMessage, useModal } from "fidesui"; +import { useRouter } from "next/router"; + +import { getErrorMessage } from "~/features/common/helpers"; +import { TrashCanOutlineIcon } from "~/features/common/Icon/TrashCanOutlineIcon"; +import { DATA_CONSUMERS_ROUTE } from "~/features/common/nav/routes"; + +import { + DataConsumer, + useDeleteDataConsumerMutation, +} from "./data-consumer.slice"; + +interface Props { + consumer: DataConsumer; +} + +const DeleteDataConsumerModal = ({ consumer }: Props) => { + const message = useMessage(); + const modal = useModal(); + const router = useRouter(); + const [deleteDataConsumer] = useDeleteDataConsumerMutation(); + + const handleDelete = (e: React.MouseEvent) => { + e.stopPropagation(); + modal.confirm({ + title: `Delete ${consumer.name}`, + content: `You are about to delete data consumer "${consumer.name}". This action is not reversible. Are you sure you want to proceed?`, + okText: "Delete", + okType: "danger", + centered: true, + onOk: async () => { + try { + await deleteDataConsumer(consumer.id).unwrap(); + message.success( + `Data consumer "${consumer.name}" deleted successfully`, + ); + router.push(DATA_CONSUMERS_ROUTE); + } catch (error) { + message.error(getErrorMessage(error as any)); + } + }, + }); + }; + + return ( + + + + } + data-testid="no-results-notice" + /> + ), + }, + }, + }), + [totalRows, isLoading, isFetching, items, router], + ); + + const { tableProps } = useAntTable(tableState, antTableConfig); + + const columns: ColumnsType = useMemo( + () => [ + { + title: "Name", + dataIndex: "name", + key: "name", + render: (_, { name, id }) => ( + {name} + ), + }, + { + title: "Type", + dataIndex: "type", + key: "type", + render: (_, { type }) => ( + {CONSUMER_TYPE_LABELS[type as DataConsumerType] ?? type} + ), + }, + { + title: "Contact", + dataIndex: "contact_email", + key: "contact_email", + render: (_, { contact_email }) => contact_email || "N/A", + }, + { + title: "Purposes", + dataIndex: "purpose_fides_keys", + key: "purpose_fides_keys", + render: (_, { purpose_fides_keys }) => + purpose_fides_keys && purpose_fides_keys.length > 0 ? ( + + {purpose_fides_keys.map((key) => ( + {key} + ))} + + ) : ( + "N/A" + ), + }, + { + title: "Tags", + dataIndex: "tags", + key: "tags", + render: (_, { tags }) => + tags && tags.length > 0 ? ( + + {tags.map((tag) => ( + {tag} + ))} + + ) : ( + "N/A" + ), + }, + { + title: "Actions", + dataIndex: "actions", + key: "actions", + render: (_, consumer) => ( + + ), + }, + ], + [], + ); + + return { tableProps, columns, error, searchQuery, updateSearch }; +}; + +export default useDataConsumersTable; diff --git a/clients/admin-ui/src/features/data-purposes/DataPurposeActionsCell.tsx b/clients/admin-ui/src/features/data-purposes/DataPurposeActionsCell.tsx new file mode 100644 index 00000000000..5aa907639f7 --- /dev/null +++ b/clients/admin-ui/src/features/data-purposes/DataPurposeActionsCell.tsx @@ -0,0 +1,48 @@ +import { Button, Flex, Icons } from "fidesui"; +import { useRouter } from "next/router"; + +import { TrashCanOutlineIcon } from "~/features/common/Icon/TrashCanOutlineIcon"; +import { DATA_PURPOSES_ROUTE } from "~/features/common/nav/routes"; +import Restrict from "~/features/common/Restrict"; +import { DataPurpose } from "~/features/data-purposes/data-purpose.slice"; +import DeleteDataPurposeModal from "~/features/data-purposes/DeleteDataPurposeModal"; +import { ScopeRegistryEnum } from "~/types/api"; + +interface Props { + purpose: DataPurpose; +} + +const DataPurposeActionsCell = ({ purpose }: Props) => { + const router = useRouter(); + + const handleEdit = () => { + router.push(`${DATA_PURPOSES_ROUTE}/${purpose.fides_key}`); + }; + + return ( + + + + + + + ); +}; + +export default DataPurposeForm; diff --git a/clients/admin-ui/src/features/data-purposes/DataPurposesTable.tsx b/clients/admin-ui/src/features/data-purposes/DataPurposesTable.tsx new file mode 100644 index 00000000000..85e5b26bf8c --- /dev/null +++ b/clients/admin-ui/src/features/data-purposes/DataPurposesTable.tsx @@ -0,0 +1,42 @@ +import { Button, Flex, Table } from "fidesui"; +import { useRouter } from "next/router"; + +import { DebouncedSearchInput } from "~/features/common/DebouncedSearchInput"; +import { DATA_PURPOSES_NEW_ROUTE } from "~/features/common/nav/routes"; +import Restrict from "~/features/common/Restrict"; +import useDataPurposesTable from "~/features/data-purposes/useDataPurposesTable"; +import { ScopeRegistryEnum } from "~/types/api"; + +const DataPurposesTable = () => { + const router = useRouter(); + const { tableProps, columns, searchQuery, updateSearch } = + useDataPurposesTable(); + + return ( + + + + + + + +
+ + ); +}; + +export default DataPurposesTable; diff --git a/clients/admin-ui/src/features/data-purposes/DeleteDataPurposeModal.tsx b/clients/admin-ui/src/features/data-purposes/DeleteDataPurposeModal.tsx new file mode 100644 index 00000000000..696e3bce4d2 --- /dev/null +++ b/clients/admin-ui/src/features/data-purposes/DeleteDataPurposeModal.tsx @@ -0,0 +1,62 @@ +import { useMessage, useModal } from "fidesui"; +import React from "react"; + +import { getErrorMessage } from "~/features/common/helpers"; +import Restrict from "~/features/common/Restrict"; +import { + DataPurpose, + useDeleteDataPurposeMutation, +} from "~/features/data-purposes/data-purpose.slice"; +import { ScopeRegistryEnum } from "~/types/api"; +import { RTKErrorResult } from "~/types/errors/api"; + +interface Props { + purpose: DataPurpose; + triggerComponent: React.ReactElement; +} + +const DeleteDataPurposeModal = ({ purpose, triggerComponent }: Props) => { + const message = useMessage(); + const modal = useModal(); + const [deleteDataPurpose] = useDeleteDataPurposeMutation(); + + const handleModalOpen = (e: React.MouseEvent) => { + e.stopPropagation(); + modal.confirm({ + title: `Delete ${purpose.name}`, + content: ( + + You are about to delete data purpose "{purpose.name}". This + action is not reversible. Are you sure you want to proceed? + + ), + okText: "Delete", + centered: true, + onOk: async () => { + try { + await deleteDataPurpose({ + fidesKey: purpose.fides_key, + force: true, + }).unwrap(); + message.success( + `Data purpose "${purpose.name}" deleted successfully`, + ); + } catch (error) { + message.error(getErrorMessage(error as RTKErrorResult["error"])); + } + }, + }); + }; + + return ( + + + {React.cloneElement(triggerComponent, { + onClick: handleModalOpen, + })} + + + ); +}; + +export default DeleteDataPurposeModal; diff --git a/clients/admin-ui/src/features/data-purposes/constants.ts b/clients/admin-ui/src/features/data-purposes/constants.ts new file mode 100644 index 00000000000..a6afc1bc69b --- /dev/null +++ b/clients/admin-ui/src/features/data-purposes/constants.ts @@ -0,0 +1 @@ +export const DATA_PURPOSE_FORM_ID = "data-purpose-form"; diff --git a/clients/admin-ui/src/features/data-purposes/data-purpose.slice.ts b/clients/admin-ui/src/features/data-purposes/data-purpose.slice.ts new file mode 100644 index 00000000000..1c235a18d99 --- /dev/null +++ b/clients/admin-ui/src/features/data-purposes/data-purpose.slice.ts @@ -0,0 +1,90 @@ +import { baseApi } from "~/features/common/api.slice"; + +interface DataPurposeParams { + page?: number; + size?: number; + search?: string; + data_use?: string; +} + +export interface DataPurpose { + id?: string; + fides_key: string; + name: string; + description?: string; + data_use: string; + data_subject?: string; + data_categories?: string[]; + legal_basis_for_processing?: string; + flexible_legal_basis_for_processing?: boolean; + special_category_legal_basis?: string; + impact_assessment_location?: string; + retention_period?: string; + features?: string[]; + created_at?: string; + updated_at?: string; +} + +export interface DataPurposePage { + items: DataPurpose[]; + total: number; + page: number; + size: number; + pages: number; +} + +export const dataPurposesApi = baseApi.injectEndpoints({ + endpoints: (builder) => ({ + getAllDataPurposes: builder.query({ + query: (params) => ({ + url: `data-purpose`, + params, + }), + providesTags: ["DataPurpose"], + }), + getDataPurposeByKey: builder.query({ + query: (fidesKey) => ({ + url: `data-purpose/${fidesKey}`, + }), + providesTags: ["DataPurpose"], + }), + createDataPurpose: builder.mutation>({ + query: (body) => ({ + url: `data-purpose`, + method: "POST", + body, + }), + invalidatesTags: ["DataPurpose"], + }), + updateDataPurpose: builder.mutation< + DataPurpose, + { fidesKey: string } & Partial + >({ + query: ({ fidesKey, ...body }) => ({ + url: `data-purpose/${fidesKey}`, + method: "PUT", + body, + }), + invalidatesTags: ["DataPurpose"], + }), + deleteDataPurpose: builder.mutation< + void, + { fidesKey: string; force?: boolean } + >({ + query: ({ fidesKey, force }) => ({ + url: `data-purpose/${fidesKey}`, + method: "DELETE", + params: force ? { force: true } : undefined, + }), + invalidatesTags: ["DataPurpose", "DataConsumer"], + }), + }), +}); + +export const { + useGetAllDataPurposesQuery, + useGetDataPurposeByKeyQuery, + useCreateDataPurposeMutation, + useUpdateDataPurposeMutation, + useDeleteDataPurposeMutation, +} = dataPurposesApi; diff --git a/clients/admin-ui/src/features/data-purposes/index.ts b/clients/admin-ui/src/features/data-purposes/index.ts new file mode 100644 index 00000000000..c82ff9818d8 --- /dev/null +++ b/clients/admin-ui/src/features/data-purposes/index.ts @@ -0,0 +1 @@ +export * from "./data-purpose.slice"; diff --git a/clients/admin-ui/src/features/data-purposes/useDataPurposesTable.tsx b/clients/admin-ui/src/features/data-purposes/useDataPurposesTable.tsx new file mode 100644 index 00000000000..15b713560af --- /dev/null +++ b/clients/admin-ui/src/features/data-purposes/useDataPurposesTable.tsx @@ -0,0 +1,133 @@ +import { + Button, + ColumnsType, + Empty, + Flex, + Space, + Tag, + Typography, +} from "fidesui"; +import { useRouter } from "next/router"; +import { useMemo } from "react"; + +import { DATA_PURPOSES_NEW_ROUTE } from "~/features/common/nav/routes"; +import { LinkCell } from "~/features/common/table/cells/LinkCell"; +import { useAntTable, useTableState } from "~/features/common/table/hooks"; +import { + DataPurpose, + useGetAllDataPurposesQuery, +} from "~/features/data-purposes/data-purpose.slice"; +import DataPurposeActionsCell from "~/features/data-purposes/DataPurposeActionsCell"; + +const useDataPurposesTable = () => { + const router = useRouter(); + + const tableState = useTableState({ + pagination: { + defaultPageSize: 25, + pageSizeOptions: [25, 50, 100], + }, + search: { + defaultSearchQuery: "", + }, + }); + + const { pageIndex, pageSize, searchQuery, updateSearch } = tableState; + + const { data, error, isLoading, isFetching } = useGetAllDataPurposesQuery({ + page: pageIndex, + size: pageSize, + search: searchQuery, + }); + + const items = useMemo(() => data?.items ?? [], [data?.items]); + const totalRows = data?.total ?? 0; + + const antTableConfig = useMemo( + () => ({ + dataSource: items, + totalRows, + isLoading, + isFetching, + getRowKey: (record: DataPurpose) => record.fides_key, + customTableProps: { + locale: { + emptyText: ( + + + No data purposes found. + + + + + + } + data-testid="no-results-notice" + /> + ), + }, + }, + }), + [totalRows, isLoading, isFetching, items, router], + ); + + const { tableProps } = useAntTable(tableState, antTableConfig); + + const columns: ColumnsType = useMemo( + () => [ + { + title: "Name", + dataIndex: "name", + key: "name", + render: (_, { name, fides_key }) => ( + {name} + ), + }, + { + title: "Key", + dataIndex: "fides_key", + key: "fides_key", + }, + { + title: "Data use", + dataIndex: "data_use", + key: "data_use", + render: (_, { data_use }) => {data_use}, + }, + { + title: "Categories", + dataIndex: "data_categories", + key: "data_categories", + render: (_, { data_categories }) => + data_categories && data_categories.length > 0 ? ( + + {data_categories.map((cat) => ( + {cat} + ))} + + ) : ( + "N/A" + ), + }, + { + title: "Actions", + dataIndex: "actions", + key: "actions", + render: (_, purpose) => , + }, + ], + [], + ); + + return { tableProps, columns, error, searchQuery, updateSearch }; +}; + +export default useDataPurposesTable; diff --git a/clients/admin-ui/src/features/integrations/add-integration/allIntegrationTypes.tsx b/clients/admin-ui/src/features/integrations/add-integration/allIntegrationTypes.tsx index 36b170adbbe..0a0f442cb0e 100644 --- a/clients/admin-ui/src/features/integrations/add-integration/allIntegrationTypes.tsx +++ b/clients/admin-ui/src/features/integrations/add-integration/allIntegrationTypes.tsx @@ -56,6 +56,18 @@ const INTEGRATION_TYPE_MAP: { [K in ConnectionType]?: IntegrationTypeInfo } = { [ConnectionType.WEBSITE]: WEBSITE_INTEGRATION_TYPE_INFO, [ConnectionType.POSTGRES]: POSTGRES_TYPE_INFO, [ConnectionType.MANUAL_TASK]: MANUAL_TYPE_INFO, + [ConnectionType.TEST_DATASTORE]: { + placeholder: { + name: "Mock query log", + key: "test_datastore_placeholder", + connection_type: ConnectionType.TEST_DATASTORE, + access: AccessLevel.READ, + created_at: "", + }, + category: ConnectionCategory.DATA_WAREHOUSE, + tags: ["Query Logging", "Mock"], + enabledFeatures: [IntegrationFeature.QUERY_LOGGING], + }, }; export const INTEGRATION_TYPE_LIST: IntegrationTypeInfo[] = [ diff --git a/clients/admin-ui/src/features/integrations/configure-query-log/ConfigureQueryLogModal.tsx b/clients/admin-ui/src/features/integrations/configure-query-log/ConfigureQueryLogModal.tsx new file mode 100644 index 00000000000..12e9c325e7a --- /dev/null +++ b/clients/admin-ui/src/features/integrations/configure-query-log/ConfigureQueryLogModal.tsx @@ -0,0 +1,215 @@ +import { + Flex, + Input, + Modal, + Select, + Switch, + Typography, + useMessage, +} from "fidesui"; +import { Form, Formik, FormikHelpers } from "formik"; +import { useCallback } from "react"; + +import { getErrorMessage } from "~/features/common/helpers"; +import { + POLL_INTERVAL_OPTIONS, + PollInterval, +} from "~/features/integrations/configure-query-log/constants"; +import { + type CreateQueryLogConfigRequest, + type QueryLogConfigResponse, + useCreateQueryLogConfigMutation, + useUpdateQueryLogConfigMutation, +} from "~/features/integrations/configure-query-log/query-log-config.slice"; +import { RTKErrorResult } from "~/types/errors/api"; + +interface ConfigureQueryLogModalProps { + isOpen: boolean; + onClose: () => void; + config?: QueryLogConfigResponse; + isEditing: boolean; + integrationKey: string; +} + +interface FormValues { + name: string; + key: string; + enabled: boolean; + poll_interval_seconds: number; +} + +const generateKey = (name: string): string => + name + .toLowerCase() + .replace(/[^a-z0-9]+/g, "_") + .replace(/^_|_$/g, ""); + +const ConfigureQueryLogModal = ({ + isOpen, + onClose, + config, + isEditing, + integrationKey, +}: ConfigureQueryLogModalProps) => { + const message = useMessage(); + const [createQueryLogConfig] = useCreateQueryLogConfigMutation(); + const [updateQueryLogConfig] = useUpdateQueryLogConfigMutation(); + + const initialValues: FormValues = { + name: config?.name ?? "", + key: config?.key ?? "", + enabled: config?.enabled ?? true, + poll_interval_seconds: + config?.poll_interval_seconds ?? PollInterval.FIVE_MINUTES, + }; + + const handleSubmit = useCallback( + async (values: FormValues, helpers: FormikHelpers) => { + try { + if (isEditing && config) { + await updateQueryLogConfig({ + configKey: config.key, + name: values.name, + enabled: values.enabled, + poll_interval_seconds: values.poll_interval_seconds, + }).unwrap(); + message.success("Query log config updated successfully"); + } else { + const body: CreateQueryLogConfigRequest = { + connection_config_key: integrationKey, + name: values.name, + key: values.key, + enabled: values.enabled, + poll_interval_seconds: values.poll_interval_seconds, + }; + await createQueryLogConfig(body).unwrap(); + message.success("Query log config created successfully"); + } + onClose(); + } catch (error) { + message.error( + getErrorMessage( + error as RTKErrorResult["error"], + "An error occurred saving the query log config.", + ), + ); + } finally { + helpers.setSubmitting(false); + } + }, + [ + isEditing, + config, + integrationKey, + createQueryLogConfig, + updateQueryLogConfig, + message, + onClose, + ], + ); + + return ( + + + initialValues={initialValues} + enableReinitialize + onSubmit={handleSubmit} + > + {({ values, setFieldValue, isSubmitting, submitForm }) => ( +
+ +
+ + Name + + { + const newName = e.target.value; + setFieldValue("name", newName); + if (!isEditing) { + setFieldValue("key", generateKey(newName)); + } + }} + placeholder="Enter config name" + data-testid="query-log-config-name-input" + /> +
+ +
+ + Key + + setFieldValue("key", e.target.value)} + placeholder="Auto-generated from name" + data-testid="query-log-config-key-input" + /> +
+ + + setFieldValue("enabled", checked)} + aria-label="Enabled" + data-testid="query-log-config-enabled-switch" + /> + + {values.enabled ? "Enabled" : "Disabled"} + + + +
+ + Poll interval + + + + )} + + + + + {enabled && existingConfig && ( + <> + + + + )} + + + + )} +
+ ); +}; + +export default QueryLogConfigTab; diff --git a/clients/admin-ui/src/features/integrations/configure-query-log/constants.ts b/clients/admin-ui/src/features/integrations/configure-query-log/constants.ts new file mode 100644 index 00000000000..ed716f31c50 --- /dev/null +++ b/clients/admin-ui/src/features/integrations/configure-query-log/constants.ts @@ -0,0 +1,21 @@ +export enum PollInterval { + ONE_MINUTE = 60, + FIVE_MINUTES = 300, + FIFTEEN_MINUTES = 900, + ONE_HOUR = 3600, + SIX_HOURS = 21600, + TWENTY_FOUR_HOURS = 86400, +} + +export const POLL_INTERVAL_LABELS: Record = { + [PollInterval.ONE_MINUTE]: "Every minute", + [PollInterval.FIVE_MINUTES]: "Every 5 minutes", + [PollInterval.FIFTEEN_MINUTES]: "Every 15 minutes", + [PollInterval.ONE_HOUR]: "Every hour", + [PollInterval.SIX_HOURS]: "Every 6 hours", + [PollInterval.TWENTY_FOUR_HOURS]: "Every 24 hours", +}; + +export const POLL_INTERVAL_OPTIONS = Object.entries(POLL_INTERVAL_LABELS).map( + ([value, label]) => ({ value: Number(value), label }), +); diff --git a/clients/admin-ui/src/features/integrations/configure-query-log/query-log-config.slice.ts b/clients/admin-ui/src/features/integrations/configure-query-log/query-log-config.slice.ts new file mode 100644 index 00000000000..9a651ed9b78 --- /dev/null +++ b/clients/admin-ui/src/features/integrations/configure-query-log/query-log-config.slice.ts @@ -0,0 +1,135 @@ +import { baseApi } from "~/features/common/api.slice"; + +export interface QueryLogConfigResponse { + key: string; + name: string; + connection_config_key: string; + enabled: boolean; + poll_interval_seconds: number; + created_at: string; + updated_at: string; +} + +export interface QueryLogConfigListResponse { + items: QueryLogConfigResponse[]; + total: number; + page: number; + size: number; +} + +export interface GetQueryLogConfigsParams { + page?: number; + size?: number; + connection_config_key?: string; +} + +export interface CreateQueryLogConfigRequest { + connection_config_key: string; + name: string; + key?: string; + enabled?: boolean; + poll_interval_seconds?: number; +} + +export interface UpdateQueryLogConfigRequest { + configKey: string; + name?: string; + enabled?: boolean; + poll_interval_seconds?: number; +} + +export interface TestQueryLogConnectionResponse { + success: boolean; + message: string; +} + +export interface TriggerQueryLogPollResponse { + entries_processed: number; + message?: string; +} + +const queryLogConfigApi = baseApi.injectEndpoints({ + endpoints: (build) => ({ + getQueryLogConfigs: build.query< + QueryLogConfigListResponse, + GetQueryLogConfigsParams + >({ + query: (params) => ({ + url: "query-log-config", + method: "GET", + params, + }), + providesTags: ["QueryLogConfig"], + }), + + getQueryLogConfigByKey: build.query({ + query: (configKey) => ({ + url: `query-log-config/${configKey}`, + method: "GET", + }), + providesTags: (_result, _error, configKey) => [ + { type: "QueryLogConfig", id: configKey }, + ], + }), + + createQueryLogConfig: build.mutation< + QueryLogConfigResponse, + CreateQueryLogConfigRequest + >({ + query: (body) => ({ + url: "query-log-config", + method: "POST", + body, + }), + invalidatesTags: ["QueryLogConfig"], + }), + + updateQueryLogConfig: build.mutation< + QueryLogConfigResponse, + UpdateQueryLogConfigRequest + >({ + query: ({ configKey, ...body }) => ({ + url: `query-log-config/${configKey}`, + method: "PUT", + body, + }), + invalidatesTags: ["QueryLogConfig"], + }), + + deleteQueryLogConfig: build.mutation({ + query: (configKey) => ({ + url: `query-log-config/${configKey}`, + method: "DELETE", + }), + invalidatesTags: ["QueryLogConfig"], + }), + + testQueryLogConnection: build.mutation< + TestQueryLogConnectionResponse, + string + >({ + query: (configKey) => ({ + url: `query-log-config/${configKey}/test`, + method: "POST", + }), + }), + + triggerQueryLogPoll: build.mutation({ + query: (configKey) => ({ + url: `query-log-config/${configKey}/poll`, + method: "POST", + }), + invalidatesTags: ["QueryLogConfig"], + }), + }), +}); + +export const { + useGetQueryLogConfigsQuery, + useGetQueryLogConfigByKeyQuery, + useCreateQueryLogConfigMutation, + useUpdateQueryLogConfigMutation, + useDeleteQueryLogConfigMutation, + useTestQueryLogConnectionMutation, + useTriggerQueryLogPollMutation, +} = queryLogConfigApi; diff --git a/clients/admin-ui/src/features/integrations/configure-query-log/useQueryLogConfigTable.tsx b/clients/admin-ui/src/features/integrations/configure-query-log/useQueryLogConfigTable.tsx new file mode 100644 index 00000000000..f8ef97ecc91 --- /dev/null +++ b/clients/admin-ui/src/features/integrations/configure-query-log/useQueryLogConfigTable.tsx @@ -0,0 +1,109 @@ +import { ColumnsType, Tag } from "fidesui"; +import { useMemo } from "react"; + +import { useAntTable, useTableState } from "~/features/common/table/hooks"; +import { + POLL_INTERVAL_LABELS, + PollInterval, +} from "~/features/integrations/configure-query-log/constants"; +import { + QueryLogConfigResponse, + useGetQueryLogConfigsQuery, +} from "~/features/integrations/configure-query-log/query-log-config.slice"; +import QueryLogConfigActionsCell from "~/features/integrations/configure-query-log/QueryLogConfigActionsCell"; + +enum QueryLogConfigColumnKeys { + NAME = "name", + KEY = "key", + STATUS = "status", + POLL_INTERVAL = "poll_interval_seconds", + ACTION = "action", +} + +interface UseQueryLogConfigTableConfig { + integrationKey: string; + onEdit: (config: QueryLogConfigResponse) => void; +} + +export const useQueryLogConfigTable = ({ + integrationKey, + onEdit, +}: UseQueryLogConfigTableConfig) => { + const tableState = useTableState(); + + const { pageIndex, pageSize } = tableState; + + const { + isLoading, + isFetching, + data: response, + } = useGetQueryLogConfigsQuery({ + page: pageIndex, + size: pageSize, + connection_config_key: integrationKey, + }); + + const antTableConfig = useMemo( + () => ({ + enableSelection: false, + getRowKey: (record: QueryLogConfigResponse) => record.key, + isLoading, + isFetching, + dataSource: response?.items ?? [], + totalRows: response?.total ?? 0, + }), + [isLoading, isFetching, response?.items, response?.total], + ); + + const antTable = useAntTable< + QueryLogConfigResponse, + QueryLogConfigColumnKeys + >(tableState, antTableConfig); + + const columns: ColumnsType = useMemo( + () => [ + { + title: "Status", + dataIndex: QueryLogConfigColumnKeys.STATUS, + key: QueryLogConfigColumnKeys.STATUS, + width: 120, + render: (_: unknown, record: QueryLogConfigResponse) => ( + + {record.enabled ? "Enabled" : "Disabled"} + + ), + }, + { + title: "Poll interval", + dataIndex: QueryLogConfigColumnKeys.POLL_INTERVAL, + key: QueryLogConfigColumnKeys.POLL_INTERVAL, + render: (_: unknown, record: QueryLogConfigResponse) => + POLL_INTERVAL_LABELS[record.poll_interval_seconds as PollInterval] ?? + `${record.poll_interval_seconds}s`, + }, + { + title: "Actions", + key: QueryLogConfigColumnKeys.ACTION, + width: 200, + render: (_: unknown, record: QueryLogConfigResponse) => ( + onEdit(record)} + /> + ), + fixed: "right" as const, + }, + ], + [onEdit], + ); + + const hasConfig = (response?.items?.length ?? 0) > 0; + + return { + columns, + tableProps: antTable.tableProps, + isLoading, + isFetching, + hasConfig, + }; +}; diff --git a/clients/admin-ui/src/features/integrations/hooks/useFeatureBasedTabs.tsx b/clients/admin-ui/src/features/integrations/hooks/useFeatureBasedTabs.tsx index ed93d985669..3ba149ff4ca 100644 --- a/clients/admin-ui/src/features/integrations/hooks/useFeatureBasedTabs.tsx +++ b/clients/admin-ui/src/features/integrations/hooks/useFeatureBasedTabs.tsx @@ -10,6 +10,7 @@ import { import { useMemo } from "react"; import MonitorConfigTab from "~/features/integrations/configure-monitor/MonitorConfigTab"; +import QueryLogConfigTab from "~/features/integrations/configure-query-log/QueryLogConfigTab"; import DatahubDataSyncTab from "~/features/integrations/configure-scan/DatahubDataSyncTab"; import TaskConditionsTab from "~/features/integrations/configure-tasks/TaskConditionsTab"; import TaskConfigTab from "~/features/integrations/configure-tasks/TaskConfigTab"; @@ -165,6 +166,14 @@ export const useFeatureBasedTabs = ({ }); } + if (enabledFeatures?.includes(IntegrationFeature.QUERY_LOGGING)) { + tabItems.push({ + label: "Query logging", + key: "query-logging", + children: , + }); + } + if (enabledFeatures?.includes(IntegrationFeature.TASKS)) { tabItems.push({ label: "Manual tasks", diff --git a/clients/admin-ui/src/features/integrations/integration-type-info/bigqueryInfo.tsx b/clients/admin-ui/src/features/integrations/integration-type-info/bigqueryInfo.tsx index 2193e351106..f7605c7d3cb 100644 --- a/clients/admin-ui/src/features/integrations/integration-type-info/bigqueryInfo.tsx +++ b/clients/admin-ui/src/features/integrations/integration-type-info/bigqueryInfo.tsx @@ -215,7 +215,10 @@ const BIGQUERY_TYPE_INFO = { overview: , instructions: , tags: BIGQUERY_TAGS, - enabledFeatures: [IntegrationFeature.DATA_DISCOVERY], + enabledFeatures: [ + IntegrationFeature.DATA_DISCOVERY, + IntegrationFeature.QUERY_LOGGING, + ], }; export default BIGQUERY_TYPE_INFO; diff --git a/clients/admin-ui/src/pages/data-consumers/[id].tsx b/clients/admin-ui/src/pages/data-consumers/[id].tsx new file mode 100644 index 00000000000..c8d197a7af2 --- /dev/null +++ b/clients/admin-ui/src/pages/data-consumers/[id].tsx @@ -0,0 +1,113 @@ +import { Flex, Spin, useMessage } from "fidesui"; +import type { NextPage } from "next"; +import { useRouter } from "next/router"; + +import ErrorPage from "~/features/common/errors/ErrorPage"; +import { getErrorMessage } from "~/features/common/helpers"; +import Layout from "~/features/common/Layout"; +import { DATA_CONSUMERS_ROUTE } from "~/features/common/nav/routes"; +import PageHeader from "~/features/common/PageHeader"; +import { + useAssignConsumerPurposesMutation, + useGetDataConsumerByIdQuery, + useUpdateDataConsumerMutation, +} from "~/features/data-consumers/data-consumer.slice"; +import DataConsumerForm, { + DataConsumerFormValues, +} from "~/features/data-consumers/DataConsumerForm"; +import { RTKErrorResult } from "~/types/errors/api"; + +const EditDataConsumerPage: NextPage = () => { + const message = useMessage(); + const router = useRouter(); + const { id: consumerId } = router.query; + + const { + data: consumer, + error, + isLoading, + } = useGetDataConsumerByIdQuery(consumerId as string, { + skip: !consumerId, + }); + + const [updateDataConsumer] = useUpdateDataConsumerMutation(); + const [assignConsumerPurposes] = useAssignConsumerPurposesMutation(); + + const handleSubmit = async (values: DataConsumerFormValues) => { + if (!consumerId) { + return; + } + + const { purposeFidesKeys, ...consumerPayload } = values; + + try { + await updateDataConsumer({ + id: consumerId as string, + ...consumerPayload, + }).unwrap(); + + // Check if purposes changed and assign if so + const existingPurposes = consumer?.purpose_fides_keys ?? []; + const purposesChanged = + JSON.stringify([...purposeFidesKeys].sort()) !== + JSON.stringify([...existingPurposes].sort()); + + if (purposesChanged) { + try { + await assignConsumerPurposes({ + id: consumerId as string, + purposeFidesKeys, + }).unwrap(); + } catch (assignErr) { + message.error( + `Data consumer updated but failed to assign purposes: ${getErrorMessage(assignErr as RTKErrorResult["error"])}`, + ); + return; + } + } + + message.success(`Data consumer "${values.name}" updated successfully`); + } catch (err) { + message.error(getErrorMessage(err as RTKErrorResult["error"])); + } + }; + + if (error) { + return ( + + ); + } + + if (isLoading) { + return ( + + + + + + ); + } + + return ( + + + + + ); +}; + +export default EditDataConsumerPage; diff --git a/clients/admin-ui/src/pages/data-consumers/index.tsx b/clients/admin-ui/src/pages/data-consumers/index.tsx new file mode 100644 index 00000000000..090f1c06f11 --- /dev/null +++ b/clients/admin-ui/src/pages/data-consumers/index.tsx @@ -0,0 +1,36 @@ +import { Typography } from "fidesui"; +import type { NextPage } from "next"; + +import ErrorPage from "~/features/common/errors/ErrorPage"; +import Layout from "~/features/common/Layout"; +import PageHeader from "~/features/common/PageHeader"; +import DataConsumersTable from "~/features/data-consumers/DataConsumersTable"; +import useDataConsumersTable from "~/features/data-consumers/useDataConsumersTable"; + +const DataConsumersPage: NextPage = () => { + const { error } = useDataConsumersTable(); + + if (error) { + return ( + + ); + } + + return ( + + + + Review and manage your data consumers below. Data consumers represent + the services, applications, groups, or users that access personal + data. + + + + + ); +}; + +export default DataConsumersPage; diff --git a/clients/admin-ui/src/pages/data-consumers/new.tsx b/clients/admin-ui/src/pages/data-consumers/new.tsx new file mode 100644 index 00000000000..a9f7572a730 --- /dev/null +++ b/clients/admin-ui/src/pages/data-consumers/new.tsx @@ -0,0 +1,71 @@ +import { useMessage } from "fidesui"; +import type { NextPage } from "next"; +import { useRouter } from "next/router"; + +import { getErrorMessage } from "~/features/common/helpers"; +import Layout from "~/features/common/Layout"; +import { DATA_CONSUMERS_ROUTE } from "~/features/common/nav/routes"; +import PageHeader from "~/features/common/PageHeader"; +import { + useAssignConsumerPurposesMutation, + useCreateDataConsumerMutation, +} from "~/features/data-consumers/data-consumer.slice"; +import DataConsumerForm, { + DataConsumerFormValues, +} from "~/features/data-consumers/DataConsumerForm"; +import { RTKErrorResult } from "~/types/errors/api"; + +const NewDataConsumerPage: NextPage = () => { + const message = useMessage(); + const router = useRouter(); + const [createDataConsumer] = useCreateDataConsumerMutation(); + const [assignConsumerPurposes] = useAssignConsumerPurposesMutation(); + + const handleSubmit = async (values: DataConsumerFormValues) => { + const { purposeFidesKeys, ...consumerPayload } = values; + + try { + const created = await createDataConsumer(consumerPayload).unwrap(); + + if (purposeFidesKeys.length > 0) { + try { + await assignConsumerPurposes({ + id: created.id, + purposeFidesKeys, + }).unwrap(); + } catch (assignErr) { + message.error( + `Data consumer created but failed to assign purposes: ${getErrorMessage(assignErr as RTKErrorResult["error"])}`, + ); + router.push(`${DATA_CONSUMERS_ROUTE}/${created.id}`); + return; + } + } + + message.success(`Data consumer "${values.name}" created successfully`); + router.push(`${DATA_CONSUMERS_ROUTE}/${created.id}`); + } catch (err) { + message.error(getErrorMessage(err as RTKErrorResult["error"])); + } + }; + + return ( + + + + + ); +}; + +export default NewDataConsumerPage; diff --git a/clients/admin-ui/src/pages/data-purposes/[fidesKey].tsx b/clients/admin-ui/src/pages/data-purposes/[fidesKey].tsx new file mode 100644 index 00000000000..f28dde3184f --- /dev/null +++ b/clients/admin-ui/src/pages/data-purposes/[fidesKey].tsx @@ -0,0 +1,78 @@ +import { Spin, useMessage } from "fidesui"; +import type { NextPage } from "next"; +import { useRouter } from "next/router"; + +import ErrorPage from "~/features/common/errors/ErrorPage"; +import { getErrorMessage } from "~/features/common/helpers"; +import Layout from "~/features/common/Layout"; +import { DATA_PURPOSES_ROUTE } from "~/features/common/nav/routes"; +import PageHeader from "~/features/common/PageHeader"; +import { + useGetDataPurposeByKeyQuery, + useUpdateDataPurposeMutation, +} from "~/features/data-purposes/data-purpose.slice"; +import DataPurposeForm, { + DataPurposeFormValues, +} from "~/features/data-purposes/DataPurposeForm"; +import { RTKErrorResult } from "~/types/errors/api"; + +const EditDataPurposePage: NextPage = () => { + const message = useMessage(); + const router = useRouter(); + const { fidesKey } = router.query; + + const { + data: purpose, + error, + isLoading, + } = useGetDataPurposeByKeyQuery(fidesKey as string, { + skip: !fidesKey, + }); + + const [updateDataPurpose] = useUpdateDataPurposeMutation(); + + const handleSubmit = async (values: DataPurposeFormValues) => { + try { + await updateDataPurpose({ + fidesKey: values.fides_key, + ...values, + }).unwrap(); + message.success(`Data purpose "${values.name}" updated successfully`); + } catch (err) { + message.error(getErrorMessage(err as RTKErrorResult["error"])); + } + }; + + if (error) { + return ( + + ); + } + + return ( + + + {isLoading ? ( + + ) : ( + + )} + + ); +}; + +export default EditDataPurposePage; diff --git a/clients/admin-ui/src/pages/data-purposes/index.tsx b/clients/admin-ui/src/pages/data-purposes/index.tsx new file mode 100644 index 00000000000..a326d3c942d --- /dev/null +++ b/clients/admin-ui/src/pages/data-purposes/index.tsx @@ -0,0 +1,35 @@ +import { Typography } from "fidesui"; +import type { NextPage } from "next"; + +import ErrorPage from "~/features/common/errors/ErrorPage"; +import Layout from "~/features/common/Layout"; +import PageHeader from "~/features/common/PageHeader"; +import DataPurposesTable from "~/features/data-purposes/DataPurposesTable"; +import useDataPurposesTable from "~/features/data-purposes/useDataPurposesTable"; + +const DataPurposesPage: NextPage = () => { + const { error } = useDataPurposesTable(); + + if (error) { + return ( + + ); + } + + return ( + + + + Review and manage your data purposes below. Data purposes define the + reasons data is collected and processed within your organization. + + + + + ); +}; + +export default DataPurposesPage; diff --git a/clients/admin-ui/src/pages/data-purposes/new.tsx b/clients/admin-ui/src/pages/data-purposes/new.tsx new file mode 100644 index 00000000000..dee270ec078 --- /dev/null +++ b/clients/admin-ui/src/pages/data-purposes/new.tsx @@ -0,0 +1,51 @@ +import { useMessage } from "fidesui"; +import type { NextPage } from "next"; +import { useRouter } from "next/router"; + +import { getErrorMessage } from "~/features/common/helpers"; +import Layout from "~/features/common/Layout"; +import { DATA_PURPOSES_ROUTE } from "~/features/common/nav/routes"; +import PageHeader from "~/features/common/PageHeader"; +import { useCreateDataPurposeMutation } from "~/features/data-purposes/data-purpose.slice"; +import DataPurposeForm, { + DataPurposeFormValues, +} from "~/features/data-purposes/DataPurposeForm"; +import { RTKErrorResult } from "~/types/errors/api"; + +const AddDataPurposePage: NextPage = () => { + const message = useMessage(); + const router = useRouter(); + const [createDataPurpose] = useCreateDataPurposeMutation(); + + const handleSubmit = async (values: DataPurposeFormValues) => { + try { + await createDataPurpose(values).unwrap(); + message.success(`Data purpose "${values.name}" created successfully`); + router.push(DATA_PURPOSES_ROUTE); + } catch (error) { + message.error(getErrorMessage(error as RTKErrorResult["error"])); + } + }; + + return ( + + +
+ +
+ + ); +}; + +export default AddDataPurposePage; diff --git a/clients/admin-ui/src/types/api/models/IntegrationFeature.ts b/clients/admin-ui/src/types/api/models/IntegrationFeature.ts index d5be835d83d..bf8887f91c1 100644 --- a/clients/admin-ui/src/types/api/models/IntegrationFeature.ts +++ b/clients/admin-ui/src/types/api/models/IntegrationFeature.ts @@ -13,4 +13,5 @@ export enum IntegrationFeature { WITHOUT_CONNECTION = "WITHOUT_CONNECTION", DSR_AUTOMATION = "DSR_AUTOMATION", CONDITIONS = "CONDITIONS", + QUERY_LOGGING = "QUERY_LOGGING", } diff --git a/clients/admin-ui/src/types/api/models/ScopeRegistryEnum.ts b/clients/admin-ui/src/types/api/models/ScopeRegistryEnum.ts index 52c992effc6..e2b6c3b94a2 100644 --- a/clients/admin-ui/src/types/api/models/ScopeRegistryEnum.ts +++ b/clients/admin-ui/src/types/api/models/ScopeRegistryEnum.ts @@ -62,10 +62,18 @@ export enum ScopeRegistryEnum { CUSTOM_REPORT_CREATE = "custom_report:create", CUSTOM_REPORT_DELETE = "custom_report:delete", CUSTOM_REPORT_READ = "custom_report:read", + DATA_CONSUMER_CREATE = "data_consumer:create", + DATA_CONSUMER_DELETE = "data_consumer:delete", + DATA_CONSUMER_READ = "data_consumer:read", + DATA_CONSUMER_UPDATE = "data_consumer:update", DATA_CATEGORY_CREATE = "data_category:create", DATA_CATEGORY_DELETE = "data_category:delete", DATA_CATEGORY_READ = "data_category:read", DATA_CATEGORY_UPDATE = "data_category:update", + DATA_PURPOSE_CREATE = "data_purpose:create", + DATA_PURPOSE_DELETE = "data_purpose:delete", + DATA_PURPOSE_READ = "data_purpose:read", + DATA_PURPOSE_UPDATE = "data_purpose:update", DATA_SUBJECT_CREATE = "data_subject:create", DATA_SUBJECT_DELETE = "data_subject:delete", DATA_SUBJECT_READ = "data_subject:read", @@ -165,6 +173,10 @@ export enum ScopeRegistryEnum { PROPERTY_DELETE = "property:delete", PROPERTY_READ = "property:read", PROPERTY_UPDATE = "property:update", + QUERY_LOG_SOURCE_CREATE = "query_log_source:create", + QUERY_LOG_SOURCE_DELETE = "query_log_source:delete", + QUERY_LOG_SOURCE_READ = "query_log_source:read", + QUERY_LOG_SOURCE_UPDATE = "query_log_source:update", RESPONDENT_CREATE = "respondent:create", RULE_CREATE_OR_UPDATE = "rule:create_or_update", RULE_DELETE = "rule:delete", diff --git a/src/fides/service/connection/connection_service.py b/src/fides/service/connection/connection_service.py index 44c279885cd..3a9a3ee83ba 100644 --- a/src/fides/service/connection/connection_service.py +++ b/src/fides/service/connection/connection_service.py @@ -248,6 +248,19 @@ def connection_status( ) -> TestStatusMessage: """Connect, verify with a trivial query or API request, and report the status.""" + # Test connection types always succeed (no real connector to test) + if connection_config.connection_type in ( + ConnectionType.test_datastore, + ConnectionType.test_website, + ): + connection_config.update_test_status( + test_status=ConnectionTestStatus.succeeded, db=self.db + ) + return TestStatusMessage( + msg=msg, + test_status=ConnectionTestStatus.succeeded, + ) + connector = get_connector(connection_config) try: From a41d17702d73c6a5bef1025150480824cdd22260 Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Wed, 18 Mar 2026 23:29:12 -0700 Subject: [PATCH 05/32] feat: add Seed Data developer page for triggering seed scenarios Adds a new "Seed Data" page under the Developer nav (dev-only) that lets users select and trigger seed scenarios via the seed API. Includes RTK Query slice with status polling and cache tag invalidation mapped per seed task. Currently supports the PBAC scenario. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../src/features/common/nav/nav-config.tsx | 6 + .../src/features/common/nav/routes.ts | 1 + .../src/features/seed-data/SeedDataPanel.tsx | 185 ++++++++++++++++++ .../src/features/seed-data/seed-data.slice.ts | 104 ++++++++++ clients/admin-ui/src/pages/poc/seed-data.tsx | 22 +++ 5 files changed, 318 insertions(+) create mode 100644 clients/admin-ui/src/features/seed-data/SeedDataPanel.tsx create mode 100644 clients/admin-ui/src/features/seed-data/seed-data.slice.ts create mode 100644 clients/admin-ui/src/pages/poc/seed-data.tsx diff --git a/clients/admin-ui/src/features/common/nav/nav-config.tsx b/clients/admin-ui/src/features/common/nav/nav-config.tsx index aae71d7d7b1..b990d574cea 100644 --- a/clients/admin-ui/src/features/common/nav/nav-config.tsx +++ b/clients/admin-ui/src/features/common/nav/nav-config.tsx @@ -373,6 +373,12 @@ if (process.env.NEXT_PUBLIC_APP_ENV === "development") { scopes: [ScopeRegistryEnum.DEVELOPER_READ], requiresPlus: true, }, + { + title: "Seed Data", + path: routes.SEED_DATA_ROUTE, + scopes: [ScopeRegistryEnum.DEVELOPER_READ], + requiresPlus: true, + }, { title: "Test Monitors", path: routes.TEST_MONITORS_ROUTE, diff --git a/clients/admin-ui/src/features/common/nav/routes.ts b/clients/admin-ui/src/features/common/nav/routes.ts index bd663541aa8..83576f9a03c 100644 --- a/clients/admin-ui/src/features/common/nav/routes.ts +++ b/clients/admin-ui/src/features/common/nav/routes.ts @@ -105,6 +105,7 @@ export const ERRORS_POC_ROUTE = "/poc/error"; export const TABLE_MIGRATION_POC_ROUTE = "/poc/table-migration"; export const FIDES_JS_DOCS = "/fides-js-docs"; export const PROMPT_EXPLORER_ROUTE = "/poc/prompt-explorer"; +export const SEED_DATA_ROUTE = "/poc/seed-data"; export const TEST_MONITORS_ROUTE = "/poc/test-monitors"; // Sandbox routes diff --git a/clients/admin-ui/src/features/seed-data/SeedDataPanel.tsx b/clients/admin-ui/src/features/seed-data/SeedDataPanel.tsx new file mode 100644 index 00000000000..73dff6e4bb4 --- /dev/null +++ b/clients/admin-ui/src/features/seed-data/SeedDataPanel.tsx @@ -0,0 +1,185 @@ +import { + Alert, + Button, + Card, + Checkbox, + Flex, + Progress, + Space, + Tag, + Typography, +} from "fidesui"; +import { useCallback, useEffect, useState } from "react"; + +import { + SeedStepStatus, + SeedTasksConfig, + useGetSeedStatusQuery, + useTriggerSeedMutation, +} from "~/features/seed-data/seed-data.slice"; + +const { Paragraph, Text } = Typography; + +interface SeedScenario { + key: keyof SeedTasksConfig; + label: string; + description: string; +} + +const SEED_SCENARIOS: SeedScenario[] = [ + { + key: "pbac", + label: "PBAC (Access Control)", + description: + "Seed data purposes, data consumers, datasets, a mock query log integration, and 60 days of access control history.", + }, +]; + +const STATUS_COLORS: Record = { + pending: "default", + in_progress: "processing", + complete: "success", + skipped: "warning", + error: "error", +}; + +const StepStatusTag = ({ status }: { status: SeedStepStatus }) => ( + + {status.status.replace("_", " ")} + +); + +const computeProgress = (steps: Record): number => { + const entries = Object.values(steps); + if (entries.length === 0) { + return 0; + } + const done = entries.filter( + (s) => + s.status === "complete" || s.status === "skipped" || s.status === "error", + ).length; + return Math.round((done / entries.length) * 100); +}; + +const SeedDataPanel = () => { + const [selectedTasks, setSelectedTasks] = useState< + Set + >(new Set()); + const [executionId, setExecutionId] = useState(null); + const [triggerSeed, { isLoading: isTriggering }] = useTriggerSeedMutation(); + + // Poll for status when an execution is in progress + const { data: statusData } = useGetSeedStatusQuery(executionId!, { + skip: !executionId, + pollingInterval: 2000, + }); + + // Stop polling when execution completes + useEffect(() => { + if ( + statusData && + (statusData.status === "complete" || statusData.status === "error") + ) { + setExecutionId(null); + } + }, [statusData]); + + const handleToggle = useCallback((key: keyof SeedTasksConfig) => { + setSelectedTasks((prev) => { + const next = new Set(prev); + if (next.has(key)) { + next.delete(key); + } else { + next.add(key); + } + return next; + }); + }, []); + + const handleSeed = useCallback(async () => { + const tasks: SeedTasksConfig = {}; + selectedTasks.forEach((key) => { + tasks[key] = true; + }); + + try { + const result = await triggerSeed({ tasks }).unwrap(); + setExecutionId(result.execution_id); + } catch { + // Error is shown via the status polling + } + }, [selectedTasks, triggerSeed]); + + const isRunning = !!executionId || isTriggering; + const showStatus = statusData && statusData.status !== "pending"; + + return ( + + + Select which data scenarios to seed into this environment. Seeding is + idempotent — it is safe to run multiple times. + + + + {SEED_SCENARIOS.map((scenario) => ( + handleToggle(scenario.key)} + disabled={isRunning} + > + {scenario.label} +
+ {scenario.description} + + ))} + + + + + + + {showStatus ? ( +
+ + + {Object.entries(statusData.steps).map(([stepName, stepStatus]) => ( + + {stepName.replace(/_/g, " ")} + + + ))} + + {statusData.error ? ( + + ) : null} + {statusData.status === "complete" ? ( + + ) : null} +
+ ) : null} + + ); +}; + +export default SeedDataPanel; diff --git a/clients/admin-ui/src/features/seed-data/seed-data.slice.ts b/clients/admin-ui/src/features/seed-data/seed-data.slice.ts new file mode 100644 index 00000000000..c9111219f43 --- /dev/null +++ b/clients/admin-ui/src/features/seed-data/seed-data.slice.ts @@ -0,0 +1,104 @@ +import { baseApi } from "~/features/common/api.slice"; + +export interface SeedTasksConfig { + sample_resources?: boolean; + messaging_mailgun?: boolean; + storage_s3?: boolean; + linked_connections?: boolean; + consent_notices?: boolean; + consent_experiences?: boolean; + compass_vendors?: boolean; + email_templates?: boolean; + discovery_monitors?: boolean; + pbac?: boolean; +} + +export interface SeedRequest { + tasks: SeedTasksConfig; +} + +export interface SeedResponse { + execution_id: string; + message: string; +} + +export interface SeedStepStatus { + status: "pending" | "in_progress" | "complete" | "skipped" | "error"; + message?: string; + started_at?: string; + completed_at?: string; +} + +export interface SeedStatusResponse { + id: string; + status: "pending" | "in_progress" | "complete" | "error"; + started_at: string; + completed_at?: string; + steps: Record; + error?: string; +} + +/** + * Cache tags to invalidate when seeding completes. + * Maps each seed task to the tags it affects. + */ +const SEED_TASK_INVALIDATION_TAGS: Record = { + pbac: [ + "DataPurpose", + "DataConsumer", + "QueryLogConfig", + "Datastore Connection", + "Dataset", + "Datasets", + ], + sample_resources: ["System", "Dataset", "Datasets", "Policies"], + linked_connections: ["Datastore Connection"], + consent_notices: ["Privacy Notices"], + consent_experiences: ["Privacy Experience Configs"], + compass_vendors: ["System", "Dictionary", "System Vendors"], + email_templates: ["Messaging Templates"], + discovery_monitors: ["Discovery Monitor Configs", "Datastore Connection"], + messaging_mailgun: ["Messaging Config"], + storage_s3: ["Configuration Settings"], +}; + +const seedDataApi = baseApi.injectEndpoints({ + endpoints: (build) => ({ + triggerSeed: build.mutation({ + query: (body) => ({ + url: "plus/seed", + method: "POST", + body, + }), + // Invalidate tags for all enabled tasks so cached data refreshes + // after the user navigates to the seeded pages. + invalidatesTags: (_result, _error, arg) => { + const tags: string[] = []; + const { tasks } = arg; + (Object.keys(tasks) as (keyof SeedTasksConfig)[]).forEach((key) => { + if (tasks[key] && SEED_TASK_INVALIDATION_TAGS[key]) { + tags.push(...SEED_TASK_INVALIDATION_TAGS[key]); + } + }); + // Deduplicate + return [...new Set(tags)]; + }, + }), + getSeedStatus: build.query({ + query: (executionId) => ({ + url: `plus/seed/status/${executionId}`, + }), + }), + getLatestSeedStatus: build.query({ + query: () => ({ + url: "plus/seed/status", + }), + }), + }), +}); + +export const { + useTriggerSeedMutation, + useGetSeedStatusQuery, + useGetLatestSeedStatusQuery, +} = seedDataApi; diff --git a/clients/admin-ui/src/pages/poc/seed-data.tsx b/clients/admin-ui/src/pages/poc/seed-data.tsx new file mode 100644 index 00000000000..a2ee4cbc2bc --- /dev/null +++ b/clients/admin-ui/src/pages/poc/seed-data.tsx @@ -0,0 +1,22 @@ +import { Layout, Typography } from "fidesui"; +import type { NextPage } from "next"; + +import PageHeader from "~/features/common/PageHeader"; +import SeedDataPanel from "~/features/seed-data/SeedDataPanel"; + +const { Content } = Layout; +const { Paragraph } = Typography; + +const SeedDataPage: NextPage = () => { + return ( + + + + Seed demo data into this environment for testing and development. + + + + ); +}; + +export default SeedDataPage; From 13850c0ed02b960cc4abd829618054e7f21571e0 Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Wed, 18 Mar 2026 23:50:48 -0700 Subject: [PATCH 06/32] fix: disable query logging via update instead of delete Toggling the query log switch off now sends PUT {enabled: false} instead of DELETE, preserving the config and its watermark so re-enabling resumes from where it left off. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../configure-query-log/QueryLogConfigTab.tsx | 21 +++++++------------ 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/clients/admin-ui/src/features/integrations/configure-query-log/QueryLogConfigTab.tsx b/clients/admin-ui/src/features/integrations/configure-query-log/QueryLogConfigTab.tsx index b483e073a61..493393d18ea 100644 --- a/clients/admin-ui/src/features/integrations/configure-query-log/QueryLogConfigTab.tsx +++ b/clients/admin-ui/src/features/integrations/configure-query-log/QueryLogConfigTab.tsx @@ -16,7 +16,6 @@ import { RTKErrorResult } from "~/types/errors/api"; import { POLL_INTERVAL_OPTIONS } from "./constants"; import { useCreateQueryLogConfigMutation, - useDeleteQueryLogConfigMutation, useGetQueryLogConfigsQuery, useTestQueryLogConnectionMutation, useTriggerQueryLogPollMutation, @@ -57,14 +56,12 @@ const QueryLogConfigTab = ({ integration }: QueryLogConfigTabProps) => { useCreateQueryLogConfigMutation(); const [updateConfig, { isLoading: isUpdating }] = useUpdateQueryLogConfigMutation(); - const [deleteConfig, { isLoading: isDeleting }] = - useDeleteQueryLogConfigMutation(); const [testConnection, { isLoading: isTesting }] = useTestQueryLogConnectionMutation(); const [triggerPoll, { isLoading: isPolling }] = useTriggerQueryLogPollMutation(); - const isSaving = isCreating || isUpdating || isDeleting; + const isSaving = isCreating || isUpdating; const enabled = Form.useWatch("enabled", form); @@ -99,22 +96,18 @@ const QueryLogConfigTab = ({ integration }: QueryLogConfigTabProps) => { }).unwrap(); message.success("Query logging settings updated"); } else if (!values.enabled && existingConfig) { - // Disable — delete the config - await deleteConfig(existingConfig.key).unwrap(); + // Disable — keep the config but stop polling + await updateConfig({ + configKey: existingConfig.key, + enabled: false, + }).unwrap(); message.success("Query logging disabled"); } } catch (err) { message.error(getErrorMessage(err as RTKErrorResult["error"])); } }, - [ - existingConfig, - integration, - createConfig, - updateConfig, - deleteConfig, - message, - ], + [existingConfig, integration, createConfig, updateConfig, message], ); const handleTest = useCallback(async () => { From 79e6e5d60a15048eb93d5cb6d8de9b595b0403bb Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Thu, 19 Mar 2026 00:02:04 -0700 Subject: [PATCH 07/32] fix: use valid page size for data purposes query in consumer form The form was requesting size=500 but the API enforces max 100, causing a validation error and an empty dropdown. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../admin-ui/src/features/data-consumers/DataConsumerForm.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clients/admin-ui/src/features/data-consumers/DataConsumerForm.tsx b/clients/admin-ui/src/features/data-consumers/DataConsumerForm.tsx index 677bb67b6c1..e7047dbc2fb 100644 --- a/clients/admin-ui/src/features/data-consumers/DataConsumerForm.tsx +++ b/clients/admin-ui/src/features/data-consumers/DataConsumerForm.tsx @@ -30,7 +30,7 @@ const DataConsumerForm = ({ const router = useRouter(); const { data: purposesData, isLoading: purposesLoading } = - useGetAllDataPurposesQuery({ size: 500 }); + useGetAllDataPurposesQuery({ size: 100 }); const purposeOptions = useMemo( () => From 3f17cde12a9a95d07622dddc7597270e5f98e9b0 Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Thu, 19 Mar 2026 00:06:54 -0700 Subject: [PATCH 08/32] fix: use white text on in-progress badge in seed data page Co-Authored-By: Claude Opus 4.6 (1M context) --- .../src/features/seed-data/SeedDataPanel.tsx | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/clients/admin-ui/src/features/seed-data/SeedDataPanel.tsx b/clients/admin-ui/src/features/seed-data/SeedDataPanel.tsx index 73dff6e4bb4..a861bf09f7f 100644 --- a/clients/admin-ui/src/features/seed-data/SeedDataPanel.tsx +++ b/clients/admin-ui/src/features/seed-data/SeedDataPanel.tsx @@ -43,11 +43,15 @@ const STATUS_COLORS: Record = { error: "error", }; -const StepStatusTag = ({ status }: { status: SeedStepStatus }) => ( - - {status.status.replace("_", " ")} - -); +const StepStatusTag = ({ status }: { status: SeedStepStatus }) => { + const color = STATUS_COLORS[status.status] ?? "default"; + const style = color === "processing" ? { color: "white" } : undefined; + return ( + + {status.status.replace("_", " ")} + + ); +}; const computeProgress = (steps: Record): number => { const entries = Object.values(steps); From d065c109ab7e4fb7e121d996f99bc075231cabb7 Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Thu, 19 Mar 2026 10:24:59 -0700 Subject: [PATCH 09/32] Adding PBAC UI plan --- plans/pbac-ui-data-purposes-consumers.md | 659 +++++++++++++++++++++++ 1 file changed, 659 insertions(+) create mode 100644 plans/pbac-ui-data-purposes-consumers.md diff --git a/plans/pbac-ui-data-purposes-consumers.md b/plans/pbac-ui-data-purposes-consumers.md new file mode 100644 index 00000000000..404dbc47c1f --- /dev/null +++ b/plans/pbac-ui-data-purposes-consumers.md @@ -0,0 +1,659 @@ +# PBAC UI: Data Purposes & Data Consumers Management + +**Source:** feat/dataset-data-purposes branch (fides repo) + +## Problem Statement + +The PBAC backend has full CRUD APIs for data purposes and data consumers, but there's no UI to manage them. Users must use the API directly or the demo script. We need two new management pages under "Core configuration" in the admin nav — one for data purposes, one for data consumers — following Ant Design v5 patterns and the conventions in the Talos frontend guidance. + +## Requirements Summary + +- **Data Purposes page**: List, create, edit, delete data purposes. Filterable by data use. +- **Data Consumers page**: List, create, edit, delete data consumers. Assign/unassign purposes to consumers. Filterable by type and assigned purpose. +- Both pages gated behind `alphaPurposeBasedAccessControl` feature flag and `requiresPlus`. +- Follow the feature-folder pattern: table with search → add/edit form pages → delete confirmation modal. + +## API Contract + +### Data Purpose API (`/api/v1/data-purpose`) + +| Method | Path | Request | Response | +|--------|------|---------|----------| +| POST | `/data-purpose` | `DataPurposeCreate` | `DataPurposeResponse` | +| GET | `/data-purpose` | `?data_use=&page=&size=` | `Page[DataPurposeResponse]` | +| GET | `/data-purpose/{fides_key}` | — | `DataPurposeResponse` | +| PUT | `/data-purpose/{fides_key}` | `DataPurposeUpdate` | `DataPurposeResponse` | +| DELETE | `/data-purpose/{fides_key}` | `?force=true` | 204 | + +**Key fields:** `fides_key` (user-provided unique key), `name`, `data_use` (required), `data_subject`, `data_categories[]`, `description` + +### Data Consumer API (`/api/v1/data-consumer`) + +| Method | Path | Request | Response | +|--------|------|---------|----------| +| POST | `/data-consumer` | `DataConsumerCreate` | `DataConsumerResponse` | +| GET | `/data-consumer` | `?type=&purpose_fides_key=&tags=&page=&size=` | `Page[DataConsumerResponse]` | +| GET | `/data-consumer/{id}` | — | `DataConsumerResponse` | +| PUT | `/data-consumer/{id}` | `DataConsumerUpdate` | `DataConsumerResponse` | +| DELETE | `/data-consumer/{id}` | — | 204 | +| PUT | `/data-consumer/{id}/purpose` | `{purpose_fides_keys: [...]}` | `DataConsumerResponse` | + +**Key fields:** `id` (UUID, system-generated), `name`, `type` (required), `contact_email`, `tags[]`, `purposes[]` (embedded refs), `purpose_fides_keys[]` + +## Frontend Conventions (Talos Guidance) + +These conventions **must** be followed — they supersede patterns in older pages like Properties: + +- **Ant Design v5 first** — always prefer Ant components over custom code +- **Mutations**: always `.unwrap()` inside `try/catch`, never `isErrorResult()` +- **User feedback**: use Ant `message.success()` / `message.error()`, never `useToast()` (deprecated Chakra) +- **Error messages**: use `getErrorMessage()` from `~/features/common/helpers` +- **Enums**: define TypeScript `enum` for finite backend value sets; keep label/color maps in `constants.ts` +- **Styling**: Ant theme tokens → FidesUI palette → Tailwind (layout only) → SCSS modules. Never arbitrary hex or sizing. +- **Navigation**: `` wrapping buttons, never `router.push()` in onClick +- **Pages are thin shells**: fetch data, handle loading/errors, compose the root feature component +- **One component per file**, split early +- **Type safety**: no `any`, no `@ts-expect-error`; use `interface` over `type`; override generated types with `Omit` + +## Implementation Approach + +Each entity gets a feature folder following the Talos file organization pattern: + +1. **Types & constants** — enums, label maps, interfaces +2. **RTK Query API slice** — CRUD endpoints with `providesTags` / `invalidatesTags` +3. **List page** — Ant `
` with search, pagination, filters +4. **Add page** — Formik form with validation +5. **Edit page** — Same form, pre-populated, with delete button +6. **Nav registration** — Under "Core configuration" with feature flag + +### File Structure + +Per Talos guidance (`features/my-feature/` template): + +``` +clients/admin-ui/src/ +├── features/ +│ ├── data-purposes/ +│ │ ├── types.d.ts # interfaces and type overrides +│ │ ├── constants.ts # enums, label maps, display config +│ │ ├── data-purpose.slice.ts # RTK Query endpoints +│ │ ├── DataPurposesTable.tsx # root feature component (Ant Table) +│ │ ├── useDataPurposesTable.tsx # table hook (state, columns, data) +│ │ ├── DataPurposeForm.tsx # Formik add/edit form +│ │ ├── DeleteDataPurposeModal.tsx # confirm delete modal +│ │ ├── DataPurposeActionsCell.tsx # edit/delete cell in table +│ │ └── index.ts # barrel export of public API +│ │ +│ └── data-consumers/ +│ ├── types.d.ts +│ ├── constants.ts +│ ├── data-consumer.slice.ts +│ ├── DataConsumersTable.tsx +│ ├── useDataConsumersTable.tsx +│ ├── DataConsumerForm.tsx # includes purpose assignment +│ ├── DeleteDataConsumerModal.tsx +│ ├── DataConsumerActionsCell.tsx +│ └── index.ts +│ +├── pages/ # thin routing shells only +│ ├── data-purposes/ +│ │ ├── index.tsx # list page +│ │ ├── new.tsx # add page +│ │ └── [fidesKey].tsx # edit page +│ │ +│ └── data-consumers/ +│ ├── index.tsx +│ ├── new.tsx +│ └── [id].tsx +``` + +Notes on Talos conventions: +- **`types.d.ts`** — interfaces and type aliases (including `Omit`-based overrides of generated types) +- **`constants.ts`** — TypeScript enums, label/color maps, static option arrays +- **`index.ts`** — required barrel file in every feature directory +- **Pages** are thin shells — fetch data, handle loading/error, compose root feature component; no business logic + +### Detailed Component Design + +#### 1. Types & Constants + +**`features/data-consumers/types.d.ts`**: +```ts +import { DataConsumerType } from "./constants"; + +// Override too-generic generated types +export interface DataConsumerResponse extends Omit { + type: DataConsumerType; +} +``` + +**`features/data-consumers/constants.ts`**: +```ts +// Enums and label maps per Talos convention + +export enum DataConsumerType { + SERVICE = "service", + APPLICATION = "application", + GROUP = "group", + USER = "user", +} + +export const CONSUMER_TYPE_LABELS: Record = { + [DataConsumerType.SERVICE]: "Service", + [DataConsumerType.APPLICATION]: "Application", + [DataConsumerType.GROUP]: "Group", + [DataConsumerType.USER]: "User", +}; + +export const CONSUMER_TYPE_OPTIONS = Object.entries(CONSUMER_TYPE_LABELS).map( + ([value, label]) => ({ value, label }) +); +``` + +#### 2. RTK Query API Slices + +**`data-purpose.slice.ts`**: +```ts +const dataPurposeApi = baseApi.injectEndpoints({ + endpoints: (builder) => ({ + getAllDataPurposes: builder.query({ + query: (params) => ({ url: "plus/data-purpose", params }), + providesTags: ["DataPurpose"], + }), + getDataPurposeByKey: builder.query({ + query: (fidesKey) => ({ url: `plus/data-purpose/${fidesKey}` }), + providesTags: ["DataPurpose"], + }), + createDataPurpose: builder.mutation({ + query: (body) => ({ url: "plus/data-purpose", method: "POST", body }), + invalidatesTags: ["DataPurpose"], + }), + updateDataPurpose: builder.mutation< + DataPurposeResponse, + { fidesKey: string } & DataPurposeUpdate + >({ + query: ({ fidesKey, ...body }) => ({ + url: `plus/data-purpose/${fidesKey}`, + method: "PUT", + body, + }), + invalidatesTags: ["DataPurpose"], + }), + deleteDataPurpose: builder.mutation({ + query: ({ fidesKey, force }) => ({ + url: `plus/data-purpose/${fidesKey}`, + method: "DELETE", + params: force ? { force: true } : undefined, + }), + invalidatesTags: ["DataPurpose", "DataConsumer"], + }), + }), +}); + +export const { + useGetAllDataPurposesQuery, + useGetDataPurposeByKeyQuery, + useCreateDataPurposeMutation, + useUpdateDataPurposeMutation, + useDeleteDataPurposeMutation, +} = dataPurposeApi; +``` + +**`data-consumer.slice.ts`** — same pattern plus: +```ts +assignConsumerPurposes: builder.mutation< + DataConsumerResponse, + { id: string; purposeFidesKeys: string[] } +>({ + query: ({ id, purposeFidesKeys }) => ({ + url: `plus/data-consumer/${id}/purpose`, + method: "PUT", + body: { purpose_fides_keys: purposeFidesKeys }, + }), + invalidatesTags: ["DataConsumer"], +}), +``` + +#### 3. Page Components (Thin Shells) + +**`pages/data-purposes/index.tsx`** (list): +```tsx +const DataPurposesPage: NextPage = () => { + return ( + + + + + ); +}; +``` + +**`pages/data-purposes/new.tsx`** (add): +```tsx +const AddDataPurposePage: NextPage = () => { + const [createDataPurpose] = useCreateDataPurposeMutation(); + const router = useRouter(); + + const handleSubmit = async (values: DataPurposeCreate) => { + try { + await createDataPurpose(values).unwrap(); + message.success(`Data purpose "${values.name}" created`); + router.push(DATA_PURPOSES_ROUTE); + } catch (error) { + message.error(getErrorMessage(error)); + } + }; + + return ( + + + + + ); +}; +``` + +**`pages/data-purposes/[fidesKey].tsx`** (edit): +```tsx +const EditDataPurposePage: NextPage = () => { + const router = useRouter(); + const { fidesKey } = router.query; + const { data: purpose, isLoading, error } = useGetDataPurposeByKeyQuery( + fidesKey as string, + { skip: !fidesKey } + ); + const [updateDataPurpose] = useUpdateDataPurposeMutation(); + + if (error) { + return ; + } + + const handleSubmit = async (values: DataPurposeUpdate) => { + try { + await updateDataPurpose({ fidesKey: fidesKey as string, ...values }).unwrap(); + message.success(`Data purpose "${values.name}" updated`); + router.push(DATA_PURPOSES_ROUTE); + } catch (error) { + message.error(getErrorMessage(error)); + } + }; + + return ( + + + {isLoading ? : } + + ); +}; +``` + +#### 4. Table Hook + +**`useDataPurposesTable.tsx`**: +```tsx +const useDataPurposesTable = () => { + const tableState = useTableState({ + pagination: { defaultPageSize: 25, pageSizeOptions: [25, 50, 100] }, + search: { defaultSearchQuery: "" }, + }); + + const { pageIndex, pageSize, searchQuery, updateSearch } = tableState; + const { data, error, isLoading, isFetching } = useGetAllDataPurposesQuery({ + page: pageIndex, + size: pageSize, + search: searchQuery, + }); + + const columns: ColumnsType = useMemo( + () => [ + { + title: "Name", + dataIndex: "name", + render: (_, record) => ( + + {record.name} + + ), + }, + { title: "Key", dataIndex: "fides_key" }, + { + title: "Data use", + dataIndex: "data_use", + render: (value: string) => {value}, + }, + { + title: "Categories", + dataIndex: "data_categories", + render: (categories: string[]) => + categories.length > 0 ? ( + {categories.map((c) => {c})} + ) : ( + "N/A" + ), + }, + { + title: "Actions", + render: (_, record) => , + }, + ], + [] + ); + + const tableProps = useAntTable({ tableState, data }); + + return { tableProps, columns, error, isLoading, isFetching, searchQuery, updateSearch }; +}; +``` + +#### 5. Delete Modal + +**`DeleteDataPurposeModal.tsx`**: +```tsx +interface DeleteDataPurposeModalProps { + purpose: DataPurposeResponse; +} + +const DeleteDataPurposeModal = ({ purpose }: DeleteDataPurposeModalProps) => { + const [modal, contextHolder] = Modal.useModal(); + const [deleteDataPurpose] = useDeleteDataPurposeMutation(); + const router = useRouter(); + + const handleDelete = () => { + modal.confirm({ + title: "Delete data purpose?", + content: `This will permanently delete "${purpose.name}". This action cannot be undone.`, + okText: "Delete", + okButtonProps: { danger: true }, + centered: true, + onOk: async () => { + try { + await deleteDataPurpose({ fidesKey: purpose.fides_key, force: true }).unwrap(); + message.success(`Data purpose "${purpose.name}" deleted`); + router.push(DATA_PURPOSES_ROUTE); + } catch (error) { + message.error(getErrorMessage(error)); + } + }, + }); + }; + + return ( + <> + {contextHolder} + + + ); +}; +``` + +#### 6. Data Purpose Form + +Fields: +- **Name** (required, Ant ``) +- **Key** (required, auto-generated from name if blank, disabled on edit, Ant ``) +- **Data use** (required, Ant ``) +- **Data categories** (Ant ``) +- **Retention period** (optional, Ant ``) + +#### 7. Data Consumer Form + +Fields: +- **Name** (required, Ant ``) +- **Type** (required, Ant ``) +- **Description** (optional, Ant ``) +- **Tags** (Ant `` — populated from `useGetAllDataPurposesQuery`; on save calls `assignConsumerPurposes` mutation with full list) +- **External ID** (optional, Ant ``, advanced section) +- **Contact Slack channel** (optional, Ant ``, advanced section) + +On the edit form, if `purpose_fides_keys` changed from the loaded value, the submit handler calls `assignConsumerPurposes` after `updateDataConsumer`. + +#### 8. Data Consumers Table + +| Column | Source | Notes | +|--------|--------|-------| +| Name | `name` | `` → `/data-consumers/{id}` | +| Type | `type` | `` with label from `CONSUMER_TYPE_LABELS` | +| Contact | `contact_email` | Text, show "N/A" if empty | +| Purposes | `purposes` | `` of `` showing purpose names | +| Tags | `tags` | `` of `` | +| Actions | — | Edit / Delete | + +- Search filters by name +- Optional `` filter by assigned purpose (fetches purposes for options) + +### 9. Navigation & Routing + +**Route constants** (`routes.ts`): +```ts +export const DATA_PURPOSES_ROUTE = "/data-purposes"; +export const DATA_PURPOSES_NEW_ROUTE = "/data-purposes/new"; +export const DATA_PURPOSES_EDIT_ROUTE = "/data-purposes/[fidesKey]"; + +export const DATA_CONSUMERS_ROUTE = "/data-consumers"; +export const DATA_CONSUMERS_NEW_ROUTE = "/data-consumers/new"; +export const DATA_CONSUMERS_EDIT_ROUTE = "/data-consumers/[id]"; +``` + +**Nav config** — add to "Core configuration" group: +```tsx +{ + title: "Data purposes", + path: routes.DATA_PURPOSES_ROUTE, + requiresPlus: true, + requiresFlag: "alphaPurposeBasedAccessControl", + scopes: [ScopeRegistryEnum.DATA_PURPOSE_READ], +}, +{ + title: "Data consumers", + path: routes.DATA_CONSUMERS_ROUTE, + requiresPlus: true, + requiresFlag: "alphaPurposeBasedAccessControl", + scopes: [ScopeRegistryEnum.DATA_CONSUMER_READ], +}, +``` + +### 10. Query Log Config Tab (Integration Detail Page) + +Query log configs sit on top of an existing `ConnectionConfig`, following the same two-step pattern as discovery monitors. The UI adds a **"Query logging" tab** to the integration detail page. + +#### Reference pattern + +The "Data discovery" tab (`MonitorConfigTab`) is the direct reference: +- Tab rendered conditionally via `useFeatureBasedTabs` when `IntegrationFeature.DATA_DISCOVERY` is enabled +- Tab lists existing configs in a table, with an "Add" button opening a modal +- Modal contains a form for create/edit; mutations invalidate cache tags to refresh the table + +#### Backend integration feature + +Add `QUERY_LOGGING = "QUERY_LOGGING"` to the `IntegrationFeature` enum (backend, in fides). BigQuery and `test_datastore` connection types include this feature. The frontend enum in `types/api/models/IntegrationFeature.ts` is auto-generated. + +#### Tab registration + +In `useFeatureBasedTabs.tsx`, add after the DATA_DISCOVERY block: + +```tsx +if (enabledFeatures?.includes(IntegrationFeature.QUERY_LOGGING)) { + tabItems.push({ + label: "Query logging", + key: "query-logging", + children: ( + + ), + }); +} +``` + +#### File structure + +``` +features/integrations/configure-query-log/ + types.d.ts # QueryLogConfig interfaces + constants.ts # poll interval options, status labels + query-log-config.slice.ts # RTK Query endpoints + QueryLogConfigTab.tsx # tab root: description + add button + table + modal + useQueryLogConfigTable.tsx # table hook (columns, data, pagination) + ConfigureQueryLogModal.tsx # create/edit modal with form + QueryLogConfigActionsCell.tsx # edit / delete / test / poll actions +``` + +#### RTK Query slice (`query-log-config.slice.ts`) + +```ts +const queryLogConfigApi = baseApi.injectEndpoints({ + endpoints: (builder) => ({ + getQueryLogConfigs: builder.query({ + query: (params) => ({ url: "plus/query-log-config", params }), + providesTags: ["QueryLogConfig"], + }), + getQueryLogConfigByKey: builder.query({ + query: (configKey) => ({ url: `plus/query-log-config/${configKey}` }), + providesTags: ["QueryLogConfig"], + }), + createQueryLogConfig: builder.mutation({ + query: (body) => ({ url: "plus/query-log-config", method: "POST", body }), + invalidatesTags: ["QueryLogConfig"], + }), + updateQueryLogConfig: builder.mutation< + QueryLogConfigResponse, + { configKey: string } & QueryLogConfigUpdate + >({ + query: ({ configKey, ...body }) => ({ + url: `plus/query-log-config/${configKey}`, + method: "PUT", + body, + }), + invalidatesTags: ["QueryLogConfig"], + }), + deleteQueryLogConfig: builder.mutation({ + query: (configKey) => ({ + url: `plus/query-log-config/${configKey}`, + method: "DELETE", + }), + invalidatesTags: ["QueryLogConfig"], + }), + testQueryLogConnection: builder.mutation({ + query: (configKey) => ({ + url: `plus/query-log-config/${configKey}/test`, + method: "POST", + }), + }), + triggerQueryLogPoll: builder.mutation({ + query: (configKey) => ({ + url: `plus/query-log-config/${configKey}/poll`, + method: "POST", + }), + }), + }), +}); +``` + +#### Table columns + +| Column | Source | Notes | +|--------|--------|-------| +| Name | `name` | Text | +| Key | `key` | Monospace | +| Status | `enabled` | Ant `` toggle (calls update mutation) | +| Poll interval | `poll_interval_seconds` | Human-readable from `POLL_INTERVAL_LABELS` | +| Last poll | `watermark` | Formatted timestamp or "Never" | +| Actions | — | Test / Poll / Edit / Delete | + +#### Modal form fields + +- **Name** (required, Ant ``) +- **Key** (auto-generated from name, disabled on edit, Ant ``) +- **Enabled** (Ant ``, default true) +- **Poll interval** (Ant ` { - const newName = e.target.value; - setFieldValue("name", newName); - if (!isEditing) { - setFieldValue("key", generateKey(newName)); - } - }} - placeholder="Enter config name" - data-testid="query-log-config-name-input" - /> - - -
- - Key - - setFieldValue("key", e.target.value)} - placeholder="Auto-generated from name" - data-testid="query-log-config-key-input" - /> -
- - - setFieldValue("enabled", checked)} - aria-label="Enabled" - data-testid="query-log-config-enabled-switch" - /> - - {values.enabled ? "Enabled" : "Disabled"} - - - -
- - Poll interval - -