From c904e386a0b01bdacac2eb85d8dd60924553a8c8 Mon Sep 17 00:00:00 2001 From: siddhant-galileo Date: Thu, 29 Jan 2026 22:18:26 +0530 Subject: [PATCH 1/7] feat: basic setup for evaluator store flow --- ui/src/core/api/client.ts | 21 + .../use-create-evaluator-config.ts | 30 ++ .../use-delete-evaluator-config.ts | 17 + .../query-hooks/use-evaluator-configs.ts | 23 + .../agent-detail/add-new-control-modal.tsx | 393 ++++++++++++++++ .../agent-detail/control-store-modal.tsx | 424 ++++++++---------- .../edit-control/edit-control-content.tsx | 106 +++-- .../edit-control/evaluator-json-view.tsx | 82 ++-- .../save-evaluator-template-modal.tsx | 156 +++++++ .../agent-detail/edit-control/types.ts | 5 + 10 files changed, 941 insertions(+), 316 deletions(-) create mode 100644 ui/src/core/hooks/query-hooks/use-create-evaluator-config.ts create mode 100644 ui/src/core/hooks/query-hooks/use-delete-evaluator-config.ts create mode 100644 ui/src/core/hooks/query-hooks/use-evaluator-configs.ts create mode 100644 ui/src/core/page-components/agent-detail/add-new-control-modal.tsx create mode 100644 ui/src/core/page-components/agent-detail/edit-control/save-evaluator-template-modal.tsx diff --git a/ui/src/core/api/client.ts b/ui/src/core/api/client.ts index 6263f4e4..dd630330 100644 --- a/ui/src/core/api/client.ts +++ b/ui/src/core/api/client.ts @@ -110,4 +110,25 @@ export const api = { params: { query: params }, }), }, + evaluatorConfigs: { + list: (params?: { + cursor?: number; + limit?: number; + name?: string; + evaluator?: string; + }) => + apiClient.GET("/api/v1/evaluator-configs", { + params: params ? { query: params } : undefined, + }), + create: (data: { + name: string; + description?: string | null; + evaluator: string; + config: Record; + }) => apiClient.POST("/api/v1/evaluator-configs", { body: data }), + delete: (configId: number) => + apiClient.DELETE("/api/v1/evaluator-configs/{config_id}", { + params: { path: { config_id: configId } }, + }), + }, }; diff --git a/ui/src/core/hooks/query-hooks/use-create-evaluator-config.ts b/ui/src/core/hooks/query-hooks/use-create-evaluator-config.ts new file mode 100644 index 00000000..c05cf5b5 --- /dev/null +++ b/ui/src/core/hooks/query-hooks/use-create-evaluator-config.ts @@ -0,0 +1,30 @@ +import { useMutation, useQueryClient } from "@tanstack/react-query"; + +import { api } from "@/core/api/client"; +import type { components } from "@/core/api/generated/api-types"; + +export type CreateEvaluatorConfigRequest = + components["schemas"]["CreateEvaluatorConfigRequest"]; +export type EvaluatorConfigItem = components["schemas"]["EvaluatorConfigItem"]; + +export function useCreateEvaluatorConfig() { + const queryClient = useQueryClient(); + + return useMutation({ + mutationFn: async ( + data: CreateEvaluatorConfigRequest + ): Promise => { + const { data: result, error } = await api.evaluatorConfigs.create(data); + + if (error) { + throw error; + } + + return result; + }, + onSuccess: () => { + // Invalidate evaluator configs list if we add that query later + queryClient.invalidateQueries({ queryKey: ["evaluator-configs"] }); + }, + }); +} diff --git a/ui/src/core/hooks/query-hooks/use-delete-evaluator-config.ts b/ui/src/core/hooks/query-hooks/use-delete-evaluator-config.ts new file mode 100644 index 00000000..da1c7e05 --- /dev/null +++ b/ui/src/core/hooks/query-hooks/use-delete-evaluator-config.ts @@ -0,0 +1,17 @@ +import { useMutation, useQueryClient } from "@tanstack/react-query"; + +import { api } from "@/core/api/client"; + +export function useDeleteEvaluatorConfig() { + const queryClient = useQueryClient(); + + return useMutation({ + mutationFn: async (configId: number) => { + const { error } = await api.evaluatorConfigs.delete(configId); + if (error) throw error; + }, + onSuccess: () => { + queryClient.invalidateQueries({ queryKey: ["evaluator-configs"] }); + }, + }); +} diff --git a/ui/src/core/hooks/query-hooks/use-evaluator-configs.ts b/ui/src/core/hooks/query-hooks/use-evaluator-configs.ts new file mode 100644 index 00000000..ae3030dc --- /dev/null +++ b/ui/src/core/hooks/query-hooks/use-evaluator-configs.ts @@ -0,0 +1,23 @@ +import { useQuery } from "@tanstack/react-query"; + +import { api } from "@/core/api/client"; +import type { components } from "@/core/api/generated/api-types"; + +export type ListEvaluatorConfigsResponse = + components["schemas"]["ListEvaluatorConfigsResponse"]; + +export function useEvaluatorConfigs(params?: { + cursor?: number; + limit?: number; + name?: string; + evaluator?: string; +}) { + return useQuery({ + queryKey: ["evaluator-configs", params], + queryFn: async () => { + const { data, error } = await api.evaluatorConfigs.list(params); + if (error) throw error; + return data!; + }, + }); +} diff --git a/ui/src/core/page-components/agent-detail/add-new-control-modal.tsx b/ui/src/core/page-components/agent-detail/add-new-control-modal.tsx new file mode 100644 index 00000000..4d08e9e8 --- /dev/null +++ b/ui/src/core/page-components/agent-detail/add-new-control-modal.tsx @@ -0,0 +1,393 @@ +import { + Box, + Divider, + Group, + Loader, + Modal, + Paper, + Stack, + Text, + TextInput, + Title, + Tooltip, +} from "@mantine/core"; +import { Button, Table } from "@rungalileo/jupiter-ds"; +import { + IconAlertCircle, + IconSearch, + IconSettings, + IconSparkles, + IconX, +} from "@tabler/icons-react"; +import { type ColumnDef } from "@tanstack/react-table"; +import { useMemo, useState } from "react"; + +import { ErrorBoundary } from "@/components/error-boundary"; +import type { EvaluatorInfo } from "@/core/api/types"; +import { useEvaluators } from "@/core/hooks/query-hooks/use-evaluators"; + +import { EditControlContent } from "./edit-control"; + +type EvaluatorWithId = EvaluatorInfo & { id: string }; + +/** + * Default evaluator configs for each evaluator type + * Based on backend models in agent_control_models/controls.py + */ +const DEFAULT_EVALUATOR_CONFIGS: Record> = { + regex: { + pattern: "^.*$", + }, + list: { + values: [], + logic: "any", + match_on: "match", + match_mode: "exact", + case_sensitive: false, + }, +}; + +function getDefaultConfigForEvaluator( + evaluatorId: string +): Record { + return DEFAULT_EVALUATOR_CONFIGS[evaluatorId] ?? {}; +} + +interface AddNewControlModalProps { + opened: boolean; + onClose: () => void; + agentId: string; +} + +export function AddNewControlModal({ + opened, + onClose, + agentId, +}: AddNewControlModalProps) { + const [selectedSource, setSelectedSource] = useState<"galileo" | "custom">( + "galileo" + ); + const [searchQuery, setSearchQuery] = useState(""); + const [selectedEvaluator, setSelectedEvaluator] = + useState(null); + const [editModalOpened, setEditModalOpened] = useState(false); + const { data: evaluatorsData, isLoading, error } = useEvaluators(); + + const handleAddClick = (evaluator: EvaluatorWithId) => { + setSelectedEvaluator(evaluator); + setEditModalOpened(true); + }; + + const handleEditModalClose = () => { + setEditModalOpened(false); + setSelectedEvaluator(null); + }; + + const handleEditModalSuccess = () => { + handleEditModalClose(); + onClose(); + }; + + // Transform evaluators record to array for table display + const evaluators = useMemo(() => { + if (!evaluatorsData) return []; + return Object.entries(evaluatorsData).map(([key, evaluator]) => ({ + ...evaluator, + id: key, + })); + }, [evaluatorsData]); + + const draftControl = useMemo(() => { + if (!selectedEvaluator) return null; + return { + id: 0, + name: selectedEvaluator.name, + control: { + description: selectedEvaluator.description, + enabled: true, + execution: "server" as const, + scope: { + step_types: ["llm"], + stages: ["post"] as ("post" | "pre")[], + }, + selector: { + path: "*", + }, + evaluator: { + name: selectedEvaluator.id, + config: getDefaultConfigForEvaluator(selectedEvaluator.id), + }, + action: { decision: "deny" as const }, + }, + }; + }, [selectedEvaluator]); + + const columns: ColumnDef[] = [ + { + id: "name", + header: "Name", + accessorKey: "name", + size: 80, + cell: ({ row }) => ( + + + {row.original.name} + + + ), + }, + { + id: "version", + header: "Version", + accessorKey: "version", + size: 80, + cell: ({ row }) => {row.original.version}, + }, + { + id: "description", + header: "Description", + accessorKey: "description", + size: 200, + cell: ({ row }) => ( + + + {row.original.description} + + + ), + }, + { + id: "actions", + header: "", + size: 80, + cell: ({ row }) => ( + + ), + }, + ]; + + const filteredEvaluators = + selectedSource === "galileo" + ? evaluators.filter((evaluator) => + evaluator.name.toLowerCase().includes(searchQuery.toLowerCase()) + ) + : []; + + return ( + + + {/* Header */} + + + + Control store + + + + + Browse and add controls to your agent + + + + + {/* Content */} + + {/* Left Sidebar */} + + + + + Source + + + setSelectedSource("galileo")} + w="100%" + p="xs" + radius="sm" + withBorder + bg={ + selectedSource === "galileo" + ? "var(--mantine-color-blue-0)" + : "transparent" + } + > + + + + OOB standard + + + + setSelectedSource("custom")} + w="100%" + p="xs" + radius="sm" + withBorder + bg={ + selectedSource === "custom" + ? "var(--mantine-color-blue-0)" + : "transparent" + } + > + + + + Custom + + + + + + + + + + {/* Right Content */} + + + {/* Search and Docs Link */} + + } + flex={1} + maw={250} + value={searchQuery} + onChange={(e) => setSearchQuery(e.target.value)} + /> + + Looking to add custom control?{" "} + + Check our Docs ↗ + + + + + {/* Table or Empty State */} + {selectedSource === "galileo" ? ( + isLoading ? ( + + + + ) : error ? ( + + + + Failed to load evaluators + + + ) : filteredEvaluators.length > 0 ? ( + + ) : ( + + No evaluators found + + ) + ) : ( + + + + + No custom controls yet + + + Create your first custom control to get started + + + + )} + + + + + + {/* Edit Control Modal */} + + + {draftControl && ( + + )} + + + + ); +} diff --git a/ui/src/core/page-components/agent-detail/control-store-modal.tsx b/ui/src/core/page-components/agent-detail/control-store-modal.tsx index 052e768a..30cf70bf 100644 --- a/ui/src/core/page-components/agent-detail/control-store-modal.tsx +++ b/ui/src/core/page-components/agent-detail/control-store-modal.tsx @@ -11,47 +11,26 @@ import { Title, Tooltip, } from "@mantine/core"; +import { notifications } from "@mantine/notifications"; import { Button, Table } from "@rungalileo/jupiter-ds"; import { IconAlertCircle, IconSearch, - IconSettings, - IconSparkles, + IconTrash, IconX, } from "@tabler/icons-react"; import { type ColumnDef } from "@tanstack/react-table"; import { useMemo, useState } from "react"; import { ErrorBoundary } from "@/components/error-boundary"; -import type { EvaluatorInfo } from "@/core/api/types"; -import { useEvaluators } from "@/core/hooks/query-hooks/use-evaluators"; +import type { components } from "@/core/api/generated/api-types"; +import { useDeleteEvaluatorConfig } from "@/core/hooks/query-hooks/use-delete-evaluator-config"; +import { useEvaluatorConfigs } from "@/core/hooks/query-hooks/use-evaluator-configs"; +import { AddNewControlModal } from "./add-new-control-modal"; import { EditControlContent } from "./edit-control"; -type EvaluatorWithId = EvaluatorInfo & { id: string }; - -/** - * Default evaluator configs for each evaluator type - * Based on backend models in agent_control_models/controls.py - */ -const DEFAULT_EVALUATOR_CONFIGS: Record> = { - regex: { - pattern: "^.*$", - }, - list: { - values: [], - logic: "any", - match_on: "match", - match_mode: "exact", - case_sensitive: false, - }, -}; - -function getDefaultConfigForEvaluator( - evaluatorId: string -): Record { - return DEFAULT_EVALUATOR_CONFIGS[evaluatorId] ?? {}; -} +type EvaluatorConfigItem = components["schemas"]["EvaluatorConfigItem"]; interface ControlStoreModalProps { opened: boolean; @@ -64,23 +43,22 @@ export function ControlStoreModal({ onClose, agentId, }: ControlStoreModalProps) { - const [selectedSource, setSelectedSource] = useState<"galileo" | "custom">( - "galileo" - ); const [searchQuery, setSearchQuery] = useState(""); - const [selectedEvaluator, setSelectedEvaluator] = - useState(null); + const [selectedConfig, setSelectedConfig] = + useState(null); const [editModalOpened, setEditModalOpened] = useState(false); - const { data: evaluatorsData, isLoading, error } = useEvaluators(); + const [addNewModalOpened, setAddNewModalOpened] = useState(false); + const deleteEvaluatorConfig = useDeleteEvaluatorConfig(); + const { data, isLoading, error } = useEvaluatorConfigs(); - const handleAddClick = (evaluator: EvaluatorWithId) => { - setSelectedEvaluator(evaluator); + const handleUseConfig = (config: EvaluatorConfigItem) => { + setSelectedConfig(config); setEditModalOpened(true); }; const handleEditModalClose = () => { setEditModalOpened(false); - setSelectedEvaluator(null); + setSelectedConfig(null); }; const handleEditModalSuccess = () => { @@ -88,35 +66,88 @@ export function ControlStoreModal({ onClose(); }; - // Transform evaluators record to array for table display - const evaluators = useMemo(() => { - if (!evaluatorsData) return []; - return Object.entries(evaluatorsData).map(([key, evaluator]) => ({ - ...evaluator, - id: key, - })); - }, [evaluatorsData]); + const handleDeleteConfig = async (config: EvaluatorConfigItem) => { + const shouldDelete = window.confirm( + `Delete evaluator config "${config.name}"? This cannot be undone.` + ); + if (!shouldDelete) return; + + try { + await deleteEvaluatorConfig.mutateAsync(config.id); + notifications.show({ + title: "Deleted", + message: `"${config.name}" has been deleted.`, + color: "green", + }); + } catch (deleteError) { + notifications.show({ + title: "Delete failed", + message: + deleteError instanceof Error + ? deleteError.message + : "Unable to delete evaluator config.", + color: "red", + }); + } + }; + + const filteredConfigs = useMemo(() => { + const configs = data?.evaluator_configs ?? []; + if (!searchQuery.trim()) return configs; + const query = searchQuery.toLowerCase(); + return configs.filter( + (config) => + config.name.toLowerCase().includes(query) || + config.evaluator.toLowerCase().includes(query) || + (config.description ?? "").toLowerCase().includes(query) + ); + }, [data, searchQuery]); - const columns: ColumnDef[] = [ + const draftControl = useMemo(() => { + if (!selectedConfig) return null; + return { + id: 0, + name: selectedConfig.name, + control: { + description: selectedConfig.description, + enabled: true, + execution: "server" as const, + scope: { + step_types: ["llm"], + stages: ["post"] as ("post" | "pre")[], + }, + selector: { + path: "*", + }, + evaluator: { + name: selectedConfig.evaluator, + config: selectedConfig.config, + }, + action: { decision: "deny" as const }, + }, + }; + }, [selectedConfig]); + + const columns: ColumnDef[] = [ { id: "name", header: "Name", accessorKey: "name", - size: 80, + size: 120, cell: ({ row }) => ( - - + + {row.original.name} ), }, { - id: "version", - header: "Version", - accessorKey: "version", - size: 80, - cell: ({ row }) => {row.original.version}, + id: "evaluator", + header: "Evaluator", + accessorKey: "evaluator", + size: 120, + cell: ({ row }) => {row.original.evaluator}, }, { id: "description", @@ -125,8 +156,8 @@ export function ControlStoreModal({ size: 200, cell: ({ row }) => ( - - {row.original.description} + + {row.original.description || "—"} ), @@ -134,27 +165,33 @@ export function ControlStoreModal({ { id: "actions", header: "", - size: 80, + size: 180, cell: ({ row }) => ( - + + + + + + ), }, ]; - const filteredEvaluators = - selectedSource === "galileo" - ? evaluators.filter((evaluator) => - evaluator.name.toLowerCase().includes(searchQuery.toLowerCase()) - ) - : []; - return ( - + {/* Header */} Control store - + + + + - Browse and add controls to your agent + Choose a saved evaluator config to create a control {/* Content */} - - {/* Left Sidebar */} - - - - - Source - - - setSelectedSource("galileo")} - w='100%' - p='xs' - radius='sm' - withBorder - bg={ - selectedSource === "galileo" - ? "var(--mantine-color-blue-0)" - : "transparent" - } - > - - - - OOB standard - - - - setSelectedSource("custom")} - w='100%' - p='xs' - radius='sm' - withBorder - bg={ - selectedSource === "custom" - ? "var(--mantine-color-blue-0)" - : "transparent" - } - > - - - - Custom - - - - - - - - - - {/* Right Content */} - - - {/* Search and Docs Link */} - - } - flex={1} - maw={250} - value={searchQuery} - onChange={(e) => setSearchQuery(e.target.value)} - /> - - Looking to add custom control?{" "} - - Check our Docs ↗ - - - + + + + } + flex={1} + maw={250} + value={searchQuery} + onChange={(e) => setSearchQuery(e.target.value)} + /> + - {/* Table or Empty State */} - {selectedSource === "galileo" ? ( - isLoading ? ( - - - - ) : error ? ( - - - - Failed to load evaluators - - - ) : filteredEvaluators.length > 0 ? ( -
+ + + ) : error ? ( + + + - ) : ( - - No evaluators found - - ) - ) : ( - - - - - No custom controls yet - - - Create your first custom control to get started - - - - )} - - - + Failed to load evaluator configs + + + ) : filteredConfigs.length > 0 ? ( +
+ ) : ( + + No evaluator configs found + + )} + + {/* Edit Control Modal */} @@ -352,37 +294,23 @@ export function ControlStoreModal({ }} > - {selectedEvaluator && ( + {draftControl && ( )} + + setAddNewModalOpened(false)} + agentId={agentId} + /> ); } diff --git a/ui/src/core/page-components/agent-detail/edit-control/edit-control-content.tsx b/ui/src/core/page-components/agent-detail/edit-control/edit-control-content.tsx index 58db702a..7cc11fcb 100644 --- a/ui/src/core/page-components/agent-detail/edit-control/edit-control-content.tsx +++ b/ui/src/core/page-components/agent-detail/edit-control/edit-control-content.tsx @@ -1,5 +1,6 @@ import { Anchor, + Box, Divider, Grid, Group, @@ -12,7 +13,7 @@ import { } from "@mantine/core"; import { useForm } from "@mantine/form"; import { Button } from "@rungalileo/jupiter-ds"; -import { IconExternalLink } from "@tabler/icons-react"; +import { IconBookmark, IconExternalLink } from "@tabler/icons-react"; import { useEffect, useMemo, useRef, useState } from "react"; import { isApiError } from "@/core/api/errors"; @@ -24,6 +25,7 @@ import { ApiErrorAlert } from "./api-error-alert"; import { ControlDefinitionForm } from "./control-definition-form"; import { EvaluatorJsonView } from "./evaluator-json-view"; import { getEvaluator } from "./evaluators"; +import { SaveEvaluatorTemplateModal } from "./save-evaluator-template-modal"; import type { ConfigViewMode, ControlDefinitionFormValues, @@ -32,7 +34,7 @@ import type { } from "./types"; import { applyApiErrorsToForms } from "./utils"; -const EVALUATOR_CONFIG_HEIGHT = 400; +const EVALUATOR_CONFIG_HEIGHT = 360; // Reduced to make room for Save as Template button export interface EditControlContentProps { /** The control to edit/create template */ @@ -76,6 +78,9 @@ export const EditControlContent = ({ ? addControlToAgent.isPending : updateControl.isPending; + // Save as Template modal state + const [templateModalOpened, setTemplateModalOpened] = useState(false); + // Track which evaluator the evaluator form has been initialized for const formInitializedForEvaluator = useRef(""); @@ -187,6 +192,11 @@ export const EditControlContent = ({ setUnmappedErrors([]); }, [evaluatorId]); + const validateEvaluatorForm = () => { + const validation = evaluatorForm.validate(); + return !validation.hasErrors; + }; + // Load control data into forms useEffect(() => { if (control && evaluator) { @@ -323,8 +333,28 @@ export const EditControlContent = ({ // Render the evaluator's form component const FormComponent = evaluator?.FormComponent; + const saveTemplateAction = ( + + ); + return ( -
+ <> + - {configViewMode === "form" && ( - - - {FormComponent ? ( - - ) : ( - - No form available for this evaluator. Use JSON view to - configure. - - )} - - - )} - - {configViewMode === "json" && ( - - )} + + {configViewMode === "form" && ( + + + {FormComponent ? ( + + ) : ( + + No form available for this evaluator. Use JSON view to + configure. + + )} + + {saveTemplateAction} + + )} + + {configViewMode === "json" && ( + + )} + @@ -432,6 +467,17 @@ export const EditControlContent = ({ {isCreating ? "Create" : "Save"} - + + + setTemplateModalOpened(false)} + evaluatorId={evaluatorId} + configViewMode={configViewMode} + rawJsonText={rawJsonText} + getEvaluatorConfig={getEvaluatorConfig} + validateEvaluatorForm={validateEvaluatorForm} + /> + ); }; diff --git a/ui/src/core/page-components/agent-detail/edit-control/evaluator-json-view.tsx b/ui/src/core/page-components/agent-detail/edit-control/evaluator-json-view.tsx index d80bf359..ee78c034 100644 --- a/ui/src/core/page-components/agent-detail/edit-control/evaluator-json-view.tsx +++ b/ui/src/core/page-components/agent-detail/edit-control/evaluator-json-view.tsx @@ -3,7 +3,6 @@ import { // Group, ScrollArea, // SegmentedControl, - Stack, Textarea, } from "@mantine/core"; @@ -11,7 +10,7 @@ import { JsonEditor } from "@/components/json-editor"; import type { EvaluatorJsonViewProps } from "./types"; -const JSON_VIEW_HEIGHT = 400; +const DEFAULT_HEIGHT = 400; export const EvaluatorJsonView = ({ config, @@ -21,29 +20,31 @@ export const EvaluatorJsonView = ({ rawJsonText, onRawJsonTextChange, rawJsonError, + height = DEFAULT_HEIGHT, + action, }: EvaluatorJsonViewProps) => { - return ( - - {/* TODO: Re-enable tree/raw toggle when needed */} - {/* - - */} + // TODO: Re-enable tree/raw toggle when needed + // + // + // - {jsonViewMode === "tree" ? ( - - + if (jsonViewMode === "tree") { + return ( + + + - ) : ( -