WIP: Data Model / Config management

.github/workflows/lib-vedana-core.yml

@@ -38,12 +38,14 @@ jobs:
           uv sync --all-groups
 
       - env:
+            CONFIG_PLANE_DB_CONN_URI: 'sqlite:///:memory:'
             DB_CONN_URI: 'sqlite:///:memory:'
             EMBEDDINGS_DIM: '1024'
             GRIST_API_KEY: e30d2f274a538c05fecd14510887f8a3b7eab718
             GRIST_DATA_DOC_ID: qAxQ1gcBKcW7
             GRIST_DATA_MODEL_DOC_ID: wEEmPY3UiwMD
             GRIST_SERVER_URL: http://0.0.0.0:8484
+            JIMS_DB_CONN_URI: 'sqlite:///:memory:'
             MEMGRAPH_PWD: ''
             MEMGRAPH_URI: bolt://localhost:7687
             MEMGRAPH_USER: ''

.github/workflows/lib-vedana-etl.yml

@@ -48,6 +48,7 @@ jobs:
 
 
       - env:
+            CONFIG_PLANE_DB_CONN_URI: 'sqlite:///:memory:'
             DB_CONN_URI: 'sqlite:///:memory:'
             EMBEDDINGS_DIM: '1536'
             GRIST_API_KEY: e30d2f274a538c05fecd14510887f8a3b7eab718

apps/vedana/.env.example

@@ -13,6 +13,9 @@ MEMGRAPH_PWD="modular-current-bonjour-senior-neptune-8618"
 # Note: by default Vedana handles all SQL in one database.
 JIMS_DB_CONN_URI="postgresql://postgres:postgres@db:5432"
 
+# Config Plane - system configuration management between deployments
+CONFIG_PLANE_DB_CONN_URI="postgresql://postgres:postgres@db:5432"
+
 # LLM Setup
 # We use LiteLLM to handle different vendors.
 # if you want to run Google/VertexAI models:

libs/jims-backoffice/pyproject.toml

@@ -15,7 +15,7 @@ dependencies = [
     "sqlalchemy>=2.0.41",
     "psycopg2-binary>=2.9.10",
     "uvicorn>=0.29.0",
-    "pydantic==2.9.2",
+    "pydantic>=2.9.2",
     "jims-core>=0.5.1",
     "python-multipart>=0.0.18",
     "asyncpg>=0.31.0",

libs/vedana-backoffice/src/vedana_backoffice/components/ui_chat.py

@@ -1,5 +1,8 @@
 import reflex as rx
 
+from vedana_backoffice.states.data_model import DataModelState
+from vedana_backoffice.ui import table_accordion
+
 
 def render_message_bubble(
     msg: dict,
@@ -79,6 +82,92 @@ def render_message_bubble(
                         width="100%",
                     ),
                 ),
+                rx.cond(
+                    msg.get("dm_snapshot_id"),
+                    rx.hstack(
+                        rx.text("Data Model ID: "),
+                        rx.text(msg.get("dm_snapshot_id")),
+                        rx.popover.root(
+                            rx.popover.trigger(
+                                rx.button(
+                                    "Open",
+                                    size="1",
+                                    variant="soft",
+                                    on_click=DataModelState.open_quick_view(snapshot_id=msg.get("dm_snapshot_id")),  # type: ignore[arg-type,call-arg,func-returns-value]
+                                )
+                            ),
+                            rx.popover.content(
+                                rx.vstack(
+                                    rx.hstack(
+                                        rx.text("Snapshot", weight="medium"),
+                                        rx.spacer(),
+                                        rx.text(DataModelState.quick_view_snapshot_id, color="gray"),
+                                        width="100%",
+                                    ),
+                                    rx.cond(
+                                        DataModelState.quick_view_error_message != "",
+                                        rx.callout(
+                                            DataModelState.quick_view_error_message,
+                                            icon="triangle_alert",
+                                            color_scheme="red",
+                                        ),
+                                        rx.fragment(),
+                                    ),
+                                    rx.cond(
+                                        DataModelState.quick_view_is_loading,
+                                        rx.center(rx.spinner(size="3"), height="200px"),
+                                        table_accordion(DataModelState.quick_view_tables),  # type: ignore[arg-type]
+                                    ),
+                                    spacing="3",
+                                    width="100%",
+                                ),
+                                style={"maxWidth": "90vw", "maxHeight": "80vh", "overflow": "auto"},
+                            ),
+                        ),
+                        rx.popover.root(
+                            rx.popover.trigger(
+                                rx.button(
+                                    "Show diff",
+                                    size="1",
+                                    variant="soft",
+                                    on_click=DataModelState.open_quick_diff(
+                                        snapshot_id=msg.get("dm_snapshot_id"),
+                                        compare_branch=DataModelState.prod_branch,
+                                    ),  # type: ignore[arg-type,call-arg,func-returns-value]
+                                )
+                            ),
+                            rx.popover.content(
+                                rx.vstack(
+                                    rx.hstack(
+                                        rx.text("Diff vs prod", weight="medium"),
+                                        rx.spacer(),
+                                        rx.text(DataModelState.quick_diff_snapshot_id, color="gray"),
+                                        width="100%",
+                                    ),
+                                    rx.cond(
+                                        DataModelState.quick_diff_error_message != "",
+                                        rx.callout(
+                                            DataModelState.quick_diff_error_message,
+                                            icon="triangle_alert",
+                                            color_scheme="red",
+                                        ),
+                                        rx.fragment(),
+                                    ),
+                                    rx.cond(
+                                        DataModelState.quick_diff_is_loading,
+                                        rx.center(rx.spinner(size="3"), height="200px"),
+                                        table_accordion(DataModelState.quick_diff_tables),  # type: ignore[arg-type]
+                                    ),
+                                    spacing="3",
+                                    width="100%",
+                                ),
+                                style={"maxWidth": "90vw", "maxHeight": "80vh", "overflow": "auto"},
+                            ),
+                        ),
+                        spacing="2",
+                        align="center",
+                    ),
+                ),
                 spacing="2",
                 width="100%",
             ),

libs/vedana-backoffice/src/vedana_backoffice/pages/chat.py

@@ -3,6 +3,7 @@
 from vedana_backoffice.components.ui_chat import render_message_bubble
 from vedana_backoffice.states.chat import ChatState
 from vedana_backoffice.ui import app_header
+from vedana_core.settings import settings as core_settings
 
 
 def _message_row(msg: dict) -> rx.Component:
@@ -89,6 +90,19 @@ def page() -> rx.Component:
                                     align="center",
                                     width="100%",
                                 ),
+                                rx.hstack(
+                                    rx.text(f"Branch: {ChatState.data_model_branch}", color="gray"),
+                                    rx.spacer(),
+                                    rx.text(
+                                        rx.cond(
+                                            ChatState.data_model_snapshot_id != "",
+                                            f"Snapshot: {ChatState.data_model_snapshot_id}",
+                                            "Snapshot: —",
+                                        ),
+                                        color="gray",
+                                    ),
+                                    width="100%",
+                                ),
                                 rx.scroll_area(
                                     rx.markdown(ChatState.data_model_text),  # type: ignore[operator]
                                     type="always",
@@ -101,6 +115,21 @@ def page() -> rx.Component:
                             max_width="70vw",
                         ),
                     ),
+                    rx.select(
+                        items=[
+                            core_settings.config_plane_dev_branch,
+                            core_settings.config_plane_prod_branch,
+                        ],
+                        value=ChatState.data_model_branch,
+                        on_change=ChatState.set_data_model_branch,
+                        width="10em",
+                    ),
+                    rx.input(
+                        placeholder="Snapshot id (optional)",
+                        value=ChatState.data_model_snapshot_input,
+                        on_change=ChatState.set_data_model_snapshot_input,
+                        width="14em",
+                    ),
                     rx.checkbox(
                         "Filter Data Model",
                         checked=ChatState.enable_dm_filtering,
@@ -147,11 +176,11 @@ def page() -> rx.Component:
                             ChatState.model_selection_allowed,
                             rx.hstack(
                                 rx.select(
-                                items=["openai", "openrouter"],
-                                value=ChatState.provider,
-                                on_change=ChatState.set_provider,
-                                width="10em",
-                                placeholder="Provider",
+                                    items=["openai", "openrouter"],
+                                    value=ChatState.provider,
+                                    on_change=ChatState.set_provider,
+                                    width="10em",
+                                    placeholder="Provider",
                                 ),
                                 rx.cond(
                                     ChatState.provider == "openrouter",
@@ -183,7 +212,6 @@ def page() -> rx.Component:
                         width="100%",
                     ),
                     on_submit=ChatState.send,
-                    width="100%",
                 ),
                 spacing="2",
                 width="100%",

libs/vedana-backoffice/src/vedana_backoffice/pages/data_model.py

@@ -0,0 +1,124 @@
+import reflex as rx
+
+from vedana_backoffice.states.data_model import DataModelState
+from vedana_backoffice.ui import app_header, table_accordion
+
+
+def _branch_row(branch: rx.Var, snapshot_id: rx.Var) -> rx.Component:
+    return rx.hstack(
+        rx.text(branch, color="gray"),
+        rx.spacer(),
+        rx.text(
+            rx.cond(
+                snapshot_id,
+                f"Snapshot: {snapshot_id}",
+                "Snapshot: —",
+            ),
+            color="gray",
+        ),
+        spacing="3",
+        width="100%",
+    )
+
+
+def page() -> rx.Component:
+    return rx.box(
+        app_header(),
+        rx.vstack(
+            rx.hstack(
+                rx.heading("Data Model", size="4"),
+                rx.spacer(),
+                rx.button(
+                    "Refresh status",
+                    size="1",
+                    variant="soft",
+                    on_click=DataModelState.load_status,
+                    loading=DataModelState.is_loading,
+                ),
+                width="100%",
+                align="center",
+            ),
+            rx.heading("Branch status", size="3"),
+            _branch_row(DataModelState.dev_branch, DataModelState.dev_snapshot_id),  # type: ignore[arg-type]
+            _branch_row(DataModelState.prod_branch, DataModelState.prod_snapshot_id),  # type: ignore[arg-type]
+            rx.cond(
+                DataModelState.error_message != "",
+                rx.callout(DataModelState.error_message, icon="triangle_alert", color_scheme="red"),
+                rx.fragment(),
+            ),
+            rx.hstack(
+                rx.button(
+                    "Sync Prod ← Dev",
+                    size="1",
+                    on_click=DataModelState.sync_prod_with_dev,
+                    color_scheme="green",
+                ),
+                spacing="3",
+            ),
+            rx.heading("Diff between branches", size="3"),
+            rx.hstack(
+                rx.select(
+                    items=[DataModelState.dev_branch, DataModelState.prod_branch],
+                    value=DataModelState.diff_branch_left,
+                    on_change=DataModelState.set_diff_branch_left,
+                    width="10em",
+                ),
+                rx.text("vs", color="gray"),
+                rx.select(
+                    items=[DataModelState.dev_branch, DataModelState.prod_branch],
+                    value=DataModelState.diff_branch_right,
+                    on_change=DataModelState.set_diff_branch_right,
+                    width="10em",
+                ),
+                rx.button(
+                    "Load diff",
+                    size="1",
+                    on_click=DataModelState.load_diff,
+                    loading=DataModelState.diff_is_loading,
+                ),
+                spacing="3",
+                align="center",
+                width="100%",
+            ),
+            rx.cond(
+                DataModelState.diff_error_message != "",
+                rx.callout(DataModelState.diff_error_message, icon="triangle_alert", color_scheme="red"),
+                rx.fragment(),
+            ),
+            table_accordion(DataModelState.diff_tables),  # type: ignore[arg-type]
+            rx.heading("Browse snapshots", size="3"),
+            rx.hstack(
+                rx.select(
+                    items=[DataModelState.dev_branch, DataModelState.prod_branch],
+                    value=DataModelState.view_branch,
+                    on_change=DataModelState.set_view_branch,
+                    width="10em",
+                ),
+                rx.input(
+                    placeholder="Snapshot id (optional)",
+                    value=DataModelState.view_snapshot_input,
+                    on_change=DataModelState.set_view_snapshot_input,
+                    width="14em",
+                ),
+                rx.button(
+                    "Load snapshot",
+                    size="1",
+                    on_click=DataModelState.load_view_snapshot,
+                    loading=DataModelState.view_is_loading,
+                ),
+                spacing="3",
+                align="center",
+                width="100%",
+            ),
+            rx.cond(
+                DataModelState.view_error_message != "",
+                rx.callout(DataModelState.view_error_message, icon="triangle_alert", color_scheme="red"),
+                rx.fragment(),
+            ),
+            table_accordion(DataModelState.view_tables),  # type: ignore[arg-type]
+            spacing="4",
+            padding="1.5em",
+            width="100%",
+        ),
+        width="100%",
+    )

libs/vedana-backoffice/src/vedana_backoffice/pages/eval.py

@@ -4,6 +4,7 @@
 from vedana_backoffice.states.eval import EvalState, RunSummary
 from vedana_backoffice.states.chat import ChatState
 from vedana_backoffice.ui import app_header
+from vedana_core.settings import settings as core_settings
 
 
 def _selection_and_actions() -> rx.Component:
@@ -199,6 +200,25 @@ def _pipeline_card() -> rx.Component:
             rx.vstack(
                 rx.box(
                     rx.text("Data model", weight="medium"),
+                    rx.hstack(
+                        rx.select(
+                            items=[
+                                core_settings.config_plane_dev_branch,
+                                core_settings.config_plane_prod_branch,
+                            ],
+                            value=EvalState.dm_branch,
+                            on_change=EvalState.set_dm_branch,
+                            width="50%",
+                        ),
+                        rx.input(
+                            placeholder="Snapshot id (optional)",
+                            value=EvalState.dm_snapshot_input,
+                            on_change=EvalState.set_dm_snapshot_input,
+                            width="50%",
+                        ),
+                        width="100%",
+                        padding_y="0.5em"
+                    ),
                     rx.button(
                         "View Data Model",
                         variant="soft",
@@ -895,7 +915,17 @@ def _data_model_dialog() -> rx.Component:
         rx.dialog.content(
             rx.dialog.title("Data Model"),
             rx.vstack(
-                rx.text(f"Model ID: {EvalState.dm_id}", size="2", color="gray"),
+                rx.text(f"Branch: {EvalState.dm_branch}", size="2", color="gray"),
+                rx.text(
+                    rx.cond(
+                        EvalState.dm_snapshot_id != "",
+                        f"Snapshot: {EvalState.dm_snapshot_id}",
+                        "Snapshot: —",
+                    ),
+                    size="2",
+                    color="gray",
+                ),
+                rx.text(f"Model ID: {EvalState.dm_id}", size="2", color="gray"),  # todo is dm_id still relevant?
                 rx.box(
                     rx.text(
                         rx.cond(