diff --git a/.fides/db_dataset.yml b/.fides/db_dataset.yml index 573599401e3..5522564b0f9 100644 --- a/.fides/db_dataset.yml +++ b/.fides/db_dataset.yml @@ -1163,6 +1163,24 @@ dataset: - name: enabled_actions description: 'The privacy actions that are enabled for this connection' data_categories: [system.operations] + - name: connection_config_saas_history + fields: + - name: id + data_categories: [system.operations] + - name: created_at + data_categories: [system.operations] + - name: updated_at + data_categories: [system.operations] + - name: connection_config_id + data_categories: [system.operations] + - name: connection_key + data_categories: [system.operations] + - name: version + data_categories: [system.operations] + - name: config + data_categories: [system.operations] + - name: dataset + data_categories: [system.operations] - name: consent description: 'A database table used to map consent preference to identities' data_categories: [] @@ -3841,6 +3859,24 @@ dataset: data_categories: [user.unique_id] - name: is_default data_categories: [system.operations] + - name: saas_config_version + fields: + - name: id + data_categories: [system.operations] + - name: created_at + data_categories: [system.operations] + - name: updated_at + data_categories: [system.operations] + - name: connector_type + data_categories: [system.operations] + - name: version + data_categories: [system.operations] + - name: config + data_categories: [system.operations] + - name: dataset + data_categories: [system.operations] + - name: is_custom + data_categories: [system.operations] - name: saas_template_dataset fields: - name: id diff --git a/changelog/7688-saas-config-version-history.yaml b/changelog/7688-saas-config-version-history.yaml new file mode 100644 index 00000000000..8168bc3afcc --- /dev/null +++ b/changelog/7688-saas-config-version-history.yaml @@ -0,0 +1,4 @@ +type: Added +description: Added SaaS config version history tracking — stores connector template versions on seed and custom template upload, with API endpoints to list versions and retrieve config/dataset snapshots by version +pr: 7688 +labels: ["db-migration"] diff --git a/src/fides/api/alembic/migrations/versions/a1ca9ddf3c3c_add_saas_version_to_executionlog.py b/src/fides/api/alembic/migrations/versions/a1ca9ddf3c3c_add_saas_version_to_executionlog.py index c49018aada6..5364ff7f544 100644 --- a/src/fides/api/alembic/migrations/versions/a1ca9ddf3c3c_add_saas_version_to_executionlog.py +++ b/src/fides/api/alembic/migrations/versions/a1ca9ddf3c3c_add_saas_version_to_executionlog.py @@ -5,7 +5,7 @@ Only populated for SaaS connectors (connection_type = 'saas'); null for all others. Revision ID: a1ca9ddf3c3c -Revises: 94273d7e8319 +Revises: ad1bb600715b Create Date: 2026-03-11 """ @@ -16,6 +16,7 @@ # revision identifiers, used by Alembic. revision = "a1ca9ddf3c3c" down_revision = "ad1bb600715b" + branch_labels = None depends_on = None diff --git a/src/fides/api/alembic/migrations/versions/xx_2026_03_13_0000_c3e5f7a9b1d2_add_saas_config_version_table.py b/src/fides/api/alembic/migrations/versions/xx_2026_03_13_0000_c3e5f7a9b1d2_add_saas_config_version_table.py new file mode 100644 index 00000000000..e8d3cb35bda --- /dev/null +++ b/src/fides/api/alembic/migrations/versions/xx_2026_03_13_0000_c3e5f7a9b1d2_add_saas_config_version_table.py @@ -0,0 +1,57 @@ +"""add saas_config_version table + +Stores a snapshot of each SaaS integration config and dataset per +(connector_type, version) pair. Rows are written on startup for bundled +OOB connectors, on custom template upload/update, and on direct SaaS config +PATCH. Rows are never deleted so that execution logs can always resolve the +config/dataset that was active when a DSR ran. + +Revision ID: c3e5f7a9b1d2 +Revises: a1ca9ddf3c3c +Create Date: 2026-03-13 + +""" + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = "c3e5f7a9b1d2" +down_revision = "a1ca9ddf3c3c" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "saas_config_version", + sa.Column("id", sa.String(length=255), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=True), + sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=True), + sa.Column("connector_type", sa.String(), nullable=False), + sa.Column("version", sa.String(), nullable=False), + sa.Column("config", postgresql.JSONB(astext_type=sa.Text()), nullable=False), + sa.Column("dataset", postgresql.JSONB(astext_type=sa.Text()), nullable=True), + sa.Column("is_custom", sa.Boolean(), nullable=False), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("connector_type", "version", "is_custom", name="uq_saas_config_version"), + ) + op.create_index( + op.f("ix_saas_config_version_connector_type"), + "saas_config_version", + ["connector_type"], + unique=False, + ) + op.create_index( + op.f("ix_saas_config_version_id"), + "saas_config_version", + ["id"], + unique=False, + ) + + +def downgrade() -> None: + op.drop_index(op.f("ix_saas_config_version_id"), table_name="saas_config_version") + op.drop_index(op.f("ix_saas_config_version_connector_type"), table_name="saas_config_version") + op.drop_table("saas_config_version") diff --git a/src/fides/api/alembic/migrations/versions/xx_2026_03_17_0000_d4e6f8a0b2c3_add_connection_config_saas_history_table.py b/src/fides/api/alembic/migrations/versions/xx_2026_03_17_0000_d4e6f8a0b2c3_add_connection_config_saas_history_table.py new file mode 100644 index 00000000000..644a07c2e41 --- /dev/null +++ b/src/fides/api/alembic/migrations/versions/xx_2026_03_17_0000_d4e6f8a0b2c3_add_connection_config_saas_history_table.py @@ -0,0 +1,81 @@ +"""add connection_config_saas_history table + +Stores a per-connection snapshot of a SaaS config (and associated datasets) +each time ConnectionConfig.update_saas_config() is called. Unlike the +template-level saas_config_version table, this table is append-only and +scoped to an individual connection instance, so divergent configs are +preserved correctly. + +Revision ID: d4e6f8a0b2c3 +Revises: c3e5f7a9b1d2 +Create Date: 2026-03-17 + +""" + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = "d4e6f8a0b2c3" +down_revision = "c3e5f7a9b1d2" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "connection_config_saas_history", + sa.Column("id", sa.String(length=255), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=True, + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=True, + ), + sa.Column("connection_config_id", sa.String(length=255), nullable=True), + sa.Column("connection_key", sa.String(), nullable=False), + sa.Column("version", sa.String(), nullable=False), + sa.Column( + "config", postgresql.JSONB(astext_type=sa.Text()), nullable=False + ), + sa.Column( + "dataset", postgresql.JSONB(astext_type=sa.Text()), nullable=True + ), + sa.ForeignKeyConstraint( + ["connection_config_id"], + ["connectionconfig.id"], + ondelete="SET NULL", + ), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index( + op.f("ix_connection_config_saas_history_id"), + "connection_config_saas_history", + ["id"], + unique=False, + ) + op.create_index( + op.f("ix_connection_config_saas_history_connection_config_id"), + "connection_config_saas_history", + ["connection_config_id"], + unique=False, + ) + + +def downgrade() -> None: + op.drop_index( + op.f("ix_connection_config_saas_history_connection_config_id"), + table_name="connection_config_saas_history", + ) + op.drop_index( + op.f("ix_connection_config_saas_history_id"), + table_name="connection_config_saas_history", + ) + op.drop_table("connection_config_saas_history") diff --git a/src/fides/api/db/base.py b/src/fides/api/db/base.py index 8b6add82bb6..1865633e98f 100644 --- a/src/fides/api/db/base.py +++ b/src/fides/api/db/base.py @@ -10,6 +10,7 @@ from fides.api.models.chat_config import ChatConfig from fides.api.models.client import ClientDetail from fides.api.models.comment import Comment, CommentReference +from fides.api.models.connection_config_saas_history import ConnectionConfigSaaSHistory from fides.api.models.connection_oauth_credentials import OAuthConfig from fides.api.models.connectionconfig import ConnectionConfig from fides.api.models.consent_automation import ConsentAutomation @@ -91,6 +92,7 @@ Property, ) from fides.api.models.questionnaire import ChatMessage, Questionnaire +from fides.api.models.saas_config_version import SaaSConfigVersion from fides.api.models.saas_template_dataset import SaasTemplateDataset from fides.api.models.storage import StorageConfig from fides.api.models.system_compass_sync import SystemCompassSync diff --git a/src/fides/api/db/seed.py b/src/fides/api/db/seed.py index 85277fab09a..7ba309d85fe 100644 --- a/src/fides/api/db/seed.py +++ b/src/fides/api/db/seed.py @@ -21,6 +21,7 @@ from fides.api.models.fides_user import FidesUser from fides.api.models.fides_user_permissions import FidesUserPermissions from fides.api.models.policy import Policy, Rule, RuleTarget +from fides.api.models.saas_config_version import SaaSConfigVersion from fides.api.models.sql_models import ( # type: ignore[attr-defined] Dataset, Organization, @@ -322,6 +323,47 @@ def load_default_dsr_policies(session: Session) -> None: log.info("All default policies & rules created") +def sync_oob_saas_config_versions(session: Session) -> None: + """ + Upserts a SaaSConfigVersion row for every bundled (OOB) SaaS connector + template currently loaded in memory. + + Called on startup so the table is bootstrapped on first deploy and + picks up new template versions automatically on each upgrade. + Rows are immutable once written, so this is safe to call repeatedly. + """ + # Import here to avoid circular imports at module load time + from fides.api.schemas.saas.saas_config import ( + SaaSConfig, # pylint: disable=import-outside-toplevel + ) + from fides.api.service.connectors.saas.connector_registry_service import ( # pylint: disable=import-outside-toplevel + FileConnectorTemplateLoader, + ) + from fides.api.util.saas_util import ( # pylint: disable=import-outside-toplevel + load_config_from_string, + load_dataset_from_string, + ) + + templates = FileConnectorTemplateLoader.get_connector_templates() + for connector_type, template in templates.items(): + try: + saas_config = SaaSConfig(**load_config_from_string(template.config)) + dataset = load_dataset_from_string(template.dataset) + SaaSConfigVersion.upsert( + db=session, + connector_type=connector_type, + version=saas_config.version, + config=saas_config.model_dump(mode="json"), + dataset=dataset, + is_custom=False, + ) + except Exception: # pylint: disable=broad-except + log.exception( + "Failed to sync SaaSConfigVersion for OOB connector '{}'", + connector_type, + ) + + def load_default_resources(session: Session) -> None: """ Seed the database with default resources that the application @@ -330,6 +372,7 @@ def load_default_resources(session: Session) -> None: load_default_organization(session) load_default_taxonomy(session) load_default_dsr_policies(session) + sync_oob_saas_config_versions(session) async def load_samples(async_session: AsyncSession) -> None: diff --git a/src/fides/api/models/connection_config_saas_history.py b/src/fides/api/models/connection_config_saas_history.py new file mode 100644 index 00000000000..97a45d58123 --- /dev/null +++ b/src/fides/api/models/connection_config_saas_history.py @@ -0,0 +1,71 @@ +from typing import Any, Dict, List, Optional + +from sqlalchemy import Column, ForeignKey, String +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.ext.declarative import declared_attr +from sqlalchemy.orm import Session + +from fides.api.db.base_class import Base + + +class ConnectionConfigSaaSHistory(Base): + """ + Append-only snapshot of a connection's SaaS config taken each time + ConnectionConfig.update_saas_config() is called. + + Unlike SaaSConfigVersion (which stores one row per connector_type/version), + this table is scoped to an individual ConnectionConfig instance. It captures + divergent configs — e.g. when a connection is individually PATCHed to a + version that differs from the shared template. + + connection_key is denormalized so that history is still queryable if the + parent ConnectionConfig row is deleted (FK uses ON DELETE SET NULL). + """ + + @declared_attr + def __tablename__(self) -> str: + return "connection_config_saas_history" + + connection_config_id = Column( + String, + ForeignKey("connectionconfig.id", ondelete="SET NULL"), + nullable=True, + index=True, + ) + connection_key = Column(String, nullable=False) + version = Column(String, nullable=False) + config = Column(JSONB, nullable=False) + dataset = Column(JSONB, nullable=True) + + def __repr__(self) -> str: + return ( + f"" + ) + + @classmethod + def create_snapshot( + cls, + db: Session, + connection_config_id: str, + connection_key: str, + version: str, + config: Dict[str, Any], + datasets: Optional[List[Dict[str, Any]]] = None, + ) -> "ConnectionConfigSaaSHistory": + """ + Appends a new history row. Always creates a new row — no upsert logic — + so every write is preserved as a distinct audit entry. + """ + return cls.create( + db=db, + data={ + "connection_config_id": connection_config_id, + "connection_key": connection_key, + "version": version, + "config": config, + "dataset": datasets or None, + }, + ) diff --git a/src/fides/api/models/connectionconfig.py b/src/fides/api/models/connectionconfig.py index 361e2c63482..3b1bcda0a82 100644 --- a/src/fides/api/models/connectionconfig.py +++ b/src/fides/api/models/connectionconfig.py @@ -358,6 +358,11 @@ def update_saas_config( Updates the SaaS config and initializes any empty secrets with connector param default values if available (will not override any existing secrets) """ + from fides.api.models.connection_config_saas_history import ( + ConnectionConfigSaaSHistory, + ) + from fides.api.models.datasetconfig import DatasetConfig + default_secrets = { connector_param.name: connector_param.default_value for connector_param in saas_config.connector_params @@ -366,6 +371,23 @@ def update_saas_config( updated_secrets = {**default_secrets, **(self.secrets or {})} self.secrets = updated_secrets self.saas_config = saas_config.model_dump(mode="json") + + datasets = [ + dc.ctl_dataset.dict() + for dc in db.query(DatasetConfig) + .filter(DatasetConfig.connection_config_id == self.id) + .all() + if dc.ctl_dataset + ] + ConnectionConfigSaaSHistory.create_snapshot( + db=db, + connection_config_id=self.id, + connection_key=self.key, + version=saas_config.version, + config=self.saas_config, + datasets=datasets if datasets else None, + ) + self.save(db) def update_test_status( diff --git a/src/fides/api/models/saas_config_version.py b/src/fides/api/models/saas_config_version.py new file mode 100644 index 00000000000..3f54ae02db9 --- /dev/null +++ b/src/fides/api/models/saas_config_version.py @@ -0,0 +1,88 @@ +from typing import Any, Dict, Optional + +from sqlalchemy import Boolean, Column, String, UniqueConstraint +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.ext.declarative import declared_attr +from sqlalchemy.orm import Session + +from fides.api.db.base_class import Base + + +class SaaSConfigVersion(Base): + """ + Stores a snapshot of each SaaS integration config and dataset per version. + + A new row is upserted whenever a SaaS integration version is seen for the + first time — on startup (for bundled OOB connectors), on custom template + upload/update, or on direct SaaS config PATCH. Rows are never deleted so + that execution logs can always resolve the config/dataset that was active + when a DSR ran. + """ + + @declared_attr + def __tablename__(self) -> str: + return "saas_config_version" + + __table_args__ = ( + # is_custom is part of the key: the same version string can exist once + # as an OOB template and once as a custom override without conflict. + UniqueConstraint( + "connector_type", "version", "is_custom", name="uq_saas_config_version" + ), + ) + + connector_type = Column(String, nullable=False, index=True) + version = Column(String, nullable=False) + config = Column(JSONB, nullable=False) + dataset = Column(JSONB, nullable=True) + is_custom = Column(Boolean, nullable=False, default=False) + + def __repr__(self) -> str: + return f"" + + @classmethod + def upsert( + cls, + db: Session, + connector_type: str, + version: str, + config: Dict[str, Any], + dataset: Optional[Dict[str, Any]] = None, + is_custom: bool = False, + ) -> "SaaSConfigVersion": + """ + Insert or update a version snapshot. + + - OOB rows (is_custom=False): treated as immutable once written — the + version string is controlled by Ethyca and the content never changes + for a given version. + - Custom rows (is_custom=True): config/dataset are updated in place so + that users can iterate on a custom template without bumping the version. + """ + existing = ( + db.query(cls) + .filter( + cls.connector_type == connector_type, + cls.version == version, + cls.is_custom == is_custom, + ) + .first() + ) + + if existing: + if is_custom: + existing.config = config + existing.dataset = dataset + existing.save(db) + return existing + + return cls.create( + db=db, + data={ + "connector_type": connector_type, + "version": version, + "config": config, + "dataset": dataset, + "is_custom": is_custom, + }, + ) diff --git a/src/fides/api/schemas/saas/connection_config_saas_history.py b/src/fides/api/schemas/saas/connection_config_saas_history.py new file mode 100644 index 00000000000..e0512bc03dd --- /dev/null +++ b/src/fides/api/schemas/saas/connection_config_saas_history.py @@ -0,0 +1,19 @@ +from datetime import datetime +from typing import Any, Dict, List, Optional + +from fides.api.schemas.base_class import FidesSchema + + +class ConnectionConfigSaaSHistoryResponse(FidesSchema): + """Summary of a per-connection SaaS config snapshot, used for list responses.""" + + id: str + version: str + created_at: datetime + + +class ConnectionConfigSaaSHistoryDetailResponse(ConnectionConfigSaaSHistoryResponse): + """Full detail for a single snapshot, including config and dataset.""" + + config: Dict[str, Any] + dataset: Optional[List[Dict[str, Any]]] = None diff --git a/src/fides/api/schemas/saas/saas_config_version.py b/src/fides/api/schemas/saas/saas_config_version.py new file mode 100644 index 00000000000..8ca832a0081 --- /dev/null +++ b/src/fides/api/schemas/saas/saas_config_version.py @@ -0,0 +1,20 @@ +from datetime import datetime +from typing import Optional + +from fides.api.schemas.base_class import FidesSchema + + +class SaaSConfigVersionResponse(FidesSchema): + """Summary of a stored SaaS integration version, used for list responses.""" + + connector_type: str + version: str + is_custom: bool + created_at: datetime + + +class SaaSConfigVersionDetailResponse(SaaSConfigVersionResponse): + """Full detail for a single version, including config and dataset as raw dicts.""" + + config: dict + dataset: Optional[dict] = None diff --git a/src/fides/api/service/connectors/saas/connector_registry_service.py b/src/fides/api/service/connectors/saas/connector_registry_service.py index 251893f2404..1c611775667 100644 --- a/src/fides/api/service/connectors/saas/connector_registry_service.py +++ b/src/fides/api/service/connectors/saas/connector_registry_service.py @@ -12,6 +12,7 @@ from fides.api.cryptography.cryptographic_util import str_to_b64_str from fides.api.models.connectionconfig import ConnectionConfig from fides.api.models.custom_connector_template import CustomConnectorTemplate +from fides.api.models.saas_config_version import SaaSConfigVersion from fides.api.models.saas_template_dataset import SaasTemplateDataset from fides.api.schemas.saas.connector_template import ( ConnectorTemplate, @@ -330,6 +331,15 @@ def save_template(cls, db: Session, zip_file: ZipFile) -> str: dataset_json=template_dataset_json, ) + SaaSConfigVersion.upsert( + db=db, + connector_type=connector_type, + version=saas_config.version, + config=saas_config.model_dump(mode="json"), + dataset=template_dataset_json, + is_custom=True, + ) + # Bump the Redis version counter and clear the local cache so # every server detects the change on its next read. cls.get_connector_templates.bump_version() # type: ignore[attr-defined] diff --git a/src/fides/api/v1/endpoints/connector_template_endpoints.py b/src/fides/api/v1/endpoints/connector_template_endpoints.py index e2c689fc02e..98126738367 100644 --- a/src/fides/api/v1/endpoints/connector_template_endpoints.py +++ b/src/fides/api/v1/endpoints/connector_template_endpoints.py @@ -2,6 +2,7 @@ from typing import List, Optional from zipfile import BadZipFile, ZipFile +import yaml from fastapi import Body, Depends, HTTPException from fastapi.params import Security from fastapi.responses import JSONResponse, Response @@ -14,16 +15,19 @@ ) from fides.api import deps +from fides.api.models.saas_config_version import SaaSConfigVersion from fides.api.oauth.utils import verify_oauth_client from fides.api.schemas.saas.connector_template import ( ConnectorTemplate, ConnectorTemplateListResponse, ) +from fides.api.schemas.saas.saas_config_version import SaaSConfigVersionResponse from fides.api.service.connectors.saas.connector_registry_service import ( ConnectorRegistry, CustomConnectorTemplateLoader, ) from fides.api.util.api_router import APIRouter +from fides.api.util.saas_util import load_config_from_string, load_dataset_from_string from fides.common.scope_registry import ( CONNECTOR_TEMPLATE_READ, CONNECTOR_TEMPLATE_REGISTER, @@ -34,6 +38,9 @@ CONNECTOR_TEMPLATES_CONFIG, CONNECTOR_TEMPLATES_DATASET, CONNECTOR_TEMPLATES_REGISTER, + CONNECTOR_TEMPLATES_VERSION_CONFIG, + CONNECTOR_TEMPLATES_VERSION_DATASET, + CONNECTOR_TEMPLATES_VERSIONS, DELETE_CUSTOM_TEMPLATE, REGISTER_CONNECTOR_TEMPLATE, V1_URL_PREFIX, @@ -252,3 +259,112 @@ def delete_custom_connector_template( return JSONResponse( content={"message": "Custom connector template successfully deleted."} ) + + +def _get_version_row( + db: Session, + connector_template_type: str, + version: str, +) -> Optional[SaaSConfigVersion]: + """ + Returns the most recent stored row for (connector_type, version). + + When both an OOB row (is_custom=False) and a custom row (is_custom=True) + exist for the same version string, the one created most recently is returned. + Callers that need to distinguish OOB from custom should filter on is_custom + before calling this helper. + """ + return ( + db.query(SaaSConfigVersion) + .filter( + SaaSConfigVersion.connector_type == connector_template_type, + SaaSConfigVersion.version == version, + ) + .order_by(SaaSConfigVersion.created_at.desc()) + .first() + ) + + +@router.get( + CONNECTOR_TEMPLATES_VERSIONS, + dependencies=[Security(verify_oauth_client, scopes=[CONNECTOR_TEMPLATE_READ])], + response_model=List[SaaSConfigVersionResponse], +) +def list_connector_template_versions( + connector_template_type: str, + db: Session = Depends(deps.get_db), +) -> List[SaaSConfigVersionResponse]: + """ + Returns all stored versions for a connector template type, newest first. + + Each entry includes the version string, whether it is a custom template, + and when it was first recorded. Use the version string with the + `/versions/{version}/config` and `/versions/{version}/dataset` endpoints + to inspect the full config or dataset for that version. + """ + rows = ( + db.query(SaaSConfigVersion) + .filter(SaaSConfigVersion.connector_type == connector_template_type) + .order_by(SaaSConfigVersion.created_at.desc()) + .all() + ) + return [SaaSConfigVersionResponse.model_validate(row) for row in rows] + + +@router.get( + CONNECTOR_TEMPLATES_VERSION_CONFIG, + dependencies=[Security(verify_oauth_client, scopes=[CONNECTOR_TEMPLATE_READ])], +) +def get_connector_template_version_config( + connector_template_type: str, + version: str, + db: Session = Depends(deps.get_db), +) -> Response: + """ + Retrieves the SaaS config for a specific version of a connector template. + + Returns the config as raw YAML, in the same format as + `GET /connector-templates/{type}/config`. + """ + row = _get_version_row(db, connector_template_type, version) + if not row: + raise HTTPException( + status_code=HTTP_404_NOT_FOUND, + detail=f"No stored version '{version}' found for connector type '{connector_template_type}'", + ) + return Response( + content=yaml.dump({"saas_config": row.config}, default_flow_style=False), + media_type="text/yaml", + ) + + +@router.get( + CONNECTOR_TEMPLATES_VERSION_DATASET, + dependencies=[Security(verify_oauth_client, scopes=[CONNECTOR_TEMPLATE_READ])], +) +def get_connector_template_version_dataset( + connector_template_type: str, + version: str, + db: Session = Depends(deps.get_db), +) -> Response: + """ + Retrieves the dataset for a specific version of a connector template. + + Returns the dataset as raw YAML, in the same format as + `GET /connector-templates/{type}/dataset`. + """ + row = _get_version_row(db, connector_template_type, version) + if not row: + raise HTTPException( + status_code=HTTP_404_NOT_FOUND, + detail=f"No stored version '{version}' found for connector type '{connector_template_type}'", + ) + if not row.dataset: + raise HTTPException( + status_code=HTTP_404_NOT_FOUND, + detail=f"No dataset stored for version '{version}' of connector type '{connector_template_type}'", + ) + return Response( + content=yaml.dump({"dataset": [row.dataset]}, default_flow_style=False), + media_type="text/yaml", + ) diff --git a/src/fides/api/v1/endpoints/saas_config_endpoints.py b/src/fides/api/v1/endpoints/saas_config_endpoints.py index 359d1439449..10359747549 100644 --- a/src/fides/api/v1/endpoints/saas_config_endpoints.py +++ b/src/fides/api/v1/endpoints/saas_config_endpoints.py @@ -1,4 +1,4 @@ -from typing import Optional +from typing import List, Optional from fastapi import Depends, HTTPException, Request from fastapi.encoders import jsonable_encoder @@ -23,9 +23,11 @@ SaaSConfigNotFoundException, ) from fides.api.common_exceptions import ValidationError as FidesValidationError +from fides.api.models.connection_config_saas_history import ConnectionConfigSaaSHistory from fides.api.models.connectionconfig import ConnectionConfig, ConnectionType from fides.api.models.datasetconfig import DatasetConfig from fides.api.models.event_audit import EventAuditStatus, EventAuditType +from fides.api.models.saas_config_version import SaaSConfigVersion from fides.api.models.sql_models import System # type: ignore from fides.api.oauth.utils import verify_oauth_client from fides.api.schemas.connection_configuration.connection_config import ( @@ -34,6 +36,10 @@ from fides.api.schemas.connection_configuration.saas_config_template_values import ( SaasConnectionTemplateValues, ) +from fides.api.schemas.saas.connection_config_saas_history import ( + ConnectionConfigSaaSHistoryDetailResponse, + ConnectionConfigSaaSHistoryResponse, +) from fides.api.schemas.saas.saas_config import ( SaaSConfig, SaaSConfigValidationDetails, @@ -65,6 +71,8 @@ from fides.common.urn_registry import ( AUTHORIZE, SAAS_CONFIG, + SAAS_CONFIG_HISTORY, + SAAS_CONFIG_HISTORY_BY_VERSION, SAAS_CONFIG_VALIDATE, SAAS_CONNECTOR_FROM_TEMPLATE, V1_URL_PREFIX, @@ -210,6 +218,16 @@ def patch_saas_config( connection_config.update_saas_config(db, saas_config=saas_config) + patched_template = ConnectorRegistry.get_connector_template(saas_config.type) + SaaSConfigVersion.upsert( + db=db, + connector_type=saas_config.type, + version=saas_config.version, + config=saas_config.model_dump(mode="json"), + dataset=None, # PATCH only updates the config; dataset is managed separately + is_custom=patched_template.custom if patched_template else True, + ) + # Create audit event for SaaS config update event_audit_service = EventAuditService(db) event_details, description = generate_connection_audit_event_details( @@ -302,6 +320,67 @@ def delete_saas_config( connection_config.update(db, data={"saas_config": None}) +@router.get( + SAAS_CONFIG_HISTORY, + dependencies=[Security(verify_oauth_client, scopes=[SAAS_CONFIG_READ])], + response_model=List[ConnectionConfigSaaSHistoryResponse], +) +def list_saas_config_history( + db: Session = Depends(deps.get_db), + connection_config: ConnectionConfig = Depends(_get_saas_connection_config), +) -> List[ConnectionConfigSaaSHistory]: + """ + Returns all per-connection SaaS config snapshots for the given connection, + ordered newest first. + """ + logger.info( + "Listing SaaS config history for connection '{}'", connection_config.key + ) + return ( + db.query(ConnectionConfigSaaSHistory) + .filter( + ConnectionConfigSaaSHistory.connection_config_id == connection_config.id + ) + .order_by(ConnectionConfigSaaSHistory.created_at.desc()) + .all() + ) + + +@router.get( + SAAS_CONFIG_HISTORY_BY_VERSION, + dependencies=[Security(verify_oauth_client, scopes=[SAAS_CONFIG_READ])], + response_model=ConnectionConfigSaaSHistoryDetailResponse, +) +def get_saas_config_history_by_version( + version: str, + db: Session = Depends(deps.get_db), + connection_config: ConnectionConfig = Depends(_get_saas_connection_config), +) -> ConnectionConfigSaaSHistory: + """ + Returns the most recent snapshot for the given connection and version string. + """ + logger.info( + "Fetching SaaS config history for connection '{}' version '{}'", + connection_config.key, + version, + ) + snapshot = ( + db.query(ConnectionConfigSaaSHistory) + .filter( + ConnectionConfigSaaSHistory.connection_config_id == connection_config.id, + ConnectionConfigSaaSHistory.version == version, + ) + .order_by(ConnectionConfigSaaSHistory.created_at.desc()) + .first() + ) + if not snapshot: + raise HTTPException( + status_code=HTTP_404_NOT_FOUND, + detail=f"No SaaS config history found for connection '{connection_config.key}' version '{version}'", + ) + return snapshot + + @router.get( AUTHORIZE, dependencies=[Security(verify_oauth_client, scopes=[CONNECTION_AUTHORIZE])], diff --git a/src/fides/common/urn_registry.py b/src/fides/common/urn_registry.py index c9490147ce4..db3bbc8113d 100644 --- a/src/fides/common/urn_registry.py +++ b/src/fides/common/urn_registry.py @@ -193,6 +193,8 @@ # SaaS Config URLs SAAS_CONFIG_VALIDATE = CONNECTION_BY_KEY + "/validate_saas_config" SAAS_CONFIG = CONNECTION_BY_KEY + "/saas_config" +SAAS_CONFIG_HISTORY = CONNECTION_BY_KEY + "/saas-history" +SAAS_CONFIG_HISTORY_BY_VERSION = CONNECTION_BY_KEY + "/saas-history/{version}" SAAS_CONNECTOR_FROM_TEMPLATE = "/connection/instantiate/{connector_template_type}" # Connector Template URLs @@ -200,6 +202,13 @@ CONNECTOR_TEMPLATES_REGISTER = "/connector-templates/register" CONNECTOR_TEMPLATES_CONFIG = "/connector-templates/{connector_template_type}/config" CONNECTOR_TEMPLATES_DATASET = "/connector-templates/{connector_template_type}/dataset" +CONNECTOR_TEMPLATES_VERSIONS = "/connector-templates/{connector_template_type}/versions" +CONNECTOR_TEMPLATES_VERSION_CONFIG = ( + "/connector-templates/{connector_template_type}/versions/{version}/config" +) +CONNECTOR_TEMPLATES_VERSION_DATASET = ( + "/connector-templates/{connector_template_type}/versions/{version}/dataset" +) DELETE_CUSTOM_TEMPLATE = "/connector-templates/{connector_template_type}" # Deprecated: Old connector template register URL diff --git a/tests/ops/api/v1/endpoints/test_saas_config_endpoints.py b/tests/ops/api/v1/endpoints/test_saas_config_endpoints.py index a6e4d1dcdf0..c8aed37cf37 100644 --- a/tests/ops/api/v1/endpoints/test_saas_config_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_saas_config_endpoints.py @@ -24,6 +24,8 @@ from fides.common.urn_registry import ( AUTHORIZE, SAAS_CONFIG, + SAAS_CONFIG_HISTORY, + SAAS_CONFIG_HISTORY_BY_VERSION, SAAS_CONFIG_VALIDATE, V1_URL_PREFIX, ) @@ -584,3 +586,238 @@ def test_get_authorize_url( response = api_client.get(authorize_url, headers=auth_header) response.raise_for_status() assert response.text == f'"{authorization_url}"' + + +@pytest.mark.unit_saas +class TestListSaaSConfigHistory: + @pytest.fixture + def history_url(self, saas_example_connection_config) -> str: + path = V1_URL_PREFIX + SAAS_CONFIG_HISTORY + return path.format(connection_key=saas_example_connection_config.key) + + def test_list_saas_config_history_unauthenticated( + self, history_url, api_client: TestClient + ) -> None: + response = api_client.get(history_url, headers={}) + assert response.status_code == 401 + + def test_list_saas_config_history_wrong_scope( + self, + history_url, + api_client: TestClient, + generate_auth_header, + ) -> None: + auth_header = generate_auth_header(scopes=[SAAS_CONFIG_CREATE_OR_UPDATE]) + response = api_client.get(history_url, headers=auth_header) + assert response.status_code == 403 + + def test_list_saas_config_history_connection_not_found( + self, + api_client: TestClient, + generate_auth_header, + ) -> None: + url = (V1_URL_PREFIX + SAAS_CONFIG_HISTORY).format( + connection_key="nonexistent_key" + ) + auth_header = generate_auth_header(scopes=[SAAS_CONFIG_READ]) + response = api_client.get(url, headers=auth_header) + assert response.status_code == 404 + + def test_list_saas_config_history_empty( + self, + history_url, + api_client: TestClient, + generate_auth_header, + ) -> None: + """Connection exists but update_saas_config has never been called — no snapshots.""" + auth_header = generate_auth_header(scopes=[SAAS_CONFIG_READ]) + response = api_client.get(history_url, headers=auth_header) + assert response.status_code == 200 + assert response.json() == [] + + def test_list_saas_config_history_after_patch( + self, + saas_example_config, + saas_example_connection_config, + api_client: TestClient, + db: Session, + generate_auth_header, + ) -> None: + """PATCH the saas config, then verify a history snapshot was created.""" + patch_url = (V1_URL_PREFIX + SAAS_CONFIG).format( + connection_key=saas_example_connection_config.key + ) + patch_auth = generate_auth_header(scopes=[SAAS_CONFIG_CREATE_OR_UPDATE]) + patch_resp = api_client.patch( + patch_url, headers=patch_auth, json=saas_example_config + ) + assert patch_resp.status_code == 200 + + history_url = (V1_URL_PREFIX + SAAS_CONFIG_HISTORY).format( + connection_key=saas_example_connection_config.key + ) + auth_header = generate_auth_header(scopes=[SAAS_CONFIG_READ]) + response = api_client.get(history_url, headers=auth_header) + assert response.status_code == 200 + items = response.json() + assert len(items) == 1 + item = items[0] + assert item["version"] == saas_example_config["version"] + assert "id" in item + assert "created_at" in item + # list response must not include the full config blob + assert "config" not in item + + def test_list_saas_config_history_multiple_patches_newest_first( + self, + saas_example_config, + saas_example_connection_config, + api_client: TestClient, + db: Session, + generate_auth_header, + ) -> None: + """Two PATCHes produce two snapshots ordered newest first.""" + patch_url = (V1_URL_PREFIX + SAAS_CONFIG).format( + connection_key=saas_example_connection_config.key + ) + patch_auth = generate_auth_header(scopes=[SAAS_CONFIG_CREATE_OR_UPDATE]) + + # first patch + api_client.patch(patch_url, headers=patch_auth, json=saas_example_config) + + # second patch — bump version so it's distinguishable + config_v2 = dict(saas_example_config) + config_v2["version"] = "0.0.2" + api_client.patch(patch_url, headers=patch_auth, json=config_v2) + + history_url = (V1_URL_PREFIX + SAAS_CONFIG_HISTORY).format( + connection_key=saas_example_connection_config.key + ) + auth_header = generate_auth_header(scopes=[SAAS_CONFIG_READ]) + response = api_client.get(history_url, headers=auth_header) + assert response.status_code == 200 + items = response.json() + assert len(items) == 2 + # newest first + assert items[0]["version"] == "0.0.2" + assert items[1]["version"] == saas_example_config["version"] + + +@pytest.mark.unit_saas +class TestGetSaaSConfigHistoryByVersion: + @pytest.fixture + def patched_connection_config( + self, + saas_example_config, + saas_example_connection_config, + api_client: TestClient, + generate_auth_header, + ) -> ConnectionConfig: + """Connection config that has had update_saas_config called once.""" + patch_url = (V1_URL_PREFIX + SAAS_CONFIG).format( + connection_key=saas_example_connection_config.key + ) + auth_header = generate_auth_header(scopes=[SAAS_CONFIG_CREATE_OR_UPDATE]) + api_client.patch(patch_url, headers=auth_header, json=saas_example_config) + return saas_example_connection_config + + def test_get_saas_config_history_by_version_unauthenticated( + self, + patched_connection_config, + api_client: TestClient, + ) -> None: + url = (V1_URL_PREFIX + SAAS_CONFIG_HISTORY_BY_VERSION).format( + connection_key=patched_connection_config.key, version="0.0.1" + ) + response = api_client.get(url, headers={}) + assert response.status_code == 401 + + def test_get_saas_config_history_by_version_wrong_scope( + self, + patched_connection_config, + api_client: TestClient, + generate_auth_header, + ) -> None: + url = (V1_URL_PREFIX + SAAS_CONFIG_HISTORY_BY_VERSION).format( + connection_key=patched_connection_config.key, version="0.0.1" + ) + auth_header = generate_auth_header(scopes=[SAAS_CONFIG_CREATE_OR_UPDATE]) + response = api_client.get(url, headers=auth_header) + assert response.status_code == 403 + + def test_get_saas_config_history_by_version_connection_not_found( + self, + api_client: TestClient, + generate_auth_header, + ) -> None: + url = (V1_URL_PREFIX + SAAS_CONFIG_HISTORY_BY_VERSION).format( + connection_key="nonexistent_key", version="0.0.1" + ) + auth_header = generate_auth_header(scopes=[SAAS_CONFIG_READ]) + response = api_client.get(url, headers=auth_header) + assert response.status_code == 404 + + def test_get_saas_config_history_by_version_not_found( + self, + patched_connection_config, + api_client: TestClient, + generate_auth_header, + ) -> None: + url = (V1_URL_PREFIX + SAAS_CONFIG_HISTORY_BY_VERSION).format( + connection_key=patched_connection_config.key, version="9.9.9" + ) + auth_header = generate_auth_header(scopes=[SAAS_CONFIG_READ]) + response = api_client.get(url, headers=auth_header) + assert response.status_code == 404 + + def test_get_saas_config_history_by_version_found( + self, + saas_example_config, + patched_connection_config, + api_client: TestClient, + generate_auth_header, + ) -> None: + version = saas_example_config["version"] + url = (V1_URL_PREFIX + SAAS_CONFIG_HISTORY_BY_VERSION).format( + connection_key=patched_connection_config.key, version=version + ) + auth_header = generate_auth_header(scopes=[SAAS_CONFIG_READ]) + response = api_client.get(url, headers=auth_header) + assert response.status_code == 200 + data = response.json() + assert data["version"] == version + assert "id" in data + assert "created_at" in data + assert "config" in data + assert data["config"]["fides_key"] == saas_example_config["fides_key"] + # no datasets associated in this fixture + assert data["dataset"] is None + + def test_get_saas_config_history_by_version_returns_most_recent( + self, + saas_example_config, + saas_example_connection_config, + api_client: TestClient, + generate_auth_header, + ) -> None: + """When the same version is patched twice, the most recent snapshot is returned.""" + patch_url = (V1_URL_PREFIX + SAAS_CONFIG).format( + connection_key=saas_example_connection_config.key + ) + patch_auth = generate_auth_header(scopes=[SAAS_CONFIG_CREATE_OR_UPDATE]) + + # patch twice with the same version + api_client.patch(patch_url, headers=patch_auth, json=saas_example_config) + modified = dict(saas_example_config) + modified["description"] = "second patch" + api_client.patch(patch_url, headers=patch_auth, json=modified) + + version = saas_example_config["version"] + url = (V1_URL_PREFIX + SAAS_CONFIG_HISTORY_BY_VERSION).format( + connection_key=saas_example_connection_config.key, version=version + ) + auth_header = generate_auth_header(scopes=[SAAS_CONFIG_READ]) + response = api_client.get(url, headers=auth_header) + assert response.status_code == 200 + data = response.json() + assert data["config"].get("description") == "second patch"