diff --git a/monitoring/fabric-unified-admin-monitoring/README.md b/monitoring/fabric-unified-admin-monitoring/README.md index 70e33575..9767081d 100644 --- a/monitoring/fabric-unified-admin-monitoring/README.md +++ b/monitoring/fabric-unified-admin-monitoring/README.md @@ -38,6 +38,7 @@ FUAM extracts the following data from the tenant: - Tenant meta data (Scanner API) - Capacity Refreshables - Git Connections +- **Feature Releases & Preview Tracking** (NEW) - Engine level insights (coming soon in optimization module) @@ -104,6 +105,7 @@ The FUAM solution accelerator template **is not an official Microsoft service**. - [Documentation - FUAM Architecture](/monitoring/fabric-unified-admin-monitoring/media/documentation/FUAM_Architecture.md) - [Documentation - FUAM Lakehouse table lineage](/monitoring/fabric-unified-admin-monitoring/media/documentation/FUAM_Documentation_Lakehouse_table_lineage.pdf) - [Documentation - FUAM Engine level analyzer reports](/monitoring/fabric-unified-admin-monitoring/media/documentation/FUAM_Engine_Level_Analyzer_Reports.md) +- [Documentation - FUAM Feature Release Tracking](/monitoring/fabric-unified-admin-monitoring/media/documentation/Feature_Release_Tracking_Documentation.md) ##### Some other Fabric Toolbox assets - [Overview - Fabric Cost Analysis](/monitoring/fabric-cost-analysis/README.md) diff --git a/monitoring/fabric-unified-admin-monitoring/config/deployment_order.json b/monitoring/fabric-unified-admin-monitoring/config/deployment_order.json index 8a8d78a0..970f84df 100644 --- a/monitoring/fabric-unified-admin-monitoring/config/deployment_order.json +++ b/monitoring/fabric-unified-admin-monitoring/config/deployment_order.json @@ -190,5 +190,17 @@ { "name": "FUAM_Gateway_Monitoring_From_Files_Report.Report", "fuam_id": "3695cd0e-da7e-3b40-ad28-5bd1bcd33eb6" + }, + { + "name": "01_Setup_Feature_Tracking.Notebook", + "fuam_id": "f8a2b1c3-4d5e-6f7a-8b9c-0d1e2f3a4b5c" + }, + { + "name": "02_Load_Feature_Tracking.Notebook", + "fuam_id": "a1b2c3d4-5e6f-7a8b-9c0d-1e2f3a4b5c6d" + }, + { + "name": "Load_Feature_Tracking_E2E.DataPipeline", + "fuam_id": "b2c3d4e5-6f7a-8b9c-0d1e-2f3a4b5c6d7e" } ] \ No newline at end of file diff --git a/monitoring/fabric-unified-admin-monitoring/src/01_Setup_Feature_Tracking.Notebook/.platform b/monitoring/fabric-unified-admin-monitoring/src/01_Setup_Feature_Tracking.Notebook/.platform new file mode 100644 index 00000000..c01c4294 --- /dev/null +++ b/monitoring/fabric-unified-admin-monitoring/src/01_Setup_Feature_Tracking.Notebook/.platform @@ -0,0 +1,12 @@ +{ + "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", + "metadata": { + "type": "Notebook", + "displayName": "Feature_Tracking_Setup", + "description": "One-time setup for feature tracking tables and views" + }, + "config": { + "version": "2.0", + "logicalId": "00000000-0000-0000-0000-000000000000" + } +} diff --git a/monitoring/fabric-unified-admin-monitoring/src/01_Setup_Feature_Tracking.Notebook/notebook-content.ipynb b/monitoring/fabric-unified-admin-monitoring/src/01_Setup_Feature_Tracking.Notebook/notebook-content.ipynb new file mode 100644 index 00000000..aebaaeb1 --- /dev/null +++ b/monitoring/fabric-unified-admin-monitoring/src/01_Setup_Feature_Tracking.Notebook/notebook-content.ipynb @@ -0,0 +1,323 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "1a0fcaaa", + "metadata": {}, + "source": [ + "# Feature Tracking - Setup Tables and Views\n", + "\n", + "**Purpose**: One-time setup for feature tracking tables and views\n", + "\n", + "**What this creates**:\n", + "- āœ… `feature_releases_roadmap` - Feature releases from Fabric GPS API (with roadmap)\n", + "- āœ… `preview_features_active` - Detected activated preview features\n", + "- āœ… `feature_alerts` - Alerts for new/risky preview features\n", + "- āœ… Helper SQL views for easy querying" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c5c91aa2", + "metadata": {}, + "outputs": [], + "source": [ + "from datetime import datetime\n", + "import pandas as pd" + ] + }, + { + "cell_type": "markdown", + "id": "505200d2", + "metadata": {}, + "source": [ + "## Step 1: Create `feature_releases_roadmap` Table" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6122dce7", + "metadata": {}, + "outputs": [], + "source": [ + "print(\"šŸ”„ Creating table: feature_releases_roadmap\")\n", + "print(\"=\" * 70)\n", + "\n", + "spark.sql(\"\"\"\n", + " CREATE TABLE IF NOT EXISTS feature_releases_roadmap (\n", + " feature_id STRING NOT NULL,\n", + " feature_name STRING NOT NULL,\n", + " feature_description STRING,\n", + " workload STRING,\n", + " product_name STRING,\n", + " release_date TIMESTAMP,\n", + " release_type STRING,\n", + " release_status STRING,\n", + " is_preview BOOLEAN NOT NULL,\n", + " is_planned BOOLEAN NOT NULL,\n", + " is_shipped BOOLEAN NOT NULL,\n", + " last_modified TIMESTAMP NOT NULL,\n", + " source_url STRING,\n", + " source STRING,\n", + " extracted_date TIMESTAMP NOT NULL\n", + " )\n", + " USING DELTA\n", + "\"\"\")\n", + "\n", + "print(\"āœ… Table created: feature_releases_roadmap\")\n", + "print(\" Schema: 15 columns\")\n", + "print(\" šŸ’” Includes planned/future features and historical tracking\")" + ] + }, + { + "cell_type": "markdown", + "id": "49385cd0", + "metadata": {}, + "source": [ + "## Step 2: Create `preview_features_active` Table" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e3dcfc16", + "metadata": {}, + "outputs": [], + "source": [ + "print(\"\\nšŸ”„ Creating table: preview_features_active\")\n", + "print(\"=\" * 70)\n", + "\n", + "spark.sql(\"\"\"\n", + " CREATE TABLE IF NOT EXISTS preview_features_active (\n", + " setting_name STRING NOT NULL,\n", + " feature_id STRING NOT NULL,\n", + " feature_name STRING NOT NULL,\n", + " workload STRING,\n", + " similarity_score DOUBLE NOT NULL,\n", + " is_enabled BOOLEAN NOT NULL,\n", + " delegate_to_tenant BOOLEAN,\n", + " detected_date TIMESTAMP NOT NULL,\n", + " release_date TIMESTAMP,\n", + " release_status STRING,\n", + " source_url STRING,\n", + " days_since_release INT\n", + " )\n", + " USING DELTA\n", + "\"\"\")\n", + "\n", + "print(\"āœ… Table created: preview_features_active\")\n", + "print(\" Schema: 12 columns\")" + ] + }, + { + "cell_type": "markdown", + "id": "b8337a78", + "metadata": {}, + "source": [ + "## Step 3: Create `feature_alerts` Table" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "66383382", + "metadata": {}, + "outputs": [], + "source": [ + "print(\"\\nšŸ”„ Creating table: feature_alerts\")\n", + "print(\"=\" * 70)\n", + "\n", + "spark.sql(\"\"\"\n", + " CREATE TABLE IF NOT EXISTS feature_alerts (\n", + " alert_id STRING NOT NULL,\n", + " feature_id STRING NOT NULL,\n", + " feature_name STRING NOT NULL,\n", + " workload STRING,\n", + " alert_type STRING NOT NULL,\n", + " severity STRING NOT NULL,\n", + " message STRING NOT NULL,\n", + " setting_name STRING,\n", + " similarity_score DOUBLE,\n", + " days_since_release INT,\n", + " alert_date TIMESTAMP NOT NULL,\n", + " acknowledged BOOLEAN NOT NULL,\n", + " acknowledged_date TIMESTAMP,\n", + " acknowledged_by STRING\n", + " )\n", + " USING DELTA\n", + "\"\"\")\n", + "\n", + "print(\"āœ… Table created: feature_alerts\")\n", + "print(\" Schema: 14 columns\")" + ] + }, + { + "cell_type": "markdown", + "id": "21110f70", + "metadata": {}, + "source": [ + "## Step 4: Create Helper SQL Views" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3189b548", + "metadata": {}, + "outputs": [], + "source": [ + "print(\"\\nšŸ”„ Creating helper SQL views...\")\n", + "print(\"=\" * 70)\n", + "\n", + "# View 1: Roadmap Upcoming Features\n", + "spark.sql(\"\"\"\n", + " CREATE OR REPLACE VIEW vw_roadmap_upcoming AS\n", + " SELECT \n", + " feature_name,\n", + " feature_description,\n", + " product_name,\n", + " workload,\n", + " release_type,\n", + " release_status,\n", + " release_date,\n", + " is_preview,\n", + " is_planned,\n", + " last_modified,\n", + " CASE \n", + " WHEN release_date IS NULL THEN NULL\n", + " ELSE DATEDIFF(release_date, CURRENT_DATE())\n", + " END as days_until_release\n", + " FROM feature_releases_roadmap\n", + " WHERE is_planned = true\n", + " AND (release_date IS NULL OR release_date >= CURRENT_DATE())\n", + " ORDER BY release_date ASC NULLS LAST, last_modified DESC\n", + "\"\"\")\n", + "print(\"āœ… vw_roadmap_upcoming - Planned/upcoming features\")\n", + "\n", + "# View 2: Active Preview Features\n", + "spark.sql(\"\"\"\n", + " CREATE OR REPLACE VIEW vw_active_preview_features AS\n", + " SELECT \n", + " feature_name,\n", + " workload,\n", + " setting_name,\n", + " days_since_release,\n", + " similarity_score,\n", + " release_date,\n", + " detected_date,\n", + " is_enabled\n", + " FROM preview_features_active\n", + " WHERE is_enabled = true\n", + " ORDER BY detected_date DESC\n", + "\"\"\")\n", + "print(\"āœ… vw_active_preview_features - Currently enabled previews\")\n", + "\n", + "# View 3: Critical Alerts\n", + "spark.sql(\"\"\"\n", + " CREATE OR REPLACE VIEW vw_critical_alerts AS\n", + " SELECT \n", + " alert_id,\n", + " feature_name,\n", + " workload,\n", + " alert_type,\n", + " severity,\n", + " message,\n", + " alert_date,\n", + " acknowledged\n", + " FROM feature_alerts\n", + " WHERE acknowledged = false \n", + " AND severity IN ('Critical', 'Warning')\n", + " ORDER BY \n", + " CASE severity \n", + " WHEN 'Critical' THEN 1 \n", + " WHEN 'Warning' THEN 2 \n", + " ELSE 3 \n", + " END,\n", + " alert_date DESC\n", + "\"\"\")\n", + "print(\"āœ… vw_critical_alerts - Unacknowledged critical/warning alerts\")\n", + "\n", + "# View 4: Feature Release Timeline\n", + "spark.sql(\"\"\"\n", + " CREATE OR REPLACE VIEW vw_feature_timeline AS\n", + " SELECT \n", + " feature_name,\n", + " product_name,\n", + " workload,\n", + " release_type,\n", + " release_status,\n", + " is_preview,\n", + " is_planned,\n", + " is_shipped,\n", + " release_date,\n", + " CASE \n", + " WHEN release_date IS NULL THEN NULL\n", + " ELSE DATEDIFF(CURRENT_DATE(), release_date)\n", + " END as days_since_release,\n", + " last_modified\n", + " FROM feature_releases_roadmap\n", + " ORDER BY release_date DESC NULLS LAST\n", + "\"\"\")\n", + "print(\"āœ… vw_feature_timeline - Complete release timeline\")\n", + "\n", + "print(\"\\nāœ… All SQL views created successfully\")" + ] + }, + { + "cell_type": "markdown", + "id": "9af215ea", + "metadata": {}, + "source": [ + "## āœ… Setup Complete!" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cb7dc56a", + "metadata": {}, + "outputs": [], + "source": [ + "print(\"\\n\" + \"=\" * 70)\n", + "print(\"šŸŽ‰ FEATURE TRACKING SETUP COMPLETED!\")\n", + "print(\"=\" * 70)\n", + "\n", + "# Verify tables\n", + "tables = [\"feature_releases_roadmap\", \"preview_features_active\", \"feature_alerts\"]\n", + "print(\"\\nšŸ“‹ Tables created:\")\n", + "for table in tables:\n", + " try:\n", + " count = spark.read.format(\"delta\").table(table).count()\n", + " print(f\" āœ… {table}: {count} rows\")\n", + " except Exception as e:\n", + " print(f\" āŒ {table}: ERROR - {e}\")\n", + "\n", + "# Verify views\n", + "views = [\"vw_roadmap_upcoming\", \"vw_active_preview_features\", \"vw_critical_alerts\", \"vw_feature_timeline\"]\n", + "print(\"\\nšŸ“‹ Views created:\")\n", + "for view in views:\n", + " try:\n", + " spark.sql(f\"SELECT * FROM {view} LIMIT 1\")\n", + " print(f\" āœ… {view}\")\n", + " except Exception as e:\n", + " print(f\" āŒ {view}: ERROR - {e}\")\n", + "\n", + "print(\"\\n\" + \"=\" * 70)\n", + "print(\"šŸ“š Next Step:\")\n", + "print(\"=\" * 70)\n", + "print(\"\\n ā†’ Run 'Load_Feature_Tracking' notebook to populate the tables\")\n", + "print(\"\\nšŸ’” Schedule Load_Feature_Tracking to run daily for continuous monitoring\")\n", + "print(\"\\n\" + \"=\" * 70)" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/monitoring/fabric-unified-admin-monitoring/src/02_Load_Feature_Tracking.Notebook/.platform b/monitoring/fabric-unified-admin-monitoring/src/02_Load_Feature_Tracking.Notebook/.platform new file mode 100644 index 00000000..10a12210 --- /dev/null +++ b/monitoring/fabric-unified-admin-monitoring/src/02_Load_Feature_Tracking.Notebook/.platform @@ -0,0 +1,12 @@ +{ + "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", + "metadata": { + "type": "Notebook", + "displayName": "Load_Feature_Tracking", + "description": "Complete feature tracking pipeline - fetch releases, detect previews, generate alerts" + }, + "config": { + "version": "2.0", + "logicalId": "00000000-0000-0000-0000-000000000000" + } +} diff --git a/monitoring/fabric-unified-admin-monitoring/src/02_Load_Feature_Tracking.Notebook/notebook-content.ipynb b/monitoring/fabric-unified-admin-monitoring/src/02_Load_Feature_Tracking.Notebook/notebook-content.ipynb new file mode 100644 index 00000000..7529966d --- /dev/null +++ b/monitoring/fabric-unified-admin-monitoring/src/02_Load_Feature_Tracking.Notebook/notebook-content.ipynb @@ -0,0 +1,786 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "e0a5125f", + "metadata": {}, + "outputs": [], + "source": [ + "import requests\n", + "from datetime import datetime, timedelta\n", + "from pyspark.sql import functions as F\n", + "from pyspark.sql.types import *\n", + "from pyspark.sql.window import Window\n", + "from difflib import SequenceMatcher" + ] + }, + { + "cell_type": "markdown", + "id": "3f0085bb", + "metadata": {}, + "source": [ + "## Parameters" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f250ccba", + "metadata": {}, + "outputs": [], + "source": [ + "# API Configuration\n", + "fabric_gps_api_url = \"https://fabric-gps.com/api/releases\"\n", + "modified_within_days = 90\n", + "page_size = 200\n", + "include_planned = True\n", + "include_shipped = True\n", + "\n", + "# Alert Thresholds\n", + "ALERT_DAYS_THRESHOLD = 90 # Alert if preview active >90 days\n", + "LOW_CONFIDENCE_THRESHOLD = 0.5 # Alert if similarity score <0.5\n", + "SIMILARITY_MATCH_THRESHOLD = 0.3 # Minimum similarity to consider a match\n", + "\n", + "# Alert Severity Levels\n", + "SEVERITY_INFO = \"Info\"\n", + "SEVERITY_WARNING = \"Warning\"\n", + "SEVERITY_CRITICAL = \"Critical\"" + ] + }, + { + "cell_type": "markdown", + "id": "8abfdf27", + "metadata": {}, + "source": [ + "---\n", + "# Part 1: Fetch Feature Releases from Fabric GPS API" + ] + }, + { + "cell_type": "markdown", + "id": "a49058b0", + "metadata": {}, + "source": [ + "## Step 1.1: Fetch from API" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e7dfa925", + "metadata": {}, + "outputs": [], + "source": [ + "def fetch_all_fabric_gps_releases(base_url, page_size=200, modified_within_days=None, include_planned=True, include_shipped=True):\n", + " \"\"\"Fetch all releases from Fabric GPS API with pagination\"\"\"\n", + " all_releases = []\n", + " page = 1\n", + " \n", + " print(f\"šŸ”„ Fetching from Fabric GPS API: {base_url}\")\n", + " \n", + " while True:\n", + " try:\n", + " params = {\"page\": page, \"page_size\": page_size}\n", + " \n", + " if modified_within_days and modified_within_days <= 30:\n", + " params[\"modified_within_days\"] = modified_within_days\n", + " \n", + " response = requests.get(base_url, params=params, timeout=30)\n", + " response.raise_for_status()\n", + " data = response.json()\n", + " \n", + " releases = data.get(\"data\", [])\n", + " if not releases:\n", + " break\n", + " \n", + " # Filter by release_status\n", + " filtered_releases = []\n", + " for release in releases:\n", + " status = release.get(\"release_status\", \"\")\n", + " if status == \"Planned\" and not include_planned:\n", + " continue\n", + " if status == \"Shipped\" and not include_shipped:\n", + " continue\n", + " filtered_releases.append(release)\n", + " \n", + " all_releases.extend(filtered_releases)\n", + " \n", + " pagination = data.get(\"pagination\", {})\n", + " has_next = pagination.get(\"has_next\", False)\n", + " total_items = pagination.get(\"total_items\", 0)\n", + " \n", + " print(f\" ā†’ Page {page}: {len(filtered_releases)} releases (Total: {len(all_releases)}/{total_items})\")\n", + " \n", + " if not has_next:\n", + " break\n", + " \n", + " page += 1\n", + " \n", + " except Exception as e:\n", + " print(f\" āŒ Error fetching page {page}: {e}\")\n", + " break\n", + " \n", + " print(f\"āœ… Total releases fetched: {len(all_releases)}\")\n", + " return all_releases\n", + "\n", + "releases_data = fetch_all_fabric_gps_releases(\n", + " fabric_gps_api_url, \n", + " page_size=page_size,\n", + " modified_within_days=modified_within_days if modified_within_days <= 30 else None,\n", + " include_planned=include_planned,\n", + " include_shipped=include_shipped\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "06bbdd25", + "metadata": {}, + "source": [ + "## Step 1.2: Transform to Schema" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "00b94a80", + "metadata": {}, + "outputs": [], + "source": [ + "def transform_fabric_gps_to_fuam(releases):\n", + " \"\"\"Transform Fabric GPS release data to FUAM schema\"\"\"\n", + " transformed = []\n", + " \n", + " workload_mapping = {\n", + " \"Power BI\": \"Power BI\",\n", + " \"Data Factory\": \"Data Factory\",\n", + " \"Data Engineering\": \"Data Engineering\",\n", + " \"Data Science\": \"Data Science\",\n", + " \"Data Warehouse\": \"Data Warehouse\",\n", + " \"Real-Time Intelligence\": \"Real-Time Intelligence\",\n", + " \"OneLake\": \"Data Engineering\",\n", + " \"Administration, Governance and Security\": \"Governance\",\n", + " \"Cosmos DB (NoSQL)\": \"Cosmos DB\"\n", + " }\n", + " \n", + " for release in releases:\n", + " try:\n", + " release_date_str = release.get(\"release_date\")\n", + " release_date = datetime.strptime(release_date_str, \"%Y-%m-%d\") if release_date_str else None\n", + " except:\n", + " release_date = None\n", + " \n", + " try:\n", + " last_modified_str = release.get(\"last_modified\")\n", + " last_modified = datetime.strptime(last_modified_str, \"%Y-%m-%d\") if last_modified_str else datetime.now()\n", + " except:\n", + " last_modified = datetime.now()\n", + " \n", + " release_type = release.get(\"release_type\", \"\")\n", + " is_preview = \"preview\" in release_type.lower()\n", + " product_name = release.get(\"product_name\", \"Unknown\")\n", + " workload = workload_mapping.get(product_name, product_name)\n", + " \n", + " feature = {\n", + " \"feature_id\": release.get(\"release_item_id\"),\n", + " \"feature_name\": release.get(\"feature_name\", \"Unknown\"),\n", + " \"feature_description\": release.get(\"feature_description\"),\n", + " \"workload\": workload,\n", + " \"product_name\": product_name,\n", + " \"release_date\": release_date,\n", + " \"release_type\": release_type,\n", + " \"release_status\": release.get(\"release_status\", \"Unknown\"),\n", + " \"is_preview\": is_preview,\n", + " \"is_planned\": release.get(\"release_status\") == \"Planned\",\n", + " \"is_shipped\": release.get(\"release_status\") == \"Shipped\",\n", + " \"last_modified\": last_modified,\n", + " \"source_url\": f\"https://fabric-gps.com/api/releases?release_item_id={release.get('release_item_id')}\",\n", + " \"source\": \"Fabric GPS API\",\n", + " \"extracted_date\": datetime.now()\n", + " }\n", + " \n", + " transformed.append(feature)\n", + " \n", + " return transformed\n", + "\n", + "print(\"šŸ”„ Transforming to schema...\")\n", + "features_data = transform_fabric_gps_to_fuam(releases_data)\n", + "\n", + "preview_count = sum(1 for f in features_data if f[\"is_preview\"])\n", + "planned_count = sum(1 for f in features_data if f[\"is_planned\"])\n", + "shipped_count = sum(1 for f in features_data if f[\"is_shipped\"])\n", + "\n", + "print(f\"āœ… Transformed {len(features_data)} features\")\n", + "print(f\" Preview: {preview_count} | Planned: {planned_count} | Shipped: {shipped_count}\")" + ] + }, + { + "cell_type": "markdown", + "id": "db647aba", + "metadata": {}, + "source": [ + "## Step 1.3: Write Feature Releases to Delta" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3f7579b9", + "metadata": {}, + "outputs": [], + "source": [ + "schema = StructType([\n", + " StructField(\"feature_id\", StringType(), False),\n", + " StructField(\"feature_name\", StringType(), False),\n", + " StructField(\"feature_description\", StringType(), True),\n", + " StructField(\"workload\", StringType(), True),\n", + " StructField(\"product_name\", StringType(), True),\n", + " StructField(\"release_date\", TimestampType(), True),\n", + " StructField(\"release_type\", StringType(), True),\n", + " StructField(\"release_status\", StringType(), True),\n", + " StructField(\"is_preview\", BooleanType(), False),\n", + " StructField(\"is_planned\", BooleanType(), False),\n", + " StructField(\"is_shipped\", BooleanType(), False),\n", + " StructField(\"last_modified\", TimestampType(), False),\n", + " StructField(\"source_url\", StringType(), True),\n", + " StructField(\"source\", StringType(), True),\n", + " StructField(\"extracted_date\", TimestampType(), False)\n", + "])\n", + "\n", + "df_features = spark.createDataFrame(features_data, schema=schema)\n", + "\n", + "table_name = \"feature_releases_roadmap\"\n", + "table_path = f\"Tables/{table_name}\"\n", + "\n", + "print(f\"šŸ”„ Writing to {table_name}...\")\n", + "\n", + "try:\n", + " from delta.tables import DeltaTable\n", + " \n", + " if DeltaTable.isDeltaTable(spark, table_path):\n", + " print(\" ā†’ Performing MERGE (upsert)...\")\n", + " \n", + " delta_table = DeltaTable.forPath(spark, table_path)\n", + " \n", + " delta_table.alias(\"target\").merge(\n", + " df_features.alias(\"source\"),\n", + " \"target.feature_id = source.feature_id\"\n", + " ).whenMatchedUpdate(\n", + " condition=\"source.last_modified > target.last_modified\",\n", + " set={\n", + " \"feature_name\": \"source.feature_name\",\n", + " \"feature_description\": \"source.feature_description\",\n", + " \"workload\": \"source.workload\",\n", + " \"product_name\": \"source.product_name\",\n", + " \"release_date\": \"source.release_date\",\n", + " \"release_type\": \"source.release_type\",\n", + " \"release_status\": \"source.release_status\",\n", + " \"is_preview\": \"source.is_preview\",\n", + " \"is_planned\": \"source.is_planned\",\n", + " \"is_shipped\": \"source.is_shipped\",\n", + " \"last_modified\": \"source.last_modified\",\n", + " \"source_url\": \"source.source_url\",\n", + " \"extracted_date\": \"source.extracted_date\"\n", + " }\n", + " ).whenNotMatchedInsertAll().execute()\n", + " \n", + " print(\" āœ… MERGE completed\")\n", + " else:\n", + " print(\" ā†’ Creating new table...\")\n", + " df_features.write.format(\"delta\").mode(\"overwrite\").save(table_path)\n", + " print(\" āœ… Table created\")\n", + " \n", + " final_count = spark.read.format(\"delta\").load(table_path).count()\n", + " print(f\"āœ… Total records in {table_name}: {final_count}\")\n", + " \n", + "except Exception as e:\n", + " print(f\"āŒ Error writing to Delta: {e}\")\n", + " raise" + ] + }, + { + "cell_type": "markdown", + "id": "1fd4034e", + "metadata": {}, + "source": [ + "---\n", + "# Part 2: Detect Activated Preview Features" + ] + }, + { + "cell_type": "markdown", + "id": "51ba2cbf", + "metadata": {}, + "source": [ + "## Step 2.1: Load Data Sources" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "67ba4890", + "metadata": {}, + "outputs": [], + "source": [ + "print(\"\\nšŸ”„ Loading preview features...\")\n", + "df_preview_features = spark.read.format(\"delta\").load(table_path).filter(F.col(\"is_preview\") == True)\n", + "preview_count = df_preview_features.count()\n", + "print(f\"āœ… Loaded {preview_count} preview features\")\n", + "\n", + "print(\"\\nšŸ”„ Loading tenant settings...\")\n", + "try:\n", + " df_tenant_settings = spark.read.format(\"delta\").load(\"Tables/tenant_settings\")\n", + " settings_count = df_tenant_settings.count()\n", + " print(f\"āœ… Loaded {settings_count} tenant settings\")\n", + "except Exception as e:\n", + " print(f\"āš ļø Warning: Could not load tenant_settings table: {e}\")\n", + " print(\" Skipping preview feature detection...\")\n", + " df_tenant_settings = None" + ] + }, + { + "cell_type": "markdown", + "id": "706b1ba5", + "metadata": {}, + "source": [ + "## Step 2.2: Map Settings to Features" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0d8aad6a", + "metadata": {}, + "outputs": [], + "source": [ + "def similarity_score(a, b):\n", + " \"\"\"Calculate similarity between two strings (0-1)\"\"\"\n", + " return SequenceMatcher(None, a.lower(), b.lower()).ratio()\n", + "\n", + "def map_settings_to_features(df_settings, df_features, threshold=SIMILARITY_MATCH_THRESHOLD):\n", + " \"\"\"Map tenant settings to preview features based on name similarity\"\"\"\n", + " \n", + " features_list = df_features.select(\"feature_id\", \"feature_name\", \"workload\", \"release_date\", \"release_status\", \"source_url\").collect()\n", + " settings_list = df_settings.select(\"settingName\", \"enabled\", \"delegateToDomain\", \"delegateToWorkspace\", \"delegateToCapacity\").collect()\n", + " \n", + " matches = []\n", + " \n", + " for setting in settings_list:\n", + " setting_name = setting[\"settingName\"]\n", + " \n", + " if not setting[\"enabled\"]:\n", + " continue\n", + " \n", + " best_match = None\n", + " best_score = 0.0\n", + " \n", + " for feature in features_list:\n", + " feature_name = feature[\"feature_name\"]\n", + " \n", + " # Calculate similarity\n", + " score = similarity_score(setting_name, feature_name)\n", + " \n", + " # Boost score for common words\n", + " setting_words = set(setting_name.lower().split())\n", + " feature_words = set(feature_name.lower().split())\n", + " common_words = setting_words & feature_words\n", + " \n", + " if common_words:\n", + " score += len(common_words) * 0.1\n", + " \n", + " if score > best_score and score > threshold:\n", + " best_score = score\n", + " best_match = feature\n", + " \n", + " if best_match:\n", + " # Calculate days since release\n", + " days_since = None\n", + " if best_match[\"release_date\"]:\n", + " days_since = (datetime.now() - best_match[\"release_date\"]).days\n", + " \n", + " # Check if any delegation is enabled\n", + " is_delegated = setting[\"delegateToDomain\"] or setting[\"delegateToWorkspace\"] or setting[\"delegateToCapacity\"]\n", + " \n", + " matches.append({\n", + " \"setting_name\": setting_name,\n", + " \"feature_id\": best_match[\"feature_id\"],\n", + " \"feature_name\": best_match[\"feature_name\"],\n", + " \"workload\": best_match[\"workload\"],\n", + " \"similarity_score\": best_score,\n", + " \"is_enabled\": setting[\"enabled\"],\n", + " \"delegate_to_tenant\": is_delegated,\n", + " \"detected_date\": datetime.now(),\n", + " \"release_date\": best_match[\"release_date\"],\n", + " \"release_status\": best_match[\"release_status\"],\n", + " \"source_url\": best_match[\"source_url\"],\n", + " \"days_since_release\": days_since\n", + " })\n", + " \n", + " return matches\n", + "\n", + "if df_tenant_settings:\n", + " print(\"šŸ”„ Mapping settings to preview features...\")\n", + " matches = map_settings_to_features(df_tenant_settings, df_preview_features)\n", + " print(f\"āœ… Found {len(matches)} activated preview features\")\n", + "else:\n", + " matches = []" + ] + }, + { + "cell_type": "markdown", + "id": "ccca10b5", + "metadata": {}, + "source": [ + "## Step 2.3: Write Active Previews to Delta" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9702e5dd", + "metadata": {}, + "outputs": [], + "source": [ + "if matches:\n", + " schema_active = StructType([\n", + " StructField(\"setting_name\", StringType(), False),\n", + " StructField(\"feature_id\", StringType(), False),\n", + " StructField(\"feature_name\", StringType(), False),\n", + " StructField(\"workload\", StringType(), True),\n", + " StructField(\"similarity_score\", DoubleType(), False),\n", + " StructField(\"is_enabled\", BooleanType(), False),\n", + " StructField(\"delegate_to_tenant\", BooleanType(), True),\n", + " StructField(\"detected_date\", TimestampType(), False),\n", + " StructField(\"release_date\", TimestampType(), True),\n", + " StructField(\"release_status\", StringType(), True),\n", + " StructField(\"source_url\", StringType(), True),\n", + " StructField(\"days_since_release\", IntegerType(), True)\n", + " ])\n", + " \n", + " df_active_previews = spark.createDataFrame(matches, schema=schema_active)\n", + " \n", + " table_name_active = \"preview_features_active\"\n", + " table_path_active = f\"Tables/{table_name_active}\"\n", + " \n", + " print(f\"šŸ”„ Writing to {table_name_active}...\")\n", + " \n", + " try:\n", + " if DeltaTable.isDeltaTable(spark, table_path_active):\n", + " print(\" ā†’ Performing MERGE...\")\n", + " \n", + " delta_table = DeltaTable.forPath(spark, table_path_active)\n", + " \n", + " delta_table.alias(\"target\").merge(\n", + " df_active_previews.alias(\"source\"),\n", + " \"target.feature_id = source.feature_id AND target.setting_name = source.setting_name\"\n", + " ).whenMatchedUpdate(\n", + " set={\n", + " \"is_enabled\": \"source.is_enabled\",\n", + " \"detected_date\": \"source.detected_date\",\n", + " \"days_since_release\": \"source.days_since_release\",\n", + " \"similarity_score\": \"source.similarity_score\"\n", + " }\n", + " ).whenNotMatchedInsertAll().execute()\n", + " \n", + " print(\" āœ… MERGE completed\")\n", + " else:\n", + " print(\" ā†’ Creating new table...\")\n", + " df_active_previews.write.format(\"delta\").mode(\"overwrite\").save(table_path_active)\n", + " print(\" āœ… Table created\")\n", + " \n", + " final_count = spark.read.format(\"delta\").load(table_path_active).count()\n", + " print(f\"āœ… Total activated previews: {final_count}\")\n", + " \n", + " except Exception as e:\n", + " print(f\"āŒ Error writing active previews: {e}\")\n", + " raise\n", + "else:\n", + " print(\"ā„¹ļø No activated preview features to write\")\n", + " df_active_previews = None" + ] + }, + { + "cell_type": "markdown", + "id": "d1c7f6dd", + "metadata": {}, + "source": [ + "---\n", + "# Part 3: Generate Feature Alerts" + ] + }, + { + "cell_type": "markdown", + "id": "28f72485", + "metadata": {}, + "source": [ + "## Step 3.1: Load Historical Alerts" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ec9f5348", + "metadata": {}, + "outputs": [], + "source": [ + "table_name_alerts = \"feature_alerts\"\n", + "table_path_alerts = f\"Tables/{table_name_alerts}\"\n", + "\n", + "try:\n", + " if DeltaTable.isDeltaTable(spark, table_path_alerts):\n", + " print(\"šŸ”„ Loading historical alerts...\")\n", + " df_historical = spark.read.format(\"delta\").load(table_path_alerts)\n", + " \n", + " # Get already alerted feature_ids + alert_type combinations to avoid duplicate alerts\n", + " alerted_combos = set([\n", + " (row[\"feature_id\"], row[\"alert_type\"]) \n", + " for row in df_historical.select(\"feature_id\", \"alert_type\").distinct().collect()\n", + " ])\n", + " \n", + " print(f\"āœ… Loaded {len(alerted_combos)} existing alert combinations\")\n", + " else:\n", + " alerted_combos = set()\n", + " print(\"ā„¹ļø No historical alerts found (first run)\")\n", + "except Exception as e:\n", + " alerted_combos = set()\n", + " print(f\"ā„¹ļø No historical alerts (first run): {e}\")" + ] + }, + { + "cell_type": "markdown", + "id": "2f2b3c9f", + "metadata": {}, + "source": [ + "## Step 3.2: Generate Alerts" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b972785f", + "metadata": {}, + "outputs": [], + "source": [ + "def generate_alerts(matches, alerted_combos):\n", + " \"\"\"Generate alerts based on business rules\"\"\"\n", + " alerts = []\n", + " \n", + " for match in matches:\n", + " feature_id = match[\"feature_id\"]\n", + " feature_name = match[\"feature_name\"]\n", + " workload = match[\"workload\"]\n", + " days_since_release = match[\"days_since_release\"]\n", + " similarity_score = match[\"similarity_score\"]\n", + " setting_name = match[\"setting_name\"]\n", + " \n", + " # Rule 1: NEW PREVIEW FEATURE ACTIVATED\n", + " alert_type = \"New Preview Activated\"\n", + " if (feature_id, alert_type) not in alerted_combos:\n", + " alerts.append({\n", + " \"alert_id\": f\"NEW_{feature_id}_{datetime.now().strftime('%Y%m%d%H%M%S')}\",\n", + " \"feature_id\": feature_id,\n", + " \"feature_name\": feature_name,\n", + " \"workload\": workload,\n", + " \"alert_type\": alert_type,\n", + " \"severity\": SEVERITY_INFO,\n", + " \"message\": f\"New preview feature '{feature_name}' activated (Setting: {setting_name})\",\n", + " \"setting_name\": setting_name,\n", + " \"similarity_score\": similarity_score,\n", + " \"days_since_release\": days_since_release,\n", + " \"alert_date\": datetime.now(),\n", + " \"acknowledged\": False,\n", + " \"acknowledged_date\": None,\n", + " \"acknowledged_by\": None\n", + " })\n", + " \n", + " # Rule 2: LONG-RUNNING PREVIEW (>90 days)\n", + " if days_since_release and days_since_release > ALERT_DAYS_THRESHOLD:\n", + " alert_type = \"Long-Running Preview\"\n", + " if (feature_id, alert_type) not in alerted_combos:\n", + " alerts.append({\n", + " \"alert_id\": f\"LONGRUN_{feature_id}_{datetime.now().strftime('%Y%m%d%H%M%S')}\",\n", + " \"feature_id\": feature_id,\n", + " \"feature_name\": feature_name,\n", + " \"workload\": workload,\n", + " \"alert_type\": alert_type,\n", + " \"severity\": SEVERITY_WARNING,\n", + " \"message\": f\"Preview feature '{feature_name}' active for {days_since_release} days. Review for GA transition.\",\n", + " \"setting_name\": setting_name,\n", + " \"similarity_score\": similarity_score,\n", + " \"days_since_release\": days_since_release,\n", + " \"alert_date\": datetime.now(),\n", + " \"acknowledged\": False,\n", + " \"acknowledged_date\": None,\n", + " \"acknowledged_by\": None\n", + " })\n", + " \n", + " # Rule 3: LOW CONFIDENCE MATCH\n", + " if similarity_score < LOW_CONFIDENCE_THRESHOLD:\n", + " alert_type = \"Low Confidence Match\"\n", + " if (feature_id, alert_type) not in alerted_combos:\n", + " alerts.append({\n", + " \"alert_id\": f\"LOWCONF_{feature_id}_{datetime.now().strftime('%Y%m%d%H%M%S')}\",\n", + " \"feature_id\": feature_id,\n", + " \"feature_name\": feature_name,\n", + " \"workload\": workload,\n", + " \"alert_type\": alert_type,\n", + " \"severity\": SEVERITY_CRITICAL,\n", + " \"message\": f\"Low confidence match ({similarity_score:.2f}) between setting '{setting_name}' and feature '{feature_name}'. Manual review recommended.\",\n", + " \"setting_name\": setting_name,\n", + " \"similarity_score\": similarity_score,\n", + " \"days_since_release\": days_since_release,\n", + " \"alert_date\": datetime.now(),\n", + " \"acknowledged\": False,\n", + " \"acknowledged_date\": None,\n", + " \"acknowledged_by\": None\n", + " })\n", + " \n", + " return alerts\n", + "\n", + "if matches:\n", + " print(\"šŸ”„ Generating alerts...\")\n", + " alerts_data = generate_alerts(matches, alerted_combos)\n", + " print(f\"āœ… Generated {len(alerts_data)} new alerts\")\n", + " \n", + " if alerts_data:\n", + " for alert in alerts_data[:3]:\n", + " print(f\" [{alert['severity']}] {alert['alert_type']}: {alert['feature_name']}\")\n", + "else:\n", + " alerts_data = []\n", + " print(\"ā„¹ļø No data for alert generation\")" + ] + }, + { + "cell_type": "markdown", + "id": "88aa50ca", + "metadata": {}, + "source": [ + "## Step 3.3: Write Alerts to Delta" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c334fef8", + "metadata": {}, + "outputs": [], + "source": [ + "if alerts_data:\n", + " schema_alerts = StructType([\n", + " StructField(\"alert_id\", StringType(), False),\n", + " StructField(\"feature_id\", StringType(), False),\n", + " StructField(\"feature_name\", StringType(), False),\n", + " StructField(\"workload\", StringType(), True),\n", + " StructField(\"alert_type\", StringType(), False),\n", + " StructField(\"severity\", StringType(), False),\n", + " StructField(\"message\", StringType(), False),\n", + " StructField(\"setting_name\", StringType(), True),\n", + " StructField(\"similarity_score\", DoubleType(), True),\n", + " StructField(\"days_since_release\", IntegerType(), True),\n", + " StructField(\"alert_date\", TimestampType(), False),\n", + " StructField(\"acknowledged\", BooleanType(), False),\n", + " StructField(\"acknowledged_date\", TimestampType(), True),\n", + " StructField(\"acknowledged_by\", StringType(), True)\n", + " ])\n", + " \n", + " df_alerts = spark.createDataFrame(alerts_data, schema=schema_alerts)\n", + " \n", + " print(f\"šŸ”„ Writing {len(alerts_data)} alerts to {table_name_alerts}...\")\n", + " \n", + " try:\n", + " if DeltaTable.isDeltaTable(spark, table_path_alerts):\n", + " print(\" ā†’ Appending to existing table...\")\n", + " df_alerts.write.format(\"delta\").mode(\"append\").save(table_path_alerts)\n", + " else:\n", + " print(\" ā†’ Creating new table...\")\n", + " df_alerts.write.format(\"delta\").mode(\"overwrite\").save(table_path_alerts)\n", + " \n", + " final_count = spark.read.format(\"delta\").load(table_path_alerts).count()\n", + " print(f\"āœ… Total alerts: {final_count}\")\n", + " \n", + " except Exception as e:\n", + " print(f\"āŒ Error writing alerts: {e}\")\n", + " raise\n", + "else:\n", + " print(\"ā„¹ļø No new alerts to write\")" + ] + }, + { + "cell_type": "markdown", + "id": "62c50df9", + "metadata": {}, + "source": [ + "---\n", + "# Summary & Statistics" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fe4c5072", + "metadata": {}, + "outputs": [], + "source": [ + "print(\"\\n\" + \"=\" * 70)\n", + "print(\"šŸ“Š FEATURE TRACKING SUMMARY\")\n", + "print(\"=\" * 70)\n", + "\n", + "# Feature Releases Stats\n", + "df_roadmap = spark.read.format(\"delta\").load(table_path)\n", + "total_features = df_roadmap.count()\n", + "preview_features = df_roadmap.filter(F.col(\"is_preview\") == True).count()\n", + "planned_features = df_roadmap.filter(F.col(\"is_planned\") == True).count()\n", + "shipped_features = df_roadmap.filter(F.col(\"is_shipped\") == True).count()\n", + "\n", + "print(f\"\\nšŸ”ø Feature Releases Roadmap:\")\n", + "print(f\" Total features: {total_features}\")\n", + "print(f\" Preview features: {preview_features}\")\n", + "print(f\" Planned (roadmap): {planned_features}\")\n", + "print(f\" Shipped: {shipped_features}\")\n", + "\n", + "# Active Previews Stats\n", + "if matches:\n", + " print(f\"\\nšŸ”ø Activated Preview Features:\")\n", + " print(f\" Total activated: {len(matches)}\")\n", + " high_conf = sum(1 for m in matches if m[\"similarity_score\"] >= 0.7)\n", + " med_conf = sum(1 for m in matches if 0.5 <= m[\"similarity_score\"] < 0.7)\n", + " low_conf = sum(1 for m in matches if m[\"similarity_score\"] < 0.5)\n", + " print(f\" High confidence (ā‰„0.7): {high_conf}\")\n", + " print(f\" Medium confidence (0.5-0.7): {med_conf}\")\n", + " print(f\" Low confidence (<0.5): {low_conf}\")\n", + "\n", + "# Alerts Stats\n", + "if alerts_data:\n", + " print(f\"\\nšŸ”ø Alerts Generated:\")\n", + " print(f\" New alerts: {len(alerts_data)}\")\n", + " info_alerts = sum(1 for a in alerts_data if a[\"severity\"] == SEVERITY_INFO)\n", + " warn_alerts = sum(1 for a in alerts_data if a[\"severity\"] == SEVERITY_WARNING)\n", + " crit_alerts = sum(1 for a in alerts_data if a[\"severity\"] == SEVERITY_CRITICAL)\n", + " print(f\" Info: {info_alerts} | Warning: {warn_alerts} | Critical: {crit_alerts}\")\n", + "\n", + "# Workload Breakdown\n", + "print(f\"\\nšŸ”ø Top Workloads (by feature count):\")\n", + "df_roadmap.groupBy(\"workload\").count().orderBy(F.desc(\"count\")).show(5, truncate=False)\n", + "\n", + "print(\"\\n\" + \"=\" * 70)\n", + "print(\"āœ… LOAD FEATURE TRACKING - COMPLETED\")\n", + "print(\"=\" * 70)\n", + "print(\"\\nšŸ’” View results in:\")\n", + "print(\" - vw_roadmap_upcoming (planned features)\")\n", + "print(\" - vw_active_preview_features (enabled previews)\")\n", + "print(\" - vw_critical_alerts (unacknowledged alerts)\")\n", + "print(\"\\n\" + \"=\" * 70)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "PySpark", + "language": "Python", + "name": "synapse_pyspark" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/monitoring/fabric-unified-admin-monitoring/src/Load_Feature_Tracking_E2E.DataPipeline/.platform b/monitoring/fabric-unified-admin-monitoring/src/Load_Feature_Tracking_E2E.DataPipeline/.platform new file mode 100644 index 00000000..d73a2f64 --- /dev/null +++ b/monitoring/fabric-unified-admin-monitoring/src/Load_Feature_Tracking_E2E.DataPipeline/.platform @@ -0,0 +1,12 @@ +{ + "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", + "metadata": { + "type": "DataPipeline", + "displayName": "Load_Feature_Tracking_E2E", + "description": "Pipeline to load feature tracking data - fetches releases, detects previews, generates alerts" + }, + "config": { + "version": "2.0", + "logicalId": "00000000-0000-0000-0000-000000000000" + } +} diff --git a/monitoring/fabric-unified-admin-monitoring/src/Load_Feature_Tracking_E2E.DataPipeline/pipeline-content.json b/monitoring/fabric-unified-admin-monitoring/src/Load_Feature_Tracking_E2E.DataPipeline/pipeline-content.json new file mode 100644 index 00000000..5cdb65ef --- /dev/null +++ b/monitoring/fabric-unified-admin-monitoring/src/Load_Feature_Tracking_E2E.DataPipeline/pipeline-content.json @@ -0,0 +1,26 @@ +{ + "properties": { + "activities": [ + { + "name": "02_Load_Feature_Tracking", + "type": "TridentNotebook", + "dependsOn": [], + "policy": { + "timeout": "0.12:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "typeProperties": { + "notebookId": "REPLACE_WITH_NOTEBOOK_ID", + "workspaceId": "REPLACE_WITH_WORKSPACE_ID", + "parameters": {}, + "sessionTag": "fuam_feature_tracking" + } + } + ], + "parameters": {}, + "annotations": [] + } +}