From 5968dec423ba1d9be5b092724a477f6adc2a882f Mon Sep 17 00:00:00 2001 From: Ayoub KEBAILI Date: Wed, 19 Nov 2025 14:40:30 -0500 Subject: [PATCH 1/4] feat: Add Load Feature Tracking E2E pipeline and setup notebooks - Created a new pipeline `Load_Feature_Tracking_E2E` for tracking feature releases, preview features, and alerts. - Added `Setup_Feature_Tracking_Tables` notebook for one-time setup of Delta tables. - Implemented `Setup_Feature_Tracking_Tables_GpsApi` notebook to enhance feature tracking with roadmap data. - Defined schemas and created Delta tables: `feature_releases`, `preview_features_active`, `feature_alerts`, and `feature_releases_roadmap`. - Added helper views for SQL querying: `vw_active_preview_features`, `vw_critical_alerts`, `vw_feature_timeline`, and `vw_roadmap_upcoming`. - Included verification steps to ensure tables and views are created successfully. --- .../fabric-unified-admin-monitoring/README.md | 2 + .../notebook-content.ipynb | 412 +++++++++++++++++ .../.platform | 12 + .../notebook-content.ipynb | 324 ++++++++++++++ .../.platform | 12 + .../notebook-content.ipynb | 372 ++++++++++++++++ .../.platform | 12 + .../notebook-content.ipynb | 417 ++++++++++++++++++ .../pipeline-content.json | 114 +++++ .../.platform | 12 + .../notebook-content.ipynb | 348 +++++++++++++++ .../notebook-content.ipynb | 382 ++++++++++++++++ 12 files changed, 2419 insertions(+) create mode 100644 monitoring/fabric-unified-admin-monitoring/src/01_Transfer_Feature_Releases_GpsApi_Unit.Notebook/notebook-content.ipynb create mode 100644 monitoring/fabric-unified-admin-monitoring/src/01_Transfer_Feature_Releases_Unit.Notebook/.platform create mode 100644 monitoring/fabric-unified-admin-monitoring/src/01_Transfer_Feature_Releases_Unit.Notebook/notebook-content.ipynb create mode 100644 monitoring/fabric-unified-admin-monitoring/src/02_Transfer_Preview_Features_Unit.Notebook/.platform create mode 100644 monitoring/fabric-unified-admin-monitoring/src/02_Transfer_Preview_Features_Unit.Notebook/notebook-content.ipynb create mode 100644 monitoring/fabric-unified-admin-monitoring/src/03_Transfer_Feature_Alerts_Unit.Notebook/.platform create mode 100644 monitoring/fabric-unified-admin-monitoring/src/03_Transfer_Feature_Alerts_Unit.Notebook/notebook-content.ipynb create mode 100644 monitoring/fabric-unified-admin-monitoring/src/Load_Feature_Tracking_E2E.DataPipeline/pipeline-content.json create mode 100644 monitoring/fabric-unified-admin-monitoring/src/Setup_Feature_Tracking_Tables.Notebook/.platform create mode 100644 monitoring/fabric-unified-admin-monitoring/src/Setup_Feature_Tracking_Tables.Notebook/notebook-content.ipynb create mode 100644 monitoring/fabric-unified-admin-monitoring/src/Setup_Feature_Tracking_Tables_GpsApi.Notebook/notebook-content.ipynb diff --git a/monitoring/fabric-unified-admin-monitoring/README.md b/monitoring/fabric-unified-admin-monitoring/README.md index 70e33575..9767081d 100644 --- a/monitoring/fabric-unified-admin-monitoring/README.md +++ b/monitoring/fabric-unified-admin-monitoring/README.md @@ -38,6 +38,7 @@ FUAM extracts the following data from the tenant: - Tenant meta data (Scanner API) - Capacity Refreshables - Git Connections +- **Feature Releases & Preview Tracking** (NEW) - Engine level insights (coming soon in optimization module) @@ -104,6 +105,7 @@ The FUAM solution accelerator template **is not an official Microsoft service**. - [Documentation - FUAM Architecture](/monitoring/fabric-unified-admin-monitoring/media/documentation/FUAM_Architecture.md) - [Documentation - FUAM Lakehouse table lineage](/monitoring/fabric-unified-admin-monitoring/media/documentation/FUAM_Documentation_Lakehouse_table_lineage.pdf) - [Documentation - FUAM Engine level analyzer reports](/monitoring/fabric-unified-admin-monitoring/media/documentation/FUAM_Engine_Level_Analyzer_Reports.md) +- [Documentation - FUAM Feature Release Tracking](/monitoring/fabric-unified-admin-monitoring/media/documentation/Feature_Release_Tracking_Documentation.md) ##### Some other Fabric Toolbox assets - [Overview - Fabric Cost Analysis](/monitoring/fabric-cost-analysis/README.md) diff --git a/monitoring/fabric-unified-admin-monitoring/src/01_Transfer_Feature_Releases_GpsApi_Unit.Notebook/notebook-content.ipynb b/monitoring/fabric-unified-admin-monitoring/src/01_Transfer_Feature_Releases_GpsApi_Unit.Notebook/notebook-content.ipynb new file mode 100644 index 00000000..013b0555 --- /dev/null +++ b/monitoring/fabric-unified-admin-monitoring/src/01_Transfer_Feature_Releases_GpsApi_Unit.Notebook/notebook-content.ipynb @@ -0,0 +1,412 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "785109b8", + "metadata": {}, + "outputs": [], + "source": [ + "import requests\n", + "import hashlib\n", + "from datetime import datetime, timedelta\n", + "from pyspark.sql import functions as F\n", + "from pyspark.sql.types import *" + ] + }, + { + "cell_type": "markdown", + "id": "48325bc2", + "metadata": {}, + "source": [ + "## Parameters" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3bab581f", + "metadata": {}, + "outputs": [], + "source": [ + "# Parameters (can be passed from Pipeline)\n", + "fabric_gps_api_url = \"https://fabric-gps.com/api/releases\"\n", + "modified_within_days = 90 # Get features modified in last N days (1-30 per API, but we'll fetch all)\n", + "page_size = 200 # Max allowed by API\n", + "include_planned = True # Include planned features (not yet shipped)\n", + "include_shipped = True # Include shipped features" + ] + }, + { + "cell_type": "markdown", + "id": "805c7883", + "metadata": {}, + "source": [ + "## Step 1: Fetch Feature Releases from Fabric GPS API" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fb075233", + "metadata": {}, + "outputs": [], + "source": [ + "def fetch_all_fabric_gps_releases(base_url, page_size=200, modified_within_days=None, include_planned=True, include_shipped=True):\n", + " \"\"\"\n", + " Fetch all releases from Fabric GPS API with pagination\n", + " Returns list of release objects\n", + " \"\"\"\n", + " all_releases = []\n", + " page = 1\n", + " \n", + " print(f\"šŸ”„ Fetching from Fabric GPS API: {base_url}\")\n", + " \n", + " while True:\n", + " try:\n", + " # Build query parameters\n", + " params = {\n", + " \"page\": page,\n", + " \"page_size\": page_size\n", + " }\n", + " \n", + " if modified_within_days and modified_within_days <= 30:\n", + " params[\"modified_within_days\"] = modified_within_days\n", + " \n", + " # Make request\n", + " response = requests.get(base_url, params=params, timeout=30)\n", + " response.raise_for_status()\n", + " data = response.json()\n", + " \n", + " # Extract releases\n", + " releases = data.get(\"data\", [])\n", + " \n", + " if not releases:\n", + " print(f\" ā†’ No more data on page {page}\")\n", + " break\n", + " \n", + " # Filter by release_status if specified\n", + " filtered_releases = []\n", + " for release in releases:\n", + " status = release.get(\"release_status\", \"\")\n", + " \n", + " # Apply filters\n", + " if status == \"Planned\" and not include_planned:\n", + " continue\n", + " if status == \"Shipped\" and not include_shipped:\n", + " continue\n", + " \n", + " filtered_releases.append(release)\n", + " \n", + " all_releases.extend(filtered_releases)\n", + " \n", + " # Check pagination\n", + " pagination = data.get(\"pagination\", {})\n", + " has_next = pagination.get(\"has_next\", False)\n", + " total_items = pagination.get(\"total_items\", 0)\n", + " \n", + " print(f\" ā†’ Page {page}: Fetched {len(filtered_releases)} releases (Total: {len(all_releases)}/{total_items})\")\n", + " \n", + " if not has_next:\n", + " print(f\" āœ… Reached end of data\")\n", + " break\n", + " \n", + " page += 1\n", + " \n", + " except Exception as e:\n", + " print(f\" āŒ Error fetching page {page}: {e}\")\n", + " break\n", + " \n", + " print(f\"\\nāœ… Total releases fetched: {len(all_releases)}\")\n", + " return all_releases\n", + "\n", + "# Fetch all releases\n", + "releases_data = fetch_all_fabric_gps_releases(\n", + " fabric_gps_api_url, \n", + " page_size=page_size,\n", + " modified_within_days=modified_within_days if modified_within_days <= 30 else None,\n", + " include_planned=include_planned,\n", + " include_shipped=include_shipped\n", + ")\n", + "\n", + "print(f\"\\nšŸ“‹ Sample releases:\")\n", + "for release in releases_data[:3]:\n", + " print(f\" - {release.get('feature_name')} ({release.get('release_type')}) - {release.get('product_name')}\")" + ] + }, + { + "cell_type": "markdown", + "id": "32c24c88", + "metadata": {}, + "source": [ + "## Step 2: Transform to FUAM Schema" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9eebb7af", + "metadata": {}, + "outputs": [], + "source": [ + "def transform_fabric_gps_to_fuam(releases):\n", + " \"\"\"\n", + " Transform Fabric GPS release data to FUAM feature_releases schema\n", + " \"\"\"\n", + " transformed = []\n", + " \n", + " for release in releases:\n", + " # Parse release date\n", + " release_date_str = release.get(\"release_date\")\n", + " try:\n", + " if release_date_str:\n", + " release_date = datetime.strptime(release_date_str, \"%Y-%m-%d\")\n", + " else:\n", + " release_date = None # Planned features may not have a date yet\n", + " except:\n", + " release_date = None\n", + " \n", + " # Parse last modified\n", + " last_modified_str = release.get(\"last_modified\")\n", + " try:\n", + " last_modified = datetime.strptime(last_modified_str, \"%Y-%m-%d\") if last_modified_str else datetime.now()\n", + " except:\n", + " last_modified = datetime.now()\n", + " \n", + " # Determine if preview\n", + " release_type = release.get(\"release_type\", \"\")\n", + " is_preview = \"preview\" in release_type.lower()\n", + " \n", + " # Map product_name to workload (align with FUAM naming)\n", + " product_name = release.get(\"product_name\", \"Unknown\")\n", + " workload_mapping = {\n", + " \"Power BI\": \"Power BI\",\n", + " \"Data Factory\": \"Data Factory\",\n", + " \"Data Engineering\": \"Data Engineering\",\n", + " \"Data Science\": \"Data Science\",\n", + " \"Data Warehouse\": \"Data Warehouse\",\n", + " \"Real-Time Intelligence\": \"Real-Time Intelligence\",\n", + " \"OneLake\": \"Data Engineering\",\n", + " \"Administration, Governance and Security\": \"Governance\",\n", + " \"Cosmos DB (NoSQL)\": \"Cosmos DB\"\n", + " }\n", + " workload = workload_mapping.get(product_name, product_name)\n", + " \n", + " # Create feature record\n", + " feature = {\n", + " \"feature_id\": release.get(\"release_item_id\"), # Use Fabric GPS UUID\n", + " \"feature_name\": release.get(\"feature_name\", \"Unknown\"),\n", + " \"feature_description\": release.get(\"feature_description\"),\n", + " \"workload\": workload,\n", + " \"product_name\": product_name,\n", + " \"release_date\": release_date,\n", + " \"release_type\": release_type,\n", + " \"release_status\": release.get(\"release_status\", \"Unknown\"),\n", + " \"is_preview\": is_preview,\n", + " \"is_planned\": release.get(\"release_status\") == \"Planned\",\n", + " \"is_shipped\": release.get(\"release_status\") == \"Shipped\",\n", + " \"last_modified\": last_modified,\n", + " \"source_url\": f\"https://fabric-gps.com/api/releases?release_item_id={release.get('release_item_id')}\",\n", + " \"source\": \"Fabric GPS API\",\n", + " \"extracted_date\": datetime.now()\n", + " }\n", + " \n", + " transformed.append(feature)\n", + " \n", + " return transformed\n", + "\n", + "print(\"šŸ”„ Transforming releases to FUAM schema...\")\n", + "features_data = transform_fabric_gps_to_fuam(releases_data)\n", + "print(f\"āœ… Transformed {len(features_data)} features\")\n", + "\n", + "# Statistics\n", + "preview_count = sum(1 for f in features_data if f[\"is_preview\"])\n", + "planned_count = sum(1 for f in features_data if f[\"is_planned\"])\n", + "shipped_count = sum(1 for f in features_data if f[\"is_shipped\"])\n", + "\n", + "print(f\"\\nšŸ“Š Feature Breakdown:\")\n", + "print(f\" - Preview features: {preview_count}\")\n", + "print(f\" - Planned features: {planned_count}\")\n", + "print(f\" - Shipped features: {shipped_count}\")" + ] + }, + { + "cell_type": "markdown", + "id": "ecd949aa", + "metadata": {}, + "source": [ + "## Step 3: Create DataFrame" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f4feed1d", + "metadata": {}, + "outputs": [], + "source": [ + "schema = StructType([\n", + " StructField(\"feature_id\", StringType(), False),\n", + " StructField(\"feature_name\", StringType(), False),\n", + " StructField(\"feature_description\", StringType(), True),\n", + " StructField(\"workload\", StringType(), True),\n", + " StructField(\"product_name\", StringType(), True),\n", + " StructField(\"release_date\", TimestampType(), True), # Nullable for planned features\n", + " StructField(\"release_type\", StringType(), True),\n", + " StructField(\"release_status\", StringType(), True),\n", + " StructField(\"is_preview\", BooleanType(), False),\n", + " StructField(\"is_planned\", BooleanType(), False),\n", + " StructField(\"is_shipped\", BooleanType(), False),\n", + " StructField(\"last_modified\", TimestampType(), False),\n", + " StructField(\"source_url\", StringType(), True),\n", + " StructField(\"source\", StringType(), True),\n", + " StructField(\"extracted_date\", TimestampType(), False)\n", + "])\n", + "\n", + "df_features = spark.createDataFrame(features_data, schema=schema)\n", + "\n", + "print(f\"āœ… Created DataFrame with {df_features.count()} rows\")\n", + "df_features.show(5, truncate=False)" + ] + }, + { + "cell_type": "markdown", + "id": "55d7f50e", + "metadata": {}, + "source": [ + "## Step 4: Write to Delta Lake" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "afc62e35", + "metadata": {}, + "outputs": [], + "source": [ + "# Define table path\n", + "table_name = \"feature_releases_roadmap\"\n", + "table_path = f\"Tables/{table_name}\"\n", + "\n", + "print(f\"šŸ”„ Writing to Delta table: {table_name}\")\n", + "\n", + "# Write with merge logic (upsert)\n", + "try:\n", + " from delta.tables import DeltaTable\n", + " \n", + " # Check if table exists\n", + " if DeltaTable.isDeltaTable(spark, table_path):\n", + " print(\" ā†’ Table exists, performing MERGE (upsert)...\")\n", + " \n", + " delta_table = DeltaTable.forPath(spark, table_path)\n", + " \n", + " # Merge: Update existing, insert new\n", + " # Use last_modified to track changes\n", + " delta_table.alias(\"target\").merge(\n", + " df_features.alias(\"source\"),\n", + " \"target.feature_id = source.feature_id\"\n", + " ).whenMatchedUpdate(\n", + " condition=\"source.last_modified > target.last_modified\",\n", + " set={\n", + " \"feature_name\": \"source.feature_name\",\n", + " \"feature_description\": \"source.feature_description\",\n", + " \"workload\": \"source.workload\",\n", + " \"product_name\": \"source.product_name\",\n", + " \"release_date\": \"source.release_date\",\n", + " \"release_type\": \"source.release_type\",\n", + " \"release_status\": \"source.release_status\",\n", + " \"is_preview\": \"source.is_preview\",\n", + " \"is_planned\": \"source.is_planned\",\n", + " \"is_shipped\": \"source.is_shipped\",\n", + " \"last_modified\": \"source.last_modified\",\n", + " \"source_url\": \"source.source_url\",\n", + " \"extracted_date\": \"source.extracted_date\"\n", + " }\n", + " ).whenNotMatchedInsertAll(\n", + " ).execute()\n", + " \n", + " print(\" āœ… MERGE completed\")\n", + " else:\n", + " print(\" ā†’ Table doesn't exist, creating new table...\")\n", + " df_features.write.format(\"delta\").mode(\"overwrite\").save(table_path)\n", + " print(\" āœ… Table created\")\n", + " \n", + " # Show final count\n", + " final_count = spark.read.format(\"delta\").load(table_path).count()\n", + " print(f\"\\nāœ… Total records in {table_name}: {final_count}\")\n", + " \n", + "except Exception as e:\n", + " print(f\"āŒ Error writing to Delta: {e}\")\n", + " raise" + ] + }, + { + "cell_type": "markdown", + "id": "7ce4ca77", + "metadata": {}, + "source": [ + "## Step 5: Summary Statistics" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5e58a863", + "metadata": {}, + "outputs": [], + "source": [ + "print(\"\\nšŸ“Š Feature Release Roadmap Statistics:\")\n", + "print(\"=\" * 70)\n", + "\n", + "df_summary = spark.read.format(\"delta\").load(table_path)\n", + "\n", + "# By workload/product\n", + "print(\"\\nšŸ”ø By Workload:\")\n", + "df_summary.groupBy(\"workload\").count().orderBy(F.desc(\"count\")).show(truncate=False)\n", + "\n", + "# By release type\n", + "print(\"\\nšŸ”ø By Release Type:\")\n", + "df_summary.groupBy(\"release_type\").count().orderBy(F.desc(\"count\")).show(truncate=False)\n", + "\n", + "# By release status\n", + "print(\"\\nšŸ”ø By Release Status:\")\n", + "df_summary.groupBy(\"release_status\").count().orderBy(F.desc(\"count\")).show(truncate=False)\n", + "\n", + "# Preview vs GA\n", + "print(\"\\nšŸ”ø Preview vs GA:\")\n", + "df_summary.groupBy(\"is_preview\").count().show(truncate=False)\n", + "\n", + "# Planned vs Shipped\n", + "print(\"\\nšŸ”ø Planned vs Shipped:\")\n", + "df_summary.select(\"is_planned\", \"is_shipped\").groupBy(\"is_planned\", \"is_shipped\").count().show(truncate=False)\n", + "\n", + "# Recent modifications (last 30 days)\n", + "recent_cutoff = datetime.now() - timedelta(days=30)\n", + "recent_count = df_summary.filter(F.col(\"last_modified\") >= recent_cutoff).count()\n", + "print(f\"\\nšŸ”ø Features modified in last 30 days: {recent_count}\")\n", + "\n", + "# Upcoming releases (planned)\n", + "planned_count = df_summary.filter(F.col(\"is_planned\") == True).count()\n", + "print(f\"šŸ”ø Planned features (roadmap): {planned_count}\")\n", + "\n", + "# Show upcoming preview features\n", + "print(\"\\nšŸ”ø Upcoming Preview Features (sample):\")\n", + "df_summary.filter((F.col(\"is_preview\") == True) & (F.col(\"is_planned\") == True)) \\\n", + " .select(\"feature_name\", \"product_name\", \"release_date\", \"release_status\") \\\n", + " .orderBy(F.col(\"release_date\").asc()) \\\n", + " .show(10, truncate=60)\n", + "\n", + "print(\"\\n\" + \"=\" * 70)\n", + "print(\"āœ… Transfer Feature Releases GpsApi Unit - COMPLETED\")\n", + "print(f\"šŸ’” Tip: Use this table for roadmap planning and preview feature tracking\")" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/monitoring/fabric-unified-admin-monitoring/src/01_Transfer_Feature_Releases_Unit.Notebook/.platform b/monitoring/fabric-unified-admin-monitoring/src/01_Transfer_Feature_Releases_Unit.Notebook/.platform new file mode 100644 index 00000000..c7df0caf --- /dev/null +++ b/monitoring/fabric-unified-admin-monitoring/src/01_Transfer_Feature_Releases_Unit.Notebook/.platform @@ -0,0 +1,12 @@ +{ + "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", + "metadata": { + "type": "Notebook", + "displayName": "01_Transfer_Feature_Releases_Unit", + "description": "Extract Microsoft Fabric feature releases from Microsoft Learn" + }, + "config": { + "version": "2.0", + "logicalId": "00000000-0000-0000-0000-000000000000" + } +} diff --git a/monitoring/fabric-unified-admin-monitoring/src/01_Transfer_Feature_Releases_Unit.Notebook/notebook-content.ipynb b/monitoring/fabric-unified-admin-monitoring/src/01_Transfer_Feature_Releases_Unit.Notebook/notebook-content.ipynb new file mode 100644 index 00000000..4e7e5449 --- /dev/null +++ b/monitoring/fabric-unified-admin-monitoring/src/01_Transfer_Feature_Releases_Unit.Notebook/notebook-content.ipynb @@ -0,0 +1,324 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "6be68f0c", + "metadata": {}, + "source": [ + "# 01 - Transfer Feature Releases Unit\n", + "\n", + "**Purpose**: Extract Microsoft Fabric feature releases from Microsoft Learn what's new documentation\n", + "\n", + "**Inputs**:\n", + "- Microsoft Learn URL: https://learn.microsoft.com/en-us/fabric/get-started/whats-new\n", + "\n", + "**Outputs**:\n", + "- Delta Table: `feature_releases`\n", + "\n", + "**Frequency**: Daily (recommended)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b00196e6", + "metadata": {}, + "outputs": [], + "source": [ + "import requests\n", + "import re\n", + "from datetime import datetime, timedelta\n", + "from pyspark.sql import functions as F\n", + "from pyspark.sql.types import *" + ] + }, + { + "cell_type": "markdown", + "id": "5ddc0cf4", + "metadata": {}, + "source": [ + "## Parameters" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b7540d3c", + "metadata": {}, + "outputs": [], + "source": [ + "# Parameters (can be passed from Pipeline)\n", + "whats_new_url = \"https://learn.microsoft.com/en-us/fabric/get-started/whats-new\"\n", + "lookback_days = 90 # Extract features from last 90 days" + ] + }, + { + "cell_type": "markdown", + "id": "ddba066b", + "metadata": {}, + "source": [ + "## Step 1: Fetch What's New Content" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "63f9cdc9", + "metadata": {}, + "outputs": [], + "source": [ + "print(f\"šŸ”„ Fetching Fabric What's New from: {whats_new_url}\")\n", + "\n", + "try:\n", + " response = requests.get(whats_new_url, timeout=30)\n", + " response.raise_for_status()\n", + " content = response.text\n", + " print(f\"āœ… Successfully fetched {len(content)} characters\")\n", + "except Exception as e:\n", + " print(f\"āŒ Error fetching content: {e}\")\n", + " raise" + ] + }, + { + "cell_type": "markdown", + "id": "39a8fb67", + "metadata": {}, + "source": [ + "## Step 2: Parse Feature Releases" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9eced003", + "metadata": {}, + "outputs": [], + "source": [ + "def parse_feature_releases(html_content):\n", + " \"\"\"\n", + " Parse feature releases from Microsoft Learn What's New page\n", + " Returns list of dicts with feature information\n", + " \"\"\"\n", + " features = []\n", + " \n", + " # Pattern to match feature sections\n", + " # Example: ## September 2024\n", + " month_pattern = r'##\\s+([A-Za-z]+\\s+\\d{4})'\n", + " \n", + " # Pattern to match feature entries\n", + " # Example: ### Data Warehouse - New feature (Preview)\n", + " feature_pattern = r'###\\s+(.+?)\\s*(?:\\(([^)]+)\\))?'\n", + " \n", + " # Split content by months\n", + " months = re.split(month_pattern, html_content)\n", + " \n", + " for i in range(1, len(months), 2):\n", + " if i+1 >= len(months):\n", + " break\n", + " \n", + " month_str = months[i].strip()\n", + " month_content = months[i+1]\n", + " \n", + " try:\n", + " release_date = datetime.strptime(month_str, \"%B %Y\")\n", + " except:\n", + " continue\n", + " \n", + " # Find all features in this month\n", + " feature_matches = re.finditer(feature_pattern, month_content)\n", + " \n", + " for match in feature_matches:\n", + " feature_name = match.group(1).strip()\n", + " status = match.group(2).strip() if match.group(2) else \"GA\"\n", + " \n", + " # Determine workload from feature name\n", + " workload = \"Unknown\"\n", + " if any(kw in feature_name.lower() for kw in [\"data warehouse\", \"warehouse\", \"sql\"]):\n", + " workload = \"Data Warehouse\"\n", + " elif any(kw in feature_name.lower() for kw in [\"data factory\", \"pipeline\", \"dataflow\"]):\n", + " workload = \"Data Factory\"\n", + " elif any(kw in feature_name.lower() for kw in [\"power bi\", \"semantic model\", \"report\"]):\n", + " workload = \"Power BI\"\n", + " elif any(kw in feature_name.lower() for kw in [\"real-time\", \"kql\", \"eventhouse\"]):\n", + " workload = \"Real-Time Intelligence\"\n", + " elif any(kw in feature_name.lower() for kw in [\"lakehouse\", \"onelake\"]):\n", + " workload = \"Data Engineering\"\n", + " elif any(kw in feature_name.lower() for kw in [\"data science\", \"ml\", \"notebook\"]):\n", + " workload = \"Data Science\"\n", + " \n", + " # Determine if it's a preview feature\n", + " is_preview = \"preview\" in status.lower()\n", + " \n", + " features.append({\n", + " \"feature_id\": f\"{release_date.strftime('%Y%m')}_{hash(feature_name) % 100000}\",\n", + " \"feature_name\": feature_name,\n", + " \"workload\": workload,\n", + " \"release_date\": release_date,\n", + " \"status\": status,\n", + " \"is_preview\": is_preview,\n", + " \"source_url\": whats_new_url,\n", + " \"extracted_date\": datetime.now()\n", + " })\n", + " \n", + " return features" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "05e9a48b", + "metadata": {}, + "outputs": [], + "source": [ + "print(\"šŸ”„ Parsing feature releases...\")\n", + "features_data = parse_feature_releases(content)\n", + "\n", + "# Filter to last N days\n", + "cutoff_date = datetime.now() - timedelta(days=lookback_days)\n", + "features_data = [f for f in features_data if f[\"release_date\"] >= cutoff_date]\n", + "\n", + "print(f\"āœ… Parsed {len(features_data)} feature releases from last {lookback_days} days\")\n", + "\n", + "# Preview\n", + "if features_data:\n", + " print(\"\\nšŸ“‹ Sample features:\")\n", + " for feat in features_data[:3]:\n", + " print(f\" - {feat['feature_name']} ({feat['status']}) - {feat['workload']}\")" + ] + }, + { + "cell_type": "markdown", + "id": "54aa947a", + "metadata": {}, + "source": [ + "## Step 3: Create DataFrame" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0df118f4", + "metadata": {}, + "outputs": [], + "source": [ + "schema = StructType([\n", + " StructField(\"feature_id\", StringType(), False),\n", + " StructField(\"feature_name\", StringType(), False),\n", + " StructField(\"workload\", StringType(), True),\n", + " StructField(\"release_date\", TimestampType(), False),\n", + " StructField(\"status\", StringType(), True),\n", + " StructField(\"is_preview\", BooleanType(), False),\n", + " StructField(\"source_url\", StringType(), True),\n", + " StructField(\"extracted_date\", TimestampType(), False)\n", + "])\n", + "\n", + "df_features = spark.createDataFrame(features_data, schema=schema)\n", + "\n", + "print(f\"āœ… Created DataFrame with {df_features.count()} rows\")\n", + "df_features.show(5, truncate=False)" + ] + }, + { + "cell_type": "markdown", + "id": "f9eeb551", + "metadata": {}, + "source": [ + "## Step 4: Write to Delta Lake" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c99f1c39", + "metadata": {}, + "outputs": [], + "source": [ + "# Define table path\n", + "table_name = \"feature_releases\"\n", + "table_path = f\"Tables/{table_name}\"\n", + "\n", + "print(f\"šŸ”„ Writing to Delta table: {table_name}\")\n", + "\n", + "# Write with merge logic (upsert)\n", + "try:\n", + " from delta.tables import DeltaTable\n", + " \n", + " # Check if table exists\n", + " if DeltaTable.isDeltaTable(spark, table_path):\n", + " print(\" ā†’ Table exists, performing MERGE (upsert)...\")\n", + " \n", + " delta_table = DeltaTable.forPath(spark, table_path)\n", + " \n", + " # Merge: Update existing, insert new\n", + " delta_table.alias(\"target\").merge(\n", + " df_features.alias(\"source\"),\n", + " \"target.feature_id = source.feature_id\"\n", + " ).whenMatchedUpdateAll(\n", + " ).whenNotMatchedInsertAll(\n", + " ).execute()\n", + " \n", + " print(\" āœ… MERGE completed\")\n", + " else:\n", + " print(\" ā†’ Table doesn't exist, creating new table...\")\n", + " df_features.write.format(\"delta\").mode(\"overwrite\").save(table_path)\n", + " print(\" āœ… Table created\")\n", + " \n", + " # Show final count\n", + " final_count = spark.read.format(\"delta\").load(table_path).count()\n", + " print(f\"\\nāœ… Total records in {table_name}: {final_count}\")\n", + " \n", + "except Exception as e:\n", + " print(f\"āŒ Error writing to Delta: {e}\")\n", + " raise" + ] + }, + { + "cell_type": "markdown", + "id": "6b8f45b2", + "metadata": {}, + "source": [ + "## Step 5: Summary Statistics" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "625df20e", + "metadata": {}, + "outputs": [], + "source": [ + "print(\"\\nšŸ“Š Feature Release Statistics:\")\n", + "print(\"=\" * 60)\n", + "\n", + "df_summary = spark.read.format(\"delta\").load(table_path)\n", + "\n", + "# By workload\n", + "print(\"\\nšŸ”ø By Workload:\")\n", + "df_summary.groupBy(\"workload\").count().orderBy(F.desc(\"count\")).show(truncate=False)\n", + "\n", + "# By status\n", + "print(\"\\nšŸ”ø By Status:\")\n", + "df_summary.groupBy(\"status\").count().orderBy(F.desc(\"count\")).show(truncate=False)\n", + "\n", + "# Preview vs GA\n", + "print(\"\\nšŸ”ø Preview vs GA:\")\n", + "df_summary.groupBy(\"is_preview\").count().show(truncate=False)\n", + "\n", + "# Recent releases (last 30 days)\n", + "recent_cutoff = datetime.now() - timedelta(days=30)\n", + "recent_count = df_summary.filter(F.col(\"release_date\") >= recent_cutoff).count()\n", + "print(f\"\\nšŸ”ø Features released in last 30 days: {recent_count}\")\n", + "\n", + "print(\"\\n\" + \"=\" * 60)\n", + "print(\"āœ… Transfer Feature Releases Unit - COMPLETED\")" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/monitoring/fabric-unified-admin-monitoring/src/02_Transfer_Preview_Features_Unit.Notebook/.platform b/monitoring/fabric-unified-admin-monitoring/src/02_Transfer_Preview_Features_Unit.Notebook/.platform new file mode 100644 index 00000000..61331b64 --- /dev/null +++ b/monitoring/fabric-unified-admin-monitoring/src/02_Transfer_Preview_Features_Unit.Notebook/.platform @@ -0,0 +1,12 @@ +{ + "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", + "metadata": { + "type": "Notebook", + "displayName": "02_Transfer_Preview_Features_Unit", + "description": "Detect activated preview features by comparing tenant settings" + }, + "config": { + "version": "2.0", + "logicalId": "00000000-0000-0000-0000-000000000000" + } +} diff --git a/monitoring/fabric-unified-admin-monitoring/src/02_Transfer_Preview_Features_Unit.Notebook/notebook-content.ipynb b/monitoring/fabric-unified-admin-monitoring/src/02_Transfer_Preview_Features_Unit.Notebook/notebook-content.ipynb new file mode 100644 index 00000000..edbda538 --- /dev/null +++ b/monitoring/fabric-unified-admin-monitoring/src/02_Transfer_Preview_Features_Unit.Notebook/notebook-content.ipynb @@ -0,0 +1,372 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "6a7f70e9", + "metadata": {}, + "source": [ + "# 02 - Transfer Preview Features Unit\n", + "\n", + "**Purpose**: Detect activated preview features by comparing tenant settings with feature releases\n", + "\n", + "**Inputs**:\n", + "- Delta Table: `tenant_settings` (from FUAM)\n", + "- Delta Table: `feature_releases` (from 01_Transfer_Feature_Releases_Unit)\n", + "\n", + "**Outputs**:\n", + "- Delta Table: `preview_features_active`\n", + "\n", + "**Frequency**: Daily (recommended)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8a78d2d0", + "metadata": {}, + "outputs": [], + "source": [ + "from pyspark.sql import functions as F\n", + "from pyspark.sql.window import Window\n", + "from pyspark.sql.types import *\n", + "from datetime import datetime\n", + "from difflib import SequenceMatcher" + ] + }, + { + "cell_type": "markdown", + "id": "89e5c031", + "metadata": {}, + "source": [ + "## Step 1: Load Feature Releases (Preview only)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3fbcbd68", + "metadata": {}, + "outputs": [], + "source": [ + "print(\"šŸ”„ Loading preview feature releases...\")\n", + "\n", + "df_features = spark.read.format(\"delta\").load(\"Tables/feature_releases\")\n", + "\n", + "# Filter to preview features only\n", + "df_preview_features = df_features.filter(F.col(\"is_preview\") == True)\n", + "\n", + "preview_count = df_preview_features.count()\n", + "print(f\"āœ… Loaded {preview_count} preview features\")\n", + "\n", + "# Show sample\n", + "df_preview_features.select(\"feature_id\", \"feature_name\", \"workload\", \"release_date\").show(5, truncate=False)" + ] + }, + { + "cell_type": "markdown", + "id": "b5353e2d", + "metadata": {}, + "source": [ + "## Step 2: Load Tenant Settings" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c88e29ee", + "metadata": {}, + "outputs": [], + "source": [ + "print(\"\\nšŸ”„ Loading tenant settings...\")\n", + "\n", + "df_tenant_settings = spark.read.format(\"delta\").load(\"Tables/tenant_settings\")\n", + "\n", + "settings_count = df_tenant_settings.count()\n", + "print(f\"āœ… Loaded {settings_count} tenant settings\")\n", + "\n", + "# Show sample\n", + "df_tenant_settings.select(\"settingName\", \"enabled\", \"delegateToTenant\").show(5, truncate=False)" + ] + }, + { + "cell_type": "markdown", + "id": "8f7422f6", + "metadata": {}, + "source": [ + "## Step 3: Map Settings to Features\n", + "\n", + "This creates a mapping between tenant setting names and preview feature names using fuzzy matching" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd21a1bc", + "metadata": {}, + "outputs": [], + "source": [ + "def similarity_score(a, b):\n", + " \"\"\"Calculate similarity between two strings (0-1)\"\"\"\n", + " return SequenceMatcher(None, a.lower(), b.lower()).ratio()\n", + "\n", + "def map_settings_to_features(df_settings, df_features):\n", + " \"\"\"\n", + " Map tenant settings to preview features based on name similarity\n", + " Returns DataFrame with matches\n", + " \"\"\"\n", + " \n", + " # Collect feature names for matching\n", + " features_list = df_features.select(\"feature_id\", \"feature_name\", \"workload\").collect()\n", + " settings_list = df_settings.select(\"settingName\", \"enabled\", \"delegateToTenant\").collect()\n", + " \n", + " matches = []\n", + " \n", + " for setting in settings_list:\n", + " setting_name = setting[\"settingName\"]\n", + " \n", + " # Skip if setting is not enabled\n", + " if not setting[\"enabled\"]:\n", + " continue\n", + " \n", + " # Find best matching feature\n", + " best_match = None\n", + " best_score = 0.0\n", + " \n", + " for feature in features_list:\n", + " feature_name = feature[\"feature_name\"]\n", + " \n", + " # Calculate similarity\n", + " score = similarity_score(setting_name, feature_name)\n", + " \n", + " # Also check for keyword matches\n", + " setting_words = set(setting_name.lower().split())\n", + " feature_words = set(feature_name.lower().split())\n", + " common_words = setting_words & feature_words\n", + " \n", + " if common_words:\n", + " score += len(common_words) * 0.1 # Boost score for common words\n", + " \n", + " if score > best_score and score > 0.3: # Threshold 0.3\n", + " best_score = score\n", + " best_match = feature\n", + " \n", + " if best_match:\n", + " matches.append({\n", + " \"setting_name\": setting_name,\n", + " \"feature_id\": best_match[\"feature_id\"],\n", + " \"feature_name\": best_match[\"feature_name\"],\n", + " \"workload\": best_match[\"workload\"],\n", + " \"similarity_score\": best_score,\n", + " \"is_enabled\": setting[\"enabled\"],\n", + " \"delegate_to_tenant\": setting[\"delegateToTenant\"],\n", + " \"detected_date\": datetime.now()\n", + " })\n", + " \n", + " return matches" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a9ec9449", + "metadata": {}, + "outputs": [], + "source": [ + "print(\"šŸ”„ Mapping settings to preview features...\")\n", + "\n", + "matches = map_settings_to_features(df_tenant_settings, df_preview_features)\n", + "\n", + "print(f\"āœ… Found {len(matches)} potential preview features activated\")\n", + "\n", + "# Preview matches\n", + "if matches:\n", + " print(\"\\nšŸ“‹ Sample matches:\")\n", + " for match in matches[:5]:\n", + " print(f\" - {match['feature_name']}\")\n", + " print(f\" ā†’ Setting: {match['setting_name']}\")\n", + " print(f\" ā†’ Similarity: {match['similarity_score']:.2f}\")\n", + " print()" + ] + }, + { + "cell_type": "markdown", + "id": "ab1740ab", + "metadata": {}, + "source": [ + "## Step 4: Create DataFrame with Activated Previews" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a271ca26", + "metadata": {}, + "outputs": [], + "source": [ + "schema = StructType([\n", + " StructField(\"setting_name\", StringType(), False),\n", + " StructField(\"feature_id\", StringType(), False),\n", + " StructField(\"feature_name\", StringType(), False),\n", + " StructField(\"workload\", StringType(), True),\n", + " StructField(\"similarity_score\", DoubleType(), False),\n", + " StructField(\"is_enabled\", BooleanType(), False),\n", + " StructField(\"delegate_to_tenant\", BooleanType(), True),\n", + " StructField(\"detected_date\", TimestampType(), False)\n", + "])\n", + "\n", + "df_active_previews = spark.createDataFrame(matches, schema=schema)\n", + "\n", + "print(f\"āœ… Created DataFrame with {df_active_previews.count()} activated preview features\")\n", + "df_active_previews.show(5, truncate=False)" + ] + }, + { + "cell_type": "markdown", + "id": "4e776aac", + "metadata": {}, + "source": [ + "## Step 5: Enrich with Feature Details" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0593e47f", + "metadata": {}, + "outputs": [], + "source": [ + "print(\"šŸ”„ Enriching with feature release details...\")\n", + "\n", + "# Join with feature_releases to get release_date, status, etc.\n", + "df_enriched = df_active_previews.join(\n", + " df_preview_features.select(\n", + " \"feature_id\",\n", + " \"release_date\",\n", + " \"status\",\n", + " \"source_url\"\n", + " ),\n", + " on=\"feature_id\",\n", + " how=\"left\"\n", + ")\n", + "\n", + "# Calculate days since release\n", + "df_enriched = df_enriched.withColumn(\n", + " \"days_since_release\",\n", + " F.datediff(F.current_date(), F.col(\"release_date\"))\n", + ")\n", + "\n", + "print(\"āœ… Enrichment completed\")\n", + "df_enriched.show(5, truncate=False)" + ] + }, + { + "cell_type": "markdown", + "id": "28244a88", + "metadata": {}, + "source": [ + "## Step 6: Write to Delta Lake" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "34b079a1", + "metadata": {}, + "outputs": [], + "source": [ + "table_name = \"preview_features_active\"\n", + "table_path = f\"Tables/{table_name}\"\n", + "\n", + "print(f\"šŸ”„ Writing to Delta table: {table_name}\")\n", + "\n", + "try:\n", + " from delta.tables import DeltaTable\n", + " \n", + " # Check if table exists\n", + " if DeltaTable.isDeltaTable(spark, table_path):\n", + " print(\" ā†’ Table exists, performing MERGE (upsert)...\")\n", + " \n", + " delta_table = DeltaTable.forPath(spark, table_path)\n", + " \n", + " # Merge: Update existing, insert new\n", + " delta_table.alias(\"target\").merge(\n", + " df_enriched.alias(\"source\"),\n", + " \"target.feature_id = source.feature_id AND target.setting_name = source.setting_name\"\n", + " ).whenMatchedUpdate(\n", + " set={\n", + " \"is_enabled\": \"source.is_enabled\",\n", + " \"detected_date\": \"source.detected_date\",\n", + " \"days_since_release\": \"source.days_since_release\"\n", + " }\n", + " ).whenNotMatchedInsertAll(\n", + " ).execute()\n", + " \n", + " print(\" āœ… MERGE completed\")\n", + " else:\n", + " print(\" ā†’ Table doesn't exist, creating new table...\")\n", + " df_enriched.write.format(\"delta\").mode(\"overwrite\").save(table_path)\n", + " print(\" āœ… Table created\")\n", + " \n", + " # Show final count\n", + " final_count = spark.read.format(\"delta\").load(table_path).count()\n", + " print(f\"\\nāœ… Total activated preview features in {table_name}: {final_count}\")\n", + " \n", + "except Exception as e:\n", + " print(f\"āŒ Error writing to Delta: {e}\")\n", + " raise" + ] + }, + { + "cell_type": "markdown", + "id": "dd1e5bdd", + "metadata": {}, + "source": [ + "## Step 7: Summary Statistics" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bc0d84f5", + "metadata": {}, + "outputs": [], + "source": [ + "print(\"\\nšŸ“Š Activated Preview Features Statistics:\")\n", + "print(\"=\" * 60)\n", + "\n", + "df_summary = spark.read.format(\"delta\").load(table_path)\n", + "\n", + "# By workload\n", + "print(\"\\nšŸ”ø By Workload:\")\n", + "df_summary.groupBy(\"workload\").count().orderBy(F.desc(\"count\")).show(truncate=False)\n", + "\n", + "# By similarity score range\n", + "print(\"\\nšŸ”ø By Confidence Level (Similarity Score):\")\n", + "df_summary.withColumn(\n", + " \"confidence\",\n", + " F.when(F.col(\"similarity_score\") >= 0.7, \"High (ā‰„0.7)\")\n", + " .when(F.col(\"similarity_score\") >= 0.5, \"Medium (0.5-0.7)\")\n", + " .otherwise(\"Low (<0.5)\")\n", + ").groupBy(\"confidence\").count().orderBy(F.desc(\"count\")).show(truncate=False)\n", + "\n", + "# Recently released (last 30 days) but already activated\n", + "recent_activated = df_summary.filter(F.col(\"days_since_release\") <= 30).count()\n", + "print(f\"\\nšŸ”ø Features activated within 30 days of release: {recent_activated}\")\n", + "\n", + "# Features that have been in preview for long time\n", + "long_preview = df_summary.filter(F.col(\"days_since_release\") > 180).count()\n", + "print(f\"šŸ”ø Features in preview for >180 days: {long_preview}\")\n", + "\n", + "print(\"\\n\" + \"=\" * 60)\n", + "print(\"āœ… Transfer Preview Features Unit - COMPLETED\")" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/monitoring/fabric-unified-admin-monitoring/src/03_Transfer_Feature_Alerts_Unit.Notebook/.platform b/monitoring/fabric-unified-admin-monitoring/src/03_Transfer_Feature_Alerts_Unit.Notebook/.platform new file mode 100644 index 00000000..3c394b8f --- /dev/null +++ b/monitoring/fabric-unified-admin-monitoring/src/03_Transfer_Feature_Alerts_Unit.Notebook/.platform @@ -0,0 +1,12 @@ +{ + "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", + "metadata": { + "type": "Notebook", + "displayName": "03_Transfer_Feature_Alerts_Unit", + "description": "Generate alerts for newly activated or high-risk preview features" + }, + "config": { + "version": "2.0", + "logicalId": "00000000-0000-0000-0000-000000000000" + } +} diff --git a/monitoring/fabric-unified-admin-monitoring/src/03_Transfer_Feature_Alerts_Unit.Notebook/notebook-content.ipynb b/monitoring/fabric-unified-admin-monitoring/src/03_Transfer_Feature_Alerts_Unit.Notebook/notebook-content.ipynb new file mode 100644 index 00000000..aafd9da2 --- /dev/null +++ b/monitoring/fabric-unified-admin-monitoring/src/03_Transfer_Feature_Alerts_Unit.Notebook/notebook-content.ipynb @@ -0,0 +1,417 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "8230571e", + "metadata": {}, + "source": [ + "# 03 - Transfer Feature Alerts Unit\n", + "\n", + "**Purpose**: Generate alerts for newly activated or high-risk preview features\n", + "\n", + "**Inputs**:\n", + "- Delta Table: `preview_features_active`\n", + "- Delta Table: `feature_alerts` (for historical tracking)\n", + "\n", + "**Outputs**:\n", + "- Delta Table: `feature_alerts`\n", + "\n", + "**Frequency**: Daily (recommended)\n", + "\n", + "**Alert Triggers**:\n", + "- New preview feature detected\n", + "- Preview feature active >90 days (approaching GA)\n", + "- Preview feature with low confidence match (similarity <0.5)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "aa08e36b", + "metadata": {}, + "outputs": [], + "source": [ + "from pyspark.sql import functions as F\n", + "from pyspark.sql.types import *\n", + "from datetime import datetime, timedelta" + ] + }, + { + "cell_type": "markdown", + "id": "b031c9ec", + "metadata": {}, + "source": [ + "## Parameters" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "88adb1d2", + "metadata": {}, + "outputs": [], + "source": [ + "# Alert thresholds\n", + "ALERT_DAYS_THRESHOLD = 90 # Alert if preview active >90 days\n", + "LOW_CONFIDENCE_THRESHOLD = 0.5 # Alert if similarity score <0.5\n", + "\n", + "# Alert severity levels\n", + "SEVERITY_INFO = \"Info\"\n", + "SEVERITY_WARNING = \"Warning\"\n", + "SEVERITY_CRITICAL = \"Critical\"" + ] + }, + { + "cell_type": "markdown", + "id": "e3ff6936", + "metadata": {}, + "source": [ + "## Step 1: Load Current Activated Preview Features" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c07b23d7", + "metadata": {}, + "outputs": [], + "source": [ + "print(\"šŸ”„ Loading activated preview features...\")\n", + "\n", + "df_active = spark.read.format(\"delta\").load(\"Tables/preview_features_active\")\n", + "\n", + "active_count = df_active.count()\n", + "print(f\"āœ… Loaded {active_count} activated preview features\")\n", + "\n", + "df_active.select(\"feature_id\", \"feature_name\", \"workload\", \"days_since_release\", \"similarity_score\").show(5, truncate=False)" + ] + }, + { + "cell_type": "markdown", + "id": "f991be8e", + "metadata": {}, + "source": [ + "## Step 2: Load Historical Alerts (to avoid duplicates)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7fe05695", + "metadata": {}, + "outputs": [], + "source": [ + "table_name = \"feature_alerts\"\n", + "table_path = f\"Tables/{table_name}\"\n", + "\n", + "try:\n", + " from delta.tables import DeltaTable\n", + " \n", + " if DeltaTable.isDeltaTable(spark, table_path):\n", + " print(\"šŸ”„ Loading historical alerts...\")\n", + " df_historical = spark.read.format(\"delta\").load(table_path)\n", + " historical_count = df_historical.count()\n", + " print(f\"āœ… Loaded {historical_count} historical alerts\")\n", + " \n", + " # Get already alerted feature_ids\n", + " alerted_features = set([row[\"feature_id\"] for row in df_historical.select(\"feature_id\").distinct().collect()])\n", + " print(f\" ā†’ {len(alerted_features)} unique features already alerted\")\n", + " else:\n", + " print(\"ā„¹ļø No historical alerts found (first run)\")\n", + " df_historical = None\n", + " alerted_features = set()\n", + "except Exception as e:\n", + " print(f\"ā„¹ļø No historical alerts (first run): {e}\")\n", + " df_historical = None\n", + " alerted_features = set()" + ] + }, + { + "cell_type": "markdown", + "id": "440eeb33", + "metadata": {}, + "source": [ + "## Step 3: Generate Alerts" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a948004d", + "metadata": {}, + "outputs": [], + "source": [ + "def generate_alerts(df_active, alerted_features):\n", + " \"\"\"\n", + " Generate alerts based on business rules\n", + " Returns list of alert dicts\n", + " \"\"\"\n", + " alerts = []\n", + " \n", + " for row in df_active.collect():\n", + " feature_id = row[\"feature_id\"]\n", + " feature_name = row[\"feature_name\"]\n", + " workload = row[\"workload\"]\n", + " days_since_release = row[\"days_since_release\"]\n", + " similarity_score = row[\"similarity_score\"]\n", + " setting_name = row[\"setting_name\"]\n", + " \n", + " # Rule 1: NEW PREVIEW FEATURE ACTIVATED\n", + " if feature_id not in alerted_features:\n", + " alerts.append({\n", + " \"alert_id\": f\"NEW_{feature_id}_{datetime.now().strftime('%Y%m%d')}\",\n", + " \"feature_id\": feature_id,\n", + " \"feature_name\": feature_name,\n", + " \"workload\": workload,\n", + " \"alert_type\": \"New Preview Activated\",\n", + " \"severity\": SEVERITY_INFO,\n", + " \"message\": f\"New preview feature '{feature_name}' has been activated (Setting: {setting_name})\",\n", + " \"days_since_release\": days_since_release,\n", + " \"similarity_score\": similarity_score,\n", + " \"alert_date\": datetime.now(),\n", + " \"acknowledged\": False\n", + " })\n", + " \n", + " # Rule 2: LONG-RUNNING PREVIEW (>90 days)\n", + " if days_since_release and days_since_release > ALERT_DAYS_THRESHOLD:\n", + " alert_id = f\"LONGRUN_{feature_id}_{datetime.now().strftime('%Y%m%d')}\"\n", + " \n", + " alerts.append({\n", + " \"alert_id\": alert_id,\n", + " \"feature_id\": feature_id,\n", + " \"feature_name\": feature_name,\n", + " \"workload\": workload,\n", + " \"alert_type\": \"Long-Running Preview\",\n", + " \"severity\": SEVERITY_WARNING,\n", + " \"message\": f\"Preview feature '{feature_name}' has been active for {days_since_release} days. Consider reviewing for GA transition.\",\n", + " \"days_since_release\": days_since_release,\n", + " \"similarity_score\": similarity_score,\n", + " \"alert_date\": datetime.now(),\n", + " \"acknowledged\": False\n", + " })\n", + " \n", + " # Rule 3: LOW CONFIDENCE MATCH\n", + " if similarity_score < LOW_CONFIDENCE_THRESHOLD:\n", + " alerts.append({\n", + " \"alert_id\": f\"LOWCONF_{feature_id}_{datetime.now().strftime('%Y%m%d')}\",\n", + " \"feature_id\": feature_id,\n", + " \"feature_name\": feature_name,\n", + " \"workload\": workload,\n", + " \"alert_type\": \"Low Confidence Match\",\n", + " \"severity\": SEVERITY_CRITICAL,\n", + " \"message\": f\"Low confidence match ({similarity_score:.2f}) between setting '{setting_name}' and feature '{feature_name}'. Manual review recommended.\",\n", + " \"days_since_release\": days_since_release,\n", + " \"similarity_score\": similarity_score,\n", + " \"alert_date\": datetime.now(),\n", + " \"acknowledged\": False\n", + " })\n", + " \n", + " return alerts" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e216ee50", + "metadata": {}, + "outputs": [], + "source": [ + "print(\"šŸ”„ Generating alerts based on business rules...\")\n", + "\n", + "alerts_data = generate_alerts(df_active, alerted_features)\n", + "\n", + "print(f\"āœ… Generated {len(alerts_data)} new alerts\")\n", + "\n", + "# Preview alerts\n", + "if alerts_data:\n", + " print(\"\\nšŸ“‹ Sample alerts:\")\n", + " for alert in alerts_data[:5]:\n", + " print(f\" [{alert['severity']}] {alert['alert_type']}\")\n", + " print(f\" ā†’ {alert['message']}\")\n", + " print()" + ] + }, + { + "cell_type": "markdown", + "id": "20a7afc0", + "metadata": {}, + "source": [ + "## Step 4: Create Alerts DataFrame" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6a8f9b72", + "metadata": {}, + "outputs": [], + "source": [ + "if alerts_data:\n", + " schema = StructType([\n", + " StructField(\"alert_id\", StringType(), False),\n", + " StructField(\"feature_id\", StringType(), False),\n", + " StructField(\"feature_name\", StringType(), False),\n", + " StructField(\"workload\", StringType(), True),\n", + " StructField(\"alert_type\", StringType(), False),\n", + " StructField(\"severity\", StringType(), False),\n", + " StructField(\"message\", StringType(), False),\n", + " StructField(\"days_since_release\", IntegerType(), True),\n", + " StructField(\"similarity_score\", DoubleType(), True),\n", + " StructField(\"alert_date\", TimestampType(), False),\n", + " StructField(\"acknowledged\", BooleanType(), False)\n", + " ])\n", + " \n", + " df_alerts = spark.createDataFrame(alerts_data, schema=schema)\n", + " \n", + " print(f\"āœ… Created alerts DataFrame with {df_alerts.count()} rows\")\n", + " df_alerts.show(5, truncate=False)\n", + "else:\n", + " print(\"ā„¹ļø No new alerts to generate\")\n", + " df_alerts = None" + ] + }, + { + "cell_type": "markdown", + "id": "8ae1e2fc", + "metadata": {}, + "source": [ + "## Step 5: Write Alerts to Delta Lake" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "49940421", + "metadata": {}, + "outputs": [], + "source": [ + "if df_alerts and df_alerts.count() > 0:\n", + " print(f\"šŸ”„ Writing alerts to Delta table: {table_name}\")\n", + " \n", + " try:\n", + " from delta.tables import DeltaTable\n", + " \n", + " # Check if table exists\n", + " if DeltaTable.isDeltaTable(spark, table_path):\n", + " print(\" ā†’ Table exists, appending new alerts...\")\n", + " \n", + " # Append mode - keep all alerts for historical tracking\n", + " df_alerts.write.format(\"delta\").mode(\"append\").save(table_path)\n", + " \n", + " print(\" āœ… Alerts appended\")\n", + " else:\n", + " print(\" ā†’ Table doesn't exist, creating new table...\")\n", + " df_alerts.write.format(\"delta\").mode(\"overwrite\").save(table_path)\n", + " print(\" āœ… Table created\")\n", + " \n", + " # Show final count\n", + " final_count = spark.read.format(\"delta\").load(table_path).count()\n", + " print(f\"\\nāœ… Total alerts in {table_name}: {final_count}\")\n", + " \n", + " except Exception as e:\n", + " print(f\"āŒ Error writing alerts: {e}\")\n", + " raise\n", + "else:\n", + " print(\"ā„¹ļø No alerts to write\")" + ] + }, + { + "cell_type": "markdown", + "id": "b709754b", + "metadata": {}, + "source": [ + "## Step 6: Alert Summary Statistics" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f9518879", + "metadata": {}, + "outputs": [], + "source": [ + "from delta.tables import DeltaTable\n", + "\n", + "if DeltaTable.isDeltaTable(spark, table_path):\n", + " print(\"\\nšŸ“Š Alert Statistics:\")\n", + " print(\"=\" * 60)\n", + " \n", + " df_all_alerts = spark.read.format(\"delta\").load(table_path)\n", + " \n", + " # By severity\n", + " print(\"\\nšŸ”ø By Severity:\")\n", + " df_all_alerts.groupBy(\"severity\").count().orderBy(F.desc(\"count\")).show(truncate=False)\n", + " \n", + " # By alert type\n", + " print(\"\\nšŸ”ø By Alert Type:\")\n", + " df_all_alerts.groupBy(\"alert_type\").count().orderBy(F.desc(\"count\")).show(truncate=False)\n", + " \n", + " # By workload\n", + " print(\"\\nšŸ”ø By Workload:\")\n", + " df_all_alerts.groupBy(\"workload\").count().orderBy(F.desc(\"count\")).show(truncate=False)\n", + " \n", + " # Unacknowledged alerts\n", + " unack_count = df_all_alerts.filter(F.col(\"acknowledged\") == False).count()\n", + " print(f\"\\nšŸ”ø Unacknowledged alerts: {unack_count}\")\n", + " \n", + " # Recent alerts (last 7 days)\n", + " recent_cutoff = datetime.now() - timedelta(days=7)\n", + " recent_count = df_all_alerts.filter(F.col(\"alert_date\") >= recent_cutoff).count()\n", + " print(f\"šŸ”ø Alerts in last 7 days: {recent_count}\")\n", + " \n", + " # Critical alerts\n", + " critical_count = df_all_alerts.filter(F.col(\"severity\") == SEVERITY_CRITICAL).count()\n", + " print(f\"šŸ”ø Critical alerts: {critical_count}\")\n", + " \n", + " print(\"\\n\" + \"=\" * 60)\n", + "\n", + "print(\"āœ… Transfer Feature Alerts Unit - COMPLETED\")" + ] + }, + { + "cell_type": "markdown", + "id": "f0650ea7", + "metadata": {}, + "source": [ + "## Step 7: Export Alerts for Data Activator (Optional)\n", + "\n", + "This creates a view that Data Activator can monitor" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "67823959", + "metadata": {}, + "outputs": [], + "source": [ + "from delta.tables import DeltaTable\n", + "\n", + "if DeltaTable.isDeltaTable(spark, table_path):\n", + " print(\"\\nšŸ”„ Creating view for Data Activator integration...\")\n", + " \n", + " df_activator = spark.read.format(\"delta\").load(table_path)\n", + " \n", + " # Filter to unacknowledged critical/warning alerts\n", + " df_activator_filtered = df_activator.filter(\n", + " (F.col(\"acknowledged\") == False) & \n", + " (F.col(\"severity\").isin([SEVERITY_CRITICAL, SEVERITY_WARNING]))\n", + " )\n", + " \n", + " # Create or replace temp view\n", + " df_activator_filtered.createOrReplaceTempView(\"vw_feature_alerts_active\")\n", + " \n", + " alert_count = df_activator_filtered.count()\n", + " print(f\"āœ… Created view 'vw_feature_alerts_active' with {alert_count} active alerts\")\n", + " print(\" ā†’ This view can be monitored by Data Activator for real-time notifications\")" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/monitoring/fabric-unified-admin-monitoring/src/Load_Feature_Tracking_E2E.DataPipeline/pipeline-content.json b/monitoring/fabric-unified-admin-monitoring/src/Load_Feature_Tracking_E2E.DataPipeline/pipeline-content.json new file mode 100644 index 00000000..a7607255 --- /dev/null +++ b/monitoring/fabric-unified-admin-monitoring/src/Load_Feature_Tracking_E2E.DataPipeline/pipeline-content.json @@ -0,0 +1,114 @@ +{ + "properties": { + "activities": [ + { + "name": "01_Transfer_Feature_Releases_Unit", + "type": "TridentNotebook", + "dependsOn": [], + "policy": { + "timeout": "0.12:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "typeProperties": { + "notebookId": "REPLACE_WITH_NOTEBOOK_ID", + "workspaceId": "REPLACE_WITH_WORKSPACE_ID", + "parameters": { + "whats_new_url": { + "value": { + "value": "@pipeline().parameters.whats_new_url", + "type": "Expression" + }, + "type": "string" + }, + "lookback_days": { + "value": { + "value": "@pipeline().parameters.lookback_days", + "type": "Expression" + }, + "type": "int" + } + }, + "sessionTag": "fuam_feature_tracking" + } + }, + { + "name": "02_Transfer_Preview_Features_Unit", + "type": "TridentNotebook", + "dependsOn": [ + { + "activity": "01_Transfer_Feature_Releases_Unit", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "timeout": "0.12:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "typeProperties": { + "notebookId": "REPLACE_WITH_NOTEBOOK_ID", + "workspaceId": "REPLACE_WITH_WORKSPACE_ID", + "parameters": {}, + "sessionTag": "fuam_feature_tracking" + } + }, + { + "name": "03_Transfer_Feature_Alerts_Unit", + "type": "TridentNotebook", + "dependsOn": [ + { + "activity": "02_Transfer_Preview_Features_Unit", + "dependencyConditions": [ + "Succeeded" + ] + } + ], + "policy": { + "timeout": "0.12:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "typeProperties": { + "notebookId": "REPLACE_WITH_NOTEBOOK_ID", + "workspaceId": "REPLACE_WITH_WORKSPACE_ID", + "parameters": { + "alert_days_threshold": { + "value": { + "value": "@pipeline().parameters.alert_days_threshold", + "type": "Expression" + }, + "type": "int" + } + }, + "sessionTag": "fuam_feature_tracking" + } + } + ], + "parameters": { + "whats_new_url": { + "type": "String", + "defaultValue": "https://learn.microsoft.com/en-us/fabric/get-started/whats-new" + }, + "lookback_days": { + "type": "Int", + "defaultValue": 90 + }, + "alert_days_threshold": { + "type": "Int", + "defaultValue": 90 + } + }, + "annotations": [], + "lastPublishTime": "2025-10-31T00:00:00Z" + }, + "name": "Load_Feature_Tracking_E2E" +} diff --git a/monitoring/fabric-unified-admin-monitoring/src/Setup_Feature_Tracking_Tables.Notebook/.platform b/monitoring/fabric-unified-admin-monitoring/src/Setup_Feature_Tracking_Tables.Notebook/.platform new file mode 100644 index 00000000..658ddafd --- /dev/null +++ b/monitoring/fabric-unified-admin-monitoring/src/Setup_Feature_Tracking_Tables.Notebook/.platform @@ -0,0 +1,12 @@ +{ + "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", + "metadata": { + "type": "Notebook", + "displayName": "Setup_Feature_Tracking_Tables", + "description": "One-time setup to create Delta tables for Feature Release Tracking" + }, + "config": { + "version": "2.0", + "logicalId": "00000000-0000-0000-0000-000000000000" + } +} diff --git a/monitoring/fabric-unified-admin-monitoring/src/Setup_Feature_Tracking_Tables.Notebook/notebook-content.ipynb b/monitoring/fabric-unified-admin-monitoring/src/Setup_Feature_Tracking_Tables.Notebook/notebook-content.ipynb new file mode 100644 index 00000000..8b806c90 --- /dev/null +++ b/monitoring/fabric-unified-admin-monitoring/src/Setup_Feature_Tracking_Tables.Notebook/notebook-content.ipynb @@ -0,0 +1,348 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "63d1ec5e", + "metadata": {}, + "source": [ + "# Setup Feature Tracking Tables\n", + "\n", + "**Purpose**: One-time setup to create Delta tables for Feature Release Tracking\n", + "\n", + "**Tables Created**:\n", + "1. `feature_releases` - Microsoft Fabric feature releases from What's New\n", + "2. `preview_features_active` - Preview features currently activated in tenant\n", + "3. `feature_alerts` - Alerts for preview feature activations\n", + "\n", + "**Run Once**: This notebook only needs to be executed once during initial setup\n", + "\n", + "**Prerequisites**: \n", + "- FUAM Lakehouse must exist\n", + "- `tenant_settings` table should already be created by FUAM" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c0bef748", + "metadata": {}, + "outputs": [], + "source": [ + "from pyspark.sql.types import *\n", + "from pyspark.sql import functions as F\n", + "from datetime import datetime" + ] + }, + { + "cell_type": "markdown", + "id": "217b4ddc", + "metadata": {}, + "source": [ + "## Step 1: Create `feature_releases` Table" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c8d499ac", + "metadata": {}, + "outputs": [], + "source": [ + "print(\"šŸ”„ Creating table: feature_releases\")\n", + "print(\"=\" * 60)\n", + "\n", + "schema_feature_releases = StructType([\n", + " StructField(\"feature_id\", StringType(), False),\n", + " StructField(\"feature_name\", StringType(), False),\n", + " StructField(\"workload\", StringType(), True),\n", + " StructField(\"release_date\", TimestampType(), False),\n", + " StructField(\"status\", StringType(), True),\n", + " StructField(\"is_preview\", BooleanType(), False),\n", + " StructField(\"source_url\", StringType(), True),\n", + " StructField(\"extracted_date\", TimestampType(), False)\n", + "])\n", + "\n", + "# Create empty DataFrame\n", + "df_feature_releases = spark.createDataFrame([], schema_feature_releases)\n", + "\n", + "# Write to Delta\n", + "table_path = \"Tables/feature_releases\"\n", + "df_feature_releases.write.format(\"delta\").mode(\"overwrite\").save(table_path)\n", + "\n", + "print(\"āœ… Table created: feature_releases\")\n", + "print(f\" Location: {table_path}\")\n", + "print(\"\\n Schema:\")\n", + "for field in schema_feature_releases.fields:\n", + " print(f\" - {field.name}: {field.dataType.simpleString()}\")" + ] + }, + { + "cell_type": "markdown", + "id": "3352b549", + "metadata": {}, + "source": [ + "## Step 2: Create `preview_features_active` Table" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8603d07d", + "metadata": {}, + "outputs": [], + "source": [ + "print(\"\\nšŸ”„ Creating table: preview_features_active\")\n", + "print(\"=\" * 60)\n", + "\n", + "schema_preview_active = StructType([\n", + " StructField(\"setting_name\", StringType(), False),\n", + " StructField(\"feature_id\", StringType(), False),\n", + " StructField(\"feature_name\", StringType(), False),\n", + " StructField(\"workload\", StringType(), True),\n", + " StructField(\"similarity_score\", DoubleType(), False),\n", + " StructField(\"is_enabled\", BooleanType(), False),\n", + " StructField(\"delegate_to_tenant\", BooleanType(), True),\n", + " StructField(\"detected_date\", TimestampType(), False),\n", + " StructField(\"release_date\", TimestampType(), True),\n", + " StructField(\"status\", StringType(), True),\n", + " StructField(\"source_url\", StringType(), True),\n", + " StructField(\"days_since_release\", IntegerType(), True)\n", + "])\n", + "\n", + "# Create empty DataFrame\n", + "df_preview_active = spark.createDataFrame([], schema_preview_active)\n", + "\n", + "# Write to Delta\n", + "table_path = \"Tables/preview_features_active\"\n", + "df_preview_active.write.format(\"delta\").mode(\"overwrite\").save(table_path)\n", + "\n", + "print(\"āœ… Table created: preview_features_active\")\n", + "print(f\" Location: {table_path}\")\n", + "print(\"\\n Schema:\")\n", + "for field in schema_preview_active.fields:\n", + " print(f\" - {field.name}: {field.dataType.simpleString()}\")" + ] + }, + { + "cell_type": "markdown", + "id": "ca05db54", + "metadata": {}, + "source": [ + "## Step 3: Create `feature_alerts` Table" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "21331f82", + "metadata": {}, + "outputs": [], + "source": [ + "print(\"\\nšŸ”„ Creating table: feature_alerts\")\n", + "print(\"=\" * 60)\n", + "\n", + "schema_alerts = StructType([\n", + " StructField(\"alert_id\", StringType(), False),\n", + " StructField(\"feature_id\", StringType(), False),\n", + " StructField(\"feature_name\", StringType(), False),\n", + " StructField(\"workload\", StringType(), True),\n", + " StructField(\"alert_type\", StringType(), False),\n", + " StructField(\"severity\", StringType(), False),\n", + " StructField(\"message\", StringType(), False),\n", + " StructField(\"days_since_release\", IntegerType(), True),\n", + " StructField(\"similarity_score\", DoubleType(), True),\n", + " StructField(\"alert_date\", TimestampType(), False),\n", + " StructField(\"acknowledged\", BooleanType(), False)\n", + "])\n", + "\n", + "# Create empty DataFrame\n", + "df_alerts = spark.createDataFrame([], schema_alerts)\n", + "\n", + "# Write to Delta\n", + "table_path = \"Tables/feature_alerts\"\n", + "df_alerts.write.format(\"delta\").mode(\"overwrite\").save(table_path)\n", + "\n", + "print(\"āœ… Table created: feature_alerts\")\n", + "print(f\" Location: {table_path}\")\n", + "print(\"\\n Schema:\")\n", + "for field in schema_alerts.fields:\n", + " print(f\" - {field.name}: {field.dataType.simpleString()}\")" + ] + }, + { + "cell_type": "markdown", + "id": "5b0f748b", + "metadata": {}, + "source": [ + "## Step 4: Verify Tables Created" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3396868e", + "metadata": {}, + "outputs": [], + "source": [ + "print(\"\\nšŸ“Š Verification Summary\")\n", + "print(\"=\" * 60)\n", + "\n", + "tables_to_verify = [\n", + " \"feature_releases\",\n", + " \"preview_features_active\", \n", + " \"feature_alerts\"\n", + "]\n", + "\n", + "all_tables_exist = True\n", + "\n", + "for table_name in tables_to_verify:\n", + " table_path = f\"Tables/{table_name}\"\n", + " try:\n", + " from delta.tables import DeltaTable\n", + " \n", + " if DeltaTable.isDeltaTable(spark, table_path):\n", + " df = spark.read.format(\"delta\").load(table_path)\n", + " count = df.count()\n", + " print(f\"āœ… {table_name}: EXISTS (rows={count})\")\n", + " else:\n", + " print(f\"āŒ {table_name}: NOT A DELTA TABLE\")\n", + " all_tables_exist = False\n", + " except Exception as e:\n", + " print(f\"āŒ {table_name}: ERROR - {e}\")\n", + " all_tables_exist = False\n", + "\n", + "print(\"=\" * 60)\n", + "\n", + "if all_tables_exist:\n", + " print(\"\\nšŸŽ‰ SUCCESS! All tables created successfully\")\n", + " print(\"\\nNext Steps:\")\n", + " print(\"1. Run '01_Transfer_Feature_Releases_Unit' to populate feature_releases\")\n", + " print(\"2. Run '02_Transfer_Preview_Features_Unit' to detect activated previews\")\n", + " print(\"3. Run '03_Transfer_Feature_Alerts_Unit' to generate alerts\")\n", + " print(\"\\nOR\")\n", + " print(\"ā†’ Run 'Load_Feature_Tracking_E2E' pipeline to execute all steps\")\n", + "else:\n", + " print(\"\\nāš ļø WARNING: Some tables failed to create. Review errors above.\")" + ] + }, + { + "cell_type": "markdown", + "id": "77be40c2", + "metadata": {}, + "source": [ + "## Step 5: Create Sample Queries View (Optional)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2810793e", + "metadata": {}, + "outputs": [], + "source": [ + "print(\"\\nšŸ”„ Creating helper views for SQL Endpoint...\")\n", + "\n", + "# View 1: Active Preview Features (for quick querying)\n", + "spark.sql(\"\"\"\n", + " CREATE OR REPLACE VIEW vw_active_preview_features AS\n", + " SELECT \n", + " feature_name,\n", + " workload,\n", + " setting_name,\n", + " days_since_release,\n", + " similarity_score,\n", + " release_date,\n", + " detected_date\n", + " FROM preview_features_active\n", + " WHERE is_enabled = true\n", + " ORDER BY detected_date DESC\n", + "\"\"\")\n", + "print(\"āœ… Created view: vw_active_preview_features\")\n", + "\n", + "# View 2: Unacknowledged Critical Alerts\n", + "spark.sql(\"\"\"\n", + " CREATE OR REPLACE VIEW vw_critical_alerts AS\n", + " SELECT \n", + " alert_id,\n", + " feature_name,\n", + " workload,\n", + " alert_type,\n", + " severity,\n", + " message,\n", + " alert_date\n", + " FROM feature_alerts\n", + " WHERE acknowledged = false \n", + " AND severity IN ('Critical', 'Warning')\n", + " ORDER BY alert_date DESC\n", + "\"\"\")\n", + "print(\"āœ… Created view: vw_critical_alerts\")\n", + "\n", + "# View 3: Feature Release Timeline\n", + "spark.sql(\"\"\"\n", + " CREATE OR REPLACE VIEW vw_feature_timeline AS\n", + " SELECT \n", + " feature_name,\n", + " workload,\n", + " status,\n", + " is_preview,\n", + " release_date,\n", + " DATEDIFF(CURRENT_DATE(), release_date) as days_since_release\n", + " FROM feature_releases\n", + " ORDER BY release_date DESC\n", + "\"\"\")\n", + "print(\"āœ… Created view: vw_feature_timeline\")\n", + "\n", + "print(\"\\nāœ… All views created successfully\")\n", + "print(\" ā†’ These views are accessible via SQL Endpoint\")" + ] + }, + { + "cell_type": "markdown", + "id": "1630563f", + "metadata": {}, + "source": [ + "## āœ… Setup Complete!" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b96f8877", + "metadata": {}, + "outputs": [], + "source": [ + "print(\"\\n\" + \"=\" * 60)\n", + "print(\"šŸŽ‰ FEATURE TRACKING SETUP COMPLETED!\")\n", + "print(\"=\" * 60)\n", + "\n", + "print(\"\\nšŸ“‹ Tables Created:\")\n", + "print(\" āœ… feature_releases\")\n", + "print(\" āœ… preview_features_active\")\n", + "print(\" āœ… feature_alerts\")\n", + "\n", + "print(\"\\nšŸ“‹ Views Created:\")\n", + "print(\" āœ… vw_active_preview_features\")\n", + "print(\" āœ… vw_critical_alerts\")\n", + "print(\" āœ… vw_feature_timeline\")\n", + "\n", + "print(\"\\nšŸš€ Ready to Run:\")\n", + "print(\" ā†’ Execute 'Load_Feature_Tracking_E2E' pipeline\")\n", + "print(\" ā†’ Or run individual Unit notebooks\")\n", + "\n", + "print(\"\\nšŸ“Š Query Examples:\")\n", + "print(\" SELECT * FROM vw_active_preview_features;\")\n", + "print(\" SELECT * FROM vw_critical_alerts;\")\n", + "print(\" SELECT * FROM vw_feature_timeline WHERE is_preview = true;\")\n", + "\n", + "print(\"\\n\" + \"=\" * 60)" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/monitoring/fabric-unified-admin-monitoring/src/Setup_Feature_Tracking_Tables_GpsApi.Notebook/notebook-content.ipynb b/monitoring/fabric-unified-admin-monitoring/src/Setup_Feature_Tracking_Tables_GpsApi.Notebook/notebook-content.ipynb new file mode 100644 index 00000000..1c4d7511 --- /dev/null +++ b/monitoring/fabric-unified-admin-monitoring/src/Setup_Feature_Tracking_Tables_GpsApi.Notebook/notebook-content.ipynb @@ -0,0 +1,382 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "33e13348", + "metadata": {}, + "outputs": [], + "source": [ + "from pyspark.sql.types import *\n", + "from datetime import datetime" + ] + }, + { + "cell_type": "markdown", + "id": "7d714890", + "metadata": {}, + "source": [ + "## Step 1: Create `feature_releases` Table (Original)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "97b8567b", + "metadata": {}, + "outputs": [], + "source": [ + "print(\"šŸ”„ Creating table: feature_releases\")\n", + "print(\"=\" * 60)\n", + "\n", + "schema_feature_releases = StructType([\n", + " StructField(\"feature_id\", StringType(), False),\n", + " StructField(\"feature_name\", StringType(), False),\n", + " StructField(\"workload\", StringType(), True),\n", + " StructField(\"release_date\", TimestampType(), False),\n", + " StructField(\"status\", StringType(), True),\n", + " StructField(\"is_preview\", BooleanType(), False),\n", + " StructField(\"source_url\", StringType(), True),\n", + " StructField(\"extracted_date\", TimestampType(), False)\n", + "])\n", + "\n", + "# Create empty DataFrame\n", + "df_feature_releases = spark.createDataFrame([], schema_feature_releases)\n", + "\n", + "# Write to Delta\n", + "table_path = \"Tables/feature_releases\"\n", + "df_feature_releases.write.format(\"delta\").mode(\"overwrite\").save(table_path)\n", + "\n", + "print(\"āœ… Table created: feature_releases\")\n", + "print(f\" Location: {table_path}\")\n", + "print(\"\\n Schema:\")\n", + "for field in schema_feature_releases.fields:\n", + " print(f\" - {field.name}: {field.dataType.simpleString()}\")" + ] + }, + { + "cell_type": "markdown", + "id": "f282f309", + "metadata": {}, + "source": [ + "## Step 1b: Create `feature_releases_roadmap` Table (Enhanced)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "560e3ede", + "metadata": {}, + "outputs": [], + "source": [ + "print(\"\\nšŸ”„ Creating table: feature_releases_roadmap (Fabric GPS)\")\n", + "print(\"=\" * 60)\n", + "\n", + "schema_roadmap = StructType([\n", + " StructField(\"feature_id\", StringType(), False),\n", + " StructField(\"feature_name\", StringType(), False),\n", + " StructField(\"feature_description\", StringType(), True),\n", + " StructField(\"workload\", StringType(), True),\n", + " StructField(\"product_name\", StringType(), True),\n", + " StructField(\"release_date\", TimestampType(), True), # Nullable for planned features\n", + " StructField(\"release_type\", StringType(), True),\n", + " StructField(\"release_status\", StringType(), True),\n", + " StructField(\"is_preview\", BooleanType(), False),\n", + " StructField(\"is_planned\", BooleanType(), False),\n", + " StructField(\"is_shipped\", BooleanType(), False),\n", + " StructField(\"last_modified\", TimestampType(), False),\n", + " StructField(\"source_url\", StringType(), True),\n", + " StructField(\"source\", StringType(), True),\n", + " StructField(\"extracted_date\", TimestampType(), False)\n", + "])\n", + "\n", + "# Create empty DataFrame\n", + "df_roadmap = spark.createDataFrame([], schema_roadmap)\n", + "\n", + "# Write to Delta\n", + "table_path = \"Tables/feature_releases_roadmap\"\n", + "df_roadmap.write.format(\"delta\").mode(\"overwrite\").save(table_path)\n", + "\n", + "print(\"āœ… Table created: feature_releases_roadmap\")\n", + "print(f\" Location: {table_path}\")\n", + "print(\"\\n Schema:\")\n", + "for field in schema_roadmap.fields:\n", + " print(f\" - {field.name}: {field.dataType.simpleString()}\")\n", + "print(\"\\n šŸ’” This table includes:\")\n", + "print(\" - Complete feature descriptions\")\n", + "print(\" - Planned/future features (roadmap)\")\n", + "print(\" - Historical change tracking (last_modified)\")\n", + "print(\" - Release status (Planned, In Development, Shipped)\")" + ] + }, + { + "cell_type": "markdown", + "id": "c1c72b90", + "metadata": {}, + "source": [ + "## Step 2: Create `preview_features_active` Table" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "399f3d8f", + "metadata": {}, + "outputs": [], + "source": [ + "print(\"\\nšŸ”„ Creating table: preview_features_active\")\n", + "print(\"=\" * 60)\n", + "\n", + "schema_preview_active = StructType([\n", + " StructField(\"setting_name\", StringType(), False),\n", + " StructField(\"feature_id\", StringType(), False),\n", + " StructField(\"feature_name\", StringType(), False),\n", + " StructField(\"workload\", StringType(), True),\n", + " StructField(\"similarity_score\", DoubleType(), False),\n", + " StructField(\"is_enabled\", BooleanType(), False),\n", + " StructField(\"delegate_to_tenant\", BooleanType(), True),\n", + " StructField(\"detected_date\", TimestampType(), False),\n", + " StructField(\"release_date\", TimestampType(), True),\n", + " StructField(\"status\", StringType(), True),\n", + " StructField(\"source_url\", StringType(), True),\n", + " StructField(\"days_since_release\", IntegerType(), True)\n", + "])\n", + "\n", + "# Create empty DataFrame\n", + "df_preview_active = spark.createDataFrame([], schema_preview_active)\n", + "\n", + "# Write to Delta\n", + "table_path = \"Tables/preview_features_active\"\n", + "df_preview_active.write.format(\"delta\").mode(\"overwrite\").save(table_path)\n", + "\n", + "print(\"āœ… Table created: preview_features_active\")\n", + "print(f\" Location: {table_path}\")\n", + "print(\"\\n Schema:\")\n", + "for field in schema_preview_active.fields:\n", + " print(f\" - {field.name}: {field.dataType.simpleString()}\")" + ] + }, + { + "cell_type": "markdown", + "id": "541c869f", + "metadata": {}, + "source": [ + "## Step 3: Create `feature_alerts` Table" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1505fe69", + "metadata": {}, + "outputs": [], + "source": [ + "print(\"\\nšŸ”„ Creating table: feature_alerts\")\n", + "print(\"=\" * 60)\n", + "\n", + "schema_alerts = StructType([\n", + " StructField(\"alert_id\", StringType(), False),\n", + " StructField(\"feature_id\", StringType(), False),\n", + " StructField(\"feature_name\", StringType(), False),\n", + " StructField(\"workload\", StringType(), True),\n", + " StructField(\"alert_type\", StringType(), False),\n", + " StructField(\"severity\", StringType(), False),\n", + " StructField(\"message\", StringType(), False),\n", + " StructField(\"setting_name\", StringType(), True),\n", + " StructField(\"similarity_score\", DoubleType(), True),\n", + " StructField(\"days_since_release\", IntegerType(), True),\n", + " StructField(\"alert_date\", TimestampType(), False),\n", + " StructField(\"acknowledged\", BooleanType(), False),\n", + " StructField(\"acknowledged_date\", TimestampType(), True),\n", + " StructField(\"acknowledged_by\", StringType(), True)\n", + "])\n", + "\n", + "# Create empty DataFrame\n", + "df_alerts = spark.createDataFrame([], schema_alerts)\n", + "\n", + "# Write to Delta\n", + "table_path = \"Tables/feature_alerts\"\n", + "df_alerts.write.format(\"delta\").mode(\"overwrite\").save(table_path)\n", + "\n", + "print(\"āœ… Table created: feature_alerts\")\n", + "print(f\" Location: {table_path}\")\n", + "print(\"\\n Schema:\")\n", + "for field in schema_alerts.fields:\n", + " print(f\" - {field.name}: {field.dataType.simpleString()}\")" + ] + }, + { + "cell_type": "markdown", + "id": "8cb9af32", + "metadata": {}, + "source": [ + "## Step 4: Create Helper Views" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d03a41fb", + "metadata": {}, + "outputs": [], + "source": [ + "print(\"\\nšŸ”„ Creating helper views for SQL Endpoint...\")\n", + "\n", + "# View 1: Active Preview Features (for quick querying)\n", + "spark.sql(\"\"\"\n", + " CREATE OR REPLACE VIEW vw_active_preview_features AS\n", + " SELECT \n", + " feature_name,\n", + " workload,\n", + " setting_name,\n", + " days_since_release,\n", + " similarity_score,\n", + " release_date,\n", + " detected_date\n", + " FROM preview_features_active\n", + " WHERE is_enabled = true\n", + " ORDER BY detected_date DESC\n", + "\"\"\")\n", + "print(\"āœ… Created view: vw_active_preview_features\")\n", + "\n", + "# View 2: Unacknowledged Critical Alerts\n", + "spark.sql(\"\"\"\n", + " CREATE OR REPLACE VIEW vw_critical_alerts AS\n", + " SELECT \n", + " alert_id,\n", + " feature_name,\n", + " workload,\n", + " alert_type,\n", + " severity,\n", + " message,\n", + " alert_date\n", + " FROM feature_alerts\n", + " WHERE acknowledged = false \n", + " AND severity IN ('Critical', 'Warning')\n", + " ORDER BY alert_date DESC\n", + "\"\"\")\n", + "print(\"āœ… Created view: vw_critical_alerts\")\n", + "\n", + "# View 3: Feature Release Timeline\n", + "spark.sql(\"\"\"\n", + " CREATE OR REPLACE VIEW vw_feature_timeline AS\n", + " SELECT \n", + " feature_name,\n", + " workload,\n", + " status,\n", + " is_preview,\n", + " release_date,\n", + " DATEDIFF(CURRENT_DATE(), release_date) as days_since_release\n", + " FROM feature_releases\n", + " ORDER BY release_date DESC\n", + "\"\"\")\n", + "print(\"āœ… Created view: vw_feature_timeline\")\n", + "\n", + "# View 4: Roadmap Upcoming Features (NEW)\n", + "spark.sql(\"\"\"\n", + " CREATE OR REPLACE VIEW vw_roadmap_upcoming AS\n", + " SELECT \n", + " feature_name,\n", + " feature_description,\n", + " product_name,\n", + " workload,\n", + " release_type,\n", + " release_status,\n", + " release_date,\n", + " is_preview,\n", + " is_planned,\n", + " last_modified,\n", + " CASE \n", + " WHEN release_date IS NULL THEN NULL\n", + " ELSE DATEDIFF(release_date, CURRENT_DATE())\n", + " END as days_until_release\n", + " FROM feature_releases_roadmap\n", + " WHERE is_planned = true\n", + " AND (release_date IS NULL OR release_date >= CURRENT_DATE())\n", + " ORDER BY release_date ASC NULLS LAST, last_modified DESC\n", + "\"\"\")\n", + "print(\"āœ… Created view: vw_roadmap_upcoming\")\n", + "\n", + "print(\"\\nāœ… All views created successfully\")\n", + "print(\" ā†’ These views are accessible via SQL Endpoint\")" + ] + }, + { + "cell_type": "markdown", + "id": "c2b37e4e", + "metadata": {}, + "source": [ + "## āœ… Setup Complete!" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d0659717", + "metadata": {}, + "outputs": [], + "source": [ + "print(\"\\n\" + \"=\" * 60)\n", + "print(\"šŸŽ‰ FEATURE TRACKING SETUP COMPLETED!\")\n", + "print(\"=\" * 60)\n", + "\n", + "# Verify all tables exist\n", + "tables = [\n", + " \"feature_releases\",\n", + " \"feature_releases_roadmap\", \n", + " \"preview_features_active\", \n", + " \"feature_alerts\"\n", + "]\n", + "\n", + "print(\"\\nšŸ“‹ Verifying tables...\")\n", + "for table in tables:\n", + " try:\n", + " count = spark.read.format(\"delta\").load(f\"Tables/{table}\").count()\n", + " print(f\" āœ… {table}: {count} rows\")\n", + " except:\n", + " print(f\" āŒ {table}: ERROR\")\n", + "\n", + "# Verify views\n", + "views = [\n", + " \"vw_active_preview_features\",\n", + " \"vw_critical_alerts\",\n", + " \"vw_feature_timeline\",\n", + " \"vw_roadmap_upcoming\"\n", + "]\n", + "\n", + "print(\"\\nšŸ“‹ Verifying views...\")\n", + "for view in views:\n", + " try:\n", + " spark.sql(f\"SELECT * FROM {view} LIMIT 1\")\n", + " print(f\" āœ… {view}\")\n", + " except:\n", + " print(f\" āŒ {view}: ERROR\")\n", + "\n", + "print(\"\\n\" + \"=\" * 60)\n", + "print(\"šŸ“š Next Steps:\")\n", + "print(\"=\" * 60)\n", + "print(\"\\n1. Choose your data source:\")\n", + "print(\" a) Microsoft Learn (original):\")\n", + "print(\" ā†’ Run '01_Transfer_Feature_Releases_Unit'\")\n", + "print(\" b) Fabric GPS API (enhanced with roadmap):\")\n", + " \"print(\\\" ā†’ Run '01_Transfer_Feature_Releases_GpsApi_Unit'\\\")\\n\",\n", + "print(\"\\n2. Run '02_Transfer_Preview_Features_Unit' to detect activated previews\")\n", + "print(\" Note: This requires 'tenant_settings' table from FUAM core\")\n", + "print(\"\\n3. Run '03_Transfer_Feature_Alerts_Unit' to generate alerts\")\n", + "print(\"\\n4. OR run the full pipeline: 'Load_Feature_Tracking_E2E'\")\n", + "print(\"\\nšŸ’” Recommended:\")\n", + "print(\" - Use Enhanced version (Fabric GPS) for complete roadmap visibility\")\n", + "print(\" - Run both versions if you want dual data sources\")\n", + "print(\" - Schedule pipeline to run daily\")\n", + "print(\"\\n\" + \"=\" * 60)" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From 69c3557b3ca07f2b1088491a8af095d6f2019e2c Mon Sep 17 00:00:00 2001 From: Ayoub KEBAILI Date: Wed, 19 Nov 2025 14:40:30 -0500 Subject: [PATCH 2/4] Add Load Feature Tracking Notebook and implementation for feature tracking pipeline - Created a new notebook for Load Feature Tracking with complete feature tracking pipeline. - Implemented API calls to fetch feature releases from Fabric GPS API. - Transformed API data to a defined schema for further processing. - Added functionality to write feature releases to Delta Lake. - Implemented detection of activated preview features and mapping to tenant settings. - Generated alerts based on business rules for new previews, long-running previews, and low confidence matches. - Summarized and displayed statistics for feature tracking, activated previews, and generated alerts. --- .../fabric-unified-admin-monitoring/README.md | 2 + .../config/deployment_order.json | 8 + .../Feature_Tracking_Setup.Notebook/.platform | 12 + .../notebook-content.ipynb | 310 +++++++ .../Load_Feature_Tracking.Notebook/.platform | 12 + .../notebook-content.ipynb | 781 ++++++++++++++++++ 6 files changed, 1125 insertions(+) create mode 100644 monitoring/fabric-unified-admin-monitoring/src/Feature_Tracking_Setup.Notebook/.platform create mode 100644 monitoring/fabric-unified-admin-monitoring/src/Feature_Tracking_Setup.Notebook/notebook-content.ipynb create mode 100644 monitoring/fabric-unified-admin-monitoring/src/Load_Feature_Tracking.Notebook/.platform create mode 100644 monitoring/fabric-unified-admin-monitoring/src/Load_Feature_Tracking.Notebook/notebook-content.ipynb diff --git a/monitoring/fabric-unified-admin-monitoring/README.md b/monitoring/fabric-unified-admin-monitoring/README.md index 70e33575..9767081d 100644 --- a/monitoring/fabric-unified-admin-monitoring/README.md +++ b/monitoring/fabric-unified-admin-monitoring/README.md @@ -38,6 +38,7 @@ FUAM extracts the following data from the tenant: - Tenant meta data (Scanner API) - Capacity Refreshables - Git Connections +- **Feature Releases & Preview Tracking** (NEW) - Engine level insights (coming soon in optimization module) @@ -104,6 +105,7 @@ The FUAM solution accelerator template **is not an official Microsoft service**. - [Documentation - FUAM Architecture](/monitoring/fabric-unified-admin-monitoring/media/documentation/FUAM_Architecture.md) - [Documentation - FUAM Lakehouse table lineage](/monitoring/fabric-unified-admin-monitoring/media/documentation/FUAM_Documentation_Lakehouse_table_lineage.pdf) - [Documentation - FUAM Engine level analyzer reports](/monitoring/fabric-unified-admin-monitoring/media/documentation/FUAM_Engine_Level_Analyzer_Reports.md) +- [Documentation - FUAM Feature Release Tracking](/monitoring/fabric-unified-admin-monitoring/media/documentation/Feature_Release_Tracking_Documentation.md) ##### Some other Fabric Toolbox assets - [Overview - Fabric Cost Analysis](/monitoring/fabric-cost-analysis/README.md) diff --git a/monitoring/fabric-unified-admin-monitoring/config/deployment_order.json b/monitoring/fabric-unified-admin-monitoring/config/deployment_order.json index 8a8d78a0..e6944de7 100644 --- a/monitoring/fabric-unified-admin-monitoring/config/deployment_order.json +++ b/monitoring/fabric-unified-admin-monitoring/config/deployment_order.json @@ -190,5 +190,13 @@ { "name": "FUAM_Gateway_Monitoring_From_Files_Report.Report", "fuam_id": "3695cd0e-da7e-3b40-ad28-5bd1bcd33eb6" + }, + { + "name": "Feature_Tracking_Setup.Notebook", + "fuam_id": "f8a2b1c3-4d5e-6f7a-8b9c-0d1e2f3a4b5c" + }, + { + "name": "Load_Feature_Tracking.Notebook", + "fuam_id": "a1b2c3d4-5e6f-7a8b-9c0d-1e2f3a4b5c6d" } ] \ No newline at end of file diff --git a/monitoring/fabric-unified-admin-monitoring/src/Feature_Tracking_Setup.Notebook/.platform b/monitoring/fabric-unified-admin-monitoring/src/Feature_Tracking_Setup.Notebook/.platform new file mode 100644 index 00000000..c01c4294 --- /dev/null +++ b/monitoring/fabric-unified-admin-monitoring/src/Feature_Tracking_Setup.Notebook/.platform @@ -0,0 +1,12 @@ +{ + "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", + "metadata": { + "type": "Notebook", + "displayName": "Feature_Tracking_Setup", + "description": "One-time setup for feature tracking tables and views" + }, + "config": { + "version": "2.0", + "logicalId": "00000000-0000-0000-0000-000000000000" + } +} diff --git a/monitoring/fabric-unified-admin-monitoring/src/Feature_Tracking_Setup.Notebook/notebook-content.ipynb b/monitoring/fabric-unified-admin-monitoring/src/Feature_Tracking_Setup.Notebook/notebook-content.ipynb new file mode 100644 index 00000000..89f98915 --- /dev/null +++ b/monitoring/fabric-unified-admin-monitoring/src/Feature_Tracking_Setup.Notebook/notebook-content.ipynb @@ -0,0 +1,310 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "c7fd3012", + "metadata": {}, + "outputs": [], + "source": [ + "from pyspark.sql.types import *\n", + "from datetime import datetime" + ] + }, + { + "cell_type": "markdown", + "id": "505200d2", + "metadata": {}, + "source": [ + "## Step 1: Create `feature_releases_roadmap` Table" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6122dce7", + "metadata": {}, + "outputs": [], + "source": [ + "print(\"šŸ”„ Creating table: feature_releases_roadmap\")\n", + "print(\"=\" * 70)\n", + "\n", + "schema_roadmap = StructType([\n", + " StructField(\"feature_id\", StringType(), False),\n", + " StructField(\"feature_name\", StringType(), False),\n", + " StructField(\"feature_description\", StringType(), True),\n", + " StructField(\"workload\", StringType(), True),\n", + " StructField(\"product_name\", StringType(), True),\n", + " StructField(\"release_date\", TimestampType(), True), # Nullable for planned features\n", + " StructField(\"release_type\", StringType(), True),\n", + " StructField(\"release_status\", StringType(), True),\n", + " StructField(\"is_preview\", BooleanType(), False),\n", + " StructField(\"is_planned\", BooleanType(), False),\n", + " StructField(\"is_shipped\", BooleanType(), False),\n", + " StructField(\"last_modified\", TimestampType(), False),\n", + " StructField(\"source_url\", StringType(), True),\n", + " StructField(\"source\", StringType(), True),\n", + " StructField(\"extracted_date\", TimestampType(), False)\n", + "])\n", + "\n", + "df_roadmap = spark.createDataFrame([], schema_roadmap)\n", + "table_path = \"Tables/feature_releases_roadmap\"\n", + "df_roadmap.write.format(\"delta\").mode(\"overwrite\").save(table_path)\n", + "\n", + "print(\"āœ… Table created: feature_releases_roadmap\")\n", + "print(f\" Schema: {len(schema_roadmap.fields)} columns\")\n", + "print(\"\\n šŸ’” Includes planned/future features and historical tracking\")" + ] + }, + { + "cell_type": "markdown", + "id": "49385cd0", + "metadata": {}, + "source": [ + "## Step 2: Create `preview_features_active` Table" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e3dcfc16", + "metadata": {}, + "outputs": [], + "source": [ + "print(\"\\nšŸ”„ Creating table: preview_features_active\")\n", + "print(\"=\" * 70)\n", + "\n", + "schema_preview_active = StructType([\n", + " StructField(\"setting_name\", StringType(), False),\n", + " StructField(\"feature_id\", StringType(), False),\n", + " StructField(\"feature_name\", StringType(), False),\n", + " StructField(\"workload\", StringType(), True),\n", + " StructField(\"similarity_score\", DoubleType(), False),\n", + " StructField(\"is_enabled\", BooleanType(), False),\n", + " StructField(\"delegate_to_tenant\", BooleanType(), True),\n", + " StructField(\"detected_date\", TimestampType(), False),\n", + " StructField(\"release_date\", TimestampType(), True),\n", + " StructField(\"release_status\", StringType(), True),\n", + " StructField(\"source_url\", StringType(), True),\n", + " StructField(\"days_since_release\", IntegerType(), True)\n", + "])\n", + "\n", + "df_preview_active = spark.createDataFrame([], schema_preview_active)\n", + "table_path = \"Tables/preview_features_active\"\n", + "df_preview_active.write.format(\"delta\").mode(\"overwrite\").save(table_path)\n", + "\n", + "print(\"āœ… Table created: preview_features_active\")\n", + "print(f\" Schema: {len(schema_preview_active.fields)} columns\")" + ] + }, + { + "cell_type": "markdown", + "id": "b8337a78", + "metadata": {}, + "source": [ + "## Step 3: Create `feature_alerts` Table" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "66383382", + "metadata": {}, + "outputs": [], + "source": [ + "print(\"\\nšŸ”„ Creating table: feature_alerts\")\n", + "print(\"=\" * 70)\n", + "\n", + "schema_alerts = StructType([\n", + " StructField(\"alert_id\", StringType(), False),\n", + " StructField(\"feature_id\", StringType(), False),\n", + " StructField(\"feature_name\", StringType(), False),\n", + " StructField(\"workload\", StringType(), True),\n", + " StructField(\"alert_type\", StringType(), False),\n", + " StructField(\"severity\", StringType(), False),\n", + " StructField(\"message\", StringType(), False),\n", + " StructField(\"setting_name\", StringType(), True),\n", + " StructField(\"similarity_score\", DoubleType(), True),\n", + " StructField(\"days_since_release\", IntegerType(), True),\n", + " StructField(\"alert_date\", TimestampType(), False),\n", + " StructField(\"acknowledged\", BooleanType(), False),\n", + " StructField(\"acknowledged_date\", TimestampType(), True),\n", + " StructField(\"acknowledged_by\", StringType(), True)\n", + "])\n", + "\n", + "df_alerts = spark.createDataFrame([], schema_alerts)\n", + "table_path = \"Tables/feature_alerts\"\n", + "df_alerts.write.format(\"delta\").mode(\"overwrite\").save(table_path)\n", + "\n", + "print(\"āœ… Table created: feature_alerts\")\n", + "print(f\" Schema: {len(schema_alerts.fields)} columns\")" + ] + }, + { + "cell_type": "markdown", + "id": "21110f70", + "metadata": {}, + "source": [ + "## Step 4: Create Helper SQL Views" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3189b548", + "metadata": {}, + "outputs": [], + "source": [ + "print(\"\\nšŸ”„ Creating helper SQL views...\")\n", + "print(\"=\" * 70)\n", + "\n", + "# View 1: Roadmap Upcoming Features\n", + "spark.sql(\"\"\"\n", + " CREATE OR REPLACE VIEW vw_roadmap_upcoming AS\n", + " SELECT \n", + " feature_name,\n", + " feature_description,\n", + " product_name,\n", + " workload,\n", + " release_type,\n", + " release_status,\n", + " release_date,\n", + " is_preview,\n", + " is_planned,\n", + " last_modified,\n", + " CASE \n", + " WHEN release_date IS NULL THEN NULL\n", + " ELSE DATEDIFF(release_date, CURRENT_DATE())\n", + " END as days_until_release\n", + " FROM feature_releases_roadmap\n", + " WHERE is_planned = true\n", + " AND (release_date IS NULL OR release_date >= CURRENT_DATE())\n", + " ORDER BY release_date ASC NULLS LAST, last_modified DESC\n", + "\"\"\")\n", + "print(\"āœ… vw_roadmap_upcoming - Planned/upcoming features\")\n", + "\n", + "# View 2: Active Preview Features\n", + "spark.sql(\"\"\"\n", + " CREATE OR REPLACE VIEW vw_active_preview_features AS\n", + " SELECT \n", + " feature_name,\n", + " workload,\n", + " setting_name,\n", + " days_since_release,\n", + " similarity_score,\n", + " release_date,\n", + " detected_date,\n", + " is_enabled\n", + " FROM preview_features_active\n", + " WHERE is_enabled = true\n", + " ORDER BY detected_date DESC\n", + "\"\"\")\n", + "print(\"āœ… vw_active_preview_features - Currently enabled previews\")\n", + "\n", + "# View 3: Critical Alerts\n", + "spark.sql(\"\"\"\n", + " CREATE OR REPLACE VIEW vw_critical_alerts AS\n", + " SELECT \n", + " alert_id,\n", + " feature_name,\n", + " workload,\n", + " alert_type,\n", + " severity,\n", + " message,\n", + " alert_date,\n", + " acknowledged\n", + " FROM feature_alerts\n", + " WHERE acknowledged = false \n", + " AND severity IN ('Critical', 'Warning')\n", + " ORDER BY \n", + " CASE severity \n", + " WHEN 'Critical' THEN 1 \n", + " WHEN 'Warning' THEN 2 \n", + " ELSE 3 \n", + " END,\n", + " alert_date DESC\n", + "\"\"\")\n", + "print(\"āœ… vw_critical_alerts - Unacknowledged critical/warning alerts\")\n", + "\n", + "# View 4: Feature Release Timeline\n", + "spark.sql(\"\"\"\n", + " CREATE OR REPLACE VIEW vw_feature_timeline AS\n", + " SELECT \n", + " feature_name,\n", + " product_name,\n", + " workload,\n", + " release_type,\n", + " release_status,\n", + " is_preview,\n", + " is_planned,\n", + " is_shipped,\n", + " release_date,\n", + " CASE \n", + " WHEN release_date IS NULL THEN NULL\n", + " ELSE DATEDIFF(CURRENT_DATE(), release_date)\n", + " END as days_since_release,\n", + " last_modified\n", + " FROM feature_releases_roadmap\n", + " ORDER BY release_date DESC NULLS LAST\n", + "\"\"\")\n", + "print(\"āœ… vw_feature_timeline - Complete release timeline\")\n", + "\n", + "print(\"\\nāœ… All SQL views created successfully\")" + ] + }, + { + "cell_type": "markdown", + "id": "9af215ea", + "metadata": {}, + "source": [ + "## āœ… Setup Complete!" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cb7dc56a", + "metadata": {}, + "outputs": [], + "source": [ + "print(\"\\n\" + \"=\" * 70)\n", + "print(\"šŸŽ‰ FEATURE TRACKING SETUP COMPLETED!\")\n", + "print(\"=\" * 70)\n", + "\n", + "# Verify tables\n", + "tables = [\"feature_releases_roadmap\", \"preview_features_active\", \"feature_alerts\"]\n", + "print(\"\\nšŸ“‹ Tables created:\")\n", + "for table in tables:\n", + " try:\n", + " count = spark.read.format(\"delta\").load(f\"Tables/{table}\").count()\n", + " print(f\" āœ… {table}: {count} rows\")\n", + " except Exception as e:\n", + " print(f\" āŒ {table}: ERROR - {e}\")\n", + "\n", + "# Verify views\n", + "views = [\"vw_roadmap_upcoming\", \"vw_active_preview_features\", \"vw_critical_alerts\", \"vw_feature_timeline\"]\n", + "print(\"\\nšŸ“‹ Views created:\")\n", + "for view in views:\n", + " try:\n", + " spark.sql(f\"SELECT * FROM {view} LIMIT 1\")\n", + " print(f\" āœ… {view}\")\n", + " except Exception as e:\n", + " print(f\" āŒ {view}: ERROR - {e}\")\n", + "\n", + "print(\"\\n\" + \"=\" * 70)\n", + "print(\"šŸ“š Next Step:\")\n", + "print(\"=\" * 70)\n", + "print(\"\\n ā†’ Run 'Load_Feature_Tracking' notebook to populate the tables\")\n", + "print(\"\\nšŸ’” Schedule Load_Feature_Tracking to run daily for continuous monitoring\")\n", + "print(\"\\n\" + \"=\" * 70)" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/monitoring/fabric-unified-admin-monitoring/src/Load_Feature_Tracking.Notebook/.platform b/monitoring/fabric-unified-admin-monitoring/src/Load_Feature_Tracking.Notebook/.platform new file mode 100644 index 00000000..10a12210 --- /dev/null +++ b/monitoring/fabric-unified-admin-monitoring/src/Load_Feature_Tracking.Notebook/.platform @@ -0,0 +1,12 @@ +{ + "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", + "metadata": { + "type": "Notebook", + "displayName": "Load_Feature_Tracking", + "description": "Complete feature tracking pipeline - fetch releases, detect previews, generate alerts" + }, + "config": { + "version": "2.0", + "logicalId": "00000000-0000-0000-0000-000000000000" + } +} diff --git a/monitoring/fabric-unified-admin-monitoring/src/Load_Feature_Tracking.Notebook/notebook-content.ipynb b/monitoring/fabric-unified-admin-monitoring/src/Load_Feature_Tracking.Notebook/notebook-content.ipynb new file mode 100644 index 00000000..d0445624 --- /dev/null +++ b/monitoring/fabric-unified-admin-monitoring/src/Load_Feature_Tracking.Notebook/notebook-content.ipynb @@ -0,0 +1,781 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "e0a5125f", + "metadata": {}, + "outputs": [], + "source": [ + "import requests\n", + "from datetime import datetime, timedelta\n", + "from pyspark.sql import functions as F\n", + "from pyspark.sql.types import *\n", + "from pyspark.sql.window import Window\n", + "from difflib import SequenceMatcher" + ] + }, + { + "cell_type": "markdown", + "id": "3f0085bb", + "metadata": {}, + "source": [ + "## Parameters" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f250ccba", + "metadata": {}, + "outputs": [], + "source": [ + "# API Configuration\n", + "fabric_gps_api_url = \"https://fabric-gps.com/api/releases\"\n", + "modified_within_days = 90\n", + "page_size = 200\n", + "include_planned = True\n", + "include_shipped = True\n", + "\n", + "# Alert Thresholds\n", + "ALERT_DAYS_THRESHOLD = 90 # Alert if preview active >90 days\n", + "LOW_CONFIDENCE_THRESHOLD = 0.5 # Alert if similarity score <0.5\n", + "SIMILARITY_MATCH_THRESHOLD = 0.3 # Minimum similarity to consider a match\n", + "\n", + "# Alert Severity Levels\n", + "SEVERITY_INFO = \"Info\"\n", + "SEVERITY_WARNING = \"Warning\"\n", + "SEVERITY_CRITICAL = \"Critical\"" + ] + }, + { + "cell_type": "markdown", + "id": "8abfdf27", + "metadata": {}, + "source": [ + "---\n", + "# Part 1: Fetch Feature Releases from Fabric GPS API" + ] + }, + { + "cell_type": "markdown", + "id": "a49058b0", + "metadata": {}, + "source": [ + "## Step 1.1: Fetch from API" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e7dfa925", + "metadata": {}, + "outputs": [], + "source": [ + "def fetch_all_fabric_gps_releases(base_url, page_size=200, modified_within_days=None, include_planned=True, include_shipped=True):\n", + " \"\"\"Fetch all releases from Fabric GPS API with pagination\"\"\"\n", + " all_releases = []\n", + " page = 1\n", + " \n", + " print(f\"šŸ”„ Fetching from Fabric GPS API: {base_url}\")\n", + " \n", + " while True:\n", + " try:\n", + " params = {\"page\": page, \"page_size\": page_size}\n", + " \n", + " if modified_within_days and modified_within_days <= 30:\n", + " params[\"modified_within_days\"] = modified_within_days\n", + " \n", + " response = requests.get(base_url, params=params, timeout=30)\n", + " response.raise_for_status()\n", + " data = response.json()\n", + " \n", + " releases = data.get(\"data\", [])\n", + " if not releases:\n", + " break\n", + " \n", + " # Filter by release_status\n", + " filtered_releases = []\n", + " for release in releases:\n", + " status = release.get(\"release_status\", \"\")\n", + " if status == \"Planned\" and not include_planned:\n", + " continue\n", + " if status == \"Shipped\" and not include_shipped:\n", + " continue\n", + " filtered_releases.append(release)\n", + " \n", + " all_releases.extend(filtered_releases)\n", + " \n", + " pagination = data.get(\"pagination\", {})\n", + " has_next = pagination.get(\"has_next\", False)\n", + " total_items = pagination.get(\"total_items\", 0)\n", + " \n", + " print(f\" ā†’ Page {page}: {len(filtered_releases)} releases (Total: {len(all_releases)}/{total_items})\")\n", + " \n", + " if not has_next:\n", + " break\n", + " \n", + " page += 1\n", + " \n", + " except Exception as e:\n", + " print(f\" āŒ Error fetching page {page}: {e}\")\n", + " break\n", + " \n", + " print(f\"āœ… Total releases fetched: {len(all_releases)}\")\n", + " return all_releases\n", + "\n", + "releases_data = fetch_all_fabric_gps_releases(\n", + " fabric_gps_api_url, \n", + " page_size=page_size,\n", + " modified_within_days=modified_within_days if modified_within_days <= 30 else None,\n", + " include_planned=include_planned,\n", + " include_shipped=include_shipped\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "06bbdd25", + "metadata": {}, + "source": [ + "## Step 1.2: Transform to Schema" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "00b94a80", + "metadata": {}, + "outputs": [], + "source": [ + "def transform_fabric_gps_to_fuam(releases):\n", + " \"\"\"Transform Fabric GPS release data to FUAM schema\"\"\"\n", + " transformed = []\n", + " \n", + " workload_mapping = {\n", + " \"Power BI\": \"Power BI\",\n", + " \"Data Factory\": \"Data Factory\",\n", + " \"Data Engineering\": \"Data Engineering\",\n", + " \"Data Science\": \"Data Science\",\n", + " \"Data Warehouse\": \"Data Warehouse\",\n", + " \"Real-Time Intelligence\": \"Real-Time Intelligence\",\n", + " \"OneLake\": \"Data Engineering\",\n", + " \"Administration, Governance and Security\": \"Governance\",\n", + " \"Cosmos DB (NoSQL)\": \"Cosmos DB\"\n", + " }\n", + " \n", + " for release in releases:\n", + " try:\n", + " release_date_str = release.get(\"release_date\")\n", + " release_date = datetime.strptime(release_date_str, \"%Y-%m-%d\") if release_date_str else None\n", + " except:\n", + " release_date = None\n", + " \n", + " try:\n", + " last_modified_str = release.get(\"last_modified\")\n", + " last_modified = datetime.strptime(last_modified_str, \"%Y-%m-%d\") if last_modified_str else datetime.now()\n", + " except:\n", + " last_modified = datetime.now()\n", + " \n", + " release_type = release.get(\"release_type\", \"\")\n", + " is_preview = \"preview\" in release_type.lower()\n", + " product_name = release.get(\"product_name\", \"Unknown\")\n", + " workload = workload_mapping.get(product_name, product_name)\n", + " \n", + " feature = {\n", + " \"feature_id\": release.get(\"release_item_id\"),\n", + " \"feature_name\": release.get(\"feature_name\", \"Unknown\"),\n", + " \"feature_description\": release.get(\"feature_description\"),\n", + " \"workload\": workload,\n", + " \"product_name\": product_name,\n", + " \"release_date\": release_date,\n", + " \"release_type\": release_type,\n", + " \"release_status\": release.get(\"release_status\", \"Unknown\"),\n", + " \"is_preview\": is_preview,\n", + " \"is_planned\": release.get(\"release_status\") == \"Planned\",\n", + " \"is_shipped\": release.get(\"release_status\") == \"Shipped\",\n", + " \"last_modified\": last_modified,\n", + " \"source_url\": f\"https://fabric-gps.com/api/releases?release_item_id={release.get('release_item_id')}\",\n", + " \"source\": \"Fabric GPS API\",\n", + " \"extracted_date\": datetime.now()\n", + " }\n", + " \n", + " transformed.append(feature)\n", + " \n", + " return transformed\n", + "\n", + "print(\"šŸ”„ Transforming to schema...\")\n", + "features_data = transform_fabric_gps_to_fuam(releases_data)\n", + "\n", + "preview_count = sum(1 for f in features_data if f[\"is_preview\"])\n", + "planned_count = sum(1 for f in features_data if f[\"is_planned\"])\n", + "shipped_count = sum(1 for f in features_data if f[\"is_shipped\"])\n", + "\n", + "print(f\"āœ… Transformed {len(features_data)} features\")\n", + "print(f\" Preview: {preview_count} | Planned: {planned_count} | Shipped: {shipped_count}\")" + ] + }, + { + "cell_type": "markdown", + "id": "db647aba", + "metadata": {}, + "source": [ + "## Step 1.3: Write Feature Releases to Delta" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3f7579b9", + "metadata": {}, + "outputs": [], + "source": [ + "schema = StructType([\n", + " StructField(\"feature_id\", StringType(), False),\n", + " StructField(\"feature_name\", StringType(), False),\n", + " StructField(\"feature_description\", StringType(), True),\n", + " StructField(\"workload\", StringType(), True),\n", + " StructField(\"product_name\", StringType(), True),\n", + " StructField(\"release_date\", TimestampType(), True),\n", + " StructField(\"release_type\", StringType(), True),\n", + " StructField(\"release_status\", StringType(), True),\n", + " StructField(\"is_preview\", BooleanType(), False),\n", + " StructField(\"is_planned\", BooleanType(), False),\n", + " StructField(\"is_shipped\", BooleanType(), False),\n", + " StructField(\"last_modified\", TimestampType(), False),\n", + " StructField(\"source_url\", StringType(), True),\n", + " StructField(\"source\", StringType(), True),\n", + " StructField(\"extracted_date\", TimestampType(), False)\n", + "])\n", + "\n", + "df_features = spark.createDataFrame(features_data, schema=schema)\n", + "\n", + "table_name = \"feature_releases_roadmap\"\n", + "table_path = f\"Tables/{table_name}\"\n", + "\n", + "print(f\"šŸ”„ Writing to {table_name}...\")\n", + "\n", + "try:\n", + " from delta.tables import DeltaTable\n", + " \n", + " if DeltaTable.isDeltaTable(spark, table_path):\n", + " print(\" ā†’ Performing MERGE (upsert)...\")\n", + " \n", + " delta_table = DeltaTable.forPath(spark, table_path)\n", + " \n", + " delta_table.alias(\"target\").merge(\n", + " df_features.alias(\"source\"),\n", + " \"target.feature_id = source.feature_id\"\n", + " ).whenMatchedUpdate(\n", + " condition=\"source.last_modified > target.last_modified\",\n", + " set={\n", + " \"feature_name\": \"source.feature_name\",\n", + " \"feature_description\": \"source.feature_description\",\n", + " \"workload\": \"source.workload\",\n", + " \"product_name\": \"source.product_name\",\n", + " \"release_date\": \"source.release_date\",\n", + " \"release_type\": \"source.release_type\",\n", + " \"release_status\": \"source.release_status\",\n", + " \"is_preview\": \"source.is_preview\",\n", + " \"is_planned\": \"source.is_planned\",\n", + " \"is_shipped\": \"source.is_shipped\",\n", + " \"last_modified\": \"source.last_modified\",\n", + " \"source_url\": \"source.source_url\",\n", + " \"extracted_date\": \"source.extracted_date\"\n", + " }\n", + " ).whenNotMatchedInsertAll().execute()\n", + " \n", + " print(\" āœ… MERGE completed\")\n", + " else:\n", + " print(\" ā†’ Creating new table...\")\n", + " df_features.write.format(\"delta\").mode(\"overwrite\").save(table_path)\n", + " print(\" āœ… Table created\")\n", + " \n", + " final_count = spark.read.format(\"delta\").load(table_path).count()\n", + " print(f\"āœ… Total records in {table_name}: {final_count}\")\n", + " \n", + "except Exception as e:\n", + " print(f\"āŒ Error writing to Delta: {e}\")\n", + " raise" + ] + }, + { + "cell_type": "markdown", + "id": "1fd4034e", + "metadata": {}, + "source": [ + "---\n", + "# Part 2: Detect Activated Preview Features" + ] + }, + { + "cell_type": "markdown", + "id": "51ba2cbf", + "metadata": {}, + "source": [ + "## Step 2.1: Load Data Sources" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "67ba4890", + "metadata": {}, + "outputs": [], + "source": [ + "print(\"\\nšŸ”„ Loading preview features...\")\n", + "df_preview_features = spark.read.format(\"delta\").load(table_path).filter(F.col(\"is_preview\") == True)\n", + "preview_count = df_preview_features.count()\n", + "print(f\"āœ… Loaded {preview_count} preview features\")\n", + "\n", + "print(\"\\nšŸ”„ Loading tenant settings...\")\n", + "try:\n", + " df_tenant_settings = spark.read.format(\"delta\").load(\"Tables/tenant_settings\")\n", + " settings_count = df_tenant_settings.count()\n", + " print(f\"āœ… Loaded {settings_count} tenant settings\")\n", + "except Exception as e:\n", + " print(f\"āš ļø Warning: Could not load tenant_settings table: {e}\")\n", + " print(\" Skipping preview feature detection...\")\n", + " df_tenant_settings = None" + ] + }, + { + "cell_type": "markdown", + "id": "706b1ba5", + "metadata": {}, + "source": [ + "## Step 2.2: Map Settings to Features" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0d8aad6a", + "metadata": {}, + "outputs": [], + "source": [ + "def similarity_score(a, b):\n", + " \"\"\"Calculate similarity between two strings (0-1)\"\"\"\n", + " return SequenceMatcher(None, a.lower(), b.lower()).ratio()\n", + "\n", + "def map_settings_to_features(df_settings, df_features, threshold=SIMILARITY_MATCH_THRESHOLD):\n", + " \"\"\"Map tenant settings to preview features based on name similarity\"\"\"\n", + " \n", + " features_list = df_features.select(\"feature_id\", \"feature_name\", \"workload\", \"release_date\", \"release_status\", \"source_url\").collect()\n", + " settings_list = df_settings.select(\"settingName\", \"enabled\", \"delegateToTenant\").collect()\n", + " \n", + " matches = []\n", + " \n", + " for setting in settings_list:\n", + " setting_name = setting[\"settingName\"]\n", + " \n", + " if not setting[\"enabled\"]:\n", + " continue\n", + " \n", + " best_match = None\n", + " best_score = 0.0\n", + " \n", + " for feature in features_list:\n", + " feature_name = feature[\"feature_name\"]\n", + " \n", + " # Calculate similarity\n", + " score = similarity_score(setting_name, feature_name)\n", + " \n", + " # Boost score for common words\n", + " setting_words = set(setting_name.lower().split())\n", + " feature_words = set(feature_name.lower().split())\n", + " common_words = setting_words & feature_words\n", + " \n", + " if common_words:\n", + " score += len(common_words) * 0.1\n", + " \n", + " if score > best_score and score > threshold:\n", + " best_score = score\n", + " best_match = feature\n", + " \n", + " if best_match:\n", + " # Calculate days since release\n", + " days_since = None\n", + " if best_match[\"release_date\"]:\n", + " days_since = (datetime.now() - best_match[\"release_date\"]).days\n", + " \n", + " matches.append({\n", + " \"setting_name\": setting_name,\n", + " \"feature_id\": best_match[\"feature_id\"],\n", + " \"feature_name\": best_match[\"feature_name\"],\n", + " \"workload\": best_match[\"workload\"],\n", + " \"similarity_score\": best_score,\n", + " \"is_enabled\": setting[\"enabled\"],\n", + " \"delegate_to_tenant\": setting[\"delegateToTenant\"],\n", + " \"detected_date\": datetime.now(),\n", + " \"release_date\": best_match[\"release_date\"],\n", + " \"release_status\": best_match[\"release_status\"],\n", + " \"source_url\": best_match[\"source_url\"],\n", + " \"days_since_release\": days_since\n", + " })\n", + " \n", + " return matches\n", + "\n", + "if df_tenant_settings:\n", + " print(\"šŸ”„ Mapping settings to preview features...\")\n", + " matches = map_settings_to_features(df_tenant_settings, df_preview_features)\n", + " print(f\"āœ… Found {len(matches)} activated preview features\")\n", + "else:\n", + " matches = []" + ] + }, + { + "cell_type": "markdown", + "id": "ccca10b5", + "metadata": {}, + "source": [ + "## Step 2.3: Write Active Previews to Delta" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9702e5dd", + "metadata": {}, + "outputs": [], + "source": [ + "if matches:\n", + " schema_active = StructType([\n", + " StructField(\"setting_name\", StringType(), False),\n", + " StructField(\"feature_id\", StringType(), False),\n", + " StructField(\"feature_name\", StringType(), False),\n", + " StructField(\"workload\", StringType(), True),\n", + " StructField(\"similarity_score\", DoubleType(), False),\n", + " StructField(\"is_enabled\", BooleanType(), False),\n", + " StructField(\"delegate_to_tenant\", BooleanType(), True),\n", + " StructField(\"detected_date\", TimestampType(), False),\n", + " StructField(\"release_date\", TimestampType(), True),\n", + " StructField(\"release_status\", StringType(), True),\n", + " StructField(\"source_url\", StringType(), True),\n", + " StructField(\"days_since_release\", IntegerType(), True)\n", + " ])\n", + " \n", + " df_active_previews = spark.createDataFrame(matches, schema=schema_active)\n", + " \n", + " table_name_active = \"preview_features_active\"\n", + " table_path_active = f\"Tables/{table_name_active}\"\n", + " \n", + " print(f\"šŸ”„ Writing to {table_name_active}...\")\n", + " \n", + " try:\n", + " if DeltaTable.isDeltaTable(spark, table_path_active):\n", + " print(\" ā†’ Performing MERGE...\")\n", + " \n", + " delta_table = DeltaTable.forPath(spark, table_path_active)\n", + " \n", + " delta_table.alias(\"target\").merge(\n", + " df_active_previews.alias(\"source\"),\n", + " \"target.feature_id = source.feature_id AND target.setting_name = source.setting_name\"\n", + " ).whenMatchedUpdate(\n", + " set={\n", + " \"is_enabled\": \"source.is_enabled\",\n", + " \"detected_date\": \"source.detected_date\",\n", + " \"days_since_release\": \"source.days_since_release\",\n", + " \"similarity_score\": \"source.similarity_score\"\n", + " }\n", + " ).whenNotMatchedInsertAll().execute()\n", + " \n", + " print(\" āœ… MERGE completed\")\n", + " else:\n", + " print(\" ā†’ Creating new table...\")\n", + " df_active_previews.write.format(\"delta\").mode(\"overwrite\").save(table_path_active)\n", + " print(\" āœ… Table created\")\n", + " \n", + " final_count = spark.read.format(\"delta\").load(table_path_active).count()\n", + " print(f\"āœ… Total activated previews: {final_count}\")\n", + " \n", + " except Exception as e:\n", + " print(f\"āŒ Error writing active previews: {e}\")\n", + " raise\n", + "else:\n", + " print(\"ā„¹ļø No activated preview features to write\")\n", + " df_active_previews = None" + ] + }, + { + "cell_type": "markdown", + "id": "d1c7f6dd", + "metadata": {}, + "source": [ + "---\n", + "# Part 3: Generate Feature Alerts" + ] + }, + { + "cell_type": "markdown", + "id": "28f72485", + "metadata": {}, + "source": [ + "## Step 3.1: Load Historical Alerts" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ec9f5348", + "metadata": {}, + "outputs": [], + "source": [ + "table_name_alerts = \"feature_alerts\"\n", + "table_path_alerts = f\"Tables/{table_name_alerts}\"\n", + "\n", + "try:\n", + " if DeltaTable.isDeltaTable(spark, table_path_alerts):\n", + " print(\"šŸ”„ Loading historical alerts...\")\n", + " df_historical = spark.read.format(\"delta\").load(table_path_alerts)\n", + " \n", + " # Get already alerted feature_ids + alert_type combinations to avoid duplicate alerts\n", + " alerted_combos = set([\n", + " (row[\"feature_id\"], row[\"alert_type\"]) \n", + " for row in df_historical.select(\"feature_id\", \"alert_type\").distinct().collect()\n", + " ])\n", + " \n", + " print(f\"āœ… Loaded {len(alerted_combos)} existing alert combinations\")\n", + " else:\n", + " alerted_combos = set()\n", + " print(\"ā„¹ļø No historical alerts found (first run)\")\n", + "except Exception as e:\n", + " alerted_combos = set()\n", + " print(f\"ā„¹ļø No historical alerts (first run): {e}\")" + ] + }, + { + "cell_type": "markdown", + "id": "2f2b3c9f", + "metadata": {}, + "source": [ + "## Step 3.2: Generate Alerts" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b972785f", + "metadata": {}, + "outputs": [], + "source": [ + "def generate_alerts(matches, alerted_combos):\n", + " \"\"\"Generate alerts based on business rules\"\"\"\n", + " alerts = []\n", + " \n", + " for match in matches:\n", + " feature_id = match[\"feature_id\"]\n", + " feature_name = match[\"feature_name\"]\n", + " workload = match[\"workload\"]\n", + " days_since_release = match[\"days_since_release\"]\n", + " similarity_score = match[\"similarity_score\"]\n", + " setting_name = match[\"setting_name\"]\n", + " \n", + " # Rule 1: NEW PREVIEW FEATURE ACTIVATED\n", + " alert_type = \"New Preview Activated\"\n", + " if (feature_id, alert_type) not in alerted_combos:\n", + " alerts.append({\n", + " \"alert_id\": f\"NEW_{feature_id}_{datetime.now().strftime('%Y%m%d%H%M%S')}\",\n", + " \"feature_id\": feature_id,\n", + " \"feature_name\": feature_name,\n", + " \"workload\": workload,\n", + " \"alert_type\": alert_type,\n", + " \"severity\": SEVERITY_INFO,\n", + " \"message\": f\"New preview feature '{feature_name}' activated (Setting: {setting_name})\",\n", + " \"setting_name\": setting_name,\n", + " \"similarity_score\": similarity_score,\n", + " \"days_since_release\": days_since_release,\n", + " \"alert_date\": datetime.now(),\n", + " \"acknowledged\": False,\n", + " \"acknowledged_date\": None,\n", + " \"acknowledged_by\": None\n", + " })\n", + " \n", + " # Rule 2: LONG-RUNNING PREVIEW (>90 days)\n", + " if days_since_release and days_since_release > ALERT_DAYS_THRESHOLD:\n", + " alert_type = \"Long-Running Preview\"\n", + " if (feature_id, alert_type) not in alerted_combos:\n", + " alerts.append({\n", + " \"alert_id\": f\"LONGRUN_{feature_id}_{datetime.now().strftime('%Y%m%d%H%M%S')}\",\n", + " \"feature_id\": feature_id,\n", + " \"feature_name\": feature_name,\n", + " \"workload\": workload,\n", + " \"alert_type\": alert_type,\n", + " \"severity\": SEVERITY_WARNING,\n", + " \"message\": f\"Preview feature '{feature_name}' active for {days_since_release} days. Review for GA transition.\",\n", + " \"setting_name\": setting_name,\n", + " \"similarity_score\": similarity_score,\n", + " \"days_since_release\": days_since_release,\n", + " \"alert_date\": datetime.now(),\n", + " \"acknowledged\": False,\n", + " \"acknowledged_date\": None,\n", + " \"acknowledged_by\": None\n", + " })\n", + " \n", + " # Rule 3: LOW CONFIDENCE MATCH\n", + " if similarity_score < LOW_CONFIDENCE_THRESHOLD:\n", + " alert_type = \"Low Confidence Match\"\n", + " if (feature_id, alert_type) not in alerted_combos:\n", + " alerts.append({\n", + " \"alert_id\": f\"LOWCONF_{feature_id}_{datetime.now().strftime('%Y%m%d%H%M%S')}\",\n", + " \"feature_id\": feature_id,\n", + " \"feature_name\": feature_name,\n", + " \"workload\": workload,\n", + " \"alert_type\": alert_type,\n", + " \"severity\": SEVERITY_CRITICAL,\n", + " \"message\": f\"Low confidence match ({similarity_score:.2f}) between setting '{setting_name}' and feature '{feature_name}'. Manual review recommended.\",\n", + " \"setting_name\": setting_name,\n", + " \"similarity_score\": similarity_score,\n", + " \"days_since_release\": days_since_release,\n", + " \"alert_date\": datetime.now(),\n", + " \"acknowledged\": False,\n", + " \"acknowledged_date\": None,\n", + " \"acknowledged_by\": None\n", + " })\n", + " \n", + " return alerts\n", + "\n", + "if matches:\n", + " print(\"šŸ”„ Generating alerts...\")\n", + " alerts_data = generate_alerts(matches, alerted_combos)\n", + " print(f\"āœ… Generated {len(alerts_data)} new alerts\")\n", + " \n", + " if alerts_data:\n", + " for alert in alerts_data[:3]:\n", + " print(f\" [{alert['severity']}] {alert['alert_type']}: {alert['feature_name']}\")\n", + "else:\n", + " alerts_data = []\n", + " print(\"ā„¹ļø No data for alert generation\")" + ] + }, + { + "cell_type": "markdown", + "id": "88aa50ca", + "metadata": {}, + "source": [ + "## Step 3.3: Write Alerts to Delta" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c334fef8", + "metadata": {}, + "outputs": [], + "source": [ + "if alerts_data:\n", + " schema_alerts = StructType([\n", + " StructField(\"alert_id\", StringType(), False),\n", + " StructField(\"feature_id\", StringType(), False),\n", + " StructField(\"feature_name\", StringType(), False),\n", + " StructField(\"workload\", StringType(), True),\n", + " StructField(\"alert_type\", StringType(), False),\n", + " StructField(\"severity\", StringType(), False),\n", + " StructField(\"message\", StringType(), False),\n", + " StructField(\"setting_name\", StringType(), True),\n", + " StructField(\"similarity_score\", DoubleType(), True),\n", + " StructField(\"days_since_release\", IntegerType(), True),\n", + " StructField(\"alert_date\", TimestampType(), False),\n", + " StructField(\"acknowledged\", BooleanType(), False),\n", + " StructField(\"acknowledged_date\", TimestampType(), True),\n", + " StructField(\"acknowledged_by\", StringType(), True)\n", + " ])\n", + " \n", + " df_alerts = spark.createDataFrame(alerts_data, schema=schema_alerts)\n", + " \n", + " print(f\"šŸ”„ Writing {len(alerts_data)} alerts to {table_name_alerts}...\")\n", + " \n", + " try:\n", + " if DeltaTable.isDeltaTable(spark, table_path_alerts):\n", + " print(\" ā†’ Appending to existing table...\")\n", + " df_alerts.write.format(\"delta\").mode(\"append\").save(table_path_alerts)\n", + " else:\n", + " print(\" ā†’ Creating new table...\")\n", + " df_alerts.write.format(\"delta\").mode(\"overwrite\").save(table_path_alerts)\n", + " \n", + " final_count = spark.read.format(\"delta\").load(table_path_alerts).count()\n", + " print(f\"āœ… Total alerts: {final_count}\")\n", + " \n", + " except Exception as e:\n", + " print(f\"āŒ Error writing alerts: {e}\")\n", + " raise\n", + "else:\n", + " print(\"ā„¹ļø No new alerts to write\")" + ] + }, + { + "cell_type": "markdown", + "id": "62c50df9", + "metadata": {}, + "source": [ + "---\n", + "# Summary & Statistics" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fe4c5072", + "metadata": {}, + "outputs": [], + "source": [ + "print(\"\\n\" + \"=\" * 70)\n", + "print(\"šŸ“Š FEATURE TRACKING SUMMARY\")\n", + "print(\"=\" * 70)\n", + "\n", + "# Feature Releases Stats\n", + "df_roadmap = spark.read.format(\"delta\").load(table_path)\n", + "total_features = df_roadmap.count()\n", + "preview_features = df_roadmap.filter(F.col(\"is_preview\") == True).count()\n", + "planned_features = df_roadmap.filter(F.col(\"is_planned\") == True).count()\n", + "shipped_features = df_roadmap.filter(F.col(\"is_shipped\") == True).count()\n", + "\n", + "print(f\"\\nšŸ”ø Feature Releases Roadmap:\")\n", + "print(f\" Total features: {total_features}\")\n", + "print(f\" Preview features: {preview_features}\")\n", + "print(f\" Planned (roadmap): {planned_features}\")\n", + "print(f\" Shipped: {shipped_features}\")\n", + "\n", + "# Active Previews Stats\n", + "if matches:\n", + " print(f\"\\nšŸ”ø Activated Preview Features:\")\n", + " print(f\" Total activated: {len(matches)}\")\n", + " high_conf = sum(1 for m in matches if m[\"similarity_score\"] >= 0.7)\n", + " med_conf = sum(1 for m in matches if 0.5 <= m[\"similarity_score\"] < 0.7)\n", + " low_conf = sum(1 for m in matches if m[\"similarity_score\"] < 0.5)\n", + " print(f\" High confidence (ā‰„0.7): {high_conf}\")\n", + " print(f\" Medium confidence (0.5-0.7): {med_conf}\")\n", + " print(f\" Low confidence (<0.5): {low_conf}\")\n", + "\n", + "# Alerts Stats\n", + "if alerts_data:\n", + " print(f\"\\nšŸ”ø Alerts Generated:\")\n", + " print(f\" New alerts: {len(alerts_data)}\")\n", + " info_alerts = sum(1 for a in alerts_data if a[\"severity\"] == SEVERITY_INFO)\n", + " warn_alerts = sum(1 for a in alerts_data if a[\"severity\"] == SEVERITY_WARNING)\n", + " crit_alerts = sum(1 for a in alerts_data if a[\"severity\"] == SEVERITY_CRITICAL)\n", + " print(f\" Info: {info_alerts} | Warning: {warn_alerts} | Critical: {crit_alerts}\")\n", + "\n", + "# Workload Breakdown\n", + "print(f\"\\nšŸ”ø Top Workloads (by feature count):\")\n", + "df_roadmap.groupBy(\"workload\").count().orderBy(F.desc(\"count\")).show(5, truncate=False)\n", + "\n", + "print(\"\\n\" + \"=\" * 70)\n", + "print(\"āœ… LOAD FEATURE TRACKING - COMPLETED\")\n", + "print(\"=\" * 70)\n", + "print(\"\\nšŸ’” View results in:\")\n", + "print(\" - vw_roadmap_upcoming (planned features)\")\n", + "print(\" - vw_active_preview_features (enabled previews)\")\n", + "print(\" - vw_critical_alerts (unacknowledged alerts)\")\n", + "print(\"\\n\" + \"=\" * 70)" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From b97bf11647fee88d5457cfa5aa4f8e6e05aa0c20 Mon Sep 17 00:00:00 2001 From: Ayoub KEBAILI Date: Thu, 20 Nov 2025 15:53:52 -0500 Subject: [PATCH 3/4] Remove obsolete notebooks and pipeline configurations for feature tracking setup --- .../notebook-content.ipynb | 412 ----------------- .../.platform | 12 - .../notebook-content.ipynb | 324 -------------- .../.platform | 12 - .../notebook-content.ipynb | 372 ---------------- .../.platform | 12 - .../notebook-content.ipynb | 417 ------------------ .../pipeline-content.json | 114 ----- .../.platform | 12 - .../notebook-content.ipynb | 348 --------------- .../notebook-content.ipynb | 382 ---------------- 11 files changed, 2417 deletions(-) delete mode 100644 monitoring/fabric-unified-admin-monitoring/src/01_Transfer_Feature_Releases_GpsApi_Unit.Notebook/notebook-content.ipynb delete mode 100644 monitoring/fabric-unified-admin-monitoring/src/01_Transfer_Feature_Releases_Unit.Notebook/.platform delete mode 100644 monitoring/fabric-unified-admin-monitoring/src/01_Transfer_Feature_Releases_Unit.Notebook/notebook-content.ipynb delete mode 100644 monitoring/fabric-unified-admin-monitoring/src/02_Transfer_Preview_Features_Unit.Notebook/.platform delete mode 100644 monitoring/fabric-unified-admin-monitoring/src/02_Transfer_Preview_Features_Unit.Notebook/notebook-content.ipynb delete mode 100644 monitoring/fabric-unified-admin-monitoring/src/03_Transfer_Feature_Alerts_Unit.Notebook/.platform delete mode 100644 monitoring/fabric-unified-admin-monitoring/src/03_Transfer_Feature_Alerts_Unit.Notebook/notebook-content.ipynb delete mode 100644 monitoring/fabric-unified-admin-monitoring/src/Load_Feature_Tracking_E2E.DataPipeline/pipeline-content.json delete mode 100644 monitoring/fabric-unified-admin-monitoring/src/Setup_Feature_Tracking_Tables.Notebook/.platform delete mode 100644 monitoring/fabric-unified-admin-monitoring/src/Setup_Feature_Tracking_Tables.Notebook/notebook-content.ipynb delete mode 100644 monitoring/fabric-unified-admin-monitoring/src/Setup_Feature_Tracking_Tables_GpsApi.Notebook/notebook-content.ipynb diff --git a/monitoring/fabric-unified-admin-monitoring/src/01_Transfer_Feature_Releases_GpsApi_Unit.Notebook/notebook-content.ipynb b/monitoring/fabric-unified-admin-monitoring/src/01_Transfer_Feature_Releases_GpsApi_Unit.Notebook/notebook-content.ipynb deleted file mode 100644 index 013b0555..00000000 --- a/monitoring/fabric-unified-admin-monitoring/src/01_Transfer_Feature_Releases_GpsApi_Unit.Notebook/notebook-content.ipynb +++ /dev/null @@ -1,412 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "id": "785109b8", - "metadata": {}, - "outputs": [], - "source": [ - "import requests\n", - "import hashlib\n", - "from datetime import datetime, timedelta\n", - "from pyspark.sql import functions as F\n", - "from pyspark.sql.types import *" - ] - }, - { - "cell_type": "markdown", - "id": "48325bc2", - "metadata": {}, - "source": [ - "## Parameters" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "3bab581f", - "metadata": {}, - "outputs": [], - "source": [ - "# Parameters (can be passed from Pipeline)\n", - "fabric_gps_api_url = \"https://fabric-gps.com/api/releases\"\n", - "modified_within_days = 90 # Get features modified in last N days (1-30 per API, but we'll fetch all)\n", - "page_size = 200 # Max allowed by API\n", - "include_planned = True # Include planned features (not yet shipped)\n", - "include_shipped = True # Include shipped features" - ] - }, - { - "cell_type": "markdown", - "id": "805c7883", - "metadata": {}, - "source": [ - "## Step 1: Fetch Feature Releases from Fabric GPS API" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "fb075233", - "metadata": {}, - "outputs": [], - "source": [ - "def fetch_all_fabric_gps_releases(base_url, page_size=200, modified_within_days=None, include_planned=True, include_shipped=True):\n", - " \"\"\"\n", - " Fetch all releases from Fabric GPS API with pagination\n", - " Returns list of release objects\n", - " \"\"\"\n", - " all_releases = []\n", - " page = 1\n", - " \n", - " print(f\"šŸ”„ Fetching from Fabric GPS API: {base_url}\")\n", - " \n", - " while True:\n", - " try:\n", - " # Build query parameters\n", - " params = {\n", - " \"page\": page,\n", - " \"page_size\": page_size\n", - " }\n", - " \n", - " if modified_within_days and modified_within_days <= 30:\n", - " params[\"modified_within_days\"] = modified_within_days\n", - " \n", - " # Make request\n", - " response = requests.get(base_url, params=params, timeout=30)\n", - " response.raise_for_status()\n", - " data = response.json()\n", - " \n", - " # Extract releases\n", - " releases = data.get(\"data\", [])\n", - " \n", - " if not releases:\n", - " print(f\" ā†’ No more data on page {page}\")\n", - " break\n", - " \n", - " # Filter by release_status if specified\n", - " filtered_releases = []\n", - " for release in releases:\n", - " status = release.get(\"release_status\", \"\")\n", - " \n", - " # Apply filters\n", - " if status == \"Planned\" and not include_planned:\n", - " continue\n", - " if status == \"Shipped\" and not include_shipped:\n", - " continue\n", - " \n", - " filtered_releases.append(release)\n", - " \n", - " all_releases.extend(filtered_releases)\n", - " \n", - " # Check pagination\n", - " pagination = data.get(\"pagination\", {})\n", - " has_next = pagination.get(\"has_next\", False)\n", - " total_items = pagination.get(\"total_items\", 0)\n", - " \n", - " print(f\" ā†’ Page {page}: Fetched {len(filtered_releases)} releases (Total: {len(all_releases)}/{total_items})\")\n", - " \n", - " if not has_next:\n", - " print(f\" āœ… Reached end of data\")\n", - " break\n", - " \n", - " page += 1\n", - " \n", - " except Exception as e:\n", - " print(f\" āŒ Error fetching page {page}: {e}\")\n", - " break\n", - " \n", - " print(f\"\\nāœ… Total releases fetched: {len(all_releases)}\")\n", - " return all_releases\n", - "\n", - "# Fetch all releases\n", - "releases_data = fetch_all_fabric_gps_releases(\n", - " fabric_gps_api_url, \n", - " page_size=page_size,\n", - " modified_within_days=modified_within_days if modified_within_days <= 30 else None,\n", - " include_planned=include_planned,\n", - " include_shipped=include_shipped\n", - ")\n", - "\n", - "print(f\"\\nšŸ“‹ Sample releases:\")\n", - "for release in releases_data[:3]:\n", - " print(f\" - {release.get('feature_name')} ({release.get('release_type')}) - {release.get('product_name')}\")" - ] - }, - { - "cell_type": "markdown", - "id": "32c24c88", - "metadata": {}, - "source": [ - "## Step 2: Transform to FUAM Schema" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9eebb7af", - "metadata": {}, - "outputs": [], - "source": [ - "def transform_fabric_gps_to_fuam(releases):\n", - " \"\"\"\n", - " Transform Fabric GPS release data to FUAM feature_releases schema\n", - " \"\"\"\n", - " transformed = []\n", - " \n", - " for release in releases:\n", - " # Parse release date\n", - " release_date_str = release.get(\"release_date\")\n", - " try:\n", - " if release_date_str:\n", - " release_date = datetime.strptime(release_date_str, \"%Y-%m-%d\")\n", - " else:\n", - " release_date = None # Planned features may not have a date yet\n", - " except:\n", - " release_date = None\n", - " \n", - " # Parse last modified\n", - " last_modified_str = release.get(\"last_modified\")\n", - " try:\n", - " last_modified = datetime.strptime(last_modified_str, \"%Y-%m-%d\") if last_modified_str else datetime.now()\n", - " except:\n", - " last_modified = datetime.now()\n", - " \n", - " # Determine if preview\n", - " release_type = release.get(\"release_type\", \"\")\n", - " is_preview = \"preview\" in release_type.lower()\n", - " \n", - " # Map product_name to workload (align with FUAM naming)\n", - " product_name = release.get(\"product_name\", \"Unknown\")\n", - " workload_mapping = {\n", - " \"Power BI\": \"Power BI\",\n", - " \"Data Factory\": \"Data Factory\",\n", - " \"Data Engineering\": \"Data Engineering\",\n", - " \"Data Science\": \"Data Science\",\n", - " \"Data Warehouse\": \"Data Warehouse\",\n", - " \"Real-Time Intelligence\": \"Real-Time Intelligence\",\n", - " \"OneLake\": \"Data Engineering\",\n", - " \"Administration, Governance and Security\": \"Governance\",\n", - " \"Cosmos DB (NoSQL)\": \"Cosmos DB\"\n", - " }\n", - " workload = workload_mapping.get(product_name, product_name)\n", - " \n", - " # Create feature record\n", - " feature = {\n", - " \"feature_id\": release.get(\"release_item_id\"), # Use Fabric GPS UUID\n", - " \"feature_name\": release.get(\"feature_name\", \"Unknown\"),\n", - " \"feature_description\": release.get(\"feature_description\"),\n", - " \"workload\": workload,\n", - " \"product_name\": product_name,\n", - " \"release_date\": release_date,\n", - " \"release_type\": release_type,\n", - " \"release_status\": release.get(\"release_status\", \"Unknown\"),\n", - " \"is_preview\": is_preview,\n", - " \"is_planned\": release.get(\"release_status\") == \"Planned\",\n", - " \"is_shipped\": release.get(\"release_status\") == \"Shipped\",\n", - " \"last_modified\": last_modified,\n", - " \"source_url\": f\"https://fabric-gps.com/api/releases?release_item_id={release.get('release_item_id')}\",\n", - " \"source\": \"Fabric GPS API\",\n", - " \"extracted_date\": datetime.now()\n", - " }\n", - " \n", - " transformed.append(feature)\n", - " \n", - " return transformed\n", - "\n", - "print(\"šŸ”„ Transforming releases to FUAM schema...\")\n", - "features_data = transform_fabric_gps_to_fuam(releases_data)\n", - "print(f\"āœ… Transformed {len(features_data)} features\")\n", - "\n", - "# Statistics\n", - "preview_count = sum(1 for f in features_data if f[\"is_preview\"])\n", - "planned_count = sum(1 for f in features_data if f[\"is_planned\"])\n", - "shipped_count = sum(1 for f in features_data if f[\"is_shipped\"])\n", - "\n", - "print(f\"\\nšŸ“Š Feature Breakdown:\")\n", - "print(f\" - Preview features: {preview_count}\")\n", - "print(f\" - Planned features: {planned_count}\")\n", - "print(f\" - Shipped features: {shipped_count}\")" - ] - }, - { - "cell_type": "markdown", - "id": "ecd949aa", - "metadata": {}, - "source": [ - "## Step 3: Create DataFrame" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f4feed1d", - "metadata": {}, - "outputs": [], - "source": [ - "schema = StructType([\n", - " StructField(\"feature_id\", StringType(), False),\n", - " StructField(\"feature_name\", StringType(), False),\n", - " StructField(\"feature_description\", StringType(), True),\n", - " StructField(\"workload\", StringType(), True),\n", - " StructField(\"product_name\", StringType(), True),\n", - " StructField(\"release_date\", TimestampType(), True), # Nullable for planned features\n", - " StructField(\"release_type\", StringType(), True),\n", - " StructField(\"release_status\", StringType(), True),\n", - " StructField(\"is_preview\", BooleanType(), False),\n", - " StructField(\"is_planned\", BooleanType(), False),\n", - " StructField(\"is_shipped\", BooleanType(), False),\n", - " StructField(\"last_modified\", TimestampType(), False),\n", - " StructField(\"source_url\", StringType(), True),\n", - " StructField(\"source\", StringType(), True),\n", - " StructField(\"extracted_date\", TimestampType(), False)\n", - "])\n", - "\n", - "df_features = spark.createDataFrame(features_data, schema=schema)\n", - "\n", - "print(f\"āœ… Created DataFrame with {df_features.count()} rows\")\n", - "df_features.show(5, truncate=False)" - ] - }, - { - "cell_type": "markdown", - "id": "55d7f50e", - "metadata": {}, - "source": [ - "## Step 4: Write to Delta Lake" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "afc62e35", - "metadata": {}, - "outputs": [], - "source": [ - "# Define table path\n", - "table_name = \"feature_releases_roadmap\"\n", - "table_path = f\"Tables/{table_name}\"\n", - "\n", - "print(f\"šŸ”„ Writing to Delta table: {table_name}\")\n", - "\n", - "# Write with merge logic (upsert)\n", - "try:\n", - " from delta.tables import DeltaTable\n", - " \n", - " # Check if table exists\n", - " if DeltaTable.isDeltaTable(spark, table_path):\n", - " print(\" ā†’ Table exists, performing MERGE (upsert)...\")\n", - " \n", - " delta_table = DeltaTable.forPath(spark, table_path)\n", - " \n", - " # Merge: Update existing, insert new\n", - " # Use last_modified to track changes\n", - " delta_table.alias(\"target\").merge(\n", - " df_features.alias(\"source\"),\n", - " \"target.feature_id = source.feature_id\"\n", - " ).whenMatchedUpdate(\n", - " condition=\"source.last_modified > target.last_modified\",\n", - " set={\n", - " \"feature_name\": \"source.feature_name\",\n", - " \"feature_description\": \"source.feature_description\",\n", - " \"workload\": \"source.workload\",\n", - " \"product_name\": \"source.product_name\",\n", - " \"release_date\": \"source.release_date\",\n", - " \"release_type\": \"source.release_type\",\n", - " \"release_status\": \"source.release_status\",\n", - " \"is_preview\": \"source.is_preview\",\n", - " \"is_planned\": \"source.is_planned\",\n", - " \"is_shipped\": \"source.is_shipped\",\n", - " \"last_modified\": \"source.last_modified\",\n", - " \"source_url\": \"source.source_url\",\n", - " \"extracted_date\": \"source.extracted_date\"\n", - " }\n", - " ).whenNotMatchedInsertAll(\n", - " ).execute()\n", - " \n", - " print(\" āœ… MERGE completed\")\n", - " else:\n", - " print(\" ā†’ Table doesn't exist, creating new table...\")\n", - " df_features.write.format(\"delta\").mode(\"overwrite\").save(table_path)\n", - " print(\" āœ… Table created\")\n", - " \n", - " # Show final count\n", - " final_count = spark.read.format(\"delta\").load(table_path).count()\n", - " print(f\"\\nāœ… Total records in {table_name}: {final_count}\")\n", - " \n", - "except Exception as e:\n", - " print(f\"āŒ Error writing to Delta: {e}\")\n", - " raise" - ] - }, - { - "cell_type": "markdown", - "id": "7ce4ca77", - "metadata": {}, - "source": [ - "## Step 5: Summary Statistics" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5e58a863", - "metadata": {}, - "outputs": [], - "source": [ - "print(\"\\nšŸ“Š Feature Release Roadmap Statistics:\")\n", - "print(\"=\" * 70)\n", - "\n", - "df_summary = spark.read.format(\"delta\").load(table_path)\n", - "\n", - "# By workload/product\n", - "print(\"\\nšŸ”ø By Workload:\")\n", - "df_summary.groupBy(\"workload\").count().orderBy(F.desc(\"count\")).show(truncate=False)\n", - "\n", - "# By release type\n", - "print(\"\\nšŸ”ø By Release Type:\")\n", - "df_summary.groupBy(\"release_type\").count().orderBy(F.desc(\"count\")).show(truncate=False)\n", - "\n", - "# By release status\n", - "print(\"\\nšŸ”ø By Release Status:\")\n", - "df_summary.groupBy(\"release_status\").count().orderBy(F.desc(\"count\")).show(truncate=False)\n", - "\n", - "# Preview vs GA\n", - "print(\"\\nšŸ”ø Preview vs GA:\")\n", - "df_summary.groupBy(\"is_preview\").count().show(truncate=False)\n", - "\n", - "# Planned vs Shipped\n", - "print(\"\\nšŸ”ø Planned vs Shipped:\")\n", - "df_summary.select(\"is_planned\", \"is_shipped\").groupBy(\"is_planned\", \"is_shipped\").count().show(truncate=False)\n", - "\n", - "# Recent modifications (last 30 days)\n", - "recent_cutoff = datetime.now() - timedelta(days=30)\n", - "recent_count = df_summary.filter(F.col(\"last_modified\") >= recent_cutoff).count()\n", - "print(f\"\\nšŸ”ø Features modified in last 30 days: {recent_count}\")\n", - "\n", - "# Upcoming releases (planned)\n", - "planned_count = df_summary.filter(F.col(\"is_planned\") == True).count()\n", - "print(f\"šŸ”ø Planned features (roadmap): {planned_count}\")\n", - "\n", - "# Show upcoming preview features\n", - "print(\"\\nšŸ”ø Upcoming Preview Features (sample):\")\n", - "df_summary.filter((F.col(\"is_preview\") == True) & (F.col(\"is_planned\") == True)) \\\n", - " .select(\"feature_name\", \"product_name\", \"release_date\", \"release_status\") \\\n", - " .orderBy(F.col(\"release_date\").asc()) \\\n", - " .show(10, truncate=60)\n", - "\n", - "print(\"\\n\" + \"=\" * 70)\n", - "print(\"āœ… Transfer Feature Releases GpsApi Unit - COMPLETED\")\n", - "print(f\"šŸ’” Tip: Use this table for roadmap planning and preview feature tracking\")" - ] - } - ], - "metadata": { - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/monitoring/fabric-unified-admin-monitoring/src/01_Transfer_Feature_Releases_Unit.Notebook/.platform b/monitoring/fabric-unified-admin-monitoring/src/01_Transfer_Feature_Releases_Unit.Notebook/.platform deleted file mode 100644 index c7df0caf..00000000 --- a/monitoring/fabric-unified-admin-monitoring/src/01_Transfer_Feature_Releases_Unit.Notebook/.platform +++ /dev/null @@ -1,12 +0,0 @@ -{ - "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", - "metadata": { - "type": "Notebook", - "displayName": "01_Transfer_Feature_Releases_Unit", - "description": "Extract Microsoft Fabric feature releases from Microsoft Learn" - }, - "config": { - "version": "2.0", - "logicalId": "00000000-0000-0000-0000-000000000000" - } -} diff --git a/monitoring/fabric-unified-admin-monitoring/src/01_Transfer_Feature_Releases_Unit.Notebook/notebook-content.ipynb b/monitoring/fabric-unified-admin-monitoring/src/01_Transfer_Feature_Releases_Unit.Notebook/notebook-content.ipynb deleted file mode 100644 index 4e7e5449..00000000 --- a/monitoring/fabric-unified-admin-monitoring/src/01_Transfer_Feature_Releases_Unit.Notebook/notebook-content.ipynb +++ /dev/null @@ -1,324 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "6be68f0c", - "metadata": {}, - "source": [ - "# 01 - Transfer Feature Releases Unit\n", - "\n", - "**Purpose**: Extract Microsoft Fabric feature releases from Microsoft Learn what's new documentation\n", - "\n", - "**Inputs**:\n", - "- Microsoft Learn URL: https://learn.microsoft.com/en-us/fabric/get-started/whats-new\n", - "\n", - "**Outputs**:\n", - "- Delta Table: `feature_releases`\n", - "\n", - "**Frequency**: Daily (recommended)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b00196e6", - "metadata": {}, - "outputs": [], - "source": [ - "import requests\n", - "import re\n", - "from datetime import datetime, timedelta\n", - "from pyspark.sql import functions as F\n", - "from pyspark.sql.types import *" - ] - }, - { - "cell_type": "markdown", - "id": "5ddc0cf4", - "metadata": {}, - "source": [ - "## Parameters" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b7540d3c", - "metadata": {}, - "outputs": [], - "source": [ - "# Parameters (can be passed from Pipeline)\n", - "whats_new_url = \"https://learn.microsoft.com/en-us/fabric/get-started/whats-new\"\n", - "lookback_days = 90 # Extract features from last 90 days" - ] - }, - { - "cell_type": "markdown", - "id": "ddba066b", - "metadata": {}, - "source": [ - "## Step 1: Fetch What's New Content" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "63f9cdc9", - "metadata": {}, - "outputs": [], - "source": [ - "print(f\"šŸ”„ Fetching Fabric What's New from: {whats_new_url}\")\n", - "\n", - "try:\n", - " response = requests.get(whats_new_url, timeout=30)\n", - " response.raise_for_status()\n", - " content = response.text\n", - " print(f\"āœ… Successfully fetched {len(content)} characters\")\n", - "except Exception as e:\n", - " print(f\"āŒ Error fetching content: {e}\")\n", - " raise" - ] - }, - { - "cell_type": "markdown", - "id": "39a8fb67", - "metadata": {}, - "source": [ - "## Step 2: Parse Feature Releases" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9eced003", - "metadata": {}, - "outputs": [], - "source": [ - "def parse_feature_releases(html_content):\n", - " \"\"\"\n", - " Parse feature releases from Microsoft Learn What's New page\n", - " Returns list of dicts with feature information\n", - " \"\"\"\n", - " features = []\n", - " \n", - " # Pattern to match feature sections\n", - " # Example: ## September 2024\n", - " month_pattern = r'##\\s+([A-Za-z]+\\s+\\d{4})'\n", - " \n", - " # Pattern to match feature entries\n", - " # Example: ### Data Warehouse - New feature (Preview)\n", - " feature_pattern = r'###\\s+(.+?)\\s*(?:\\(([^)]+)\\))?'\n", - " \n", - " # Split content by months\n", - " months = re.split(month_pattern, html_content)\n", - " \n", - " for i in range(1, len(months), 2):\n", - " if i+1 >= len(months):\n", - " break\n", - " \n", - " month_str = months[i].strip()\n", - " month_content = months[i+1]\n", - " \n", - " try:\n", - " release_date = datetime.strptime(month_str, \"%B %Y\")\n", - " except:\n", - " continue\n", - " \n", - " # Find all features in this month\n", - " feature_matches = re.finditer(feature_pattern, month_content)\n", - " \n", - " for match in feature_matches:\n", - " feature_name = match.group(1).strip()\n", - " status = match.group(2).strip() if match.group(2) else \"GA\"\n", - " \n", - " # Determine workload from feature name\n", - " workload = \"Unknown\"\n", - " if any(kw in feature_name.lower() for kw in [\"data warehouse\", \"warehouse\", \"sql\"]):\n", - " workload = \"Data Warehouse\"\n", - " elif any(kw in feature_name.lower() for kw in [\"data factory\", \"pipeline\", \"dataflow\"]):\n", - " workload = \"Data Factory\"\n", - " elif any(kw in feature_name.lower() for kw in [\"power bi\", \"semantic model\", \"report\"]):\n", - " workload = \"Power BI\"\n", - " elif any(kw in feature_name.lower() for kw in [\"real-time\", \"kql\", \"eventhouse\"]):\n", - " workload = \"Real-Time Intelligence\"\n", - " elif any(kw in feature_name.lower() for kw in [\"lakehouse\", \"onelake\"]):\n", - " workload = \"Data Engineering\"\n", - " elif any(kw in feature_name.lower() for kw in [\"data science\", \"ml\", \"notebook\"]):\n", - " workload = \"Data Science\"\n", - " \n", - " # Determine if it's a preview feature\n", - " is_preview = \"preview\" in status.lower()\n", - " \n", - " features.append({\n", - " \"feature_id\": f\"{release_date.strftime('%Y%m')}_{hash(feature_name) % 100000}\",\n", - " \"feature_name\": feature_name,\n", - " \"workload\": workload,\n", - " \"release_date\": release_date,\n", - " \"status\": status,\n", - " \"is_preview\": is_preview,\n", - " \"source_url\": whats_new_url,\n", - " \"extracted_date\": datetime.now()\n", - " })\n", - " \n", - " return features" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "05e9a48b", - "metadata": {}, - "outputs": [], - "source": [ - "print(\"šŸ”„ Parsing feature releases...\")\n", - "features_data = parse_feature_releases(content)\n", - "\n", - "# Filter to last N days\n", - "cutoff_date = datetime.now() - timedelta(days=lookback_days)\n", - "features_data = [f for f in features_data if f[\"release_date\"] >= cutoff_date]\n", - "\n", - "print(f\"āœ… Parsed {len(features_data)} feature releases from last {lookback_days} days\")\n", - "\n", - "# Preview\n", - "if features_data:\n", - " print(\"\\nšŸ“‹ Sample features:\")\n", - " for feat in features_data[:3]:\n", - " print(f\" - {feat['feature_name']} ({feat['status']}) - {feat['workload']}\")" - ] - }, - { - "cell_type": "markdown", - "id": "54aa947a", - "metadata": {}, - "source": [ - "## Step 3: Create DataFrame" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "0df118f4", - "metadata": {}, - "outputs": [], - "source": [ - "schema = StructType([\n", - " StructField(\"feature_id\", StringType(), False),\n", - " StructField(\"feature_name\", StringType(), False),\n", - " StructField(\"workload\", StringType(), True),\n", - " StructField(\"release_date\", TimestampType(), False),\n", - " StructField(\"status\", StringType(), True),\n", - " StructField(\"is_preview\", BooleanType(), False),\n", - " StructField(\"source_url\", StringType(), True),\n", - " StructField(\"extracted_date\", TimestampType(), False)\n", - "])\n", - "\n", - "df_features = spark.createDataFrame(features_data, schema=schema)\n", - "\n", - "print(f\"āœ… Created DataFrame with {df_features.count()} rows\")\n", - "df_features.show(5, truncate=False)" - ] - }, - { - "cell_type": "markdown", - "id": "f9eeb551", - "metadata": {}, - "source": [ - "## Step 4: Write to Delta Lake" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c99f1c39", - "metadata": {}, - "outputs": [], - "source": [ - "# Define table path\n", - "table_name = \"feature_releases\"\n", - "table_path = f\"Tables/{table_name}\"\n", - "\n", - "print(f\"šŸ”„ Writing to Delta table: {table_name}\")\n", - "\n", - "# Write with merge logic (upsert)\n", - "try:\n", - " from delta.tables import DeltaTable\n", - " \n", - " # Check if table exists\n", - " if DeltaTable.isDeltaTable(spark, table_path):\n", - " print(\" ā†’ Table exists, performing MERGE (upsert)...\")\n", - " \n", - " delta_table = DeltaTable.forPath(spark, table_path)\n", - " \n", - " # Merge: Update existing, insert new\n", - " delta_table.alias(\"target\").merge(\n", - " df_features.alias(\"source\"),\n", - " \"target.feature_id = source.feature_id\"\n", - " ).whenMatchedUpdateAll(\n", - " ).whenNotMatchedInsertAll(\n", - " ).execute()\n", - " \n", - " print(\" āœ… MERGE completed\")\n", - " else:\n", - " print(\" ā†’ Table doesn't exist, creating new table...\")\n", - " df_features.write.format(\"delta\").mode(\"overwrite\").save(table_path)\n", - " print(\" āœ… Table created\")\n", - " \n", - " # Show final count\n", - " final_count = spark.read.format(\"delta\").load(table_path).count()\n", - " print(f\"\\nāœ… Total records in {table_name}: {final_count}\")\n", - " \n", - "except Exception as e:\n", - " print(f\"āŒ Error writing to Delta: {e}\")\n", - " raise" - ] - }, - { - "cell_type": "markdown", - "id": "6b8f45b2", - "metadata": {}, - "source": [ - "## Step 5: Summary Statistics" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "625df20e", - "metadata": {}, - "outputs": [], - "source": [ - "print(\"\\nšŸ“Š Feature Release Statistics:\")\n", - "print(\"=\" * 60)\n", - "\n", - "df_summary = spark.read.format(\"delta\").load(table_path)\n", - "\n", - "# By workload\n", - "print(\"\\nšŸ”ø By Workload:\")\n", - "df_summary.groupBy(\"workload\").count().orderBy(F.desc(\"count\")).show(truncate=False)\n", - "\n", - "# By status\n", - "print(\"\\nšŸ”ø By Status:\")\n", - "df_summary.groupBy(\"status\").count().orderBy(F.desc(\"count\")).show(truncate=False)\n", - "\n", - "# Preview vs GA\n", - "print(\"\\nšŸ”ø Preview vs GA:\")\n", - "df_summary.groupBy(\"is_preview\").count().show(truncate=False)\n", - "\n", - "# Recent releases (last 30 days)\n", - "recent_cutoff = datetime.now() - timedelta(days=30)\n", - "recent_count = df_summary.filter(F.col(\"release_date\") >= recent_cutoff).count()\n", - "print(f\"\\nšŸ”ø Features released in last 30 days: {recent_count}\")\n", - "\n", - "print(\"\\n\" + \"=\" * 60)\n", - "print(\"āœ… Transfer Feature Releases Unit - COMPLETED\")" - ] - } - ], - "metadata": { - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/monitoring/fabric-unified-admin-monitoring/src/02_Transfer_Preview_Features_Unit.Notebook/.platform b/monitoring/fabric-unified-admin-monitoring/src/02_Transfer_Preview_Features_Unit.Notebook/.platform deleted file mode 100644 index 61331b64..00000000 --- a/monitoring/fabric-unified-admin-monitoring/src/02_Transfer_Preview_Features_Unit.Notebook/.platform +++ /dev/null @@ -1,12 +0,0 @@ -{ - "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", - "metadata": { - "type": "Notebook", - "displayName": "02_Transfer_Preview_Features_Unit", - "description": "Detect activated preview features by comparing tenant settings" - }, - "config": { - "version": "2.0", - "logicalId": "00000000-0000-0000-0000-000000000000" - } -} diff --git a/monitoring/fabric-unified-admin-monitoring/src/02_Transfer_Preview_Features_Unit.Notebook/notebook-content.ipynb b/monitoring/fabric-unified-admin-monitoring/src/02_Transfer_Preview_Features_Unit.Notebook/notebook-content.ipynb deleted file mode 100644 index edbda538..00000000 --- a/monitoring/fabric-unified-admin-monitoring/src/02_Transfer_Preview_Features_Unit.Notebook/notebook-content.ipynb +++ /dev/null @@ -1,372 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "6a7f70e9", - "metadata": {}, - "source": [ - "# 02 - Transfer Preview Features Unit\n", - "\n", - "**Purpose**: Detect activated preview features by comparing tenant settings with feature releases\n", - "\n", - "**Inputs**:\n", - "- Delta Table: `tenant_settings` (from FUAM)\n", - "- Delta Table: `feature_releases` (from 01_Transfer_Feature_Releases_Unit)\n", - "\n", - "**Outputs**:\n", - "- Delta Table: `preview_features_active`\n", - "\n", - "**Frequency**: Daily (recommended)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8a78d2d0", - "metadata": {}, - "outputs": [], - "source": [ - "from pyspark.sql import functions as F\n", - "from pyspark.sql.window import Window\n", - "from pyspark.sql.types import *\n", - "from datetime import datetime\n", - "from difflib import SequenceMatcher" - ] - }, - { - "cell_type": "markdown", - "id": "89e5c031", - "metadata": {}, - "source": [ - "## Step 1: Load Feature Releases (Preview only)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "3fbcbd68", - "metadata": {}, - "outputs": [], - "source": [ - "print(\"šŸ”„ Loading preview feature releases...\")\n", - "\n", - "df_features = spark.read.format(\"delta\").load(\"Tables/feature_releases\")\n", - "\n", - "# Filter to preview features only\n", - "df_preview_features = df_features.filter(F.col(\"is_preview\") == True)\n", - "\n", - "preview_count = df_preview_features.count()\n", - "print(f\"āœ… Loaded {preview_count} preview features\")\n", - "\n", - "# Show sample\n", - "df_preview_features.select(\"feature_id\", \"feature_name\", \"workload\", \"release_date\").show(5, truncate=False)" - ] - }, - { - "cell_type": "markdown", - "id": "b5353e2d", - "metadata": {}, - "source": [ - "## Step 2: Load Tenant Settings" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c88e29ee", - "metadata": {}, - "outputs": [], - "source": [ - "print(\"\\nšŸ”„ Loading tenant settings...\")\n", - "\n", - "df_tenant_settings = spark.read.format(\"delta\").load(\"Tables/tenant_settings\")\n", - "\n", - "settings_count = df_tenant_settings.count()\n", - "print(f\"āœ… Loaded {settings_count} tenant settings\")\n", - "\n", - "# Show sample\n", - "df_tenant_settings.select(\"settingName\", \"enabled\", \"delegateToTenant\").show(5, truncate=False)" - ] - }, - { - "cell_type": "markdown", - "id": "8f7422f6", - "metadata": {}, - "source": [ - "## Step 3: Map Settings to Features\n", - "\n", - "This creates a mapping between tenant setting names and preview feature names using fuzzy matching" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "bd21a1bc", - "metadata": {}, - "outputs": [], - "source": [ - "def similarity_score(a, b):\n", - " \"\"\"Calculate similarity between two strings (0-1)\"\"\"\n", - " return SequenceMatcher(None, a.lower(), b.lower()).ratio()\n", - "\n", - "def map_settings_to_features(df_settings, df_features):\n", - " \"\"\"\n", - " Map tenant settings to preview features based on name similarity\n", - " Returns DataFrame with matches\n", - " \"\"\"\n", - " \n", - " # Collect feature names for matching\n", - " features_list = df_features.select(\"feature_id\", \"feature_name\", \"workload\").collect()\n", - " settings_list = df_settings.select(\"settingName\", \"enabled\", \"delegateToTenant\").collect()\n", - " \n", - " matches = []\n", - " \n", - " for setting in settings_list:\n", - " setting_name = setting[\"settingName\"]\n", - " \n", - " # Skip if setting is not enabled\n", - " if not setting[\"enabled\"]:\n", - " continue\n", - " \n", - " # Find best matching feature\n", - " best_match = None\n", - " best_score = 0.0\n", - " \n", - " for feature in features_list:\n", - " feature_name = feature[\"feature_name\"]\n", - " \n", - " # Calculate similarity\n", - " score = similarity_score(setting_name, feature_name)\n", - " \n", - " # Also check for keyword matches\n", - " setting_words = set(setting_name.lower().split())\n", - " feature_words = set(feature_name.lower().split())\n", - " common_words = setting_words & feature_words\n", - " \n", - " if common_words:\n", - " score += len(common_words) * 0.1 # Boost score for common words\n", - " \n", - " if score > best_score and score > 0.3: # Threshold 0.3\n", - " best_score = score\n", - " best_match = feature\n", - " \n", - " if best_match:\n", - " matches.append({\n", - " \"setting_name\": setting_name,\n", - " \"feature_id\": best_match[\"feature_id\"],\n", - " \"feature_name\": best_match[\"feature_name\"],\n", - " \"workload\": best_match[\"workload\"],\n", - " \"similarity_score\": best_score,\n", - " \"is_enabled\": setting[\"enabled\"],\n", - " \"delegate_to_tenant\": setting[\"delegateToTenant\"],\n", - " \"detected_date\": datetime.now()\n", - " })\n", - " \n", - " return matches" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "a9ec9449", - "metadata": {}, - "outputs": [], - "source": [ - "print(\"šŸ”„ Mapping settings to preview features...\")\n", - "\n", - "matches = map_settings_to_features(df_tenant_settings, df_preview_features)\n", - "\n", - "print(f\"āœ… Found {len(matches)} potential preview features activated\")\n", - "\n", - "# Preview matches\n", - "if matches:\n", - " print(\"\\nšŸ“‹ Sample matches:\")\n", - " for match in matches[:5]:\n", - " print(f\" - {match['feature_name']}\")\n", - " print(f\" ā†’ Setting: {match['setting_name']}\")\n", - " print(f\" ā†’ Similarity: {match['similarity_score']:.2f}\")\n", - " print()" - ] - }, - { - "cell_type": "markdown", - "id": "ab1740ab", - "metadata": {}, - "source": [ - "## Step 4: Create DataFrame with Activated Previews" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "a271ca26", - "metadata": {}, - "outputs": [], - "source": [ - "schema = StructType([\n", - " StructField(\"setting_name\", StringType(), False),\n", - " StructField(\"feature_id\", StringType(), False),\n", - " StructField(\"feature_name\", StringType(), False),\n", - " StructField(\"workload\", StringType(), True),\n", - " StructField(\"similarity_score\", DoubleType(), False),\n", - " StructField(\"is_enabled\", BooleanType(), False),\n", - " StructField(\"delegate_to_tenant\", BooleanType(), True),\n", - " StructField(\"detected_date\", TimestampType(), False)\n", - "])\n", - "\n", - "df_active_previews = spark.createDataFrame(matches, schema=schema)\n", - "\n", - "print(f\"āœ… Created DataFrame with {df_active_previews.count()} activated preview features\")\n", - "df_active_previews.show(5, truncate=False)" - ] - }, - { - "cell_type": "markdown", - "id": "4e776aac", - "metadata": {}, - "source": [ - "## Step 5: Enrich with Feature Details" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "0593e47f", - "metadata": {}, - "outputs": [], - "source": [ - "print(\"šŸ”„ Enriching with feature release details...\")\n", - "\n", - "# Join with feature_releases to get release_date, status, etc.\n", - "df_enriched = df_active_previews.join(\n", - " df_preview_features.select(\n", - " \"feature_id\",\n", - " \"release_date\",\n", - " \"status\",\n", - " \"source_url\"\n", - " ),\n", - " on=\"feature_id\",\n", - " how=\"left\"\n", - ")\n", - "\n", - "# Calculate days since release\n", - "df_enriched = df_enriched.withColumn(\n", - " \"days_since_release\",\n", - " F.datediff(F.current_date(), F.col(\"release_date\"))\n", - ")\n", - "\n", - "print(\"āœ… Enrichment completed\")\n", - "df_enriched.show(5, truncate=False)" - ] - }, - { - "cell_type": "markdown", - "id": "28244a88", - "metadata": {}, - "source": [ - "## Step 6: Write to Delta Lake" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "34b079a1", - "metadata": {}, - "outputs": [], - "source": [ - "table_name = \"preview_features_active\"\n", - "table_path = f\"Tables/{table_name}\"\n", - "\n", - "print(f\"šŸ”„ Writing to Delta table: {table_name}\")\n", - "\n", - "try:\n", - " from delta.tables import DeltaTable\n", - " \n", - " # Check if table exists\n", - " if DeltaTable.isDeltaTable(spark, table_path):\n", - " print(\" ā†’ Table exists, performing MERGE (upsert)...\")\n", - " \n", - " delta_table = DeltaTable.forPath(spark, table_path)\n", - " \n", - " # Merge: Update existing, insert new\n", - " delta_table.alias(\"target\").merge(\n", - " df_enriched.alias(\"source\"),\n", - " \"target.feature_id = source.feature_id AND target.setting_name = source.setting_name\"\n", - " ).whenMatchedUpdate(\n", - " set={\n", - " \"is_enabled\": \"source.is_enabled\",\n", - " \"detected_date\": \"source.detected_date\",\n", - " \"days_since_release\": \"source.days_since_release\"\n", - " }\n", - " ).whenNotMatchedInsertAll(\n", - " ).execute()\n", - " \n", - " print(\" āœ… MERGE completed\")\n", - " else:\n", - " print(\" ā†’ Table doesn't exist, creating new table...\")\n", - " df_enriched.write.format(\"delta\").mode(\"overwrite\").save(table_path)\n", - " print(\" āœ… Table created\")\n", - " \n", - " # Show final count\n", - " final_count = spark.read.format(\"delta\").load(table_path).count()\n", - " print(f\"\\nāœ… Total activated preview features in {table_name}: {final_count}\")\n", - " \n", - "except Exception as e:\n", - " print(f\"āŒ Error writing to Delta: {e}\")\n", - " raise" - ] - }, - { - "cell_type": "markdown", - "id": "dd1e5bdd", - "metadata": {}, - "source": [ - "## Step 7: Summary Statistics" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "bc0d84f5", - "metadata": {}, - "outputs": [], - "source": [ - "print(\"\\nšŸ“Š Activated Preview Features Statistics:\")\n", - "print(\"=\" * 60)\n", - "\n", - "df_summary = spark.read.format(\"delta\").load(table_path)\n", - "\n", - "# By workload\n", - "print(\"\\nšŸ”ø By Workload:\")\n", - "df_summary.groupBy(\"workload\").count().orderBy(F.desc(\"count\")).show(truncate=False)\n", - "\n", - "# By similarity score range\n", - "print(\"\\nšŸ”ø By Confidence Level (Similarity Score):\")\n", - "df_summary.withColumn(\n", - " \"confidence\",\n", - " F.when(F.col(\"similarity_score\") >= 0.7, \"High (ā‰„0.7)\")\n", - " .when(F.col(\"similarity_score\") >= 0.5, \"Medium (0.5-0.7)\")\n", - " .otherwise(\"Low (<0.5)\")\n", - ").groupBy(\"confidence\").count().orderBy(F.desc(\"count\")).show(truncate=False)\n", - "\n", - "# Recently released (last 30 days) but already activated\n", - "recent_activated = df_summary.filter(F.col(\"days_since_release\") <= 30).count()\n", - "print(f\"\\nšŸ”ø Features activated within 30 days of release: {recent_activated}\")\n", - "\n", - "# Features that have been in preview for long time\n", - "long_preview = df_summary.filter(F.col(\"days_since_release\") > 180).count()\n", - "print(f\"šŸ”ø Features in preview for >180 days: {long_preview}\")\n", - "\n", - "print(\"\\n\" + \"=\" * 60)\n", - "print(\"āœ… Transfer Preview Features Unit - COMPLETED\")" - ] - } - ], - "metadata": { - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/monitoring/fabric-unified-admin-monitoring/src/03_Transfer_Feature_Alerts_Unit.Notebook/.platform b/monitoring/fabric-unified-admin-monitoring/src/03_Transfer_Feature_Alerts_Unit.Notebook/.platform deleted file mode 100644 index 3c394b8f..00000000 --- a/monitoring/fabric-unified-admin-monitoring/src/03_Transfer_Feature_Alerts_Unit.Notebook/.platform +++ /dev/null @@ -1,12 +0,0 @@ -{ - "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", - "metadata": { - "type": "Notebook", - "displayName": "03_Transfer_Feature_Alerts_Unit", - "description": "Generate alerts for newly activated or high-risk preview features" - }, - "config": { - "version": "2.0", - "logicalId": "00000000-0000-0000-0000-000000000000" - } -} diff --git a/monitoring/fabric-unified-admin-monitoring/src/03_Transfer_Feature_Alerts_Unit.Notebook/notebook-content.ipynb b/monitoring/fabric-unified-admin-monitoring/src/03_Transfer_Feature_Alerts_Unit.Notebook/notebook-content.ipynb deleted file mode 100644 index aafd9da2..00000000 --- a/monitoring/fabric-unified-admin-monitoring/src/03_Transfer_Feature_Alerts_Unit.Notebook/notebook-content.ipynb +++ /dev/null @@ -1,417 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "8230571e", - "metadata": {}, - "source": [ - "# 03 - Transfer Feature Alerts Unit\n", - "\n", - "**Purpose**: Generate alerts for newly activated or high-risk preview features\n", - "\n", - "**Inputs**:\n", - "- Delta Table: `preview_features_active`\n", - "- Delta Table: `feature_alerts` (for historical tracking)\n", - "\n", - "**Outputs**:\n", - "- Delta Table: `feature_alerts`\n", - "\n", - "**Frequency**: Daily (recommended)\n", - "\n", - "**Alert Triggers**:\n", - "- New preview feature detected\n", - "- Preview feature active >90 days (approaching GA)\n", - "- Preview feature with low confidence match (similarity <0.5)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "aa08e36b", - "metadata": {}, - "outputs": [], - "source": [ - "from pyspark.sql import functions as F\n", - "from pyspark.sql.types import *\n", - "from datetime import datetime, timedelta" - ] - }, - { - "cell_type": "markdown", - "id": "b031c9ec", - "metadata": {}, - "source": [ - "## Parameters" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "88adb1d2", - "metadata": {}, - "outputs": [], - "source": [ - "# Alert thresholds\n", - "ALERT_DAYS_THRESHOLD = 90 # Alert if preview active >90 days\n", - "LOW_CONFIDENCE_THRESHOLD = 0.5 # Alert if similarity score <0.5\n", - "\n", - "# Alert severity levels\n", - "SEVERITY_INFO = \"Info\"\n", - "SEVERITY_WARNING = \"Warning\"\n", - "SEVERITY_CRITICAL = \"Critical\"" - ] - }, - { - "cell_type": "markdown", - "id": "e3ff6936", - "metadata": {}, - "source": [ - "## Step 1: Load Current Activated Preview Features" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c07b23d7", - "metadata": {}, - "outputs": [], - "source": [ - "print(\"šŸ”„ Loading activated preview features...\")\n", - "\n", - "df_active = spark.read.format(\"delta\").load(\"Tables/preview_features_active\")\n", - "\n", - "active_count = df_active.count()\n", - "print(f\"āœ… Loaded {active_count} activated preview features\")\n", - "\n", - "df_active.select(\"feature_id\", \"feature_name\", \"workload\", \"days_since_release\", \"similarity_score\").show(5, truncate=False)" - ] - }, - { - "cell_type": "markdown", - "id": "f991be8e", - "metadata": {}, - "source": [ - "## Step 2: Load Historical Alerts (to avoid duplicates)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7fe05695", - "metadata": {}, - "outputs": [], - "source": [ - "table_name = \"feature_alerts\"\n", - "table_path = f\"Tables/{table_name}\"\n", - "\n", - "try:\n", - " from delta.tables import DeltaTable\n", - " \n", - " if DeltaTable.isDeltaTable(spark, table_path):\n", - " print(\"šŸ”„ Loading historical alerts...\")\n", - " df_historical = spark.read.format(\"delta\").load(table_path)\n", - " historical_count = df_historical.count()\n", - " print(f\"āœ… Loaded {historical_count} historical alerts\")\n", - " \n", - " # Get already alerted feature_ids\n", - " alerted_features = set([row[\"feature_id\"] for row in df_historical.select(\"feature_id\").distinct().collect()])\n", - " print(f\" ā†’ {len(alerted_features)} unique features already alerted\")\n", - " else:\n", - " print(\"ā„¹ļø No historical alerts found (first run)\")\n", - " df_historical = None\n", - " alerted_features = set()\n", - "except Exception as e:\n", - " print(f\"ā„¹ļø No historical alerts (first run): {e}\")\n", - " df_historical = None\n", - " alerted_features = set()" - ] - }, - { - "cell_type": "markdown", - "id": "440eeb33", - "metadata": {}, - "source": [ - "## Step 3: Generate Alerts" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "a948004d", - "metadata": {}, - "outputs": [], - "source": [ - "def generate_alerts(df_active, alerted_features):\n", - " \"\"\"\n", - " Generate alerts based on business rules\n", - " Returns list of alert dicts\n", - " \"\"\"\n", - " alerts = []\n", - " \n", - " for row in df_active.collect():\n", - " feature_id = row[\"feature_id\"]\n", - " feature_name = row[\"feature_name\"]\n", - " workload = row[\"workload\"]\n", - " days_since_release = row[\"days_since_release\"]\n", - " similarity_score = row[\"similarity_score\"]\n", - " setting_name = row[\"setting_name\"]\n", - " \n", - " # Rule 1: NEW PREVIEW FEATURE ACTIVATED\n", - " if feature_id not in alerted_features:\n", - " alerts.append({\n", - " \"alert_id\": f\"NEW_{feature_id}_{datetime.now().strftime('%Y%m%d')}\",\n", - " \"feature_id\": feature_id,\n", - " \"feature_name\": feature_name,\n", - " \"workload\": workload,\n", - " \"alert_type\": \"New Preview Activated\",\n", - " \"severity\": SEVERITY_INFO,\n", - " \"message\": f\"New preview feature '{feature_name}' has been activated (Setting: {setting_name})\",\n", - " \"days_since_release\": days_since_release,\n", - " \"similarity_score\": similarity_score,\n", - " \"alert_date\": datetime.now(),\n", - " \"acknowledged\": False\n", - " })\n", - " \n", - " # Rule 2: LONG-RUNNING PREVIEW (>90 days)\n", - " if days_since_release and days_since_release > ALERT_DAYS_THRESHOLD:\n", - " alert_id = f\"LONGRUN_{feature_id}_{datetime.now().strftime('%Y%m%d')}\"\n", - " \n", - " alerts.append({\n", - " \"alert_id\": alert_id,\n", - " \"feature_id\": feature_id,\n", - " \"feature_name\": feature_name,\n", - " \"workload\": workload,\n", - " \"alert_type\": \"Long-Running Preview\",\n", - " \"severity\": SEVERITY_WARNING,\n", - " \"message\": f\"Preview feature '{feature_name}' has been active for {days_since_release} days. Consider reviewing for GA transition.\",\n", - " \"days_since_release\": days_since_release,\n", - " \"similarity_score\": similarity_score,\n", - " \"alert_date\": datetime.now(),\n", - " \"acknowledged\": False\n", - " })\n", - " \n", - " # Rule 3: LOW CONFIDENCE MATCH\n", - " if similarity_score < LOW_CONFIDENCE_THRESHOLD:\n", - " alerts.append({\n", - " \"alert_id\": f\"LOWCONF_{feature_id}_{datetime.now().strftime('%Y%m%d')}\",\n", - " \"feature_id\": feature_id,\n", - " \"feature_name\": feature_name,\n", - " \"workload\": workload,\n", - " \"alert_type\": \"Low Confidence Match\",\n", - " \"severity\": SEVERITY_CRITICAL,\n", - " \"message\": f\"Low confidence match ({similarity_score:.2f}) between setting '{setting_name}' and feature '{feature_name}'. Manual review recommended.\",\n", - " \"days_since_release\": days_since_release,\n", - " \"similarity_score\": similarity_score,\n", - " \"alert_date\": datetime.now(),\n", - " \"acknowledged\": False\n", - " })\n", - " \n", - " return alerts" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "e216ee50", - "metadata": {}, - "outputs": [], - "source": [ - "print(\"šŸ”„ Generating alerts based on business rules...\")\n", - "\n", - "alerts_data = generate_alerts(df_active, alerted_features)\n", - "\n", - "print(f\"āœ… Generated {len(alerts_data)} new alerts\")\n", - "\n", - "# Preview alerts\n", - "if alerts_data:\n", - " print(\"\\nšŸ“‹ Sample alerts:\")\n", - " for alert in alerts_data[:5]:\n", - " print(f\" [{alert['severity']}] {alert['alert_type']}\")\n", - " print(f\" ā†’ {alert['message']}\")\n", - " print()" - ] - }, - { - "cell_type": "markdown", - "id": "20a7afc0", - "metadata": {}, - "source": [ - "## Step 4: Create Alerts DataFrame" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6a8f9b72", - "metadata": {}, - "outputs": [], - "source": [ - "if alerts_data:\n", - " schema = StructType([\n", - " StructField(\"alert_id\", StringType(), False),\n", - " StructField(\"feature_id\", StringType(), False),\n", - " StructField(\"feature_name\", StringType(), False),\n", - " StructField(\"workload\", StringType(), True),\n", - " StructField(\"alert_type\", StringType(), False),\n", - " StructField(\"severity\", StringType(), False),\n", - " StructField(\"message\", StringType(), False),\n", - " StructField(\"days_since_release\", IntegerType(), True),\n", - " StructField(\"similarity_score\", DoubleType(), True),\n", - " StructField(\"alert_date\", TimestampType(), False),\n", - " StructField(\"acknowledged\", BooleanType(), False)\n", - " ])\n", - " \n", - " df_alerts = spark.createDataFrame(alerts_data, schema=schema)\n", - " \n", - " print(f\"āœ… Created alerts DataFrame with {df_alerts.count()} rows\")\n", - " df_alerts.show(5, truncate=False)\n", - "else:\n", - " print(\"ā„¹ļø No new alerts to generate\")\n", - " df_alerts = None" - ] - }, - { - "cell_type": "markdown", - "id": "8ae1e2fc", - "metadata": {}, - "source": [ - "## Step 5: Write Alerts to Delta Lake" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "49940421", - "metadata": {}, - "outputs": [], - "source": [ - "if df_alerts and df_alerts.count() > 0:\n", - " print(f\"šŸ”„ Writing alerts to Delta table: {table_name}\")\n", - " \n", - " try:\n", - " from delta.tables import DeltaTable\n", - " \n", - " # Check if table exists\n", - " if DeltaTable.isDeltaTable(spark, table_path):\n", - " print(\" ā†’ Table exists, appending new alerts...\")\n", - " \n", - " # Append mode - keep all alerts for historical tracking\n", - " df_alerts.write.format(\"delta\").mode(\"append\").save(table_path)\n", - " \n", - " print(\" āœ… Alerts appended\")\n", - " else:\n", - " print(\" ā†’ Table doesn't exist, creating new table...\")\n", - " df_alerts.write.format(\"delta\").mode(\"overwrite\").save(table_path)\n", - " print(\" āœ… Table created\")\n", - " \n", - " # Show final count\n", - " final_count = spark.read.format(\"delta\").load(table_path).count()\n", - " print(f\"\\nāœ… Total alerts in {table_name}: {final_count}\")\n", - " \n", - " except Exception as e:\n", - " print(f\"āŒ Error writing alerts: {e}\")\n", - " raise\n", - "else:\n", - " print(\"ā„¹ļø No alerts to write\")" - ] - }, - { - "cell_type": "markdown", - "id": "b709754b", - "metadata": {}, - "source": [ - "## Step 6: Alert Summary Statistics" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f9518879", - "metadata": {}, - "outputs": [], - "source": [ - "from delta.tables import DeltaTable\n", - "\n", - "if DeltaTable.isDeltaTable(spark, table_path):\n", - " print(\"\\nšŸ“Š Alert Statistics:\")\n", - " print(\"=\" * 60)\n", - " \n", - " df_all_alerts = spark.read.format(\"delta\").load(table_path)\n", - " \n", - " # By severity\n", - " print(\"\\nšŸ”ø By Severity:\")\n", - " df_all_alerts.groupBy(\"severity\").count().orderBy(F.desc(\"count\")).show(truncate=False)\n", - " \n", - " # By alert type\n", - " print(\"\\nšŸ”ø By Alert Type:\")\n", - " df_all_alerts.groupBy(\"alert_type\").count().orderBy(F.desc(\"count\")).show(truncate=False)\n", - " \n", - " # By workload\n", - " print(\"\\nšŸ”ø By Workload:\")\n", - " df_all_alerts.groupBy(\"workload\").count().orderBy(F.desc(\"count\")).show(truncate=False)\n", - " \n", - " # Unacknowledged alerts\n", - " unack_count = df_all_alerts.filter(F.col(\"acknowledged\") == False).count()\n", - " print(f\"\\nšŸ”ø Unacknowledged alerts: {unack_count}\")\n", - " \n", - " # Recent alerts (last 7 days)\n", - " recent_cutoff = datetime.now() - timedelta(days=7)\n", - " recent_count = df_all_alerts.filter(F.col(\"alert_date\") >= recent_cutoff).count()\n", - " print(f\"šŸ”ø Alerts in last 7 days: {recent_count}\")\n", - " \n", - " # Critical alerts\n", - " critical_count = df_all_alerts.filter(F.col(\"severity\") == SEVERITY_CRITICAL).count()\n", - " print(f\"šŸ”ø Critical alerts: {critical_count}\")\n", - " \n", - " print(\"\\n\" + \"=\" * 60)\n", - "\n", - "print(\"āœ… Transfer Feature Alerts Unit - COMPLETED\")" - ] - }, - { - "cell_type": "markdown", - "id": "f0650ea7", - "metadata": {}, - "source": [ - "## Step 7: Export Alerts for Data Activator (Optional)\n", - "\n", - "This creates a view that Data Activator can monitor" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "67823959", - "metadata": {}, - "outputs": [], - "source": [ - "from delta.tables import DeltaTable\n", - "\n", - "if DeltaTable.isDeltaTable(spark, table_path):\n", - " print(\"\\nšŸ”„ Creating view for Data Activator integration...\")\n", - " \n", - " df_activator = spark.read.format(\"delta\").load(table_path)\n", - " \n", - " # Filter to unacknowledged critical/warning alerts\n", - " df_activator_filtered = df_activator.filter(\n", - " (F.col(\"acknowledged\") == False) & \n", - " (F.col(\"severity\").isin([SEVERITY_CRITICAL, SEVERITY_WARNING]))\n", - " )\n", - " \n", - " # Create or replace temp view\n", - " df_activator_filtered.createOrReplaceTempView(\"vw_feature_alerts_active\")\n", - " \n", - " alert_count = df_activator_filtered.count()\n", - " print(f\"āœ… Created view 'vw_feature_alerts_active' with {alert_count} active alerts\")\n", - " print(\" ā†’ This view can be monitored by Data Activator for real-time notifications\")" - ] - } - ], - "metadata": { - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/monitoring/fabric-unified-admin-monitoring/src/Load_Feature_Tracking_E2E.DataPipeline/pipeline-content.json b/monitoring/fabric-unified-admin-monitoring/src/Load_Feature_Tracking_E2E.DataPipeline/pipeline-content.json deleted file mode 100644 index a7607255..00000000 --- a/monitoring/fabric-unified-admin-monitoring/src/Load_Feature_Tracking_E2E.DataPipeline/pipeline-content.json +++ /dev/null @@ -1,114 +0,0 @@ -{ - "properties": { - "activities": [ - { - "name": "01_Transfer_Feature_Releases_Unit", - "type": "TridentNotebook", - "dependsOn": [], - "policy": { - "timeout": "0.12:00:00", - "retry": 0, - "retryIntervalInSeconds": 30, - "secureOutput": false, - "secureInput": false - }, - "typeProperties": { - "notebookId": "REPLACE_WITH_NOTEBOOK_ID", - "workspaceId": "REPLACE_WITH_WORKSPACE_ID", - "parameters": { - "whats_new_url": { - "value": { - "value": "@pipeline().parameters.whats_new_url", - "type": "Expression" - }, - "type": "string" - }, - "lookback_days": { - "value": { - "value": "@pipeline().parameters.lookback_days", - "type": "Expression" - }, - "type": "int" - } - }, - "sessionTag": "fuam_feature_tracking" - } - }, - { - "name": "02_Transfer_Preview_Features_Unit", - "type": "TridentNotebook", - "dependsOn": [ - { - "activity": "01_Transfer_Feature_Releases_Unit", - "dependencyConditions": [ - "Succeeded" - ] - } - ], - "policy": { - "timeout": "0.12:00:00", - "retry": 0, - "retryIntervalInSeconds": 30, - "secureOutput": false, - "secureInput": false - }, - "typeProperties": { - "notebookId": "REPLACE_WITH_NOTEBOOK_ID", - "workspaceId": "REPLACE_WITH_WORKSPACE_ID", - "parameters": {}, - "sessionTag": "fuam_feature_tracking" - } - }, - { - "name": "03_Transfer_Feature_Alerts_Unit", - "type": "TridentNotebook", - "dependsOn": [ - { - "activity": "02_Transfer_Preview_Features_Unit", - "dependencyConditions": [ - "Succeeded" - ] - } - ], - "policy": { - "timeout": "0.12:00:00", - "retry": 0, - "retryIntervalInSeconds": 30, - "secureOutput": false, - "secureInput": false - }, - "typeProperties": { - "notebookId": "REPLACE_WITH_NOTEBOOK_ID", - "workspaceId": "REPLACE_WITH_WORKSPACE_ID", - "parameters": { - "alert_days_threshold": { - "value": { - "value": "@pipeline().parameters.alert_days_threshold", - "type": "Expression" - }, - "type": "int" - } - }, - "sessionTag": "fuam_feature_tracking" - } - } - ], - "parameters": { - "whats_new_url": { - "type": "String", - "defaultValue": "https://learn.microsoft.com/en-us/fabric/get-started/whats-new" - }, - "lookback_days": { - "type": "Int", - "defaultValue": 90 - }, - "alert_days_threshold": { - "type": "Int", - "defaultValue": 90 - } - }, - "annotations": [], - "lastPublishTime": "2025-10-31T00:00:00Z" - }, - "name": "Load_Feature_Tracking_E2E" -} diff --git a/monitoring/fabric-unified-admin-monitoring/src/Setup_Feature_Tracking_Tables.Notebook/.platform b/monitoring/fabric-unified-admin-monitoring/src/Setup_Feature_Tracking_Tables.Notebook/.platform deleted file mode 100644 index 658ddafd..00000000 --- a/monitoring/fabric-unified-admin-monitoring/src/Setup_Feature_Tracking_Tables.Notebook/.platform +++ /dev/null @@ -1,12 +0,0 @@ -{ - "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", - "metadata": { - "type": "Notebook", - "displayName": "Setup_Feature_Tracking_Tables", - "description": "One-time setup to create Delta tables for Feature Release Tracking" - }, - "config": { - "version": "2.0", - "logicalId": "00000000-0000-0000-0000-000000000000" - } -} diff --git a/monitoring/fabric-unified-admin-monitoring/src/Setup_Feature_Tracking_Tables.Notebook/notebook-content.ipynb b/monitoring/fabric-unified-admin-monitoring/src/Setup_Feature_Tracking_Tables.Notebook/notebook-content.ipynb deleted file mode 100644 index 8b806c90..00000000 --- a/monitoring/fabric-unified-admin-monitoring/src/Setup_Feature_Tracking_Tables.Notebook/notebook-content.ipynb +++ /dev/null @@ -1,348 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "63d1ec5e", - "metadata": {}, - "source": [ - "# Setup Feature Tracking Tables\n", - "\n", - "**Purpose**: One-time setup to create Delta tables for Feature Release Tracking\n", - "\n", - "**Tables Created**:\n", - "1. `feature_releases` - Microsoft Fabric feature releases from What's New\n", - "2. `preview_features_active` - Preview features currently activated in tenant\n", - "3. `feature_alerts` - Alerts for preview feature activations\n", - "\n", - "**Run Once**: This notebook only needs to be executed once during initial setup\n", - "\n", - "**Prerequisites**: \n", - "- FUAM Lakehouse must exist\n", - "- `tenant_settings` table should already be created by FUAM" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c0bef748", - "metadata": {}, - "outputs": [], - "source": [ - "from pyspark.sql.types import *\n", - "from pyspark.sql import functions as F\n", - "from datetime import datetime" - ] - }, - { - "cell_type": "markdown", - "id": "217b4ddc", - "metadata": {}, - "source": [ - "## Step 1: Create `feature_releases` Table" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c8d499ac", - "metadata": {}, - "outputs": [], - "source": [ - "print(\"šŸ”„ Creating table: feature_releases\")\n", - "print(\"=\" * 60)\n", - "\n", - "schema_feature_releases = StructType([\n", - " StructField(\"feature_id\", StringType(), False),\n", - " StructField(\"feature_name\", StringType(), False),\n", - " StructField(\"workload\", StringType(), True),\n", - " StructField(\"release_date\", TimestampType(), False),\n", - " StructField(\"status\", StringType(), True),\n", - " StructField(\"is_preview\", BooleanType(), False),\n", - " StructField(\"source_url\", StringType(), True),\n", - " StructField(\"extracted_date\", TimestampType(), False)\n", - "])\n", - "\n", - "# Create empty DataFrame\n", - "df_feature_releases = spark.createDataFrame([], schema_feature_releases)\n", - "\n", - "# Write to Delta\n", - "table_path = \"Tables/feature_releases\"\n", - "df_feature_releases.write.format(\"delta\").mode(\"overwrite\").save(table_path)\n", - "\n", - "print(\"āœ… Table created: feature_releases\")\n", - "print(f\" Location: {table_path}\")\n", - "print(\"\\n Schema:\")\n", - "for field in schema_feature_releases.fields:\n", - " print(f\" - {field.name}: {field.dataType.simpleString()}\")" - ] - }, - { - "cell_type": "markdown", - "id": "3352b549", - "metadata": {}, - "source": [ - "## Step 2: Create `preview_features_active` Table" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8603d07d", - "metadata": {}, - "outputs": [], - "source": [ - "print(\"\\nšŸ”„ Creating table: preview_features_active\")\n", - "print(\"=\" * 60)\n", - "\n", - "schema_preview_active = StructType([\n", - " StructField(\"setting_name\", StringType(), False),\n", - " StructField(\"feature_id\", StringType(), False),\n", - " StructField(\"feature_name\", StringType(), False),\n", - " StructField(\"workload\", StringType(), True),\n", - " StructField(\"similarity_score\", DoubleType(), False),\n", - " StructField(\"is_enabled\", BooleanType(), False),\n", - " StructField(\"delegate_to_tenant\", BooleanType(), True),\n", - " StructField(\"detected_date\", TimestampType(), False),\n", - " StructField(\"release_date\", TimestampType(), True),\n", - " StructField(\"status\", StringType(), True),\n", - " StructField(\"source_url\", StringType(), True),\n", - " StructField(\"days_since_release\", IntegerType(), True)\n", - "])\n", - "\n", - "# Create empty DataFrame\n", - "df_preview_active = spark.createDataFrame([], schema_preview_active)\n", - "\n", - "# Write to Delta\n", - "table_path = \"Tables/preview_features_active\"\n", - "df_preview_active.write.format(\"delta\").mode(\"overwrite\").save(table_path)\n", - "\n", - "print(\"āœ… Table created: preview_features_active\")\n", - "print(f\" Location: {table_path}\")\n", - "print(\"\\n Schema:\")\n", - "for field in schema_preview_active.fields:\n", - " print(f\" - {field.name}: {field.dataType.simpleString()}\")" - ] - }, - { - "cell_type": "markdown", - "id": "ca05db54", - "metadata": {}, - "source": [ - "## Step 3: Create `feature_alerts` Table" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "21331f82", - "metadata": {}, - "outputs": [], - "source": [ - "print(\"\\nšŸ”„ Creating table: feature_alerts\")\n", - "print(\"=\" * 60)\n", - "\n", - "schema_alerts = StructType([\n", - " StructField(\"alert_id\", StringType(), False),\n", - " StructField(\"feature_id\", StringType(), False),\n", - " StructField(\"feature_name\", StringType(), False),\n", - " StructField(\"workload\", StringType(), True),\n", - " StructField(\"alert_type\", StringType(), False),\n", - " StructField(\"severity\", StringType(), False),\n", - " StructField(\"message\", StringType(), False),\n", - " StructField(\"days_since_release\", IntegerType(), True),\n", - " StructField(\"similarity_score\", DoubleType(), True),\n", - " StructField(\"alert_date\", TimestampType(), False),\n", - " StructField(\"acknowledged\", BooleanType(), False)\n", - "])\n", - "\n", - "# Create empty DataFrame\n", - "df_alerts = spark.createDataFrame([], schema_alerts)\n", - "\n", - "# Write to Delta\n", - "table_path = \"Tables/feature_alerts\"\n", - "df_alerts.write.format(\"delta\").mode(\"overwrite\").save(table_path)\n", - "\n", - "print(\"āœ… Table created: feature_alerts\")\n", - "print(f\" Location: {table_path}\")\n", - "print(\"\\n Schema:\")\n", - "for field in schema_alerts.fields:\n", - " print(f\" - {field.name}: {field.dataType.simpleString()}\")" - ] - }, - { - "cell_type": "markdown", - "id": "5b0f748b", - "metadata": {}, - "source": [ - "## Step 4: Verify Tables Created" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "3396868e", - "metadata": {}, - "outputs": [], - "source": [ - "print(\"\\nšŸ“Š Verification Summary\")\n", - "print(\"=\" * 60)\n", - "\n", - "tables_to_verify = [\n", - " \"feature_releases\",\n", - " \"preview_features_active\", \n", - " \"feature_alerts\"\n", - "]\n", - "\n", - "all_tables_exist = True\n", - "\n", - "for table_name in tables_to_verify:\n", - " table_path = f\"Tables/{table_name}\"\n", - " try:\n", - " from delta.tables import DeltaTable\n", - " \n", - " if DeltaTable.isDeltaTable(spark, table_path):\n", - " df = spark.read.format(\"delta\").load(table_path)\n", - " count = df.count()\n", - " print(f\"āœ… {table_name}: EXISTS (rows={count})\")\n", - " else:\n", - " print(f\"āŒ {table_name}: NOT A DELTA TABLE\")\n", - " all_tables_exist = False\n", - " except Exception as e:\n", - " print(f\"āŒ {table_name}: ERROR - {e}\")\n", - " all_tables_exist = False\n", - "\n", - "print(\"=\" * 60)\n", - "\n", - "if all_tables_exist:\n", - " print(\"\\nšŸŽ‰ SUCCESS! All tables created successfully\")\n", - " print(\"\\nNext Steps:\")\n", - " print(\"1. Run '01_Transfer_Feature_Releases_Unit' to populate feature_releases\")\n", - " print(\"2. Run '02_Transfer_Preview_Features_Unit' to detect activated previews\")\n", - " print(\"3. Run '03_Transfer_Feature_Alerts_Unit' to generate alerts\")\n", - " print(\"\\nOR\")\n", - " print(\"ā†’ Run 'Load_Feature_Tracking_E2E' pipeline to execute all steps\")\n", - "else:\n", - " print(\"\\nāš ļø WARNING: Some tables failed to create. Review errors above.\")" - ] - }, - { - "cell_type": "markdown", - "id": "77be40c2", - "metadata": {}, - "source": [ - "## Step 5: Create Sample Queries View (Optional)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2810793e", - "metadata": {}, - "outputs": [], - "source": [ - "print(\"\\nšŸ”„ Creating helper views for SQL Endpoint...\")\n", - "\n", - "# View 1: Active Preview Features (for quick querying)\n", - "spark.sql(\"\"\"\n", - " CREATE OR REPLACE VIEW vw_active_preview_features AS\n", - " SELECT \n", - " feature_name,\n", - " workload,\n", - " setting_name,\n", - " days_since_release,\n", - " similarity_score,\n", - " release_date,\n", - " detected_date\n", - " FROM preview_features_active\n", - " WHERE is_enabled = true\n", - " ORDER BY detected_date DESC\n", - "\"\"\")\n", - "print(\"āœ… Created view: vw_active_preview_features\")\n", - "\n", - "# View 2: Unacknowledged Critical Alerts\n", - "spark.sql(\"\"\"\n", - " CREATE OR REPLACE VIEW vw_critical_alerts AS\n", - " SELECT \n", - " alert_id,\n", - " feature_name,\n", - " workload,\n", - " alert_type,\n", - " severity,\n", - " message,\n", - " alert_date\n", - " FROM feature_alerts\n", - " WHERE acknowledged = false \n", - " AND severity IN ('Critical', 'Warning')\n", - " ORDER BY alert_date DESC\n", - "\"\"\")\n", - "print(\"āœ… Created view: vw_critical_alerts\")\n", - "\n", - "# View 3: Feature Release Timeline\n", - "spark.sql(\"\"\"\n", - " CREATE OR REPLACE VIEW vw_feature_timeline AS\n", - " SELECT \n", - " feature_name,\n", - " workload,\n", - " status,\n", - " is_preview,\n", - " release_date,\n", - " DATEDIFF(CURRENT_DATE(), release_date) as days_since_release\n", - " FROM feature_releases\n", - " ORDER BY release_date DESC\n", - "\"\"\")\n", - "print(\"āœ… Created view: vw_feature_timeline\")\n", - "\n", - "print(\"\\nāœ… All views created successfully\")\n", - "print(\" ā†’ These views are accessible via SQL Endpoint\")" - ] - }, - { - "cell_type": "markdown", - "id": "1630563f", - "metadata": {}, - "source": [ - "## āœ… Setup Complete!" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b96f8877", - "metadata": {}, - "outputs": [], - "source": [ - "print(\"\\n\" + \"=\" * 60)\n", - "print(\"šŸŽ‰ FEATURE TRACKING SETUP COMPLETED!\")\n", - "print(\"=\" * 60)\n", - "\n", - "print(\"\\nšŸ“‹ Tables Created:\")\n", - "print(\" āœ… feature_releases\")\n", - "print(\" āœ… preview_features_active\")\n", - "print(\" āœ… feature_alerts\")\n", - "\n", - "print(\"\\nšŸ“‹ Views Created:\")\n", - "print(\" āœ… vw_active_preview_features\")\n", - "print(\" āœ… vw_critical_alerts\")\n", - "print(\" āœ… vw_feature_timeline\")\n", - "\n", - "print(\"\\nšŸš€ Ready to Run:\")\n", - "print(\" ā†’ Execute 'Load_Feature_Tracking_E2E' pipeline\")\n", - "print(\" ā†’ Or run individual Unit notebooks\")\n", - "\n", - "print(\"\\nšŸ“Š Query Examples:\")\n", - "print(\" SELECT * FROM vw_active_preview_features;\")\n", - "print(\" SELECT * FROM vw_critical_alerts;\")\n", - "print(\" SELECT * FROM vw_feature_timeline WHERE is_preview = true;\")\n", - "\n", - "print(\"\\n\" + \"=\" * 60)" - ] - } - ], - "metadata": { - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/monitoring/fabric-unified-admin-monitoring/src/Setup_Feature_Tracking_Tables_GpsApi.Notebook/notebook-content.ipynb b/monitoring/fabric-unified-admin-monitoring/src/Setup_Feature_Tracking_Tables_GpsApi.Notebook/notebook-content.ipynb deleted file mode 100644 index 1c4d7511..00000000 --- a/monitoring/fabric-unified-admin-monitoring/src/Setup_Feature_Tracking_Tables_GpsApi.Notebook/notebook-content.ipynb +++ /dev/null @@ -1,382 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "id": "33e13348", - "metadata": {}, - "outputs": [], - "source": [ - "from pyspark.sql.types import *\n", - "from datetime import datetime" - ] - }, - { - "cell_type": "markdown", - "id": "7d714890", - "metadata": {}, - "source": [ - "## Step 1: Create `feature_releases` Table (Original)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "97b8567b", - "metadata": {}, - "outputs": [], - "source": [ - "print(\"šŸ”„ Creating table: feature_releases\")\n", - "print(\"=\" * 60)\n", - "\n", - "schema_feature_releases = StructType([\n", - " StructField(\"feature_id\", StringType(), False),\n", - " StructField(\"feature_name\", StringType(), False),\n", - " StructField(\"workload\", StringType(), True),\n", - " StructField(\"release_date\", TimestampType(), False),\n", - " StructField(\"status\", StringType(), True),\n", - " StructField(\"is_preview\", BooleanType(), False),\n", - " StructField(\"source_url\", StringType(), True),\n", - " StructField(\"extracted_date\", TimestampType(), False)\n", - "])\n", - "\n", - "# Create empty DataFrame\n", - "df_feature_releases = spark.createDataFrame([], schema_feature_releases)\n", - "\n", - "# Write to Delta\n", - "table_path = \"Tables/feature_releases\"\n", - "df_feature_releases.write.format(\"delta\").mode(\"overwrite\").save(table_path)\n", - "\n", - "print(\"āœ… Table created: feature_releases\")\n", - "print(f\" Location: {table_path}\")\n", - "print(\"\\n Schema:\")\n", - "for field in schema_feature_releases.fields:\n", - " print(f\" - {field.name}: {field.dataType.simpleString()}\")" - ] - }, - { - "cell_type": "markdown", - "id": "f282f309", - "metadata": {}, - "source": [ - "## Step 1b: Create `feature_releases_roadmap` Table (Enhanced)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "560e3ede", - "metadata": {}, - "outputs": [], - "source": [ - "print(\"\\nšŸ”„ Creating table: feature_releases_roadmap (Fabric GPS)\")\n", - "print(\"=\" * 60)\n", - "\n", - "schema_roadmap = StructType([\n", - " StructField(\"feature_id\", StringType(), False),\n", - " StructField(\"feature_name\", StringType(), False),\n", - " StructField(\"feature_description\", StringType(), True),\n", - " StructField(\"workload\", StringType(), True),\n", - " StructField(\"product_name\", StringType(), True),\n", - " StructField(\"release_date\", TimestampType(), True), # Nullable for planned features\n", - " StructField(\"release_type\", StringType(), True),\n", - " StructField(\"release_status\", StringType(), True),\n", - " StructField(\"is_preview\", BooleanType(), False),\n", - " StructField(\"is_planned\", BooleanType(), False),\n", - " StructField(\"is_shipped\", BooleanType(), False),\n", - " StructField(\"last_modified\", TimestampType(), False),\n", - " StructField(\"source_url\", StringType(), True),\n", - " StructField(\"source\", StringType(), True),\n", - " StructField(\"extracted_date\", TimestampType(), False)\n", - "])\n", - "\n", - "# Create empty DataFrame\n", - "df_roadmap = spark.createDataFrame([], schema_roadmap)\n", - "\n", - "# Write to Delta\n", - "table_path = \"Tables/feature_releases_roadmap\"\n", - "df_roadmap.write.format(\"delta\").mode(\"overwrite\").save(table_path)\n", - "\n", - "print(\"āœ… Table created: feature_releases_roadmap\")\n", - "print(f\" Location: {table_path}\")\n", - "print(\"\\n Schema:\")\n", - "for field in schema_roadmap.fields:\n", - " print(f\" - {field.name}: {field.dataType.simpleString()}\")\n", - "print(\"\\n šŸ’” This table includes:\")\n", - "print(\" - Complete feature descriptions\")\n", - "print(\" - Planned/future features (roadmap)\")\n", - "print(\" - Historical change tracking (last_modified)\")\n", - "print(\" - Release status (Planned, In Development, Shipped)\")" - ] - }, - { - "cell_type": "markdown", - "id": "c1c72b90", - "metadata": {}, - "source": [ - "## Step 2: Create `preview_features_active` Table" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "399f3d8f", - "metadata": {}, - "outputs": [], - "source": [ - "print(\"\\nšŸ”„ Creating table: preview_features_active\")\n", - "print(\"=\" * 60)\n", - "\n", - "schema_preview_active = StructType([\n", - " StructField(\"setting_name\", StringType(), False),\n", - " StructField(\"feature_id\", StringType(), False),\n", - " StructField(\"feature_name\", StringType(), False),\n", - " StructField(\"workload\", StringType(), True),\n", - " StructField(\"similarity_score\", DoubleType(), False),\n", - " StructField(\"is_enabled\", BooleanType(), False),\n", - " StructField(\"delegate_to_tenant\", BooleanType(), True),\n", - " StructField(\"detected_date\", TimestampType(), False),\n", - " StructField(\"release_date\", TimestampType(), True),\n", - " StructField(\"status\", StringType(), True),\n", - " StructField(\"source_url\", StringType(), True),\n", - " StructField(\"days_since_release\", IntegerType(), True)\n", - "])\n", - "\n", - "# Create empty DataFrame\n", - "df_preview_active = spark.createDataFrame([], schema_preview_active)\n", - "\n", - "# Write to Delta\n", - "table_path = \"Tables/preview_features_active\"\n", - "df_preview_active.write.format(\"delta\").mode(\"overwrite\").save(table_path)\n", - "\n", - "print(\"āœ… Table created: preview_features_active\")\n", - "print(f\" Location: {table_path}\")\n", - "print(\"\\n Schema:\")\n", - "for field in schema_preview_active.fields:\n", - " print(f\" - {field.name}: {field.dataType.simpleString()}\")" - ] - }, - { - "cell_type": "markdown", - "id": "541c869f", - "metadata": {}, - "source": [ - "## Step 3: Create `feature_alerts` Table" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1505fe69", - "metadata": {}, - "outputs": [], - "source": [ - "print(\"\\nšŸ”„ Creating table: feature_alerts\")\n", - "print(\"=\" * 60)\n", - "\n", - "schema_alerts = StructType([\n", - " StructField(\"alert_id\", StringType(), False),\n", - " StructField(\"feature_id\", StringType(), False),\n", - " StructField(\"feature_name\", StringType(), False),\n", - " StructField(\"workload\", StringType(), True),\n", - " StructField(\"alert_type\", StringType(), False),\n", - " StructField(\"severity\", StringType(), False),\n", - " StructField(\"message\", StringType(), False),\n", - " StructField(\"setting_name\", StringType(), True),\n", - " StructField(\"similarity_score\", DoubleType(), True),\n", - " StructField(\"days_since_release\", IntegerType(), True),\n", - " StructField(\"alert_date\", TimestampType(), False),\n", - " StructField(\"acknowledged\", BooleanType(), False),\n", - " StructField(\"acknowledged_date\", TimestampType(), True),\n", - " StructField(\"acknowledged_by\", StringType(), True)\n", - "])\n", - "\n", - "# Create empty DataFrame\n", - "df_alerts = spark.createDataFrame([], schema_alerts)\n", - "\n", - "# Write to Delta\n", - "table_path = \"Tables/feature_alerts\"\n", - "df_alerts.write.format(\"delta\").mode(\"overwrite\").save(table_path)\n", - "\n", - "print(\"āœ… Table created: feature_alerts\")\n", - "print(f\" Location: {table_path}\")\n", - "print(\"\\n Schema:\")\n", - "for field in schema_alerts.fields:\n", - " print(f\" - {field.name}: {field.dataType.simpleString()}\")" - ] - }, - { - "cell_type": "markdown", - "id": "8cb9af32", - "metadata": {}, - "source": [ - "## Step 4: Create Helper Views" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d03a41fb", - "metadata": {}, - "outputs": [], - "source": [ - "print(\"\\nšŸ”„ Creating helper views for SQL Endpoint...\")\n", - "\n", - "# View 1: Active Preview Features (for quick querying)\n", - "spark.sql(\"\"\"\n", - " CREATE OR REPLACE VIEW vw_active_preview_features AS\n", - " SELECT \n", - " feature_name,\n", - " workload,\n", - " setting_name,\n", - " days_since_release,\n", - " similarity_score,\n", - " release_date,\n", - " detected_date\n", - " FROM preview_features_active\n", - " WHERE is_enabled = true\n", - " ORDER BY detected_date DESC\n", - "\"\"\")\n", - "print(\"āœ… Created view: vw_active_preview_features\")\n", - "\n", - "# View 2: Unacknowledged Critical Alerts\n", - "spark.sql(\"\"\"\n", - " CREATE OR REPLACE VIEW vw_critical_alerts AS\n", - " SELECT \n", - " alert_id,\n", - " feature_name,\n", - " workload,\n", - " alert_type,\n", - " severity,\n", - " message,\n", - " alert_date\n", - " FROM feature_alerts\n", - " WHERE acknowledged = false \n", - " AND severity IN ('Critical', 'Warning')\n", - " ORDER BY alert_date DESC\n", - "\"\"\")\n", - "print(\"āœ… Created view: vw_critical_alerts\")\n", - "\n", - "# View 3: Feature Release Timeline\n", - "spark.sql(\"\"\"\n", - " CREATE OR REPLACE VIEW vw_feature_timeline AS\n", - " SELECT \n", - " feature_name,\n", - " workload,\n", - " status,\n", - " is_preview,\n", - " release_date,\n", - " DATEDIFF(CURRENT_DATE(), release_date) as days_since_release\n", - " FROM feature_releases\n", - " ORDER BY release_date DESC\n", - "\"\"\")\n", - "print(\"āœ… Created view: vw_feature_timeline\")\n", - "\n", - "# View 4: Roadmap Upcoming Features (NEW)\n", - "spark.sql(\"\"\"\n", - " CREATE OR REPLACE VIEW vw_roadmap_upcoming AS\n", - " SELECT \n", - " feature_name,\n", - " feature_description,\n", - " product_name,\n", - " workload,\n", - " release_type,\n", - " release_status,\n", - " release_date,\n", - " is_preview,\n", - " is_planned,\n", - " last_modified,\n", - " CASE \n", - " WHEN release_date IS NULL THEN NULL\n", - " ELSE DATEDIFF(release_date, CURRENT_DATE())\n", - " END as days_until_release\n", - " FROM feature_releases_roadmap\n", - " WHERE is_planned = true\n", - " AND (release_date IS NULL OR release_date >= CURRENT_DATE())\n", - " ORDER BY release_date ASC NULLS LAST, last_modified DESC\n", - "\"\"\")\n", - "print(\"āœ… Created view: vw_roadmap_upcoming\")\n", - "\n", - "print(\"\\nāœ… All views created successfully\")\n", - "print(\" ā†’ These views are accessible via SQL Endpoint\")" - ] - }, - { - "cell_type": "markdown", - "id": "c2b37e4e", - "metadata": {}, - "source": [ - "## āœ… Setup Complete!" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d0659717", - "metadata": {}, - "outputs": [], - "source": [ - "print(\"\\n\" + \"=\" * 60)\n", - "print(\"šŸŽ‰ FEATURE TRACKING SETUP COMPLETED!\")\n", - "print(\"=\" * 60)\n", - "\n", - "# Verify all tables exist\n", - "tables = [\n", - " \"feature_releases\",\n", - " \"feature_releases_roadmap\", \n", - " \"preview_features_active\", \n", - " \"feature_alerts\"\n", - "]\n", - "\n", - "print(\"\\nšŸ“‹ Verifying tables...\")\n", - "for table in tables:\n", - " try:\n", - " count = spark.read.format(\"delta\").load(f\"Tables/{table}\").count()\n", - " print(f\" āœ… {table}: {count} rows\")\n", - " except:\n", - " print(f\" āŒ {table}: ERROR\")\n", - "\n", - "# Verify views\n", - "views = [\n", - " \"vw_active_preview_features\",\n", - " \"vw_critical_alerts\",\n", - " \"vw_feature_timeline\",\n", - " \"vw_roadmap_upcoming\"\n", - "]\n", - "\n", - "print(\"\\nšŸ“‹ Verifying views...\")\n", - "for view in views:\n", - " try:\n", - " spark.sql(f\"SELECT * FROM {view} LIMIT 1\")\n", - " print(f\" āœ… {view}\")\n", - " except:\n", - " print(f\" āŒ {view}: ERROR\")\n", - "\n", - "print(\"\\n\" + \"=\" * 60)\n", - "print(\"šŸ“š Next Steps:\")\n", - "print(\"=\" * 60)\n", - "print(\"\\n1. Choose your data source:\")\n", - "print(\" a) Microsoft Learn (original):\")\n", - "print(\" ā†’ Run '01_Transfer_Feature_Releases_Unit'\")\n", - "print(\" b) Fabric GPS API (enhanced with roadmap):\")\n", - " \"print(\\\" ā†’ Run '01_Transfer_Feature_Releases_GpsApi_Unit'\\\")\\n\",\n", - "print(\"\\n2. Run '02_Transfer_Preview_Features_Unit' to detect activated previews\")\n", - "print(\" Note: This requires 'tenant_settings' table from FUAM core\")\n", - "print(\"\\n3. Run '03_Transfer_Feature_Alerts_Unit' to generate alerts\")\n", - "print(\"\\n4. OR run the full pipeline: 'Load_Feature_Tracking_E2E'\")\n", - "print(\"\\nšŸ’” Recommended:\")\n", - "print(\" - Use Enhanced version (Fabric GPS) for complete roadmap visibility\")\n", - "print(\" - Run both versions if you want dual data sources\")\n", - "print(\" - Schedule pipeline to run daily\")\n", - "print(\"\\n\" + \"=\" * 60)" - ] - } - ], - "metadata": { - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} From c0b763b2204115790801041a45f566f39fa50a3c Mon Sep 17 00:00:00 2001 From: Ayoub KEBAILI Date: Fri, 21 Nov 2025 15:49:42 -0500 Subject: [PATCH 4/4] Add Load Feature Tracking Notebook and Data Pipeline - Created a new notebook for the Load Feature Tracking process, which includes fetching releases, detecting previews, and generating alerts. - Implemented the necessary code to transform and write feature release data to Delta tables. - Added a new Data Pipeline to orchestrate the execution of the Load Feature Tracking notebook. - Configured the pipeline with appropriate parameters and dependencies for seamless execution. --- .../config/deployment_order.json | 8 +- .../.platform | 0 .../notebook-content.ipynb | 147 ++++++++++-------- .../.platform | 0 .../notebook-content.ipynb | 13 +- .../.platform | 12 ++ .../pipeline-content.json | 26 ++++ 7 files changed, 133 insertions(+), 73 deletions(-) rename monitoring/fabric-unified-admin-monitoring/src/{Feature_Tracking_Setup.Notebook => 01_Setup_Feature_Tracking.Notebook}/.platform (100%) rename monitoring/fabric-unified-admin-monitoring/src/{Feature_Tracking_Setup.Notebook => 01_Setup_Feature_Tracking.Notebook}/notebook-content.ipynb (63%) rename monitoring/fabric-unified-admin-monitoring/src/{Load_Feature_Tracking.Notebook => 02_Load_Feature_Tracking.Notebook}/.platform (100%) rename monitoring/fabric-unified-admin-monitoring/src/{Load_Feature_Tracking.Notebook => 02_Load_Feature_Tracking.Notebook}/notebook-content.ipynb (98%) create mode 100644 monitoring/fabric-unified-admin-monitoring/src/Load_Feature_Tracking_E2E.DataPipeline/.platform create mode 100644 monitoring/fabric-unified-admin-monitoring/src/Load_Feature_Tracking_E2E.DataPipeline/pipeline-content.json diff --git a/monitoring/fabric-unified-admin-monitoring/config/deployment_order.json b/monitoring/fabric-unified-admin-monitoring/config/deployment_order.json index e6944de7..970f84df 100644 --- a/monitoring/fabric-unified-admin-monitoring/config/deployment_order.json +++ b/monitoring/fabric-unified-admin-monitoring/config/deployment_order.json @@ -192,11 +192,15 @@ "fuam_id": "3695cd0e-da7e-3b40-ad28-5bd1bcd33eb6" }, { - "name": "Feature_Tracking_Setup.Notebook", + "name": "01_Setup_Feature_Tracking.Notebook", "fuam_id": "f8a2b1c3-4d5e-6f7a-8b9c-0d1e2f3a4b5c" }, { - "name": "Load_Feature_Tracking.Notebook", + "name": "02_Load_Feature_Tracking.Notebook", "fuam_id": "a1b2c3d4-5e6f-7a8b-9c0d-1e2f3a4b5c6d" + }, + { + "name": "Load_Feature_Tracking_E2E.DataPipeline", + "fuam_id": "b2c3d4e5-6f7a-8b9c-0d1e-2f3a4b5c6d7e" } ] \ No newline at end of file diff --git a/monitoring/fabric-unified-admin-monitoring/src/Feature_Tracking_Setup.Notebook/.platform b/monitoring/fabric-unified-admin-monitoring/src/01_Setup_Feature_Tracking.Notebook/.platform similarity index 100% rename from monitoring/fabric-unified-admin-monitoring/src/Feature_Tracking_Setup.Notebook/.platform rename to monitoring/fabric-unified-admin-monitoring/src/01_Setup_Feature_Tracking.Notebook/.platform diff --git a/monitoring/fabric-unified-admin-monitoring/src/Feature_Tracking_Setup.Notebook/notebook-content.ipynb b/monitoring/fabric-unified-admin-monitoring/src/01_Setup_Feature_Tracking.Notebook/notebook-content.ipynb similarity index 63% rename from monitoring/fabric-unified-admin-monitoring/src/Feature_Tracking_Setup.Notebook/notebook-content.ipynb rename to monitoring/fabric-unified-admin-monitoring/src/01_Setup_Feature_Tracking.Notebook/notebook-content.ipynb index 89f98915..aebaaeb1 100644 --- a/monitoring/fabric-unified-admin-monitoring/src/Feature_Tracking_Setup.Notebook/notebook-content.ipynb +++ b/monitoring/fabric-unified-admin-monitoring/src/01_Setup_Feature_Tracking.Notebook/notebook-content.ipynb @@ -1,14 +1,30 @@ { "cells": [ + { + "cell_type": "markdown", + "id": "1a0fcaaa", + "metadata": {}, + "source": [ + "# Feature Tracking - Setup Tables and Views\n", + "\n", + "**Purpose**: One-time setup for feature tracking tables and views\n", + "\n", + "**What this creates**:\n", + "- āœ… `feature_releases_roadmap` - Feature releases from Fabric GPS API (with roadmap)\n", + "- āœ… `preview_features_active` - Detected activated preview features\n", + "- āœ… `feature_alerts` - Alerts for new/risky preview features\n", + "- āœ… Helper SQL views for easy querying" + ] + }, { "cell_type": "code", "execution_count": null, - "id": "c7fd3012", + "id": "c5c91aa2", "metadata": {}, "outputs": [], "source": [ - "from pyspark.sql.types import *\n", - "from datetime import datetime" + "from datetime import datetime\n", + "import pandas as pd" ] }, { @@ -29,31 +45,30 @@ "print(\"šŸ”„ Creating table: feature_releases_roadmap\")\n", "print(\"=\" * 70)\n", "\n", - "schema_roadmap = StructType([\n", - " StructField(\"feature_id\", StringType(), False),\n", - " StructField(\"feature_name\", StringType(), False),\n", - " StructField(\"feature_description\", StringType(), True),\n", - " StructField(\"workload\", StringType(), True),\n", - " StructField(\"product_name\", StringType(), True),\n", - " StructField(\"release_date\", TimestampType(), True), # Nullable for planned features\n", - " StructField(\"release_type\", StringType(), True),\n", - " StructField(\"release_status\", StringType(), True),\n", - " StructField(\"is_preview\", BooleanType(), False),\n", - " StructField(\"is_planned\", BooleanType(), False),\n", - " StructField(\"is_shipped\", BooleanType(), False),\n", - " StructField(\"last_modified\", TimestampType(), False),\n", - " StructField(\"source_url\", StringType(), True),\n", - " StructField(\"source\", StringType(), True),\n", - " StructField(\"extracted_date\", TimestampType(), False)\n", - "])\n", - "\n", - "df_roadmap = spark.createDataFrame([], schema_roadmap)\n", - "table_path = \"Tables/feature_releases_roadmap\"\n", - "df_roadmap.write.format(\"delta\").mode(\"overwrite\").save(table_path)\n", + "spark.sql(\"\"\"\n", + " CREATE TABLE IF NOT EXISTS feature_releases_roadmap (\n", + " feature_id STRING NOT NULL,\n", + " feature_name STRING NOT NULL,\n", + " feature_description STRING,\n", + " workload STRING,\n", + " product_name STRING,\n", + " release_date TIMESTAMP,\n", + " release_type STRING,\n", + " release_status STRING,\n", + " is_preview BOOLEAN NOT NULL,\n", + " is_planned BOOLEAN NOT NULL,\n", + " is_shipped BOOLEAN NOT NULL,\n", + " last_modified TIMESTAMP NOT NULL,\n", + " source_url STRING,\n", + " source STRING,\n", + " extracted_date TIMESTAMP NOT NULL\n", + " )\n", + " USING DELTA\n", + "\"\"\")\n", "\n", "print(\"āœ… Table created: feature_releases_roadmap\")\n", - "print(f\" Schema: {len(schema_roadmap.fields)} columns\")\n", - "print(\"\\n šŸ’” Includes planned/future features and historical tracking\")" + "print(\" Schema: 15 columns\")\n", + "print(\" šŸ’” Includes planned/future features and historical tracking\")" ] }, { @@ -74,27 +89,26 @@ "print(\"\\nšŸ”„ Creating table: preview_features_active\")\n", "print(\"=\" * 70)\n", "\n", - "schema_preview_active = StructType([\n", - " StructField(\"setting_name\", StringType(), False),\n", - " StructField(\"feature_id\", StringType(), False),\n", - " StructField(\"feature_name\", StringType(), False),\n", - " StructField(\"workload\", StringType(), True),\n", - " StructField(\"similarity_score\", DoubleType(), False),\n", - " StructField(\"is_enabled\", BooleanType(), False),\n", - " StructField(\"delegate_to_tenant\", BooleanType(), True),\n", - " StructField(\"detected_date\", TimestampType(), False),\n", - " StructField(\"release_date\", TimestampType(), True),\n", - " StructField(\"release_status\", StringType(), True),\n", - " StructField(\"source_url\", StringType(), True),\n", - " StructField(\"days_since_release\", IntegerType(), True)\n", - "])\n", - "\n", - "df_preview_active = spark.createDataFrame([], schema_preview_active)\n", - "table_path = \"Tables/preview_features_active\"\n", - "df_preview_active.write.format(\"delta\").mode(\"overwrite\").save(table_path)\n", + "spark.sql(\"\"\"\n", + " CREATE TABLE IF NOT EXISTS preview_features_active (\n", + " setting_name STRING NOT NULL,\n", + " feature_id STRING NOT NULL,\n", + " feature_name STRING NOT NULL,\n", + " workload STRING,\n", + " similarity_score DOUBLE NOT NULL,\n", + " is_enabled BOOLEAN NOT NULL,\n", + " delegate_to_tenant BOOLEAN,\n", + " detected_date TIMESTAMP NOT NULL,\n", + " release_date TIMESTAMP,\n", + " release_status STRING,\n", + " source_url STRING,\n", + " days_since_release INT\n", + " )\n", + " USING DELTA\n", + "\"\"\")\n", "\n", "print(\"āœ… Table created: preview_features_active\")\n", - "print(f\" Schema: {len(schema_preview_active.fields)} columns\")" + "print(\" Schema: 12 columns\")" ] }, { @@ -115,29 +129,28 @@ "print(\"\\nšŸ”„ Creating table: feature_alerts\")\n", "print(\"=\" * 70)\n", "\n", - "schema_alerts = StructType([\n", - " StructField(\"alert_id\", StringType(), False),\n", - " StructField(\"feature_id\", StringType(), False),\n", - " StructField(\"feature_name\", StringType(), False),\n", - " StructField(\"workload\", StringType(), True),\n", - " StructField(\"alert_type\", StringType(), False),\n", - " StructField(\"severity\", StringType(), False),\n", - " StructField(\"message\", StringType(), False),\n", - " StructField(\"setting_name\", StringType(), True),\n", - " StructField(\"similarity_score\", DoubleType(), True),\n", - " StructField(\"days_since_release\", IntegerType(), True),\n", - " StructField(\"alert_date\", TimestampType(), False),\n", - " StructField(\"acknowledged\", BooleanType(), False),\n", - " StructField(\"acknowledged_date\", TimestampType(), True),\n", - " StructField(\"acknowledged_by\", StringType(), True)\n", - "])\n", - "\n", - "df_alerts = spark.createDataFrame([], schema_alerts)\n", - "table_path = \"Tables/feature_alerts\"\n", - "df_alerts.write.format(\"delta\").mode(\"overwrite\").save(table_path)\n", + "spark.sql(\"\"\"\n", + " CREATE TABLE IF NOT EXISTS feature_alerts (\n", + " alert_id STRING NOT NULL,\n", + " feature_id STRING NOT NULL,\n", + " feature_name STRING NOT NULL,\n", + " workload STRING,\n", + " alert_type STRING NOT NULL,\n", + " severity STRING NOT NULL,\n", + " message STRING NOT NULL,\n", + " setting_name STRING,\n", + " similarity_score DOUBLE,\n", + " days_since_release INT,\n", + " alert_date TIMESTAMP NOT NULL,\n", + " acknowledged BOOLEAN NOT NULL,\n", + " acknowledged_date TIMESTAMP,\n", + " acknowledged_by STRING\n", + " )\n", + " USING DELTA\n", + "\"\"\")\n", "\n", "print(\"āœ… Table created: feature_alerts\")\n", - "print(f\" Schema: {len(schema_alerts.fields)} columns\")" + "print(\" Schema: 14 columns\")" ] }, { @@ -276,7 +289,7 @@ "print(\"\\nšŸ“‹ Tables created:\")\n", "for table in tables:\n", " try:\n", - " count = spark.read.format(\"delta\").load(f\"Tables/{table}\").count()\n", + " count = spark.read.format(\"delta\").table(table).count()\n", " print(f\" āœ… {table}: {count} rows\")\n", " except Exception as e:\n", " print(f\" āŒ {table}: ERROR - {e}\")\n", diff --git a/monitoring/fabric-unified-admin-monitoring/src/Load_Feature_Tracking.Notebook/.platform b/monitoring/fabric-unified-admin-monitoring/src/02_Load_Feature_Tracking.Notebook/.platform similarity index 100% rename from monitoring/fabric-unified-admin-monitoring/src/Load_Feature_Tracking.Notebook/.platform rename to monitoring/fabric-unified-admin-monitoring/src/02_Load_Feature_Tracking.Notebook/.platform diff --git a/monitoring/fabric-unified-admin-monitoring/src/Load_Feature_Tracking.Notebook/notebook-content.ipynb b/monitoring/fabric-unified-admin-monitoring/src/02_Load_Feature_Tracking.Notebook/notebook-content.ipynb similarity index 98% rename from monitoring/fabric-unified-admin-monitoring/src/Load_Feature_Tracking.Notebook/notebook-content.ipynb rename to monitoring/fabric-unified-admin-monitoring/src/02_Load_Feature_Tracking.Notebook/notebook-content.ipynb index d0445624..7529966d 100644 --- a/monitoring/fabric-unified-admin-monitoring/src/Load_Feature_Tracking.Notebook/notebook-content.ipynb +++ b/monitoring/fabric-unified-admin-monitoring/src/02_Load_Feature_Tracking.Notebook/notebook-content.ipynb @@ -362,7 +362,7 @@ " \"\"\"Map tenant settings to preview features based on name similarity\"\"\"\n", " \n", " features_list = df_features.select(\"feature_id\", \"feature_name\", \"workload\", \"release_date\", \"release_status\", \"source_url\").collect()\n", - " settings_list = df_settings.select(\"settingName\", \"enabled\", \"delegateToTenant\").collect()\n", + " settings_list = df_settings.select(\"settingName\", \"enabled\", \"delegateToDomain\", \"delegateToWorkspace\", \"delegateToCapacity\").collect()\n", " \n", " matches = []\n", " \n", @@ -399,6 +399,9 @@ " if best_match[\"release_date\"]:\n", " days_since = (datetime.now() - best_match[\"release_date\"]).days\n", " \n", + " # Check if any delegation is enabled\n", + " is_delegated = setting[\"delegateToDomain\"] or setting[\"delegateToWorkspace\"] or setting[\"delegateToCapacity\"]\n", + " \n", " matches.append({\n", " \"setting_name\": setting_name,\n", " \"feature_id\": best_match[\"feature_id\"],\n", @@ -406,7 +409,7 @@ " \"workload\": best_match[\"workload\"],\n", " \"similarity_score\": best_score,\n", " \"is_enabled\": setting[\"enabled\"],\n", - " \"delegate_to_tenant\": setting[\"delegateToTenant\"],\n", + " \"delegate_to_tenant\": is_delegated,\n", " \"detected_date\": datetime.now(),\n", " \"release_date\": best_match[\"release_date\"],\n", " \"release_status\": best_match[\"release_status\"],\n", @@ -772,8 +775,10 @@ } ], "metadata": { - "language_info": { - "name": "python" + "kernelspec": { + "display_name": "PySpark", + "language": "Python", + "name": "synapse_pyspark" } }, "nbformat": 4, diff --git a/monitoring/fabric-unified-admin-monitoring/src/Load_Feature_Tracking_E2E.DataPipeline/.platform b/monitoring/fabric-unified-admin-monitoring/src/Load_Feature_Tracking_E2E.DataPipeline/.platform new file mode 100644 index 00000000..d73a2f64 --- /dev/null +++ b/monitoring/fabric-unified-admin-monitoring/src/Load_Feature_Tracking_E2E.DataPipeline/.platform @@ -0,0 +1,12 @@ +{ + "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", + "metadata": { + "type": "DataPipeline", + "displayName": "Load_Feature_Tracking_E2E", + "description": "Pipeline to load feature tracking data - fetches releases, detects previews, generates alerts" + }, + "config": { + "version": "2.0", + "logicalId": "00000000-0000-0000-0000-000000000000" + } +} diff --git a/monitoring/fabric-unified-admin-monitoring/src/Load_Feature_Tracking_E2E.DataPipeline/pipeline-content.json b/monitoring/fabric-unified-admin-monitoring/src/Load_Feature_Tracking_E2E.DataPipeline/pipeline-content.json new file mode 100644 index 00000000..5cdb65ef --- /dev/null +++ b/monitoring/fabric-unified-admin-monitoring/src/Load_Feature_Tracking_E2E.DataPipeline/pipeline-content.json @@ -0,0 +1,26 @@ +{ + "properties": { + "activities": [ + { + "name": "02_Load_Feature_Tracking", + "type": "TridentNotebook", + "dependsOn": [], + "policy": { + "timeout": "0.12:00:00", + "retry": 0, + "retryIntervalInSeconds": 30, + "secureOutput": false, + "secureInput": false + }, + "typeProperties": { + "notebookId": "REPLACE_WITH_NOTEBOOK_ID", + "workspaceId": "REPLACE_WITH_WORKSPACE_ID", + "parameters": {}, + "sessionTag": "fuam_feature_tracking" + } + } + ], + "parameters": {}, + "annotations": [] + } +}