feature/oracle_profiler

.gitignore

@@ -1,11 +1,13 @@
+.venv/
 .venv
 .python-version
 .sdkmanrc
-.DS_Store
+.DS_Store/
 *.pyc
 __pycache__
 dist
 .idea
+.idea/
 /htmlcov/
 *.iml
 target/

pyproject.toml

@@ -40,6 +40,7 @@ dependencies = [
   "SQLAlchemy~=2.0.40",
   "pygls~=2.0.0",
   "duckdb>=1.2.2,<1.5.0",
+   "oracledb==3.4.0",
   "databricks-switch-plugin~=0.1.7",  # Temporary, until Switch is migrated to be a transpiler (LSP) plugin.
   "requests>=2.28.1,<3",  # Matches databricks-sdk (and 'types-requests' below), to avoid conflicts.
   "pandas~=2.3.1", # Required for new configure assessment

src/databricks/labs/lakebridge/assessments/_constants.py

@@ -5,10 +5,11 @@
 
 PLATFORM_TO_SOURCE_TECHNOLOGY_CFG = {
     "synapse": "src/databricks/labs/lakebridge/resources/assessments/synapse/pipeline_config.yml",
+    "oracle": "src/databricks/labs/lakebridge/resources/assessments/oracle/pipeline_config.yml",
 }
 
 # TODO modify this PLATFORM_TO_SOURCE_TECHNOLOGY.keys() once all platforms are supported
-PROFILER_SOURCE_SYSTEM = ["synapse"]
+PROFILER_SOURCE_SYSTEM = ["mssql", "synapse", "oracle"]
 
 
 # This flag indicates whether a connector is required for the source system when pipeline is trigger
@@ -18,4 +19,5 @@
 CONNECTOR_REQUIRED = {
     "synapse": False,
     "mssql": True,
+    "oracle": True,
 }

src/databricks/labs/lakebridge/assessments/configure_assessment.py

@@ -1,4 +1,5 @@
 from abc import ABC, abstractmethod
+from collections.abc import Callable
 from pathlib import Path
 import logging
 import shutil
@@ -72,6 +73,43 @@ def run(self):
         logger.info(f"{source.capitalize()} Assessment Configuration Completed")
 
 
+class ConfigureOracleAssessment(AssessmentConfigurator):
+    """Oracle specific assessment configuration."""
+
+    def _configure_credentials(self) -> str:
+        cred_file = self._credential_file
+        source = self._source_name
+
+        logger.info(
+            "\n(local | env) \nlocal means values are read as plain text \nenv means values are read "
+            "from environment variables fall back to plain text if not variable is not found\n",
+        )
+        secret_vault_type = str(self.prompts.choice("Enter secret vault type (local | env)", ["local", "env"])).lower()
+        secret_vault_name = None
+
+        logger.info("Please refer to the documentation to understand the difference between local and env.")
+
+        credential = {
+            "secret_vault_type": secret_vault_type,
+            "secret_vault_name": secret_vault_name,
+            source: {
+                "host": self.prompts.question("Enter the host details (Server name, IP address, SCAN Name)"),
+                "tnsPort": int(
+                    self.prompts.question("Enter the TNS Listener port number", default=str(1521), valid_number=True)
+                ),
+                "tnsService": self.prompts.question(
+                    "Enter the TNS service name as registered in the Oracle listener", default="orcl"
+                ),
+                "user": self.prompts.question("Enter user name with system privileges", default="SYSTEM"),
+                "password": self.prompts.password("Enter user password"),
+            },
+        }
+
+        _save_to_disk(credential, cred_file)
+        logger.info(f"Credential template created for {source}.")
+        return source
+
+
 class ConfigureSqlServerAssessment(AssessmentConfigurator):
     """SQL Server specific assessment configuration."""
 
@@ -182,9 +220,10 @@ def create_assessment_configurator(
     source_system: str, product_name: str, prompts: Prompts, credential_file=None
 ) -> AssessmentConfigurator:
     """Factory function to create the appropriate assessment configurator."""
-    configurators = {
+    configurators: dict[str, Callable[..., AssessmentConfigurator]] = {
         "mssql": ConfigureSqlServerAssessment,
         "synapse": ConfigureSynapseAssessment,
+        "oracle": ConfigureOracleAssessment,
     }
 
     if source_system not in configurators:

src/databricks/labs/lakebridge/connections/database_manager.py

@@ -79,7 +79,7 @@ def _create_connector(db_type: str, config: JsonObject) -> DatabaseConnector:
         "snowflake": SnowflakeConnector,
         "mssql": MSSQLConnector,
         "tsql": MSSQLConnector,
-        "synapse": MSSQLConnector,  # Synapse uses MSSQL protocol
+        "oracle": OracleConnector,
     }
 
     connector_class = connectors.get(db_type.lower())
@@ -125,9 +125,26 @@ def _connect(self) -> Engine:
         return create_engine(connection_string)
 
 
+class OracleConnector(_BaseConnector):
+    def _connect(self) -> Engine:
+
+        db_name = self.config.get('tnsService')
+        connection_string = URL.create(
+            drivername="oracle+oracledb",
+            username=str(self.config['user']),
+            password=str(self.config['password']),
+            host=str(self.config['host']),
+            port=int(str(self.config.get('tnsPort', 1521))),
+            database=str(db_name),
+        )
+
+        return create_engine(connection_string)
+
+
 class DatabaseManager:
     def __init__(self, db_type: str, config: JsonObject):
         self.connector = _create_connector(db_type, config)
+        self._db_type = db_type
 
     def __enter__(self) -> "DatabaseManager":
         """Support context manager protocol for resource management."""
@@ -152,6 +169,8 @@ def fetch(self, query: str) -> FetchResult:
 
     def check_connection(self) -> bool:
         query = "SELECT 101 AS test_column"
+        if self._db_type.lower() == "oracle":
+            query = "SELECT 101 AS test_column FROM dual"
         result = self.fetch(query)
         if result is None:
             return False

src/databricks/labs/lakebridge/resources/assessments/oracle/config_containers.sql

@@ -0,0 +1,4 @@
+-- Has to be executed on CDB
+
+select listagg(inst_id, ',') within group (order by name) as inst_ids,name,open_mode,pdb_count from gv$containers group by name,open_mode,pdb_count
+

src/databricks/labs/lakebridge/resources/assessments/oracle/config_containers_ddl.sql

@@ -0,0 +1,6 @@
+CREATE TABLE config_containers (
+    inst_ids VARCHAR,
+    name VARCHAR,
+    open_mode VARCHAR,
+    pdb_count INTEGER
+);

src/databricks/labs/lakebridge/resources/assessments/oracle/config_db_features.sql

@@ -0,0 +1,29 @@
+-- Has to be executed on CDB
+with t as (select version from product_component_version where product like 'Oracle Database%'),
+	  inst_cnt as (select count(*) as cnt from gv$instance),
+	  pdb_cnt as (select count(distinct con_id ) as cnt from gv$pdbs where name!='PDB$SEED'),
+	  pdb_namelist as (select listagg(name,',') within group (order by name) as pdbnames from (select distinct name from gv$pdbs where name !='PDB$SEED')),
+--	select listagg(distinct name,',') as pdbnames from gv$pdbs where name !='PDB$SEED'),
+	  cpu_cores_global as (select 'CLUSTER' as scope, null , stat_name,'CPU GLOBAL (Cluster): '||stat_name as detailed_stat_name,sum(to_number(value)) as value
+			from gv$osstat
+			where stat_name in ('NUM_CPUS','NUM_CPU_CORES','NUM_CPU_SOCKETS')
+			group by stat_name),
+	  cpu_cores_details as (select 'INSTANCE' as scope,inst_id, stat_name,'CPU per Instance Id: '||inst_id||' - '||stat_name as detailed_stat_name, to_number(value) as value
+				from gv$osstat
+				where stat_name in ('NUM_CPUS','NUM_CPU_CORES','NUM_CPU_SOCKETS')
+				order by inst_id,stat_name)
+select * from
+(
+select null as scope, null as inst_id, null as stat_name, 'VERSION' as name,to_char(t.version) as value from t
+union
+select null as scope, null as inst_id, null as stat_name, 'INSTANCE COUNT', to_char(inst_cnt.cnt) from inst_cnt
+union
+select null as scope, null as inst_id, null as stat_name, 'PDB COUNT',to_char(pdb_cnt.cnt) from pdb_cnt
+union
+select null as scope, null as inst_id, null as stat_name, 'PDB LIST', to_char(pdb_namelist.pdbnames) from pdb_namelist
+union
+select scope, null as inst_id, stat_name,detailed_stat_name, to_char(value) from cpu_cores_global
+union
+select scope, inst_id, stat_name, detailed_stat_name, to_char(value) from cpu_cores_details
+)
+order by name

src/databricks/labs/lakebridge/resources/assessments/oracle/config_db_features_ddl.sql

@@ -0,0 +1,7 @@
+CREATE TABLE config_db_features (
+    scope VARCHAR,
+    inst_id INTEGER,
+    stat_name VARCHAR,
+    name VARCHAR,
+    value VARCHAR
+);

src/databricks/labs/lakebridge/resources/assessments/oracle/config_instance.sql

@@ -0,0 +1,4 @@
+-- Has to be executed on CDB
+-- spool results/config_instance.csv
+select inst_id,instance_name,version,database_type from gv$instance
+

src/databricks/labs/lakebridge/resources/assessments/oracle/config_instance_ddl.sql

@@ -0,0 +1,6 @@
+CREATE TABLE config_instance (
+    inst_id INTEGER,
+    instance_name VARCHAR,
+    version VARCHAR,
+    database_type VARCHAR
+);

src/databricks/labs/lakebridge/resources/assessments/oracle/config_memory_evolution.sql

@@ -0,0 +1,27 @@
+select NVL(con.name,'Entire CDB/Non CDB') con_name,
+	    param.instance_number,
+	    snap.snap_time,
+	    parameter_name,
+	    value
+from cdb_hist_parameter param,
+(select con_id,name from v$containers) con,
+(select con_id,snap_id,instance_number,begin_interval_time snap_time from cdb_hist_snapshot) snap
+where 1=1
+and param.snap_id=snap.snap_id
+and param.con_id=con.con_id(+)
+and param.instance_number=snap.instance_number
+and param.parameter_name in ('sga_target',
+                             'pga_aggregate_target',
+                             'db_cache_size',
+                             'shared_pool_size',
+                             'large_pool_size',
+                             'java_pool_size',
+                             'streams_pool_size',
+                             'db_16k_cache_size',
+                             'db_2k_cache_size',
+                             'db_32k_cache_size',
+                             'db_4k_cache_size',
+                             'db_8k_cache_size',
+                             'memory_target',
+                             'memory_max_target')
+order by 1,3,2

src/databricks/labs/lakebridge/resources/assessments/oracle/config_memory_evolution_ddl.sql

@@ -0,0 +1,7 @@
+CREATE TABLE config_memory_evolution (
+    con_name VARCHAR,
+    instance_number INTEGER,
+    snap_time TIMESTAMP,
+    parameter_name VARCHAR,
+    value VARCHAR
+);

src/databricks/labs/lakebridge/resources/assessments/oracle/config_pdb_objects.sql

@@ -0,0 +1,10 @@
+-- Has to be executed on CDB
+
+SELECT cont.NAME as PDB_NAME,OWNER,OBJECT_TYPE,COUNT(*) as CNT
+FROM CDB_OBJECTS o,
+     (select distinct con_id as con_id, name from gv$containers) cont
+WHERE o.CON_ID=cont.con_id
+AND OWNER in (select username from cdb_users where oracle_maintained='N' and cont.con_id=con_id)
+GROUP BY cont.name,OWNER,OBJECT_TYPE
+ORDER BY 1,2
+

src/databricks/labs/lakebridge/resources/assessments/oracle/config_pdb_objects_ddl.sql

@@ -0,0 +1,6 @@
+CREATE TABLE config_pdb_objects (
+    pdb_name VARCHAR,
+    owner VARCHAR,
+    object_type VARCHAR,
+    cnt INTEGER
+);

src/databricks/labs/lakebridge/resources/assessments/oracle/config_pdb_partitions.sql

@@ -0,0 +1,21 @@
+-- Has to be executed on CDB
+SELECT cont.NAME as PDB_NAME,OWNER,OBJECT_TYPE,CNT
+from
+(
+    select owner,con_id,'TABLE (NON PARTITIONED)' as OBJECT_TYPE,count(*) as cnt from cdb_tables where partitioned='NO' group by owner,con_id
+    union
+    select owner,con_id,'TABLE (PARTITIONED)',count(*) as cnt from cdb_tables where partitioned='YES' group by owner,con_id
+    union
+    select owner,con_id,'INDEX (NON PARTITIONED)',count(*) as cnt from cdb_indexes where partitioned='NO' group by owner,con_id
+    union
+    select owner,con_id,'INDEX (PARTITIONED)',count(*) as cnt from cdb_indexes where partitioned='YES' group by owner,con_id
+    union
+    select owner,con_id,'LOBS (NON PARTITIONED)',count(*) as cnt from cdb_lobs where partitioned='NO' group by owner,con_id
+    union
+    select owner,con_id,'LOBS (PARTITIONED)',count(*) as cnt from cdb_lobs where partitioned='YES' group by owner,con_id
+) u,
+(select distinct con_id as con_id, name from gv$containers) cont
+WHERE u.CON_ID=cont.con_id
+AND OWNER in (select username from cdb_users where oracle_maintained='N' and cont.con_id=u.con_id)
+ORDER BY 1,2
+

src/databricks/labs/lakebridge/resources/assessments/oracle/config_pdb_partitions_ddl.sql

@@ -0,0 +1,6 @@
+CREATE TABLE config_pdb_partitions (
+    pdb_name VARCHAR,
+    owner VARCHAR,
+    object_type VARCHAR,
+    cnt INTEGER
+);

src/databricks/labs/lakebridge/resources/assessments/oracle/config_storage.sql

@@ -0,0 +1,36 @@
+select con_name,
+--       sub.tablespace_name,
+       case
+         when tablespace_name in ('SYSTEM','SYSAUX') then 'SYSTEM'
+         when tablespace_name in (select tablespace_name from cdb_tablespaces where contents='UNDO' and con_id=sub.con_id) then 'UNDO'
+         -- when tablespace_name in (select tablespace_name from cdb_tablespaces where contents='TEMPORARY' and con_id=sub.con_id) then 'TEMP'
+         ELSE 'USER_DATA'
+       end as tablespace_type,
+       sum(gb) gb,
+       sum(freegb) freegb,
+       sum(maxgb) maxgb
+from
+(
+    select c.name as con_name,c.con_id,
+           f.tablespace_name,
+           f.bytes/1024/1024 mb, f.bytes/1024/1024/1024 gb,
+           t.free_bytes/1024/1024 freemb, t.free_bytes/1024/1024/1024 freegb,
+           f.maxbytes/1024/1024 maxmb, f.maxbytes/1024/1024/1024 maxgb
+    from
+    (select con_id,tablespace_name,bytes,maxbytes from cdb_data_files ) f,
+    (select con_id,tablespace_name,sum(bytes) free_bytes from cdb_free_space group by con_id,tablespace_name ) t,
+    (select distinct con_id,name from gv$containers) c
+    where 1=1
+          and t.con_id=f.con_id
+          and t.con_id=c.con_id
+          and t.tablespace_name=f.tablespace_name
+) sub
+group by con_name,
+--tablespace_name
+       case
+         when tablespace_name in ('SYSTEM','SYSAUX') then 'SYSTEM'
+         when tablespace_name in (select tablespace_name from cdb_tablespaces where contents='UNDO' and con_id=sub.con_id) then 'UNDO'
+         -- when tablespace_name in (select tablespace_name from cdb_tablespaces where contents='TEMPORARY' and con_id=sub.con_id) then 'TEMP'
+         ELSE 'USER_DATA'
+       end
+order by 1

src/databricks/labs/lakebridge/resources/assessments/oracle/config_storage_ddl.sql

@@ -0,0 +1,7 @@
+CREATE TABLE config_storage (
+    con_name VARCHAR,
+    tablespace_type VARCHAR,
+    gb DOUBLE,
+    freegb DOUBLE,
+    maxgb DOUBLE
+);

src/databricks/labs/lakebridge/resources/assessments/oracle/perf_cpu_waits.sql

@@ -0,0 +1,25 @@
+ -- Has to be executed on CDB
+
+select cont.name as pdb_name,
+       ash.instance_number,
+       TO_DATE(ash.mtime,'YYYY-MM-DD HH24') as mtime,
+       ash.event,
+       ash.wait_class,
+       ash.total_wait_time
+from (SELECT instance_number,con_id,CON_DBID,
+        TO_CHAR(sample_time,'YYYY-MM-DD HH24') mtime,
+        NVL(a.event, 'ON CPU') AS event,
+        NVL(a.wait_class, 'ON CPU') AS wait_class,
+        COUNT(*)*10 AS total_wait_time
+      FROM   cdb_hist_active_sess_history a
+      GROUP BY instance_number,
+            con_id, CON_DBID,
+            TO_CHAR(sample_time,'YYYY-MM-DD HH24'),
+            a.event,
+            a.wait_class
+     ) ash,
+     (select distinct con_id,name,dbid from gv$containers) cont
+where cont.con_id=ash.con_id
+      and cont.dbid=ash.CON_DBID
+ORDER BY pdb_name,mtime
+

src/databricks/labs/lakebridge/resources/assessments/oracle/perf_cpu_waits_ddl.sql

@@ -0,0 +1,8 @@
+CREATE TABLE perf_cpu_waits (
+    pdb_name VARCHAR,
+    instance_number INTEGER,
+    mtime TIMESTAMP,
+    event VARCHAR,
+    wait_class VARCHAR,
+    total_wait_time BIGINT
+);