diff --git a/Databricks/ACTIVE/APPEALS/GOLD_APPEAL_SUBMITTED_JSON.ipynb b/Databricks/ACTIVE/APPEALS/GOLD_APPEAL_SUBMITTED_JSON.ipynb index 1d5fc3f8..80e324dd 100644 --- a/Databricks/ACTIVE/APPEALS/GOLD_APPEAL_SUBMITTED_JSON.ipynb +++ b/Databricks/ACTIVE/APPEALS/GOLD_APPEAL_SUBMITTED_JSON.ipynb @@ -41,6 +41,7 @@ "outputs": [], "source": [ "import shared_functions.paymentPending as PP\n", + "import shared_functions.paymentPendingDetained as PPD\n", "import shared_functions.appealSubmitted as APS\n", "from shared_functions.DQRules import base_DQRules, build_rule_expression, build_dq_rules_dependencies" ] @@ -218,11 +219,7 @@ "cell_type": "markdown", "metadata": { "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "batchId": -6018114963734290, - "byteLimit": 2048000, - "rowLimit": 10000 - }, + "cellMetadata": {}, "inputWidgets": {}, "nuid": "e818dd62-dc2f-4b3b-bac0-f9216f747357", "showTitle": false, @@ -264,7 +261,8 @@ "silver_h = spark.table(\"hive_metastore.ariadm_active_appeals.silver_history_detail\")\n", "bronze_hearing_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_hearing_centres\")\n", "bronze_derive_hearing_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_derive_hearing_centres\")\n", - "bronze_HORef_cleansing = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_HORef_cleansing\")" + "bronze_HORef_cleansing = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_HORef_cleansing\")\n", + "bronze_detention_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_detention_centres\")" ] }, { @@ -411,21 +409,14 @@ "inputWidgets": {}, "nuid": "3c102932-86cd-491c-ba77-39fc6a8d92e9", "showTitle": true, - "tableResultSettingsMap": { - "0": { - "dataGridStateBlob": "{\"version\":1,\"tableState\":{\"columnPinning\":{\"left\":[\"#row_number#\"],\"right\":[]},\"columnSizing\":{},\"columnVisibility\":{}},\"settings\":{\"columns\":{}},\"syncTimestamp\":1775056717427}", - "filterBlob": "{\"version\":1,\"filterGroups\":[],\"syncTimestamp\":1775051018238}", - "queryPlanFiltersBlob": "[]", - "tableResultIndex": 0 - } - }, + "tableResultSettingsMap": {}, "title": "Function: paymentType" } }, "outputs": [], "source": [ "df, df_audit = APS.paymentType(silver_m1, silver_m4)\n", - "# df.display()" + "# display(df)" ] }, { @@ -600,26 +591,26 @@ }, "outputs": [], "source": [ - "def mainAppealSubmitted(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m4, silver_c, silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres):\n", + "def mainAppealSubmitted(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m4, silver_c, silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres, bronze_detention_centres):\n", "\n", " AppealState = \"appealSubmitted\"\n", "\n", " # Aggregate details\n", " AppealType_df, AppealType_df_audit = PP.appealType(silver_m1)\n", - " caseData_df, caseData_df_audit = PP.caseData(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres)\n", + " caseData_df, caseData_df_audit = PPD.caseData(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres,bronze_detention_centres)\n", " flagsLabels_df, flagsLabels_df_audit = PP.flagsLabels(silver_m1, silver_m2, silver_c)\n", - " appellantDetails_df, appellantDetails_df_audit = PP.appellantDetails(silver_m1, silver_m2, silver_c, bronze_countryFromAddress,bronze_HORef_cleansing)\n", + " appellantDetails_df, appellantDetails_df_audit = PPD.appellantDetails(silver_m1, silver_m2, silver_c, bronze_countryFromAddress,bronze_HORef_cleansing)\n", " legalRepDetails_df, legalRepDetails_df_audit = PP.legalRepDetails(silver_m1, bronze_countryFromAddress)\n", " partyID_df, partyID_df_audit = PP.partyID(silver_m1, silver_m3, silver_c)\n", " payment_df, payment_df_audit = APS.paymentType(silver_m1, silver_m4)\n", " homeOfficeDetails_df, homeOfficeDetails_df_audit = PP.homeOfficeDetails(silver_m1, silver_m2, silver_c, bronze_HORef_cleansing)\n", " remissionTypes_df, remissionTypes_df_audit = APS.remissionTypes(silver_m1, bronze_remission_lookup_df, silver_m4)\n", - " sponsorDetails_df, sponsorDetails_df_audit = PP.sponsorDetails(silver_m1, silver_c)\n", - " general_df, general_df_audit = PP.general(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres)\n", + " sponsorDetails_df, sponsorDetails_df_audit = PPD.sponsorDetails(silver_m1, silver_c)\n", + " general_df, general_df_audit = PPD.general(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres,bronze_detention_centres)\n", " generalDefault_df = PP.generalDefault(silver_m1)\n", " documents_df, documents_df_audit = PP.documents(silver_m1)\n", " caseState_df, caseState_df_audit = PP.caseState(silver_m1, AppealState)\n", - " detainedState_df, detainedState_df_audit = PP.detainedState(silver_m1)\n", + " detainedState_df, detainedState_df_audit = PPD.detained(silver_m1,silver_m2,bronze_detention_centres)\n", " silver_segmentation_df = silver_segmentation\n", "\n", " # Join all aggregated data with Appeal Case Details\n", @@ -686,7 +677,7 @@ "\n", "# bronze_HORef_cleansing = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_HORef_cleansing\")\n", "\n", - "# df_final, df_audit = mainAppealSubmitted(silver_segmentation,silver_m1, silver_m2, silver_m3,silver_m4, silver_c,silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing)\n", + "# df_final, df_audit = mainAppealSubmitted(silver_segmentation,silver_m1, silver_m2, silver_m3,silver_m4, silver_c,silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing,bronze_detention_centres)\n", "\n", "# display(df_final)" ] @@ -882,6 +873,7 @@ " bronze_derive_hearing_centres = dlt.read(\"bronze_derive_hearing_centres\")\n", " bronze_interpreter_languages = dlt.read(\"bronze_interpreter_languages\")\n", " bronze_listing_location = dlt.read(\"bronze_listing_location\")\n", + " bronze_detention_centres = dlt.read(\"bronze_detention_centres\")\n", " silver_segmentation = dlt.read(\"stg_segmentation_states\").filter(col(\"TargetState\") == lit(AppealState))\n", " except:\n", " silver_m1 = spark.table(\"hive_metastore.ariadm_active_appeals.silver_appealcase_detail\").filter(col(\"dv_targetState\") == lit(AppealState)).distinct()\n", @@ -899,15 +891,16 @@ " bronze_derive_hearing_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_derive_hearing_centres\")\n", " bronze_interpreter_languages = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_interpreter_languages\")\n", " bronze_listing_location = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_listing_location\")\n", + " bronze_detention_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_detention_centres\")\n", " silver_segmentation = spark.table(\"hive_metastore.ariadm_active_appeals.stg_segmentation_states\").filter(col(\"TargetState\") == lit(AppealState))\n", " bronze_ended_states = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_ended_states\")\n", " \n", - " df_final, df_audit = mainAppealSubmitted(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m4, silver_c, silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres)\n", + " df_final, df_audit = mainAppealSubmitted(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m4, silver_c, silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres,bronze_detention_centres)\n", " \n", " df_final = build_dq_rules_dependencies(df_final, silver_m1, silver_m2, silver_m3, silver_m4, silver_m6, silver_c,silver_h,\n", " bronze_countries_postal_lookup_df, bronze_HORef_cleansing, bronze_remissions,\n", " bronze_interpreter_languages, bronze_listing_location,bronze_ended_states,\n", - " bronze_hearing_centres, bronze_derive_hearing_centres)\n", + " bronze_hearing_centres, bronze_derive_hearing_centres, bronze_detention_centres)\n", " \n", " df_final = df_final.withColumn(\"is_valid\", expr(dq_rules))\n", "\n", @@ -1052,6 +1045,7 @@ " bronze_remissions = dlt.read(\"bronze_remissions\").distinct()\n", " bronze_countryFromAddress = dlt.read(\"bronze_countries_countryFromAddress\")\n", " bronze_HORef_cleansing = dlt.read(\"bronze_HORef_cleansing\")\n", + " bronze_detention_centres = dlt.read(\"bronze_detention_centres\")\n", " silver_segmentation = dlt.read(\"stg_segmentation_states\").filter(col(\"TargetState\") == lit(AppealState))\n", " except:\n", " silver_m1 = spark.table(\"hive_metastore.ariadm_active_appeals.silver_appealcase_detail\").filter(col(\"dv_targetState\") == lit(AppealState)).distinct()\n", @@ -1071,9 +1065,10 @@ " bronze_remissions = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_remissions\").distinct()\n", " bronze_countryFromAddress = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_countries_countryFromAddress\")\n", " bronze_HORef_cleansing = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_HORef_cleansing\")\n", + " bronze_detention_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_detention_centres\")\n", " silver_segmentation = spark.table(\"hive_metastore.ariadm_active_appeals.stg_segmentation_states\").filter(col(\"TargetState\") == lit(AppealState))\n", " \n", - " df_final,df_audit = mainAppealSubmitted(silver_segmentation,silver_m1, silver_m2, silver_m3, silver_m4, silver_c, silver_h, bronze_remission_lookup_df, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres)\n", + " df_final,df_audit = mainAppealSubmitted(silver_segmentation,silver_m1, silver_m2, silver_m3, silver_m4, silver_c, silver_h, bronze_remission_lookup_df, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres,bronze_detention_centres)\n", "\n", " return df_audit" ] diff --git a/Databricks/ACTIVE/APPEALS/GOLD_AWAITING_EVIDENCE_RESPONDENT_A_JSON.ipynb b/Databricks/ACTIVE/APPEALS/GOLD_AWAITING_EVIDENCE_RESPONDENT_A_JSON.ipynb index 19590458..f783ca41 100644 --- a/Databricks/ACTIVE/APPEALS/GOLD_AWAITING_EVIDENCE_RESPONDENT_A_JSON.ipynb +++ b/Databricks/ACTIVE/APPEALS/GOLD_AWAITING_EVIDENCE_RESPONDENT_A_JSON.ipynb @@ -42,6 +42,7 @@ "source": [ "import importlib\n", "import shared_functions.paymentPending as PP\n", + "import shared_functions.paymentPendingDetained as PPD\n", "import shared_functions.appealSubmitted as APS\n", "import shared_functions.AwaitingEvidenceRespondant_a as AERa\n", "import shared_functions.AwaitingEvidenceRespondant_b as AERb\n", @@ -248,7 +249,8 @@ "bronze_countryFromAddress = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_countries_countryFromAddress\")\n", "bronze_HORef_cleansing = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_HORef_cleansing\")\n", "bronze_hearing_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_hearing_centres\")\n", - "bronze_derive_hearing_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_derive_hearing_centres\")" + "bronze_derive_hearing_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_derive_hearing_centres\")\n", + "bronze_detention_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_detention_centres\")" ] }, { @@ -581,13 +583,13 @@ "from pyspark.sql.functions import col, lit, to_json, struct, concat, regexp_replace\n", "from datetime import datetime\n", "\n", - "def mainAwaitingRespondentEvidence(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m4, silver_c, silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres):\n", + "def mainAwaitingRespondentEvidence(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m4, silver_c, silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres, bronze_detention_centres):\n", "\n", " AppealState = \"awaitingRespondentEvidence(a)\"\n", "\n", " # Aggregate details\n", " AppealType_df, AppealType_df_audit = PP.appealType(silver_m1)\n", - " caseData_df, caseData_df_audit = PP.caseData(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres)\n", + " caseData_df, caseData_df_audit = PPD.caseData(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres, bronze_detention_centres)\n", " flagsLabels_df, flagsLabels_df_audit = PP.flagsLabels(silver_m1, silver_m2, silver_c)\n", " appellantDetails_df, appellantDetails_df_audit = AERa.appellantDetails(silver_m1, silver_m2, silver_c, bronze_countryFromAddress, bronze_HORef_cleansing)\n", " legalRepDetails_df, legalRepDetails_df_audit = PP.legalRepDetails(silver_m1, bronze_countryFromAddress)\n", @@ -595,12 +597,12 @@ " payment_df, payment_df_audit = APS.paymentType(silver_m1, silver_m4)\n", " homeOfficeDetails_df, homeOfficeDetails_df_audit = PP.homeOfficeDetails(silver_m1, silver_m2, silver_c, bronze_HORef_cleansing)\n", " remissionTypes_df, remissionTypes_df_audit = APS.remissionTypes(silver_m1, bronze_remissions, silver_m4)\n", - " sponsorDetails_df, sponsorDetails_df_audit = PP.sponsorDetails(silver_m1, silver_c)\n", - " general_df, general_df_audit = PP.general(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres)\n", + " sponsorDetails_df, sponsorDetails_df_audit = PPD.sponsorDetails(silver_m1, silver_c)\n", + " general_df, general_df_audit = PPD.general(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres, bronze_detention_centres)\n", " generalDefault_df = AERa.generalDefault(silver_m1)\n", " documents_df, documents_df_audit = PP.documents(silver_m1)\n", " caseState_df, caseState_df_audit = PP.caseState(silver_m1, \"awaitingRespondentEvidence\")\n", - " detainedState_df, detainedState_df_audit = PP.detainedState(silver_m1)\n", + " detainedState_df, detainedState_df_audit = PPD.detained(silver_m1, silver_m2, bronze_detention_centres)\n", " silver_segmentation_df = silver_segmentation\n", "\n", " # Join all aggregated data with Appeal Case Details\n", @@ -674,7 +676,10 @@ "cell_type": "markdown", "metadata": { "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, "inputWidgets": {}, "nuid": "b1e06424-6966-4125-9563-3d73261b37c0", "showTitle": false, @@ -780,7 +785,10 @@ "cell_type": "markdown", "metadata": { "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, "inputWidgets": {}, "nuid": "db97feb8-762a-40b6-8cb9-5afa8f134fe3", "showTitle": false, @@ -861,6 +869,7 @@ " bronze_derive_hearing_centres = dlt.read(\"bronze_derive_hearing_centres\")\n", " bronze_interpreter_languages = dlt.read(\"bronze_interpreter_languages\")\n", " bronze_listing_location = dlt.read(\"bronze_listing_location\")\n", + " bronze_detention_centres = dlt.read(\"bronze_detention_centres\")\n", " silver_segmentation = dlt.read(\"stg_segmentation_states\").filter(col(\"TargetState\") == lit(AppealState))\n", " except:\n", " silver_m1 = spark.table(\"hive_metastore.ariadm_active_appeals.silver_appealcase_detail\").filter(col(\"dv_targetState\") == lit(AppealState)).distinct()\n", @@ -878,15 +887,16 @@ " bronze_derive_hearing_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_derive_hearing_centres\")\n", " bronze_interpreter_languages = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_interpreter_languages\")\n", " bronze_listing_location = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_listing_location\")\n", + " bronze_detention_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_detention_centres\")\n", " silver_segmentation = spark.table(\"hive_metastore.ariadm_active_appeals.stg_segmentation_states\").filter(col(\"TargetState\") == lit(AppealState))\n", " bronze_ended_states = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_ended_states\")\n", " \n", - " df_final, df_audit = mainAwaitingRespondentEvidence(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m4, silver_c, silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres)\n", + " df_final, df_audit = mainAwaitingRespondentEvidence(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m4, silver_c, silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres, bronze_detention_centres)\n", " \n", " df_final = build_dq_rules_dependencies(df_final, silver_m1, silver_m2, silver_m3, silver_m4, silver_m6, silver_c,silver_h,\n", " bronze_countries_postal_lookup_df, bronze_HORef_cleansing, bronze_remissions,\n", " bronze_interpreter_languages, bronze_listing_location,bronze_ended_states,\n", - " bronze_hearing_centres, bronze_derive_hearing_centres)\n", + " bronze_hearing_centres, bronze_derive_hearing_centres, bronze_detention_centres)\n", "\n", " df_final = df_final.withColumn(\"is_valid\", expr(dq_rules))\n", "\n", @@ -1025,6 +1035,7 @@ " bronze_remissions = dlt.read(\"bronze_remissions\").distinct()\n", " bronze_countryFromAddress = dlt.read(\"bronze_countries_countryFromAddress\")\n", " bronze_HORef_cleansing = dlt.read(\"bronze_HORef_cleansing\")\n", + " bronze_detention_centres = dlt.read(\"bronze_detention_centres\")\n", " silver_segmentation = dlt.read(\"stg_segmentation_states\").filter(col(\"TargetState\") == lit(AppealState))\n", " \n", " except:\n", @@ -1038,9 +1049,10 @@ " bronze_remissions = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_remissions\").distinct()\n", " bronze_countryFromAddress = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_countries_countryFromAddress\")\n", " bronze_HORef_cleansing = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_HORef_cleansing\")\n", + " bronze_detention_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_detention_centres\")\n", " silver_segmentation = spark.table(\"hive_metastore.ariadm_active_appeals.stg_segmentation_states\").filter(col(\"TargetState\") == lit(AppealState))\n", " \n", - " df_final,df_audit = mainAwaitingRespondentEvidence(silver_segmentation, silver_m1,silver_m2, silver_m3, silver_m4, silver_c, silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres)\n", + " df_final,df_audit = mainAwaitingRespondentEvidence(silver_segmentation, silver_m1,silver_m2, silver_m3, silver_m4, silver_c, silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres, bronze_detention_centres)\n", "\n", " return df_audit" ] @@ -1079,7 +1091,7 @@ "language": "python", "notebookMetadata": { "mostRecentlyExecutedCommandWithImplicitDF": { - "commandId": 6755913017508968, + "commandId": 8743555699811783, "dataframes": [ "_sqldf" ] diff --git a/Databricks/ACTIVE/APPEALS/GOLD_AWAITING_EVIDENCE_RESPONDENT_B_JSON.ipynb b/Databricks/ACTIVE/APPEALS/GOLD_AWAITING_EVIDENCE_RESPONDENT_B_JSON.ipynb index c4c07313..d603d500 100644 --- a/Databricks/ACTIVE/APPEALS/GOLD_AWAITING_EVIDENCE_RESPONDENT_B_JSON.ipynb +++ b/Databricks/ACTIVE/APPEALS/GOLD_AWAITING_EVIDENCE_RESPONDENT_B_JSON.ipynb @@ -42,6 +42,7 @@ "source": [ "import importlib\n", "import shared_functions.paymentPending as PP\n", + "import shared_functions.paymentPendingDetained as PPD\n", "import shared_functions.appealSubmitted as APS\n", "import shared_functions.AwaitingEvidenceRespondant_a as AERa\n", "import shared_functions.AwaitingEvidenceRespondant_b as AERb\n", @@ -245,7 +246,8 @@ "bronze_countryFromAddress = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_countries_countryFromAddress\")\n", "bronze_HORef_cleansing = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_HORef_cleansing\")\n", "bronze_hearing_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_hearing_centres\")\n", - "bronze_derive_hearing_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_derive_hearing_centres\")" + "bronze_derive_hearing_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_derive_hearing_centres\")\n", + "bronze_detention_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_detention_centres\")" ] }, { @@ -578,12 +580,12 @@ "from pyspark.sql.functions import col, lit, to_json, struct, concat, regexp_replace\n", "from datetime import datetime\n", "\n", - "def mainAwaitingRespondentEvidenceB(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m4, silver_c, silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres):\n", + "def mainAwaitingRespondentEvidenceB(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m4, silver_c, silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres, bronze_detention_centres):\n", " AppealState = \"awaitingRespondentEvidence(b)\"\n", "\n", " # Aggregate details\n", " AppealType_df, AppealType_df_audit = PP.appealType(silver_m1)\n", - " caseData_df, caseData_df_audit = PP.caseData(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres)\n", + " caseData_df, caseData_df_audit = PPD.caseData(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres, bronze_detention_centres)\n", " flagsLabels_df, flagsLabels_df_audit = PP.flagsLabels(silver_m1, silver_m2, silver_c)\n", " appellantDetails_df, appellantDetails_df_audit = AERa.appellantDetails(silver_m1, silver_m2, silver_c, bronze_countryFromAddress,bronze_HORef_cleansing)\n", " legalRepDetails_df, legalRepDetails_df_audit = PP.legalRepDetails(silver_m1, bronze_countryFromAddress)\n", @@ -591,12 +593,12 @@ " payment_df, payment_df_audit = APS.paymentType(silver_m1, silver_m4)\n", " homeOfficeDetails_df, homeOfficeDetails_df_audit = PP.homeOfficeDetails(silver_m1, silver_m2, silver_c, bronze_HORef_cleansing)\n", " remissionTypes_df, remissionTypes_df_audit = APS.remissionTypes(silver_m1, bronze_remissions, silver_m4)\n", - " sponsorDetails_df, sponsorDetails_df_audit = PP.sponsorDetails(silver_m1, silver_c)\n", - " general_df, general_df_audit = PP.general(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres)\n", + " sponsorDetails_df, sponsorDetails_df_audit = PPD.sponsorDetails(silver_m1, silver_c)\n", + " general_df, general_df_audit = PPD.general(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres,bronze_detention_centres)\n", " generalDefault_df = AERb.generalDefault(silver_m1)\n", " documents_df, documents_df_audit = AERb.documents(silver_m1)\n", " caseState_df, caseState_df_audit = PP.caseState(silver_m1,\"awaitingRespondentEvidence\")\n", - " detainedState_df, detainedState_df_audit = PP.detainedState(silver_m1)\n", + " detainedState_df, detainedState_df_audit = PPD.detained(silver_m1,silver_m2,bronze_detention_centres)\n", " silver_segmentation_df = silver_segmentation\n", "\n", " # Join all aggregated data with Appeal Case Details\n", @@ -672,7 +674,10 @@ "cell_type": "markdown", "metadata": { "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, "inputWidgets": {}, "nuid": "b1e06424-6966-4125-9563-3d73261b37c0", "showTitle": false, @@ -778,7 +783,10 @@ "cell_type": "markdown", "metadata": { "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, "inputWidgets": {}, "nuid": "db97feb8-762a-40b6-8cb9-5afa8f134fe3", "showTitle": false, @@ -859,6 +867,7 @@ " bronze_derive_hearing_centres = dlt.read(\"bronze_derive_hearing_centres\")\n", " bronze_interpreter_languages = dlt.read(\"bronze_interpreter_languages\")\n", " bronze_listing_location = dlt.read(\"bronze_listing_location\")\n", + " bronze_detention_centres = dlt.read(\"bronze_detention_centres\")\n", " silver_segmentation = dlt.read(\"stg_segmentation_states\").filter(col(\"TargetState\") == lit(AppealState))\n", " except:\n", " silver_m1 = spark.table(\"hive_metastore.ariadm_active_appeals.silver_appealcase_detail\").filter(col(\"dv_targetState\") == lit(AppealState)).distinct()\n", @@ -876,15 +885,16 @@ " bronze_derive_hearing_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_derive_hearing_centres\")\n", " bronze_interpreter_languages = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_interpreter_languages\")\n", " bronze_listing_location = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_listing_location\")\n", + " bronze_detention_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_detention_centres\")\n", " silver_segmentation = spark.table(\"hive_metastore.ariadm_active_appeals.stg_segmentation_states\").filter(col(\"TargetState\") == lit(AppealState))\n", " bronze_ended_states = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_ended_states\")\n", " \n", - " df_final, df_audit = mainAwaitingRespondentEvidenceB(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m4, silver_c, silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres)\n", + " df_final, df_audit = mainAwaitingRespondentEvidenceB(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m4, silver_c, silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres,bronze_detention_centres)\n", " \n", " df_final = build_dq_rules_dependencies(df_final, silver_m1, silver_m2, silver_m3, silver_m4, silver_m6, silver_c,silver_h,\n", " bronze_countries_postal_lookup_df, bronze_HORef_cleansing, bronze_remissions,\n", " bronze_interpreter_languages, bronze_listing_location,bronze_ended_states,\n", - " bronze_hearing_centres, bronze_derive_hearing_centres)\n", + " bronze_hearing_centres, bronze_derive_hearing_centres, bronze_detention_centres)\n", " \n", " df_final = df_final.withColumn(\"is_valid\", expr(dq_rules))\n", "\n", @@ -1026,6 +1036,7 @@ " bronze_HORef_cleansing = dlt.read(\"bronze_HORef_cleansing\")\n", " bronze_hearing_centres = dlt.read(\"bronze_hearing_centres\")\n", " bronze_derive_hearing_centres = dlt.read(\"bronze_derive_hearing_centres\")\n", + " bronze_detention_centres = dlt.read(\"bronze_detention_centres\")\n", " silver_segmentation = dlt.read(\"stg_segmentation_states\").filter(col(\"TargetState\") == lit(AppealState))\n", " \n", " except:\n", @@ -1041,10 +1052,11 @@ " bronze_HORef_cleansing = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_HORef_cleansing\")\n", " bronze_hearing_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_hearing_centres\")\n", " bronze_derive_hearing_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_derive_hearing_centres\")\n", + " bronze_detention_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_detention_centres\")\n", " silver_segmentation = spark.table(\"hive_metastore.ariadm_active_appeals.stg_segmentation_states\").filter(col(\"TargetState\") == lit(AppealState))\n", "\n", " \n", - " df_final,df_audit = mainAwaitingRespondentEvidenceB(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m4, silver_c,silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing,bronze_hearing_centres,bronze_derive_hearing_centres)\n", + " df_final,df_audit = mainAwaitingRespondentEvidenceB(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m4, silver_c,silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing,bronze_hearing_centres,bronze_derive_hearing_centres,bronze_detention_centres)\n", "\n", " return df_audit" ] diff --git a/Databricks/ACTIVE/APPEALS/GOLD_CASE_UNDER_REVIEW_JSON.ipynb b/Databricks/ACTIVE/APPEALS/GOLD_CASE_UNDER_REVIEW_JSON.ipynb index 9fdc9e94..a8ca6b30 100644 --- a/Databricks/ACTIVE/APPEALS/GOLD_CASE_UNDER_REVIEW_JSON.ipynb +++ b/Databricks/ACTIVE/APPEALS/GOLD_CASE_UNDER_REVIEW_JSON.ipynb @@ -211,6 +211,7 @@ "outputs": [], "source": [ "import shared_functions.paymentPending as PP\n", + "import shared_functions.paymentPendingDetained as PPD\n", "import shared_functions.appealSubmitted as APS\n", "import shared_functions.AwaitingEvidenceRespondant_a as AERa\n", "import shared_functions.AwaitingEvidenceRespondant_b as AERb\n", @@ -605,14 +606,14 @@ "from pyspark.sql.functions import col, lit, to_json, struct, concat, regexp_replace\n", "from datetime import datetime\n", "\n", - "def mainCaseUnderReview(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m4, silver_m6, silver_c,silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres):\n", + "def mainCaseUnderReview(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m4, silver_m6, silver_c,silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres, bronze_detention_centres):\n", " AppealState = \"caseUnderReview\"\n", "\n", " silver_m1 = silver_m1.filter(col(\"dv_representation\") == \"LR\")\n", "\n", " # Aggregate details\n", " AppealType_df, AppealType_df_audit = PP.appealType(silver_m1)\n", - " caseData_df, caseData_df_audit = PP.caseData(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres)\n", + " caseData_df, caseData_df_audit = PPD.caseData(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres,bronze_detention_centres)\n", " flagsLabels_df, flagsLabels_df_audit = PP.flagsLabels(silver_m1, silver_m2, silver_c)\n", " appellantDetails_df, appellantDetails_df_audit = AERa.appellantDetails(silver_m1, silver_m2, silver_c, bronze_countryFromAddress,bronze_HORef_cleansing)\n", " legalRepDetails_df, legalRepDetails_df_audit = PP.legalRepDetails(silver_m1, bronze_countryFromAddress)\n", @@ -620,13 +621,13 @@ " payment_df, payment_df_audit = APS.paymentType(silver_m1, silver_m4)\n", " homeOfficeDetails_df, homeOfficeDetails_df_audit = PP.homeOfficeDetails(silver_m1, silver_m2, silver_c, bronze_HORef_cleansing)\n", " remissionTypes_df, remissionTypes_df_audit = APS.remissionTypes(silver_m1, bronze_remissions, silver_m4)\n", - " sponsorDetails_df, sponsorDetails_df_audit = PP.sponsorDetails(silver_m1, silver_c)\n", - " general_df, general_df_audit = PP.general(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres)\n", + " sponsorDetails_df, sponsorDetails_df_audit = PPD.sponsorDetails(silver_m1, silver_c)\n", + " general_df, general_df_audit = PPD.general(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres,bronze_detention_centres)\n", " generalDefault_df = CUR.generalDefault(silver_m1)\n", " documents_df, documents_df_audit = AERb.documents(silver_m1)\n", " caseState_df, caseState_df_audit = PP.caseState(silver_m1,AppealState)\n", " hearingResponse_df,hearingResponse_df_audit = CUR.hearingResponse(silver_m1,silver_m3,silver_m6)\n", - " detainedState_df, detainedState_df_audit = PP.detainedState(silver_m1)\n", + " detainedState_df, detainedState_df_audit = PPD.detained(silver_m1,silver_m2,bronze_detention_centres)\n", " silver_segmentation_df = silver_segmentation\n", " \n", "\n", @@ -707,7 +708,10 @@ "cell_type": "markdown", "metadata": { "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, "inputWidgets": {}, "nuid": "b1e06424-6966-4125-9563-3d73261b37c0", "showTitle": false, @@ -813,7 +817,10 @@ "cell_type": "markdown", "metadata": { "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, "inputWidgets": {}, "nuid": "db97feb8-762a-40b6-8cb9-5afa8f134fe3", "showTitle": false, @@ -895,6 +902,7 @@ " bronze_derive_hearing_centres = dlt.read(\"bronze_derive_hearing_centres\")\n", " bronze_interpreter_languages = dlt.read(\"bronze_interpreter_languages\")\n", " bronze_listing_location = dlt.read(\"bronze_listing_location\")\n", + " bronze_detention_centres = dlt.read(\"bronze_detention_centres\")\n", " silver_segmentation = dlt.read(\"stg_segmentation_states\").filter(col(\"TargetState\") == lit(AppealState))\n", " except:\n", " silver_m1 = spark.table(\"hive_metastore.ariadm_active_appeals.silver_appealcase_detail\").filter(col(\"dv_targetState\") == lit(AppealState)).distinct()\n", @@ -912,15 +920,16 @@ " bronze_derive_hearing_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_derive_hearing_centres\")\n", " bronze_interpreter_languages = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_interpreter_languages\")\n", " bronze_listing_location = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_listing_location\")\n", + " bronze_detention_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_detention_centres\")\n", " silver_segmentation = spark.table(\"hive_metastore.ariadm_active_appeals.stg_segmentation_states\").filter(col(\"TargetState\") == lit(AppealState))\n", " bronze_ended_states = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_ended_states\")\n", " \n", - " df_final, df_audit = mainCaseUnderReview(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m4, silver_m6, silver_c, silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres)\n", + " df_final, df_audit = mainCaseUnderReview(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m4, silver_m6, silver_c, silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres,bronze_detention_centres)\n", " \n", " df_final = build_dq_rules_dependencies(df_final, silver_m1, silver_m2, silver_m3, silver_m4, silver_m6, silver_c,silver_h,\n", " bronze_countries_postal_lookup_df, bronze_HORef_cleansing, bronze_remissions,\n", " bronze_interpreter_languages, bronze_listing_location,bronze_ended_states,\n", - " bronze_hearing_centres, bronze_derive_hearing_centres)\n", + " bronze_hearing_centres, bronze_derive_hearing_centres,bronze_detention_centres)\n", " \n", " df_final = df_final.withColumn(\"is_valid\", expr(dq_rules))\n", "\n", @@ -1061,6 +1070,7 @@ " bronze_HORef_cleansing = dlt.read(\"bronze_HORef_cleansing\")\n", " bronze_hearing_centres = dlt.read(\"bronze_hearing_centres\")\n", " bronze_derive_hearing_centres = dlt.read(\"bronze_derive_hearing_centres\")\n", + " bronze_detention_centres = dlt.read(\"bronze_detention_centres\")\n", " silver_segmentation = dlt.read(\"stg_segmentation_states\").filter(col(\"TargetState\") == lit(AppealState))\n", " \n", " except:\n", @@ -1076,10 +1086,11 @@ " bronze_HORef_cleansing = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_HORef_cleansing\")\n", " bronze_hearing_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_hearing_centres\")\n", " bronze_derive_hearing_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_derive_hearing_centres\")\n", + " bronze_detention_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_detention_centres\")\n", " silver_segmentation = spark.table(\"hive_metastore.ariadm_active_appeals.stg_segmentation_states\").filter(col(\"TargetState\") == lit(AppealState))\n", "\n", " \n", - " df_final,df_audit = mainCaseUnderReview(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m4, silver_m6, silver_c, silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing,bronze_hearing_centres,bronze_derive_hearing_centres)\n", + " df_final,df_audit = mainCaseUnderReview(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m4, silver_m6, silver_c, silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing,bronze_hearing_centres,bronze_derive_hearing_centres,bronze_detention_centres)\n", "\n", " return df_audit" ] diff --git a/Databricks/ACTIVE/APPEALS/GOLD_DECIDED_A_JSON.ipynb b/Databricks/ACTIVE/APPEALS/GOLD_DECIDED_A_JSON.ipynb index bb3bbc12..37767b57 100644 --- a/Databricks/ACTIVE/APPEALS/GOLD_DECIDED_A_JSON.ipynb +++ b/Databricks/ACTIVE/APPEALS/GOLD_DECIDED_A_JSON.ipynb @@ -82,6 +82,7 @@ "source": [ "import importlib\n", "import shared_functions.paymentPending as PP\n", + "import shared_functions.paymentPendingDetained as PPD\n", "import shared_functions.appealSubmitted as APS\n", "import shared_functions.AwaitingEvidenceRespondant_a as AERa\n", "import shared_functions.listing as L\n", @@ -291,7 +292,8 @@ "bronze_hearing_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_hearing_centres\")\n", "bronze_derive_hearing_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_derive_hearing_centres\")\n", "bronze_interpreter_languages = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_interpreter_languages\")\n", - "bronze_listing_location = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_listing_location\")" + "bronze_listing_location = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_listing_location\")\n", + "bronze_detention_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_detention_centres\")" ] }, { @@ -554,7 +556,7 @@ }, "outputs": [], "source": [ - "df, df_audit = D.general(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres)\n", + "df, df_audit = D.general(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres,bronze_detention_centres)\n", "# display(df)" ] }, @@ -763,12 +765,12 @@ "from pyspark.sql.functions import col, lit, to_json, struct, concat, regexp_replace\n", "from datetime import datetime\n", "\n", - "def mainDecidedA(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m4, silver_m6, silver_c, silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres, bronze_interpreter_languages, bronze_listing_location):\n", + "def mainDecidedA(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m4, silver_m6, silver_c, silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres, bronze_interpreter_languages, bronze_listing_location,bronze_detention_centres):\n", " AppealState = \"decided\"\n", " \n", " # Aggregate details\n", " appealType_df, appealType_df_audit = PP.appealType(silver_m1)\n", - " caseData_df, caseData_df_audit = PP.caseData(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres)\n", + " caseData_df, caseData_df_audit = PPD.caseData(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres,bronze_detention_centres)\n", " flagsLabels_df, flagsLabels_df_audit = PP.flagsLabels(silver_m1, silver_m2, silver_c)\n", " appellantDetails_df, appellantDetails_df_audit = AERa.appellantDetails(silver_m1, silver_m2, silver_c, bronze_countryFromAddress,bronze_HORef_cleansing)\n", " legalRepDetails_df, legalRepDetails_df_audit = PP.legalRepDetails(silver_m1,bronze_countryFromAddress)\n", @@ -776,8 +778,8 @@ " payment_df, payment_df_audit = APS.paymentType(silver_m1, silver_m4)\n", " homeOfficeDetails_df, homeOfficeDetails_df_audit = PP.homeOfficeDetails(silver_m1, silver_m2, silver_c, bronze_HORef_cleansing)\n", " remissionTypes_df, remissionTypes_df_audit = APS.remissionTypes(silver_m1, bronze_remissions, silver_m4)\n", - " sponsorDetails_df, sponsorDetails_df_audit = PP.sponsorDetails(silver_m1, silver_c)\n", - " general_df, general_df_audit = D.general(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres)\n", + " sponsorDetails_df, sponsorDetails_df_audit = PPD.sponsorDetails(silver_m1, silver_c)\n", + " general_df, general_df_audit = D.general(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres,bronze_detention_centres)\n", " generalDefault_df = DA.generalDefault(silver_m1)\n", " documents_df, documents_df_audit = DA.documents(silver_m1)\n", " hearingResponse_df, hearingResponse_df_audit = PFH.hearingResponse(silver_m1, silver_m3, silver_m6)\n", @@ -787,7 +789,7 @@ " substantiveDecision_df, substantiveDecision_df_audit = DA.substantiveDecision(silver_m1,silver_m3)\n", " hearingActuals_df, hearingActuals_df_audit = DA.hearingActuals(silver_m3)\n", " ftpa_df, ftpa_df_audit = DA.ftpa(silver_m3,silver_c)\n", - " detainedState_df, detainedState_df_audit = PP.detainedState(silver_m1)\n", + " detainedState_df, detainedState_df_audit = PPD.detained(silver_m1,silver_m2,bronze_detention_centres)\n", "\n", " silver_segmentation_df = silver_segmentation\n", "\n", @@ -1065,6 +1067,7 @@ " bronze_derive_hearing_centres = dlt.read(\"bronze_derive_hearing_centres\")\n", " bronze_interpreter_languages = dlt.read(\"bronze_interpreter_languages\")\n", " bronze_listing_location = dlt.read(\"bronze_listing_location\")\n", + " bronze_detention_centres = dlt.read(\"bronze_detention_centres\")\n", " silver_segmentation = dlt.read(\"stg_segmentation_states\").filter(col(\"TargetState\") == lit(AppealState))\n", " except:\n", " silver_m1 = spark.table(\"hive_metastore.ariadm_active_appeals.silver_appealcase_detail\").filter(col(\"dv_targetState\") == lit(AppealState)).distinct()\n", @@ -1082,15 +1085,16 @@ " bronze_derive_hearing_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_derive_hearing_centres\")\n", " bronze_interpreter_languages = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_interpreter_languages\")\n", " bronze_listing_location = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_listing_location\")\n", + " bronze_detention_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_detention_centres\")\n", " silver_segmentation = spark.table(\"hive_metastore.ariadm_active_appeals.stg_segmentation_states\").filter(col(\"TargetState\") == lit(AppealState))\n", " bronze_ended_states = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_ended_states\")\n", " \n", - " df_final, df_audit = mainDecidedA(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m4, silver_m6, silver_c, silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres, bronze_interpreter_languages,bronze_listing_location)\n", + " df_final, df_audit = mainDecidedA(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m4, silver_m6, silver_c, silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres, bronze_interpreter_languages,bronze_listing_location,bronze_detention_centres)\n", " \n", " df_final = build_dq_rules_dependencies(df_final, silver_m1, silver_m2, silver_m3, silver_m4, silver_m6, silver_c,silver_h,\n", " bronze_countries_postal_lookup_df, bronze_HORef_cleansing, bronze_remissions,\n", " bronze_interpreter_languages, bronze_listing_location,bronze_ended_states,\n", - " bronze_hearing_centres, bronze_derive_hearing_centres)\n", + " bronze_hearing_centres, bronze_derive_hearing_centres,bronze_detention_centres)\n", " \n", " df_final = df_final.withColumn(\"is_valid\", expr(dq_rules))\n", "\n", @@ -1227,6 +1231,7 @@ " bronze_derive_hearing_centres = dlt.read(\"bronze_derive_hearing_centres\")\n", " bronze_interpreter_languages = dlt.read(\"bronze_interpreter_languages\")\n", " bronze_listing_location = spark.table(\"bronze_listing_location\")\n", + " bronze_detention_centres = dlt.read(\"bronze_detention_centres\")\n", " silver_segmentation = dlt.read(\"stg_segmentation_states\").filter(col(\"TargetState\") == lit(AppealState))\n", " \n", " except:\n", @@ -1245,10 +1250,11 @@ " bronze_derive_hearing_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_derive_hearing_centres\")\n", " bronze_interpreter_languages = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_interpreter_languages\")\n", " bronze_listing_location = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_listing_location\")\n", + " bronze_detention_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_detention_centres\")\n", " silver_segmentation = spark.table(\"hive_metastore.ariadm_active_appeals.stg_segmentation_states\").filter(col(\"TargetState\") == lit(AppealState))\n", "\n", " \n", - " df_final, df_audit = mainDecidedA(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m4, silver_m6, silver_c,silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres, bronze_interpreter_languages, bronze_listing_location)\n", + " df_final, df_audit = mainDecidedA(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m4, silver_m6, silver_c,silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres, bronze_interpreter_languages, bronze_listing_location,bronze_detention_centres)\n", "\n", " return df_audit" ] diff --git a/Databricks/ACTIVE/APPEALS/GOLD_DECIDED_B_JSON.ipynb b/Databricks/ACTIVE/APPEALS/GOLD_DECIDED_B_JSON.ipynb index e0e41a41..ef3a1032 100644 --- a/Databricks/ACTIVE/APPEALS/GOLD_DECIDED_B_JSON.ipynb +++ b/Databricks/ACTIVE/APPEALS/GOLD_DECIDED_B_JSON.ipynb @@ -82,6 +82,7 @@ "source": [ "import importlib\n", "import shared_functions.paymentPending as PP\n", + "import shared_functions.paymentPendingDetained as PPD\n", "import shared_functions.appealSubmitted as APS\n", "import shared_functions.AwaitingEvidenceRespondant_a as AERa\n", "import shared_functions.listing as L\n", @@ -293,7 +294,8 @@ "bronze_hearing_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_hearing_centres\")\n", "bronze_derive_hearing_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_derive_hearing_centres\")\n", "bronze_interpreter_languages = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_interpreter_languages\")\n", - "bronze_listing_location = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_listing_location\")" + "bronze_listing_location = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_listing_location\")\n", + "bronze_detention_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_detention_centres\")" ] }, { @@ -556,7 +558,7 @@ }, "outputs": [], "source": [ - "df, df_audit = DB.general(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres)\n", + "df, df_audit = DB.general(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres,bronze_detention_centres)\n", "# display(df)" ] }, @@ -794,12 +796,12 @@ "from pyspark.sql.functions import col, lit, to_json, struct, concat, regexp_replace\n", "from datetime import datetime\n", "\n", - "def mainDecidedB(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m4, silver_m6, silver_c, silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres, bronze_interpreter_languages, bronze_listing_location):\n", + "def mainDecidedB(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m4, silver_m6, silver_c, silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres, bronze_interpreter_languages, bronze_listing_location,bronze_detention_centres):\n", " AppealState = \"decided\"\n", " \n", " # Aggregate details\n", " appealType_df, appealType_df_audit = PP.appealType(silver_m1)\n", - " caseData_df, caseData_df_audit = PP.caseData(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres)\n", + " caseData_df, caseData_df_audit = PPD.caseData(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres,bronze_detention_centres)\n", " flagsLabels_df, flagsLabels_df_audit = PP.flagsLabels(silver_m1, silver_m2, silver_c)\n", " appellantDetails_df, appellantDetails_df_audit = AERa.appellantDetails(silver_m1, silver_m2, silver_c, bronze_countryFromAddress,bronze_HORef_cleansing)\n", " legalRepDetails_df, legalRepDetails_df_audit = PP.legalRepDetails(silver_m1,bronze_countryFromAddress)\n", @@ -807,8 +809,8 @@ " payment_df, payment_df_audit = APS.paymentType(silver_m1, silver_m4)\n", " homeOfficeDetails_df, homeOfficeDetails_df_audit = PP.homeOfficeDetails(silver_m1, silver_m2, silver_c, bronze_HORef_cleansing)\n", " remissionTypes_df, remissionTypes_df_audit = APS.remissionTypes(silver_m1, bronze_remissions, silver_m4)\n", - " sponsorDetails_df, sponsorDetails_df_audit = PP.sponsorDetails(silver_m1, silver_c)\n", - " general_df, general_df_audit = DB.general(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres)\n", + " sponsorDetails_df, sponsorDetails_df_audit = PPD.sponsorDetails(silver_m1, silver_c)\n", + " general_df, general_df_audit = DB.general(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres,bronze_detention_centres)\n", " generalDefault_df = DB.generalDefault(silver_m1)\n", " documents_df, documents_df_audit = DB.documents(silver_m1,silver_m3)\n", " hearingResponse_df, hearingResponse_df_audit = PFH.hearingResponse(silver_m1, silver_m3, silver_m6)\n", @@ -819,7 +821,7 @@ " hearingActuals_df, hearingActuals_df_audit = DA.hearingActuals(silver_m3)\n", " ftpa_df, ftpa_df_audit = DB.ftpa(silver_m3,silver_c)\n", " setAside_df, setAside_df_audit = DB.setAside(silver_m1,silver_m3,silver_m6)\n", - " detainedState_df, detainedState_df_audit = PP.detainedState(silver_m1)\n", + " detainedState_df, detainedState_df_audit = PPD.detained(silver_m1,silver_m2,bronze_detention_centres)\n", "\n", " silver_segmentation_df = silver_segmentation\n", "\n", @@ -1097,6 +1099,7 @@ " bronze_derive_hearing_centres = dlt.read(\"bronze_derive_hearing_centres\")\n", " bronze_interpreter_languages = dlt.read(\"bronze_interpreter_languages\")\n", " bronze_listing_location = dlt.read(\"bronze_listing_location\")\n", + " bronze_detention_centres = dlt.read(\"bronze_detention_centres\")\n", " silver_segmentation = dlt.read(\"stg_segmentation_states\").filter(col(\"TargetState\") == lit(AppealState))\n", " except:\n", " silver_m1 = spark.table(\"hive_metastore.ariadm_active_appeals.silver_appealcase_detail\").filter(col(\"dv_targetState\") == lit(AppealState)).distinct()\n", @@ -1114,15 +1117,16 @@ " bronze_derive_hearing_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_derive_hearing_centres\")\n", " bronze_interpreter_languages = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_interpreter_languages\")\n", " bronze_listing_location = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_listing_location\")\n", + " bronze_detention_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_detention_centres\")\n", " silver_segmentation = spark.table(\"hive_metastore.ariadm_active_appeals.stg_segmentation_states\").filter(col(\"TargetState\") == lit(AppealState))\n", " bronze_ended_states = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_ended_states\")\n", " \n", - " df_final, df_audit = mainDecidedB(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m4, silver_m6, silver_c, silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres, bronze_interpreter_languages, bronze_listing_location)\n", + " df_final, df_audit = mainDecidedB(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m4, silver_m6, silver_c, silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres, bronze_interpreter_languages, bronze_listing_location,bronze_detention_centres)\n", " \n", " df_final = build_dq_rules_dependencies(df_final, silver_m1, silver_m2, silver_m3, silver_m4, silver_m6, silver_c,silver_h,\n", " bronze_countries_postal_lookup_df, bronze_HORef_cleansing, bronze_remissions,\n", " bronze_interpreter_languages, bronze_listing_location,bronze_ended_states,\n", - " bronze_hearing_centres, bronze_derive_hearing_centres)\n", + " bronze_hearing_centres, bronze_derive_hearing_centres,bronze_detention_centres)\n", " \n", " df_final = df_final.withColumn(\"is_valid\", expr(dq_rules))\n", "\n", @@ -1259,6 +1263,7 @@ " bronze_derive_hearing_centres = dlt.read(\"bronze_derive_hearing_centres\")\n", " bronze_interpreter_languages = dlt.read(\"bronze_interpreter_languages\")\n", " bronze_listing_location = spark.table(\"bronze_listing_location\")\n", + " bronze_detention_centres = dlt.read(\"bronze_detention_centres\")\n", " silver_segmentation = dlt.read(\"stg_segmentation_states\").filter(col(\"TargetState\") == lit(AppealState))\n", " \n", " except:\n", @@ -1277,10 +1282,11 @@ " bronze_derive_hearing_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_derive_hearing_centres\")\n", " bronze_interpreter_languages = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_interpreter_languages\")\n", " bronze_listing_location = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_listing_location\")\n", + " bronze_detention_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_detention_centres\")\n", " silver_segmentation = spark.table(\"hive_metastore.ariadm_active_appeals.stg_segmentation_states\").filter(col(\"TargetState\") == lit(AppealState))\n", "\n", " \n", - " df_final, df_audit = mainDecidedB(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m4, silver_m6, silver_c,silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres, bronze_interpreter_languages, bronze_listing_location)\n", + " df_final, df_audit = mainDecidedB(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m4, silver_m6, silver_c,silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres, bronze_interpreter_languages, bronze_listing_location,bronze_detention_centres)\n", "\n", " return df_audit" ] diff --git a/Databricks/ACTIVE/APPEALS/GOLD_DECISION_JSON.ipynb b/Databricks/ACTIVE/APPEALS/GOLD_DECISION_JSON.ipynb index 68e07768..fe6fe6ca 100644 --- a/Databricks/ACTIVE/APPEALS/GOLD_DECISION_JSON.ipynb +++ b/Databricks/ACTIVE/APPEALS/GOLD_DECISION_JSON.ipynb @@ -82,6 +82,7 @@ "source": [ "import importlib\n", "import shared_functions.paymentPending as PP\n", + "import shared_functions.paymentPendingDetained as PPD\n", "import shared_functions.appealSubmitted as APS\n", "import shared_functions.AwaitingEvidenceRespondant_a as AERa\n", "import shared_functions.listing as L\n", @@ -290,7 +291,8 @@ "bronze_hearing_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_hearing_centres\")\n", "bronze_derive_hearing_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_derive_hearing_centres\")\n", "bronze_interpreter_languages = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_interpreter_languages\")\n", - "bronze_listing_location = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_listing_location\")" + "bronze_listing_location = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_listing_location\")\n", + "bronze_detention_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_detention_centres\")" ] }, { @@ -553,7 +555,7 @@ }, "outputs": [], "source": [ - "df, df_audit = D.general(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres)\n", + "df, df_audit = D.general(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres,bronze_detention_centres)\n", "# display(df)" ] }, @@ -718,12 +720,12 @@ "from pyspark.sql.functions import col, lit, to_json, struct, concat, regexp_replace\n", "from datetime import datetime\n", "\n", - "def mainDecision(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m4, silver_m6, silver_c, silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres, bronze_interpreter_languages, bronze_listing_location):\n", + "def mainDecision(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m4, silver_m6, silver_c, silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres, bronze_interpreter_languages, bronze_listing_location,bronze_detention_centres):\n", " AppealState = \"decision\"\n", " \n", " # Aggregate details\n", " appealType_df, appealType_df_audit = PP.appealType(silver_m1)\n", - " caseData_df, caseData_df_audit = PP.caseData(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres)\n", + " caseData_df, caseData_df_audit = PPD.caseData(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres,bronze_detention_centres)\n", " flagsLabels_df, flagsLabels_df_audit = PP.flagsLabels(silver_m1, silver_m2, silver_c)\n", " appellantDetails_df, appellantDetails_df_audit = AERa.appellantDetails(silver_m1, silver_m2, silver_c, bronze_countryFromAddress,bronze_HORef_cleansing)\n", " legalRepDetails_df, legalRepDetails_df_audit = PP.legalRepDetails(silver_m1,bronze_countryFromAddress)\n", @@ -731,8 +733,8 @@ " payment_df, payment_df_audit = APS.paymentType(silver_m1, silver_m4)\n", " homeOfficeDetails_df, homeOfficeDetails_df_audit = PP.homeOfficeDetails(silver_m1, silver_m2, silver_c, bronze_HORef_cleansing)\n", " remissionTypes_df, remissionTypes_df_audit = APS.remissionTypes(silver_m1, bronze_remissions, silver_m4)\n", - " sponsorDetails_df, sponsorDetails_df_audit = PP.sponsorDetails(silver_m1, silver_c)\n", - " general_df, general_df_audit = D.general(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres)\n", + " sponsorDetails_df, sponsorDetails_df_audit = PPD.sponsorDetails(silver_m1, silver_c)\n", + " general_df, general_df_audit = D.general(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres,bronze_detention_centres)\n", " generalDefault_df = D.generalDefault(silver_m1)\n", " documents_df, documents_df_audit = D.documents(silver_m1)\n", " hearingResponse_df, hearingResponse_df_audit = PFH.hearingResponse(silver_m1, silver_m3, silver_m6)\n", @@ -740,7 +742,7 @@ " caseState_df, caseState_df_audit = PP.caseState(silver_m1, AppealState)\n", " hearingRequirements_df, hearingRequirements_df_audit = L.hearingRequirements(silver_m1, silver_m3, silver_c, bronze_interpreter_languages)\n", " substantiveDecision, substantiveDecision_df_audit = D.substantiveDecision(silver_m1)\n", - " detainedState_df, detainedState_df_audit = PP.detainedState(silver_m1)\n", + " detainedState_df, detainedState_df_audit = PPD.detained(silver_m1,silver_m2,bronze_detention_centres)\n", "\n", " silver_segmentation_df = silver_segmentation\n", "\n", @@ -1015,6 +1017,7 @@ " bronze_derive_hearing_centres = dlt.read(\"bronze_derive_hearing_centres\")\n", " bronze_interpreter_languages = dlt.read(\"bronze_interpreter_languages\")\n", " bronze_listing_location = dlt.read(\"bronze_listing_location\")\n", + " bronze_detention_centres = dlt.read(\"bronze_detention_centres\")\n", " silver_segmentation = dlt.read(\"stg_segmentation_states\").filter(col(\"TargetState\") == lit(AppealState))\n", " except:\n", " silver_m1 = spark.table(\"hive_metastore.ariadm_active_appeals.silver_appealcase_detail\").filter(col(\"dv_targetState\") == lit(AppealState)).distinct()\n", @@ -1032,15 +1035,16 @@ " bronze_derive_hearing_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_derive_hearing_centres\")\n", " bronze_interpreter_languages = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_interpreter_languages\")\n", " bronze_listing_location = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_listing_location\")\n", + " bronze_detention_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_detention_centres\")\n", " silver_segmentation = spark.table(\"hive_metastore.ariadm_active_appeals.stg_segmentation_states\").filter(col(\"TargetState\") == lit(AppealState))\n", " bronze_ended_states = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_ended_states\")\n", " \n", - " df_final, df_audit = mainDecision(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m4, silver_m6, silver_c, silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres, bronze_interpreter_languages, bronze_listing_location)\n", + " df_final, df_audit = mainDecision(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m4, silver_m6, silver_c, silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres, bronze_interpreter_languages, bronze_listing_location,bronze_detention_centres)\n", " \n", " df_final = build_dq_rules_dependencies(df_final, silver_m1, silver_m2, silver_m3, silver_m4, silver_m6, silver_c,silver_h,\n", " bronze_countries_postal_lookup_df, bronze_HORef_cleansing, bronze_remissions,\n", " bronze_interpreter_languages, bronze_listing_location,bronze_ended_states,\n", - " bronze_hearing_centres, bronze_derive_hearing_centres)\n", + " bronze_hearing_centres, bronze_derive_hearing_centres,bronze_detention_centres)\n", " \n", " df_final = df_final.withColumn(\"is_valid\", expr(dq_rules))\n", "\n", @@ -1177,6 +1181,7 @@ " bronze_derive_hearing_centres = dlt.read(\"bronze_derive_hearing_centres\")\n", " bronze_interpreter_languages = dlt.read(\"bronze_interpreter_languages\")\n", " bronze_listing_location = spark.table(\"bronze_listing_location\")\n", + " bronze_detention_centres = dlt.read(\"bronze_detention_centres\")\n", " silver_segmentation = dlt.read(\"stg_segmentation_states\").filter(col(\"TargetState\") == lit(AppealState))\n", " \n", " except:\n", @@ -1195,10 +1200,11 @@ " bronze_derive_hearing_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_derive_hearing_centres\")\n", " bronze_interpreter_languages = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_interpreter_languages\")\n", " bronze_listing_location = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_listing_location\")\n", + " bronze_detention_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_detention_centres\")\n", " silver_segmentation = spark.table(\"hive_metastore.ariadm_active_appeals.stg_segmentation_states\").filter(col(\"TargetState\") == lit(AppealState))\n", "\n", " \n", - " df_final, df_audit = mainDecision(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m4, silver_m6, silver_c,silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres, bronze_interpreter_languages, bronze_listing_location)\n", + " df_final, df_audit = mainDecision(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m4, silver_m6, silver_c,silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres, bronze_interpreter_languages, bronze_listing_location,bronze_detention_centres)\n", "\n", " return df_audit" ] diff --git a/Databricks/ACTIVE/APPEALS/GOLD_ENDED_JSON.ipynb b/Databricks/ACTIVE/APPEALS/GOLD_ENDED_JSON.ipynb index 1637a8a1..bdc7f438 100644 --- a/Databricks/ACTIVE/APPEALS/GOLD_ENDED_JSON.ipynb +++ b/Databricks/ACTIVE/APPEALS/GOLD_ENDED_JSON.ipynb @@ -82,6 +82,7 @@ "source": [ "import importlib\n", "import shared_functions.paymentPending as PP\n", + "import shared_functions.paymentPendingDetained as PPD\n", "import shared_functions.appealSubmitted as APS\n", "import shared_functions.AwaitingEvidenceRespondant_a as AERa\n", "import shared_functions.listing as L\n", @@ -293,6 +294,7 @@ "bronze_derive_hearing_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_derive_hearing_centres\")\n", "bronze_interpreter_languages = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_interpreter_languages\")\n", "bronze_listing_location = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_listing_location\")\n", + "bronze_detention_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_detention_centres\")\n", "bronze_ended_states = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_ended_states\")" ] }, @@ -556,7 +558,7 @@ }, "outputs": [], "source": [ - "df, df_audit = E.general(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres)\n", + "df, df_audit = E.general(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres,bronze_detention_centres)\n", "# display(df)" ] }, @@ -787,12 +789,12 @@ "from pyspark.sql.functions import col, lit, to_json, struct, concat, regexp_replace\n", "from datetime import datetime\n", "\n", - "def mainEnded(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m6, silver_c, silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres, bronze_interpreter_languages, bronze_listing_location,bronze_ended_states):\n", + "def mainEnded(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m6, silver_c, silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres, bronze_interpreter_languages, bronze_listing_location,bronze_ended_states,bronze_detention_centres):\n", " AppealState = \"ended\"\n", " \n", " # Aggregate details\n", " appealType_df, appealType_df_audit = PP.appealType(silver_m1)\n", - " caseData_df, caseData_df_audit = PP.caseData(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres)\n", + " caseData_df, caseData_df_audit = PPD.caseData(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres,bronze_detention_centres)\n", " flagsLabels_df, flagsLabels_df_audit = PP.flagsLabels(silver_m1, silver_m2, silver_c)\n", " appellantDetails_df, appellantDetails_df_audit = AERa.appellantDetails(silver_m1, silver_m2, silver_c, bronze_countryFromAddress,bronze_HORef_cleansing)\n", " legalRepDetails_df, legalRepDetails_df_audit = PP.legalRepDetails(silver_m1,bronze_countryFromAddress)\n", @@ -800,8 +802,8 @@ " payment_df, payment_df_audit = APS.paymentType(silver_m1, silver_m4)\n", " homeOfficeDetails_df, homeOfficeDetails_df_audit = PP.homeOfficeDetails(silver_m1, silver_m2, silver_c, bronze_HORef_cleansing)\n", " remissionTypes_df, remissionTypes_df_audit = APS.remissionTypes(silver_m1, bronze_remissions, silver_m4)\n", - " sponsorDetails_df, sponsorDetails_df_audit = PP.sponsorDetails(silver_m1, silver_c)\n", - " general_df, general_df_audit = E.general(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres)\n", + " sponsorDetails_df, sponsorDetails_df_audit = PPD.sponsorDetails(silver_m1, silver_c)\n", + " general_df, general_df_audit = E.general(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres,bronze_detention_centres)\n", " generalDefault_df = E.generalDefault(silver_m1,silver_m3)\n", " documents_df, documents_df_audit = E.documents(silver_m1,silver_m3)\n", " hearingResponse_df, hearingResponse_df_audit = E.hearingResponse(silver_m1, silver_m3, silver_m6)\n", @@ -812,7 +814,7 @@ " hearingActuals_df, hearingActuals_df_audit = E.hearingActuals(silver_m3)\n", " ftpa_df, ftpa_df_audit = E.ftpa(silver_m3,silver_c)\n", " ended_df, ended_df_audit = E.ended(silver_m1,silver_m3,bronze_ended_states)\n", - " detainedState_df, detainedState_df_audit = PP.detainedState(silver_m1)\n", + " detainedState_df, detainedState_df_audit = PPD.detained(silver_m1,silver_m2,bronze_detention_centres)\n", "\n", " silver_segmentation_df = silver_segmentation\n", "\n", @@ -1089,6 +1091,7 @@ " bronze_derive_hearing_centres = dlt.read(\"bronze_derive_hearing_centres\")\n", " bronze_interpreter_languages = dlt.read(\"bronze_interpreter_languages\")\n", " bronze_listing_location = spark.table(\"bronze_listing_location\")\n", + " bronze_detention_centres = dlt.read(\"bronze_detention_centres\")\n", " bronze_ended_states = spark.table(\"bronze_ended_states\")\n", " silver_segmentation = dlt.read(\"stg_segmentation_states\").filter(col(\"TargetState\") == lit(AppealState))\n", " \n", @@ -1108,14 +1111,15 @@ " bronze_interpreter_languages = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_interpreter_languages\")\n", " bronze_listing_location = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_listing_location\")\n", " bronze_ended_states = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_ended_states\")\n", + " bronze_detention_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_detention_centres\")\n", " silver_segmentation = spark.table(\"hive_metastore.ariadm_active_appeals.stg_segmentation_states\").filter(col(\"TargetState\") == lit(AppealState))\n", "\n", - " df_final, df_audit = mainEnded(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m6, silver_c, silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres, bronze_interpreter_languages,bronze_listing_location,bronze_ended_states)\n", + " df_final, df_audit = mainEnded(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m6, silver_c, silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres, bronze_interpreter_languages,bronze_listing_location,bronze_ended_states,bronze_detention_centres)\n", "\n", " df_final = build_dq_rules_dependencies(df_final, silver_m1, silver_m2, silver_m3, silver_m4, silver_m6, silver_c,silver_h,\n", " bronze_countries_postal_lookup_df, bronze_HORef_cleansing, bronze_remissions,\n", " bronze_interpreter_languages, bronze_listing_location,bronze_ended_states,\n", - " bronze_hearing_centres, bronze_derive_hearing_centres)\n", + " bronze_hearing_centres, bronze_derive_hearing_centres,bronze_detention_centres)\n", " \n", " df_final = df_final.withColumn(\"is_valid\", expr(dq_rules))\n", "\n", @@ -1253,6 +1257,7 @@ " bronze_interpreter_languages = dlt.read(\"bronze_interpreter_languages\")\n", " bronze_listing_location = spark.table(\"bronze_listing_location\")\n", " bronze_ended_states = spark.table(\"bronze_ended_states\")\n", + " bronze_detention_centres = dlt.read(\"bronze_detention_centres\")\n", " silver_segmentation = dlt.read(\"stg_segmentation_states\").filter(col(\"TargetState\") == lit(AppealState))\n", " \n", " \n", @@ -1273,10 +1278,11 @@ " bronze_interpreter_languages = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_interpreter_languages\")\n", " bronze_listing_location = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_listing_location\")\n", " bronze_ended_states = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_ended_states\")\n", + " bronze_detention_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_detention_centres\")\n", " silver_segmentation = spark.table(\"hive_metastore.ariadm_active_appeals.stg_segmentation_states\").filter(col(\"TargetState\") == lit(AppealState))\n", "\n", " \n", - " df_final, df_audit = mainEnded(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m6, silver_c,silver_h, bronze_remissions_lookup_df, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres, bronze_interpreter_languages,bronze_listing_location,bronze_ended_states)\n", + " df_final, df_audit = mainEnded(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m6, silver_c,silver_h, bronze_remissions_lookup_df, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres, bronze_interpreter_languages,bronze_listing_location,bronze_ended_states,bronze_detention_centres)\n", "\n", " return df_audit" ] diff --git a/Databricks/ACTIVE/APPEALS/GOLD_FTPA_DECIDED_JSON.ipynb b/Databricks/ACTIVE/APPEALS/GOLD_FTPA_DECIDED_JSON.ipynb index 8c030066..0d3d2f1b 100644 --- a/Databricks/ACTIVE/APPEALS/GOLD_FTPA_DECIDED_JSON.ipynb +++ b/Databricks/ACTIVE/APPEALS/GOLD_FTPA_DECIDED_JSON.ipynb @@ -82,6 +82,7 @@ "source": [ "import importlib\n", "import shared_functions.paymentPending as PP\n", + "import shared_functions.paymentPendingDetained as PPD\n", "import shared_functions.appealSubmitted as APS\n", "import shared_functions.AwaitingEvidenceRespondant_a as AERa\n", "import shared_functions.listing as L\n", @@ -294,7 +295,8 @@ "bronze_hearing_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_hearing_centres\")\n", "bronze_derive_hearing_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_derive_hearing_centres\")\n", "bronze_interpreter_languages = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_interpreter_languages\")\n", - "bronze_listing_location = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_listing_location\")" + "bronze_listing_location = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_listing_location\")\n", + "bronze_detention_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_detention_centres\")" ] }, { @@ -557,7 +559,7 @@ }, "outputs": [], "source": [ - "df, df_audit = FTD.general(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres)\n", + "df, df_audit = FTD.general(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres,bronze_detention_centres)\n", "# display(df)" ] }, @@ -787,12 +789,12 @@ "from pyspark.sql.functions import col, lit, to_json, struct, concat, regexp_replace\n", "from datetime import datetime\n", "\n", - "def mainftpaDecided(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m6, silver_c, silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres, bronze_interpreter_languages, bronze_listing_location):\n", + "def mainftpaDecided(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m6, silver_c, silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres, bronze_interpreter_languages, bronze_listing_location,bronze_detention_centres):\n", " AppealState = \"ftpaDecided\"\n", " \n", " # Aggregate details\n", " appealType_df, appealType_df_audit = PP.appealType(silver_m1)\n", - " caseData_df, caseData_df_audit = PP.caseData(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres)\n", + " caseData_df, caseData_df_audit = PPD.caseData(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres,bronze_detention_centres)\n", " flagsLabels_df, flagsLabels_df_audit = PP.flagsLabels(silver_m1, silver_m2, silver_c)\n", " appellantDetails_df, appellantDetails_df_audit = AERa.appellantDetails(silver_m1, silver_m2, silver_c, bronze_countryFromAddress,bronze_HORef_cleansing)\n", " legalRepDetails_df, legalRepDetails_df_audit = PP.legalRepDetails(silver_m1,bronze_countryFromAddress)\n", @@ -800,8 +802,8 @@ " payment_df, payment_df_audit = APS.paymentType(silver_m1, silver_m4)\n", " homeOfficeDetails_df, homeOfficeDetails_df_audit = PP.homeOfficeDetails(silver_m1, silver_m2, silver_c, bronze_HORef_cleansing)\n", " remissionTypes_df, remissionTypes_df_audit = APS.remissionTypes(silver_m1, bronze_remissions, silver_m4)\n", - " sponsorDetails_df, sponsorDetails_df_audit = PP.sponsorDetails(silver_m1, silver_c)\n", - " general_df, general_df_audit = FTD.general(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres)\n", + " sponsorDetails_df, sponsorDetails_df_audit = PPD.sponsorDetails(silver_m1, silver_c)\n", + " general_df, general_df_audit = FTD.general(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres,bronze_detention_centres)\n", " generalDefault_df = FSA.generalDefault(silver_m1)\n", " documents_df, documents_df_audit = FTD.documents(silver_m1,silver_m3)\n", " hearingResponse_df, hearingResponse_df_audit = PFH.hearingResponse(silver_m1, silver_m3, silver_m6)\n", @@ -811,7 +813,7 @@ " substantiveDecision_df, substantiveDecision_df_audit = DA.substantiveDecision(silver_m1,silver_m3)\n", " hearingActuals_df, hearingActuals_df_audit = DA.hearingActuals(silver_m3)\n", " ftpaDec_df, ftpa_df_audit = FTD.ftpa(silver_m3,silver_c) \n", - " detainedState_df, detainedState_df_audit = PP.detainedState(silver_m1)\n", + " detainedState_df, detainedState_df_audit = PPD.detained(silver_m1,silver_m2,bronze_detention_centres)\n", "\n", " silver_segmentation_df = silver_segmentation\n", "\n", @@ -1088,6 +1090,7 @@ " bronze_derive_hearing_centres = dlt.read(\"bronze_derive_hearing_centres\")\n", " bronze_interpreter_languages = dlt.read(\"bronze_interpreter_languages\")\n", " bronze_listing_location = spark.table(\"bronze_listing_location\")\n", + " bronze_detention_centres = dlt.read(\"bronze_detention_centres\")\n", " silver_segmentation = dlt.read(\"stg_segmentation_states\").filter(col(\"TargetState\") == lit(AppealState))\n", " \n", " except:\n", @@ -1105,10 +1108,11 @@ " bronze_derive_hearing_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_derive_hearing_centres\")\n", " bronze_interpreter_languages = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_interpreter_languages\")\n", " bronze_listing_location = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_listing_location\")\n", + " bronze_detention_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_detention_centres\")\n", " silver_segmentation = spark.table(\"hive_metastore.ariadm_active_appeals.stg_segmentation_states\").filter(col(\"TargetState\") == lit(AppealState))\n", " bronze_ended_states = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_ended_states\")\n", " \n", - " df_final, df_audit = mainftpaDecided(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m6, silver_c, silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres, bronze_interpreter_languages,bronze_listing_location)\n", + " df_final, df_audit = mainftpaDecided(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m6, silver_c, silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres, bronze_interpreter_languages,bronze_listing_location,bronze_detention_centres)\n", "\n", " \n", "\n", @@ -1116,7 +1120,7 @@ " df_final = build_dq_rules_dependencies(df_final, silver_m1, silver_m2, silver_m3, silver_m4, silver_m6, silver_c,silver_h,\n", " bronze_countries_postal_lookup_df, bronze_HORef_cleansing, bronze_remissions,\n", " bronze_interpreter_languages, bronze_listing_location,bronze_ended_states,\n", - " bronze_hearing_centres, bronze_derive_hearing_centres)\n", + " bronze_hearing_centres, bronze_derive_hearing_centres,bronze_detention_centres)\n", " \n", " df_final = df_final.withColumn(\"is_valid\", expr(dq_rules))\n", "\n", @@ -1252,6 +1256,7 @@ " bronze_derive_hearing_centres = dlt.read(\"bronze_derive_hearing_centres\")\n", " bronze_interpreter_languages = dlt.read(\"bronze_interpreter_languages\")\n", " bronze_listing_location = spark.table(\"bronze_listing_location\")\n", + " bronze_detention_centres = dlt.read(\"bronze_detention_centres\")\n", " silver_segmentation = dlt.read(\"stg_segmentation_states\").filter(col(\"TargetState\") == lit(AppealState))\n", " \n", " except:\n", @@ -1270,10 +1275,11 @@ " bronze_derive_hearing_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_derive_hearing_centres\")\n", " bronze_interpreter_languages = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_interpreter_languages\")\n", " bronze_listing_location = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_listing_location\")\n", + " bronze_detention_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_detention_centres\")\n", " silver_segmentation = spark.table(\"hive_metastore.ariadm_active_appeals.stg_segmentation_states\").filter(col(\"TargetState\") == lit(AppealState))\n", "\n", " \n", - " df_final, df_audit = mainftpaDecided(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m6, silver_c,silver_h, bronze_remissions_lookup_df, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres, bronze_interpreter_languages,bronze_listing_location)\n", + " df_final, df_audit = mainftpaDecided(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m6, silver_c,silver_h, bronze_remissions_lookup_df, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres, bronze_interpreter_languages,bronze_listing_location,bronze_detention_centres)\n", "\n", " return df_audit" ] diff --git a/Databricks/ACTIVE/APPEALS/GOLD_FTPA_SUBMITTED_A_JSON.ipynb b/Databricks/ACTIVE/APPEALS/GOLD_FTPA_SUBMITTED_A_JSON.ipynb index 3a0a372d..d5a89427 100644 --- a/Databricks/ACTIVE/APPEALS/GOLD_FTPA_SUBMITTED_A_JSON.ipynb +++ b/Databricks/ACTIVE/APPEALS/GOLD_FTPA_SUBMITTED_A_JSON.ipynb @@ -82,6 +82,7 @@ "source": [ "import importlib\n", "import shared_functions.paymentPending as PP\n", + "import shared_functions.paymentPendingDetained as PPD\n", "import shared_functions.appealSubmitted as APS\n", "import shared_functions.AwaitingEvidenceRespondant_a as AERa\n", "import shared_functions.listing as L\n", @@ -292,7 +293,8 @@ "bronze_hearing_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_hearing_centres\")\n", "bronze_derive_hearing_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_derive_hearing_centres\")\n", "bronze_interpreter_languages = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_interpreter_languages\")\n", - "bronze_listing_location = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_listing_location\")" + "bronze_listing_location = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_listing_location\")\n", + "bronze_detention_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_detention_centres\")" ] }, { @@ -555,7 +557,7 @@ }, "outputs": [], "source": [ - "df, df_audit = FSA.general(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres)\n", + "df, df_audit = FSA.general(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres,bronze_detention_centres)\n", "# display(df)" ] }, @@ -764,12 +766,12 @@ "from pyspark.sql.functions import col, lit, to_json, struct, concat, regexp_replace\n", "from datetime import datetime\n", "\n", - "def mainFTPASubmittedA(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m4, silver_m6, silver_c, silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres, bronze_interpreter_languages, bronze_listing_location):\n", + "def mainFTPASubmittedA(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m4, silver_m6, silver_c, silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres, bronze_interpreter_languages, bronze_listing_location,bronze_detention_centres):\n", " AppealState = \"ftpaSubmitted\"\n", " \n", " # Aggregate details\n", " appealType_df, appealType_df_audit = PP.appealType(silver_m1)\n", - " caseData_df, caseData_df_audit = PP.caseData(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres)\n", + " caseData_df, caseData_df_audit = PPD.caseData(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres,bronze_detention_centres)\n", " flagsLabels_df, flagsLabels_df_audit = PP.flagsLabels(silver_m1, silver_m2, silver_c)\n", " appellantDetails_df, appellantDetails_df_audit = AERa.appellantDetails(silver_m1, silver_m2, silver_c, bronze_countryFromAddress,bronze_HORef_cleansing)\n", " legalRepDetails_df, legalRepDetails_df_audit = PP.legalRepDetails(silver_m1,bronze_countryFromAddress)\n", @@ -777,8 +779,8 @@ " payment_df, payment_df_audit = APS.paymentType(silver_m1, silver_m4)\n", " homeOfficeDetails_df, homeOfficeDetails_df_audit = PP.homeOfficeDetails(silver_m1, silver_m2, silver_c, bronze_HORef_cleansing)\n", " remissionTypes_df, remissionTypes_df_audit = APS.remissionTypes(silver_m1, bronze_remissions, silver_m4)\n", - " sponsorDetails_df, sponsorDetails_df_audit = PP.sponsorDetails(silver_m1, silver_c)\n", - " general_df, general_df_audit = FSA.general(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres)\n", + " sponsorDetails_df, sponsorDetails_df_audit = PPD.sponsorDetails(silver_m1, silver_c)\n", + " general_df, general_df_audit = FSA.general(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres,bronze_detention_centres)\n", " generalDefault_df = FSA.generalDefault(silver_m1)\n", " documents_df, documents_df_audit = FSA.documents(silver_m1,silver_m3)\n", " hearingResponse_df, hearingResponse_df_audit = PFH.hearingResponse(silver_m1, silver_m3, silver_m6)\n", @@ -788,7 +790,7 @@ " substantiveDecision_df, substantiveDecision_df_audit = DA.substantiveDecision(silver_m1,silver_m3)\n", " hearingActuals_df, hearingActuals_df_audit = DA.hearingActuals(silver_m3)\n", " ftpa_df, ftpa_df_audit = FSA.ftpa(silver_m3,silver_c)\n", - " detainedState_df, detainedState_df_audit = PP.detainedState(silver_m1)\n", + " detainedState_df, detainedState_df_audit = PPD.detained(silver_m1,silver_m2,bronze_detention_centres)\n", "\n", " silver_segmentation_df = silver_segmentation\n", "\n", @@ -1065,6 +1067,7 @@ " bronze_derive_hearing_centres = dlt.read(\"bronze_derive_hearing_centres\")\n", " bronze_interpreter_languages = dlt.read(\"bronze_interpreter_languages\")\n", " bronze_listing_location = dlt.read(\"bronze_listing_location\")\n", + " bronze_detention_centres = dlt.read(\"bronze_detention_centres\")\n", " silver_segmentation = dlt.read(\"stg_segmentation_states\").filter(col(\"TargetState\") == lit(AppealState))\n", " except:\n", " silver_m1 = spark.table(\"hive_metastore.ariadm_active_appeals.silver_appealcase_detail\").filter(col(\"dv_targetState\") == lit(AppealState)).distinct()\n", @@ -1082,15 +1085,16 @@ " bronze_derive_hearing_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_derive_hearing_centres\")\n", " bronze_interpreter_languages = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_interpreter_languages\")\n", " bronze_listing_location = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_listing_location\")\n", + " bronze_detention_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_detention_centres\")\n", " silver_segmentation = spark.table(\"hive_metastore.ariadm_active_appeals.stg_segmentation_states\").filter(col(\"TargetState\") == lit(AppealState))\n", " bronze_ended_states = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_ended_states\")\n", " \n", - " df_final, df_audit = mainFTPASubmittedA(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m4, silver_m6, silver_c, silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres, bronze_interpreter_languages, bronze_listing_location)\n", + " df_final, df_audit = mainFTPASubmittedA(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m4, silver_m6, silver_c, silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres, bronze_interpreter_languages, bronze_listing_location,bronze_detention_centres)\n", " \n", " df_final = build_dq_rules_dependencies(df_final, silver_m1, silver_m2, silver_m3, silver_m4, silver_m6, silver_c,silver_h,\n", " bronze_countries_postal_lookup_df, bronze_HORef_cleansing, bronze_remissions,\n", " bronze_interpreter_languages, bronze_listing_location,bronze_ended_states,\n", - " bronze_hearing_centres, bronze_derive_hearing_centres)\n", + " bronze_hearing_centres, bronze_derive_hearing_centres,bronze_detention_centres)\n", " \n", " df_final = df_final.withColumn(\"is_valid\", expr(dq_rules))\n", "\n", @@ -1227,6 +1231,7 @@ " bronze_derive_hearing_centres = dlt.read(\"bronze_derive_hearing_centres\")\n", " bronze_interpreter_languages = dlt.read(\"bronze_interpreter_languages\")\n", " bronze_listing_location = spark.table(\"bronze_listing_location\")\n", + " bronze_detention_centres = dlt.read(\"bronze_detention_centres\")\n", " silver_segmentation = dlt.read(\"stg_segmentation_states\").filter(col(\"TargetState\") == lit(AppealState))\n", " \n", " except:\n", @@ -1245,10 +1250,11 @@ " bronze_derive_hearing_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_derive_hearing_centres\")\n", " bronze_interpreter_languages = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_interpreter_languages\")\n", " bronze_listing_location = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_listing_location\")\n", + " bronze_detention_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_detention_centres\")\n", " silver_segmentation = spark.table(\"hive_metastore.ariadm_active_appeals.stg_segmentation_states\").filter(col(\"TargetState\") == lit(AppealState))\n", "\n", " \n", - " df_final, df_audit = mainFTPASubmittedA(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m4, silver_m6, silver_c,silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres, bronze_interpreter_languages, bronze_listing_location)\n", + " df_final, df_audit = mainFTPASubmittedA(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m4, silver_m6, silver_c,silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres, bronze_interpreter_languages, bronze_listing_location,bronze_detention_centres)\n", "\n", " return df_audit" ] diff --git a/Databricks/ACTIVE/APPEALS/GOLD_FTPA_SUBMITTED_B_JSON.ipynb b/Databricks/ACTIVE/APPEALS/GOLD_FTPA_SUBMITTED_B_JSON.ipynb index f9d11bca..11e63776 100644 --- a/Databricks/ACTIVE/APPEALS/GOLD_FTPA_SUBMITTED_B_JSON.ipynb +++ b/Databricks/ACTIVE/APPEALS/GOLD_FTPA_SUBMITTED_B_JSON.ipynb @@ -82,6 +82,7 @@ "source": [ "import importlib\n", "import shared_functions.paymentPending as PP\n", + "import shared_functions.paymentPendingDetained as PPD\n", "import shared_functions.appealSubmitted as APS\n", "import shared_functions.AwaitingEvidenceRespondant_a as AERa\n", "import shared_functions.listing as L\n", @@ -293,7 +294,8 @@ "bronze_hearing_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_hearing_centres\")\n", "bronze_derive_hearing_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_derive_hearing_centres\")\n", "bronze_interpreter_languages = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_interpreter_languages\")\n", - "bronze_listing_location = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_listing_location\")" + "bronze_listing_location = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_listing_location\")\n", + "bronze_detention_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_detention_centres\")" ] }, { @@ -556,7 +558,7 @@ }, "outputs": [], "source": [ - "df, df_audit = FSA.general(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres)\n", + "df, df_audit = FSA.general(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres,bronze_detention_centres)\n", "# display(df)" ] }, @@ -765,12 +767,12 @@ "from pyspark.sql.functions import col, lit, to_json, struct, concat, regexp_replace\n", "from datetime import datetime\n", "\n", - "def mainFTPASubmittedB(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m4, silver_m6, silver_c, silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres, bronze_interpreter_languages, bronze_listing_location):\n", + "def mainFTPASubmittedB(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m4, silver_m6, silver_c, silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres, bronze_interpreter_languages, bronze_listing_location,bronze_detention_centres):\n", " AppealState = \"ftpaSubmitted\"\n", " \n", " # Aggregate details\n", " appealType_df, appealType_df_audit = PP.appealType(silver_m1)\n", - " caseData_df, caseData_df_audit = PP.caseData(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres)\n", + " caseData_df, caseData_df_audit = PPD.caseData(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres,bronze_detention_centres)\n", " flagsLabels_df, flagsLabels_df_audit = PP.flagsLabels(silver_m1, silver_m2, silver_c)\n", " appellantDetails_df, appellantDetails_df_audit = AERa.appellantDetails(silver_m1, silver_m2, silver_c, bronze_countryFromAddress,bronze_HORef_cleansing)\n", " legalRepDetails_df, legalRepDetails_df_audit = PP.legalRepDetails(silver_m1,bronze_countryFromAddress)\n", @@ -778,8 +780,8 @@ " payment_df, payment_df_audit = APS.paymentType(silver_m1, silver_m4)\n", " homeOfficeDetails_df, homeOfficeDetails_df_audit = PP.homeOfficeDetails(silver_m1, silver_m2, silver_c, bronze_HORef_cleansing)\n", " remissionTypes_df, remissionTypes_df_audit = APS.remissionTypes(silver_m1, bronze_remissions, silver_m4)\n", - " sponsorDetails_df, sponsorDetails_df_audit = PP.sponsorDetails(silver_m1, silver_c)\n", - " general_df, general_df_audit = FSA.general(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres)\n", + " sponsorDetails_df, sponsorDetails_df_audit = PPD.sponsorDetails(silver_m1, silver_c)\n", + " general_df, general_df_audit = FSA.general(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres,bronze_detention_centres)\n", " generalDefault_df = FSA.generalDefault(silver_m1)\n", " documents_df, documents_df_audit = FSA.documents(silver_m1,silver_m3)\n", " hearingResponse_df, hearingResponse_df_audit = PFH.hearingResponse(silver_m1, silver_m3, silver_m6)\n", @@ -789,7 +791,7 @@ " substantiveDecision_df, substantiveDecision_df_audit = DA.substantiveDecision(silver_m1,silver_m3)\n", " hearingActuals_df, hearingActuals_df_audit = DA.hearingActuals(silver_m3)\n", " ftpa_df, ftpa_df_audit = FSB.ftpa(silver_m3,silver_c)\n", - " detainedState_df, detainedState_df_audit = PP.detainedState(silver_m1)\n", + " detainedState_df, detainedState_df_audit = PPD.detained(silver_m1,silver_m2,bronze_detention_centres)\n", "\n", " silver_segmentation_df = silver_segmentation\n", "\n", @@ -1066,6 +1068,7 @@ " bronze_derive_hearing_centres = dlt.read(\"bronze_derive_hearing_centres\")\n", " bronze_interpreter_languages = dlt.read(\"bronze_interpreter_languages\")\n", " bronze_listing_location = dlt.read(\"bronze_listing_location\")\n", + " bronze_detention_centres = dlt.read(\"bronze_detention_centres\")\n", " silver_segmentation = dlt.read(\"stg_segmentation_states\").filter(col(\"TargetState\") == lit(AppealState))\n", " except:\n", " silver_m1 = spark.table(\"hive_metastore.ariadm_active_appeals.silver_appealcase_detail\").filter(col(\"dv_targetState\") == lit(AppealState)).distinct()\n", @@ -1083,15 +1086,16 @@ " bronze_derive_hearing_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_derive_hearing_centres\")\n", " bronze_interpreter_languages = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_interpreter_languages\")\n", " bronze_listing_location = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_listing_location\")\n", + " bronze_detention_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_detention_centres\")\n", " silver_segmentation = spark.table(\"hive_metastore.ariadm_active_appeals.stg_segmentation_states\").filter(col(\"TargetState\") == lit(AppealState))\n", " bronze_ended_states = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_ended_states\")\n", " \n", - " df_final, df_audit = mainFTPASubmittedB(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m4, silver_m6, silver_c, silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres, bronze_interpreter_languages, bronze_listing_location)\n", + " df_final, df_audit = mainFTPASubmittedB(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m4, silver_m6, silver_c, silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres, bronze_interpreter_languages, bronze_listing_location,bronze_detention_centres)\n", " \n", " df_final = build_dq_rules_dependencies(df_final, silver_m1, silver_m2, silver_m3, silver_m4, silver_m6, silver_c,silver_h,\n", " bronze_countries_postal_lookup_df, bronze_HORef_cleansing, bronze_remissions,\n", " bronze_interpreter_languages, bronze_listing_location,bronze_ended_states,\n", - " bronze_hearing_centres, bronze_derive_hearing_centres)\n", + " bronze_hearing_centres, bronze_derive_hearing_centres,bronze_detention_centres)\n", " \n", " df_final = df_final.withColumn(\"is_valid\", expr(dq_rules))\n", "\n", @@ -1228,6 +1232,7 @@ " bronze_derive_hearing_centres = dlt.read(\"bronze_derive_hearing_centres\")\n", " bronze_interpreter_languages = dlt.read(\"bronze_interpreter_languages\")\n", " bronze_listing_location = spark.table(\"bronze_listing_location\")\n", + " bronze_detention_centres = dlt.read(\"bronze_detention_centres\")\n", " silver_segmentation = dlt.read(\"stg_segmentation_states\").filter(col(\"TargetState\") == lit(AppealState))\n", " \n", " except:\n", @@ -1246,10 +1251,11 @@ " bronze_derive_hearing_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_derive_hearing_centres\")\n", " bronze_interpreter_languages = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_interpreter_languages\")\n", " bronze_listing_location = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_listing_location\")\n", + " bronze_detention_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_detention_centres\")\n", " silver_segmentation = spark.table(\"hive_metastore.ariadm_active_appeals.stg_segmentation_states\").filter(col(\"TargetState\") == lit(AppealState))\n", "\n", " \n", - " df_final, df_audit = mainFTPASubmittedB(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m4, silver_m6, silver_c,silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres, bronze_interpreter_languages, bronze_listing_location)\n", + " df_final, df_audit = mainFTPASubmittedB(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m4, silver_m6, silver_c,silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres, bronze_interpreter_languages, bronze_listing_location,bronze_detention_centres)\n", "\n", " return df_audit" ] diff --git a/Databricks/ACTIVE/APPEALS/GOLD_LISTING_JSON.ipynb b/Databricks/ACTIVE/APPEALS/GOLD_LISTING_JSON.ipynb index 002588dd..9a34eb37 100644 --- a/Databricks/ACTIVE/APPEALS/GOLD_LISTING_JSON.ipynb +++ b/Databricks/ACTIVE/APPEALS/GOLD_LISTING_JSON.ipynb @@ -26,7 +26,10 @@ "cell_type": "markdown", "metadata": { "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, "inputWidgets": {}, "nuid": "cc4f34c6-1b52-4600-a8bd-ecf5f1539523", "showTitle": false, @@ -82,6 +85,7 @@ "source": [ "import importlib\n", "import shared_functions.paymentPending as PP\n", + "import shared_functions.paymentPendingDetained as PPD\n", "import shared_functions.appealSubmitted as APS\n", "import shared_functions.AwaitingEvidenceRespondant_a as AERa\n", "import shared_functions.listing as L\n", @@ -287,7 +291,8 @@ "bronze_HORef_cleansing = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_HORef_cleansing\")\n", "bronze_hearing_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_hearing_centres\")\n", "bronze_derive_hearing_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_derive_hearing_centres\")\n", - "bronze_interpreter_languages = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_interpreter_languages\")\n" + "bronze_interpreter_languages = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_interpreter_languages\")\n", + "bronze_detention_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_detention_centres\")" ] }, { @@ -545,13 +550,20 @@ "inputWidgets": {}, "nuid": "19921c7b-4c43-4878-96a8-3f8738309915", "showTitle": true, - "tableResultSettingsMap": {}, + "tableResultSettingsMap": { + "0": { + "dataGridStateBlob": "{\"version\":1,\"tableState\":{\"columnPinning\":{\"left\":[\"#row_number#\"],\"right\":[]},\"columnSizing\":{},\"columnVisibility\":{}},\"settings\":{\"columns\":{}},\"syncTimestamp\":1775137542975}", + "filterBlob": null, + "queryPlanFiltersBlob": null, + "tableResultIndex": 0 + } + }, "title": "Function: GeneralFunctions" } }, "outputs": [], "source": [ - "df, df_audit = L.general(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres)\n", + "df, df_audit = L.general(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres,bronze_detention_centres)\n", "# display(df)" ] }, @@ -650,12 +662,12 @@ "from pyspark.sql.functions import col, lit, to_json, struct, concat, regexp_replace\n", "from datetime import datetime\n", "\n", - "def mainListing(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m4, silver_c, silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres, bronze_interpreter_languages):\n", + "def mainListing(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m4, silver_c, silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres, bronze_interpreter_languages,bronze_detention_centres):\n", " AppealState = \"listing\"\n", "\n", " # Aggregate details\n", " appealType_df, appealType_df_audit = PP.appealType(silver_m1)\n", - " caseData_df, caseData_df_audit = PP.caseData(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres)\n", + " caseData_df, caseData_df_audit = PPD.caseData(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres,bronze_detention_centres)\n", " flagsLabels_df, flagsLabels_df_audit = PP.flagsLabels(silver_m1, silver_m2, silver_c)\n", " appellantDetails_df, appellantDetails_df_audit = AERa.appellantDetails(silver_m1, silver_m2, silver_c, bronze_countryFromAddress,bronze_HORef_cleansing)\n", " legalRepDetails_df, legalRepDetails_df_audit = PP.legalRepDetails(silver_m1, bronze_countryFromAddress)\n", @@ -663,13 +675,13 @@ " payment_df, payment_df_audit = APS.paymentType(silver_m1, silver_m4)\n", " homeOfficeDetails_df, homeOfficeDetails_df_audit = PP.homeOfficeDetails(silver_m1, silver_m2, silver_c, bronze_HORef_cleansing)\n", " remissionTypes_df, remissionTypes_df_audit = APS.remissionTypes(silver_m1, bronze_remissions, silver_m4)\n", - " sponsorDetails_df, sponsorDetails_df_audit = PP.sponsorDetails(silver_m1, silver_c)\n", - " general_df, general_df_audit = L.general(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres)\n", + " sponsorDetails_df, sponsorDetails_df_audit = PPD.sponsorDetails(silver_m1, silver_c)\n", + " general_df, general_df_audit = L.general(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres,bronze_detention_centres)\n", " generalDefault_df = L.generalDefault(silver_m1)\n", " documents_df, documents_df_audit = L.documents(silver_m1)\n", " caseState_df, caseState_df_audit = PP.caseState(silver_m1, AppealState)\n", " hearingRequirements_df, hearingRequirements_df_audit = L.hearingRequirements(silver_m1, silver_m3, silver_c, bronze_interpreter_languages)\n", - " detainedState_df, detainedState_df_audit = PP.detainedState(silver_m1)\n", + " detainedState_df, detainedState_df_audit = PPD.detained(silver_m1,silver_m2,bronze_detention_centres)\n", " silver_segmentation_df = silver_segmentation\n", "\n", " # Join all aggregated data with Appeal Case Details\n", @@ -748,7 +760,10 @@ "cell_type": "markdown", "metadata": { "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, "inputWidgets": {}, "nuid": "b1e06424-6966-4125-9563-3d73261b37c0", "showTitle": false, @@ -854,7 +869,10 @@ "cell_type": "markdown", "metadata": { "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, "inputWidgets": {}, "nuid": "db97feb8-762a-40b6-8cb9-5afa8f134fe3", "showTitle": false, @@ -936,6 +954,7 @@ " bronze_derive_hearing_centres = dlt.read(\"bronze_derive_hearing_centres\")\n", " bronze_interpreter_languages = dlt.read(\"bronze_interpreter_languages\")\n", " bronze_listing_location = dlt.read(\"bronze_listing_location\")\n", + " bronze_detention_centres = dlt.read(\"bronze_detention_centres\")\n", " silver_segmentation = dlt.read(\"stg_segmentation_states\").filter(col(\"TargetState\") == lit(AppealState))\n", " except:\n", " silver_m1 = spark.table(\"hive_metastore.ariadm_active_appeals.silver_appealcase_detail\").filter(col(\"dv_targetState\") == lit(AppealState)).distinct()\n", @@ -953,15 +972,16 @@ " bronze_derive_hearing_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_derive_hearing_centres\")\n", " bronze_interpreter_languages = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_interpreter_languages\")\n", " bronze_listing_location = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_listing_location\")\n", + " bronze_detention_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_detention_centres\")\n", " silver_segmentation = spark.table(\"hive_metastore.ariadm_active_appeals.stg_segmentation_states\").filter(col(\"TargetState\") == lit(AppealState))\n", " bronze_ended_states = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_ended_states\")\n", " \n", - " df_final, df_audit = mainListing(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m4, silver_c, silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres, bronze_interpreter_languages)\n", + " df_final, df_audit = mainListing(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m4, silver_c, silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres, bronze_interpreter_languages,bronze_detention_centres)\n", " \n", " df_final = build_dq_rules_dependencies(df_final, silver_m1, silver_m2, silver_m3, silver_m4, silver_m6, silver_c,silver_h,\n", " bronze_countries_postal_lookup_df, bronze_HORef_cleansing, bronze_remissions,\n", " bronze_interpreter_languages, bronze_listing_location,bronze_ended_states,\n", - " bronze_hearing_centres, bronze_derive_hearing_centres)\n", + " bronze_hearing_centres, bronze_derive_hearing_centres,bronze_detention_centres)\n", " \n", " df_final = df_final.withColumn(\"is_valid\", expr(dq_rules))\n", "\n", @@ -1096,6 +1116,7 @@ " bronze_hearing_centres = dlt.read(\"bronze_hearing_centres\")\n", " bronze_derive_hearing_centres = dlt.read(\"bronze_derive_hearing_centres\")\n", " bronze_interpreter_languages = dlt.read(\"bronze_interpreter_languages\")\n", + " bronze_detention_centres = dlt.read(\"bronze_detention_centres\")\n", " silver_segmentation = dlt.read(\"stg_segmentation_states\").filter(col(\"TargetState\") == lit(AppealState))\n", " \n", " except:\n", @@ -1112,10 +1133,11 @@ " bronze_hearing_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_hearing_centres\")\n", " bronze_derive_hearing_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_derive_hearing_centres\")\n", " bronze_interpreter_languages = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_interpreter_languages\")\n", + " bronze_detention_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_detention_centres\")\n", " silver_segmentation = spark.table(\"hive_metastore.ariadm_active_appeals.stg_segmentation_states\").filter(col(\"TargetState\") == lit(AppealState))\n", "\n", " \n", - " df_final, df_audit = mainListing(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m4, silver_c, silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres, bronze_interpreter_languages)\n", + " df_final, df_audit = mainListing(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m4, silver_c, silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres, bronze_interpreter_languages,bronze_detention_centres)\n", "\n", " return df_audit" ] @@ -1154,7 +1176,7 @@ "language": "python", "notebookMetadata": { "mostRecentlyExecutedCommandWithImplicitDF": { - "commandId": 4671000946785512, + "commandId": 8743555699811799, "dataframes": [ "_sqldf" ] diff --git a/Databricks/ACTIVE/APPEALS/GOLD_PAYMENT_PENDING_JSON.ipynb b/Databricks/ACTIVE/APPEALS/GOLD_PAYMENT_PENDING_JSON.ipynb index 81a751f0..bee9d582 100644 --- a/Databricks/ACTIVE/APPEALS/GOLD_PAYMENT_PENDING_JSON.ipynb +++ b/Databricks/ACTIVE/APPEALS/GOLD_PAYMENT_PENDING_JSON.ipynb @@ -26,7 +26,10 @@ "cell_type": "markdown", "metadata": { "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, "inputWidgets": {}, "nuid": "18504bdf-794e-4966-b290-b635328c3ed0", "showTitle": false, @@ -285,7 +288,10 @@ "cell_type": "markdown", "metadata": { "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, "inputWidgets": {}, "nuid": "a6b49dac-308d-447a-8532-b56207d5350c", "showTitle": false, @@ -450,7 +456,10 @@ "cell_type": "markdown", "metadata": { "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, "inputWidgets": {}, "nuid": "d1b86460-a200-4c6b-9bf1-382b6bc1e8db", "showTitle": false, @@ -545,7 +554,10 @@ "cell_type": "markdown", "metadata": { "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, "inputWidgets": {}, "nuid": "9a2e4c75-b6e3-403f-99f8-ca27a6e03b05", "showTitle": false, @@ -561,7 +573,10 @@ "cell_type": "markdown", "metadata": { "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, "inputWidgets": {}, "nuid": "b9dc2f07-0f34-46eb-a4fc-73e29b7966e1", "showTitle": false, @@ -573,6 +588,35 @@ "## Transformation Functions" ] }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "f1d49249-68cd-4f66-aecc-5f30cd6ee616", + "showTitle": false, + "tableResultSettingsMap": { + "0": { + "dataGridStateBlob": "{\"version\":1,\"tableState\":{\"columnPinning\":{\"left\":[\"#row_number#\"],\"right\":[]},\"columnSizing\":{},\"columnVisibility\":{}},\"settings\":{\"columns\":{}},\"syncTimestamp\":1775060931451}", + "filterBlob": null, + "queryPlanFiltersBlob": null, + "tableResultIndex": 0 + } + }, + "title": "" + } + }, + "outputs": [], + "source": [ + "\n", + "silver_m1.display()\n" + ] + }, { "cell_type": "code", "execution_count": 0, @@ -607,14 +651,7 @@ "inputWidgets": {}, "nuid": "e272d878-a11a-4888-a02a-11c64be6ed87", "showTitle": false, - "tableResultSettingsMap": { - "0": { - "dataGridStateBlob": "{\"version\":1,\"tableState\":{\"columnPinning\":{\"left\":[\"#row_number#\"],\"right\":[]},\"columnSizing\":{},\"columnVisibility\":{}},\"settings\":{\"columns\":{}},\"syncTimestamp\":1774896626100}", - "filterBlob": null, - "queryPlanFiltersBlob": null, - "tableResultIndex": 0 - } - }, + "tableResultSettingsMap": {}, "title": "Function: caseData" } }, @@ -665,7 +702,7 @@ "outputs": [], "source": [ "df, df_audit = PP.legalRepDetails(silver_m1, bronze_countryFromAddress)\n", - "# display(df)" + "display(df)" ] }, { @@ -712,6 +749,27 @@ "# display(df)" ] }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "be4a6601-637b-45bd-b777-3aa6750c40d0", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "outputs": [], + "source": [ + "silver_m1.display()" + ] + }, { "cell_type": "code", "execution_count": 0, @@ -1035,7 +1093,10 @@ "cell_type": "markdown", "metadata": { "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, "inputWidgets": {}, "nuid": "cd194be7-d459-40fb-b7fb-e1754f659817", "showTitle": false, @@ -1141,7 +1202,10 @@ "cell_type": "markdown", "metadata": { "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, "inputWidgets": {}, "nuid": "8ee255ab-a380-4564-b454-2c10e65707a5", "showTitle": false, @@ -1157,7 +1221,10 @@ "cell_type": "markdown", "metadata": { "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, "inputWidgets": {}, "nuid": "3966a5d7-de3a-467e-b1ed-04809e2d256b", "showTitle": false, @@ -1513,7 +1580,10 @@ "cell_type": "markdown", "metadata": { "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, "inputWidgets": {}, "nuid": "89fdf409-78a7-47f2-bbcf-d4b001ea7051", "showTitle": false, @@ -1529,7 +1599,10 @@ "cell_type": "markdown", "metadata": { "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, "inputWidgets": {}, "nuid": "9edb7ba9-63ab-47ee-ac6a-245d6f2c0dc5", "showTitle": false, diff --git a/Databricks/ACTIVE/APPEALS/GOLD_PREPARE_FOR_HEARING_JSON.ipynb b/Databricks/ACTIVE/APPEALS/GOLD_PREPARE_FOR_HEARING_JSON.ipynb index 32dd2897..25fb6878 100644 --- a/Databricks/ACTIVE/APPEALS/GOLD_PREPARE_FOR_HEARING_JSON.ipynb +++ b/Databricks/ACTIVE/APPEALS/GOLD_PREPARE_FOR_HEARING_JSON.ipynb @@ -82,6 +82,7 @@ "source": [ "import importlib\n", "import shared_functions.paymentPending as PP\n", + "import shared_functions.paymentPendingDetained as PPD\n", "import shared_functions.appealSubmitted as APS\n", "import shared_functions.AwaitingEvidenceRespondant_a as AERa\n", "import shared_functions.listing as L\n", @@ -289,7 +290,8 @@ "bronze_hearing_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_hearing_centres\")\n", "bronze_derive_hearing_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_derive_hearing_centres\")\n", "bronze_interpreter_languages = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_interpreter_languages\")\n", - "bronze_listing_location = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_listing_location\")" + "bronze_listing_location = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_listing_location\")\n", + "bronze_detention_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_detention_centres\")" ] }, { @@ -553,7 +555,7 @@ }, "outputs": [], "source": [ - "df, df_audit = L.general(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres)\n", + "df, df_audit = L.general(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres,bronze_detention_centres)\n", "# display(df)" ] }, @@ -696,12 +698,12 @@ "from pyspark.sql.functions import col, lit, to_json, struct, concat, regexp_replace\n", "from datetime import datetime\n", "\n", - "def mainPrepareForHearing(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m4, silver_m6, silver_c, silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres, bronze_interpreter_languages, bronze_listing_location):\n", + "def mainPrepareForHearing(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m4, silver_m6, silver_c, silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres, bronze_interpreter_languages, bronze_listing_location,bronze_detention_centres):\n", " AppealState = \"prepareForHearing\"\n", " \n", " # Aggregate details\n", " appealType_df, appealType_df_audit = PP.appealType(silver_m1)\n", - " caseData_df, caseData_df_audit = PP.caseData(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres)\n", + " caseData_df, caseData_df_audit = PPD.caseData(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres,bronze_detention_centres)\n", " flagsLabels_df, flagsLabels_df_audit = PP.flagsLabels(silver_m1, silver_m2, silver_c)\n", " appellantDetails_df, appellantDetails_df_audit = AERa.appellantDetails(silver_m1, silver_m2, silver_c, bronze_countryFromAddress,bronze_HORef_cleansing)\n", " legalRepDetails_df, legalRepDetails_df_audit = PP.legalRepDetails(silver_m1, bronze_countryFromAddress)\n", @@ -709,15 +711,15 @@ " payment_df, payment_df_audit = APS.paymentType(silver_m1, silver_m4)\n", " homeOfficeDetails_df, homeOfficeDetails_df_audit = PP.homeOfficeDetails(silver_m1, silver_m2, silver_c, bronze_HORef_cleansing)\n", " remissionTypes_df, remissionTypes_df_audit = APS.remissionTypes(silver_m1, bronze_remissions, silver_m4)\n", - " sponsorDetails_df, sponsorDetails_df_audit = PP.sponsorDetails(silver_m1, silver_c)\n", - " general_df, general_df_audit = L.general(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres)\n", + " sponsorDetails_df, sponsorDetails_df_audit = PPD.sponsorDetails(silver_m1, silver_c)\n", + " general_df, general_df_audit = L.general(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres,bronze_detention_centres)\n", " generalDefault_df = L.generalDefault(silver_m1)\n", " documents_df, documents_df_audit = PFH.documents(silver_m1)\n", " hearingResponse_df, hearingResponse_df_audit = PFH.hearingResponse(silver_m1, silver_m3, silver_m6)\n", " hearingDetails_df, hearingDetails_df_audit = PFH.hearingDetails(silver_m1,silver_m3,bronze_listing_location)\n", " caseState_df, caseState_df_audit = PP.caseState(silver_m1, AppealState)\n", " hearingRequirements_df, hearingRequirements_df_audit = L.hearingRequirements(silver_m1, silver_m3, silver_c, bronze_interpreter_languages)\n", - " detainedState_df, detainedState_df_audit = PP.detainedState(silver_m1)\n", + " detainedState_df, detainedState_df_audit = PPD.detainedState(silver_m1,silver_m2,bronze_detention_centres)\n", " silver_segmentation_df = silver_segmentation\n", "\n", " # Join all aggregated data with Appeal Case Details\n", @@ -989,6 +991,7 @@ " bronze_derive_hearing_centres = dlt.read(\"bronze_derive_hearing_centres\")\n", " bronze_interpreter_languages = dlt.read(\"bronze_interpreter_languages\")\n", " bronze_listing_location = dlt.read(\"bronze_listing_location\")\n", + " bronze_detention_centres = dlt.read(\"bronze_detention_centres\")\n", " silver_segmentation = dlt.read(\"stg_segmentation_states\").filter(col(\"TargetState\") == lit(AppealState))\n", " except:\n", " silver_m1 = spark.table(\"hive_metastore.ariadm_active_appeals.silver_appealcase_detail\").filter(col(\"dv_targetState\") == lit(AppealState)).distinct()\n", @@ -1006,15 +1009,16 @@ " bronze_derive_hearing_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_derive_hearing_centres\")\n", " bronze_interpreter_languages = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_interpreter_languages\")\n", " bronze_listing_location = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_listing_location\")\n", + " bronze_detention_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_detention_centres\")\n", " silver_segmentation = spark.table(\"hive_metastore.ariadm_active_appeals.stg_segmentation_states\").filter(col(\"TargetState\") == lit(AppealState))\n", " bronze_ended_states = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_ended_states\")\n", " \n", - " df_final, df_audit = mainPrepareForHearing(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m4, silver_m6, silver_c, silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres, bronze_interpreter_languages, bronze_listing_location)\n", + " df_final, df_audit = mainPrepareForHearing(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m4, silver_m6, silver_c, silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres, bronze_interpreter_languages, bronze_listing_location,bronze_detention_centres)\n", " \n", " df_final = build_dq_rules_dependencies(df_final, silver_m1, silver_m2, silver_m3, silver_m4, silver_m6, silver_c,silver_h,\n", " bronze_countries_postal_lookup_df, bronze_HORef_cleansing, bronze_remissions,\n", " bronze_interpreter_languages, bronze_listing_location,bronze_ended_states,\n", - " bronze_hearing_centres, bronze_derive_hearing_centres)\n", + " bronze_hearing_centres, bronze_derive_hearing_centres,bronze_detention_centres)\n", " \n", " df_final = df_final.withColumn(\"is_valid\", expr(dq_rules))\n", "\n", @@ -1151,6 +1155,7 @@ " bronze_derive_hearing_centres = dlt.read(\"bronze_derive_hearing_centres\")\n", " bronze_interpreter_languages = dlt.read(\"bronze_interpreter_languages\")\n", " bronze_listing_location = spark.table(\"bronze_listing_location\")\n", + " bronze_detention_centres = dlt.read(\"bronze_detention_centres\")\n", " silver_segmentation = dlt.read(\"stg_segmentation_states\").filter(col(\"TargetState\") == lit(AppealState))\n", " \n", " except:\n", @@ -1169,10 +1174,11 @@ " bronze_derive_hearing_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_derive_hearing_centres\")\n", " bronze_interpreter_languages = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_interpreter_languages\")\n", " bronze_listing_location = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_listing_location\")\n", + " bronze_detention_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_detention_centres\")\n", " silver_segmentation = spark.table(\"hive_metastore.ariadm_active_appeals.stg_segmentation_states\").filter(col(\"TargetState\") == lit(AppealState))\n", "\n", " \n", - " df_final, df_audit = mainPrepareForHearing(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m4, silver_m6, silver_c,silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres, bronze_interpreter_languages,bronze_listing_location) \n", + " df_final, df_audit = mainPrepareForHearing(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m4, silver_m6, silver_c,silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres, bronze_interpreter_languages,bronze_listing_location,bronze_detention_centres) \n", "\n", " return df_audit" ] diff --git a/Databricks/ACTIVE/APPEALS/GOLD_REASON_FOR_APPEAL_SUBMITTED_JSON.ipynb b/Databricks/ACTIVE/APPEALS/GOLD_REASON_FOR_APPEAL_SUBMITTED_JSON.ipynb index 7a5eb209..0f5a0a6e 100644 --- a/Databricks/ACTIVE/APPEALS/GOLD_REASON_FOR_APPEAL_SUBMITTED_JSON.ipynb +++ b/Databricks/ACTIVE/APPEALS/GOLD_REASON_FOR_APPEAL_SUBMITTED_JSON.ipynb @@ -42,6 +42,7 @@ "source": [ "import importlib\n", "import shared_functions.paymentPending as PP\n", + "import shared_functions.paymentPendingDetained as PPD\n", "import shared_functions.appealSubmitted as APS\n", "import shared_functions.AwaitingEvidenceRespondant_a as AERa\n", "import shared_functions.AwaitingEvidenceRespondant_b as AERb\n", @@ -252,7 +253,8 @@ "bronze_countryFromAddress = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_countries_countryFromAddress\")\n", "bronze_HORef_cleansing = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_HORef_cleansing\")\n", "bronze_hearing_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_hearing_centres\")\n", - "bronze_derive_hearing_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_derive_hearing_centres\")" + "bronze_derive_hearing_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_derive_hearing_centres\")\n", + "bronze_detention_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_detention_centres\")" ] }, { @@ -516,7 +518,7 @@ }, "outputs": [], "source": [ - "df, df_audit = PP.general(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres)\n", + "df, df_audit = PPD.general(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres,bronze_detention_centres)\n", "# display(df)" ] }, @@ -615,14 +617,14 @@ "from pyspark.sql.functions import col, lit, to_json, struct, concat, regexp_replace\n", "from datetime import datetime\n", "\n", - "def mainReasonForAppealSubmitted(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m4, silver_c, silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres):\n", + "def mainReasonForAppealSubmitted(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m4, silver_c, silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres,bronze_detention_centres):\n", " AppealState = \"reasonsForAppealSubmitted\"\n", "\n", " silver_m1 = silver_m1.filter(col(\"dv_representation\") == \"AIP\")\n", "\n", " # Aggregate details\n", " AppealType_df, AppealType_df_audit = PP.appealType(silver_m1)\n", - " caseData_df, caseData_df_audit = PP.caseData(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres)\n", + " caseData_df, caseData_df_audit = PPD.caseData(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres,bronze_detention_centres)\n", " flagsLabels_df, flagsLabels_df_audit = PP.flagsLabels(silver_m1, silver_m2, silver_c)\n", " appellantDetails_df, appellantDetails_df_audit = AERa.appellantDetails(silver_m1, silver_m2, silver_c, bronze_countryFromAddress,bronze_HORef_cleansing)\n", " legalRepDetails_df, legalRepDetails_df_audit = PP.legalRepDetails(silver_m1, bronze_countryFromAddress)\n", @@ -630,13 +632,13 @@ " payment_df, payment_df_audit = APS.paymentType(silver_m1, silver_m4)\n", " homeOfficeDetails_df, homeOfficeDetails_df_audit = PP.homeOfficeDetails(silver_m1, silver_m2, silver_c, bronze_HORef_cleansing)\n", " remissionTypes_df, remissionTypes_df_audit = APS.remissionTypes(silver_m1, bronze_remissions, silver_m4)\n", - " sponsorDetails_df, sponsorDetails_df_audit = PP.sponsorDetails(silver_m1, silver_c)\n", - " general_df, general_df_audit = PP.general(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres)\n", + " sponsorDetails_df, sponsorDetails_df_audit = PPD.sponsorDetails(silver_m1, silver_c)\n", + " general_df, general_df_audit = PPD.general(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres,bronze_detention_centres)\n", " generalDefault_df = RFPAS.generalDefault(silver_m1)\n", " documents_df, documents_df_audit = AERb.documents(silver_m1)\n", " caseState_df, caseState_df_audit = PP.caseState(silver_m1, \"reasonsForAppealSubmitted\")\n", " hearingResponse_df, hearingResponse_df_audit = RFPAS.hearingResponse(silver_m1, silver_m3, silver_m6)\n", - " detainedState_df, detainedState_df_audit = PP.detainedState(silver_m1)\n", + " detainedState_df, detainedState_df_audit = PPD.detained(silver_m1,silver_m2,bronze_detention_centres)\n", " silver_segmentation_df = silver_segmentation\n", "\n", " # Join all aggregated data with Appeal Case Details\n", @@ -902,6 +904,7 @@ " bronze_derive_hearing_centres = dlt.read(\"bronze_derive_hearing_centres\")\n", " bronze_interpreter_languages = dlt.read(\"bronze_interpreter_languages\")\n", " bronze_listing_location = dlt.read(\"bronze_listing_location\")\n", + " bronze_detention_centres = dlt.read(\"bronze_detention_centres\")\n", " silver_segmentation = dlt.read(\"stg_segmentation_states\").filter(col(\"TargetState\") == lit(AppealState))\n", " except:\n", " silver_m1 = spark.table(\"hive_metastore.ariadm_active_appeals.silver_appealcase_detail\").filter(col(\"dv_targetState\") == lit(AppealState)).distinct()\n", @@ -919,15 +922,16 @@ " bronze_derive_hearing_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_derive_hearing_centres\")\n", " bronze_interpreter_languages = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_interpreter_languages\")\n", " bronze_listing_location = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_listing_location\")\n", + " bronze_detention_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_detention_centres\")\n", " silver_segmentation = spark.table(\"hive_metastore.ariadm_active_appeals.stg_segmentation_states\").filter(col(\"TargetState\") == lit(AppealState))\n", " bronze_ended_states = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_ended_states\")\n", " \n", - " df_final, df_audit = mainReasonForAppealSubmitted(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m4, silver_c, silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres)\n", + " df_final, df_audit = mainReasonForAppealSubmitted(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m4, silver_c, silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres,bronze_detention_centres)\n", " \n", " df_final = build_dq_rules_dependencies(df_final, silver_m1, silver_m2, silver_m3, silver_m4, silver_m6, silver_c,silver_h,\n", " bronze_countries_postal_lookup_df, bronze_HORef_cleansing, bronze_remissions,\n", " bronze_interpreter_languages, bronze_listing_location,bronze_ended_states,\n", - " bronze_hearing_centres, bronze_derive_hearing_centres)\n", + " bronze_hearing_centres, bronze_derive_hearing_centres,bronze_detention_centres)\n", " \n", " df_final = df_final.withColumn(\"is_valid\", expr(dq_rules))\n", "\n", @@ -1068,6 +1072,7 @@ " bronze_HORef_cleansing = dlt.read(\"bronze_HORef_cleansing\")\n", " bronze_hearing_centres = dlt.read(\"bronze_hearing_centres\")\n", " bronze_derive_hearing_centres = dlt.read(\"bronze_derive_hearing_centres\")\n", + " bronze_detention_centres = dlt.read(\"bronze_detention_centres\")\n", " silver_segmentation = dlt.read(\"stg_segmentation_states\").filter(col(\"TargetState\") == lit(AppealState))\n", " \n", " except:\n", @@ -1083,10 +1088,11 @@ " bronze_HORef_cleansing = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_HORef_cleansing\")\n", " bronze_hearing_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_hearing_centres\")\n", " bronze_derive_hearing_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_derive_hearing_centres\")\n", + " bronze_detention_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_detention_centres\")\n", " silver_segmentation = spark.table(\"hive_metastore.ariadm_active_appeals.stg_segmentation_states\").filter(col(\"TargetState\") == lit(AppealState))\n", "\n", " \n", - " df_final,df_audit = mainReasonForAppealSubmitted(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m4, silver_c, silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing,bronze_hearing_centres,bronze_derive_hearing_centres)\n", + " df_final,df_audit = mainReasonForAppealSubmitted(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m4, silver_c, silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing,bronze_hearing_centres,bronze_derive_hearing_centres,bronze_detention_centres)\n", "\n", " return df_audit" ] diff --git a/Databricks/ACTIVE/APPEALS/GOLD_REMITTED_JSON.ipynb b/Databricks/ACTIVE/APPEALS/GOLD_REMITTED_JSON.ipynb index f02ec896..f075dffc 100644 --- a/Databricks/ACTIVE/APPEALS/GOLD_REMITTED_JSON.ipynb +++ b/Databricks/ACTIVE/APPEALS/GOLD_REMITTED_JSON.ipynb @@ -82,6 +82,7 @@ "source": [ "import importlib\n", "import shared_functions.paymentPending as PP\n", + "import shared_functions.paymentPendingDetained as PPD\n", "import shared_functions.appealSubmitted as APS\n", "import shared_functions.AwaitingEvidenceRespondant_a as AERa\n", "import shared_functions.listing as L\n", @@ -295,7 +296,8 @@ "bronze_hearing_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_hearing_centres\")\n", "bronze_derive_hearing_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_derive_hearing_centres\")\n", "bronze_interpreter_languages = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_interpreter_languages\")\n", - "bronze_listing_location = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_listing_location\")" + "bronze_listing_location = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_listing_location\")\n", + "bronze_detention_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_detention_centres\")" ] }, { @@ -558,7 +560,7 @@ }, "outputs": [], "source": [ - "df, df_audit = FTD.general(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres)\n", + "df, df_audit = FTD.general(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres,bronze_detention_centres)\n", "# display(df)" ] }, @@ -789,12 +791,12 @@ "from pyspark.sql.functions import col, lit, to_json, struct, concat, regexp_replace\n", "from datetime import datetime\n", "\n", - "def mainRemitted(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m6, silver_c, silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres, bronze_interpreter_languages, bronze_listing_location):\n", + "def mainRemitted(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m6, silver_c, silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres, bronze_interpreter_languages, bronze_listing_location,bronze_detention_centres):\n", " AppealState = \"remitted\"\n", " \n", " # Aggregate details\n", " appealType_df, appealType_df_audit = PP.appealType(silver_m1)\n", - " caseData_df, caseData_df_audit = PP.caseData(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres)\n", + " caseData_df, caseData_df_audit = PPD.caseData(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres,bronze_detention_centres)\n", " flagsLabels_df, flagsLabels_df_audit = PP.flagsLabels(silver_m1, silver_m2, silver_c)\n", " appellantDetails_df, appellantDetails_df_audit = AERa.appellantDetails(silver_m1, silver_m2, silver_c, bronze_countryFromAddress,bronze_HORef_cleansing)\n", " legalRepDetails_df, legalRepDetails_df_audit = PP.legalRepDetails(silver_m1,bronze_countryFromAddress)\n", @@ -802,8 +804,8 @@ " payment_df, payment_df_audit = APS.paymentType(silver_m1, silver_m4)\n", " homeOfficeDetails_df, homeOfficeDetails_df_audit = PP.homeOfficeDetails(silver_m1, silver_m2, silver_c, bronze_HORef_cleansing)\n", " remissionTypes_df, remissionTypes_df_audit = APS.remissionTypes(silver_m1, bronze_remissions, silver_m4)\n", - " sponsorDetails_df, sponsorDetails_df_audit = PP.sponsorDetails(silver_m1, silver_c)\n", - " general_df, general_df_audit = FTD.general(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres)\n", + " sponsorDetails_df, sponsorDetails_df_audit = PPD.sponsorDetails(silver_m1, silver_c)\n", + " general_df, general_df_audit = FTD.general(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres,bronze_detention_centres)\n", " generalDefault_df = R.generalDefault(silver_m1)\n", " documents_df, documents_df_audit = R.documents(silver_m1,silver_m3)\n", " hearingResponse_df, hearingResponse_df_audit = PFH.hearingResponse(silver_m1, silver_m3, silver_m6)\n", @@ -814,7 +816,7 @@ " hearingActuals_df, hearingActuals_df_audit = DA.hearingActuals(silver_m3)\n", " ftpaDec_df, ftpa_df_audit = FTD.ftpa(silver_m3,silver_c)\n", " remittal_df, remittal_df_audit = R.remittal(silver_m3)\n", - " detainedState_df, detainedState_df_audit = PP.detainedState(silver_m1)\n", + " detainedState_df, detainedState_df_audit = PP.detained(silver_m1,silver_m2,bronze_detention_centres)\n", "\n", " silver_segmentation_df = silver_segmentation\n", "\n", @@ -1093,6 +1095,7 @@ " bronze_derive_hearing_centres = dlt.read(\"bronze_derive_hearing_centres\")\n", " bronze_interpreter_languages = dlt.read(\"bronze_interpreter_languages\")\n", " bronze_listing_location = spark.table(\"bronze_listing_location\")\n", + " bronze_detention_centres = dlt.read(\"bronze_detention_centres\")\n", " silver_segmentation = dlt.read(\"stg_segmentation_states\").filter(col(\"TargetState\") == lit(AppealState))\n", " \n", " except:\n", @@ -1110,16 +1113,17 @@ " bronze_derive_hearing_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_derive_hearing_centres\")\n", " bronze_interpreter_languages = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_interpreter_languages\")\n", " bronze_listing_location = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_listing_location\")\n", + " bronze_detention_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_detention_centres\")\n", " silver_segmentation = spark.table(\"hive_metastore.ariadm_active_appeals.stg_segmentation_states\").filter(col(\"TargetState\") == lit(AppealState))\n", " bronze_ended_states = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_ended_states\")\n", " \n", - " df_final, df_audit = mainRemitted(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m6, silver_c, silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres, bronze_interpreter_languages,bronze_listing_location)\n", + " df_final, df_audit = mainRemitted(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m6, silver_c, silver_h, bronze_remissions, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres, bronze_interpreter_languages,bronze_listing_location,bronze_detention_centres)\n", "\n", " \n", " df_final = build_dq_rules_dependencies(df_final, silver_m1, silver_m2, silver_m3, silver_m4, silver_m6, silver_c,silver_h,\n", " bronze_countries_postal_lookup_df, bronze_HORef_cleansing, bronze_remissions,\n", " bronze_interpreter_languages, bronze_listing_location,bronze_ended_states,\n", - " bronze_hearing_centres, bronze_derive_hearing_centres)\n", + " bronze_hearing_centres, bronze_derive_hearing_centres,bronze_detention_centres)\n", " \n", " df_final = df_final.withColumn(\"is_valid\", expr(dq_rules))\n", "\n", @@ -1255,6 +1259,7 @@ " bronze_derive_hearing_centres = dlt.read(\"bronze_derive_hearing_centres\")\n", " bronze_interpreter_languages = dlt.read(\"bronze_interpreter_languages\")\n", " bronze_listing_location = spark.table(\"bronze_listing_location\")\n", + " bronze_detention_centres = dlt.read(\"bronze_detention_centres\")\n", " silver_segmentation = dlt.read(\"stg_segmentation_states\").filter(col(\"TargetState\") == lit(AppealState))\n", " \n", " except:\n", @@ -1273,10 +1278,11 @@ " bronze_derive_hearing_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_derive_hearing_centres\")\n", " bronze_interpreter_languages = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_interpreter_languages\")\n", " bronze_listing_location = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_listing_location\")\n", + " bronze_detention_centres = spark.table(\"hive_metastore.ariadm_active_appeals.bronze_detention_centres\")\n", " silver_segmentation = spark.table(\"hive_metastore.ariadm_active_appeals.stg_segmentation_states\").filter(col(\"TargetState\") == lit(AppealState))\n", "\n", " \n", - " df_final, df_audit = mainRemitted(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m6, silver_c,silver_h, bronze_remissions_lookup_df, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres, bronze_interpreter_languages,bronze_listing_location)\n", + " df_final, df_audit = mainRemitted(silver_segmentation, silver_m1, silver_m2, silver_m3, silver_m6, silver_c,silver_h, bronze_remissions_lookup_df, bronze_countryFromAddress, bronze_HORef_cleansing, bronze_hearing_centres, bronze_derive_hearing_centres, bronze_interpreter_languages,bronze_listing_location,bronze_detention_centres)\n", "\n", " return df_audit" ] diff --git a/Databricks/ACTIVE/APPEALS/shared_functions/AwaitingEvidenceRespondant_a.py b/Databricks/ACTIVE/APPEALS/shared_functions/AwaitingEvidenceRespondant_a.py index 391def61..c06e09ef 100644 --- a/Databricks/ACTIVE/APPEALS/shared_functions/AwaitingEvidenceRespondant_a.py +++ b/Databricks/ACTIVE/APPEALS/shared_functions/AwaitingEvidenceRespondant_a.py @@ -1,5 +1,6 @@ from pyspark.sql.functions import col, struct, lit, concat, array from . import paymentPending as PP +from . import paymentPendingDetained as PPD ########################## @@ -7,7 +8,7 @@ ########################## def appellantDetails(silver_m1, silver_m2, silver_c, bronze_countryFromAddress, bronze_HORef_cleansing): - df_apellantDetails, df_audit_appellantDetails = PP.appellantDetails(silver_m1, silver_m2, silver_c, bronze_countryFromAddress, bronze_HORef_cleansing) + df_apellantDetails, df_audit_appellantDetails = PPD.appellantDetails(silver_m1, silver_m2, silver_c, bronze_countryFromAddress, bronze_HORef_cleansing) # Update column changeDirectionDueDateActionAvailable and add two new colums per mapping document df_apellantDetails = df_apellantDetails.withColumn("appellantFullName", concat(col("appellantGivenNames"), lit(" "), col("appellantFamilyName"))) diff --git a/Databricks/ACTIVE/APPEALS/shared_functions/DQRules.py b/Databricks/ACTIVE/APPEALS/shared_functions/DQRules.py index 93eb4437..0656ad9b 100644 --- a/Databricks/ACTIVE/APPEALS/shared_functions/DQRules.py +++ b/Databricks/ACTIVE/APPEALS/shared_functions/DQRules.py @@ -409,7 +409,7 @@ def build_dq_rules_dependencies(df_final, silver_m1, silver_m2, silver_m3, silve df_documents, df_documents_audit = E.documents(silver_m1,silver_m3) df_ftpa, df_ftpa_audit = E.ftpa(silver_m3,silver_c) - df_general, df_general_audit = E.general(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres) + df_general, df_general_audit = E.general(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres,bronze_detention_centres) df_generalDefault = E.generalDefault(silver_m1,silver_m3) df_hearingRequirements, df_hearingRequirements_audit = E.hearingRequirements(silver_m1, silver_m3, silver_c, bronze_interpreter_languages) df_hearingResponse, df_hearingResponse_audit = E.hearingResponse(silver_m1, silver_m3, silver_m6) @@ -489,7 +489,7 @@ def build_dq_rules_dependencies(df_final, silver_m1, silver_m2, silver_m3, silve silver_m1.alias("m1") .join(silver_m2.alias("m2"),on="CaseNo",how="left") .join(bronze_detention_centres.alias("det"),on="DetentionCentreId",how="left") - .select(col("m1.CaseNo"),col("m1.RemovalDate"),col("m2.PrisonRef"),col("m2.Detained"),col("m2.DetentionCentreId"), + .select(col("m1.CaseNo"),col("m1.RemovalDate"),col("m2.PrisonRef"),col("m2.Detained"),col("m2.DetentionCentreId").alias("DetentionCentreId"), *[col(f"det.{c}").alias(f"{c}_det") for c in bronze_detention_centres.columns ] diff --git a/Databricks/ACTIVE/APPEALS/shared_functions/decided_b.py b/Databricks/ACTIVE/APPEALS/shared_functions/decided_b.py index 4b36d277..ce61df7f 100644 --- a/Databricks/ACTIVE/APPEALS/shared_functions/decided_b.py +++ b/Databricks/ACTIVE/APPEALS/shared_functions/decided_b.py @@ -353,9 +353,9 @@ def ftpa(silver_m3,silver_c): ########## general ########### ################################################################ -def general(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres): +def general(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres,bronze_detention_centres): - general_df,general_audit = FSA.general(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres) + general_df,general_audit = FSA.general(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres,bronze_detention_centres) window_spec = Window.partitionBy("CaseNo").orderBy(col("StatusId").desc()) # Add row_number to get the row with the highest StatusId per CaseNo diff --git a/Databricks/ACTIVE/APPEALS/shared_functions/decision.py b/Databricks/ACTIVE/APPEALS/shared_functions/decision.py index e36403d8..86eca637 100644 --- a/Databricks/ACTIVE/APPEALS/shared_functions/decision.py +++ b/Databricks/ACTIVE/APPEALS/shared_functions/decision.py @@ -277,9 +277,9 @@ def generalDefault(silver_m1): ########## general ########### ################################################################ -def general(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres): +def general(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres,bronze_detention_centres): - general_df, general_audit = L.general(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres) + general_df, general_audit = L.general(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres,bronze_detention_centres) bundleFileNamePrefix_df = ( silver_m1.alias("m1") diff --git a/Databricks/ACTIVE/APPEALS/shared_functions/ended.py b/Databricks/ACTIVE/APPEALS/shared_functions/ended.py index 36edb109..71e5601b 100644 --- a/Databricks/ACTIVE/APPEALS/shared_functions/ended.py +++ b/Databricks/ACTIVE/APPEALS/shared_functions/ended.py @@ -1675,9 +1675,9 @@ def generalDefault(silver_m1,silver_m3): ################################################################ -def general(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres): +def general(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres,bronze_detention_centres): - general_df, general_audit = FSA.general(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres) + general_df, general_audit = FSA.general(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres,bronze_detention_centres) df = ( silver_m3 diff --git a/Databricks/ACTIVE/APPEALS/shared_functions/ftpa_submitted_a.py b/Databricks/ACTIVE/APPEALS/shared_functions/ftpa_submitted_a.py index 5e6381f9..419e0a99 100644 --- a/Databricks/ACTIVE/APPEALS/shared_functions/ftpa_submitted_a.py +++ b/Databricks/ACTIVE/APPEALS/shared_functions/ftpa_submitted_a.py @@ -288,9 +288,9 @@ def ftpa(silver_m3,silver_c): ########## general ########### ################################################################ -def general(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres): +def general(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres,bronze_detention_centres): - general_df,general_audit = D.general(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres) + general_df,general_audit = D.general(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres,bronze_detention_centres) window_spec = Window.partitionBy("CaseNo").orderBy(col("StatusId").desc()) # Add row_number to get the row with the highest StatusId per CaseNo diff --git a/Databricks/ACTIVE/APPEALS/shared_functions/listing.py b/Databricks/ACTIVE/APPEALS/shared_functions/listing.py index adfe5bba..87a5f4fd 100644 --- a/Databricks/ACTIVE/APPEALS/shared_functions/listing.py +++ b/Databricks/ACTIVE/APPEALS/shared_functions/listing.py @@ -4,6 +4,7 @@ from pyspark.sql.window import Window from . import paymentPending as PP +from . import paymentPendingDetained as PPD from . import AwaitingEvidenceRespondant_b as AERb @@ -444,8 +445,8 @@ def hearingRequirements(silver_m1, silver_m3, silver_c, bronze_interpreter_langu return df_hearingRequirements, df_audit_hearingRequirements -def general(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres): - df, df_audit = PP.general(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres) +def general(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres,bronze_detention_centres): + df, df_audit = PPD.general(silver_m1, silver_m2, silver_m3, silver_h, bronze_hearing_centres, bronze_derive_hearing_centres,bronze_detention_centres) df_representation = silver_m1.select("CaseNo", "dv_representation", "lu_appealType") df = df.join(df_representation, on="CaseNo", how="left") diff --git a/Databricks/ACTIVE/APPEALS/shared_functions/paymentPending.py b/Databricks/ACTIVE/APPEALS/shared_functions/paymentPending.py index decb87fc..38f3d0fe 100644 --- a/Databricks/ACTIVE/APPEALS/shared_functions/paymentPending.py +++ b/Databricks/ACTIVE/APPEALS/shared_functions/paymentPending.py @@ -1055,9 +1055,9 @@ def legalRepDetails(silver_m1, bronze_countryFromAddress): col("CaseRep_Address4"), col("CaseRep_Address5") ).alias("AddressLine1"), - col("CaseRep_Address2").alias("AddressLine2"), - col("CaseRep_Address3").alias("PostTown"), - col("CaseRep_Address4").alias("County"), + coalesce(col("CaseRep_Address2"),col("CaseRep_Address3"),col("CaseRep_Address4"),col("CaseRep_Address5")).alias("AddressLine2"), + coalesce(col("CaseRep_Address3"),col("CaseRep_Address4"),col("CaseRep_Address5")).alias("PostTown"), + coalesce(col("CaseRep_Address4"),col("CaseRep_Address5")).alias("County"), col("CaseRep_Address5").alias("Country"), col("CaseRep_Postcode").alias("PostCode") ) @@ -1430,8 +1430,11 @@ def appellantDetails(silver_m1, silver_m2, silver_c, bronze_countryFromAddress, # appellantAddress logic # Only include if CategoryIdList contains 37 and conditions - include_appellant_address = (conditions & expr("array_contains(CategoryIdList, 37)") & + include_appellant_address = (conditions & (expr("array_contains(CategoryIdList, 37)") | expr("array_contains(CategoryIdList, 38)")) & (coalesce(col("Appellant_Address1"), col("Appellant_Address2"), col("Appellant_Address3"), col("Appellant_Address4"), col("Appellant_Address5"), col("Appellant_Postcode")).isNotNull())) + # include_appellant_out_of_uk_address = (conditions & expr("array_contains(CategoryIdList, 38)") & + # (coalesce(col("Appellant_Address1"), col("Appellant_Address2"), col("Appellant_Address3"), col("Appellant_Address4"), col("Appellant_Address5"), col("Appellant_Postcode")).isNotNull())) + appellant_address_struct = when( include_appellant_address, struct( @@ -1452,6 +1455,7 @@ def appellantDetails(silver_m1, silver_m2, silver_c, bronze_countryFromAddress, ) ).otherwise(None) + # addressLine1AdminJ: mandatory for OOC, fallback through address fields address_line1_adminj_expr = when( conditions & expr("array_contains(CategoryIdList, 38)"), @@ -1634,6 +1638,19 @@ def appellantDetails(silver_m1, silver_m2, silver_c, bronze_countryFromAddress, ooc_appeal_adminj_expr.alias("oocAppealAdminJ"), appellant_has_fixed_address_expr.alias("appellantHasFixedAddress"), appellant_has_fixed_address_adminj_expr.alias("appellantHasFixedAddressAdminJ"), + + # when(include_appellant_address, appellant_address_struct) + # .when( + # include_appellant_out_of_uk_address, + # struct( + # address_line1_adminj_expr.alias("AddressLine1"), + # address_line2_adminj_expr.alias("AddressLine2"), + # address_line3_adminj_expr.alias("PostTown") + # ) + # ) + # .otherwise(None) + # .alias("appellantAddress"), + appellant_address_struct.alias("appellantAddress"), address_line1_adminj_expr.alias("addressLine1AdminJ"), address_line2_adminj_expr.alias("addressLine2AdminJ"), @@ -2126,7 +2143,10 @@ def homeOfficeDetails(silver_m1, silver_m2, silver_c, bronze_HORef_cleansing): date_entry_clearance_decision_expr.alias("dateEntryClearanceDecision"), home_office_reference_number_expr.alias("homeOfficeReferenceNumber"), gwf_reference_number_expr.alias("gwfReferenceNumber"), - lit("Yes").alias("isHomeOfficeIntegrationEnabled"), + when(col("dv_CCDAppealType").isin("RP", "PA"), "Yes") + .otherwise(None) + .alias("isHomeOfficeIntegrationEnabled"), + # lit("Yes").alias("isHomeOfficeIntegrationEnabled"), lit("Yes").alias("homeOfficeNotificationsEligible") # col("HORef"), # col("FCONumber"), @@ -2171,8 +2191,8 @@ def homeOfficeDetails(silver_m1, silver_m2, silver_c, bronze_HORef_cleansing): lit("yes").alias("gwfReferenceNumber_Transformed"), #isHomeOfficeIntegrationEnabled - ARIADM-797 - array(struct(*common_inputFields, lit("isHomeOfficeIntegrationEnabled"))).alias("isHomeOfficeIntegrationEnabled_inputFields"), - array(struct(*common_inputValues, lit("null"))).alias("isHomeOfficeIntegrationEnabled_inputValues"), + array(struct(*common_inputFields, lit("isHomeOfficeIntegrationEnabled"), lit("dv_CCDAppealType"))).alias("isHomeOfficeIntegrationEnabled_inputFields"), + array(struct(*common_inputValues, lit("null"),col("dv_CCDAppealType"))).alias("isHomeOfficeIntegrationEnabled_inputValues"), col("content.isHomeOfficeIntegrationEnabled"), lit("no").alias("isHomeOfficeIntegrationEnabled_Transformation"),