From 6fbae798c2d9653129cf1719f9dfadbd4a4372eb Mon Sep 17 00:00:00 2001 From: Michael Wood Date: Tue, 21 Jan 2025 14:07:25 +0000 Subject: [PATCH] additional_data: codelist: Add geoCodeType support Revert "additional_data: codelist: Remove changes needed to support geoCodeType" This reverts commit a948b84d85c14e61ad75da3193e81341febc3eac. --- .../additional_data/sources/codelist_code.py | 58 ++++++++++++++++--- .../test_additional_data_codelist_code.py | 8 +++ 2 files changed, 59 insertions(+), 7 deletions(-) diff --git a/datastore/additional_data/sources/codelist_code.py b/datastore/additional_data/sources/codelist_code.py index e66ca1e2..0ecfe28a 100644 --- a/datastore/additional_data/sources/codelist_code.py +++ b/datastore/additional_data/sources/codelist_code.py @@ -9,8 +9,8 @@ "https://raw.githubusercontent.com/ThreeSixtyGiving/standard/main/codelists/grantToIndividualsReason.csv", "https://raw.githubusercontent.com/ThreeSixtyGiving/standard/main/codelists/regrantType.csv", "https://raw.githubusercontent.com/ThreeSixtyGiving/standard/main/codelists/locationScope.csv", + "https://raw.githubusercontent.com/ThreeSixtyGiving/standard/main/codelists/geoCodeType.csv", # These codelists aren't yet processed - # "https://raw.githubusercontent.com/ThreeSixtyGiving/standard/main/codelists/geoCodeType.csv", # "https://raw.githubusercontent.com/ThreeSixtyGiving/standard/main/codelists/countryCode.csv", # "https://raw.githubusercontent.com/ThreeSixtyGiving/standard/main/codelists/currency.csv", ] @@ -35,12 +35,21 @@ def import_codelists(self): r.iter_lines(decode_unicode=True), delimiter="," ) for value in file_data: - CodelistCode.objects.create( - code=value["Code"], - title=value["Title"], - description=value["Description"], - list_name=list_name, - ) + # In https://github.com/ThreeSixtyGiving/standard/blob/main/codelists/geoCodeType.csv + # we have non unique codes with differing descriptions. We have to just take the first + # one we come accross to avoid an integrity error on the unique constraints. + # https://github.com/ThreeSixtyGiving/standard/issues/391 + try: + CodelistCode.objects.get( + code=value["Code"], list_name=list_name + ) + except CodelistCode.DoesNotExist: + CodelistCode.objects.create( + code=value["Code"], + title=value["Title"], + description=value["Description"], + list_name=list_name, + ) def update_additional_data(self, grant, source_file, additional_data): # check All the fields in the grant data that use codelists and make additional data field versions of them @@ -50,6 +59,9 @@ def update_additional_data(self, grant, source_file, additional_data): grantPurpose = [] regrantType = "" locationScope = "" + beneficiary_geoCodeTypes = [] + recipient_organization_geoCodeType = "" + funding_organization_geoCodeType = "" try: code = grant["toIndividualsDetails"]["primaryGrantReason"] @@ -94,6 +106,33 @@ def update_additional_data(self, grant, source_file, additional_data): except (KeyError, CodelistCode.DoesNotExist): pass + for location in grant["beneficiaryLocation"]: + try: + code = location["geoCodeType"] + if code_title := CodelistCode.objects.get( + code=location["geoCodeType"], list_name="geoCodeType" + ).title: + beneficiary_geoCodeTypes.append(code_title) + + except (KeyError, IndexError, CodelistCode.DoesNotExist): + continue + + try: + code = grant["fundingOrganization"][0]["location"][0]["geoCodeType"] + funding_organization_geoCodeType = CodelistCode.objects.get( + code=code, list_name="geoCodeType" + ).title + except (KeyError, IndexError, CodelistCode.DoesNotExist): + pass + + try: + code = grant["recipientOrganization"][0]["location"][0]["geoCodeType"] + recipient_organization_geoCodeType = CodelistCode.objects.get( + code=code, list_name="geoCodeType" + ).title + except (KeyError, IndexError, CodelistCode.DoesNotExist): + pass + additional_data["codeListLookup"] = { "toIndividualsDetails": { "primaryGrantReason": primaryGrantReason, @@ -102,4 +141,9 @@ def update_additional_data(self, grant, source_file, additional_data): }, "regrantType": regrantType, "locationScope": locationScope, + "geoCodeType": { + "beneficiaryLocations": beneficiary_geoCodeTypes, + "recipientOrganization0": recipient_organization_geoCodeType, + "fundingOrganization0": funding_organization_geoCodeType, + }, } diff --git a/datastore/tests/test_additional_data_codelist_code.py b/datastore/tests/test_additional_data_codelist_code.py index e1714c4e..969feb31 100644 --- a/datastore/tests/test_additional_data_codelist_code.py +++ b/datastore/tests/test_additional_data_codelist_code.py @@ -16,6 +16,9 @@ def test_code_list(self): }, "regrantType": "FRG010", "locationScope": "GLS040", + "fundingOrganization": [{"location": [{"geoCodeType": "CTY"}]}], + "recipientOrganization": [{"location": [{"geoCodeType": "LONB"}]}], + "beneficiaryLocation": [{"geoCodeType": "MD"}], } source_file = {} @@ -31,6 +34,11 @@ def test_code_list(self): }, "regrantType": "Common Regrant", "locationScope": "Subnational region", + "geoCodeType": { + "beneficiaryLocations": ["Metropolitan Districts"], + "recipientOrganization0": "London Boroughs", + "fundingOrganization0": "Counties", + }, } }