From a876b45ec4315c271df66847fa6f4422d5f3171d Mon Sep 17 00:00:00 2001 From: Diana Boiangiu Date: Fri, 9 Jan 2026 16:59:35 +0200 Subject: [PATCH 1/2] Add Country modules scripts/Refactor import_resources_v2 --- core/admin/__init__.py | 1 + core/admin/base.py | 15 + core/admin/country.py | 2 +- core/api/tests/conftest.py | 28 +- core/api/views/countries.py | 2 +- .../import_project_resources_v2.py | 662 ------------------ core/import_data/utils.py | 1 + core/import_data_v2/__init__.py | 0 .../import_project_resources_v2.py | 112 +++ .../countries/MLF Countries and Regions.xlsx | Bin 0 -> 8508 bytes .../projects}/ClusterTypeSectorLinks.json | 0 .../projects}/Fields_24_10_2025.json | 0 .../resources/projects/modules.json | 0 .../project_clusters_06_05_2025.xlsx | Bin .../project_specific_fields_24_10_2025.xlsx | Bin .../project_specific_fields_27_10_2025.xlsx | Bin .../project_submission_statuses.json | 0 .../projects}/tbSector_15_10_2025.json | 0 .../projects}/tbSubsector_06_05_2025.json | 0 .../projects}/tbTypeOfProject_06_05_2025.json | 0 .../resources/users/groups.json | 0 .../resources/users/permissions.json | 0 core/import_data_v2/scripts/__init__.py | 0 .../scripts/clean_up_countries.py | 58 ++ .../scripts/clean_up_project_statuses.py | 67 ++ .../generate_new_cluster_type_sector_file.py | 45 ++ .../import_cluster_type_sector_links.py | 58 ++ core/import_data_v2/scripts/import_fields.py | 214 ++++++ core/import_data_v2/scripts/import_modules.py | 20 + .../scripts/import_project_clusters.py | 59 ++ .../scripts/import_project_type.py | 36 + .../scripts/import_sector_subsector.py | 103 +++ .../commands/import_resources_v2.py | 4 +- .../commands/import_user_permissions.py | 8 +- .../migrations/0268_module_country_modules.py | 35 + core/models/base.py | 9 + core/models/country.py | 74 +- 37 files changed, 864 insertions(+), 749 deletions(-) create mode 100644 core/admin/base.py delete mode 100644 core/import_data/import_project_resources_v2.py create mode 100644 core/import_data_v2/__init__.py create mode 100644 core/import_data_v2/import_project_resources_v2.py create mode 100644 core/import_data_v2/resources/countries/MLF Countries and Regions.xlsx rename core/{import_data/resources/projects_v2 => import_data_v2/resources/projects}/ClusterTypeSectorLinks.json (100%) rename core/{import_data/resources/projects_v2 => import_data_v2/resources/projects}/Fields_24_10_2025.json (100%) create mode 100644 core/import_data_v2/resources/projects/modules.json rename core/{import_data/resources/projects_v2 => import_data_v2/resources/projects}/project_clusters_06_05_2025.xlsx (100%) rename core/{import_data/resources/projects_v2 => import_data_v2/resources/projects}/project_specific_fields_24_10_2025.xlsx (100%) rename core/{import_data/resources/projects_v2 => import_data_v2/resources/projects}/project_specific_fields_27_10_2025.xlsx (100%) rename core/{import_data/resources/projects_v2 => import_data_v2/resources/projects}/project_submission_statuses.json (100%) rename core/{import_data/resources/projects_v2 => import_data_v2/resources/projects}/tbSector_15_10_2025.json (100%) rename core/{import_data/resources/projects_v2 => import_data_v2/resources/projects}/tbSubsector_06_05_2025.json (100%) rename core/{import_data/resources/projects_v2 => import_data_v2/resources/projects}/tbTypeOfProject_06_05_2025.json (100%) rename core/{import_data => import_data_v2}/resources/users/groups.json (100%) rename core/{import_data => import_data_v2}/resources/users/permissions.json (100%) create mode 100644 core/import_data_v2/scripts/__init__.py create mode 100644 core/import_data_v2/scripts/clean_up_countries.py create mode 100644 core/import_data_v2/scripts/clean_up_project_statuses.py create mode 100644 core/import_data_v2/scripts/generate_new_cluster_type_sector_file.py create mode 100644 core/import_data_v2/scripts/import_cluster_type_sector_links.py create mode 100644 core/import_data_v2/scripts/import_fields.py create mode 100644 core/import_data_v2/scripts/import_modules.py create mode 100644 core/import_data_v2/scripts/import_project_clusters.py create mode 100644 core/import_data_v2/scripts/import_project_type.py create mode 100644 core/import_data_v2/scripts/import_sector_subsector.py create mode 100644 core/migrations/0268_module_country_modules.py diff --git a/core/admin/__init__.py b/core/admin/__init__.py index 299e4eb4d..3e79381af 100644 --- a/core/admin/__init__.py +++ b/core/admin/__init__.py @@ -1,5 +1,6 @@ from .adm import * from .agency import * +from .base import * from .blend import * from .business_plan import * from .country import * diff --git a/core/admin/base.py b/core/admin/base.py new file mode 100644 index 000000000..0a2dc5d29 --- /dev/null +++ b/core/admin/base.py @@ -0,0 +1,15 @@ +from django.contrib import admin + +from core.models.base import Module + + +@admin.register(Module) +class AgencyAdmin(admin.ModelAdmin): + search_fields = [ + "name", + ] + list_display = [ + "code", + "name", + "description", + ] diff --git a/core/admin/country.py b/core/admin/country.py index db450816d..1720c9a55 100644 --- a/core/admin/country.py +++ b/core/admin/country.py @@ -14,7 +14,7 @@ class CountryAdmin(admin.ModelAdmin): "abbr", "abbr_alt", ] - list_filter = ["location_type"] + list_filter = ["location_type", "modules"] def get_list_display(self, request): exclude = [ diff --git a/core/api/tests/conftest.py b/core/api/tests/conftest.py index 4cff8fc23..13c8b7079 100644 --- a/core/api/tests/conftest.py +++ b/core/api/tests/conftest.py @@ -63,6 +63,7 @@ from core.models.adm import AdmRecordArchive from core.models.business_plan import BusinessPlan from core.models.country_programme_archive import CPReportArchive +from core.models.base import Module from core.utils import get_project_sub_code # pylint: disable=C0302,W0613 @@ -72,6 +73,13 @@ def user(): return UserFactory(username="FlorinSalam", email="salam@reggaeton.ta") +@pytest.fixture +def project_module(): + return Module.objects.get_or_create(name="Projects", code="Projects")[0] + +@pytest.fixture +def business_plan_module(): + return Module.objects.get_or_create(name="Business Plans", code="BP")[0] @pytest.fixture def secretariat_user(): @@ -282,19 +290,23 @@ def apr_mlfs_full_access_user(): @pytest.fixture -def new_country(): - return CountryFactory.create(iso3="NwC") +def new_country(project_module, business_plan_module): + country = CountryFactory.create(iso3="NwC") + country.modules.add(project_module, business_plan_module) + return country @pytest.fixture -def country_europe(): - return CountryFactory(name="Europe", location_type=Country.LocationType.REGION) - +def country_europe(project_module, business_plan_module): + country = CountryFactory(name="Europe", location_type=Country.LocationType.REGION) + country.modules.add(project_module, business_plan_module) + return country @pytest.fixture -def country_ro(): - return CountryFactory.create(name="Romania", iso3="ROM") - +def country_ro(project_module, business_plan_module): + country = CountryFactory.create(name="Romania", iso3="ROM") + country.modules.add(project_module, business_plan_module) + return country @pytest.fixture def country_viewer_user(country_ro): diff --git a/core/api/views/countries.py b/core/api/views/countries.py index 53c809976..65b1de21d 100644 --- a/core/api/views/countries.py +++ b/core/api/views/countries.py @@ -32,7 +32,7 @@ def get_queryset(self): if values_exclusive_for == "projects": queryset = queryset.filter( - location_type="Country", + modules__code="Projects", ) elif values_exclusive_for == "business_plan": queryset = Country.get_business_plan_countries() diff --git a/core/import_data/import_project_resources_v2.py b/core/import_data/import_project_resources_v2.py deleted file mode 100644 index 2151156c9..000000000 --- a/core/import_data/import_project_resources_v2.py +++ /dev/null @@ -1,662 +0,0 @@ -import json -import logging -import pandas as pd - -from django.db import transaction - -from core.import_data.utils import ( - IMPORT_RESOURCES_DIR, -) - -from core.models.group import Group -from core.models.project_metadata import ( - ProjectCluster, - ProjectSpecificFields, - ProjectField, - ProjectSector, - ProjectStatus, - ProjectSubmissionStatus, - ProjectSubSector, - ProjectType, -) -from core.models.project import Project - -logger = logging.getLogger(__name__) - -# pylint: disable=C0301 - -SUBSTANCE_FIELDS = [ - "Substance - baseline technology", - "Replacement technology/ies", - "Phase out (CO2-eq t)", - "Phase out (ODP t)", - "Phase out (Mt)", - "Ods type", - "Sort order", -] - -NEW_SUBSECTORS = [ - { - "SEC": "OTH", - "SUBSECTOR": "Policy paper", - "SORT_SUBSECTOR": 99, - }, - { - "SEC": "KIP", - "SUBSECTOR": "Preparation of project proposal", - "SORT_SUBSECTOR": 99, - }, - { - "SEC": "FFI", - "SUBSECTOR": "Servicing Fire Protections Systems", - "SORT_SUBSECTOR": 99, - }, - { - "SEC": "AC", - "SUBSECTOR": "Compressor", - "SORT_SUBSECTOR": 99, - }, -] -OUTDATED_SUBSECTORS = { - "Filling plant", - "Demonstration", - "Regulations", - "Information exchange", - "Technical assistance/support", - "Training programme/workshop", - "Flexible PU", - "Several PU foam", - "Polyol production", - "Agency programme", - "Ozone unit support", - "Process conversion", - "Project monitoring and coordination unit (PMU)", - "Domestic/commercial refrigeration (refrigerant)", - "Multiple-subsectors", -} - - -NEW_STATUSES = [ - { - "STATUS": "N/A", - "STATUS_CODE": "NA", - }, - { - "STATUS": "Unknow", - "STATUS_CODE": "UNK", - }, -] - - -FIELDS_WITH_ACTUAL_VALUES = [ - "Total number of technicians trained", - "Number of female technicians trained", - "Total number of trainers trained", - "Number of female trainers trained", - "Total number of technicians certified", - "Number of female technicians certified", - "Number of training institutions newly  assisted", - "Number of tools sets distributed", - "Total number of customs officers trained", - "Number of female customs officers trained", - "Total number of NOU personnel supported", - "Number of female NOU personnel supported", - "Certification system for technicians established or further enhanced (yes or no)", - "Operation of recovery and recycling scheme (yes or no)", - "Operation of reclamation scheme (yes or no)", - "Establishment or upgrade of Import/export licensing (yes or no)", - "Establishment of quota systems (yes or no)", - "Ban of equipment (number)", - "Ban of substances (number)", - "kWh/year saved", - "MEPS developed for domestic refrigeration (yes/no)", - "MEPS developed for commercial refrigeration (yes/no)", - "MEPS developed for residential air-conditioning (yes/no)", - "MEPS developed for commercial AC (yes/no)", - "Capacity building programmes (checklist (yes/no) for technicians, end-users, operators, consultants, procurement officers and other Government entities)", - "EE demonstration project included (yes/no)", - "Quantity of controlled substances destroyed (M t)", - "Quantity of controlled substances destroyed (CO2-eq t)", - "Checklist of regulations or policies enacted", - "Quantity of HFC-23 by-product (Generated)", - "Quantity of HFC-23 by-product (by product generation rate)", - "Quantity of HFC-23 by-product (Destroyed)", - "Quantity of HFC-23 by-product (Emitted)", - "Number of Production Lines assisted", - "Number of enterprises assisted", - "Number of enterprises", - "Aggregated consumption", - "Cost effectiveness (US$/ Kg)", - "Cost effectiveness (US$/ CO2-eq)", -] - - -def import_project_clusters(file_path): - """ - Import project clusters from file - Please make sure that the file has the correct extention - (xls, xlsx, xlsm, xlsb, odf, ods, odt) - - @param file_path = str (file path for import file) - """ - - df = pd.read_excel(file_path).fillna("") - - for index, row in df.iterrows(): - if row["Action"] == "Outdated": - continue - if row["Action"] == "Rename": - ProjectCluster.objects.filter(name=row["Old name"]).update(name=row["Name"]) - - production = False - if row["Production"] == "Y": - production = True - elif row["Production"] == "Both": - production = None - - # get annex groups - annex_groups = [] - if row["Annex groups"]: - annex_groups_name_alt = row["Annex groups"].split(",") - annex_groups = Group.objects.filter(name__in=annex_groups_name_alt) - if annex_groups.count() != len(annex_groups_name_alt): - logger.warning( - f"⚠️ Some annex groups not found for cluster {row['Name']}" - ) - cluster_data = { - "name": row["Name"], - "code": row["Acronym"], - "category": row["Category"].upper(), - "group": row["Dashboard group"], - "production": production, - "sort_order": index, - } - - cluster, _ = ProjectCluster.objects.update_or_create( - name=cluster_data["name"], defaults=cluster_data - ) - if annex_groups: - cluster.annex_groups.set(annex_groups) - - -def clean_up_project_statuses(): - """ - Clean up project statuses - Remove outdated statuses and add new ones - """ - # remove Unknown status only if there are no projects with this status - - if Project.objects.really_all().filter(status__code="UNK").exists(): - logger.warning( - "⚠️ Cannot remove 'Unknown' status, there are projects with this status." - ) - else: - ProjectStatus.objects.filter(code="UNK").delete() - - # change the status 'New submission' into 'N/A' and delete status 'New submission' - - new_submission_status, _ = ProjectStatus.objects.update_or_create( - name="N/A", - defaults={ - "code": "NA", - }, - ) - Project.objects.really_all().filter(status__code="NEWSUB").update( - status=new_submission_status - ) - ProjectStatus.objects.filter(code="NEWSUB").delete() - - # change the status 'Newly approved' into 'Ongoing' and delete status 'Newly approved' - - on_going_status = ProjectStatus.objects.filter(name="Ongoing").first() - Project.objects.really_all().filter(status__code="NEW").update( - status=on_going_status - ) - ProjectStatus.objects.filter(code="NEW").delete() - - -def import_project_type(file_path): - """ - Import project type from file - - @param file_path = str (file path for import file) - """ - with open(file_path, "r", encoding="utf8") as f: - types_json = json.load(f) - - # add other types that are not in the file - for type_json in types_json: - if type_json.get("ACTION", None) == "RENAME": - ProjectType.objects.filter(name=type_json["OLD_NAME"]).update( - name=type_json["TYPE_PRO"] - ) - else: - type_data = { - "code": type_json["TYPE"], - "name": type_json["TYPE_PRO"], - "sort_order": type_json["SORT_TYPE"], - } - ProjectType.objects.update_or_create( - name=type_data["name"], defaults=type_data - ) - - -def import_sector(file_path): - """ - Import sectors and subsectors from file - - @param file_path = str (file path for import file) - """ - - with open(file_path, "r", encoding="utf8") as f: - sectors_json = json.load(f) - - for sector_json in sectors_json: - - if sector_json.get("ACTION", None) == "RENAME": - ProjectSector.objects.filter(name=sector_json["OLD_NAME"]).update( - name=sector_json["SECTOR"] - ) - sector_data = { - "name": sector_json["SECTOR"].strip(), - "code": sector_json["SEC"].strip(), - "sort_order": sector_json["SORT_SECTOR"], - } - ProjectSector.objects.update_or_create( - code=sector_data["code"], defaults=sector_data - ) - - -def import_subsector(file_path): - """ - Import sectors and subsectors from file - Please make sure that the file has the correct extention - (xls, xlsx, xlsm, xlsb, odf, ods, odt) - - @param file_path = str (file path for import file) - """ - - with open(file_path, "r", encoding="utf8") as f: - subsectors_json = json.load(f) - - for subsector_json in subsectors_json: - subsector_name = subsector_json["SUBSECTOR"].strip() - if subsector_json.get("ACTION", None) == "RENAME": - project_sub_sector = ProjectSubSector.objects.filter( - name=subsector_json["OLD_NAME"] - ) - if subsector_json.get("OLD_SEC", None): - project_sub_sector = project_sub_sector.filter( - sector__code=subsector_json["OLD_SEC"] - ) - project_sub_sector.update(name=subsector_name) - - # get sector - sector = ProjectSector.objects.filter(code=subsector_json["SEC"]).first() - if not sector: - logger.warning( - f"⚠️ {subsector_json['SEC']} sector not found => {subsector_json['SUBSECTOR']} not imported" - ) - continue - - # set subsector data - subsector_code = ( - subsector_json["CODE_SUBSECTOR"].strip() - if subsector_json.get("CODE_SUBSECTOR") - else None - ) - subsector_data = { - "name": subsector_json["SUBSECTOR"].strip(), - "code": subsector_code, - "sector": sector, - "sort_order": subsector_json["SORT_SUBSECTOR"], - } - - project_sub_sector = ProjectSubSector.objects.filter( - name=subsector_data["name"] - ) - if project_sub_sector.exists(): - project_sub_sector.update( - name=subsector_data["name"], - code=subsector_data["code"], - sector=subsector_data["sector"], - sort_order=subsector_data["sort_order"], - ) - else: - ProjectSubSector.objects.create( - name=subsector_data["name"], - code=subsector_data["code"], - sector=subsector_data["sector"], - sort_order=subsector_data["sort_order"], - ) - - -def import_project_specific_fields(file_path): - """ - Import project clusters from file - Please make sure that the file has the correct extention - (xls, xlsx, xlsm, xlsb, odf, ods, odt) - - @param file_path = str (file path for import file) - """ - - def _clean_up_field_name(field_name, mya=False): - """ - Clean up field name - """ - mya_clean_up = { - "Cost effectiveness (US$/ CO2-ep) (MYA)": "Cost effectiveness (US$/ CO2-eq) (MYA)", - "Cost effectiveness (US$/ CO2-ep)": "Cost effectiveness (US$/ CO2-eq) (MYA)", - "Cost effectiveness (US$/ CO2-eq)": "Cost effectiveness (US$/ CO2-eq) (MYA)", - "Aggregated consumption": "Aggregated consumption (MYA)", - "Cost effectiveness (US$/ Kg)": "Cost effectiveness (US$/ Kg) (MYA)", - "Number of enterprises assisted": "Number of enterprises assisted (MYA)", - "Number of enterprises": "Number of enterprises (MYA)", - "Number of Production Lines assisted": "Number of Production Lines assisted (MYA)", - } - - individual_field_clean_up = { - "Cost effectiveness (US$/ CO2-ep)": "Cost effectiveness (US$/ CO2-eq)", - "Cost effectiveness (US$/ CO2-ep) (MYA)": "Cost effectiveness (US$/ CO2-eq)", - "Phase out (Mt) (MYA)": "Phase out (Mt)", - "Phase out (M t)": "Phase out (Mt)", - "Phase out (CO2-eq t) (MYA)": "Phase out (CO2-eq t)", - "Cost effectiveness (US$/ CO2-eq)": "Cost effectiveness (US$/ CO2-eq)", - "Phase out (ODP t) (MYA)": "Phase out (ODP t)", - } - if mya: - if field_name in mya_clean_up: - return mya_clean_up[field_name] - else: - if field_name in individual_field_clean_up: - return individual_field_clean_up[field_name] - return field_name.strip().replace(" ", " ") - - df = pd.read_excel(file_path).fillna("") - - for _, row in df.iterrows(): - if row["Project type name"].strip() == "Project preparation": - row["Project type name"] = "Preparation" - if row["Sector name"].strip() == "Control Submstance Monitoring": - row["Sector name"] = "Control Substance Monitoring" - if row["Sector name"].strip() == "Compliance Assistance Program": - row["Sector name"] = "Compliance Assistance Programme" - if row["Sector name"] == "Other Sector": - continue - try: - cluster_sector_type = ProjectSpecificFields.objects.get( - cluster__name__iexact=row["Cluster name"].strip(), - type__name__iexact=row["Project type name"].strip(), - sector__name__iexact=row["Sector name"].strip(), - ) - except ProjectSpecificFields.DoesNotExist: - logger.warning( - f"⚠️ {row['Cluster name']}/{row['Project type name']}/{row['Sector name']} not found." - ) - continue - - cluster_sector_type.fields.clear() - - # particular fields start from row 22 - # Extract MYA fields separately as some names are dupliated in the impact section - field_names_excluding_mya = [ - _clean_up_field_name(row[field_index].strip()) - for field_index in range(22, 49) - if row[field_index] != "" - ] - - # search for fields that also have an actual field that is not in the file - # and add them to the list of fields to be added (for Impact fields) - actual_field_names = [ - f"{field_name} actual" - for field_name in field_names_excluding_mya - if field_name in FIELDS_WITH_ACTUAL_VALUES - ] - field_names_excluding_mya.extend(actual_field_names) - project_fields = ProjectField.objects.exclude(section="MYA").filter( - import_name__in=field_names_excluding_mya - ) - - missing_fields = set(field_names_excluding_mya) - set( - project_fields.values_list("import_name", flat=True) - ) - - for missing_field in missing_fields: - logger.warning( - f"⚠️ {missing_field} field not found =>" - + f"{row['Cluster name']}/{row['Project type name']}/{row['Sector name']}" - ) - cluster_sector_type.fields.add(*project_fields) - - mya_field_names = [ - _clean_up_field_name(row[field_index].strip(), mya=True) - for field_index in range(49, len(row) - 1) - if row[field_index] != "" - ] - project_fields = ProjectField.objects.filter( - import_name__in=mya_field_names, section="MYA" - ) - missing_fields = set(mya_field_names) - set( - project_fields.values_list("import_name", flat=True) - ) - for missing_field in missing_fields: - logger.warning( - f"⚠️ {missing_field} field not found =>" - + f"{row['Cluster name']}/{row['Project type name']}/{row['Sector name']}" - ) - - cluster_sector_type.fields.add(*project_fields) - - -def import_project_submission_statuses(file_path): - """ - Import project submission statuses from file - - @param file_path = str (file path for import file) - """ - - with open(file_path, "r", encoding="utf8") as f: - statuses_json = json.load(f) - - for status_json in statuses_json: - status_data = { - "name": status_json["STATUS"], - "code": status_json["STATUS_CODE"], - } - ProjectSubmissionStatus.objects.update_or_create( - name=status_data["name"], defaults=status_data - ) - - -def import_cluster_type_sector_links(file_path): - """ - Import links between cluster, type and sector from file - - @param file_path = str (file path for import file) - """ - with open(file_path, "r", encoding="utf8") as f: - data = json.load(f) - - for cluster_json in data: - cluster = ProjectCluster.objects.filter(name=cluster_json["cluster"]).first() - if not cluster: - logger.warning( - f"⚠️ {cluster_json['cluster']} cluster not found => {cluster_json['cluster']} not imported" - ) - continue - for type_json in cluster_json["types"]: - type_name = type_json["type"] - if type_name == "Project Support": - type_name = "Project support" - type_obj = ProjectType.objects.filter(name=type_name).first() - if not type_obj: - logger.warning( - f"⚠️ {type_name} type not found => {cluster_json['cluster']} not imported" - ) - continue - for sector_name in type_json["sectors"]: - if sector_name == "Control Submstance Monitoring": - sector_name = "Control Substance Monitoring" - if sector_name == "Compliance Assistance Program": - sector_name = "Compliance Assistance Programme" - sector = ProjectSector.objects.filter(name=sector_name).first() - if not sector: - logger.warning( - f"⚠️ {sector_name} sector not found => {cluster_json['cluster']} not imported" - ) - continue - ProjectSpecificFields.objects.update_or_create( - cluster=cluster, - type=type_obj, - sector=sector, - ) - - -def generate_new_cluster_type_sector_file(file_path): - """ - Generate new cluster type sector file based on the current data in the database - - @param file_path = str (file path for import file) - """ - combinations = {} # {cluster: {type: [sectors]}} - df = pd.read_excel(file_path).fillna("") - - for _, row in df.iterrows(): - if row["Project type name"].strip() == "Project preparation": - row["Project type name"] = "Preparation" - - if row["Sector name"].strip() == "Other Sector": - continue - - combinations.setdefault(row["Cluster name"].strip(), {}) - combinations[row["Cluster name"].strip()].setdefault( - row["Project type name"].strip(), [] - ) - combinations[row["Cluster name"].strip()][ - row["Project type name"].strip() - ].append(row["Sector name"].strip()) - - new_data = [] - for cluster_name, types in combinations.items(): - new_data.append( - { - "cluster": cluster_name, - "types": [ - { - "type": type_name, - "sectors": sorted(list(set(sector_names))), # remove duplicates - } - for type_name, sector_names in types.items() - ], - } - ) - - with open("new_ClusterTypeSectorLinks.json", "w", encoding="utf8") as f: - json.dump(new_data, f, indent=4) - - -def import_fields(file_path): - """ - Import project type from file - - @param file_path = str (file path for import file) - """ - - ProjectField.objects.all().delete() - with open(file_path, "r", encoding="utf8") as f: - fields_json = json.load(f) - - # add other types that are not in the file - ProjectField.objects.all().delete() - for field_json in fields_json: - - field_data = { - "import_name": field_json["IMPORT_NAME"], - "label": field_json["LABEL"], - "read_field_name": field_json["READ_FIELD_NAME"], - "write_field_name": field_json["WRITE_FIELD_NAME"], - "table": field_json["TABLE"], - "data_type": field_json["DATA_TYPE"], - "mlfs_only": field_json.get("MLFS_ONLY", False), - "section": field_json["SECTION"], - "is_actual": field_json.get("IS_ACTUAL", False), - "sort_order": field_json["SORT_ORDER"], - "editable_in_versions": ",".join( - [str(version) for version in field_json["EDITABLE_IN_VERSIONS"]] - ), - "visible_in_versions": ",".join( - [str(version) for version in field_json["VISIBLE_IN_VERSIONS"]] - ), - } - - ProjectField.objects.update_or_create( - import_name=field_data["import_name"], defaults=field_data - ) - - -@transaction.atomic -def import_project_resources_v2(option): - - if option in ["all", "import_project_clusters"]: - file_path = ( - IMPORT_RESOURCES_DIR / "projects_v2" / "project_clusters_06_05_2025.xlsx" - ) - import_project_clusters(file_path) - logger.info("✔ project clusters imported") - - if option in ["all", "import_project_type"]: - file_path = ( - IMPORT_RESOURCES_DIR / "projects_v2" / "tbTypeOfProject_06_05_2025.json" - ) - import_project_type(file_path) - logger.info("✔ project types imported") - - if option in ["all", "import_sector"]: - file_path = IMPORT_RESOURCES_DIR / "projects_v2" / "tbSector_15_10_2025.json" - import_sector(file_path) - logger.info("✔ sectors imported") - - if option in ["all", "import_subsector"]: - file_path = IMPORT_RESOURCES_DIR / "projects_v2" / "tbSubsector_06_05_2025.json" - import_subsector(file_path) - logger.info("✔ subsectors imported") - - if option in ["all", "import_project_submission_statuses"]: - file_path = ( - IMPORT_RESOURCES_DIR / "projects_v2" / "project_submission_statuses.json" - ) - import_project_submission_statuses(file_path) - logger.info("✔ project submission statuses imported") - - if option in ["all", "clean_up_project_statuses"]: - clean_up_project_statuses() - logger.info("✔ project statuses cleaned up") - - if option in ["all", "import_cluster_type_sector_links"]: - file_path = IMPORT_RESOURCES_DIR / "projects_v2" / "ClusterTypeSectorLinks.json" - import_cluster_type_sector_links(file_path) - logger.info("✔ cluster type sector links imported") - - if option in ["all", "import_fields"]: - file_path = IMPORT_RESOURCES_DIR / "projects_v2" / "Fields_24_10_2025.json" - import_fields(file_path) - logger.info("✔ fields imported") - - if option in ["all", "import_project_specific_fields"]: - file_path = ( - IMPORT_RESOURCES_DIR - / "projects_v2" - / "project_specific_fields_27_10_2025.xlsx" - ) - import_project_specific_fields(file_path) - logger.info("✔ cluster type sector fields imported") - - if option == "generate_new_cluster_type_sector_file": - # use to generate new ClusterTypeSectorLinks.json file - file_path = ( - IMPORT_RESOURCES_DIR - / "projects_v2" - / "project_specific_fields_27_10_2025.xlsx" - ) - generate_new_cluster_type_sector_file(file_path) - logger.info("✔ new cluster type sector file generated") diff --git a/core/import_data/utils.py b/core/import_data/utils.py index 6e6895517..c92faacc6 100644 --- a/core/import_data/utils.py +++ b/core/import_data/utils.py @@ -45,6 +45,7 @@ logger = logging.getLogger(__name__) IMPORT_RESOURCES_DIR = settings.ROOT_DIR / "import_data" / "resources" +IMPORT_RESOURCES_V2_DIR = settings.ROOT_DIR / "import_data_v2" / "resources" IMPORT_PROJECTS_DIR = settings.IMPORT_DATA_DIR / "project_database" PCR_DIR_LIST = ["pcr2023", "hpmppcr2023"] diff --git a/core/import_data_v2/__init__.py b/core/import_data_v2/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/core/import_data_v2/import_project_resources_v2.py b/core/import_data_v2/import_project_resources_v2.py new file mode 100644 index 000000000..df5500cc1 --- /dev/null +++ b/core/import_data_v2/import_project_resources_v2.py @@ -0,0 +1,112 @@ +import logging + +from django.db import transaction + +from core.import_data.utils import ( + IMPORT_RESOURCES_V2_DIR, +) +from core.import_data_v2.scripts.import_project_type import import_project_type +from core.import_data_v2.scripts.generate_new_cluster_type_sector_file import ( + generate_new_cluster_type_sector_file, +) +from core.import_data_v2.scripts.import_cluster_type_sector_links import ( + import_cluster_type_sector_links, +) +from core.import_data_v2.scripts.import_fields import ( + import_fields, + import_project_specific_fields, +) +from core.import_data_v2.scripts.import_project_clusters import ( + import_project_clusters, +) +from core.import_data_v2.scripts.clean_up_project_statuses import ( + clean_up_project_statuses, + import_project_submission_statuses, +) +from core.import_data_v2.scripts.import_sector_subsector import ( + import_sector, + import_subsector, +) +from core.import_data_v2.scripts.clean_up_countries import ( + clean_up_countries, +) +from core.import_data_v2.scripts.import_modules import import_modules + +logger = logging.getLogger(__name__) + + +@transaction.atomic +def import_project_resources_v2(option): + + if option in ["all", "import_project_clusters"]: + file_path = ( + IMPORT_RESOURCES_V2_DIR / "projects" / "project_clusters_06_05_2025.xlsx" + ) + import_project_clusters(file_path) + logger.info("✔ project clusters imported") + + if option in ["all", "import_project_type"]: + file_path = ( + IMPORT_RESOURCES_V2_DIR / "projects" / "tbTypeOfProject_06_05_2025.json" + ) + import_project_type(file_path) + logger.info("✔ project types imported") + + if option in ["all", "import_sector"]: + file_path = IMPORT_RESOURCES_V2_DIR / "projects" / "tbSector_15_10_2025.json" + import_sector(file_path) + logger.info("✔ sectors imported") + + if option in ["all", "import_subsector"]: + file_path = IMPORT_RESOURCES_V2_DIR / "projects" / "tbSubsector_06_05_2025.json" + import_subsector(file_path) + logger.info("✔ subsectors imported") + + if option in ["all", "import_project_submission_statuses"]: + file_path = ( + IMPORT_RESOURCES_V2_DIR / "projects" / "project_submission_statuses.json" + ) + import_project_submission_statuses(file_path) + logger.info("✔ project submission statuses imported") + + if option in ["all", "clean_up_project_statuses"]: + clean_up_project_statuses() + logger.info("✔ project statuses cleaned up") + + if option in ["all", "import_cluster_type_sector_links"]: + file_path = IMPORT_RESOURCES_V2_DIR / "projects" / "ClusterTypeSectorLinks.json" + import_cluster_type_sector_links(file_path) + logger.info("✔ cluster type sector links imported") + + if option in ["all", "import_fields"]: + file_path = IMPORT_RESOURCES_V2_DIR / "projects" / "Fields_24_10_2025.json" + import_fields(file_path) + logger.info("✔ fields imported") + + if option in ["all", "import_project_specific_fields"]: + file_path = ( + IMPORT_RESOURCES_V2_DIR + / "projects" + / "project_specific_fields_27_10_2025.xlsx" + ) + import_project_specific_fields(file_path) + logger.info("✔ cluster type sector fields imported") + if option in ["all", "import_modules"]: + import_modules() + logger.info("✔ modules imported") + if option in [ + "all", + "clean_up_countries", + ]: + clean_up_countries() + logger.info("✔ countries cleaned up") + + if option == "generate_new_cluster_type_sector_file": + # use to generate new ClusterTypeSectorLinks.json file + file_path = ( + IMPORT_RESOURCES_V2_DIR + / "projects" + / "project_specific_fields_27_10_2025.xlsx" + ) + generate_new_cluster_type_sector_file(file_path) + logger.info("✔ new cluster type sector file generated") diff --git a/core/import_data_v2/resources/countries/MLF Countries and Regions.xlsx b/core/import_data_v2/resources/countries/MLF Countries and Regions.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..4d59f77c96184f5e664665217889fe75d1922e42 GIT binary patch literal 8508 zcmaKR1zZ&E_CH;Vw3Jc`NG?d1q|{Qh357ldHpX zCnpD>rz7|;KWRwWf8i$jxO(^b?uV^+p&A^kBzW+Gb@b4Wb;@8xTS3SD^I{d-`#8DS z*zmLcWeHQiJ3N{hw}(FLQ!o@pZMZFYh~Xz zZ4eJRwn{qT+VHzQm%&Uku+^Y-g<;Bpm}!sDc;JSdCfFX+MU#U5MPcCe(2>drm5~pW za&V6Z&}~PNoD;`c9c=2!9L*6G^fB99+zM+^Gv4ivOGo?`h30g?X~U!Ft_h3)Exh9c zv8_d~VPI*TUJF#YlfT?j+4OTC$P4N^CV1XfWkLg|p^O+9o zG;Mmh+I?@Oy2PHrfuYmwTQe3L18FviCBaH%^wt!WTh@mVuVL$KyXNDy=IX~KL>aO~ zlOKa&GeHnZ^>?EMD+&cun01d^0fwk%oFV3vsv$5yyGUr*DILD?>jx3B{e=81Tax$+ zabM?k-d2G>aD;^xmNUc)B5X`Zw?lg~F-vr(O=DB{d;@GXdgAoE^D*sfhYPu;Jh~D^ z7BzS(kT^qP_m5cvByHa*v&r<2?=d1R13C>y80PIBn$$`I>)}Hh9r%cE2CO(%fzzJFb0l#_Kft@^&S-OtL9yFBh1S$M2CEyHj%5;MMp#ZDoJ)?N zDkAp<6xJfioszv4@&Hf0w_Ea5kpiQNxz^#rU0Bx#okTeCeyI8A=g(9U25zg`iW(!A zT>OfORBJz%-hKTTLj3rA&tD{yxn_Nkt~@W1PPU%(`irgD-pCAJej8JU<(n%;ec@5e z%cQtZvN8)@=qWRRvZGl=@HPu(%Gjl-wNypq7}ssi^v$qq9iyV4K+ykbY?1zzGj6t4 zj#ga1_K$Bz*4!&&rx|*(@ZDo}qGNWK*{6zRg|OGMP48=?-I^uZCJRu(sE(-t68Nf% z6-6U_b{oE=1Y*0rLD1(FyZ#D6CTRj6wQ%Dg%#@F1ds9lp+7{?u&{0rQ0oX9r*T=ZKrJtNFc0ca9K9L|q$=_l5 z(wv9+HMzmu@EK*?GSNWl;CI8Ty}$?|>nGPIJqrXUVR=4R^}7B7$aPmmE|1~h&(7kX z)$$_UJqhmymn^Nt*6{jCRcrXtwA<9+kVjy(XuKN}m#P34rW^P%ek&Rk#XebVRMqZ! z!}c20efAMT%1p~eDVjzBs0x&gu|=Kc-h8R=bcY>_6Y=^<+Y;||2#*tc(J3#}Iuj7? zU0B$_tU4XGZy!wP1{r}tKGnG6qovn$o=W{Y!H4tla`=# z1@4>9nzM}@S0`#+L%|enm|31{jO6l8o@Ch(!pP2|-t1*U@B*urf@`nafxpJ?NcvB% z2rH_*BMRP$!7|L`p|KXil}EQZh9j3b_VA`AP2l`f zj@|hy$DnRrU@Pctc5Umc$7cu;`R`U#Uii1;>Ql2xe@JE1HZ3lm1HDo`Mg}tI#Ky*q zsGMy&SV+@5?9N%MHI|a?qrBM@JlI1_Mt|&XGH!2GdvK5K1WncCAr8*KQ&lf`;KGhh zKp%rG*^01ra5Cm>GS(+0Q>NHy-hFqONHh^NCvxXJtxEwM7+8jF_9Hp}w`olree?W? zYJ8XhNv0r{R+ZVyXJlGwu4&A^hGlGHfy!X`FrPPRWqmwcK#ms&+>qbJ|B5h_G>Qs; zm~jp5og%}@F`HT=sNBPfn#m}_vMR0Uz&DVAjq0&J zI0X?(9{bi;@S~Rd^o854UVhoqSiH3KKG;Q#HRBWH=&?GVt~TOA+J_A~-VbN)I~#p0 z--RaB(rumgnRJN8fR0=jz9AB6zlgi=c zk0=;G`i!HdD~Ul-Ac_0j;gPYk5^F!H%(qJPFBYQTd?8!UX8&NCEMyuN8yTa)w`gpQDV>`xW!YV(^A57vX2CNSj2=_uNxDB)3 zrazuK4wFpGI(tA?r_NXC6SXPoxF!2e%!$;&v?ej-x`C8^y6JSYfoV9ttYaBvL5>J1G+-9-%NPE9q;3k^H z)s)LI@u<`Oy{L}p;biajo;i`o{S5g2$=g$Ifj3<7GrY&22KG>f+8uX(_BE%$DJ-@i z10?C$T=DlvH+wG&T<(XfFKfEN-FWoQ+@c?Z`VRTd;(t!^V-f3 z%l>=P^z0__Pui{*Y9W9nYoc)Rc>P%a_>7>am6Bli^pi`=YXX9EZ|92&)3bZGWdLN| z_PXz;&4}Fpr!qkKdwK%3wX$-9a{c=H)o+@!Zu*TJ`Axsk#c5m{8(*YyJxT0g3$|=F zrjHE~atvzDYfqF7eW0IIy>xMw#lWP0lun8vdweHhk{V<)pPqOa`j&QJf0e7r{)JQH zC-6qEurbC#deMZ6`nX5x!GgH)Wm><@Wx|B43I7LprUcCY>R^MeH=^Kt)+v_Cx=wP= zw7>q#;3or)v%~SN{j0ZjDvt4B7$lYO$l9ZRZKi*@{Upcu!!s!&t`}bV1BG}Mjk`rd zV1e(`A05Zfvx*{$6l{upEBDSFr;a-9O<(R7>$;tW&}=*&(EnOLlBoIQrCzf!dl%otxp-CKI0 zrb~a{lU^Oc+}-_7YYElfZ|~2i8XaK+<=ta@KeWL1muECTKV{@BQ)l+=G&Bd)6OikgwE#!;f~|xC7T{_$5`@w!de7Va{;g5 z>dMR>zo0)jQ9=)gfIO}(xW=r9jsp!lTCZ}DPB|pRt}f-~54s`5Zy~|iEq)i=|tNZTvylewqH#x!%_pJTdtC^QP5CvVo~tC z1W?gJ3s7XmGFnlkLjr|x6(mt}^(F6PF%AX}3eoqYlLSiIJ|L1u!&Ro|Ba74w!r)-x zBPZ)b!!>$l3w$31_;z~jl0SlW;)S{zOwy1uX!I--073>!L#14REI|M~1ZF!t2Ls5$ zM?v;)1>9@lyZ^1p00S`GY7o7J6YU>_Jwr-Iu1I&mi9oxQZ3{e$MmU(jP+MSVCLp&p zj0h2y0A(QirRxd|MWc}BqmcIhM1*IAGh~D#Od%Zv_+?I-#hF1G%J2*P)E4+O6Oh;% zhJy%;gfiURC>KuT0(?5$j&U-E(I2LOlnz~yUWUJ={}(0_1NbL~#VWPpK@fl#1|VLQ zW^|UygXf^XAfUuG8ngx4_G5DXIwMR`+lm>x5=<}!vkwALBQO#0ocztc9{@xqzyKL6 z2bJ;x3IP5Ca(Ws}-jFk7G#ooPmsJ6eN5UB#3$FZ=Ij(S|^>Fr>D{ne!=&A zpK2qegIA#8)HGlMLY;k%InCh(mt@LeXLs5OiX5tagF zxZ9BM7M|b=e1}G%!AGIdAEt$rez_vu15W@gqRkp1{|%w_kwQv`uSj>m6Yz|>293Ie zD0qSZR4@S5sx-4RgA9~m$*2nhXlVdpGCY0LTJEr1~6Mw>tn?ZHTOzLsRqsAU_!*iRaO}JpsrbL!`?*8ZNab)wG4o zOqYh(N22ZA@b^!bvTQ5JIMvZBWY0g^s>gI>41Eod!HR1fknm9z|FDcy%I%|Qe22$vV0YDcNKGkr3!ZM z+36x^G|*>!nYc`mrwq5;Ai47cvtk`{gQE)V9E@=RhWyO$72}YU;t(u4LFP4|n^fu$ z>@cysSFT7>feSIwT+R(&HX$|o#2oLZlH{-Ad<}a*$j^wLfy5J#rL|We5oqHC58wQt z6=;(M@25>N7ENM!RISrnNEjM*cQ7}^snF78FfwEy>u%nzy|JbwK@$V(*cyS#iPU8NYV|vu_x7kDCl9GPZpdGD=brT&atWhK;7F zlwxIVn!Sy7O7#WHE6-tNB`$H+vhb-%_u`1EE6qNO19$a{&8zj(Mgsxa$Bx6*XABD; zB$}zNFTV}=@m~BanoajVniuO67iju;wJHy-uh@r(2+hZjo_s%a-P7vO&is7w;Uz@X zdg++1qdLN%xBXOT8!-ZrE4fC!of1BY7~?|VqM#7l{nM0?>|bw&w&t!@mYQy^b`YCi zFNj(CDw+=kiJDh+-}`nsPB3aK<_olvQ*v`mx>Pc}YCi1a<88KmCq;KO-6HOIuPX_Z zKfi2yUNp8*am+~Pe=%)PvrN?9l-EYMONbS2yw2`&ID~|_?CqomMMW9Oj*E&@mkJYz z(*}4@TJSWEw&zJDYaE*k#aMmeY(M^FQP?7OGN>^epG(^T)bM=?OWjJ~bK`AJJQWQa9@q;g zD7Wiw18}rdiRLAQ>dBge^xLgD(47pbSNrr#mdF%X4iols8L3yDpiRDrEw;8wZc}%KMg3Zy-vP+tJ;=$fR7tJs_mW*Yi=dVm zAYEhe*oMqC#r#@^}+k=wg%;x6%cbQ=h~Hw;tB^C?Y+55e6AU75Q$WB}ODt05!V zk`Ezc(~$-Q^A=3J`(=b?6R{r;9Z_HRc5NaB>(@}mJp=RkdoZ!Y&L4yg;AW86e7YMU za*np6uVgcO#`7#`-#u%Ou&GF7A4JwbjA!_e_bBW8ZqdQs#7W2U2i={BzN}RJLP%@n zVWSe|Fid!g33AG+xeqEHkEszFJ*LMMx3T}~@g!r*cuZxY0DJJwPwf4=2NOtOb1_3Sx(aPV(l=+C{_>pRkeKULCq|Wpbh*p zwv*_7t=877@nG~9a`7h*r<4!nqP;|BpU-ZcN;pU+MJhUm+m12wj6E@HQ8YBBGYI8i%N zT8LZ$ZmVO216TD)yWykW#Qy%b-Q*JtI-PyxuH9b*%Nln zlxgjWh>FTw?caeEM~ej~z>$Vvjv3tUg$YS|y9^m8QPHKT)Z|n3lzG(JZ?BYzaiy1o zEVcWs65k@czb!l*&|`xd=#Xf*p@pnW8GqL5W4puEsFA~oy(Ik5?stfJN`5b z4AWZjT5dtQ+P+sB&ORE>n$yW4XSOHwrJEw}<;zA7)qc$EdyU0yK&MJyqV^;~A3h$D zVH92exFtz^rwgCA+K-CkSaS+p5r;{DvkO7EU3qDf zx%`!akdcP|*4$^_ge&VYP`DPfCyBbrO@6_(i%zwFNv?cXArpE{XGfAeHKx!kWvKrp zNHBowq2my$fN{oU=OgZ!SUwaALx|Gau;0n5g!RI0faX!M`Jdk`p3nRv;X?FhfXtnp zZ{w4dXzcKr``5}jXtGSD*inZm7Qdrh;C?CTE(n%sFP9O@OnP}7-7~tGRdUdN@P@EX z4BxLhV#vXPEhQg=$?To>G?@LTc8e+~<%B+lq?!L&?nBSoWx_3sT7r(8wnvF(zF{vI z8ZbLdcFc&xW35QpvuLo1p9?{%wnaF9Kcv1Ew10t`N#q>iHq?y!DyAWpN z2Z5<5-h(~zpi!CqcF(fajR5q@5x1<5S5_f|Jq=MK-R(O4ie`PeG+)zz+qc4<#Kcs* zt>yvKQ2(6(;czGU)s|at+8q1VJPDSakGNBnQD1CTS1L)9onpg zP`hzx%YiOOGpr2RMp@I=7N3_CX`?w)UsnA=#gORA#}q*F6_+G3m4DNxrK0}(_dMAd zMrfs~y>^1s_#<4F@-QuJDHCSGb$+TlKBty;txS_8MpE{>h&%al6N5adcIbHQ`E{HJ zC-&_2l-bpIYgR1*vTz;R6g592{g(>@kbr})YfJ0qE8u5E*QO1C2BvujUlMP;6?Jhf zsZFknPau@(#caNV4b=mS_>z1ViWYWC?QNVc0Y<>_T%-s3T!NltA8FhS2(>#+|GLWn zoKwMoi|EfRRI(c?Z5OZ2`egS6QF)pjTF(@(Iby{sjDxV@l;*MF;QnTh62G#P{z17I z{fst1Pwxr#BcDhX^Z0c1aoVA7TOC^6DjOm*OV?rZq=$jzZ=R&4E#8uaSW&Is@M{2=Eb(l4l<0Ld%ARIZ>PSO+MD}fz4&!6Jt!^UJ zg?Sq#R5UV_-#qo7MC~n4{h#s&YyEfNpUmYgkNaDwZf^CzS>3JN@-{1Ai`X-m)3LWeV$;f#2N5-=Tj_gl|_se+v@t z|NZg*S`YoZl|LKC?WFm)9NZk {cluster_json['cluster']} not imported" + ) + continue + for type_json in cluster_json["types"]: + type_name = type_json["type"] + if type_name == "Project Support": + type_name = "Project support" + type_obj = ProjectType.objects.filter(name=type_name).first() + if not type_obj: + logger.warning( + f"⚠️ {type_name} type not found => {cluster_json['cluster']} not imported" + ) + continue + for sector_name in type_json["sectors"]: + if sector_name == "Control Submstance Monitoring": + sector_name = "Control Substance Monitoring" + if sector_name == "Compliance Assistance Program": + sector_name = "Compliance Assistance Programme" + sector = ProjectSector.objects.filter(name=sector_name).first() + if not sector: + logger.warning( + f"⚠️ {sector_name} sector not found => {cluster_json['cluster']} not imported" + ) + continue + ProjectSpecificFields.objects.update_or_create( + cluster=cluster, + type=type_obj, + sector=sector, + ) diff --git a/core/import_data_v2/scripts/import_fields.py b/core/import_data_v2/scripts/import_fields.py new file mode 100644 index 000000000..5e4374bb3 --- /dev/null +++ b/core/import_data_v2/scripts/import_fields.py @@ -0,0 +1,214 @@ +import logging +import json +import pandas as pd + +from django.db import transaction + +from core.models.project_metadata import ( + ProjectSpecificFields, + ProjectField, +) + +logger = logging.getLogger(__name__) + +# pylint: disable=C0301 + +FIELDS_WITH_ACTUAL_VALUES = [ + "Total number of technicians trained", + "Number of female technicians trained", + "Total number of trainers trained", + "Number of female trainers trained", + "Total number of technicians certified", + "Number of female technicians certified", + "Number of training institutions newly  assisted", + "Number of tools sets distributed", + "Total number of customs officers trained", + "Number of female customs officers trained", + "Total number of NOU personnel supported", + "Number of female NOU personnel supported", + "Certification system for technicians established or further enhanced (yes or no)", + "Operation of recovery and recycling scheme (yes or no)", + "Operation of reclamation scheme (yes or no)", + "Establishment or upgrade of Import/export licensing (yes or no)", + "Establishment of quota systems (yes or no)", + "Ban of equipment (number)", + "Ban of substances (number)", + "kWh/year saved", + "MEPS developed for domestic refrigeration (yes/no)", + "MEPS developed for commercial refrigeration (yes/no)", + "MEPS developed for residential air-conditioning (yes/no)", + "MEPS developed for commercial AC (yes/no)", + "Capacity building programmes (checklist (yes/no) for technicians, end-users, operators, consultants, procurement officers and other Government entities)", + "EE demonstration project included (yes/no)", + "Quantity of controlled substances destroyed (M t)", + "Quantity of controlled substances destroyed (CO2-eq t)", + "Checklist of regulations or policies enacted", + "Quantity of HFC-23 by-product (Generated)", + "Quantity of HFC-23 by-product (by product generation rate)", + "Quantity of HFC-23 by-product (Destroyed)", + "Quantity of HFC-23 by-product (Emitted)", + "Number of Production Lines assisted", + "Number of enterprises assisted", + "Number of enterprises", + "Aggregated consumption", + "Cost effectiveness (US$/ Kg)", + "Cost effectiveness (US$/ CO2-eq)", +] + + +@transaction.atomic +def import_fields(file_path): + """ + Import project type from file + + @param file_path = str (file path for import file) + """ + + ProjectField.objects.all().delete() + with open(file_path, "r", encoding="utf8") as f: + fields_json = json.load(f) + + # add other types that are not in the file + ProjectField.objects.all().delete() + for field_json in fields_json: + + field_data = { + "import_name": field_json["IMPORT_NAME"], + "label": field_json["LABEL"], + "read_field_name": field_json["READ_FIELD_NAME"], + "write_field_name": field_json["WRITE_FIELD_NAME"], + "table": field_json["TABLE"], + "data_type": field_json["DATA_TYPE"], + "mlfs_only": field_json.get("MLFS_ONLY", False), + "section": field_json["SECTION"], + "is_actual": field_json.get("IS_ACTUAL", False), + "sort_order": field_json["SORT_ORDER"], + "editable_in_versions": ",".join( + [str(version) for version in field_json["EDITABLE_IN_VERSIONS"]] + ), + "visible_in_versions": ",".join( + [str(version) for version in field_json["VISIBLE_IN_VERSIONS"]] + ), + } + + ProjectField.objects.update_or_create( + import_name=field_data["import_name"], defaults=field_data + ) + + +@transaction.atomic +def import_project_specific_fields(file_path): + """ + Import project clusters from file + Please make sure that the file has the correct extention + (xls, xlsx, xlsm, xlsb, odf, ods, odt) + + @param file_path = str (file path for import file) + """ + + def _clean_up_field_name(field_name, mya=False): + """ + Clean up field name + """ + mya_clean_up = { + "Cost effectiveness (US$/ CO2-ep) (MYA)": "Cost effectiveness (US$/ CO2-eq) (MYA)", + "Cost effectiveness (US$/ CO2-ep)": "Cost effectiveness (US$/ CO2-eq) (MYA)", + "Cost effectiveness (US$/ CO2-eq)": "Cost effectiveness (US$/ CO2-eq) (MYA)", + "Aggregated consumption": "Aggregated consumption (MYA)", + "Cost effectiveness (US$/ Kg)": "Cost effectiveness (US$/ Kg) (MYA)", + "Number of enterprises assisted": "Number of enterprises assisted (MYA)", + "Number of enterprises": "Number of enterprises (MYA)", + "Number of Production Lines assisted": "Number of Production Lines assisted (MYA)", + } + + individual_field_clean_up = { + "Cost effectiveness (US$/ CO2-ep)": "Cost effectiveness (US$/ CO2-eq)", + "Cost effectiveness (US$/ CO2-ep) (MYA)": "Cost effectiveness (US$/ CO2-eq)", + "Phase out (Mt) (MYA)": "Phase out (Mt)", + "Phase out (M t)": "Phase out (Mt)", + "Phase out (CO2-eq t) (MYA)": "Phase out (CO2-eq t)", + "Cost effectiveness (US$/ CO2-eq)": "Cost effectiveness (US$/ CO2-eq)", + "Phase out (ODP t) (MYA)": "Phase out (ODP t)", + } + if mya: + if field_name in mya_clean_up: + return mya_clean_up[field_name] + else: + if field_name in individual_field_clean_up: + return individual_field_clean_up[field_name] + return field_name.strip().replace(" ", " ") + + df = pd.read_excel(file_path).fillna("") + + for _, row in df.iterrows(): + if row["Project type name"].strip() == "Project preparation": + row["Project type name"] = "Preparation" + if row["Sector name"].strip() == "Control Submstance Monitoring": + row["Sector name"] = "Control Substance Monitoring" + if row["Sector name"].strip() == "Compliance Assistance Program": + row["Sector name"] = "Compliance Assistance Programme" + if row["Sector name"] == "Other Sector": + continue + try: + cluster_sector_type = ProjectSpecificFields.objects.get( + cluster__name__iexact=row["Cluster name"].strip(), + type__name__iexact=row["Project type name"].strip(), + sector__name__iexact=row["Sector name"].strip(), + ) + except ProjectSpecificFields.DoesNotExist: + logger.warning( + f"⚠️ {row['Cluster name']}/{row['Project type name']}/{row['Sector name']} not found." + ) + continue + + cluster_sector_type.fields.clear() + + # particular fields start from row 22 + # Extract MYA fields separately as some names are dupliated in the impact section + field_names_excluding_mya = [ + _clean_up_field_name(row[field_index].strip()) + for field_index in range(22, 49) + if row[field_index] != "" + ] + + # search for fields that also have an actual field that is not in the file + # and add them to the list of fields to be added (for Impact fields) + actual_field_names = [ + f"{field_name} actual" + for field_name in field_names_excluding_mya + if field_name in FIELDS_WITH_ACTUAL_VALUES + ] + field_names_excluding_mya.extend(actual_field_names) + project_fields = ProjectField.objects.exclude(section="MYA").filter( + import_name__in=field_names_excluding_mya + ) + + missing_fields = set(field_names_excluding_mya) - set( + project_fields.values_list("import_name", flat=True) + ) + + for missing_field in missing_fields: + logger.warning( + f"⚠️ {missing_field} field not found =>" + + f"{row['Cluster name']}/{row['Project type name']}/{row['Sector name']}" + ) + cluster_sector_type.fields.add(*project_fields) + + mya_field_names = [ + _clean_up_field_name(row[field_index].strip(), mya=True) + for field_index in range(49, len(row) - 1) + if row[field_index] != "" + ] + project_fields = ProjectField.objects.filter( + import_name__in=mya_field_names, section="MYA" + ) + missing_fields = set(mya_field_names) - set( + project_fields.values_list("import_name", flat=True) + ) + for missing_field in missing_fields: + logger.warning( + f"⚠️ {missing_field} field not found =>" + + f"{row['Cluster name']}/{row['Project type name']}/{row['Sector name']}" + ) + + cluster_sector_type.fields.add(*project_fields) diff --git a/core/import_data_v2/scripts/import_modules.py b/core/import_data_v2/scripts/import_modules.py new file mode 100644 index 000000000..6c1b6b9ba --- /dev/null +++ b/core/import_data_v2/scripts/import_modules.py @@ -0,0 +1,20 @@ +from django.db import transaction + +from core.models.base import Module + + +@transaction.atomic +def import_modules(): + """ + Import modules + """ + modules = [ + {"name": "Projects", "code": "Projects"}, + {"name": "Business Plans", "code": "BP"}, + {"name": "Country Programmes", "code": "CP"}, + ] + + for module_data in modules: + Module.objects.update_or_create( + code=module_data["code"], defaults={"name": module_data["name"]} + ) diff --git a/core/import_data_v2/scripts/import_project_clusters.py b/core/import_data_v2/scripts/import_project_clusters.py new file mode 100644 index 000000000..3d4c009a6 --- /dev/null +++ b/core/import_data_v2/scripts/import_project_clusters.py @@ -0,0 +1,59 @@ +import logging +import pandas as pd + +from django.db import transaction + +from core.models.group import Group +from core.models.project_metadata import ProjectCluster + + +logger = logging.getLogger(__name__) + + +@transaction.atomic +def import_project_clusters(file_path): + """ + Import project clusters from file + Please make sure that the file has the correct extention + (xls, xlsx, xlsm, xlsb, odf, ods, odt) + + @param file_path = str (file path for import file) + """ + + df = pd.read_excel(file_path).fillna("") + + for index, row in df.iterrows(): + if row["Action"] == "Outdated": + continue + if row["Action"] == "Rename": + ProjectCluster.objects.filter(name=row["Old name"]).update(name=row["Name"]) + + production = False + if row["Production"] == "Y": + production = True + elif row["Production"] == "Both": + production = None + + # get annex groups + annex_groups = [] + if row["Annex groups"]: + annex_groups_name_alt = row["Annex groups"].split(",") + annex_groups = Group.objects.filter(name__in=annex_groups_name_alt) + if annex_groups.count() != len(annex_groups_name_alt): + logger.warning( + f"⚠️ Some annex groups not found for cluster {row['Name']}" + ) + cluster_data = { + "name": row["Name"], + "code": row["Acronym"], + "category": row["Category"].upper(), + "group": row["Dashboard group"], + "production": production, + "sort_order": index, + } + + cluster, _ = ProjectCluster.objects.update_or_create( + name=cluster_data["name"], defaults=cluster_data + ) + if annex_groups: + cluster.annex_groups.set(annex_groups) diff --git a/core/import_data_v2/scripts/import_project_type.py b/core/import_data_v2/scripts/import_project_type.py new file mode 100644 index 000000000..ae3353896 --- /dev/null +++ b/core/import_data_v2/scripts/import_project_type.py @@ -0,0 +1,36 @@ +import json +import logging + +from django.db import transaction + +from core.models.project_metadata import ProjectType + + +logger = logging.getLogger(__name__) + + +@transaction.atomic +def import_project_type(file_path): + """ + Import project type from file + + @param file_path = str (file path for import file) + """ + with open(file_path, "r", encoding="utf8") as f: + types_json = json.load(f) + + # add other types that are not in the file + for type_json in types_json: + if type_json.get("ACTION", None) == "RENAME": + ProjectType.objects.filter(name=type_json["OLD_NAME"]).update( + name=type_json["TYPE_PRO"] + ) + else: + type_data = { + "code": type_json["TYPE"], + "name": type_json["TYPE_PRO"], + "sort_order": type_json["SORT_TYPE"], + } + ProjectType.objects.update_or_create( + name=type_data["name"], defaults=type_data + ) diff --git a/core/import_data_v2/scripts/import_sector_subsector.py b/core/import_data_v2/scripts/import_sector_subsector.py new file mode 100644 index 000000000..db82eb00a --- /dev/null +++ b/core/import_data_v2/scripts/import_sector_subsector.py @@ -0,0 +1,103 @@ +import json +import logging + +from django.db import transaction + +from core.models.project_metadata import ( + ProjectSector, + ProjectSubSector, +) + +logger = logging.getLogger(__name__) + + +@transaction.atomic +def import_sector(file_path): + """ + Import sectors and subsectors from file + + @param file_path = str (file path for import file) + """ + + with open(file_path, "r", encoding="utf8") as f: + sectors_json = json.load(f) + + for sector_json in sectors_json: + + if sector_json.get("ACTION", None) == "RENAME": + ProjectSector.objects.filter(name=sector_json["OLD_NAME"]).update( + name=sector_json["SECTOR"] + ) + sector_data = { + "name": sector_json["SECTOR"].strip(), + "code": sector_json["SEC"].strip(), + "sort_order": sector_json["SORT_SECTOR"], + } + ProjectSector.objects.update_or_create( + code=sector_data["code"], defaults=sector_data + ) + + +@transaction.atomic +def import_subsector(file_path): + """ + Import sectors and subsectors from file + Please make sure that the file has the correct extention + (xls, xlsx, xlsm, xlsb, odf, ods, odt) + + @param file_path = str (file path for import file) + """ + + with open(file_path, "r", encoding="utf8") as f: + subsectors_json = json.load(f) + + for subsector_json in subsectors_json: + subsector_name = subsector_json["SUBSECTOR"].strip() + if subsector_json.get("ACTION", None) == "RENAME": + project_sub_sector = ProjectSubSector.objects.filter( + name=subsector_json["OLD_NAME"] + ) + if subsector_json.get("OLD_SEC", None): + project_sub_sector = project_sub_sector.filter( + sector__code=subsector_json["OLD_SEC"] + ) + project_sub_sector.update(name=subsector_name) + + # get sector + sector = ProjectSector.objects.filter(code=subsector_json["SEC"]).first() + if not sector: + logger.warning( + f"⚠️ {subsector_json['SEC']} sector not found => {subsector_json['SUBSECTOR']} not imported" + ) + continue + + # set subsector data + subsector_code = ( + subsector_json["CODE_SUBSECTOR"].strip() + if subsector_json.get("CODE_SUBSECTOR") + else None + ) + subsector_data = { + "name": subsector_json["SUBSECTOR"].strip(), + "code": subsector_code, + "sector": sector, + "sort_order": subsector_json["SORT_SUBSECTOR"], + } + + project_sub_sector = ProjectSubSector.objects.filter( + name=subsector_data["name"] + ) + if project_sub_sector.exists(): + project_sub_sector.update( + name=subsector_data["name"], + code=subsector_data["code"], + sector=subsector_data["sector"], + sort_order=subsector_data["sort_order"], + ) + else: + ProjectSubSector.objects.create( + name=subsector_data["name"], + code=subsector_data["code"], + sector=subsector_data["sector"], + sort_order=subsector_data["sort_order"], + ) diff --git a/core/management/commands/import_resources_v2.py b/core/management/commands/import_resources_v2.py index 82444221b..ef84c2608 100644 --- a/core/management/commands/import_resources_v2.py +++ b/core/management/commands/import_resources_v2.py @@ -1,6 +1,6 @@ from django.core.management import BaseCommand -from core.import_data.import_project_resources_v2 import import_project_resources_v2 +from core.import_data_v2.import_project_resources_v2 import import_project_resources_v2 class Command(BaseCommand): @@ -23,7 +23,9 @@ def add_arguments(self, parser): "import_project_type", "import_sector", "import_subsector", + "import_modules", "import_project_submission_statuses", + "clean_up_countries", "clean_up_project_statuses", "import_cluster_type_sector_links", "import_fields", diff --git a/core/management/commands/import_user_permissions.py b/core/management/commands/import_user_permissions.py index 2a20bd5ff..aa1b8effe 100644 --- a/core/management/commands/import_user_permissions.py +++ b/core/management/commands/import_user_permissions.py @@ -5,9 +5,7 @@ from django.contrib.contenttypes.models import ContentType from django.core.management import BaseCommand -from core.import_data.utils import ( - IMPORT_RESOURCES_DIR, -) +from core.import_data.utils import IMPORT_RESOURCES_V2_DIR logger = logging.getLogger(__name__) @@ -91,11 +89,11 @@ def handle(self, *args, **kwargs): logger.info("✔ permissions cleaned up") if actions in ["all", "import_permissions"]: - file_path = IMPORT_RESOURCES_DIR / "users" / "permissions.json" + file_path = IMPORT_RESOURCES_V2_DIR / "users" / "permissions.json" import_permissions(file_path) logger.info("✔ permissions imported") if actions in ["all", "import_user_groups"]: - file_path = IMPORT_RESOURCES_DIR / "users" / "groups.json" + file_path = IMPORT_RESOURCES_V2_DIR / "users" / "groups.json" import_user_groups(file_path) logger.info("✔ user groups imported") diff --git a/core/migrations/0268_module_country_modules.py b/core/migrations/0268_module_country_modules.py new file mode 100644 index 000000000..40f413d49 --- /dev/null +++ b/core/migrations/0268_module_country_modules.py @@ -0,0 +1,35 @@ +# Generated by Django 4.2.17 on 2026-01-09 10:43 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("core", "0267_alter_projectenterprise_capital_cost_approved_and_more"), + ] + + operations = [ + migrations.CreateModel( + name="Module", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("code", models.CharField(max_length=8, unique=True)), + ("name", models.CharField(max_length=100, unique=True)), + ("description", models.TextField(blank=True, null=True)), + ], + ), + migrations.AddField( + model_name="country", + name="modules", + field=models.ManyToManyField(blank=True, to="core.module"), + ), + ] diff --git a/core/models/base.py b/core/models/base.py index eb63586d2..e6e05ad32 100644 --- a/core/models/base.py +++ b/core/models/base.py @@ -90,3 +90,12 @@ class AbstractSingleton(models.Model): class Meta: abstract = True + + +class Module(models.Model): + code = models.CharField(max_length=8, unique=True) + name = models.CharField(max_length=100, unique=True) + description = models.TextField(blank=True, null=True) + + def __str__(self): + return self.name diff --git a/core/models/country.py b/core/models/country.py index b61cab481..547e9d317 100644 --- a/core/models/country.py +++ b/core/models/country.py @@ -54,6 +54,7 @@ class LocationType(models.TextChoices): location_type = models.CharField( max_length=16, choices=LocationType.choices, default=LocationType.COUNTRY ) + modules = models.ManyToManyField("core.Module", blank=True) import_id = models.IntegerField(null=True, blank=True) is_a2 = models.BooleanField(default=False) consumption_category = models.CharField(max_length=100, blank=True) @@ -78,77 +79,8 @@ def get_business_plan_countries(): """ Returns a list of business plan countries. """ - regions_subregions_entries = ( - Country.objects.filter( - location_type__in=[ - Country.LocationType.REGION, - Country.LocationType.SUBREGION, - ] - ) - .exclude( - name__in=[ - "Africa", - "Europe", - "Asia and the Pacific", - "Global", - "Latin America and the Caribbean", - ] - ) - .values_list("id", flat=True) - ) - return ( - Country.objects.filter( - is_a2=False, - ) - .exclude(id__in=regions_subregions_entries) - .exclude( - name__in=[ - "Andorra", - "Australia", - "Austria", - "Azerbaijan", - "Belarus", - "Belgium", - "Cyprus", - "Czechia", - "Denmark", - "European Union", - "Finland", - "Germany", - "Greece", - "Holy See", - "Hungary", - "Iceland", - "Ireland", - "Israel", - "Italy", - "Japan", - "Latvia", - "Liechtenstein", - "Lithuania", - "Luxembourg", - "Malta", - "Monaco", - "Netherlands (Kingdom of the)", - "New Zealand", - "Norway", - "Poland", - "Portugal", - "Republic of Korea", - "Russian Federation", - "San Marino", - "Singapore", - "Slovakia", - "Slovenia", - "Spain", - "State of Palestine", - "Sweden", - "Switzerland", - "Tajikistan", - "United Kingdom of Great Britain and Northern Ireland", - "Uzbekistan", - ] - ) + return Country.objects.filter( + modules__code="BP", ) class Meta: From a25114843f87a3892ebc19c82eafd18381448560 Mon Sep 17 00:00:00 2001 From: Diana Boiangiu Date: Tue, 17 Feb 2026 16:24:41 +0200 Subject: [PATCH 2/2] Fix merge --- core/api/tests/conftest.py | 5 +++ .../migrations/0268_module_country_modules.py | 35 ------------------- 2 files changed, 5 insertions(+), 35 deletions(-) delete mode 100644 core/migrations/0268_module_country_modules.py diff --git a/core/api/tests/conftest.py b/core/api/tests/conftest.py index 1622df658..123c2eb64 100644 --- a/core/api/tests/conftest.py +++ b/core/api/tests/conftest.py @@ -73,14 +73,17 @@ def user(): return UserFactory(username="FlorinSalam", email="salam@reggaeton.ta") + @pytest.fixture def project_module(): return Module.objects.get_or_create(name="Projects", code="Projects")[0] + @pytest.fixture def business_plan_module(): return Module.objects.get_or_create(name="Business Plans", code="BP")[0] + @pytest.fixture def project_module(): return Module.objects.get_or_create(name="Projects", code="Projects")[0] @@ -312,12 +315,14 @@ def country_europe(project_module, business_plan_module): country.modules.add(project_module, business_plan_module) return country + @pytest.fixture def country_ro(project_module, business_plan_module): country = CountryFactory.create(name="Romania", iso3="ROM") country.modules.add(project_module, business_plan_module) return country + @pytest.fixture def country_viewer_user(country_ro): group = Group.objects.get(name="CP - Viewer") diff --git a/core/migrations/0268_module_country_modules.py b/core/migrations/0268_module_country_modules.py deleted file mode 100644 index 40f413d49..000000000 --- a/core/migrations/0268_module_country_modules.py +++ /dev/null @@ -1,35 +0,0 @@ -# Generated by Django 4.2.17 on 2026-01-09 10:43 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ("core", "0267_alter_projectenterprise_capital_cost_approved_and_more"), - ] - - operations = [ - migrations.CreateModel( - name="Module", - fields=[ - ( - "id", - models.BigAutoField( - auto_created=True, - primary_key=True, - serialize=False, - verbose_name="ID", - ), - ), - ("code", models.CharField(max_length=8, unique=True)), - ("name", models.CharField(max_length=100, unique=True)), - ("description", models.TextField(blank=True, null=True)), - ], - ), - migrations.AddField( - model_name="country", - name="modules", - field=models.ManyToManyField(blank=True, to="core.module"), - ), - ]