diff --git a/core/api/tests/conftest.py b/core/api/tests/conftest.py index b77056826..123c2eb64 100644 --- a/core/api/tests/conftest.py +++ b/core/api/tests/conftest.py @@ -84,6 +84,16 @@ def business_plan_module(): return Module.objects.get_or_create(name="Business Plans", code="BP")[0] +@pytest.fixture +def project_module(): + return Module.objects.get_or_create(name="Projects", code="Projects")[0] + + +@pytest.fixture +def business_plan_module(): + return Module.objects.get_or_create(name="Business Plans", code="BP")[0] + + @pytest.fixture def secretariat_user(): secretariat_group = Group.objects.get(name="CP - Secretariat") diff --git a/core/import_data/utils.py b/core/import_data/utils.py index 6e6895517..c92faacc6 100644 --- a/core/import_data/utils.py +++ b/core/import_data/utils.py @@ -45,6 +45,7 @@ logger = logging.getLogger(__name__) IMPORT_RESOURCES_DIR = settings.ROOT_DIR / "import_data" / "resources" +IMPORT_RESOURCES_V2_DIR = settings.ROOT_DIR / "import_data_v2" / "resources" IMPORT_PROJECTS_DIR = settings.IMPORT_DATA_DIR / "project_database" PCR_DIR_LIST = ["pcr2023", "hpmppcr2023"] diff --git a/core/import_data_v2/__init__.py b/core/import_data_v2/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/core/import_data_v2/import_project_resources_v2.py b/core/import_data_v2/import_project_resources_v2.py new file mode 100644 index 000000000..df5500cc1 --- /dev/null +++ b/core/import_data_v2/import_project_resources_v2.py @@ -0,0 +1,112 @@ +import logging + +from django.db import transaction + +from core.import_data.utils import ( + IMPORT_RESOURCES_V2_DIR, +) +from core.import_data_v2.scripts.import_project_type import import_project_type +from core.import_data_v2.scripts.generate_new_cluster_type_sector_file import ( + generate_new_cluster_type_sector_file, +) +from core.import_data_v2.scripts.import_cluster_type_sector_links import ( + import_cluster_type_sector_links, +) +from core.import_data_v2.scripts.import_fields import ( + import_fields, + import_project_specific_fields, +) +from core.import_data_v2.scripts.import_project_clusters import ( + import_project_clusters, +) +from core.import_data_v2.scripts.clean_up_project_statuses import ( + clean_up_project_statuses, + import_project_submission_statuses, +) +from core.import_data_v2.scripts.import_sector_subsector import ( + import_sector, + import_subsector, +) +from core.import_data_v2.scripts.clean_up_countries import ( + clean_up_countries, +) +from core.import_data_v2.scripts.import_modules import import_modules + +logger = logging.getLogger(__name__) + + +@transaction.atomic +def import_project_resources_v2(option): + + if option in ["all", "import_project_clusters"]: + file_path = ( + IMPORT_RESOURCES_V2_DIR / "projects" / "project_clusters_06_05_2025.xlsx" + ) + import_project_clusters(file_path) + logger.info("✔ project clusters imported") + + if option in ["all", "import_project_type"]: + file_path = ( + IMPORT_RESOURCES_V2_DIR / "projects" / "tbTypeOfProject_06_05_2025.json" + ) + import_project_type(file_path) + logger.info("✔ project types imported") + + if option in ["all", "import_sector"]: + file_path = IMPORT_RESOURCES_V2_DIR / "projects" / "tbSector_15_10_2025.json" + import_sector(file_path) + logger.info("✔ sectors imported") + + if option in ["all", "import_subsector"]: + file_path = IMPORT_RESOURCES_V2_DIR / "projects" / "tbSubsector_06_05_2025.json" + import_subsector(file_path) + logger.info("✔ subsectors imported") + + if option in ["all", "import_project_submission_statuses"]: + file_path = ( + IMPORT_RESOURCES_V2_DIR / "projects" / "project_submission_statuses.json" + ) + import_project_submission_statuses(file_path) + logger.info("✔ project submission statuses imported") + + if option in ["all", "clean_up_project_statuses"]: + clean_up_project_statuses() + logger.info("✔ project statuses cleaned up") + + if option in ["all", "import_cluster_type_sector_links"]: + file_path = IMPORT_RESOURCES_V2_DIR / "projects" / "ClusterTypeSectorLinks.json" + import_cluster_type_sector_links(file_path) + logger.info("✔ cluster type sector links imported") + + if option in ["all", "import_fields"]: + file_path = IMPORT_RESOURCES_V2_DIR / "projects" / "Fields_24_10_2025.json" + import_fields(file_path) + logger.info("✔ fields imported") + + if option in ["all", "import_project_specific_fields"]: + file_path = ( + IMPORT_RESOURCES_V2_DIR + / "projects" + / "project_specific_fields_27_10_2025.xlsx" + ) + import_project_specific_fields(file_path) + logger.info("✔ cluster type sector fields imported") + if option in ["all", "import_modules"]: + import_modules() + logger.info("✔ modules imported") + if option in [ + "all", + "clean_up_countries", + ]: + clean_up_countries() + logger.info("✔ countries cleaned up") + + if option == "generate_new_cluster_type_sector_file": + # use to generate new ClusterTypeSectorLinks.json file + file_path = ( + IMPORT_RESOURCES_V2_DIR + / "projects" + / "project_specific_fields_27_10_2025.xlsx" + ) + generate_new_cluster_type_sector_file(file_path) + logger.info("✔ new cluster type sector file generated") diff --git a/core/import_data_v2/resources/countries/MLF Countries and Regions.xlsx b/core/import_data_v2/resources/countries/MLF Countries and Regions.xlsx new file mode 100644 index 000000000..4d59f77c9 Binary files /dev/null and b/core/import_data_v2/resources/countries/MLF Countries and Regions.xlsx differ diff --git a/core/import_data/resources/projects_v2/ClusterTypeSectorLinks.json b/core/import_data_v2/resources/projects/ClusterTypeSectorLinks.json similarity index 100% rename from core/import_data/resources/projects_v2/ClusterTypeSectorLinks.json rename to core/import_data_v2/resources/projects/ClusterTypeSectorLinks.json diff --git a/core/import_data/resources/projects_v2/Fields_24_10_2025.json b/core/import_data_v2/resources/projects/Fields_24_10_2025.json similarity index 100% rename from core/import_data/resources/projects_v2/Fields_24_10_2025.json rename to core/import_data_v2/resources/projects/Fields_24_10_2025.json diff --git a/core/import_data/resources/projects_v2/MLF_Countries_and_Regions_for_BP_Projects_modules.xlsx b/core/import_data_v2/resources/projects/MLF_Countries_and_Regions_for_BP_Projects_modules.xlsx similarity index 100% rename from core/import_data/resources/projects_v2/MLF_Countries_and_Regions_for_BP_Projects_modules.xlsx rename to core/import_data_v2/resources/projects/MLF_Countries_and_Regions_for_BP_Projects_modules.xlsx diff --git a/core/import_data_v2/resources/projects/modules.json b/core/import_data_v2/resources/projects/modules.json new file mode 100644 index 000000000..e69de29bb diff --git a/core/import_data/resources/projects_v2/project_clusters_06_05_2025.xlsx b/core/import_data_v2/resources/projects/project_clusters_06_05_2025.xlsx similarity index 100% rename from core/import_data/resources/projects_v2/project_clusters_06_05_2025.xlsx rename to core/import_data_v2/resources/projects/project_clusters_06_05_2025.xlsx diff --git a/core/import_data/resources/projects_v2/project_specific_fields_15_01_2026.xlsx b/core/import_data_v2/resources/projects/project_specific_fields_15_01_2026.xlsx similarity index 100% rename from core/import_data/resources/projects_v2/project_specific_fields_15_01_2026.xlsx rename to core/import_data_v2/resources/projects/project_specific_fields_15_01_2026.xlsx diff --git a/core/import_data_v2/resources/projects/project_specific_fields_24_10_2025.xlsx b/core/import_data_v2/resources/projects/project_specific_fields_24_10_2025.xlsx new file mode 100644 index 000000000..64651f4e9 Binary files /dev/null and b/core/import_data_v2/resources/projects/project_specific_fields_24_10_2025.xlsx differ diff --git a/core/import_data_v2/resources/projects/project_specific_fields_27_10_2025.xlsx b/core/import_data_v2/resources/projects/project_specific_fields_27_10_2025.xlsx new file mode 100644 index 000000000..391f97c77 Binary files /dev/null and b/core/import_data_v2/resources/projects/project_specific_fields_27_10_2025.xlsx differ diff --git a/core/import_data/resources/projects_v2/project_submission_statuses.json b/core/import_data_v2/resources/projects/project_submission_statuses.json similarity index 100% rename from core/import_data/resources/projects_v2/project_submission_statuses.json rename to core/import_data_v2/resources/projects/project_submission_statuses.json diff --git a/core/import_data/resources/projects_v2/tbSector_15_10_2025.json b/core/import_data_v2/resources/projects/tbSector_15_10_2025.json similarity index 100% rename from core/import_data/resources/projects_v2/tbSector_15_10_2025.json rename to core/import_data_v2/resources/projects/tbSector_15_10_2025.json diff --git a/core/import_data/resources/projects_v2/tbSubsector_06_05_2025.json b/core/import_data_v2/resources/projects/tbSubsector_06_05_2025.json similarity index 100% rename from core/import_data/resources/projects_v2/tbSubsector_06_05_2025.json rename to core/import_data_v2/resources/projects/tbSubsector_06_05_2025.json diff --git a/core/import_data/resources/projects_v2/tbTypeOfProject_06_05_2025.json b/core/import_data_v2/resources/projects/tbTypeOfProject_06_05_2025.json similarity index 100% rename from core/import_data/resources/projects_v2/tbTypeOfProject_06_05_2025.json rename to core/import_data_v2/resources/projects/tbTypeOfProject_06_05_2025.json diff --git a/core/import_data/resources/users/groups.json b/core/import_data_v2/resources/users/groups.json similarity index 100% rename from core/import_data/resources/users/groups.json rename to core/import_data_v2/resources/users/groups.json diff --git a/core/import_data/resources/users/permissions.json b/core/import_data_v2/resources/users/permissions.json similarity index 100% rename from core/import_data/resources/users/permissions.json rename to core/import_data_v2/resources/users/permissions.json diff --git a/core/import_data_v2/scripts/__init__.py b/core/import_data_v2/scripts/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/core/import_data_v2/scripts/clean_up_countries.py b/core/import_data_v2/scripts/clean_up_countries.py new file mode 100644 index 000000000..95efcc441 --- /dev/null +++ b/core/import_data_v2/scripts/clean_up_countries.py @@ -0,0 +1,58 @@ +import logging +import pandas as pd + +from django.db import transaction + +from core.models.base import Module +from core.models.country import Country +from core.import_data.utils import ( + IMPORT_RESOURCES_V2_DIR, +) + +logger = logging.getLogger(__name__) + + +@transaction.atomic +def clean_up_countries(): + """ + Clean up country names + Set modules for countries + """ + # clean up country names + country_name_corrections = { + "Bolivia (Plurinational State of)": "Bolivia", + "Cabo Verde": "Cape Verde", + "Côte d'Ivoire": "Cote D'Ivoire", + "Syrian Arab Republic": "Syria", + "Timor-Leste": "Timor Leste", + "Viet Nam": "Vietnam", + "Venezuela (Bolivarian Republic of)": "Venezuela", + "Türkiye": "Turkey", + "United Republic of Tanzania": "Tanzania", + "Iran (Islamic Republic of)": "Iran", + "Guinea Bissau": "Guinea-Bissau", + "Micronesia (Federated States of)": "Micronesia", + } + + # for old_name, new_name in country_name_corrections.items(): + # country = Country.objects.filter(name=old_name).first() + # if country: + # country.name = new_name + # country.save() + # logger.info(f"✔ Country name updated from '{old_name}' to '{new_name}'") + + # set modules for countries + projects_module = Module.objects.filter(code="Projects").first() + business_plans_module = Module.objects.filter(code="BP").first() + file_path = IMPORT_RESOURCES_V2_DIR / "countries" / "MLF Countries and Regions.xlsx" + df = pd.read_excel(file_path).fillna("") + + for _, row in df.iterrows(): + country = Country.objects.find_by_name(row["Countries"]) + if not country: + logger.warning(f"⚠️ Country '{row['Countries']}' not found") + continue + country.modules.clear() + country.modules.add(projects_module) + country.modules.add(business_plans_module) + country.save() diff --git a/core/import_data_v2/scripts/clean_up_project_statuses.py b/core/import_data_v2/scripts/clean_up_project_statuses.py new file mode 100644 index 000000000..8360cc6c9 --- /dev/null +++ b/core/import_data_v2/scripts/clean_up_project_statuses.py @@ -0,0 +1,67 @@ +import json +import logging + +from django.db import transaction + +from core.models.project_metadata import ProjectStatus, ProjectSubmissionStatus +from core.models.project import Project + +logger = logging.getLogger(__name__) + + +@transaction.atomic +def clean_up_project_statuses(): + """ + Clean up project statuses + Remove outdated statuses and add new ones + """ + # remove Unknown status only if there are no projects with this status + + if Project.objects.really_all().filter(status__code="UNK").exists(): + logger.warning( + "⚠️ Cannot remove 'Unknown' status, there are projects with this status." + ) + else: + ProjectStatus.objects.filter(code="UNK").delete() + + # change the status 'New submission' into 'N/A' and delete status 'New submission' + + new_submission_status, _ = ProjectStatus.objects.update_or_create( + name="N/A", + defaults={ + "code": "NA", + }, + ) + Project.objects.really_all().filter(status__code="NEWSUB").update( + status=new_submission_status + ) + ProjectStatus.objects.filter(code="NEWSUB").delete() + + # change the status 'Newly approved' into 'Ongoing' and delete status 'Newly approved' + + on_going_status = ProjectStatus.objects.filter(name="Ongoing").first() + Project.objects.really_all().filter(status__code="NEW").update( + status=on_going_status + ) + ProjectStatus.objects.filter(code="NEW").delete() + + +@transaction.atomic +def import_project_submission_statuses(file_path): + """ + Import project submission statuses from file + + @param file_path = str (file path for import file) + """ + + with open(file_path, "r", encoding="utf8") as f: + statuses_json = json.load(f) + + for status_json in statuses_json: + status_data = { + "name": status_json["STATUS"], + "code": status_json["STATUS_CODE"], + } + ProjectSubmissionStatus.objects.update_or_create( + name=status_data["name"], defaults=status_data + ) diff --git a/core/import_data_v2/scripts/generate_new_cluster_type_sector_file.py b/core/import_data_v2/scripts/generate_new_cluster_type_sector_file.py new file mode 100644 index 000000000..3f1cf25fa --- /dev/null +++ b/core/import_data_v2/scripts/generate_new_cluster_type_sector_file.py @@ -0,0 +1,45 @@ +import json +import pandas as pd + + +def generate_new_cluster_type_sector_file(file_path): + """ + Generate new cluster type sector file based on the current data in the database + + @param file_path = str (file path for import file) + """ + combinations = {} # {cluster: {type: [sectors]}} + df = pd.read_excel(file_path).fillna("") + + for _, row in df.iterrows(): + if row["Project type name"].strip() == "Project preparation": + row["Project type name"] = "Preparation" + + if row["Sector name"].strip() == "Other Sector": + continue + + combinations.setdefault(row["Cluster name"].strip(), {}) + combinations[row["Cluster name"].strip()].setdefault( + row["Project type name"].strip(), [] + ) + combinations[row["Cluster name"].strip()][ + row["Project type name"].strip() + ].append(row["Sector name"].strip()) + + new_data = [] + for cluster_name, types in combinations.items(): + new_data.append( + { + "cluster": cluster_name, + "types": [ + { + "type": type_name, + "sectors": sorted(list(set(sector_names))), # remove duplicates + } + for type_name, sector_names in types.items() + ], + } + ) + + with open("new_ClusterTypeSectorLinks.json", "w", encoding="utf8") as f: + json.dump(new_data, f, indent=4) diff --git a/core/import_data_v2/scripts/import_cluster_type_sector_links.py b/core/import_data_v2/scripts/import_cluster_type_sector_links.py new file mode 100644 index 000000000..50a6b75c1 --- /dev/null +++ b/core/import_data_v2/scripts/import_cluster_type_sector_links.py @@ -0,0 +1,58 @@ +import json +import logging + +from django.db import transaction + +from core.models.project_metadata import ( + ProjectCluster, + ProjectSpecificFields, + ProjectSector, + ProjectType, +) + +logger = logging.getLogger(__name__) + + +@transaction.atomic +def import_cluster_type_sector_links(file_path): + """ + Import links between cluster, type and sector from file + + @param file_path = str (file path for import file) + """ + with open(file_path, "r", encoding="utf8") as f: + data = json.load(f) + + for cluster_json in data: + cluster = ProjectCluster.objects.filter(name=cluster_json["cluster"]).first() + if not cluster: + logger.warning( + f"⚠️ {cluster_json['cluster']} cluster not found => {cluster_json['cluster']} not imported" + ) + continue + for type_json in cluster_json["types"]: + type_name = type_json["type"] + if type_name == "Project Support": + type_name = "Project support" + type_obj = ProjectType.objects.filter(name=type_name).first() + if not type_obj: + logger.warning( + f"⚠️ {type_name} type not found => {cluster_json['cluster']} not imported" + ) + continue + for sector_name in type_json["sectors"]: + if sector_name == "Control Submstance Monitoring": + sector_name = "Control Substance Monitoring" + if sector_name == "Compliance Assistance Program": + sector_name = "Compliance Assistance Programme" + sector = ProjectSector.objects.filter(name=sector_name).first() + if not sector: + logger.warning( + f"⚠️ {sector_name} sector not found => {cluster_json['cluster']} not imported" + ) + continue + ProjectSpecificFields.objects.update_or_create( + cluster=cluster, + type=type_obj, + sector=sector, + ) diff --git a/core/import_data_v2/scripts/import_fields.py b/core/import_data_v2/scripts/import_fields.py new file mode 100644 index 000000000..5e4374bb3 --- /dev/null +++ b/core/import_data_v2/scripts/import_fields.py @@ -0,0 +1,214 @@ +import logging +import json +import pandas as pd + +from django.db import transaction + +from core.models.project_metadata import ( + ProjectSpecificFields, + ProjectField, +) + +logger = logging.getLogger(__name__) + +# pylint: disable=C0301 + +FIELDS_WITH_ACTUAL_VALUES = [ + "Total number of technicians trained", + "Number of female technicians trained", + "Total number of trainers trained", + "Number of female trainers trained", + "Total number of technicians certified", + "Number of female technicians certified", + "Number of training institutions newly  assisted", + "Number of tools sets distributed", + "Total number of customs officers trained", + "Number of female customs officers trained", + "Total number of NOU personnel supported", + "Number of female NOU personnel supported", + "Certification system for technicians established or further enhanced (yes or no)", + "Operation of recovery and recycling scheme (yes or no)", + "Operation of reclamation scheme (yes or no)", + "Establishment or upgrade of Import/export licensing (yes or no)", + "Establishment of quota systems (yes or no)", + "Ban of equipment (number)", + "Ban of substances (number)", + "kWh/year saved", + "MEPS developed for domestic refrigeration (yes/no)", + "MEPS developed for commercial refrigeration (yes/no)", + "MEPS developed for residential air-conditioning (yes/no)", + "MEPS developed for commercial AC (yes/no)", + "Capacity building programmes (checklist (yes/no) for technicians, end-users, operators, consultants, procurement officers and other Government entities)", + "EE demonstration project included (yes/no)", + "Quantity of controlled substances destroyed (M t)", + "Quantity of controlled substances destroyed (CO2-eq t)", + "Checklist of regulations or policies enacted", + "Quantity of HFC-23 by-product (Generated)", + "Quantity of HFC-23 by-product (by product generation rate)", + "Quantity of HFC-23 by-product (Destroyed)", + "Quantity of HFC-23 by-product (Emitted)", + "Number of Production Lines assisted", + "Number of enterprises assisted", + "Number of enterprises", + "Aggregated consumption", + "Cost effectiveness (US$/ Kg)", + "Cost effectiveness (US$/ CO2-eq)", +] + + +@transaction.atomic +def import_fields(file_path): + """ + Import project type from file + + @param file_path = str (file path for import file) + """ + + ProjectField.objects.all().delete() + with open(file_path, "r", encoding="utf8") as f: + fields_json = json.load(f) + + # add other types that are not in the file + ProjectField.objects.all().delete() + for field_json in fields_json: + + field_data = { + "import_name": field_json["IMPORT_NAME"], + "label": field_json["LABEL"], + "read_field_name": field_json["READ_FIELD_NAME"], + "write_field_name": field_json["WRITE_FIELD_NAME"], + "table": field_json["TABLE"], + "data_type": field_json["DATA_TYPE"], + "mlfs_only": field_json.get("MLFS_ONLY", False), + "section": field_json["SECTION"], + "is_actual": field_json.get("IS_ACTUAL", False), + "sort_order": field_json["SORT_ORDER"], + "editable_in_versions": ",".join( + [str(version) for version in field_json["EDITABLE_IN_VERSIONS"]] + ), + "visible_in_versions": ",".join( + [str(version) for version in field_json["VISIBLE_IN_VERSIONS"]] + ), + } + + ProjectField.objects.update_or_create( + import_name=field_data["import_name"], defaults=field_data + ) + + +@transaction.atomic +def import_project_specific_fields(file_path): + """ + Import project clusters from file + Please make sure that the file has the correct extention + (xls, xlsx, xlsm, xlsb, odf, ods, odt) + + @param file_path = str (file path for import file) + """ + + def _clean_up_field_name(field_name, mya=False): + """ + Clean up field name + """ + mya_clean_up = { + "Cost effectiveness (US$/ CO2-ep) (MYA)": "Cost effectiveness (US$/ CO2-eq) (MYA)", + "Cost effectiveness (US$/ CO2-ep)": "Cost effectiveness (US$/ CO2-eq) (MYA)", + "Cost effectiveness (US$/ CO2-eq)": "Cost effectiveness (US$/ CO2-eq) (MYA)", + "Aggregated consumption": "Aggregated consumption (MYA)", + "Cost effectiveness (US$/ Kg)": "Cost effectiveness (US$/ Kg) (MYA)", + "Number of enterprises assisted": "Number of enterprises assisted (MYA)", + "Number of enterprises": "Number of enterprises (MYA)", + "Number of Production Lines assisted": "Number of Production Lines assisted (MYA)", + } + + individual_field_clean_up = { + "Cost effectiveness (US$/ CO2-ep)": "Cost effectiveness (US$/ CO2-eq)", + "Cost effectiveness (US$/ CO2-ep) (MYA)": "Cost effectiveness (US$/ CO2-eq)", + "Phase out (Mt) (MYA)": "Phase out (Mt)", + "Phase out (M t)": "Phase out (Mt)", + "Phase out (CO2-eq t) (MYA)": "Phase out (CO2-eq t)", + "Cost effectiveness (US$/ CO2-eq)": "Cost effectiveness (US$/ CO2-eq)", + "Phase out (ODP t) (MYA)": "Phase out (ODP t)", + } + if mya: + if field_name in mya_clean_up: + return mya_clean_up[field_name] + else: + if field_name in individual_field_clean_up: + return individual_field_clean_up[field_name] + return field_name.strip().replace(" ", " ") + + df = pd.read_excel(file_path).fillna("") + + for _, row in df.iterrows(): + if row["Project type name"].strip() == "Project preparation": + row["Project type name"] = "Preparation" + if row["Sector name"].strip() == "Control Submstance Monitoring": + row["Sector name"] = "Control Substance Monitoring" + if row["Sector name"].strip() == "Compliance Assistance Program": + row["Sector name"] = "Compliance Assistance Programme" + if row["Sector name"] == "Other Sector": + continue + try: + cluster_sector_type = ProjectSpecificFields.objects.get( + cluster__name__iexact=row["Cluster name"].strip(), + type__name__iexact=row["Project type name"].strip(), + sector__name__iexact=row["Sector name"].strip(), + ) + except ProjectSpecificFields.DoesNotExist: + logger.warning( + f"⚠️ {row['Cluster name']}/{row['Project type name']}/{row['Sector name']} not found." + ) + continue + + cluster_sector_type.fields.clear() + + # particular fields start from row 22 + # Extract MYA fields separately as some names are dupliated in the impact section + field_names_excluding_mya = [ + _clean_up_field_name(row[field_index].strip()) + for field_index in range(22, 49) + if row[field_index] != "" + ] + + # search for fields that also have an actual field that is not in the file + # and add them to the list of fields to be added (for Impact fields) + actual_field_names = [ + f"{field_name} actual" + for field_name in field_names_excluding_mya + if field_name in FIELDS_WITH_ACTUAL_VALUES + ] + field_names_excluding_mya.extend(actual_field_names) + project_fields = ProjectField.objects.exclude(section="MYA").filter( + import_name__in=field_names_excluding_mya + ) + + missing_fields = set(field_names_excluding_mya) - set( + project_fields.values_list("import_name", flat=True) + ) + + for missing_field in missing_fields: + logger.warning( + f"⚠️ {missing_field} field not found =>" + + f"{row['Cluster name']}/{row['Project type name']}/{row['Sector name']}" + ) + cluster_sector_type.fields.add(*project_fields) + + mya_field_names = [ + _clean_up_field_name(row[field_index].strip(), mya=True) + for field_index in range(49, len(row) - 1) + if row[field_index] != "" + ] + project_fields = ProjectField.objects.filter( + import_name__in=mya_field_names, section="MYA" + ) + missing_fields = set(mya_field_names) - set( + project_fields.values_list("import_name", flat=True) + ) + for missing_field in missing_fields: + logger.warning( + f"⚠️ {missing_field} field not found =>" + + f"{row['Cluster name']}/{row['Project type name']}/{row['Sector name']}" + ) + + cluster_sector_type.fields.add(*project_fields) diff --git a/core/import_data_v2/scripts/import_modules.py b/core/import_data_v2/scripts/import_modules.py new file mode 100644 index 000000000..6c1b6b9ba --- /dev/null +++ b/core/import_data_v2/scripts/import_modules.py @@ -0,0 +1,20 @@ +from django.db import transaction + +from core.models.base import Module + + +@transaction.atomic +def import_modules(): + """ + Import modules + """ + modules = [ + {"name": "Projects", "code": "Projects"}, + {"name": "Business Plans", "code": "BP"}, + {"name": "Country Programmes", "code": "CP"}, + ] + + for module_data in modules: + Module.objects.update_or_create( + code=module_data["code"], defaults={"name": module_data["name"]} + ) diff --git a/core/import_data_v2/scripts/import_project_clusters.py b/core/import_data_v2/scripts/import_project_clusters.py new file mode 100644 index 000000000..3d4c009a6 --- /dev/null +++ b/core/import_data_v2/scripts/import_project_clusters.py @@ -0,0 +1,59 @@ +import logging +import pandas as pd + +from django.db import transaction + +from core.models.group import Group +from core.models.project_metadata import ProjectCluster + + +logger = logging.getLogger(__name__) + + +@transaction.atomic +def import_project_clusters(file_path): + """ + Import project clusters from file + Please make sure that the file has the correct extention + (xls, xlsx, xlsm, xlsb, odf, ods, odt) + + @param file_path = str (file path for import file) + """ + + df = pd.read_excel(file_path).fillna("") + + for index, row in df.iterrows(): + if row["Action"] == "Outdated": + continue + if row["Action"] == "Rename": + ProjectCluster.objects.filter(name=row["Old name"]).update(name=row["Name"]) + + production = False + if row["Production"] == "Y": + production = True + elif row["Production"] == "Both": + production = None + + # get annex groups + annex_groups = [] + if row["Annex groups"]: + annex_groups_name_alt = row["Annex groups"].split(",") + annex_groups = Group.objects.filter(name__in=annex_groups_name_alt) + if annex_groups.count() != len(annex_groups_name_alt): + logger.warning( + f"⚠️ Some annex groups not found for cluster {row['Name']}" + ) + cluster_data = { + "name": row["Name"], + "code": row["Acronym"], + "category": row["Category"].upper(), + "group": row["Dashboard group"], + "production": production, + "sort_order": index, + } + + cluster, _ = ProjectCluster.objects.update_or_create( + name=cluster_data["name"], defaults=cluster_data + ) + if annex_groups: + cluster.annex_groups.set(annex_groups) diff --git a/core/import_data_v2/scripts/import_project_type.py b/core/import_data_v2/scripts/import_project_type.py new file mode 100644 index 000000000..ae3353896 --- /dev/null +++ b/core/import_data_v2/scripts/import_project_type.py @@ -0,0 +1,36 @@ +import json +import logging + +from django.db import transaction + +from core.models.project_metadata import ProjectType + + +logger = logging.getLogger(__name__) + + +@transaction.atomic +def import_project_type(file_path): + """ + Import project type from file + + @param file_path = str (file path for import file) + """ + with open(file_path, "r", encoding="utf8") as f: + types_json = json.load(f) + + # add other types that are not in the file + for type_json in types_json: + if type_json.get("ACTION", None) == "RENAME": + ProjectType.objects.filter(name=type_json["OLD_NAME"]).update( + name=type_json["TYPE_PRO"] + ) + else: + type_data = { + "code": type_json["TYPE"], + "name": type_json["TYPE_PRO"], + "sort_order": type_json["SORT_TYPE"], + } + ProjectType.objects.update_or_create( + name=type_data["name"], defaults=type_data + ) diff --git a/core/import_data_v2/scripts/import_sector_subsector.py b/core/import_data_v2/scripts/import_sector_subsector.py new file mode 100644 index 000000000..db82eb00a --- /dev/null +++ b/core/import_data_v2/scripts/import_sector_subsector.py @@ -0,0 +1,103 @@ +import json +import logging + +from django.db import transaction + +from core.models.project_metadata import ( + ProjectSector, + ProjectSubSector, +) + +logger = logging.getLogger(__name__) + + +@transaction.atomic +def import_sector(file_path): + """ + Import sectors and subsectors from file + + @param file_path = str (file path for import file) + """ + + with open(file_path, "r", encoding="utf8") as f: + sectors_json = json.load(f) + + for sector_json in sectors_json: + + if sector_json.get("ACTION", None) == "RENAME": + ProjectSector.objects.filter(name=sector_json["OLD_NAME"]).update( + name=sector_json["SECTOR"] + ) + sector_data = { + "name": sector_json["SECTOR"].strip(), + "code": sector_json["SEC"].strip(), + "sort_order": sector_json["SORT_SECTOR"], + } + ProjectSector.objects.update_or_create( + code=sector_data["code"], defaults=sector_data + ) + + +@transaction.atomic +def import_subsector(file_path): + """ + Import sectors and subsectors from file + Please make sure that the file has the correct extention + (xls, xlsx, xlsm, xlsb, odf, ods, odt) + + @param file_path = str (file path for import file) + """ + + with open(file_path, "r", encoding="utf8") as f: + subsectors_json = json.load(f) + + for subsector_json in subsectors_json: + subsector_name = subsector_json["SUBSECTOR"].strip() + if subsector_json.get("ACTION", None) == "RENAME": + project_sub_sector = ProjectSubSector.objects.filter( + name=subsector_json["OLD_NAME"] + ) + if subsector_json.get("OLD_SEC", None): + project_sub_sector = project_sub_sector.filter( + sector__code=subsector_json["OLD_SEC"] + ) + project_sub_sector.update(name=subsector_name) + + # get sector + sector = ProjectSector.objects.filter(code=subsector_json["SEC"]).first() + if not sector: + logger.warning( + f"⚠️ {subsector_json['SEC']} sector not found => {subsector_json['SUBSECTOR']} not imported" + ) + continue + + # set subsector data + subsector_code = ( + subsector_json["CODE_SUBSECTOR"].strip() + if subsector_json.get("CODE_SUBSECTOR") + else None + ) + subsector_data = { + "name": subsector_json["SUBSECTOR"].strip(), + "code": subsector_code, + "sector": sector, + "sort_order": subsector_json["SORT_SUBSECTOR"], + } + + project_sub_sector = ProjectSubSector.objects.filter( + name=subsector_data["name"] + ) + if project_sub_sector.exists(): + project_sub_sector.update( + name=subsector_data["name"], + code=subsector_data["code"], + sector=subsector_data["sector"], + sort_order=subsector_data["sort_order"], + ) + else: + ProjectSubSector.objects.create( + name=subsector_data["name"], + code=subsector_data["code"], + sector=subsector_data["sector"], + sort_order=subsector_data["sort_order"], + ) diff --git a/core/management/commands/import_resources_v2.py b/core/management/commands/import_resources_v2.py index a114bcbcb..ef84c2608 100644 --- a/core/management/commands/import_resources_v2.py +++ b/core/management/commands/import_resources_v2.py @@ -1,6 +1,6 @@ from django.core.management import BaseCommand -from core.import_data.import_project_resources_v2 import import_project_resources_v2 +from core.import_data_v2.import_project_resources_v2 import import_project_resources_v2 class Command(BaseCommand): diff --git a/core/management/commands/import_user_permissions.py b/core/management/commands/import_user_permissions.py index 2a20bd5ff..aa1b8effe 100644 --- a/core/management/commands/import_user_permissions.py +++ b/core/management/commands/import_user_permissions.py @@ -5,9 +5,7 @@ from django.contrib.contenttypes.models import ContentType from django.core.management import BaseCommand -from core.import_data.utils import ( - IMPORT_RESOURCES_DIR, -) +from core.import_data.utils import IMPORT_RESOURCES_V2_DIR logger = logging.getLogger(__name__) @@ -91,11 +89,11 @@ def handle(self, *args, **kwargs): logger.info("✔ permissions cleaned up") if actions in ["all", "import_permissions"]: - file_path = IMPORT_RESOURCES_DIR / "users" / "permissions.json" + file_path = IMPORT_RESOURCES_V2_DIR / "users" / "permissions.json" import_permissions(file_path) logger.info("✔ permissions imported") if actions in ["all", "import_user_groups"]: - file_path = IMPORT_RESOURCES_DIR / "users" / "groups.json" + file_path = IMPORT_RESOURCES_V2_DIR / "users" / "groups.json" import_user_groups(file_path) logger.info("✔ user groups imported")