Add Country modules scripts/Refactor import_resources_v2

core/api/tests/conftest.py

@@ -84,6 +84,16 @@ def business_plan_module():
     return Module.objects.get_or_create(name="Business Plans", code="BP")[0]
 
 
+@pytest.fixture
+def project_module():
+    return Module.objects.get_or_create(name="Projects", code="Projects")[0]
+
+
+@pytest.fixture
+def business_plan_module():
+    return Module.objects.get_or_create(name="Business Plans", code="BP")[0]
+
+
 @pytest.fixture
 def secretariat_user():
     secretariat_group = Group.objects.get(name="CP - Secretariat")

core/import_data/utils.py

@@ -45,6 +45,7 @@
 logger = logging.getLogger(__name__)
 
 IMPORT_RESOURCES_DIR = settings.ROOT_DIR / "import_data" / "resources"
+IMPORT_RESOURCES_V2_DIR = settings.ROOT_DIR / "import_data_v2" / "resources"
 IMPORT_PROJECTS_DIR = settings.IMPORT_DATA_DIR / "project_database"
 
 PCR_DIR_LIST = ["pcr2023", "hpmppcr2023"]

core/import_data_v2/import_project_resources_v2.py

@@ -0,0 +1,112 @@
+import logging
+
+from django.db import transaction
+
+from core.import_data.utils import (
+    IMPORT_RESOURCES_V2_DIR,
+)
+from core.import_data_v2.scripts.import_project_type import import_project_type
+from core.import_data_v2.scripts.generate_new_cluster_type_sector_file import (
+    generate_new_cluster_type_sector_file,
+)
+from core.import_data_v2.scripts.import_cluster_type_sector_links import (
+    import_cluster_type_sector_links,
+)
+from core.import_data_v2.scripts.import_fields import (
+    import_fields,
+    import_project_specific_fields,
+)
+from core.import_data_v2.scripts.import_project_clusters import (
+    import_project_clusters,
+)
+from core.import_data_v2.scripts.clean_up_project_statuses import (
+    clean_up_project_statuses,
+    import_project_submission_statuses,
+)
+from core.import_data_v2.scripts.import_sector_subsector import (
+    import_sector,
+    import_subsector,
+)
+from core.import_data_v2.scripts.clean_up_countries import (
+    clean_up_countries,
+)
+from core.import_data_v2.scripts.import_modules import import_modules
+
+logger = logging.getLogger(__name__)
+
+
+@transaction.atomic
+def import_project_resources_v2(option):
+
+    if option in ["all", "import_project_clusters"]:
+        file_path = (
+            IMPORT_RESOURCES_V2_DIR / "projects" / "project_clusters_06_05_2025.xlsx"
+        )
+        import_project_clusters(file_path)
+        logger.info("✔ project clusters imported")
+
+    if option in ["all", "import_project_type"]:
+        file_path = (
+            IMPORT_RESOURCES_V2_DIR / "projects" / "tbTypeOfProject_06_05_2025.json"
+        )
+        import_project_type(file_path)
+        logger.info("✔ project types imported")
+
+    if option in ["all", "import_sector"]:
+        file_path = IMPORT_RESOURCES_V2_DIR / "projects" / "tbSector_15_10_2025.json"
+        import_sector(file_path)
+        logger.info("✔ sectors imported")
+
+    if option in ["all", "import_subsector"]:
+        file_path = IMPORT_RESOURCES_V2_DIR / "projects" / "tbSubsector_06_05_2025.json"
+        import_subsector(file_path)
+        logger.info("✔ subsectors imported")
+
+    if option in ["all", "import_project_submission_statuses"]:
+        file_path = (
+            IMPORT_RESOURCES_V2_DIR / "projects" / "project_submission_statuses.json"
+        )
+        import_project_submission_statuses(file_path)
+        logger.info("✔ project submission statuses imported")
+
+    if option in ["all", "clean_up_project_statuses"]:
+        clean_up_project_statuses()
+        logger.info("✔ project statuses cleaned up")
+
+    if option in ["all", "import_cluster_type_sector_links"]:
+        file_path = IMPORT_RESOURCES_V2_DIR / "projects" / "ClusterTypeSectorLinks.json"
+        import_cluster_type_sector_links(file_path)
+        logger.info("✔ cluster type sector links imported")
+
+    if option in ["all", "import_fields"]:
+        file_path = IMPORT_RESOURCES_V2_DIR / "projects" / "Fields_24_10_2025.json"
+        import_fields(file_path)
+        logger.info("✔ fields imported")
+
+    if option in ["all", "import_project_specific_fields"]:
+        file_path = (
+            IMPORT_RESOURCES_V2_DIR
+            / "projects"
+            / "project_specific_fields_27_10_2025.xlsx"
+        )
+        import_project_specific_fields(file_path)
+        logger.info("✔ cluster type sector fields imported")
+    if option in ["all", "import_modules"]:
+        import_modules()
+        logger.info("✔ modules imported")
+    if option in [
+        "all",
+        "clean_up_countries",
+    ]:
+        clean_up_countries()
+        logger.info("✔ countries cleaned up")
+
+    if option == "generate_new_cluster_type_sector_file":
+        # use to generate new ClusterTypeSectorLinks.json file
+        file_path = (
+            IMPORT_RESOURCES_V2_DIR
+            / "projects"
+            / "project_specific_fields_27_10_2025.xlsx"
+        )
+        generate_new_cluster_type_sector_file(file_path)
+        logger.info("✔ new cluster type sector file generated")

core/import_data_v2/scripts/clean_up_countries.py

@@ -0,0 +1,58 @@
+import logging
+import pandas as pd
+
+from django.db import transaction
+
+from core.models.base import Module
+from core.models.country import Country
+from core.import_data.utils import (
+    IMPORT_RESOURCES_V2_DIR,
+)
+
+logger = logging.getLogger(__name__)
+
+
+@transaction.atomic
+def clean_up_countries():
+    """
+    Clean up country names
+    Set modules for countries
+    """
+    # clean up country names
+    country_name_corrections = {
+        "Bolivia (Plurinational State of)": "Bolivia",
+        "Cabo Verde": "Cape Verde",
+        "Côte d'Ivoire": "Cote D'Ivoire",
+        "Syrian Arab Republic": "Syria",
+        "Timor-Leste": "Timor Leste",
+        "Viet Nam": "Vietnam",
+        "Venezuela (Bolivarian Republic of)": "Venezuela",
+        "Türkiye": "Turkey",
+        "United Republic of Tanzania": "Tanzania",
+        "Iran (Islamic Republic of)": "Iran",
+        "Guinea Bissau": "Guinea-Bissau",
+        "Micronesia (Federated States of)": "Micronesia",
+    }
+
+    # for old_name, new_name in country_name_corrections.items():
+    #     country = Country.objects.filter(name=old_name).first()
+    #     if country:
+    #         country.name = new_name
+    #         country.save()
+    #         logger.info(f"✔ Country name updated from '{old_name}' to '{new_name}'")
+
+    # set modules for countries
+    projects_module = Module.objects.filter(code="Projects").first()
+    business_plans_module = Module.objects.filter(code="BP").first()
+    file_path = IMPORT_RESOURCES_V2_DIR / "countries" / "MLF Countries and Regions.xlsx"
+    df = pd.read_excel(file_path).fillna("")
+
+    for _, row in df.iterrows():
+        country = Country.objects.find_by_name(row["Countries"])
+        if not country:
+            logger.warning(f"⚠️ Country '{row['Countries']}' not found")
+            continue
+        country.modules.clear()
+        country.modules.add(projects_module)
+        country.modules.add(business_plans_module)
+        country.save()

core/import_data_v2/scripts/clean_up_project_statuses.py

@@ -0,0 +1,67 @@
+import json
+import logging
+
+from django.db import transaction
+
+from core.models.project_metadata import ProjectStatus, ProjectSubmissionStatus
+from core.models.project import Project
+
+logger = logging.getLogger(__name__)
+
+
+@transaction.atomic
+def clean_up_project_statuses():
+    """
+    Clean up project statuses
+    Remove outdated statuses and add new ones
+    """
+    # remove Unknown status only if there are no projects with this status
+
+    if Project.objects.really_all().filter(status__code="UNK").exists():
+        logger.warning(
+            "⚠️ Cannot remove 'Unknown' status, there are projects with this status."
+        )
+    else:
+        ProjectStatus.objects.filter(code="UNK").delete()
+
+    # change the status 'New submission' into 'N/A' and delete status 'New submission'
+
+    new_submission_status, _ = ProjectStatus.objects.update_or_create(
+        name="N/A",
+        defaults={
+            "code": "NA",
+        },
+    )
+    Project.objects.really_all().filter(status__code="NEWSUB").update(
+        status=new_submission_status
+    )
+    ProjectStatus.objects.filter(code="NEWSUB").delete()
+
+    # change the status 'Newly approved' into 'Ongoing' and delete status 'Newly approved'
+
+    on_going_status = ProjectStatus.objects.filter(name="Ongoing").first()
+    Project.objects.really_all().filter(status__code="NEW").update(
+        status=on_going_status
+    )
+    ProjectStatus.objects.filter(code="NEW").delete()
+
+
+@transaction.atomic
+def import_project_submission_statuses(file_path):
+    """
+    Import project submission statuses from file
+
+    @param file_path = str (file path for import file)
+    """
+
+    with open(file_path, "r", encoding="utf8") as f:
+        statuses_json = json.load(f)
+
+    for status_json in statuses_json:
+        status_data = {
+            "name": status_json["STATUS"],
+            "code": status_json["STATUS_CODE"],
+        }
+        ProjectSubmissionStatus.objects.update_or_create(
+            name=status_data["name"], defaults=status_data
+        )

core/import_data_v2/scripts/generate_new_cluster_type_sector_file.py

@@ -0,0 +1,45 @@
+import json
+import pandas as pd
+
+
+def generate_new_cluster_type_sector_file(file_path):
+    """
+    Generate new cluster type sector file based on the current data in the database
+
+    @param file_path = str (file path for import file)
+    """
+    combinations = {}  # {cluster: {type: [sectors]}}
+    df = pd.read_excel(file_path).fillna("")
+
+    for _, row in df.iterrows():
+        if row["Project type name"].strip() == "Project preparation":
+            row["Project type name"] = "Preparation"
+
+        if row["Sector name"].strip() == "Other Sector":
+            continue
+
+        combinations.setdefault(row["Cluster name"].strip(), {})
+        combinations[row["Cluster name"].strip()].setdefault(
+            row["Project type name"].strip(), []
+        )
+        combinations[row["Cluster name"].strip()][
+            row["Project type name"].strip()
+        ].append(row["Sector name"].strip())
+
+    new_data = []
+    for cluster_name, types in combinations.items():
+        new_data.append(
+            {
+                "cluster": cluster_name,
+                "types": [
+                    {
+                        "type": type_name,
+                        "sectors": sorted(list(set(sector_names))),  # remove duplicates
+                    }
+                    for type_name, sector_names in types.items()
+                ],
+            }
+        )
+
+    with open("new_ClusterTypeSectorLinks.json", "w", encoding="utf8") as f:
+        json.dump(new_data, f, indent=4)

core/import_data_v2/scripts/import_cluster_type_sector_links.py

@@ -0,0 +1,58 @@
+import json
+import logging
+
+from django.db import transaction
+
+from core.models.project_metadata import (
+    ProjectCluster,
+    ProjectSpecificFields,
+    ProjectSector,
+    ProjectType,
+)
+
+logger = logging.getLogger(__name__)
+
+
+@transaction.atomic
+def import_cluster_type_sector_links(file_path):
+    """
+    Import links between cluster, type and sector from file
+
+    @param file_path = str (file path for import file)
+    """
+    with open(file_path, "r", encoding="utf8") as f:
+        data = json.load(f)
+
+    for cluster_json in data:
+        cluster = ProjectCluster.objects.filter(name=cluster_json["cluster"]).first()
+        if not cluster:
+            logger.warning(
+                f"⚠️ {cluster_json['cluster']} cluster not found => {cluster_json['cluster']} not imported"
+            )
+            continue
+        for type_json in cluster_json["types"]:
+            type_name = type_json["type"]
+            if type_name == "Project Support":
+                type_name = "Project support"
+            type_obj = ProjectType.objects.filter(name=type_name).first()
+            if not type_obj:
+                logger.warning(
+                    f"⚠️ {type_name} type not found => {cluster_json['cluster']} not imported"
+                )
+                continue
+            for sector_name in type_json["sectors"]:
+                if sector_name == "Control Submstance Monitoring":
+                    sector_name = "Control Substance Monitoring"
+                if sector_name == "Compliance Assistance Program":
+                    sector_name = "Compliance Assistance Programme"
+                sector = ProjectSector.objects.filter(name=sector_name).first()
+                if not sector:
+                    logger.warning(
+                        f"⚠️ {sector_name} sector not found => {cluster_json['cluster']} not imported"
+                    )
+                    continue
+                ProjectSpecificFields.objects.update_or_create(
+                    cluster=cluster,
+                    type=type_obj,
+                    sector=sector,
+                )