From 3c02f07442a93927517a98083fcaed3a8f83631f Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Thu, 5 Mar 2026 14:30:02 +0700 Subject: [PATCH 01/11] feat(spp_mis_demo_v2): add demo API client and statistics data Add demo data files for API testing and statistics indicators: - demo_api_client.xml: pre-configured QGIS API client with GIS scopes - demo_statistics.xml: 9 CEL variables + statistics for GIS/dashboard - test_demo_statistics.py: tests for statistics loading and aggregation Add spp_statistic, spp_aggregation, spp_studio as dependencies. --- spp_mis_demo_v2/__manifest__.py | 6 + spp_mis_demo_v2/data/demo_api_client.xml | 49 ++++ spp_mis_demo_v2/data/demo_statistics.xml | 267 ++++++++++++++++++ spp_mis_demo_v2/tests/__init__.py | 1 + spp_mis_demo_v2/tests/test_demo_statistics.py | 155 ++++++++++ 5 files changed, 478 insertions(+) create mode 100644 spp_mis_demo_v2/data/demo_api_client.xml create mode 100644 spp_mis_demo_v2/data/demo_statistics.xml create mode 100644 spp_mis_demo_v2/tests/test_demo_statistics.py diff --git a/spp_mis_demo_v2/__manifest__.py b/spp_mis_demo_v2/__manifest__.py index e7acd270..404373e4 100644 --- a/spp_mis_demo_v2/__manifest__.py +++ b/spp_mis_demo_v2/__manifest__.py @@ -20,6 +20,10 @@ "spp_demo", # GIS Reports for geographic visualization "spp_gis_report", + # Statistics and aggregation for demo indicators + "spp_statistic", + "spp_aggregation", + "spp_studio", # QR Credentials (Claim 169) "spp_claim_169", # Demo-specific extensions @@ -38,6 +42,8 @@ "data/change_request_types.xml", "data/demo_change_requests_ux.xml", "data/demo_gis_reports.xml", + "data/demo_statistics.xml", + "data/demo_api_client.xml", "views/mis_demo_wizard_view.xml", ], "assets": {}, diff --git a/spp_mis_demo_v2/data/demo_api_client.xml b/spp_mis_demo_v2/data/demo_api_client.xml new file mode 100644 index 00000000..9c4f4366 --- /dev/null +++ b/spp_mis_demo_v2/data/demo_api_client.xml @@ -0,0 +1,49 @@ + + + + + + + QGIS Demo Client + Demo API client for testing QGIS plugin integration. Grants read access to GIS layers, reports, and statistics queries. + + + + + + + + + gis + read + Read access to GIS layers, reports catalog, and spatial statistics queries. + + + + + + gis + all + Full GIS access including creating and managing geofences. + + + diff --git a/spp_mis_demo_v2/data/demo_statistics.xml b/spp_mis_demo_v2/data/demo_statistics.xml new file mode 100644 index 00000000..ccba4e5c --- /dev/null +++ b/spp_mis_demo_v2/data/demo_statistics.xml @@ -0,0 +1,267 @@ + + + + + + + + demo_total_members + total_members + aggregate + count + members + true + number + group + + active + 10 + + + + + + + demo_children_under_18 + children_under_18 + aggregate + count + members + age_years(m.birthdate) < 18 + number + group + + active + 30 + + + + + demo_elderly_60_plus + elderly_60_plus + aggregate + count + members + age_years(m.birthdate) >= 60 + number + group + + active + 40 + + + + + demo_disabled_members + disabled_members + aggregate + count + members + m.disabled != null + number + group + + active + 50 + + + + + demo_female_members + female_members + aggregate + count + members + is_female(m.gender_id) + number + group + + active + 45 + + + + + demo_male_members + male_members + aggregate + count + members + is_male(m.gender_id) + number + group + + active + 46 + + + + + demo_total_households + total_households + computed + true + number + group + + active + 5 + + + + + demo_enrolled_any_program + enrolled_any_program + computed + enrollments.exists(true) + number + group + + active + 60 + + + + + + + total_households + Total Households + Count of household groups in the selected area + + count + households + + 10 + + + + + + + + total_members + Total Members + Total count of individual household members + + count + people + + 15 + + + + + + + children_under_5 + Children Under 5 + Count of children under 5 years old + + count + children + + 20 + + + + + + + children_under_18 + Children Under 18 + Count of children under 18 years old + + count + children + + 30 + + + + + + + elderly_60_plus + Elderly (60+) + Count of elderly persons aged 60 and above + + count + people + + 40 + + + + + + + female_members + Female Members + Count of female household members + + count + people + + 45 + + + + + + + male_members + Male Members + Count of male household members + + count + people + + 46 + + + + + + + disabled_members + Disabled Members + Count of household members with disabilities + + count + people + + 50 + + + + + + + enrolled_any_program + Enrolled (Any Program) + Count of households enrolled in at least one program + + count + households + + 60 + + + + diff --git a/spp_mis_demo_v2/tests/__init__.py b/spp_mis_demo_v2/tests/__init__.py index 5205e81a..44ca7fb5 100644 --- a/spp_mis_demo_v2/tests/__init__.py +++ b/spp_mis_demo_v2/tests/__init__.py @@ -8,3 +8,4 @@ from . import test_formula_configuration from . import test_mis_demo_generator from . import test_registry_variables +from . import test_demo_statistics diff --git a/spp_mis_demo_v2/tests/test_demo_statistics.py b/spp_mis_demo_v2/tests/test_demo_statistics.py new file mode 100644 index 00000000..6390cfc4 --- /dev/null +++ b/spp_mis_demo_v2/tests/test_demo_statistics.py @@ -0,0 +1,155 @@ +# Part of OpenSPP. See LICENSE file for full copyright and licensing details. +"""Test demo statistics configuration and accessibility. + +These tests verify that: +1. All demo statistics are properly loaded into the database +2. Each statistic has a valid CEL variable reference +3. Statistics are published to GIS context +4. Statistics can be computed via the aggregation service +""" + +from odoo.tests.common import TransactionCase + + +class TestDemoStatistics(TransactionCase): + """Test that demo statistics are properly loaded and accessible.""" + + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.stat_model = cls.env["spp.statistic"] + cls.aggregation_service = cls.env["spp.aggregation.service"] + + # Required statistics that should be in the database + cls.required_stats = [ + "total_households", + "total_members", + "children_under_5", + "children_under_18", + "elderly_60_plus", + "female_members", + "male_members", + "disabled_members", + "enrolled_any_program", + ] + + def test_all_demo_statistics_exist(self): + """Verify all 9 demo statistics are in the database.""" + for stat_name in self.required_stats: + with self.subTest(statistic=stat_name): + stat = self.stat_model.search([("name", "=", stat_name)], limit=1) + self.assertTrue( + stat, + f"Statistic '{stat_name}' not found in database. Check that demo_statistics.xml was loaded.", + ) + + def test_statistics_have_variables(self): + """Verify each statistic has a valid CEL variable reference.""" + for stat_name in self.required_stats: + with self.subTest(statistic=stat_name): + stat = self.stat_model.search([("name", "=", stat_name)], limit=1) + if stat: # Only test if statistic exists + self.assertTrue( + stat.variable_id, + f"Statistic '{stat_name}' has no variable_id", + ) + self.assertEqual( + stat.variable_id.state, + "active", + f"Variable for '{stat_name}' is not active", + ) + + def test_statistics_published_to_gis(self): + """Verify statistics are published to GIS context.""" + for stat_name in self.required_stats: + with self.subTest(statistic=stat_name): + stat = self.stat_model.search([("name", "=", stat_name)], limit=1) + if stat: # Only test if statistic exists + self.assertTrue( + stat.is_published_gis, + f"Statistic '{stat_name}' not published to GIS", + ) + + def test_statistics_accessible_via_aggregation_service(self): + """Verify statistics can be computed via aggregation service.""" + # Get some test registrants (limit to 10 for performance) + registrants = self.env["res.partner"].search([("is_group", "=", True)], limit=10) + + if not registrants: + self.skipTest("No registrants found in database for testing") + + registrant_ids = registrants.ids + + # Test a subset of statistics for performance + test_stats = ["total_households", "total_members", "children_under_5"] + + for stat_name in test_stats: + with self.subTest(statistic=stat_name): + # Verify statistic exists first + stat = self.stat_model.search([("name", "=", stat_name)], limit=1) + if not stat: + self.skipTest(f"Statistic '{stat_name}' not found in database") + + # Compute via aggregation service + result = self.aggregation_service.compute_aggregation( + registrant_ids=registrant_ids, statistics=[stat_name] + ) + + self.assertIn( + stat_name, + result, + f"Statistic '{stat_name}' not in aggregation result", + ) + self.assertIsNotNone( + result[stat_name], + f"Statistic '{stat_name}' returned None. Check variable configuration and CEL expression.", + ) + + def test_statistics_categories_exist(self): + """Verify statistics are assigned to categories.""" + category_mapping = { + "demographics": [ + "total_households", + "total_members", + "children_under_5", + "children_under_18", + "elderly_60_plus", + "female_members", + "male_members", + ], + "vulnerability": ["disabled_members"], + "programs": ["enrolled_any_program"], + } + + for category_code, stat_names in category_mapping.items(): + for stat_name in stat_names: + with self.subTest(statistic=stat_name, category=category_code): + stat = self.stat_model.search([("name", "=", stat_name)], limit=1) + if stat: # Only test if statistic exists + self.assertTrue( + stat.category_id, + f"Statistic '{stat_name}' has no category", + ) + self.assertEqual( + stat.category_id.code, + category_code, + f"Statistic '{stat_name}' in wrong category. " + f"Expected '{category_code}', got '{stat.category_id.code}'", + ) + + def test_gis_discovery_endpoint_returns_statistics(self): + """Verify GIS statistics discovery returns our demo statistics.""" + # Get all GIS-published statistics + gis_stats = self.stat_model.get_published_for_context("gis") + + # Extract names + gis_stat_names = [stat.name for stat in gis_stats] + + # Verify our required statistics are included + for stat_name in self.required_stats: + with self.subTest(statistic=stat_name): + self.assertIn( + stat_name, + gis_stat_names, + f"Statistic '{stat_name}' not in GIS discovery endpoint. Check is_published_gis flag.", + ) From 225237389255b34c5214259f9b4d5740f6ff10cd Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Thu, 5 Mar 2026 14:41:30 +0700 Subject: [PATCH 02/11] fix(spp_mis_demo_v2): add spp_api_v2_gis dependency for demo API client The demo_api_client.xml creates scopes with resource='gis', which is a selection value registered by spp_api_v2_gis. Without this dependency, the XML load fails with "Wrong value for spp.api.client.scope.resource: 'gis'". --- spp_mis_demo_v2/__manifest__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/spp_mis_demo_v2/__manifest__.py b/spp_mis_demo_v2/__manifest__.py index 404373e4..0ae0c968 100644 --- a/spp_mis_demo_v2/__manifest__.py +++ b/spp_mis_demo_v2/__manifest__.py @@ -24,6 +24,8 @@ "spp_statistic", "spp_aggregation", "spp_studio", + # GIS API for QGIS plugin (registers 'gis' scope for demo API client) + "spp_api_v2_gis", # QR Credentials (Claim 169) "spp_claim_169", # Demo-specific extensions From 4c3ffb499251a5155d67d93399e5aa0e80c603dd Mon Sep 17 00:00:00 2001 From: Jeremi Joslin Date: Thu, 5 Mar 2026 15:02:30 +0700 Subject: [PATCH 03/11] feat(spp_mis_demo_v2): add geographic data loading and GIS report refresh Port the geographic data pipeline from the internal branch: - Add load_geographic_data/country_code wizard fields and presets - Add _load_geographic_data() to load area shapes via spp_demo loader - Add _assign_registrant_areas() to assign municipalities to registrants - Add _generate_coordinates() for GPS points within area polygons - Add _refresh_gis_reports() to populate report data immediately - Add geographic data summary to success notification - Add spp_registrant_gis dependency for GPS coordinate field Without this, GIS reports are empty and QGIS plugin queries return no data. --- spp_mis_demo_v2/__manifest__.py | 2 + spp_mis_demo_v2/models/mis_demo_generator.py | 256 ++++++++++++++++++ .../views/mis_demo_wizard_view.xml | 4 + 3 files changed, 262 insertions(+) diff --git a/spp_mis_demo_v2/__manifest__.py b/spp_mis_demo_v2/__manifest__.py index 0ae0c968..d8505699 100644 --- a/spp_mis_demo_v2/__manifest__.py +++ b/spp_mis_demo_v2/__manifest__.py @@ -20,6 +20,8 @@ "spp_demo", # GIS Reports for geographic visualization "spp_gis_report", + # Registrant GPS coordinates for QGIS plugin demo + "spp_registrant_gis", # Statistics and aggregation for demo indicators "spp_statistic", "spp_aggregation", diff --git a/spp_mis_demo_v2/models/mis_demo_generator.py b/spp_mis_demo_v2/models/mis_demo_generator.py index 83efec58..78abd844 100644 --- a/spp_mis_demo_v2/models/mis_demo_generator.py +++ b/spp_mis_demo_v2/models/mis_demo_generator.py @@ -181,6 +181,23 @@ class SPPMISDemoGenerator(models.TransientModel): help="Generate QR credentials for demo story personas (Maria Santos, etc.)", ) + # Geographic data options + load_geographic_data = fields.Boolean( + string="Load Geographic Data", + default=True, + help="Load area data with GIS shapes and assign GPS coordinates to registrants for QGIS plugin demo", + ) + country_code = fields.Selection( + [ + ("phl", "Philippines"), + ("lka", "Sri Lanka"), + ("tgo", "Togo"), + ], + string="Country", + default="phl", + help="Country for geographic data (areas and GIS shapes)", + ) + # Locale settings locale_origin = fields.Many2one( "res.country", @@ -256,6 +273,8 @@ def _onchange_demo_mode(self): "case_volume_count": 10, "generate_claim169_demo": True, "generate_credentials_for_stories": True, + "load_geographic_data": True, + "country_code": "phl", }, "training": { "create_demo_programs": True, @@ -278,6 +297,8 @@ def _onchange_demo_mode(self): "case_volume_count": 25, "generate_claim169_demo": True, "generate_credentials_for_stories": True, + "load_geographic_data": True, + "country_code": "phl", }, "testing": { "create_demo_programs": True, @@ -300,6 +321,8 @@ def _onchange_demo_mode(self): "case_volume_count": 200, "generate_claim169_demo": True, "generate_credentials_for_stories": True, + "load_geographic_data": True, + "country_code": "phl", }, "complete": { "create_demo_programs": True, @@ -322,6 +345,8 @@ def _onchange_demo_mode(self): "case_volume_count": 50, "generate_claim169_demo": True, "generate_credentials_for_stories": True, + "load_geographic_data": True, + "country_code": "phl", }, } defaults = mode_defaults.get(self.demo_mode, mode_defaults["sales"]) @@ -453,6 +478,13 @@ def action_generate(self): self._create_test_personas() stats["test_personas_created"] = True + # Step 0.4: Load geographic data (if enabled) + if self.load_geographic_data: + _logger.info(f"Loading geographic data for {self.country_code}...") + geo_result = self._load_geographic_data(stats) + if geo_result: + stats["areas_loaded"] = geo_result.get("shapes_loaded", 0) + # Step 0.5: Ensure demo stories exist (auto-generate if needed) stories_created = self._ensure_demo_stories_exist(stats) if stories_created: @@ -522,6 +554,16 @@ def action_generate(self): _logger.info("Generating Claim 169 demo data...") self._generate_claim169_demo(stats) + # Step 11: Assign areas and generate GPS coordinates (if geographic data loaded) + if self.load_geographic_data: + _logger.info("Assigning areas to registrants...") + self._assign_registrant_areas(stats) + _logger.info("Generating GPS coordinates for registrants...") + self._generate_coordinates(stats) + + # Step 12: Refresh GIS reports so map data is available immediately + self._refresh_gis_reports(stats) + self.state = "completed" # Return success notification with detailed summary @@ -3618,6 +3660,9 @@ def _show_success_notification(self, stats): if claim169_parts: message_parts.append(_("QR Credentials: %s created") % ", ".join(claim169_parts)) + # Geographic Data + self._append_geographic_summary(stats, message_parts) + # Warnings if stats["missing_registrants"]: message_parts.append("") @@ -3660,6 +3705,217 @@ def _show_success_notification(self, stats): }, } + def _append_geographic_summary(self, stats, message_parts): + """Append geographic data summary to notification message parts.""" + if not self.load_geographic_data: + return + geo_parts = [] + if stats.get("areas_loaded", 0) > 0: + geo_parts.append(_("%(count)s areas with GIS shapes", count=stats["areas_loaded"])) + if stats.get("areas_assigned", 0) > 0: + geo_parts.append(_("%(count)s groups assigned to areas", count=stats["areas_assigned"])) + if stats.get("coordinates_generated", 0) > 0: + geo_parts.append(_("%(count)s registrants with GPS coordinates", count=stats["coordinates_generated"])) + if geo_parts: + message_parts.append(_("Geographic Data: %s") % ", ".join(geo_parts)) + + def _load_geographic_data(self, stats): + """Load geographic area data with GIS shapes. + + Uses the DemoAreaLoader from spp_demo to load country-specific + area hierarchies with GIS polygon data for spatial queries. + + Args: + stats: Statistics dictionary to update + + Returns: + dict: Result with counts of loaded data + """ + try: + loader = self.env["spp.demo.area.loader"] + result = loader.load_country_areas(self.country_code, load_shapes=True) + _logger.info( + "[spp.mis.demo] Loaded geographic data for %s: %d areas with GIS shapes", + self.country_code, + result.get("shapes_loaded", 0), + ) + return result + except Exception as e: + _logger.warning("[spp.mis.demo] Failed to load geographic data: %s", e) + return None + + def _assign_registrant_areas(self, stats): + """Assign geographic areas to registrants. + + Strategy: + - Get all municipalities (level 3 areas) from the loaded country + - For each group, assign a random municipality to area_id + - Individual members inherit area_id from their group + + Args: + stats: Statistics dictionary to update + """ + Area = self.env["spp.area"] + Partner = self.env["res.partner"] + + # Get all level 3 areas (municipalities) that have geo_polygon data + municipalities = Area.search([("area_level", "=", 3), ("geo_polygon", "!=", False)]) + + if not municipalities: + _logger.warning("[spp.mis.demo] No municipalities with GIS data found, skipping area assignment") + stats["areas_assigned"] = 0 + return + + _logger.info("[spp.mis.demo] Found %d municipalities with GIS data", len(municipalities)) + + # Get all groups (households) + groups = Partner.search([("is_group", "=", True), ("is_registrant", "=", True)]) + + if not groups: + _logger.warning("[spp.mis.demo] No groups found, skipping area assignment") + stats["areas_assigned"] = 0 + return + + # Assign random municipality to each group + groups_assigned = 0 + for group in groups: + municipality = random.choice(municipalities) + group.write({"area_id": municipality.id}) + groups_assigned += 1 + + # Members inherit area from group + members = Partner.search([("group_membership_ids.group", "=", group.id)]) + if members: + members.write({"area_id": municipality.id}) + + stats["areas_assigned"] = groups_assigned + _logger.info("[spp.mis.demo] Assigned areas to %d groups", groups_assigned) + + def _generate_coordinates(self, stats): + """Generate GPS coordinates for registrants. + + For each registrant with an area_id that has geo_polygon data, + generates a random point within the area polygon and sets + the coordinates field (if spp_registrant_gis is installed). + + Uses shapely to generate random points within polygons. + + Args: + stats: Statistics dictionary to update + """ + # Check if spp_registrant_gis is installed + if "coordinates" not in self.env["res.partner"]._fields: + _logger.info("[spp.mis.demo] spp_registrant_gis not installed, skipping coordinate generation") + stats["coordinates_generated"] = 0 + return + + try: + from shapely.geometry import shape + from shapely.wkb import loads as wkbloads + except ImportError: + _logger.warning("[spp.mis.demo] shapely not available, skipping coordinate generation") + stats["coordinates_generated"] = 0 + return + + Partner = self.env["res.partner"] + Area = self.env["spp.area"] + + # Get all registrants with an area_id + registrants = Partner.search( + [ + ("is_registrant", "=", True), + ("area_id", "!=", False), + ] + ) + + if not registrants: + _logger.warning("[spp.mis.demo] No registrants with areas found") + stats["coordinates_generated"] = 0 + return + + _logger.info("[spp.mis.demo] Generating coordinates for %d registrants", len(registrants)) + + coordinates_generated = 0 + + # Group registrants by area to minimize queries + registrants_by_area = {} + for registrant in registrants: + area_id = registrant.area_id.id + if area_id not in registrants_by_area: + registrants_by_area[area_id] = [] + registrants_by_area[area_id].append(registrant) + + # Process each area + for area_id, area_registrants in registrants_by_area.items(): + area = Area.browse(area_id) + + # Skip if no polygon data + if not area.geo_polygon: + continue + + try: + # Convert WKB to shapely polygon + polygon = wkbloads(bytes(area.geo_polygon.data)) + + # Generate random points for all registrants in this area + minx, miny, maxx, maxy = polygon.bounds + + for registrant in area_registrants: + # Generate random point within bounding box, retry if outside polygon + max_attempts = 10 + for _attempt in range(max_attempts): + point_x = random.uniform(minx, maxx) + point_y = random.uniform(miny, maxy) + point = shape({"type": "Point", "coordinates": [point_x, point_y]}) + + if polygon.contains(point): + # Set the coordinates field (GeoPointField expects WKB) + registrant.write( + { + "coordinates": f"POINT({point_x} {point_y})", + } + ) + coordinates_generated += 1 + break + else: + # If we couldn't find a point inside after max_attempts, use centroid + centroid = polygon.centroid + registrant.write( + { + "coordinates": f"POINT({centroid.x} {centroid.y})", + } + ) + coordinates_generated += 1 + + except Exception as e: + _logger.warning("[spp.mis.demo] Failed to generate coordinates for area %s: %s", area.name, e) + continue + + stats["coordinates_generated"] = coordinates_generated + _logger.info("[spp.mis.demo] Generated coordinates for %d registrants", coordinates_generated) + + def _refresh_gis_reports(self, stats): + """Refresh all active GIS reports so map data is available immediately.""" + GISReport = self.env["spp.gis.report"] + reports = GISReport.search([("active", "=", True)]) + + if not reports: + _logger.info("[spp.mis.demo] No active GIS reports found to refresh") + stats["gis_reports_refreshed"] = 0 + return + + refreshed = 0 + for report in reports: + try: + report._refresh_data() + refreshed += 1 + _logger.info("[spp.mis.demo] Refreshed GIS report: %s", report.name) + except Exception: + _logger.exception("[spp.mis.demo] Failed to refresh GIS report: %s", report.name) + + stats["gis_reports_refreshed"] = refreshed + _logger.info("[spp.mis.demo] Refreshed %d GIS reports", refreshed) + class SPPMISDemoWizard(models.TransientModel): """Wizard interface for MIS Demo Generator.""" diff --git a/spp_mis_demo_v2/views/mis_demo_wizard_view.xml b/spp_mis_demo_v2/views/mis_demo_wizard_view.xml index 928c4992..3a416835 100644 --- a/spp_mis_demo_v2/views/mis_demo_wizard_view.xml +++ b/spp_mis_demo_v2/views/mis_demo_wizard_view.xml @@ -24,6 +24,10 @@
  • Fairness analysis data
    • + + + +