From c7db98e3b0a46a6c47c36f4a2d8ad1a33d6a91ce Mon Sep 17 00:00:00 2001 From: James B Date: Wed, 19 Mar 2025 14:47:38 +0000 Subject: [PATCH] On Recipients, store count of funders https://github.com/ThreeSixtyGiving/grantnav/issues/940 --- datastore/db/models.py | 48 ++++++++++++ datastore/tests/test_models.py | 136 +++++++++++++++++++++++++++++++++ 2 files changed, 184 insertions(+) diff --git a/datastore/db/models.py b/datastore/db/models.py index d05ba93f..d284f02b 100644 --- a/datastore/db/models.py +++ b/datastore/db/models.py @@ -8,6 +8,8 @@ from django.db.utils import DataError from django.utils import timezone +from additional_data.sources.find_that_charity import non_primary_org_ids_lookup_maps + class Latest(models.Model): """Latest best data we have""" @@ -277,6 +279,8 @@ def update_aggregate(self, grant): # "GBP": { "grants": 0, "total": 0, "avg": 0, min: 0, max: 0 } }, # ... # }, + # This only covers common stats. + # See classes that inherit this - they may override update_aggregate() and add more. amount = grant["amountAwarded"] currency = grant["currency"] @@ -362,6 +366,14 @@ class Meta: class Recipient(Entity): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + # While collecting aggregate info on funders, we need to store some data temporarily that we don't want to store in the database. + # This stores all primary ids so we can count unique funders. + self._aggregate_funders_primary_ids = set() + # This stores information by currency. + self._aggregate_funders_currencies = {} + class Meta: constraints = [ models.UniqueConstraint(fields=["org_id"], name="recipient_unique_org_id") @@ -373,6 +385,42 @@ class Meta: non_primary_org_ids = ArrayField(models.TextField()) + def update_aggregate(self, grant): + # Step 1: Call parent + super().update_aggregate(grant) + + # Step 2: update _aggregate_funders_* vars with info from this grant + # This function is called repeatedly from datastore/db/management/commands/manage_entities_data.py + # and it's inefficient to call non_primary_org_ids_lookup_maps every time. + # But after discussion with MW that is ok. + ( + non_primary_to_primary_org_ids_lookup, + primary_to_non_primary_org_ids_lookup, + ) = non_primary_org_ids_lookup_maps() + currency = grant["currency"] + if currency not in self._aggregate_funders_currencies: + self._aggregate_funders_currencies[currency] = { + "funders_primary_ids": set(), + } + for funder in grant["fundingOrganization"]: + # If the org-id provided is a non-primary org-id return the primary + # otherwise return the specified org-id + funder_primary_id = non_primary_to_primary_org_ids_lookup.get( + funder["id"], funder["id"] + ) + self._aggregate_funders_primary_ids.add(funder_primary_id) + self._aggregate_funders_currencies[currency]["funders_primary_ids"].add( + funder_primary_id + ) + + # Step 3: copy info from _aggregate_funders_* vars to aggregate for saving to the database + self.aggregate["funders"] = len(self._aggregate_funders_primary_ids) + + for currency_id, currency_data in self._aggregate_funders_currencies.items(): + self.aggregate["currencies"][currency_id]["funders"] = len( + self._aggregate_funders_currencies[currency_id]["funders_primary_ids"] + ) + class Funder(Entity): class Meta: diff --git a/datastore/tests/test_models.py b/datastore/tests/test_models.py index e3e12473..c1ef46fb 100644 --- a/datastore/tests/test_models.py +++ b/datastore/tests/test_models.py @@ -2,6 +2,8 @@ import db.models as db +import unittest.mock + class GetterRunTest(TransactionTestCase): fixtures = ["test_data.json"] @@ -91,3 +93,137 @@ def test_convenience_fields_from_data(self): self.assertSetEqual(set(grant.funding_org_ids), {"GB-CHC-12345"}) self.assertEqual(grant.publisher_org_id, "XI-EXAMPLE-EXAMPLE") + + +def mock_non_primary_org_ids_lookup_maps(): + return {"GB-SECONDARY-12345": "GB-PRIMARY-12345"}, {} + + +@unittest.mock.patch( + "db.models.non_primary_org_ids_lookup_maps", mock_non_primary_org_ids_lookup_maps +) +class RecipientUpdateAggregateTest(TestCase): + def test_single_grant(self): + recipient = db.Recipient() + recipient.update_aggregate( + { + "currency": "GBP", + "amountAwarded": 100, + "awardDate": "2019-10-03T00:00:00+00:00", + "fundingOrganization": [{"id": "GB-CHC-12345"}], + "recipientOrganization": [ + {"id": "GB-COH-12345"}, + ], + } + ) + self.assertEqual(recipient.aggregate["funders"], 1) + self.assertEqual(recipient.aggregate["currencies"]["GBP"]["funders"], 1) + + def test_two_grants_from_same_funder(self): + recipient = db.Recipient() + recipient.update_aggregate( + { + "currency": "GBP", + "amountAwarded": 100, + "awardDate": "2019-10-03T00:00:00+00:00", + "fundingOrganization": [{"id": "GB-CHC-12345"}], + "recipientOrganization": [ + {"id": "GB-COH-12345"}, + ], + } + ) + recipient.update_aggregate( + { + "currency": "GBP", + "amountAwarded": 10000, + "awardDate": "2020-10-03T00:00:00+00:00", + "fundingOrganization": [{"id": "GB-CHC-12345"}], + "recipientOrganization": [ + {"id": "GB-COH-12345"}, + ], + } + ) + self.assertEqual(recipient.aggregate["funders"], 1) + self.assertEqual(recipient.aggregate["currencies"]["GBP"]["funders"], 1) + + def test_two_grants_from_different_funders(self): + recipient = db.Recipient() + recipient.update_aggregate( + { + "currency": "GBP", + "amountAwarded": 100, + "awardDate": "2019-10-03T00:00:00+00:00", + "fundingOrganization": [{"id": "GB-CHC-12345"}], + "recipientOrganization": [ + {"id": "GB-COH-12345"}, + ], + } + ) + recipient.update_aggregate( + { + "currency": "GBP", + "amountAwarded": 10000, + "awardDate": "2020-10-03T00:00:00+00:00", + "fundingOrganization": [{"id": "GB-CHC-67890"}], + "recipientOrganization": [ + {"id": "GB-COH-12345"}, + ], + } + ) + self.assertEqual(recipient.aggregate["funders"], 2) + self.assertEqual(recipient.aggregate["currencies"]["GBP"]["funders"], 2) + + def test_two_grants_from_different_funders_in_different_currencies(self): + recipient = db.Recipient() + recipient.update_aggregate( + { + "currency": "GBP", + "amountAwarded": 100, + "awardDate": "2019-10-03T00:00:00+00:00", + "fundingOrganization": [{"id": "GB-CHC-12345"}], + "recipientOrganization": [ + {"id": "GB-COH-12345"}, + ], + } + ) + recipient.update_aggregate( + { + "currency": "EUR", + "amountAwarded": 10000, + "awardDate": "2020-10-03T00:00:00+00:00", + "fundingOrganization": [{"id": "GB-CHC-67890"}], + "recipientOrganization": [ + {"id": "GB-COH-12345"}, + ], + } + ) + self.assertEqual(recipient.aggregate["funders"], 2) + self.assertEqual(recipient.aggregate["currencies"]["GBP"]["funders"], 1) + self.assertEqual(recipient.aggregate["currencies"]["EUR"]["funders"], 1) + + def test_two_grants_from_same_funder_but_different_funder_ids_used(self): + recipient = db.Recipient() + recipient.update_aggregate( + { + "currency": "GBP", + "amountAwarded": 100, + "awardDate": "2019-10-03T00:00:00+00:00", + "fundingOrganization": [{"id": "GB-PRIMARY-12345"}], + "recipientOrganization": [ + {"id": "GB-COH-12345"}, + ], + } + ) + recipient.update_aggregate( + { + "currency": "GBP", + "amountAwarded": 10000, + "awardDate": "2020-10-03T00:00:00+00:00", + "fundingOrganization": [{"id": "GB-SECONDARY-12345"}], + "recipientOrganization": [ + {"id": "GB-COH-12345"}, + ], + } + ) + self.assertEqual(recipient.aggregate["funders"], 1) + self.assertEqual(recipient.aggregate["currencies"]["GBP"]["funders"], 1)