On Recipients, store count of funders

datastore/db/models.py

@@ -8,6 +8,8 @@
 from django.db.utils import DataError
 from django.utils import timezone
 
+from additional_data.sources.find_that_charity import non_primary_org_ids_lookup_maps
+
 
 class Latest(models.Model):
     """Latest best data we have"""
@@ -277,6 +279,8 @@ def update_aggregate(self, grant):
         #       "GBP": { "grants": 0, "total": 0, "avg": 0, min: 0, max: 0 } },
         #        ...
         # },
+        # This only covers common stats.
+        # See classes that inherit this - they may override update_aggregate() and add more.
 
         amount = grant["amountAwarded"]
         currency = grant["currency"]
@@ -362,6 +366,14 @@ class Meta:
 
 
 class Recipient(Entity):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        # While collecting aggregate info on funders, we need to store some data temporarily that we don't want to store in the database.
+        # This stores all primary ids so we can count unique funders.
+        self._aggregate_funders_primary_ids = set()
+        # This stores information by currency.
+        self._aggregate_funders_currencies = {}
+
     class Meta:
         constraints = [
             models.UniqueConstraint(fields=["org_id"], name="recipient_unique_org_id")
@@ -373,6 +385,42 @@ class Meta:
 
     non_primary_org_ids = ArrayField(models.TextField())
 
+    def update_aggregate(self, grant):
+        # Step 1: Call parent
+        super().update_aggregate(grant)
+
+        # Step 2: update _aggregate_funders_* vars with info from this grant
+        # This function is called repeatedly from datastore/db/management/commands/manage_entities_data.py
+        # and it's inefficient to call non_primary_org_ids_lookup_maps every time.
+        # But after discussion with MW that is ok.
+        (
+            non_primary_to_primary_org_ids_lookup,
+            primary_to_non_primary_org_ids_lookup,
+        ) = non_primary_org_ids_lookup_maps()
+        currency = grant["currency"]
+        if currency not in self._aggregate_funders_currencies:
+            self._aggregate_funders_currencies[currency] = {
+                "funders_primary_ids": set(),
+            }
+        for funder in grant["fundingOrganization"]:
+            # If the org-id provided is a non-primary org-id return the primary
+            # otherwise return the specified org-id
+            funder_primary_id = non_primary_to_primary_org_ids_lookup.get(
+                funder["id"], funder["id"]
+            )
+            self._aggregate_funders_primary_ids.add(funder_primary_id)
+            self._aggregate_funders_currencies[currency]["funders_primary_ids"].add(
+                funder_primary_id
+            )
+
+        # Step 3: copy info from _aggregate_funders_* vars to aggregate for saving to the database
+        self.aggregate["funders"] = len(self._aggregate_funders_primary_ids)
+
+        for currency_id, currency_data in self._aggregate_funders_currencies.items():
+            self.aggregate["currencies"][currency_id]["funders"] = len(
+                self._aggregate_funders_currencies[currency_id]["funders_primary_ids"]
+            )
+
 
 class Funder(Entity):
     class Meta:

datastore/tests/test_models.py

@@ -2,6 +2,8 @@
 
 import db.models as db
 
+import unittest.mock
+
 
 class GetterRunTest(TransactionTestCase):
     fixtures = ["test_data.json"]
@@ -91,3 +93,137 @@ def test_convenience_fields_from_data(self):
         self.assertSetEqual(set(grant.funding_org_ids), {"GB-CHC-12345"})
 
         self.assertEqual(grant.publisher_org_id, "XI-EXAMPLE-EXAMPLE")
+
+
+def mock_non_primary_org_ids_lookup_maps():
+    return {"GB-SECONDARY-12345": "GB-PRIMARY-12345"}, {}
+
+
+@unittest.mock.patch(
+    "db.models.non_primary_org_ids_lookup_maps", mock_non_primary_org_ids_lookup_maps
+)
+class RecipientUpdateAggregateTest(TestCase):
+    def test_single_grant(self):
+        recipient = db.Recipient()
+        recipient.update_aggregate(
+            {
+                "currency": "GBP",
+                "amountAwarded": 100,
+                "awardDate": "2019-10-03T00:00:00+00:00",
+                "fundingOrganization": [{"id": "GB-CHC-12345"}],
+                "recipientOrganization": [
+                    {"id": "GB-COH-12345"},
+                ],
+            }
+        )
+        self.assertEqual(recipient.aggregate["funders"], 1)
+        self.assertEqual(recipient.aggregate["currencies"]["GBP"]["funders"], 1)
+
+    def test_two_grants_from_same_funder(self):
+        recipient = db.Recipient()
+        recipient.update_aggregate(
+            {
+                "currency": "GBP",
+                "amountAwarded": 100,
+                "awardDate": "2019-10-03T00:00:00+00:00",
+                "fundingOrganization": [{"id": "GB-CHC-12345"}],
+                "recipientOrganization": [
+                    {"id": "GB-COH-12345"},
+                ],
+            }
+        )
+        recipient.update_aggregate(
+            {
+                "currency": "GBP",
+                "amountAwarded": 10000,
+                "awardDate": "2020-10-03T00:00:00+00:00",
+                "fundingOrganization": [{"id": "GB-CHC-12345"}],
+                "recipientOrganization": [
+                    {"id": "GB-COH-12345"},
+                ],
+            }
+        )
+        self.assertEqual(recipient.aggregate["funders"], 1)
+        self.assertEqual(recipient.aggregate["currencies"]["GBP"]["funders"], 1)
+
+    def test_two_grants_from_different_funders(self):
+        recipient = db.Recipient()
+        recipient.update_aggregate(
+            {
+                "currency": "GBP",
+                "amountAwarded": 100,
+                "awardDate": "2019-10-03T00:00:00+00:00",
+                "fundingOrganization": [{"id": "GB-CHC-12345"}],
+                "recipientOrganization": [
+                    {"id": "GB-COH-12345"},
+                ],
+            }
+        )
+        recipient.update_aggregate(
+            {
+                "currency": "GBP",
+                "amountAwarded": 10000,
+                "awardDate": "2020-10-03T00:00:00+00:00",
+                "fundingOrganization": [{"id": "GB-CHC-67890"}],
+                "recipientOrganization": [
+                    {"id": "GB-COH-12345"},
+                ],
+            }
+        )
+        self.assertEqual(recipient.aggregate["funders"], 2)
+        self.assertEqual(recipient.aggregate["currencies"]["GBP"]["funders"], 2)
+
+    def test_two_grants_from_different_funders_in_different_currencies(self):
+        recipient = db.Recipient()
+        recipient.update_aggregate(
+            {
+                "currency": "GBP",
+                "amountAwarded": 100,
+                "awardDate": "2019-10-03T00:00:00+00:00",
+                "fundingOrganization": [{"id": "GB-CHC-12345"}],
+                "recipientOrganization": [
+                    {"id": "GB-COH-12345"},
+                ],
+            }
+        )
+        recipient.update_aggregate(
+            {
+                "currency": "EUR",
+                "amountAwarded": 10000,
+                "awardDate": "2020-10-03T00:00:00+00:00",
+                "fundingOrganization": [{"id": "GB-CHC-67890"}],
+                "recipientOrganization": [
+                    {"id": "GB-COH-12345"},
+                ],
+            }
+        )
+        self.assertEqual(recipient.aggregate["funders"], 2)
+        self.assertEqual(recipient.aggregate["currencies"]["GBP"]["funders"], 1)
+        self.assertEqual(recipient.aggregate["currencies"]["EUR"]["funders"], 1)
+
+    def test_two_grants_from_same_funder_but_different_funder_ids_used(self):
+        recipient = db.Recipient()
+        recipient.update_aggregate(
+            {
+                "currency": "GBP",
+                "amountAwarded": 100,
+                "awardDate": "2019-10-03T00:00:00+00:00",
+                "fundingOrganization": [{"id": "GB-PRIMARY-12345"}],
+                "recipientOrganization": [
+                    {"id": "GB-COH-12345"},
+                ],
+            }
+        )
+        recipient.update_aggregate(
+            {
+                "currency": "GBP",
+                "amountAwarded": 10000,
+                "awardDate": "2020-10-03T00:00:00+00:00",
+                "fundingOrganization": [{"id": "GB-SECONDARY-12345"}],
+                "recipientOrganization": [
+                    {"id": "GB-COH-12345"},
+                ],
+            }
+        )
+        self.assertEqual(recipient.aggregate["funders"], 1)
+        self.assertEqual(recipient.aggregate["currencies"]["GBP"]["funders"], 1)