fix(#799): batch contact IDs in move-contacts to avoid URI length errors

src/lib/api.js

@@ -256,20 +256,23 @@ const api = {
     }
   },
 
-  async getReportsByCreatedByIds (createdByIds, limit, skip) {
+  async getReportsByFreetext (createdByIds, limit, bookmark) {
     const queryString = createdByIds.map(id => `exact_match:"contact:${id}"`).join(' OR ');
-    const res = await request.get(`${environment.apiUrl}/_design/medic/_nouveau/reports_by_freetext`, {
-      qs: {
-        // sorting by this field ensures that the output are same with the clouseau views
-        // https://github.com/medic/cht-core/pull/9541
-        sort: '"reported_date"',
-        q: queryString,
-        include_docs: true,
-        limit,
-        skip
-      }, json: true
+    const body = {
+      // sorting by this field ensures that the output are same with the clouseau views
+      // https://github.com/medic/cht-core/pull/9541
+      sort: 'reported_date',
+      q: queryString,
+      include_docs: true,
+      limit,
+    };
+    if (bookmark) {
+      body.bookmark = bookmark;
+    }
+    const res = await request.post(`${environment.apiUrl}/_design/medic/_nouveau/reports_by_freetext`, {
+      body, json: true
     });
-    return res.hits.map(item => item.doc);
+    return { docs: res.hits.map(item => item.doc), bookmark: res.bookmark };
   }
 };
 

src/lib/hierarchy-operations/delete-hierarchy.js

@@ -34,7 +34,7 @@ async function deleteReportsForContact(db, options, contact) {
   let skip = 0;
   let reportBatch;
   do {
-    reportBatch = await DataSource.getReportsForContacts(db, [], [contact._id], skip);
+    reportBatch = await DataSource.fetchReportsBySubject(db, [contact._id], skip);
 
     for (const report of reportBatch) {
       JsDocs.deleteDoc(options, report);

src/lib/hierarchy-operations/hierarchy-data-source.js

@@ -1,10 +1,11 @@
 const lineageManipulation = require('./lineage-manipulation');
-const {getValidApiVersion} = require('../get-api-version');
+const { getValidApiVersion } = require('../get-api-version');
 const semver = require('semver');
 const api = require('../api');
 
 const HIERARCHY_ROOT = 'root';
 const BATCH_SIZE = 10000;
+const QUERY_IDS_BATCH_SIZE = 100;
 const NOUVEAU_MIN_VERSION = '5.0.0';
 const SUBJECT_IDS = ['patient_id', 'patient_uuid', 'place_id', 'place_uuid'];
 
@@ -71,18 +72,39 @@ const getFromDbView = async (db, view, keys, skip) => {
   return res.rows.map(row => row.doc);
 };
 
-const fetchReportsByCreator = async (db, createdByIds, skip) => {
+const useNouveauSearch = async () => {
+  const coreVersion = await getValidApiVersion();
+  return coreVersion && semver.gte(coreVersion, NOUVEAU_MIN_VERSION);
+};
+
+const fetchReportsByCreator = async (db, createdByIds, cursor, useNouveau) => {
   if (createdByIds.length === 0) {
-    return [];
+    return { docs: [], cursor: null };
   }
 
-  const coreVersion = await getValidApiVersion();
-  if (coreVersion && semver.gte(coreVersion, NOUVEAU_MIN_VERSION)) {
-    return await api().getReportsByCreatedByIds(createdByIds, BATCH_SIZE, skip);
+  if (useNouveau) {
+    return await fetchAllReportsFromNouveau(createdByIds);
   }
 
+  const skip = cursor || 0;
   const createdByKeys = createdByIds.map(id => [`contact:${id}`]);
-  return await getFromDbView(db, 'medic-client/reports_by_freetext', createdByKeys, skip);
+  const docs = await getFromDbView(db, 'medic-client/reports_by_freetext', createdByKeys, skip);
+  return { docs, cursor: skip + docs.length };
+};
+
+const fetchAllReportsFromNouveau = async (createdByIds) => {
+  const allDocs = [];
+  for (let i = 0; i < createdByIds.length; i += QUERY_IDS_BATCH_SIZE) {
+    const idsBatch = createdByIds.slice(i, i + QUERY_IDS_BATCH_SIZE);
+    let bookmark = null;
+    let batch;
+    do {
+      batch = await api().getReportsByFreetext(idsBatch, BATCH_SIZE, bookmark);
+      allDocs.push(...batch.docs);
+      bookmark = batch.bookmark;
+    } while (batch.docs.length >= BATCH_SIZE);
+  }
+  return { docs: allDocs, cursor: null };
 };
 
 const fetchReportsBySubject = async (db, createdAtIds, skip) => {
@@ -92,18 +114,6 @@ const fetchReportsBySubject = async (db, createdAtIds, skip) => {
   return await getFromDbView(db, 'medic-client/reports_by_subject', createdAtIds, skip);
 };
 
-const getReportsForContacts = async (db, createdByIds, createdAtIds, skip) => {
-  const [creatorReports, subjectReports] = await Promise.all([
-    fetchReportsByCreator(db, createdByIds, skip),
-    fetchReportsBySubject(db, createdAtIds, skip)
-  ]);
-
-  const allRows = [...creatorReports, ...subjectReports];
-
-  const docsWithId = allRows.map(( doc ) => [doc._id, doc]);
-  return Array.from(new Map(docsWithId).values());
-};
-
 async function getAncestorsOf(db, contactDoc) {
   const ancestorIds = lineageManipulation.pluckIdsFromLineage(contactDoc.parent);
   const ancestors = await db.allDocs({
@@ -130,5 +140,7 @@ module.exports = {
   getContactWithDescendants,
   getContact,
   getContactsByIds,
-  getReportsForContacts,
+  useNouveauSearch,
+  fetchReportsByCreator,
+  fetchReportsBySubject,
 };

src/lib/hierarchy-operations/index.js

@@ -74,24 +74,39 @@ function getPrimaryContactId(doc) {
 
 async function updateReports(db, options, moveContext) {
   const descendantIds = moveContext.descendantsAndSelf.map(contact => contact._id);
+  const createdAtIds = getReportsCreatedAtIds(moveContext);
 
-  let skip = 0;
-  let reportDocsBatch;
-  do {
-    info(`Processing ${skip} to ${skip + DataSource.BATCH_SIZE} report docs`);
-    const createdAtIds = getReportsCreatedAtIds(moveContext);
-    reportDocsBatch = await DataSource.getReportsForContacts(db, descendantIds, createdAtIds, skip);
+  let totalCount = 0;
+  const useNouveau = await DataSource.useNouveauSearch();
 
-    const lineageUpdates = replaceLineageOfReportCreator(reportDocsBatch, moveContext);
-    const reassignUpdates = reassignReports(reportDocsBatch, moveContext);
-    const updatedReports = reportDocsBatch.filter(doc => lineageUpdates.has(doc._id) || reassignUpdates.has(doc._id));
-
-    minifyLineageAndWriteToDisk(options, updatedReports);
+  let cursor = null;
+  let result;
+  do {
+    info(`Processing creator reports ${totalCount} to ${totalCount + DataSource.BATCH_SIZE}`);
+    result = await DataSource.fetchReportsByCreator(db, descendantIds, cursor, useNouveau);
+    processAndWriteReportBatch(result.docs, options, moveContext);
+    cursor = result.cursor;
+    totalCount += result.docs.length;
+  } while (result.cursor && result.docs.length >= DataSource.BATCH_SIZE);
 
-    skip += reportDocsBatch.length;
-  } while (reportDocsBatch.length >= DataSource.BATCH_SIZE);
+  let skip = 0;
+  let batch;
+  do {
+    info(`Processing subject reports ${skip} to ${skip + DataSource.BATCH_SIZE}`);
+    batch = await DataSource.fetchReportsBySubject(db, createdAtIds, skip);
+    processAndWriteReportBatch(batch, options, moveContext);
+    skip += batch.length;
+    totalCount += batch.length;
+  } while (batch.length >= DataSource.BATCH_SIZE);
+
+  return totalCount;
+}
 
-  return skip;
+function processAndWriteReportBatch(batch, options, moveContext) {
+  const lineageUpdates = replaceLineageOfReportCreator(batch, moveContext);
+  const reassignUpdates = reassignReports(batch, moveContext);
+  const updatedReports = batch.filter(doc => lineageUpdates.has(doc._id) || reassignUpdates.has(doc._id));
+  minifyLineageAndWriteToDisk(options, updatedReports);
 }
 
 function getReportsCreatedAtIds(moveContext) {

test/lib/api.spec.js

@@ -264,7 +264,7 @@ describe('api', () => {
       });
     });
 
-    describe('getReportsByCreatedByIds', async () => {
+    describe('getReportsByFreetext', async () => {
       beforeEach(() => {
         sinon.stub(environment, 'isArchiveMode').get(() => false);
       });
@@ -274,90 +274,87 @@ describe('api', () => {
           // Arrange
           const createdByIds = ['user1', 'user2'];
           const limit = 10;
-          const skip = 5;
+          const bookmark = 'abc123';
           const mockResponse = {
             hits: [
               {doc: {_id: 'report1', content: 'data1'}},
               {doc: {_id: 'report2', content: 'data2'}}
-            ]
+            ],
+            bookmark: 'next_page_bookmark'
           };
 
-          mockRequest.get.resolves(mockResponse);
+          mockRequest.post.resolves(mockResponse);
 
           // Act
-          const result = await api().getReportsByCreatedByIds(createdByIds, limit, skip);
+          const result = await api().getReportsByFreetext(createdByIds, limit, bookmark);
 
           // Assert - Check API call details
-          expect(mockRequest.get.callCount).to.equal(1);
-          const [url, options] = mockRequest.get.getCall(0).args;
+          expect(mockRequest.post.callCount).to.equal(1);
+          const [url, options] = mockRequest.post.getCall(0).args;
           expect(url).to.equal('http://example.com/db-name/_design/medic/_nouveau/reports_by_freetext');
-          expect(options.qs).to.deep.equal({
-            sort: '"reported_date"',
+          expect(options.body).to.deep.equal({
+            sort: 'reported_date',
             q: 'exact_match:"contact:user1" OR exact_match:"contact:user2"',
             include_docs: true,
             limit: 10,
-            skip: 5
+            bookmark: 'abc123'
           });
           expect(options.json).to.be.true;
 
           // Assert - Check return value
-          expect(result).to.deep.equal([
-            {_id: 'report1', content: 'data1'},
-            {_id: 'report2', content: 'data2'}
-          ]);
+          expect(result).to.deep.equal({
+            docs: [
+              {_id: 'report1', content: 'data1'},
+              {_id: 'report2', content: 'data2'}
+            ],
+            bookmark: 'next_page_bookmark'
+          });
         });
 
         it('should handle single ID and empty results', async () => {
           // Arrange
           const createdByIds = ['user1'];
-          const mockResponse = { hits: [] };
+          const mockResponse = { hits: [], bookmark: null };
 
-          mockRequest.get.resolves(mockResponse);
+          mockRequest.post.resolves(mockResponse);
 
           // Act
-          const result = await api().getReportsByCreatedByIds(createdByIds, undefined, null);
+          const result = await api().getReportsByFreetext(createdByIds, undefined, null);
 
           // Assert
-          const [, options] = mockRequest.get.getCall(0).args;
-          expect(options.qs.q).to.equal('exact_match:"contact:user1"');
-          expect(options.qs.limit).to.be.undefined;
-          expect(options.qs.skip).to.be.null;
-          expect(result).to.deep.equal([]);
+          const [, options] = mockRequest.post.getCall(0).args;
+          expect(options.body.q).to.equal('exact_match:"contact:user1"');
+          expect(options.body.bookmark).to.be.undefined;
+          expect(result).to.deep.equal({ docs: [], bookmark: null });
         });
       });
 
       describe('edge cases', () => {
         it('should handle various edge case inputs', async () => {
           const mockResponse = { hits: [] };
-          mockRequest.get.resolves(mockResponse);
+          mockRequest.post.resolves(mockResponse);
 
           // Test empty array
-          await api().getReportsByCreatedByIds([], 10, 0);
-          expect(mockRequest.get.getCall(0).args[1].qs.q).to.equal('');
+          await api().getReportsByFreetext([], 10, 0);
+          expect(mockRequest.post.getCall(0).args[1].body.q).to.equal('');
 
           // Test special characters in IDs
-          await api().getReportsByCreatedByIds(['user@domain.com', 'user-with-dashes'], 10, 0);
+          await api().getReportsByFreetext(['user@domain.com', 'user-with-dashes'], 10, 0);
           expect(
-            mockRequest.get.getCall(1).args[1].qs.q
+            mockRequest.post.getCall(1).args[1].body.q
           ).to.equal('exact_match:"contact:user@domain.com" OR exact_match:"contact:user-with-dashes"');
-
-          // Test zero values
-          await api().getReportsByCreatedByIds(['user1'], 0, 0);
-          const options = mockRequest.get.getCall(2).args[1];
-          expect(options.qs.limit).to.equal(0);
-          expect(options.qs.skip).to.equal(0);
         });
       });
 
       describe('error handling', () => {
         it('should propagate request errors', async () => {
           // Arrange
           const expectedError = new Error('Network error');
-          mockRequest.get.rejects(expectedError);
+          mockRequest.post.rejects(expectedError);
 
           // Act & Assert
           try {
-            await api().getReportsByCreatedByIds(['user1'], 10, 0);
+            await api().getReportsByFreetext(['user1'], 10, 0);
             expect.fail('Should have thrown an error');
           } catch (error) {
             expect(error).to.equal(expectedError);
@@ -366,29 +363,30 @@ describe('api', () => {
 
         it('should handle malformed responses', async () => {
           // Test missing hits property
-          mockRequest.get.resolves({});
+          mockRequest.post.resolves({});
 
           try {
-            await api().getReportsByCreatedByIds(['user1'], 10, 0);
+            await api().getReportsByFreetext(['user1'], 10, 0);
             expect.fail('Should have thrown an error');
           } catch (error) {
             expect(error).to.be.instanceOf(TypeError);
           }
 
           // Test hits with missing doc properties
-          mockRequest.get.resolves({
+          mockRequest.post.resolves({
             hits: [
               { doc: { _id: 'report1' } },
               {}, // Missing doc property
               { doc: { _id: 'report3' } }
-            ]
+            ],
+            bookmark: 'some_bookmark'
           });
 
-          const result = await api().getReportsByCreatedByIds(['user1'], 10, 0);
-          expect(result).to.have.length(3);
-          expect(result[0]).to.deep.equal({ _id: 'report1' });
-          expect(result[1]).to.be.undefined;
-          expect(result[2]).to.deep.equal({ _id: 'report3' });
+          const result = await api().getReportsByFreetext(['user1'], 10, null);
+          expect(result.docs).to.have.length(3);
+          expect(result.docs[0]).to.deep.equal({ _id: 'report1' });
+          expect(result.docs[1]).to.be.undefined;
+          expect(result.docs[2]).to.deep.equal({ _id: 'report3' });
         });
       });
     });