Adds OriginalFailureReasons/CurrentFailureReasons columns,

  refresh_failedmeasurements_current procedure, validation endpoint,
  auto-reingest for ready rows, and coordinate null/zero handling fixes.Merge branch 'dev-consolidate-failures' into forestgeo-app-development
Adds OriginalFailureReasons/CurrentFailureReasons columns,
  refresh_failedmeasurements_current procedure, validation endpoint,
  auto-reingest for ready rows, and coordinate null/zero handling fixes.# the commit.

Author: MasonStilwell

frontend/app/api/batchedupload/[schema]/[[...slugs]]/route.test.ts

@@ -91,7 +91,7 @@ describe('batchedupload POST route', () => {
 
     // ⟵ FIX: default import already *is* the mocked module
     const exec = connectionmanager.getInstance().executeQuery as ReturnType<typeof vi.fn>;
-    // Route calls executeQuery twice: INSERT + CALL reviewfailed()
+    // Route calls executeQuery twice: INSERT + CALL refresh_failedmeasurements_current()
     expect(exec).toHaveBeenCalledTimes(2);
 
     // With mocked mysql2.format we can sanity-check the SQL + params
@@ -110,9 +110,9 @@ describe('batchedupload POST route', () => {
     expect(sqlArg).toContain('42'); // plotID
     expect(sqlArg).toContain('7'); // censusID
 
-    // Verify the second call is the reviewfailed procedure
-    const reviewFailedCall = exec.mock.calls[1][0] as string;
-    expect(reviewFailedCall).toMatch(/CALL myschema\.reviewfailed\(\)/i);
+    // Verify the second call is the refresh_failedmeasurements_current procedure
+    const refreshCall = exec.mock.calls[1][0] as string;
+    expect(refreshCall).toMatch(/CALL myschema\.refresh_failedmeasurements_current/i);
   });
 
   it('500s and logs on DB error', async () => {

frontend/app/api/batchedupload/[schema]/[[...slugs]]/route.ts

@@ -55,7 +55,9 @@ export async function POST(request: NextRequest, props: { params: Promise<{ sche
   errorRows = errorRows.map(row => ({
     ...row,
     plotID,
-    censusID
+    censusID,
+    originalFailureReasons: row.originalFailureReasons ?? row.failureReasons ?? null,
+    currentFailureReasons: row.currentFailureReasons ?? row.failureReasons ?? null
   }));
 
   const connectionManager = connectionmanager.getInstance();
@@ -67,9 +69,9 @@ export async function POST(request: NextRequest, props: { params: Promise<{ sche
   try {
     await connectionManager.executeQuery(insertQuery);
 
-    // Populate failure reasons for the newly inserted failed measurements
-    const reviewFailedQuery = `CALL ${schema}.reviewfailed()`;
-    await connectionManager.executeQuery(reviewFailedQuery);
+    // Refresh current failure reasons for the affected plot/census
+    const refreshFailedQuery = `CALL ${schema}.refresh_failedmeasurements_current(?, ?)`;
+    await connectionManager.executeQuery(refreshFailedQuery, [plotID, censusID]);
 
     return new NextResponse(JSON.stringify({ message: 'Insert to SQL successful' }), { status: HTTPResponses.OK });
   } catch (error: any) {

frontend/app/api/reingest/[schema]/[plotID]/[censusID]/route.test.ts

@@ -6,16 +6,10 @@ import ConnectionManager from '@/config/connectionmanager';
 // Mock ConnectionManager
 vi.mock('@/config/connectionmanager', () => {
   const executeQuery = vi.fn();
-  const beginTransaction = vi.fn();
-  const commitTransaction = vi.fn();
-  const rollbackTransaction = vi.fn();
   const closeConnection = vi.fn();
   const cleanupStaleTransactions = vi.fn();
   const instance = {
     executeQuery,
-    beginTransaction,
-    commitTransaction,
-    rollbackTransaction,
     closeConnection,
     cleanupStaleTransactions
   };
@@ -61,6 +55,11 @@ function makeParams() {
   } as any;
 }
 
+/** Helper: generate mock FailedMeasurementID rows */
+function mockFailedIds(count: number) {
+  return Array.from({ length: count }, (_, i) => ({ FailedMeasurementID: i + 1 }));
+}
+
 describe('reingest API routes', () => {
   let mockConnectionManager: any;
   let mockValidateContextualValues: any;
@@ -69,9 +68,6 @@ describe('reingest API routes', () => {
     vi.clearAllMocks();
     mockConnectionManager = ConnectionManager.getInstance();
     mockConnectionManager.cleanupStaleTransactions.mockResolvedValue(undefined);
-    mockConnectionManager.beginTransaction.mockResolvedValue('test-transaction-id');
-    mockConnectionManager.commitTransaction.mockResolvedValue(undefined);
-    mockConnectionManager.rollbackTransaction.mockResolvedValue(undefined);
     mockConnectionManager.closeConnection.mockResolvedValue(undefined);
 
     // Get the mocked function
@@ -91,12 +87,12 @@ describe('reingest API routes', () => {
 
   describe('POST route (move rows only)', () => {
     it('moves rows from failedmeasurements to temporarymeasurements', async () => {
-      // Mock count query
       mockConnectionManager.executeQuery
-        .mockResolvedValueOnce([{ total: 5 }]) // Count query
-        .mockResolvedValueOnce(undefined) // DELETE temporarymeasurements
-        .mockResolvedValueOnce(undefined) // INSERT INTO temporarymeasurements
-        .mockResolvedValueOnce(undefined); // DELETE failedmeasurements
+        .mockResolvedValueOnce([{ total: 5 }]) // 1. COUNT(*)
+        .mockResolvedValueOnce(mockFailedIds(5)) // 2. SELECT FailedMeasurementID
+        .mockResolvedValueOnce(undefined) // 3. DELETE temporarymeasurements
+        .mockResolvedValueOnce(undefined) // 4. INSERT INTO temporarymeasurements
+        .mockResolvedValueOnce(undefined); // 5. DELETE failedmeasurements by IDs
 
       const req = makeRequest('POST');
       const res = await POST(req, makeParams());
@@ -107,13 +103,8 @@ describe('reingest API routes', () => {
       expect(body.fileID).toBe('reingestion.csv');
       expect(body.batchID).toBe('test-batch-id-12345');
 
-      // Verify transaction management
-      expect(mockConnectionManager.beginTransaction).toHaveBeenCalledTimes(1);
-      expect(mockConnectionManager.commitTransaction).toHaveBeenCalledTimes(1);
       expect(mockConnectionManager.closeConnection).toHaveBeenCalledTimes(1);
-
-      // Verify queries were called
-      expect(mockConnectionManager.executeQuery).toHaveBeenCalledTimes(4);
+      expect(mockConnectionManager.executeQuery).toHaveBeenCalledTimes(5);
     });
 
     it('returns 200 with rowsMoved=0 when no failed measurements exist', async () => {
@@ -128,7 +119,7 @@ describe('reingest API routes', () => {
       expect(body.responseMessage).toMatch(/No failed measurements found/i);
     });
 
-    it('rolls back transaction on error', async () => {
+    it('returns 500 on error', async () => {
       mockConnectionManager.executeQuery.mockRejectedValueOnce(new Error('Database error'));
 
       const req = makeRequest('POST');
@@ -137,21 +128,22 @@ describe('reingest API routes', () => {
       expect(res.status).toBe(500);
       const body = await res.json();
       expect(body.error).toBe('Database error');
-      expect(mockConnectionManager.rollbackTransaction).toHaveBeenCalledTimes(1);
+      expect(mockConnectionManager.closeConnection).toHaveBeenCalledTimes(1);
     });
   });
 
   describe('GET route (full reingestion)', () => {
     it('moves rows and runs batch ingestion process', async () => {
-      // Mock all queries for full reingestion
       mockConnectionManager.executeQuery
-        .mockResolvedValueOnce([{ total: 10 }]) // Count query
-        .mockResolvedValueOnce(undefined) // DELETE temporarymeasurements
-        .mockResolvedValueOnce(undefined) // INSERT INTO temporarymeasurements
-        .mockResolvedValueOnce(undefined) // DELETE failedmeasurements
-        .mockResolvedValueOnce(undefined) // CALL bulkingestionprocess
-        .mockResolvedValueOnce([{ remaining: 2 }]) // Count remaining failures
-        .mockResolvedValueOnce(undefined); // CALL reviewfailed
+        .mockResolvedValueOnce([{ total: 10 }]) // 1. COUNT(*)
+        .mockResolvedValueOnce(mockFailedIds(10)) // 2. SELECT FailedMeasurementID
+        .mockResolvedValueOnce(undefined) // 3. DELETE temporarymeasurements
+        .mockResolvedValueOnce(undefined) // 4. INSERT INTO temporarymeasurements
+        .mockResolvedValueOnce(undefined) // 5. CALL bulkingestionprocess
+        .mockResolvedValueOnce(undefined) // 6. DELETE failedmeasurements by IDs
+        .mockResolvedValueOnce(undefined) // 7. CALL refresh_failedmeasurements_current
+        .mockResolvedValueOnce([{ cnt: 0 }]) // 8. SELECT COUNT(*) ready for reingestion
+        .mockResolvedValueOnce([{ remaining: 2 }]); // 9. COUNT(*) remaining failures
 
       const req = makeRequest('GET');
       const res = await GET(req, makeParams());
@@ -167,9 +159,9 @@ describe('reingest API routes', () => {
       const bulkIngestionCall = calls.find((call: any) => call[0]?.includes('bulkingestionprocess'));
       expect(bulkIngestionCall).toBeDefined();
 
-      // Verify reviewfailed was called
-      const reviewFailedCall = calls.find((call: any) => call[0]?.includes('reviewfailed'));
-      expect(reviewFailedCall).toBeDefined();
+      // Verify refresh was called
+      const refreshCall = calls.find((call: any) => call[0]?.includes('refresh_failedmeasurements_current'));
+      expect(refreshCall).toBeDefined();
     });
 
     it('returns 200 with 0 processed when no failed measurements exist', async () => {
@@ -187,13 +179,15 @@ describe('reingest API routes', () => {
 
     it('handles all rows successfully reingested', async () => {
       mockConnectionManager.executeQuery
-        .mockResolvedValueOnce([{ total: 5 }])
-        .mockResolvedValueOnce(undefined) // DELETE temporarymeasurements
-        .mockResolvedValueOnce(undefined) // INSERT INTO temporarymeasurements
-        .mockResolvedValueOnce(undefined) // DELETE failedmeasurements
-        .mockResolvedValueOnce(undefined) // CALL bulkingestionprocess
-        .mockResolvedValueOnce([{ remaining: 0 }]) // All successful
-        .mockResolvedValueOnce(undefined); // CALL reviewfailed
+        .mockResolvedValueOnce([{ total: 5 }]) // 1. COUNT(*)
+        .mockResolvedValueOnce(mockFailedIds(5)) // 2. SELECT FailedMeasurementID
+        .mockResolvedValueOnce(undefined) // 3. DELETE temporarymeasurements
+        .mockResolvedValueOnce(undefined) // 4. INSERT INTO temporarymeasurements
+        .mockResolvedValueOnce(undefined) // 5. CALL bulkingestionprocess
+        .mockResolvedValueOnce(undefined) // 6. DELETE failedmeasurements by IDs
+        .mockResolvedValueOnce(undefined) // 7. CALL refresh_failedmeasurements_current
+        .mockResolvedValueOnce([{ cnt: 0 }]) // 8. SELECT COUNT(*) ready
+        .mockResolvedValueOnce([{ remaining: 0 }]); // 9. All successful
 
       const req = makeRequest('GET');
       const res = await GET(req, makeParams());
@@ -205,23 +199,47 @@ describe('reingest API routes', () => {
       expect(body.remainingFailures).toBe(0);
     });
 
-    it('rolls back and tries to run reviewfailed on error', async () => {
+    it('returns 500 on error', async () => {
       mockConnectionManager.executeQuery
-        .mockResolvedValueOnce([{ total: 5 }])
-        .mockResolvedValueOnce(undefined)
-        .mockRejectedValueOnce(new Error('Bulk ingestion failed'));
+        .mockResolvedValueOnce([{ total: 5 }]) // 1. COUNT(*)
+        .mockResolvedValueOnce(mockFailedIds(5)) // 2. SELECT FailedMeasurementID
+        .mockResolvedValueOnce(undefined) // 3. DELETE temporarymeasurements
+        .mockRejectedValueOnce(new Error('Bulk ingestion failed')); // 4. INSERT fails
 
       const req = makeRequest('GET');
       const res = await GET(req, makeParams());
 
       expect(res.status).toBe(500);
       const body = await res.json();
       expect(body.error).toBe('Bulk ingestion failed');
-      expect(mockConnectionManager.rollbackTransaction).toHaveBeenCalledTimes(1);
+      expect(mockConnectionManager.closeConnection).toHaveBeenCalledTimes(1);
+    });
+
+    it('auto-reingests rows marked Ready for reingestion', async () => {
+      mockConnectionManager.executeQuery
+        .mockResolvedValueOnce([{ total: 5 }]) // 1. COUNT(*)
+        .mockResolvedValueOnce(mockFailedIds(5)) // 2. SELECT FailedMeasurementID
+        .mockResolvedValueOnce(undefined) // 3. DELETE temporarymeasurements
+        .mockResolvedValueOnce(undefined) // 4. INSERT INTO temporarymeasurements
+        .mockResolvedValueOnce(undefined) // 5. CALL bulkingestionprocess
+        .mockResolvedValueOnce(undefined) // 6. DELETE failedmeasurements by IDs
+        .mockResolvedValueOnce(undefined) // 7. CALL refresh_failedmeasurements_current
+        .mockResolvedValueOnce([{ cnt: 2 }]) // 8. 2 rows ready for reingestion
+        .mockResolvedValueOnce(undefined) // 9. INSERT ready rows into temporarymeasurements
+        .mockResolvedValueOnce(undefined) // 10. CALL bulkingestionprocess (auto-reingest)
+        .mockResolvedValueOnce(undefined) // 11. DELETE ready rows from failedmeasurements
+        .mockResolvedValueOnce([{ remaining: 1 }]); // 12. COUNT(*) remaining
 
-      // Should attempt to run reviewfailed even after error
-      const reviewFailedCall = mockConnectionManager.executeQuery.mock.calls.find((call: any) => call[0]?.includes('reviewfailed'));
-      expect(reviewFailedCall).toBeDefined();
+      const req = makeRequest('GET');
+      const res = await GET(req, makeParams());
+
+      expect(res.status).toBe(200);
+      const body = await res.json();
+      expect(body.totalProcessed).toBe(5);
+      expect(body.successfulReingestions).toBe(4);
+      expect(body.remainingFailures).toBe(1);
+
+      expect(mockConnectionManager.executeQuery).toHaveBeenCalledTimes(12);
     });
   });
 
@@ -250,12 +268,12 @@ describe('reingest API routes', () => {
 
   describe('Attribute persistence regression tests', () => {
     it('should preserve Codes field when moving to temporarymeasurements', async () => {
-      // Mock count query
       mockConnectionManager.executeQuery
-        .mockResolvedValueOnce([{ total: 1 }]) // Count query
-        .mockResolvedValueOnce(undefined) // DELETE temporarymeasurements
-        .mockResolvedValueOnce({ insertId: 1, affectedRows: 1 }) // INSERT INTO temporarymeasurements
-        .mockResolvedValueOnce(undefined); // DELETE failedmeasurements
+        .mockResolvedValueOnce([{ total: 1 }]) // 1. COUNT(*)
+        .mockResolvedValueOnce(mockFailedIds(1)) // 2. SELECT FailedMeasurementID
+        .mockResolvedValueOnce(undefined) // 3. DELETE temporarymeasurements
+        .mockResolvedValueOnce({ insertId: 1, affectedRows: 1 }) // 4. INSERT INTO temporarymeasurements
+        .mockResolvedValueOnce(undefined); // 5. DELETE failedmeasurements by IDs
 
       const req = makeRequest('POST');
       const res = await POST(req, makeParams());
@@ -272,39 +290,38 @@ describe('reingest API routes', () => {
       expect(insertCall[0]).toContain('fm.Codes'); // Maps from failedmeasurements
     });
 
-    it('should call reviewfailed after successful GET reingestion', async () => {
+    it('should complete GET reingestion without extra validation calls', async () => {
       mockConnectionManager.executeQuery
-        .mockResolvedValueOnce([{ total: 5 }])
-        .mockResolvedValueOnce(undefined) // DELETE temporarymeasurements
-        .mockResolvedValueOnce(undefined) // INSERT INTO temporarymeasurements
-        .mockResolvedValueOnce(undefined) // DELETE failedmeasurements
-        .mockResolvedValueOnce(undefined) // CALL bulkingestionprocess
-        .mockResolvedValueOnce([{ remaining: 0 }]) // Count remaining
-        .mockResolvedValueOnce(undefined); // CALL reviewfailed
+        .mockResolvedValueOnce([{ total: 5 }]) // 1. COUNT(*)
+        .mockResolvedValueOnce(mockFailedIds(5)) // 2. SELECT FailedMeasurementID
+        .mockResolvedValueOnce(undefined) // 3. DELETE temporarymeasurements
+        .mockResolvedValueOnce(undefined) // 4. INSERT INTO temporarymeasurements
+        .mockResolvedValueOnce(undefined) // 5. CALL bulkingestionprocess
+        .mockResolvedValueOnce(undefined) // 6. DELETE failedmeasurements by IDs
+        .mockResolvedValueOnce(undefined) // 7. CALL refresh_failedmeasurements_current
+        .mockResolvedValueOnce([{ cnt: 0 }]) // 8. SELECT COUNT(*) ready
+        .mockResolvedValueOnce([{ remaining: 0 }]); // 9. Count remaining
 
       const req = makeRequest('GET');
       const res = await GET(req, makeParams());
 
       expect(res.status).toBe(200);
-
-      // Verify reviewfailed was called to update failure reasons
-      const reviewFailedCall = mockConnectionManager.executeQuery.mock.calls.find((call: any) => call[0]?.includes('reviewfailed'));
-
-      expect(reviewFailedCall).toBeDefined();
     });
 
     it('should handle rows with codes correctly in bulk ingestion', async () => {
       // This test verifies the complete flow:
       // failedmeasurements (with Codes) → temporarymeasurements → bulkingestionprocess → cmattributes
 
       mockConnectionManager.executeQuery
-        .mockResolvedValueOnce([{ total: 1 }]) // Count
-        .mockResolvedValueOnce(undefined) // DELETE temp
-        .mockResolvedValueOnce(undefined) // INSERT temp
-        .mockResolvedValueOnce(undefined) // DELETE failed
-        .mockResolvedValueOnce(undefined) // bulkingestionprocess
-        .mockResolvedValueOnce([{ remaining: 0 }]) // Count remaining - all succeeded
-        .mockResolvedValueOnce(undefined); // reviewfailed
+        .mockResolvedValueOnce([{ total: 1 }]) // 1. COUNT(*)
+        .mockResolvedValueOnce(mockFailedIds(1)) // 2. SELECT FailedMeasurementID
+        .mockResolvedValueOnce(undefined) // 3. DELETE temporarymeasurements
+        .mockResolvedValueOnce(undefined) // 4. INSERT INTO temporarymeasurements
+        .mockResolvedValueOnce(undefined) // 5. CALL bulkingestionprocess
+        .mockResolvedValueOnce(undefined) // 6. DELETE failedmeasurements by IDs
+        .mockResolvedValueOnce(undefined) // 7. CALL refresh_failedmeasurements_current
+        .mockResolvedValueOnce([{ cnt: 0 }]) // 8. SELECT COUNT(*) ready
+        .mockResolvedValueOnce([{ remaining: 0 }]); // 9. All succeeded
 
       const req = makeRequest('GET');
       const res = await GET(req, makeParams());