feat[auth-export-report]: remove the Linked bib fields column

NEWS.md

@@ -1,6 +1,7 @@
 ## Unreleased v3.5.0
 ### Stories
 * [MODEXPW-576](https://folio-org.atlassian.net/browse/MODEXPW-576) Enhance "MARC authority headings update" Report with Record Type Column Based on Consortium Environment.
+* [MODEXPW-619](https://folio-org.atlassian.net/browse/MODEXPW-619) Remove the Linked bib fields column for the Authority headings change report
 
 ### Bug fixes
 * [MODEXPW-617](https://folio-org.atlassian.net/browse/MODEXPW-617) Update Apache Kafka topic template to prevent ambiguous env name matches.

descriptors/ModuleDescriptor-template.json

@@ -40,7 +40,7 @@
     },
     {
       "id": "instance-authority-links-statistics",
-      "version": "2.1"
+      "version": "2.1 3.0"
     },
     {
       "id": "holdings-note-types",

src/main/java/org/folio/dew/ModDataExportWorkerApplication.java

@@ -6,7 +6,10 @@
 import org.springframework.boot.autoconfigure.domain.EntityScan;
 import org.springframework.cloud.openfeign.EnableFeignClients;
 
-@SpringBootApplication
+@SpringBootApplication(scanBasePackages = {
+  "org.folio.dew",
+  "org.folio.spring.scope"
+})
 @EnableFeignClients(basePackages = "org.folio.dew.client")
 @EnableBatchProcessing(isolationLevelForCreate = "ISOLATION_READ_COMMITTED")
 @EntityScan("org.folio.de.entity")

src/main/java/org/folio/dew/batch/authoritycontrol/AuthorityControlToExportFormatMapper.java

@@ -38,7 +38,6 @@ public AuthUpdateHeadingExportFormat convertToAuthUpdateHeadingsExportFormat(Aut
     exportFormat.setOriginalHeadingType(dto.getHeadingTypeOld());
     exportFormat.setIdentifier(dto.getNaturalIdNew());
     exportFormat.setAuthoritySourceFileName(dto.getSourceFileNew());
-    exportFormat.setNumberOfBibliographicRecordsLinked(dto.getLbTotal().toString());
     if (folioTenantService.isConsortiumTenant()) {
       exportFormat.setSource(Boolean.TRUE.equals(dto.getShared()) ? "shared" : "local");
     }

src/main/java/org/folio/dew/batch/authoritycontrol/readers/AuthUpdateHeadingsItemReader.java

@@ -2,26 +2,117 @@
 
 import static org.folio.dew.domain.dto.authority.control.AuthorityDataStatDto.ActionEnum.UPDATE_HEADING;
 
+import java.time.OffsetDateTime;
+import java.util.ArrayDeque;
+import java.util.Deque;
 import org.folio.dew.client.EntitiesLinksStatsClient;
 import org.folio.dew.config.properties.AuthorityControlJobProperties;
 import org.folio.dew.domain.dto.authority.control.AuthorityControlExportConfig;
 import org.folio.dew.domain.dto.authority.control.AuthorityDataStatDto;
 import org.folio.dew.domain.dto.authority.control.AuthorityDataStatDtoCollection;
+import org.folio.dew.service.FolioTenantService;
+import org.folio.spring.FolioExecutionContext;
+import org.folio.spring.scope.FolioExecutionContextService;
 import org.springframework.batch.core.configuration.annotation.StepScope;
 import org.springframework.stereotype.Component;
 
 @StepScope
 @Component
 public class AuthUpdateHeadingsItemReader extends AuthorityControlItemReader<AuthorityDataStatDto> {
 
+  private final FolioExecutionContext context;
+  private final FolioExecutionContextService executionService;
+  private final String consortiumCentralTenant;
+  private final boolean isConsortiumMemberTenant;
+  private final boolean isConsortiumCentralTenant;
+  private OffsetDateTime toConsortiumDate;
+
+  private final Deque<AuthorityDataStatDto> memberTenantStats = new ArrayDeque<>();
+  private final Deque<AuthorityDataStatDto> centralTenantStats = new ArrayDeque<>();
+
   public AuthUpdateHeadingsItemReader(EntitiesLinksStatsClient entitiesLinksStatsClient,
                                       AuthorityControlExportConfig exportConfig,
-                                      AuthorityControlJobProperties jobProperties) {
+                                      AuthorityControlJobProperties jobProperties,
+                                      FolioTenantService folioTenantService,
+                                      FolioExecutionContext context,
+                                      FolioExecutionContextService executionService) {
     super(entitiesLinksStatsClient, exportConfig, jobProperties);
+    this.context = context;
+    this.executionService = executionService;
+    this.consortiumCentralTenant = folioTenantService.getConsortiumCentralTenant();
+    this.isConsortiumMemberTenant = consortiumCentralTenant != null && !consortiumCentralTenant.equals(this.context.getTenantId());
+    this.isConsortiumCentralTenant = consortiumCentralTenant != null && consortiumCentralTenant.equals(this.context.getTenantId());
+    this.toConsortiumDate = this.toDate;
+  }
+
+  @Override
+  protected AuthorityDataStatDto doRead() {
+    if (!isConsortiumMemberTenant) {
+      return super.doRead();
+    }
+    return getConsortiumAuthorityDataStat(limit);
   }
 
   @Override
   protected AuthorityDataStatDtoCollection getCollection(int limit) {
-    return entitiesLinksStatsClient.getAuthorityStats(limit, UPDATE_HEADING, fromDate(), toDate());
+    var authorityStats = entitiesLinksStatsClient.getAuthorityStats(limit, UPDATE_HEADING, fromDate(), toDate());
+    if (isConsortiumCentralTenant && authorityStats != null && !authorityStats.getStats().isEmpty()) {
+      authorityStats.getStats().forEach(stat -> stat.setShared(true));
+    }
+    return authorityStats;
+  }
+
+  private AuthorityDataStatDto getConsortiumAuthorityDataStat(int limit) {
+    if (memberTenantStats.isEmpty()) {
+      loadNextMemberTenantStatsPage(limit);
+    }
+    if (centralTenantStats.isEmpty()) {
+      loadNextCentralTenantStatsPage(limit);
+    }
+    // if there are no stats in both member and central tenant return null to finish reading
+    if (memberTenantStats.isEmpty() && centralTenantStats.isEmpty()) {
+      return null;
+    }
+
+    if (memberTenantStats.isEmpty()) {
+      return centralTenantStats.pollFirst();
+    }
+    if (centralTenantStats.isEmpty()) {
+      return memberTenantStats.pollFirst();
+    }
+
+    var memberStat = memberTenantStats.peekFirst();
+    var centralStat = centralTenantStats.peekFirst();
+
+    return memberStat.getMetadata().getStartedAt().isAfter(centralStat.getMetadata().getStartedAt())
+      ? memberTenantStats.pollFirst()
+      : centralTenantStats.pollFirst();
+  }
+
+  private void loadNextMemberTenantStatsPage(int limit) {
+    if (toDate() != null) {
+      var authorityStats = entitiesLinksStatsClient.getAuthorityStats(limit, UPDATE_HEADING, fromDate(), toDate());
+      if (authorityStats != null) {
+        toDate = authorityStats.getNext();
+        memberTenantStats.addAll(authorityStats.getStats());
+      }
+    }
+  }
+
+  private void loadNextCentralTenantStatsPage(int limit) {
+    if (formatConsortiumDate() != null) {
+      var authorityStats = executionService.execute(consortiumCentralTenant, context, () ->
+        entitiesLinksStatsClient.getAuthorityStats(limit, UPDATE_HEADING, fromDate(), formatConsortiumDate()));
+
+      if (authorityStats != null && !authorityStats.getStats().isEmpty()) {
+        authorityStats.getStats().forEach(stat -> stat.setShared(true));
+        toConsortiumDate = authorityStats.getNext();
+        centralTenantStats.addAll(authorityStats.getStats());
+      }
+    }
+  }
+
+  private String formatConsortiumDate() {
+    return toConsortiumDate == null ? null : toConsortiumDate.toString();
   }
 }

src/main/java/org/folio/dew/batch/authoritycontrol/readers/AuthorityControlItemReader.java

@@ -16,11 +16,11 @@
 public abstract class AuthorityControlItemReader<T extends DataStatDTO>
   extends AbstractItemCountingItemStreamItemReader<T> {
   protected final EntitiesLinksStatsClient entitiesLinksStatsClient;
-  private final int limit;
-  private OffsetDateTime fromDate;
-  private OffsetDateTime toDate;
+  protected final int limit;
   private int currentChunkOffset;
   private List<T> currentChunk;
+  private OffsetDateTime fromDate;
+  protected OffsetDateTime toDate;
 
   protected AuthorityControlItemReader(EntitiesLinksStatsClient entitiesLinksStatsClient,
                                        AuthorityControlExportConfig exportConfig,
@@ -54,7 +54,6 @@ protected T doRead() {
     if (currentChunk.isEmpty()) {
       return null;
     }
-
     return currentChunk.get(currentChunkOffset++);
   }
 

src/main/java/org/folio/dew/domain/dto/authoritycontrol/exportformat/AuthUpdateHeadingExportFormat.java

@@ -20,7 +20,6 @@ public class AuthUpdateHeadingExportFormat implements AuthorityControlExportForm
   @ExportHeader(NEW_1XX)
   private String newHeadingType;
   private String authoritySourceFileName;
-  private String numberOfBibliographicRecordsLinked;
   @ExportHeader(AUTHORITY_RECORD_TYPE)
   private String source;
   private String updater;

src/main/java/org/folio/dew/service/FolioTenantService.java

@@ -63,7 +63,17 @@ protected boolean tenantExists() {
    * Check if current context tenant is a part of consortium
    * */
   public boolean isConsortiumTenant() {
-    var centralTenant =  userTenantsService.getCentralTenant(context.getTenantId());
+    var centralTenant = userTenantsService.getCentralTenant(context.getTenantId());
     return centralTenant.isPresent() && StringUtils.isNotEmpty(centralTenant.get());
   }
+
+  /**
+   * Get consortium central tenant if current context tenant is a part of consortium
+   *
+   * @return consortium central tenant or null if current context tenant is not a part of consortium
+   */
+  public String getConsortiumCentralTenant() {
+    var centralTenant = userTenantsService.getCentralTenant(context.getTenantId());
+    return centralTenant.orElse(null);
+  }
 }

src/test/java/org/folio/dew/AuthorityControlConsortiumTest.java

@@ -15,6 +15,7 @@
 import com.fasterxml.jackson.core.JsonProcessingException;
 import java.io.File;
 import java.time.LocalDate;
+import java.util.List;
 import java.util.UUID;
 import lombok.SneakyThrows;
 import org.apache.commons.io.FileUtils;
@@ -40,10 +41,12 @@
 
 class AuthorityControlConsortiumTest extends BaseBatchTest {
 
-  private static final String EXPECTED_AUTH_HEADING_UPDATE_OUTPUT =
+  private static final String EXPECTED_CONSORTIUM_OUTPUT =
     "src/test/resources/output/authority_control/auth_heading_update_consortium.csv";
+  private static final String EXPECTED_CONSORTIUM_MEMBER_TENANT_OUTPUT =
+    "src/test/resources/output/authority_control/auth_heading_update_consortium_member_tenant.csv";
   private static final String EXPECTED_S3_FILE_PATH =
-    "mod-data-export-worker/authority_control_export/consortium/";
+    "mod-data-export-worker/authority_control_export/college/";
   @Autowired
   private Job getAuthHeadingJob;
   @Autowired
@@ -55,12 +58,12 @@ class AuthorityControlConsortiumTest extends BaseBatchTest {
 
   @BeforeAll
   static void beforeAll() {
-    setUpTenant(CONSORTIUM_TENANT);
+    setUpConsortiumTenant(CONSORTIUM_TENANT, List.of(CONSORTIUM_MEMBER_TENANT), CONSORTIUM_MEMBER_TENANT);
   }
 
   @Test
   @DisplayName("Run AuthHeadingJob export in the consortium tenant successfully")
-  void authHeadingJobTest() throws Exception {
+  void authHeadingJobTestReceiveStatsFromMemberAndCentralTenantTest() throws Exception {
     var exportConfig = buildExportConfig("2024-01-01", "2024-12-01");
     var testLauncher = createTestLauncher(getAuthHeadingJob);
     var jobParameters = prepareJobParameters(exportConfig);
@@ -69,27 +72,46 @@ void authHeadingJobTest() throws Exception {
 
     assertThat(jobExecution.getExitStatus()).isEqualTo(ExitStatus.COMPLETED);
 
-    verifyFile(jobExecution);
+    verifyFile(jobExecution, EXPECTED_CONSORTIUM_OUTPUT);
     wireMockServer.verify(getRequestedFor(urlEqualTo(
       "/links/stats/authority?limit=2&action=UPDATE_HEADING&fromDate=2024-01-01T00%3A00Z&toDate=2024-12-01T23%3A59%3A59.999999999Z")));
     wireMockServer.verify(getRequestedFor(urlEqualTo(
-      "/links/stats/authority?limit=2&action=UPDATE_HEADING&fromDate=2024-01-01T00%3A00Z&toDate=2024-08-01T12%3A00Z")));
+      "/links/stats/authority?limit=2&action=UPDATE_HEADING&fromDate=2024-01-01T00%3A00Z&toDate=2024-08-01T11%3A00Z")));
     verifyJobEvent();
   }
 
-  private void verifyFile(JobExecution jobExecution) throws Exception {
+  @Test
+  @DisplayName("Run AuthHeadingJob export in the consortium tenant successfully")
+  void authHeadingJobReceiveStatsOnlyFromMemberTenantTest() throws Exception {
+    var exportConfig = buildExportConfig("2025-02-14", "2025-02-17");
+    var testLauncher = createTestLauncher(getAuthHeadingJob);
+    var jobParameters = prepareJobParameters(exportConfig);
+
+    var jobExecution = testLauncher.launchJob(jobParameters);
+
+    assertThat(jobExecution.getExitStatus()).isEqualTo(ExitStatus.COMPLETED);
+
+    verifyFile(jobExecution, EXPECTED_CONSORTIUM_MEMBER_TENANT_OUTPUT);
+    wireMockServer.verify(getRequestedFor(urlEqualTo(
+      "/links/stats/authority?limit=2&action=UPDATE_HEADING&fromDate=2025-02-14T00%3A00Z&toDate=2025-02-17T23%3A59%3A59.999999999Z")));
+    wireMockServer.verify(getRequestedFor(urlEqualTo(
+      "/links/stats/authority?limit=2&action=UPDATE_HEADING&fromDate=2025-02-14T00%3A00Z&toDate=2025-02-15T11%3A00Z")));
+    verifyJobEvent();
+  }
+
+  private void verifyFile(JobExecution jobExecution, String expectedReportOutput) throws Exception {
     var executionContext = jobExecution.getExecutionContext();
     var fileInStorage = executionContext.getString(OUTPUT_FILES_IN_STORAGE);
     var fileName = executionContext.getString(AUTHORITY_CONTROL_FILE_NAME);
 
     assertEquals(EXPECTED_S3_FILE_PATH + fileName, fileInStorage);
-    verifyFileOutput(fileInStorage);
+    verifyFileOutput(fileInStorage, expectedReportOutput);
   }
 
-  private void verifyFileOutput(String fileInStorage) throws Exception {
+  private void verifyFileOutput(String fileInStorage, String expectedReportOutput) throws Exception {
     var presignedUrl = remoteFilesStorage.objectToPresignedObjectUrl(fileInStorage);
     var actualOutput = actualFileOutput(presignedUrl);
-    var expectedOutput = new FileSystemResource(EXPECTED_AUTH_HEADING_UPDATE_OUTPUT);
+    var expectedOutput = new FileSystemResource(expectedReportOutput);
     assertTrue(FileUtils.contentEqualsIgnoreEOL(expectedOutput.getFile(), actualOutput.getFile(), "UTF-8")
       , "Files are not identical!");
   }