HDDS-13891. SCM-based health monitoring and batch processing in Recon

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/NullReplicationQueue.java → hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/MonitoringReplicationQueue.java

@@ -18,11 +18,11 @@
 package org.apache.hadoop.hdds.scm.container.replication;
 
 /**
- * A class which extents ReplicationQueue and does nothing. This is used when
+ * A class which extends ReplicationQueue and does nothing. This is used when
  * checking containers in a read-only mode, where we don't want to queue them
  * for replication.
  */
-public class NullReplicationQueue extends ReplicationQueue {
+public class MonitoringReplicationQueue extends ReplicationQueue {
 
   @Override
   public void enqueue(ContainerHealthResult.UnderReplicatedHealthResult

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationManager.java

@@ -189,8 +189,8 @@ public class ReplicationManager implements SCMService, ContainerReplicaPendingOp
   private final UnderReplicatedProcessor underReplicatedProcessor;
   private final OverReplicatedProcessor overReplicatedProcessor;
   private final HealthCheck containerCheckChain;
-  private final ReplicationQueue nullReplicationQueue =
-      new NullReplicationQueue();
+  private final ReplicationQueue noOpsReplicationQueue =
+      new MonitoringReplicationQueue();
 
   /**
    * Constructs ReplicationManager instance with the given configuration.
@@ -853,18 +853,24 @@ protected void processContainer(ContainerInfo containerInfo,
   protected boolean processContainer(ContainerInfo containerInfo,
       ReplicationQueue repQueue, ReplicationManagerReport report,
       boolean readOnly) throws ContainerNotFoundException {
+    ContainerID containerID = containerInfo.containerID();
+    Set<ContainerReplica> replicas = containerManager.getContainerReplicas(
+        containerID);
+    List<ContainerReplicaOp> pendingOps =
+        containerReplicaPendingOps.getPendingOps(containerID);
+    return processContainer(containerInfo, replicas, pendingOps, repQueue, report,
+        readOnly);
+  }
+
+  protected boolean processContainer(ContainerInfo containerInfo,
+      Set<ContainerReplica> replicas, List<ContainerReplicaOp> pendingOps,
+      ReplicationQueue repQueue, ReplicationManagerReport report,
+      boolean readOnly) throws ContainerNotFoundException {
     synchronized (containerInfo) {
       // Reset health state to HEALTHY before processing this container
       report.resetContainerHealthState();
-
-      ContainerID containerID = containerInfo.containerID();
       final boolean isEC = isEC(containerInfo.getReplicationConfig());
 
-      Set<ContainerReplica> replicas = containerManager.getContainerReplicas(
-          containerID);
-      List<ContainerReplicaOp> pendingOps =
-          containerReplicaPendingOps.getPendingOps(containerID);
-
       ContainerCheckRequest checkRequest = new ContainerCheckRequest.Builder()
           .setContainerInfo(containerInfo)
           .setContainerReplicas(replicas)
@@ -1006,7 +1012,7 @@ public ContainerHealthResult getContainerReplicationHealth(
   public boolean checkContainerStatus(ContainerInfo containerInfo,
       ReplicationManagerReport report) throws ContainerNotFoundException {
     report.increment(containerInfo.getState());
-    return processContainer(containerInfo, nullReplicationQueue, report, true);
+    return processContainer(containerInfo, noOpsReplicationQueue, report, true);
   }
 
   /**

hadoop-ozone/integration-test-recon/pom.xml

@@ -102,6 +102,11 @@
       <artifactId>hdds-interface-client</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.apache.ozone</groupId>
+      <artifactId>hdds-interface-server</artifactId>
+      <scope>test</scope>
+    </dependency>
     <dependency>
       <groupId>org.apache.ozone</groupId>
       <artifactId>hdds-server-framework</artifactId>
@@ -209,6 +214,16 @@
           <proc>none</proc>
         </configuration>
       </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-dependency-plugin</artifactId>
+        <configuration>
+          <ignoredUnusedDeclaredDependencies>
+            <ignoredUnusedDeclaredDependency>org.mockito:mockito-inline:jar</ignoredUnusedDeclaredDependency>
+            <ignoredUnusedDeclaredDependency>org.mockito:mockito-junit-jupiter:jar</ignoredUnusedDeclaredDependency>
+          </ignoredUnusedDeclaredDependencies>
+        </configuration>
+      </plugin>
     </plugins>
   </build>
 

hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/TestReconContainerEndpoint.java

@@ -39,9 +39,7 @@
 import org.apache.hadoop.ozone.recon.api.ContainerEndpoint;
 import org.apache.hadoop.ozone.recon.api.types.KeyMetadata;
 import org.apache.hadoop.ozone.recon.api.types.KeysResponse;
-import org.apache.hadoop.ozone.recon.persistence.ContainerHealthSchemaManager;
 import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager;
-import org.apache.hadoop.ozone.recon.scm.ReconContainerManager;
 import org.apache.hadoop.ozone.recon.spi.impl.OzoneManagerServiceProviderImpl;
 import org.apache.hadoop.ozone.recon.tasks.ReconTaskControllerImpl;
 import org.apache.ozone.test.GenericTestUtils;
@@ -215,16 +213,13 @@ public void testContainerEndpointForOBSBucket() throws Exception {
   private Response getContainerEndpointResponse(long containerId) {
     OzoneStorageContainerManager reconSCM =
         recon.getReconServer().getReconStorageContainerManager();
-    ReconContainerManager reconContainerManager =
-        (ReconContainerManager) reconSCM.getContainerManager();
-    ContainerHealthSchemaManager containerHealthSchemaManager =
-        reconContainerManager.getContainerSchemaManager();
     ReconOMMetadataManager omMetadataManagerInstance =
         (ReconOMMetadataManager)
             recon.getReconServer().getOzoneManagerServiceProvider()
                 .getOMMetadataManagerInstance();
     ContainerEndpoint containerEndpoint =
-        new ContainerEndpoint(reconSCM, containerHealthSchemaManager,
+        new ContainerEndpoint(reconSCM,
+            null, // ContainerHealthSchemaManager - not needed for this test
             recon.getReconServer().getReconNamespaceSummaryManager(),
             recon.getReconServer().getReconContainerMetadataManager(),
             omMetadataManagerInstance);