HDDS-14862. Log volume failures and database errors as errors

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java

@@ -412,6 +412,7 @@ private void startRatisForTest() throws IOException {
       HddsVolume hddsVolume = (HddsVolume) storageVolume;
       boolean result = StorageVolumeUtil.checkVolume(hddsVolume, clusterId, clusterId, conf, LOG, null);
       if (!result) {
+        LOG.error("Marking volume {} as failed", hddsVolume.getStorageDir().getPath());
         volumeSet.failVolume(hddsVolume.getHddsRootDir().getPath());
       }
     }

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/VersionEndpointTask.java

@@ -120,6 +120,7 @@ private void checkVolumeSet(MutableVolumeSet volumeSet,
             scmId, clusterId, configuration, LOG,
             ozoneContainer.getDbVolumeSet());
         if (!result) {
+          LOG.error("Marking volume {} as failed", volume.getStorageDir().getPath());
           volumeSet.failVolume(volume.getStorageDir().getPath());
         }
       }

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/HddsVolume.java

@@ -299,7 +299,7 @@ public synchronized VolumeCheckResult check(@Nullable Boolean unused)
     VolumeCheckResult result = super.check(unused);
 
     if (isDbLoadFailure()) {
-      LOG.warn("Volume {} failed to access RocksDB: RocksDB parent directory is null, " +
+      LOG.error("Volume {} failed to access RocksDB: RocksDB parent directory is null, " +
           "the volume might not have been loaded properly.", getStorageDir());
       return VolumeCheckResult.FAILED;
     }
@@ -312,8 +312,7 @@ public synchronized VolumeCheckResult check(@Nullable Boolean unused)
     // Check that per-volume RocksDB is present.
     File dbFile = new File(dbParentDir, CONTAINER_DB_NAME);
     if (!dbFile.exists() || !dbFile.canRead()) {
-      LOG.warn("Volume {} failed health check. Could not access RocksDB at " +
-          "{}", getStorageDir(), dbFile);
+      LOG.error("Volume {} failed health check. Could not access RocksDB at {}", getStorageDir(), dbFile);
       return VolumeCheckResult.FAILED;
     }
 

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/MutableVolumeSet.java

@@ -223,8 +223,7 @@ public void checkAllVolumes(StorageVolumeChecker checker)
     }
 
     if (!failedVolumes.isEmpty()) {
-      LOG.warn("checkAllVolumes got {} failed volumes - {}",
-          failedVolumes.size(), failedVolumes);
+      LOG.error("checkAllVolumes got {} failed volumes - {}", failedVolumes.size(), failedVolumes);
       handleVolumeFailures(failedVolumes);
     } else {
       LOG.debug("checkAllVolumes encountered no failures");
@@ -242,6 +241,7 @@ private void handleVolumeFailures(
       for (StorageVolume v : failedVolumes) {
         // Immediately mark the volume as failed so it is unavailable
         // for new containers.
+        LOG.error("Marking volume {} as failed", v.getStorageDir().getPath());
         failVolume(v.getStorageDir().getPath());
       }
 
@@ -337,11 +337,10 @@ public void failVolume(String volumeRoot) {
         failedVolumeMap.put(volumeRoot, volume);
         volumeHealthMetrics.decrementHealthyVolumes();
         volumeHealthMetrics.incrementFailedVolumes();
-        LOG.info("Moving Volume : {} to failed Volumes", volumeRoot);
       } else if (failedVolumeMap.containsKey(volumeRoot)) {
-        LOG.info("Volume : {} is not active", volumeRoot);
+        LOG.warn("Unable to fail the volume: {} as it is inactive", volumeRoot);
       } else {
-        LOG.warn("Volume : {} does not exist in VolumeSet", volumeRoot);
+        LOG.warn("Unable to fail the volume: {} as it does not exist in the VolumeSet", volumeRoot);
       }
     } finally {
       this.writeUnlock();

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/StorageVolume.java

@@ -605,6 +605,7 @@ public DatanodeConfiguration getDatanodeConfig() {
   }
 
   public void failVolume() {
+    LOG.warn("Volume {} failed", this);
     setState(VolumeState.FAILED);
     if (volumeUsage != null) {
       volumeUsage.shutdown();
@@ -685,6 +686,7 @@ public synchronized VolumeCheckResult check(@Nullable Boolean unused)
         throw new InterruptedException("Directory check of volume " + this +
             " interrupted.");
       }
+      LOG.error("Directory check of volume {} failed", this);
       return VolumeCheckResult.FAILED;
     }
 

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerReader.java

@@ -96,8 +96,8 @@ public void run() {
     try {
       readVolume(hddsVolumeDir);
     } catch (Throwable t) {
-      LOG.error("Caught an exception during reading container files" +
-          " from Volume {} {}", hddsVolumeDir, t);
+      LOG.error("Could not read container files from the volume {}. " +
+          "Marking the volume as failed", hddsVolumeDir, t);
       volumeSet.failVolume(hddsVolumeDir.getPath());
     }
   }

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/ScmHAFinalizeUpgradeActionDatanode.java

@@ -54,6 +54,7 @@ public void execute(DatanodeStateMachine dsm) throws Exception {
         if (volume instanceof HddsVolume) {
           HddsVolume hddsVolume = (HddsVolume) volume;
           if (!upgradeVolume(hddsVolume, hddsVolume.getClusterID())) {
+            LOG.error("Marking volume {} as failed", volume.getStorageDir().getAbsolutePath());
             volumeSet.failVolume(volume.getStorageDir().getAbsolutePath());
           }
         }