diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java index 9b0d87479331..c49978ce6f27 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java @@ -412,6 +412,7 @@ private void startRatisForTest() throws IOException { HddsVolume hddsVolume = (HddsVolume) storageVolume; boolean result = StorageVolumeUtil.checkVolume(hddsVolume, clusterId, clusterId, conf, LOG, null); if (!result) { + LOG.error("Marking volume {} as failed", hddsVolume.getStorageDir().getPath()); volumeSet.failVolume(hddsVolume.getHddsRootDir().getPath()); } } diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/VersionEndpointTask.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/VersionEndpointTask.java index b9326c07c5b1..40844c563e9a 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/VersionEndpointTask.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/VersionEndpointTask.java @@ -120,6 +120,7 @@ private void checkVolumeSet(MutableVolumeSet volumeSet, scmId, clusterId, configuration, LOG, ozoneContainer.getDbVolumeSet()); if (!result) { + LOG.error("Marking volume {} as failed", volume.getStorageDir().getPath()); volumeSet.failVolume(volume.getStorageDir().getPath()); } } diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/HddsVolume.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/HddsVolume.java index f331db7defc3..a4f1afa08ea8 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/HddsVolume.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/HddsVolume.java @@ -299,7 +299,7 @@ public synchronized VolumeCheckResult check(@Nullable Boolean unused) VolumeCheckResult result = super.check(unused); if (isDbLoadFailure()) { - LOG.warn("Volume {} failed to access RocksDB: RocksDB parent directory is null, " + + LOG.error("Volume {} failed to access RocksDB: RocksDB parent directory is null, " + "the volume might not have been loaded properly.", getStorageDir()); return VolumeCheckResult.FAILED; } @@ -312,8 +312,7 @@ public synchronized VolumeCheckResult check(@Nullable Boolean unused) // Check that per-volume RocksDB is present. File dbFile = new File(dbParentDir, CONTAINER_DB_NAME); if (!dbFile.exists() || !dbFile.canRead()) { - LOG.warn("Volume {} failed health check. Could not access RocksDB at " + - "{}", getStorageDir(), dbFile); + LOG.error("Volume {} failed health check. Could not access RocksDB at {}", getStorageDir(), dbFile); return VolumeCheckResult.FAILED; } diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/MutableVolumeSet.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/MutableVolumeSet.java index 9ce69fa14bdf..a79a06b6541f 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/MutableVolumeSet.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/MutableVolumeSet.java @@ -223,8 +223,7 @@ public void checkAllVolumes(StorageVolumeChecker checker) } if (!failedVolumes.isEmpty()) { - LOG.warn("checkAllVolumes got {} failed volumes - {}", - failedVolumes.size(), failedVolumes); + LOG.error("checkAllVolumes got {} failed volumes - {}", failedVolumes.size(), failedVolumes); handleVolumeFailures(failedVolumes); } else { LOG.debug("checkAllVolumes encountered no failures"); @@ -242,6 +241,7 @@ private void handleVolumeFailures( for (StorageVolume v : failedVolumes) { // Immediately mark the volume as failed so it is unavailable // for new containers. + LOG.error("Marking volume {} as failed", v.getStorageDir().getPath()); failVolume(v.getStorageDir().getPath()); } @@ -337,11 +337,10 @@ public void failVolume(String volumeRoot) { failedVolumeMap.put(volumeRoot, volume); volumeHealthMetrics.decrementHealthyVolumes(); volumeHealthMetrics.incrementFailedVolumes(); - LOG.info("Moving Volume : {} to failed Volumes", volumeRoot); } else if (failedVolumeMap.containsKey(volumeRoot)) { - LOG.info("Volume : {} is not active", volumeRoot); + LOG.warn("Unable to fail the volume: {} as it is inactive", volumeRoot); } else { - LOG.warn("Volume : {} does not exist in VolumeSet", volumeRoot); + LOG.warn("Unable to fail the volume: {} as it does not exist in the VolumeSet", volumeRoot); } } finally { this.writeUnlock(); diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/StorageVolume.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/StorageVolume.java index 5260f8468930..68be2ae227ae 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/StorageVolume.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/StorageVolume.java @@ -605,6 +605,7 @@ public DatanodeConfiguration getDatanodeConfig() { } public void failVolume() { + LOG.warn("Volume {} failed", this); setState(VolumeState.FAILED); if (volumeUsage != null) { volumeUsage.shutdown(); @@ -685,6 +686,7 @@ public synchronized VolumeCheckResult check(@Nullable Boolean unused) throw new InterruptedException("Directory check of volume " + this + " interrupted."); } + LOG.error("Directory check of volume {} failed", this); return VolumeCheckResult.FAILED; } diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerReader.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerReader.java index a89a4958aa7a..43aa05c850c5 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerReader.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerReader.java @@ -96,8 +96,8 @@ public void run() { try { readVolume(hddsVolumeDir); } catch (Throwable t) { - LOG.error("Caught an exception during reading container files" + - " from Volume {} {}", hddsVolumeDir, t); + LOG.error("Could not read container files from the volume {}. " + + "Marking the volume as failed", hddsVolumeDir, t); volumeSet.failVolume(hddsVolumeDir.getPath()); } } diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/ScmHAFinalizeUpgradeActionDatanode.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/ScmHAFinalizeUpgradeActionDatanode.java index 74ccdfb87986..37d3b241f0d3 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/ScmHAFinalizeUpgradeActionDatanode.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/upgrade/ScmHAFinalizeUpgradeActionDatanode.java @@ -54,6 +54,7 @@ public void execute(DatanodeStateMachine dsm) throws Exception { if (volume instanceof HddsVolume) { HddsVolume hddsVolume = (HddsVolume) volume; if (!upgradeVolume(hddsVolume, hddsVolume.getClusterID())) { + LOG.error("Marking volume {} as failed", volume.getStorageDir().getAbsolutePath()); volumeSet.failVolume(volume.getStorageDir().getAbsolutePath()); } }