diff --git a/hadoop-hdds/container-service/dev-support/findbugsExcludeFile.xml b/hadoop-hdds/container-service/dev-support/findbugsExcludeFile.xml
index 40d78d0cd6ce..85eca243dd61 100644
--- a/hadoop-hdds/container-service/dev-support/findbugsExcludeFile.xml
+++ b/hadoop-hdds/container-service/dev-support/findbugsExcludeFile.xml
@@ -15,4 +15,8 @@
limitations under the License.
-->
+
+
+
+
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/DiskCheckUtil.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/DiskCheckUtil.java
index 73e69eddc15b..f20c24c00e45 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/DiskCheckUtil.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/DiskCheckUtil.java
@@ -42,6 +42,7 @@
* where the disk is mounted.
*/
public final class DiskCheckUtil {
+ public static final String TOO_MANY_OPEN_FILES = "Too many open files";
// For testing purposes, an alternate check implementation can be provided
// to inject failures.
private static DiskChecks impl = new DiskChecksImpl();
@@ -156,6 +157,14 @@ public boolean checkReadWrite(File storageDir,
logError(storageDir, String.format("Could not write file %s " +
"for volume check.", testFile.getAbsolutePath()), ioEx);
return false;
+ } catch (Throwable ex) {
+ String msg = ex.getMessage();
+ String causeMsg = ex.getCause() != null ? ex.getCause().getMessage() : null;
+ causeMsg = causeMsg != null ? causeMsg : "";
+ if ((msg != null && msg.contains(TOO_MANY_OPEN_FILES)) || causeMsg.contains(TOO_MANY_OPEN_FILES)) {
+ LOG.warn("Could not write file {} for volume check.", testFile.getAbsolutePath(), ex);
+ return true;
+ }
}
// Read data back from the test file.
@@ -173,9 +182,19 @@ public boolean checkReadWrite(File storageDir,
"for volume check.", testFile.getAbsolutePath()), notFoundEx);
return false;
} catch (IOException ioEx) {
+ String ioem = ioEx.getMessage();
+ String causeMsg = ioEx.getCause() != null ? ioEx.getCause().getMessage() : null;
+ causeMsg = causeMsg != null ? causeMsg : "";
+ if ((ioem != null && ioem.contains(TOO_MANY_OPEN_FILES)) || causeMsg.contains(TOO_MANY_OPEN_FILES)) {
+ LOG.warn("Could not read file {} for volume check.", testFile.getAbsolutePath(), ioEx);
+ return true;
+ }
logError(storageDir, String.format("Could not read file %s " +
"for volume check.", testFile.getAbsolutePath()), ioEx);
return false;
+ } catch (Throwable t) {
+ logError(storageDir, String.format("Could not read file %s " +
+ "for volume check.", testFile.getAbsolutePath()), t);
}
// Check that test file has the expected content.
@@ -201,7 +220,7 @@ private void logError(File storageDir, String message) {
LOG.error("Volume {} failed health check. {}", storageDir, message);
}
- private void logError(File storageDir, String message, Exception ex) {
+ private void logError(File storageDir, String message, Throwable ex) {
LOG.error("Volume {} failed health check. {}", storageDir, message, ex);
}
}
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/utils/TestDiskCheckUtil.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/utils/TestDiskCheckUtil.java
index d66f39455949..d4a3e671c5ae 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/utils/TestDiskCheckUtil.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/utils/TestDiskCheckUtil.java
@@ -20,11 +20,23 @@
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mockStatic;
import java.io.File;
+import java.nio.file.FileSystemException;
+import java.nio.file.Files;
+import java.nio.file.OpenOption;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import org.apache.ratis.util.FileUtils;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.api.parallel.Execution;
+import org.junit.jupiter.api.parallel.ExecutionMode;
+import org.mockito.MockedStatic;
/**
* Tests {@link DiskCheckUtil} does not incorrectly identify an unhealthy
@@ -32,6 +44,7 @@
* Tests that it identifies an improperly configured directory mount point.
*
*/
+@Execution(ExecutionMode.SAME_THREAD)
public class TestDiskCheckUtil {
@TempDir
@@ -80,4 +93,38 @@ public void testReadWrite() {
assertNotNull(children);
assertEquals(0, children.length);
}
+
+ @Test
+ public void testWriteFailureDueToTooManyOpenFiles() {
+ try (MockedStatic mockService = mockStatic(FileUtils.class)) {
+
+ mockService.when(() -> FileUtils.newOutputStreamForceAtClose(any(File.class), any(OpenOption[].class)))
+ .thenThrow(new ExceptionInInitializerError("java.io.IOException: Too many open files"));
+
+ String path = "/Volumes/DiskFullTest/disk-check-c967569c";
+ assertThrows(ExceptionInInitializerError.class,
+ () -> FileUtils.newOutputStreamForceAtClose(new File(path), new OpenOption[2]));
+
+ // Test that checkReadWrite returns true for the too many open file case
+ boolean result = DiskCheckUtil.checkReadWrite(testDir, testDir, 1024);
+ assertTrue(result, "checkReadWrite should return true when too many open files");
+ }
+ }
+
+ @Test
+ public void testReadFailureDueToTooManyOpenFiles() {
+ try (MockedStatic mockService = mockStatic(Files.class)) {
+
+ String path = "/Volumes/DiskFullTest/disk-check-c967569c";
+ mockService.when(() -> Files.newInputStream(any(Path.class), any(OpenOption[].class)))
+ .thenThrow(new FileSystemException(path + ": Too many open files"));
+
+ assertThrows(FileSystemException.class,
+ () -> Files.newInputStream(Paths.get(path), new OpenOption[2]));
+
+ // Test that checkReadWrite returns true for the too many open file case
+ boolean result = DiskCheckUtil.checkReadWrite(testDir, testDir, 1024);
+ assertTrue(result, "checkReadWrite should return true when too many open files");
+ }
+ }
}