HDDS-14895. Improve DiskCheckUtil.checkReadWrite to tolerate not enough file handler

hadoop-hdds/container-service/dev-support/findbugsExcludeFile.xml

@@ -15,4 +15,8 @@
    limitations under the License.
 -->
 <FindBugsFilter>
+  <Match>
+    <Class name="org.apache.hadoop.ozone.container.common.utils.TestDiskCheckUtil"/>
+    <Bug pattern="OS_OPEN_STREAM" />
+  </Match>
 </FindBugsFilter>

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/DiskCheckUtil.java

@@ -42,6 +42,7 @@
  * where the disk is mounted.
  */
 public final class DiskCheckUtil {
+  public static final String TOO_MANY_OPEN_FILES = "Too many open files";
   // For testing purposes, an alternate check implementation can be provided
   // to inject failures.
   private static DiskChecks impl = new DiskChecksImpl();
@@ -156,6 +157,14 @@ public boolean checkReadWrite(File storageDir,
         logError(storageDir, String.format("Could not write file %s " +
             "for volume check.", testFile.getAbsolutePath()), ioEx);
         return false;
+      } catch (Throwable ex) {
+        String msg = ex.getMessage();
+        String causeMsg = ex.getCause() != null ? ex.getCause().getMessage() : null;
+        causeMsg = causeMsg != null ? causeMsg : "";
+        if ((msg != null && msg.contains(TOO_MANY_OPEN_FILES)) || causeMsg.contains(TOO_MANY_OPEN_FILES)) {
+          LOG.warn("Could not write file {} for volume check.", testFile.getAbsolutePath(), ex);
+          return true;
+        }
       }
 
       // Read data back from the test file.
@@ -173,9 +182,19 @@ public boolean checkReadWrite(File storageDir,
             "for volume check.", testFile.getAbsolutePath()), notFoundEx);
         return false;
       } catch (IOException ioEx) {
+        String ioem = ioEx.getMessage();
+        String causeMsg = ioEx.getCause() != null ? ioEx.getCause().getMessage() : null;
+        causeMsg = causeMsg != null ? causeMsg : "";
+        if ((ioem != null && ioem.contains(TOO_MANY_OPEN_FILES)) || causeMsg.contains(TOO_MANY_OPEN_FILES)) {
+          LOG.warn("Could not read file {} for volume check.", testFile.getAbsolutePath(), ioEx);
+          return true;
+        }
         logError(storageDir, String.format("Could not read file %s " +
             "for volume check.", testFile.getAbsolutePath()), ioEx);
         return false;
+      } catch (Throwable t) {
+        logError(storageDir, String.format("Could not read file %s " +
+            "for volume check.", testFile.getAbsolutePath()), t);
       }
 
       // Check that test file has the expected content.
@@ -201,7 +220,7 @@ private void logError(File storageDir, String message) {
       LOG.error("Volume {} failed health check. {}", storageDir, message);
     }
 
-    private void logError(File storageDir, String message, Exception ex) {
+    private void logError(File storageDir, String message, Throwable ex) {
       LOG.error("Volume {} failed health check. {}", storageDir, message, ex);
     }
   }

hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/utils/TestDiskCheckUtil.java

@@ -20,18 +20,31 @@
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mockStatic;
 
 import java.io.File;
+import java.nio.file.FileSystemException;
+import java.nio.file.Files;
+import java.nio.file.OpenOption;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import org.apache.ratis.util.FileUtils;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.api.parallel.Execution;
+import org.junit.jupiter.api.parallel.ExecutionMode;
+import org.mockito.MockedStatic;
 
 /**
  * Tests {@link DiskCheckUtil} does not incorrectly identify an unhealthy
  * disk or mount point.
  * Tests that it identifies an improperly configured directory mount point.
  *
  */
+@Execution(ExecutionMode.SAME_THREAD)
 public class TestDiskCheckUtil {
 
   @TempDir
@@ -80,4 +93,38 @@ public void testReadWrite() {
     assertNotNull(children);
     assertEquals(0, children.length);
   }
+
+  @Test
+  public void testWriteFailureDueToTooManyOpenFiles() {
+    try (MockedStatic<FileUtils> mockService = mockStatic(FileUtils.class)) {
+
+      mockService.when(() -> FileUtils.newOutputStreamForceAtClose(any(File.class), any(OpenOption[].class)))
+          .thenThrow(new ExceptionInInitializerError("java.io.IOException: Too many open files"));
+
+      String path = "/Volumes/DiskFullTest/disk-check-c967569c";
+      assertThrows(ExceptionInInitializerError.class,
+          () -> FileUtils.newOutputStreamForceAtClose(new File(path), new OpenOption[2]));
+
+      // Test that checkReadWrite returns true for the too many open file case
+      boolean result = DiskCheckUtil.checkReadWrite(testDir, testDir, 1024);
+      assertTrue(result, "checkReadWrite should return true when too many open files");
+    }
+  }
+
+  @Test
+  public void testReadFailureDueToTooManyOpenFiles() {
+    try (MockedStatic<Files> mockService = mockStatic(Files.class)) {
+
+      String path = "/Volumes/DiskFullTest/disk-check-c967569c";
+      mockService.when(() -> Files.newInputStream(any(Path.class), any(OpenOption[].class)))
+          .thenThrow(new FileSystemException(path + ": Too many open files"));
+
+      assertThrows(FileSystemException.class,
+          () -> Files.newInputStream(Paths.get(path), new OpenOption[2]));
+
+      // Test that checkReadWrite returns true for the too many open file case
+      boolean result = DiskCheckUtil.checkReadWrite(testDir, testDir, 1024);
+      assertTrue(result, "checkReadWrite should return true when too many open files");
+    }
+  }
 }