diff --git a/hadoop-hdds/container-service/dev-support/findbugsExcludeFile.xml b/hadoop-hdds/container-service/dev-support/findbugsExcludeFile.xml index 40d78d0cd6ce..85eca243dd61 100644 --- a/hadoop-hdds/container-service/dev-support/findbugsExcludeFile.xml +++ b/hadoop-hdds/container-service/dev-support/findbugsExcludeFile.xml @@ -15,4 +15,8 @@ limitations under the License. --> + + + + diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/DiskCheckUtil.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/DiskCheckUtil.java index 73e69eddc15b..f20c24c00e45 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/DiskCheckUtil.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/DiskCheckUtil.java @@ -42,6 +42,7 @@ * where the disk is mounted. */ public final class DiskCheckUtil { + public static final String TOO_MANY_OPEN_FILES = "Too many open files"; // For testing purposes, an alternate check implementation can be provided // to inject failures. private static DiskChecks impl = new DiskChecksImpl(); @@ -156,6 +157,14 @@ public boolean checkReadWrite(File storageDir, logError(storageDir, String.format("Could not write file %s " + "for volume check.", testFile.getAbsolutePath()), ioEx); return false; + } catch (Throwable ex) { + String msg = ex.getMessage(); + String causeMsg = ex.getCause() != null ? ex.getCause().getMessage() : null; + causeMsg = causeMsg != null ? causeMsg : ""; + if ((msg != null && msg.contains(TOO_MANY_OPEN_FILES)) || causeMsg.contains(TOO_MANY_OPEN_FILES)) { + LOG.warn("Could not write file {} for volume check.", testFile.getAbsolutePath(), ex); + return true; + } } // Read data back from the test file. @@ -173,9 +182,19 @@ public boolean checkReadWrite(File storageDir, "for volume check.", testFile.getAbsolutePath()), notFoundEx); return false; } catch (IOException ioEx) { + String ioem = ioEx.getMessage(); + String causeMsg = ioEx.getCause() != null ? ioEx.getCause().getMessage() : null; + causeMsg = causeMsg != null ? causeMsg : ""; + if ((ioem != null && ioem.contains(TOO_MANY_OPEN_FILES)) || causeMsg.contains(TOO_MANY_OPEN_FILES)) { + LOG.warn("Could not read file {} for volume check.", testFile.getAbsolutePath(), ioEx); + return true; + } logError(storageDir, String.format("Could not read file %s " + "for volume check.", testFile.getAbsolutePath()), ioEx); return false; + } catch (Throwable t) { + logError(storageDir, String.format("Could not read file %s " + + "for volume check.", testFile.getAbsolutePath()), t); } // Check that test file has the expected content. @@ -201,7 +220,7 @@ private void logError(File storageDir, String message) { LOG.error("Volume {} failed health check. {}", storageDir, message); } - private void logError(File storageDir, String message, Exception ex) { + private void logError(File storageDir, String message, Throwable ex) { LOG.error("Volume {} failed health check. {}", storageDir, message, ex); } } diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/utils/TestDiskCheckUtil.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/utils/TestDiskCheckUtil.java index d66f39455949..d4a3e671c5ae 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/utils/TestDiskCheckUtil.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/utils/TestDiskCheckUtil.java @@ -20,11 +20,23 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mockStatic; import java.io.File; +import java.nio.file.FileSystemException; +import java.nio.file.Files; +import java.nio.file.OpenOption; +import java.nio.file.Path; +import java.nio.file.Paths; +import org.apache.ratis.util.FileUtils; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.api.parallel.ExecutionMode; +import org.mockito.MockedStatic; /** * Tests {@link DiskCheckUtil} does not incorrectly identify an unhealthy @@ -32,6 +44,7 @@ * Tests that it identifies an improperly configured directory mount point. * */ +@Execution(ExecutionMode.SAME_THREAD) public class TestDiskCheckUtil { @TempDir @@ -80,4 +93,38 @@ public void testReadWrite() { assertNotNull(children); assertEquals(0, children.length); } + + @Test + public void testWriteFailureDueToTooManyOpenFiles() { + try (MockedStatic mockService = mockStatic(FileUtils.class)) { + + mockService.when(() -> FileUtils.newOutputStreamForceAtClose(any(File.class), any(OpenOption[].class))) + .thenThrow(new ExceptionInInitializerError("java.io.IOException: Too many open files")); + + String path = "/Volumes/DiskFullTest/disk-check-c967569c"; + assertThrows(ExceptionInInitializerError.class, + () -> FileUtils.newOutputStreamForceAtClose(new File(path), new OpenOption[2])); + + // Test that checkReadWrite returns true for the too many open file case + boolean result = DiskCheckUtil.checkReadWrite(testDir, testDir, 1024); + assertTrue(result, "checkReadWrite should return true when too many open files"); + } + } + + @Test + public void testReadFailureDueToTooManyOpenFiles() { + try (MockedStatic mockService = mockStatic(Files.class)) { + + String path = "/Volumes/DiskFullTest/disk-check-c967569c"; + mockService.when(() -> Files.newInputStream(any(Path.class), any(OpenOption[].class))) + .thenThrow(new FileSystemException(path + ": Too many open files")); + + assertThrows(FileSystemException.class, + () -> Files.newInputStream(Paths.get(path), new OpenOption[2])); + + // Test that checkReadWrite returns true for the too many open file case + boolean result = DiskCheckUtil.checkReadWrite(testDir, testDir, 1024); + assertTrue(result, "checkReadWrite should return true when too many open files"); + } + } }