HDDS-14888. Improve DiskCheckUtil.checkReadWrite to tolerate disk full

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/DiskCheckUtil.java

@@ -42,6 +42,7 @@
  * where the disk is mounted.
  */
 public final class DiskCheckUtil {
+  public static final String LINUX_DISK_FULL_MESSAGE = "No space left on device";
   // For testing purposes, an alternate check implementation can be provided
   // to inject failures.
   private static DiskChecks impl = new DiskChecksImpl();
@@ -149,10 +150,15 @@ public boolean checkReadWrite(File storageDir,
             "volume check.", testFile.getAbsolutePath()), notFoundEx);
         return false;
       } catch (SyncFailedException syncEx) {
-        logError(storageDir, String.format("Could sync file %s to disk.",
+        logError(storageDir, String.format("Could not sync file %s to disk.",
             testFile.getAbsolutePath()), syncEx);
         return false;
       } catch (IOException ioEx) {
+        String msg = ioEx.getMessage();
+        if (msg != null && msg.contains(LINUX_DISK_FULL_MESSAGE)) {
+          LOG.warn("Could not write file {} for volume check", testFile.getAbsolutePath(), ioEx);
+          return true;
+        }
         logError(storageDir, String.format("Could not write file %s " +
             "for volume check.", testFile.getAbsolutePath()), ioEx);
         return false;

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/StorageVolume.java

@@ -713,14 +713,6 @@ public synchronized VolumeCheckResult check(@Nullable Boolean unused)
           " interrupted.");
     }
 
-    // As WRITE keeps happening there is probability, disk has become full during above check.
-    // We can check again if disk is full. If it is full,
-    // in this case keep volume as healthy so that READ can still be served
-    if (!diskChecksPassed && getCurrentUsage().getAvailable() < minimumDiskSpace) {
-      ioTestSlidingWindow.add(true);
-      return VolumeCheckResult.HEALTHY;
-    }
-
     // Move the sliding window of IO test results forward 1 by adding the
     // latest entry and removing the oldest entry from the window.
     // Update the failure counter for the new window.

hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/utils/TestDiskCheckUtil.java

@@ -20,18 +20,28 @@
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mockStatic;
 
 import java.io.File;
+import java.nio.file.FileSystemException;
+import java.nio.file.OpenOption;
+import org.apache.ratis.util.FileUtils;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.api.parallel.Execution;
+import org.junit.jupiter.api.parallel.ExecutionMode;
+import org.mockito.MockedStatic;
 
 /**
  * Tests {@link DiskCheckUtil} does not incorrectly identify an unhealthy
  * disk or mount point.
  * Tests that it identifies an improperly configured directory mount point.
  *
  */
+@Execution(ExecutionMode.SAME_THREAD)
 public class TestDiskCheckUtil {
 
   @TempDir
@@ -80,4 +90,21 @@ public void testReadWrite() {
     assertNotNull(children);
     assertEquals(0, children.length);
   }
+
+  @Test
+  public void testCheckReadWriteDiskFull() {
+    try (MockedStatic<FileUtils> mockService = mockStatic(FileUtils.class)) {
+      // fos.write(writtenBytes) also through FileSystemException with the message
+      mockService.when(() -> FileUtils.newOutputStreamForceAtClose(any(File.class), any(OpenOption[].class)))
+          .thenThrow(new FileSystemException("No space left on device"));
+
+      String path = testDir.getAbsolutePath();
+      assertThrows(FileSystemException.class,
+          () -> FileUtils.newOutputStreamForceAtClose(new File(path), new OpenOption[2]));
+
+      // Test that checkReadWrite returns true for the disk full case
+      boolean result = DiskCheckUtil.checkReadWrite(testDir, testDir, 1024);
+      assertTrue(result, "checkReadWrite should return true when disk is full");
+    }
+  }
 }