HDDS-14895. Improve DiskCheckUtil.checkReadWrite to tolerate not enough file handler

[ChenSammi]

What changes were proposed in this pull request?

If DiskCheckUtil.checkReadWrite failed due to no open file handlers during file write, the Error type exception is not caught by checkReadWrite, so the volume will be marked as failure immediately.

This task aims to catch the exception, and if it's too many open files case, tolerate it, and don't return false result.

What is the link to the Apache JIRA

https://issues.apache.org/jira/browse/HDDS-14895

How was this patch tested?

new unit tests

real Files.newInputStream exception

  java.nio.file.FileSystemException: /Volumes/DiskFullTest/disk-check-c967569c-a7a4-4924-b6c0-63e16d71eda0: Too many open files
  at java.base/sun.nio.fs.UnixException.translateToIOException(UnixException.java:100)
  at java.base/sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:111)
  at java.base/sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:116)
  at java.base/sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:219)
  at java.base/java.nio.file.Files.newByteChannel(Files.java:371)
  at java.base/java.nio.file.Files.newByteChannel(Files.java:422)
  at java.base/java.nio.file.spi.FileSystemProvider.newInputStream(FileSystemProvider.java:420)
  at java.base/java.nio.file.Files.newInputStream(Files.java:156)

real FileUtils.newOutputStreamForceAtClose exception

java.lang.ExceptionInInitializerError
  at java.base/sun.nio.ch.FileChannelImpl.<init>(FileChannelImpl.java:123)
  at java.base/sun.nio.ch.FileChannelImpl.open(FileChannelImpl.java:145)
  at java.base/sun.nio.fs.UnixChannelFactory.newFileChannel(UnixChannelFactory.java:144)
  at java.base/sun.nio.fs.UnixChannelFactory.newFileChannel(UnixChannelFactory.java:156)
  at java.base/sun.nio.fs.UnixFileSystemProvider.newFileChannel(UnixFileSystemProvider.java:180)
  at java.base/java.nio.channels.FileChannel.open(FileChannel.java:292)
  at java.base/java.nio.channels.FileChannel.open(FileChannel.java:345)
  at org.apache.ratis.util.FileUtils.lambda$newFileChannel$6(FileUtils.java:178)
  at org.apache.ratis.util.LogUtils.supplyAndLog(LogUtils.java:58)
  at org.apache.ratis.util.FileUtils.newFileChannel(FileUtils.java:177)
  at org.apache.ratis.util.FileUtils.newOutputStreamForceAtClose(FileUtils.java:165)
  at org.apache.ratis.util.FileUtils.newOutputStreamForceAtClose(FileUtils.java:169)
  at org.apache.hadoop.hdds.scm.TestHddsServerUtils.testTooManyOpenFiles(TestHddsServerUtils.java:411)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
  at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
  at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
  at java.base/java.lang.reflect.Method.invoke(Method.java:566)
  at org.junit.platform.commons.util.ReflectionUtils.invokeMethod(ReflectionUtils.java:786)
  at org.junit.platform.commons.support.ReflectionSupport.invokeMethod(ReflectionSupport.java:514)
  at org.junit.jupiter.engine.execution.MethodInvocation.proceed(MethodInvocation.java:60)
  at org.junit.jupiter.engine.execution.InvocationInterceptorChain$ValidatingInvocation.proceed(InvocationInterceptorChain.java:131)
  at org.junit.jupiter.engine.extension.TimeoutExtension.intercept(TimeoutExtension.java:161)
  at org.junit.jupiter.engine.extension.TimeoutExtension.interceptTestableMethod(TimeoutExtension.java:152)
  at org.junit.jupiter.engine.extension.TimeoutExtension.interceptTestMethod(TimeoutExtension.java:91)
  at org.junit.jupiter.engine.execution.InterceptingExecutableInvoker$ReflectiveInterceptorCall.lambda$ofVoidMethod$0(InterceptingExecutableInvoker.java:112)
  at org.junit.jupiter.engine.execution.InterceptingExecutableInvoker.lambda$invoke$0(InterceptingExecutableInvoker.java:94)
  at org.junit.jupiter.engine.execution.InvocationInterceptorChain$InterceptedInvocation.proceed(InvocationInterceptorChain.java:106)
  at org.junit.jupiter.engine.execution.InvocationInterceptorChain.proceed(InvocationInterceptorChain.java:64)
  at org.junit.jupiter.engine.execution.InvocationInterceptorChain.chainAndInvoke(InvocationInterceptorChain.java:45)
  at org.junit.jupiter.engine.execution.InvocationInterceptorChain.invoke(InvocationInterceptorChain.java:37)
  at org.junit.jupiter.engine.execution.InterceptingExecutableInvoker.invoke(InterceptingExecutableInvoker.java:93)
  at org.junit.jupiter.engine.execution.InterceptingExecutableInvoker.invoke(InterceptingExecutableInvoker.java:87)
  at org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.lambda$invokeTestMethod$4(TestMethodTestDescriptor.java:221)
  at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
  at org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.invokeTestMethod(TestMethodTestDescriptor.java:217)
  at org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.execute(TestMethodTestDescriptor.java:159)
  at org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.execute(TestMethodTestDescriptor.java:70)
  at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$6(NodeTestTask.java:157)
  at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
  at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:147)
  at org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137)
  at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$9(NodeTestTask.java:145)
  at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
  at org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:144)
  at org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:101)
  at java.base/java.util.ArrayList.forEach(ArrayList.java:1541)
  at org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.invokeAll(SameThreadHierarchicalTestExecutorService.java:41)
  at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$6(NodeTestTask.java:161)
  at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
  at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:147)
  at org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137)
  at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$9(NodeTestTask.java:145)
  at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
  at org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:144)
  at org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:101)
  at java.base/java.util.ArrayList.forEach(ArrayList.java:1541)
  at org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.invokeAll(SameThreadHierarchicalTestExecutorService.java:41)
  at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$6(NodeTestTask.java:161)
  at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
  at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:147)
  at org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137)
  at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$9(NodeTestTask.java:145)
  at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
  at org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:144)
  at org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:101)
  at org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.submit(SameThreadHierarchicalTestExecutorService.java:35)
  at org.junit.platform.engine.support.hierarchical.HierarchicalTestExecutor.execute(HierarchicalTestExecutor.java:57)
  at org.junit.platform.engine.support.hierarchical.HierarchicalTestEngine.execute(HierarchicalTestEngine.java:54)
  at org.junit.platform.launcher.core.EngineExecutionOrchestrator.executeEngine(EngineExecutionOrchestrator.java:230)
  at org.junit.platform.launcher.core.EngineExecutionOrchestrator.failOrExecuteEngine(EngineExecutionOrchestrator.java:204)
  at org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:172)
  at org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:101)
  at org.junit.platform.launcher.core.EngineExecutionOrchestrator.lambda$execute$0(EngineExecutionOrchestrator.java:64)
  at org.junit.platform.launcher.core.EngineExecutionOrchestrator.withInterceptedStreams(EngineExecutionOrchestrator.java:150)
  at org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:63)
  at org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:109)
  at org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:91)
  at org.junit.platform.launcher.core.DelegatingLauncher.execute(DelegatingLauncher.java:47)
  at org.junit.platform.launcher.core.InterceptingLauncher.lambda$execute$1(InterceptingLauncher.java:39)
  at org.junit.platform.launcher.core.ClasspathAlignmentCheckingLauncherInterceptor.intercept(ClasspathAlignmentCheckingLauncherInterceptor.java:25)
  at org.junit.platform.launcher.core.InterceptingLauncher.execute(InterceptingLauncher.java:38)
  at org.junit.platform.launcher.core.DelegatingLauncher.execute(DelegatingLauncher.java:47)
  at org.junit.platform.launcher.core.SessionPerRequestLauncher.execute(SessionPerRequestLauncher.java:66)
  at com.intellij.junit5.JUnit5IdeaTestRunner.startRunnerWithArgs(JUnit5IdeaTestRunner.java:66)
  at com.intellij.rt.junit.IdeaTestRunner$Repeater$1.execute(IdeaTestRunner.java:38)
  at com.intellij.rt.execution.junit.TestsRepeater.repeat(TestsRepeater.java:11)
  at com.intellij.rt.junit.IdeaTestRunner$Repeater.startRunnerWithArgs(IdeaTestRunner.java:35)
  at com.intellij.rt.junit.JUnitStarter.prepareStreamsAndStart(JUnitStarter.java:237)
  at com.intellij.rt.junit.JUnitStarter.main(JUnitStarter.java:58)
Caused by: java.io.IOException: Too many open files
  at java.base/sun.nio.ch.FileDispatcherImpl.init(Native Method)
  at java.base/sun.nio.ch.FileDispatcherImpl.<clinit>(FileDispatcherImpl.java:38)