diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/control/HeapMemoryMonitor.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/control/HeapMemoryMonitor.java index 93dcb30f9..93d362936 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/control/HeapMemoryMonitor.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/control/HeapMemoryMonitor.java @@ -21,7 +21,9 @@ import java.lang.management.MemoryPoolMXBean; import java.lang.management.MemoryType; import java.lang.management.MemoryUsage; +import java.text.ParseException; import java.text.SimpleDateFormat; +import java.util.Arrays; import java.util.List; import java.util.Set; import java.util.concurrent.Executors; @@ -93,9 +95,6 @@ public final class HeapMemoryMonitor implements NotificationListener, private static final int POLLER_INTERVAL = Integer.getInteger(POLLER_INTERVAL_PROP, 500); - // Duration in millis to wait for jmap -histo to finish - private static final int JMAP_HISTO_SLEEP_DURATION = 3 * 1000; - // This holds a new event as it transitions from updateStateAndSendEvent(...) to fillInProfile() private ThreadLocal upcomingEvent = new ThreadLocal(); @@ -693,17 +692,34 @@ public void jmapDump(String pid) { SimpleDateFormat dateFormat = new SimpleDateFormat("yyyyMMdd_HHmmss"); String dateSuffix = dateFormat.format(new java.util.Date( System.currentTimeMillis())); + Integer waitTime; + if (inputArgs.contains("-XX:+HeapDumpOnOutOfMemoryError")) { + Integer heapDumpTimeout = 30; + try { + Object[] filtArgs = inputArgs.stream() + .filter(e -> e.contains("snappydata.onCriticalHeapDumpTimeoutSeconds=")) + .map(e -> e.split("=")[1]).toArray(); + if (filtArgs.length > 0) { + heapDumpTimeout = Integer.parseInt(filtArgs[0].toString()); + } + } catch (NumberFormatException e) { + logger.warn("Failed to parse user provided value for " + + "OnCriticalHeapDumpTimeoutSeconds. default value of 30 will be used"); + } + jmapCommand = new String[] { "/bin/sh", "-c", "jmap -dump:format=b,file=" + "java_pid" + pid + "-" + dateSuffix + ".hprof " + pid }; + waitTime = heapDumpTimeout * 1000; } else { jmapCommand = new String[] { "/bin/sh", "-c", "jmap -histo " + pid + " > " + "java_pid" + pid + "-" + dateSuffix + ".jmap" }; + waitTime = 3 * 1000; } Process jmapProcess = Runtime.getRuntime().exec(jmapCommand); - jmapProcess.waitFor(JMAP_HISTO_SLEEP_DURATION, TimeUnit.MILLISECONDS); + jmapProcess.waitFor(waitTime, TimeUnit.MILLISECONDS); } catch (Exception e) { logger.error("Failed to log heap histogram for pid: " + pid, e); } diff --git a/gemfirexd/core/lib/libgemfirexd.so b/gemfirexd/core/lib/libgemfirexd.so index f938619ff..4a8e34f7a 100644 Binary files a/gemfirexd/core/lib/libgemfirexd.so and b/gemfirexd/core/lib/libgemfirexd.so differ diff --git a/gemfirexd/core/lib/libgemfirexd64.so b/gemfirexd/core/lib/libgemfirexd64.so index 4dce332e6..890dd956c 100644 Binary files a/gemfirexd/core/lib/libgemfirexd64.so and b/gemfirexd/core/lib/libgemfirexd64.so differ diff --git a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/jvmkill.c b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/jvmkill.c index 53096e82c..89dd7509d 100644 --- a/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/jvmkill.c +++ b/gemfirexd/core/src/main/java/com/pivotal/gemfirexd/internal/engine/jvmkill.c @@ -45,7 +45,23 @@ resourceExhausted( jint flags, const void *reserved, const char *description) { - logMessage("ResourceExhausted: %s: killing current process!", description); + int sleepSeconds = 0; + if (getenv("JVMKILL_SLEEP_SECONDS")) { + const char* s = getenv("JVMKILL_SLEEP_SECONDS"); + char *ptr; + sleepSeconds = strtol(s, &ptr, 10); + } + if (sleepSeconds < 1) { + sleepSeconds = 30; + } + + logMessage("ResourceExhausted: %s: terminating current process!\n", description); + kill(getpid(), SIGTERM); + + logMessage("SIGKILL will be issued after %d seconds if pid doesn't exit\n", sleepSeconds); + sleep(sleepSeconds); + + logMessage("ResourceExhausted: %s: killing current process!\n", description); kill(getpid(), SIGKILL); } @@ -78,4 +94,4 @@ Agent_OnLoad(JavaVM *vm, char *options, void *reserved) { } return JNI_OK; -} +} \ No newline at end of file