Fix BrokerPool shutdown hang causing CI timeouts

exist-core/src/main/java/org/exist/storage/BrokerPool.java

@@ -92,6 +92,7 @@
 
 import static com.evolvedbinary.j8fu.fsm.TransitionTable.transitionTable;
 import static org.exist.util.ThreadUtils.nameInstanceThreadGroup;
+import static org.exist.util.ThreadUtils.newInstanceDaemonThread;
 import static org.exist.util.ThreadUtils.newInstanceThread;
 
 /**
@@ -543,7 +544,7 @@ private void _initialize() throws EXistException, DatabaseConfigurationException
             statusReporter = new StatusReporter(SIGNAL_STARTUP);
             statusObservers.forEach(statusReporter::addObserver);
 
-            final Thread statusThread = newInstanceThread(this, "startup-status-reporter", statusReporter);
+            final Thread statusThread = newInstanceDaemonThread(this, "startup-status-reporter", statusReporter);
             statusThread.start();
 
             // statusReporter may have to be terminated or the thread can/will hang.
@@ -1225,12 +1226,16 @@ public DBBroker get(final Optional<Subject> subject) throws EXistException {
         //No active broker : get one ASAP
 
         while(serviceModeUser != null && subject.isPresent() && !subject.equals(Optional.ofNullable(serviceModeUser))) {
+            if(isShuttingDown()) {
+                break;
+            }
             try {
                 LOG.debug("Db instance is in service mode. Waiting for db to become available again ...");
-                wait();
+                wait(1000);
             } catch(final InterruptedException e) {
                 Thread.currentThread().interrupt();
-                LOG.error("Interrupt detected");
+                LOG.warn("Interrupted while waiting for service mode to end");
+                break;
             }
         }
 
@@ -1245,11 +1250,15 @@ public DBBroker get(final Optional<Subject> subject) throws EXistException {
                 } else
                     //... or wait until there is one available
                     while(inactiveBrokers.isEmpty()) {
+                        if(isShuttingDown()) {
+                            throw new EXistException("BrokerPool is shutting down");
+                        }
                         LOG.debug("waiting for a broker to become available");
                         try {
-                            this.wait();
+                            this.wait(1000);
                         } catch(final InterruptedException e) {
-                            //nothing to be done!
+                            Thread.currentThread().interrupt();
+                            throw new EXistException("Interrupted while waiting for a broker", e);
                         }
                     }
             }
@@ -1400,10 +1409,14 @@ public DBBroker enterServiceMode(final Subject user) throws PermissionDeniedExce
         synchronized(this) {
             if(!activeBrokers.isEmpty()) {
                 while(!inServiceMode) {
+                    if(isShuttingDown()) {
+                        break;
+                    }
                     try {
-                        wait();
+                        wait(1000);
                     } catch(final InterruptedException e) {
-                        //nothing to be done
+                        Thread.currentThread().interrupt();
+                        break;
                     }
                 }
             }
@@ -1610,7 +1623,7 @@ void shutdown(final boolean killed, final Consumer<String> shutdownInstanceConsu
                 statusObservers.forEach(statusReporter::addObserver);
 
                 synchronized (this) {
-                    final Thread statusThread = newInstanceThread(this, "shutdown-status-reporter", statusReporter);
+                    final Thread statusThread = newInstanceDaemonThread(this, "shutdown-status-reporter", statusReporter);
                     statusThread.start();
 
                     // DW: only in debug mode
@@ -1637,7 +1650,9 @@ void shutdown(final boolean killed, final Consumer<String> shutdownInstanceConsu
                                 //Wait until they become inactive...
                                 this.wait(1000);
                             } catch (final InterruptedException e) {
-                                //nothing to be done
+                                Thread.currentThread().interrupt();
+                                LOG.warn("Interrupted while waiting for active brokers to return during shutdown");
+                                break;
                             }
 
                             //...or force the shutdown
@@ -1745,7 +1760,13 @@ void shutdown(final boolean killed, final Consumer<String> shutdownInstanceConsu
                 statusReporter.terminate();
                 statusReporter = null;
 
-//                instanceThreadGroup.destroy();
+                // interrupt any remaining threads in the instance thread group
+                // to prevent the JVM from hanging on non-daemon threads
+                try {
+                    instanceThreadGroup.interrupt();
+                } catch (final SecurityException e) {
+                    LOG.warn("Could not interrupt instance thread group: {}", e.getMessage());
+                }
             }
         } finally {
             status.process(Event.FINISHED_SHUTDOWN);

exist-core/src/main/java/org/exist/util/ThreadUtils.java

@@ -58,6 +58,12 @@ public static Thread newInstanceThread(final ThreadGroup threadGroup, final Stri
         return new Thread(threadGroup, runnable, nameInstanceThread(instanceId, threadName));
     }
 
+    public static Thread newInstanceDaemonThread(final Database database, final String threadName, final Runnable runnable) {
+        final Thread thread = new Thread(database.getThreadGroup(), runnable, nameInstanceThread(database, threadName));
+        thread.setDaemon(true);
+        return thread;
+    }
+
     public static String nameGlobalThread(final String threadName) {
         return "global." + threadName;
     }