fix: Added error messages instead of recurring while hashing.

src/main/java/io/deephaven/hash/KHash.java

@@ -210,6 +210,22 @@ protected final void postInsertHook(boolean usedFreeSlot) {
       // if we've exhausted the free spots, rehash to the same capacity,
       // which will free up any stale removed slots for reuse.
       int newCapacity = _size > _maxSize ? PrimeFinder.nextPrime(capacity() << 1) : capacity();
+
+      // Before rehashing, make sure we have not reduced the capacity below the current size.
+      if (newCapacity < capacity()) {
+        throw new IllegalStateException(
+            "Internal error: newCapacity < capacity, newCapacity="
+                + newCapacity
+                + ", capacity="
+                + capacity()
+                + ", _free="
+                + _free
+                + ", _size="
+                + _size
+                + ", _maxSize="
+                + _maxSize);
+      }
+
       rehash(newCapacity);
       computeMaxSize(capacity());
     }

src/main/java/io/deephaven/hash/KeyedObjectHash.java

@@ -192,6 +192,29 @@ protected void rehash(int newCapacity) {
     h.storage = (V[]) new Object[newCapacity];
     for (V v : storage) {
       if (v != null && v != DELETED) {
+        // Before adding the next value, make sure we won't recurse by verifying h._free > 2 before
+        // the add
+        if (h._free <= 2) {
+          throw new IllegalStateException(
+              "Internal error: h._free <= 2, newCapacity="
+                  + newCapacity
+                  + ", capacity="
+                  + capacity()
+                  + ", _free="
+                  + _free
+                  + ", _size="
+                  + _size
+                  + ", _maxSize="
+                  + _maxSize
+                  + ", h.capacity="
+                  + h.capacity()
+                  + ", h._free="
+                  + h._free
+                  + ", h._size="
+                  + h._size
+                  + ", h._maxSize="
+                  + h._maxSize);
+        }
         h.internalPut(v, NORMAL, null);
       }
     }