Fix performance of fiber pool stack tests. (ruby#16570)

Author: samuel-williams-shopify

cont.c

@@ -42,7 +42,11 @@ extern int madvise(caddr_t, size_t, int);
 #include "id_table.h"
 #include "ractor_core.h"
 
-static const int DEBUG = 0;
+enum {
+    DEBUG = 0,
+    DEBUG_EXPAND = 0,
+    DEBUG_ACQUIRE = 0,
+};
 
 #define RB_PAGE_SIZE (pagesize)
 #define RB_PAGE_MASK (~(RB_PAGE_SIZE - 1))
@@ -62,11 +66,11 @@ static VALUE rb_cFiberPool;
 // Defined in `coroutine/$arch/Context.h`:
 #ifdef COROUTINE_LIMITED_ADDRESS_SPACE
 #define FIBER_POOL_ALLOCATION_FREE
-#define FIBER_POOL_INITIAL_SIZE 8
-#define FIBER_POOL_ALLOCATION_MAXIMUM_SIZE 32
+#define FIBER_POOL_MINIMUM_COUNT 8
+#define FIBER_POOL_MAXIMUM_ALLOCATIONS 32
 #else
-#define FIBER_POOL_INITIAL_SIZE 32
-#define FIBER_POOL_ALLOCATION_MAXIMUM_SIZE 1024
+#define FIBER_POOL_MINIMUM_COUNT 32
+#define FIBER_POOL_MAXIMUM_ALLOCATIONS 1024
 #endif
 #ifdef RB_EXPERIMENTAL_FIBER_POOL
 #define FIBER_POOL_ALLOCATION_FREE
@@ -189,7 +193,11 @@ struct fiber_pool {
     size_t count;
 
     // The initial number of stacks to allocate.
-    size_t initial_count;
+    size_t minimum_count;
+
+    // If positive, total stacks in this pool cannot exceed this (shared pool only:
+    // set via RUBY_SHARED_FIBER_POOL_MAXIMUM_COUNT). Expansion fails with errno EAGAIN.
+    size_t maximum_count;
 
     // Whether to madvise(free) the stack or not.
     // If this value is set to 1, the stack will be madvise(free)ed
@@ -470,7 +478,7 @@ fiber_pool_allocate_memory(size_t * count, size_t stride)
     // the system would allow (e.g. overcommit * physical memory + swap), we
     // divide count by two and try again. This condition should only be
     // encountered in edge cases, but we handle it here gracefully.
-    while (*count > 1) {
+    while (*count) {
 #if defined(_WIN32)
         void * base = VirtualAlloc(0, (*count)*stride, MEM_COMMIT, PAGE_READWRITE);
 
@@ -518,11 +526,28 @@ fiber_pool_allocate_memory(size_t * count, size_t stride)
 static struct fiber_pool_allocation *
 fiber_pool_expand(struct fiber_pool * fiber_pool, size_t count)
 {
+    if (count == 0) {
+        errno = EAGAIN;
+        return NULL;
+    }
+
     STACK_GROW_DIR_DETECTION;
 
     size_t size = fiber_pool->size;
     size_t stride = size + RB_PAGE_SIZE;
 
+    // If the maximum number of stacks is set, and we have reached it, return NULL.
+    if (fiber_pool->maximum_count > 0) {
+        if (fiber_pool->count >= fiber_pool->maximum_count) {
+            errno = EAGAIN;
+            return NULL;
+        }
+        size_t remaining = fiber_pool->maximum_count - fiber_pool->count;
+        if (count > remaining) {
+            count = remaining;
+        }
+    }
+
     // Allocate metadata before mmap: ruby_xmalloc (RB_ALLOC) raises on failure and
     // must not run after base is mapped, or the region would leak.
     struct fiber_pool_allocation * allocation = RB_ALLOC(struct fiber_pool_allocation);
@@ -548,7 +573,7 @@ fiber_pool_expand(struct fiber_pool * fiber_pool, size_t count)
 #endif
     allocation->pool = fiber_pool;
 
-    if (DEBUG) {
+    if (DEBUG_EXPAND) {
         fprintf(stderr, "fiber_pool_expand(%"PRIuSIZE"): %p, %"PRIuSIZE"/%"PRIuSIZE" x [%"PRIuSIZE":%"PRIuSIZE"]\n",
                 count, (void*)fiber_pool, fiber_pool->used, fiber_pool->count, size, fiber_pool->vm_stack_size);
     }
@@ -613,22 +638,24 @@ fiber_pool_expand(struct fiber_pool * fiber_pool, size_t count)
 // Initialize the specified fiber pool with the given number of stacks.
 // @param vm_stack_size The size of the vm stack to allocate.
 static void
-fiber_pool_initialize(struct fiber_pool * fiber_pool, size_t size, size_t count, size_t vm_stack_size)
+fiber_pool_initialize(struct fiber_pool * fiber_pool, size_t size, size_t minimum_count, size_t maximum_count, size_t vm_stack_size)
 {
     VM_ASSERT(vm_stack_size < size);
 
     fiber_pool->allocations = NULL;
     fiber_pool->vacancies = NULL;
     fiber_pool->size = ((size / RB_PAGE_SIZE) + 1) * RB_PAGE_SIZE;
     fiber_pool->count = 0;
-    fiber_pool->initial_count = count;
+    fiber_pool->minimum_count = minimum_count;
+    fiber_pool->maximum_count = maximum_count;
     fiber_pool->free_stacks = 1;
     fiber_pool->used = 0;
-
     fiber_pool->vm_stack_size = vm_stack_size;
 
-    if (RB_UNLIKELY(!fiber_pool_expand(fiber_pool, count))) {
-        rb_raise(rb_eFiberError, "can't allocate initial fiber stacks (%"PRIuSIZE" x %"PRIuSIZE" bytes): %s", count, fiber_pool->size, strerror(errno));
+    if (fiber_pool->minimum_count > 0) {
+        if (RB_UNLIKELY(!fiber_pool_expand(fiber_pool, fiber_pool->minimum_count))) {
+            rb_raise(rb_eFiberError, "can't allocate initial fiber stacks (%"PRIuSIZE" x %"PRIuSIZE" bytes): %s", fiber_pool->minimum_count, fiber_pool->size, strerror(errno));
+        }
     }
 }
 
@@ -678,15 +705,30 @@ fiber_pool_allocation_free(struct fiber_pool_allocation * allocation)
 #endif
 
 // Number of stacks to request when expanding the pool (clamped to min/max).
-static inline size_t
+static size_t
 fiber_pool_stack_expand_count(const struct fiber_pool *pool)
 {
-    const size_t maximum = FIBER_POOL_ALLOCATION_MAXIMUM_SIZE;
-    const size_t minimum = pool->initial_count;
+    const size_t maximum_allocations = FIBER_POOL_MAXIMUM_ALLOCATIONS;
+    const size_t minimum_count = FIBER_POOL_MINIMUM_COUNT;
 
+    // We are going try and double the number of stacks in the pool:
     size_t count = pool->count;
-    if (count > maximum) count = maximum;
-    if (count < minimum) count = minimum;
+    if (count > maximum_allocations) count = maximum_allocations;
+    if (count < minimum_count) count = minimum_count;
+
+    // If we have a maximum count, we need to clamp the number of stacks to the maximum:
+    if (pool->maximum_count > 0) {
+        if (pool->count >= pool->maximum_count) {
+            // No expansion is possible:
+            return 0;
+        }
+
+        // Otherwise, compute the number of stacks we can allocate to bring us to the maximum:
+        size_t remaining = pool->maximum_count - pool->count;
+        if (count > remaining) {
+            count = remaining;
+        }
+    }
 
     return count;
 }
@@ -698,15 +740,15 @@ fiber_pool_stack_acquire_expand(struct fiber_pool *fiber_pool)
 {
     size_t count = fiber_pool_stack_expand_count(fiber_pool);
 
-    if (DEBUG) fprintf(stderr, "fiber_pool_stack_acquire: expanding fiber pool by %"PRIuSIZE" stacks\n", count);
+    if (DEBUG_ACQUIRE) fprintf(stderr, "fiber_pool_stack_acquire: expanding fiber pool by %"PRIuSIZE" stacks\n", count);
 
     struct fiber_pool_vacancy *vacancy = NULL;
 
     if (RB_LIKELY(fiber_pool_expand(fiber_pool, count))) {
         return fiber_pool_vacancy_pop(fiber_pool);
     }
     else {
-        if (DEBUG) fprintf(stderr, "fiber_pool_stack_acquire: expand failed (%s), collecting garbage\n", strerror(errno));
+        if (DEBUG_ACQUIRE) fprintf(stderr, "fiber_pool_stack_acquire: expand failed (%s), collecting garbage\n", strerror(errno));
 
         rb_gc();
 
@@ -716,6 +758,9 @@ fiber_pool_stack_acquire_expand(struct fiber_pool *fiber_pool)
             return vacancy;
         }
 
+        // Recompute count as gc may have freed up some allocations:
+        count = fiber_pool_stack_expand_count(fiber_pool);
+
         // Try to expand the fiber pool again:
         if (RB_LIKELY(fiber_pool_expand(fiber_pool, count))) {
             return fiber_pool_vacancy_pop(fiber_pool);
@@ -3526,7 +3571,7 @@ rb_fiber_pool_initialize(int argc, VALUE* argv, VALUE self)
 
     TypedData_Get_Struct(self, struct fiber_pool, &FiberPoolDataType, fiber_pool);
 
-    fiber_pool_initialize(fiber_pool, NUM2SIZET(size), NUM2SIZET(count), NUM2SIZET(vm_stack_size));
+    fiber_pool_initialize(fiber_pool, NUM2SIZET(size), NUM2SIZET(count), 0, NUM2SIZET(vm_stack_size));
 
     return self;
 }
@@ -3545,6 +3590,46 @@ rb_fiber_pool_initialize(int argc, VALUE* argv, VALUE self)
  *     fiber.resume #=> FiberError: dead fiber called
  */
 
+static size_t
+shared_fiber_pool_minimum_count()
+{
+    size_t minimum_count = FIBER_POOL_MINIMUM_COUNT;
+
+    const char *minimum_count_env = getenv("RUBY_SHARED_FIBER_POOL_MINIMUM_COUNT");
+    if (minimum_count_env && minimum_count_env[0]) {
+        char *end;
+        unsigned long value = strtoul(minimum_count_env, &end, 10);
+        if (end != minimum_count_env && *end == '\0') {
+            minimum_count = (size_t)value;
+        }
+        else {
+            rb_warn("invalid RUBY_SHARED_FIBER_POOL_MINIMUM_COUNT=%s (expected a non-negative integer)", minimum_count_env);
+        }
+    }
+
+    return minimum_count;
+}
+
+static size_t
+shared_fiber_pool_maximum_count()
+{
+    size_t maximum_count = 0;
+
+    const char *maximum_count_env = getenv("RUBY_SHARED_FIBER_POOL_MAXIMUM_COUNT");
+    if (maximum_count_env && maximum_count_env[0]) {
+        char *end;
+        unsigned long value = strtoul(maximum_count_env, &end, 10);
+        if (end != maximum_count_env && *end == '\0') {
+            maximum_count = (size_t)value;
+        }
+        else {
+            rb_warn("invalid RUBY_SHARED_FIBER_POOL_MAXIMUM_COUNT=%s (expected a non-negative integer)", maximum_count_env);
+        }
+    }
+
+    return maximum_count;
+}
+
 void
 Init_Cont(void)
 {
@@ -3562,7 +3647,9 @@ Init_Cont(void)
 #endif
     SET_MACHINE_STACK_END(&th->ec->machine.stack_end);
 
-    fiber_pool_initialize(&shared_fiber_pool, stack_size, FIBER_POOL_INITIAL_SIZE, vm_stack_size);
+    size_t minimum_count = shared_fiber_pool_minimum_count();
+    size_t maximum_count = shared_fiber_pool_maximum_count();
+    fiber_pool_initialize(&shared_fiber_pool, stack_size, minimum_count, maximum_count, vm_stack_size);
 
     fiber_initialize_keywords[0] = rb_intern_const("blocking");
     fiber_initialize_keywords[1] = rb_intern_const("pool");

man/erb.1

@@ -1,5 +1,5 @@
 .\"Ruby is copyrighted by Yukihiro Matsumoto <matz@netlab.jp>.
-.Dd February 24, 2026
+.Dd March 27, 2026
 .Dt ERB 1 "Ruby Programmer's Reference Guide"
 .Os UNIX
 .Sh NAME

man/goruby.1

@@ -1,5 +1,5 @@
 .\"Ruby is copyrighted by Yukihiro Matsumoto <matz@netlab.jp>.
-.Dd February 24, 2026
+.Dd March 27, 2026
 .Dt GORUBY 1 "Ruby Programmer's Reference Guide"
 .Os UNIX
 .Sh NAME

man/ruby.1

@@ -1,5 +1,5 @@
 .\"Ruby is copyrighted by Yukihiro Matsumoto <matz@netlab.jp>.
-.Dd February 24, 2026
+.Dd March 27, 2026
 .Dt RUBY 1 "Ruby Programmer's Reference Guide"
 .Os UNIX
 .Sh NAME
@@ -700,6 +700,16 @@ Frees stacks of pooled fibers, if set to 1.
 Do not free the stacks if set to 0.
 Introduced in Ruby 2.7, default: 1 (no growth)
 .Pp
+.It Ev RUBY_SHARED_FIBER_POOL_MAXIMUM_COUNT
+If set to a non-negative integer, the shared fiber pool cannot allocate more
+than that many stacks; further fiber creation may fail with
+.Va FiberError .
+Unset or 0 means no explicit cap (subject to process limits).
+.Pp
+.It Ev RUBY_SHARED_FIBER_POOL_MINIMUM_COUNT
+Initial and minimum growth chunk size for the shared fiber pool (stacks).
+Unset uses the implementation default.
+.Pp
 .El
 .Sh STACK SIZE ENVIRONMENT
 Stack size environment variables are implementation-dependent and

test/ruby/test_fiber.rb

@@ -508,20 +508,43 @@ def test_machine_stack_gc
   end
 
   def test_fiber_pool_stack_acquire_failure
-    omit "cannot determine max_map_count" unless File.exist?("/proc/sys/vm/max_map_count")
-    # On these platforms, excessive memory usage can cause the test to fail unexpectedly.
-    omit "not supported on IBM platforms" if RUBY_PLATFORM =~ /s390x|powerpc/
-    omit "not supported with YJIT" if defined?(RubyVM::YJIT) && RubyVM::YJIT.enabled?
-    omit "not supported with ZJIT" if defined?(RubyVM::ZJIT) && RubyVM::ZJIT.enabled?
-
-    assert_separately([], <<~RUBY, timeout: 120)
-      max_map_count = File.read("/proc/sys/vm/max_map_count").to_i
+    environment = {
+      "RUBY_SHARED_FIBER_POOL_MINIMUM_COUNT" => "0",
+      "RUBY_SHARED_FIBER_POOL_MAXIMUM_COUNT" => "128"
+    }
+
+    # This program requires, effectively, at most one fiber stack, since the fiber immediately becomes unreachable.
+    assert_separately([environment], <<~RUBY, timeout: 30)
       GC.disable
+      count_before = GC.count
+
+      # Create more fibers than the pool can handle (but they become immediately unreachable):
       assert_nothing_raised do
-        (max_map_count + 10).times do
-          Fiber.new { Fiber.yield }.resume
+        256.times do
+          Fiber.new{Fiber.yield}.resume
+        end
+      end
+
+      # Major GC should have happened at least once:
+      assert_operator(GC.count, :>, count_before)
+    RUBY
+  end
+
+  def test_fiber_pool_stack_acquire_failure_at_maximum_count
+    environment = {
+      "RUBY_SHARED_FIBER_POOL_MAXIMUM_COUNT" => "128"
+    }
+
+    assert_separately([environment], <<~RUBY, timeout: 30)
+      GC.disable
+      fibers = []
+      assert_raise(FiberError) do
+        loop do
+          Fiber.new{fibers << Fiber.current; Fiber.yield}.resume
+          raise "expected FiberError before this" if fibers.size > 128
         end
       end
+      assert_operator fibers.size, :>=, 128
     RUBY
   end
 end