From b3c837f66ea64f52af0678cdd6e23bd18c0a26b2 Mon Sep 17 00:00:00 2001 From: Rolando Santamaria Maso Date: Sun, 24 Aug 2025 23:45:17 +0200 Subject: [PATCH 1/6] feat: Enhance load balancing strategies with power-of-two-choices, latency, and weighted least connections; update README for new features --- README.md | 5 +- src/interfaces/load-balancer.ts | 9 ++ src/load-balancer/http-load-balancer.ts | 109 ++++++++++++++++++++++-- test/cluster/fixtures/worker.ts | 2 +- test/e2e/random-loadbalancer.test.ts | 20 +++-- 5 files changed, 132 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 3cae06d..aa10a0f 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ - **🔥 Blazing Fast**: Built on Bun - up to 4x faster than Node.js alternatives - **🎯 Zero Config**: Works out of the box with sensible defaults -- **🧠 Smart Load Balancing**: 5 load balancing algorithms: `round-robin`, `least-connections`, `random`, `weighted`, `ip-hash` +- **🧠 Smart Load Balancing**: Multiple algorithms: `round-robin`, `least-connections`, `random`, `weighted`, `ip-hash`, `p2c` (power-of-two-choices), `latency`, `weighted-least-connections` - **🛡️ Production Ready**: Circuit breakers, health checks, and auto-failover - **🔐 Built-in Authentication**: JWT, API keys, JWKS, and OAuth2 support out of the box - **🎨 Developer Friendly**: Full TypeScript support with intuitive APIs @@ -107,6 +107,9 @@ console.log('🚀 Bungate running on http://localhost:3000') - **Least Connections**: Route to the least busy server - **IP Hash**: Consistent routing based on client IP for session affinity - **Random**: Randomized distribution for even load +- **Power of Two Choices (p2c)**: Pick the better of two random targets by load/latency +- **Latency**: Prefer the target with the lowest average response time +- **Weighted Least Connections**: Prefer targets with fewer connections normalized by weight - **Sticky Sessions**: Session affinity with cookie-based persistence ### 🛡️ **Reliability & Resilience** diff --git a/src/interfaces/load-balancer.ts b/src/interfaces/load-balancer.ts index 6beece2..50d9650 100644 --- a/src/interfaces/load-balancer.ts +++ b/src/interfaces/load-balancer.ts @@ -69,6 +69,10 @@ export interface LoadBalancerConfig { | 'random' // Randomly selects a target | 'weighted' // Uses target weights for distribution | 'ip-hash' // Routes based on client IP hash for session affinity + | 'p2c' // Power of two choices: pick best of two random targets + | 'power-of-two-choices' // Alias for p2c + | 'latency' // Chooses target with the lowest avg response time + | 'weighted-least-connections' // Least connections normalized by weight /** * List of backend targets to load balance across @@ -113,6 +117,11 @@ export interface LoadBalancerConfig { * @example 'OK' or 'healthy' */ expectedBody?: string + /** + * HTTP method to use for health checks + * @default 'GET' + */ + method?: 'GET' | 'HEAD' } /** diff --git a/src/load-balancer/http-load-balancer.ts b/src/load-balancer/http-load-balancer.ts index e4902fd..c05b68e 100644 --- a/src/load-balancer/http-load-balancer.ts +++ b/src/load-balancer/http-load-balancer.ts @@ -151,6 +151,18 @@ export class HttpLoadBalancer implements LoadBalancer { return null } + // Fast path: only one healthy target + if (healthyTargets.length === 1) { + const only = healthyTargets[0]! + this.recordRequest(only.url) + this.logger.logLoadBalancing(this.config.strategy, only.url, { + reason: 'single-healthy', + duration: Date.now() - startTime, + healthyTargets: 1, + }) + return only + } + // Check for sticky session first if (this.config.stickySession?.enabled) { const stickyTarget = this.getStickyTarget(request) @@ -184,6 +196,16 @@ export class HttpLoadBalancer implements LoadBalancer { case 'ip-hash': selectedTarget = this.selectIpHash(request, healthyTargets) break + case 'p2c': + case 'power-of-two-choices': + selectedTarget = this.selectPowerOfTwoChoices(healthyTargets) + break + case 'latency': + selectedTarget = this.selectByLatency(healthyTargets) + break + case 'weighted-least-connections': + selectedTarget = this.selectWeightedLeastConnections(healthyTargets) + break default: selectedTarget = this.selectRoundRobin(healthyTargets) } @@ -378,7 +400,8 @@ export class HttpLoadBalancer implements LoadBalancer { const target = this.targets.get(url) if (target) { target.totalResponseTime += responseTime - target.averageResponseTime = target.totalResponseTime / target.requests + const reqs = target.requests || 1 // avoid division by zero before first selectTarget + target.averageResponseTime = target.totalResponseTime / reqs if (isError) { target.errors++ @@ -478,6 +501,69 @@ export class HttpLoadBalancer implements LoadBalancer { return targets[index]! // Guaranteed to exist due to length check } + private selectPowerOfTwoChoices( + targets: LoadBalancerTarget[], + ): LoadBalancerTarget { + // Sample two distinct targets at random and choose the better one by connections then latency + const i = Math.floor(Math.random() * targets.length) + let j = Math.floor(Math.random() * targets.length) + if (j === i) j = (j + 1) % targets.length + const a = targets[i]! + const b = targets[j]! + return this.betterByLoadThenLatency(a, b) + } + + private selectByLatency(targets: LoadBalancerTarget[]): LoadBalancerTarget { + // Choose the target with the smallest averageResponseTime; fallback to round-robin if missing data + let best = targets[0]! + let bestLatency = best.averageResponseTime ?? Number.POSITIVE_INFINITY + for (let k = 1; k < targets.length; k++) { + const t = targets[k]! + const lat = t.averageResponseTime ?? Number.POSITIVE_INFINITY + if (lat < bestLatency) { + best = t + bestLatency = lat + } + } + if (!isFinite(bestLatency)) { + // No latency data available yet + return this.selectRoundRobin(targets) + } + return best + } + + private selectWeightedLeastConnections( + targets: LoadBalancerTarget[], + ): LoadBalancerTarget { + // Choose by minimal (connections + 1) / weight + return targets.reduce((best, curr) => { + const bestScore = + (Math.max(0, best.connections ?? 0) + 1) / Math.max(1, best.weight ?? 1) + const currScore = + (Math.max(0, curr.connections ?? 0) + 1) / Math.max(1, curr.weight ?? 1) + if (currScore < bestScore) return curr + if (currScore === bestScore) return this.betterByLatency(best, curr) + return best + }) + } + + private betterByLatency(a: LoadBalancerTarget, b: LoadBalancerTarget) { + const la = a.averageResponseTime ?? Number.POSITIVE_INFINITY + const lb = b.averageResponseTime ?? Number.POSITIVE_INFINITY + return la <= lb ? a : b + } + + private betterByLoadThenLatency( + a: LoadBalancerTarget, + b: LoadBalancerTarget, + ) { + const ca = a.connections ?? 0 + const cb = b.connections ?? 0 + if (ca < cb) return a + if (cb < ca) return b + return this.betterByLatency(a, b) + } + private recordRequest(url: string): void { this.totalRequests++ const target = this.targets.get(url) @@ -552,10 +638,21 @@ export class HttpLoadBalancer implements LoadBalancer { } private getClientId(request: Request): string { - // In real scenario, would extract actual client IP - // For now, use a combination of headers as identifier - const userAgent = request.headers.get('user-agent') ?? '' - const accept = request.headers.get('accept') ?? '' + // Prefer real client IP headers commonly set by proxies/CDNs; fallback to UA+Accept + const headers = request.headers + const xff = headers.get('x-forwarded-for') || headers.get('X-Forwarded-For') + if (xff) { + const ip = xff.split(',')[0]!.trim() + if (ip) return ip + } + const realIp = + headers.get('x-real-ip') || + headers.get('cf-connecting-ip') || + headers.get('x-client-ip') || + '' + if (realIp) return realIp + const userAgent = headers.get('user-agent') ?? '' + const accept = headers.get('accept') ?? '' return userAgent + accept } @@ -596,7 +693,7 @@ export class HttpLoadBalancer implements LoadBalancer { const response = await fetch(url.toString(), { signal: controller.signal, - method: 'GET', + method: healthCheckConfig.method ?? 'GET', }) clearTimeout(timeoutId) diff --git a/test/cluster/fixtures/worker.ts b/test/cluster/fixtures/worker.ts index dc13b6b..8657037 100644 --- a/test/cluster/fixtures/worker.ts +++ b/test/cluster/fixtures/worker.ts @@ -3,5 +3,5 @@ process.on('SIGTERM', () => process.exit(0)) // Keep alive for a very long time (about 1 billion ms ~ 11.5 days) -const KEEP_ALIVE_INTERVAL = 1 << 30; +const KEEP_ALIVE_INTERVAL = 1 << 30 setInterval(() => {}, KEEP_ALIVE_INTERVAL) diff --git a/test/e2e/random-loadbalancer.test.ts b/test/e2e/random-loadbalancer.test.ts index ba61508..485206f 100644 --- a/test/e2e/random-loadbalancer.test.ts +++ b/test/e2e/random-loadbalancer.test.ts @@ -271,11 +271,17 @@ describe('Random Load Balancer E2E Tests', () => { // Each server should get roughly 1/3 of requests (allow generous variance for randomness) const expectedPerServer = requestCount / 3 expect(serverCounts['echo-1'] || 0).toBeGreaterThan(expectedPerServer * 0.3) // At least 30% of expected - expect(serverCounts['echo-1'] || 0).toBeLessThan(expectedPerServer * 1.7) // At most 170% of expected + expect(serverCounts['echo-1'] || 0).toBeLessThanOrEqual( + expectedPerServer * 1.7, + ) // At most 170% of expected expect(serverCounts['echo-2'] || 0).toBeGreaterThan(expectedPerServer * 0.3) - expect(serverCounts['echo-2'] || 0).toBeLessThan(expectedPerServer * 1.7) + expect(serverCounts['echo-2'] || 0).toBeLessThanOrEqual( + expectedPerServer * 1.7, + ) expect(serverCounts['echo-3'] || 0).toBeGreaterThan(expectedPerServer * 0.3) - expect(serverCounts['echo-3'] || 0).toBeLessThan(expectedPerServer * 1.7) + expect(serverCounts['echo-3'] || 0).toBeLessThanOrEqual( + expectedPerServer * 1.7, + ) }) test('should randomly distribute requests between two servers', async () => { @@ -309,9 +315,13 @@ describe('Random Load Balancer E2E Tests', () => { // Each server should get roughly half of requests (allow variance for randomness) const expectedPerServer = requestCount / 2 expect(serverCounts['echo-1'] || 0).toBeGreaterThan(expectedPerServer * 0.3) // At least 30% of expected - expect(serverCounts['echo-1'] || 0).toBeLessThan(expectedPerServer * 1.7) // At most 170% of expected + expect(serverCounts['echo-1'] || 0).toBeLessThanOrEqual( + expectedPerServer * 1.7, + ) // At most 170% of expected expect(serverCounts['echo-2'] || 0).toBeGreaterThan(expectedPerServer * 0.3) - expect(serverCounts['echo-2'] || 0).toBeLessThan(expectedPerServer * 1.7) + expect(serverCounts['echo-2'] || 0).toBeLessThanOrEqual( + expectedPerServer * 1.7, + ) }) test('should show randomness across multiple test runs', async () => { From d95cbce50ab125d6af0d05860d8493917d105942 Mon Sep 17 00:00:00 2001 From: Rolando Santamaria Maso <4096860+jkyberneees@users.noreply.github.com> Date: Sun, 24 Aug 2025 23:46:53 +0200 Subject: [PATCH 2/6] Update src/load-balancer/http-load-balancer.ts Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/load-balancer/http-load-balancer.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/load-balancer/http-load-balancer.ts b/src/load-balancer/http-load-balancer.ts index c05b68e..f971425 100644 --- a/src/load-balancer/http-load-balancer.ts +++ b/src/load-balancer/http-load-balancer.ts @@ -400,8 +400,11 @@ export class HttpLoadBalancer implements LoadBalancer { const target = this.targets.get(url) if (target) { target.totalResponseTime += responseTime - const reqs = target.requests || 1 // avoid division by zero before first selectTarget - target.averageResponseTime = target.totalResponseTime / reqs + if (target.requests > 0) { + target.averageResponseTime = target.totalResponseTime / target.requests + } else { + target.averageResponseTime = 0 + } if (isError) { target.errors++ From 3435df63d0c1735c53d353a50488bf00b8363ddf Mon Sep 17 00:00:00 2001 From: Rolando Santamaria Maso Date: Mon, 25 Aug 2025 00:00:39 +0200 Subject: [PATCH 3/6] test: Increase request iterations in load balancer tests to reduce randomness flakiness in CI --- test/e2e/hooks.test.ts | 35 ++++++++++++++---------- test/load-balancer/load-balancer.test.ts | 6 ++-- 2 files changed, 24 insertions(+), 17 deletions(-) diff --git a/test/e2e/hooks.test.ts b/test/e2e/hooks.test.ts index 5a89012..345cb2b 100644 --- a/test/e2e/hooks.test.ts +++ b/test/e2e/hooks.test.ts @@ -192,7 +192,7 @@ describe('Hooks E2E Tests', () => { const afterCall = afterResponseCalls[afterResponseCalls.length - 1] expect(afterCall?.req.url).toContain('/api/hooks/hello') expect(afterCall?.res.status).toBe(200) - }) + }, 20000) test('should trigger circuit breaker hooks on successful request', async () => { const initialBeforeCount = beforeCircuitBreakerCalls.length @@ -218,7 +218,7 @@ describe('Hooks E2E Tests', () => { expect(afterCall?.req.url).toContain('/api/error/health') expect(afterCall?.result.state).toBe('CLOSED') expect(afterCall?.result.success).toBe(true) - }) + }, 20000) test('should trigger onError hook on server error', async () => { const initialErrorCount = onErrorCalls.length @@ -237,7 +237,7 @@ describe('Hooks E2E Tests', () => { const errorCall = onErrorCalls[onErrorCalls.length - 1] expect(errorCall?.req.url).toContain('/api/error/error') expect(errorCall?.error.message).toContain('Server error') - }) + }, 20000) test('should trigger onError hook on timeout', async () => { const initialErrorCount = onErrorCalls.length @@ -256,7 +256,7 @@ describe('Hooks E2E Tests', () => { const errorCall = onErrorCalls[onErrorCalls.length - 1] expect(errorCall?.req.url).toContain('/api/error/timeout') expect(errorCall?.error.message).toContain('timeout') - }) + }, 20000) test('should trigger circuit breaker hooks with failure state', async () => { // Wait for circuit breaker to potentially reset @@ -281,7 +281,7 @@ describe('Hooks E2E Tests', () => { afterCircuitBreakerCalls[afterCircuitBreakerCalls.length - 1] expect(afterCall?.req.url).toContain('/api/error/error') expect(afterCall?.result.success).toBe(false) - }) + }, 20000) test('should trigger all hooks in correct order for successful request', async () => { // Reset counters @@ -323,7 +323,7 @@ describe('Hooks E2E Tests', () => { // Response should be successful expect(afterResponse?.res.status).toBe(200) - }) + }, 20000) test('should trigger all hooks in correct order for failed request', async () => { // Wait for circuit breaker to potentially reset @@ -369,7 +369,7 @@ describe('Hooks E2E Tests', () => { expect(onError?.error.message).toMatch( /Server error|Circuit breaker is OPEN/, ) - }) + }, 20000) test('should pass correct proxy configuration to beforeRequest hook', async () => { const initialCount = beforeRequestCalls.length @@ -385,7 +385,7 @@ describe('Hooks E2E Tests', () => { expect(beforeCall?.opts).toBeDefined() expect(beforeCall?.opts.pathRewrite).toBeDefined() expect(beforeCall?.opts.pathRewrite['^/api/hooks']).toBe('') - }) + }, 20000) test('should provide response body to afterResponse hook', async () => { const initialCount = afterResponseCalls.length @@ -401,7 +401,7 @@ describe('Hooks E2E Tests', () => { expect(afterCall?.res.status).toBe(200) expect(afterCall?.res.headers.get('content-type')).toContain('text/plain') expect(afterCall?.body).toBeDefined() - }) + }, 20000) test('should use fallback response from onError hook when returned', async () => { // Create a new gateway with onError hook that returns a fallback response @@ -444,6 +444,8 @@ describe('Hooks E2E Tests', () => { fallbackGateway.addRoute(fallbackRouteConfig) const fallbackServer = await fallbackGateway.listen(fallbackGatewayPort) + // Allow the server a brief moment to be fully ready in slower CI environments + await new Promise((resolve) => setTimeout(resolve, 150)) try { const response = await fetch( @@ -464,7 +466,7 @@ describe('Hooks E2E Tests', () => { } finally { fallbackServer.stop() } - }) + }, 20000) test('should use fallback response from onError hook on timeout', async () => { // Create a new gateway with onError hook that handles timeouts @@ -516,6 +518,8 @@ describe('Hooks E2E Tests', () => { timeoutGateway.addRoute(timeoutRouteConfig) const timeoutServer = await timeoutGateway.listen(timeoutGatewayPort) + // Allow the server a brief moment to be fully ready in slower CI environments + await new Promise((resolve) => setTimeout(resolve, 150)) try { const response = await fetch( @@ -539,7 +543,7 @@ describe('Hooks E2E Tests', () => { } finally { timeoutServer.stop() } - }) + }, 20000) test('should fallback to default error handling when onError hook throws', async () => { // Create a new failing server for this test @@ -601,6 +605,8 @@ describe('Hooks E2E Tests', () => { selectiveGateway.addRoute(selectiveRouteConfig) const selectiveServer = await selectiveGateway.listen(selectiveGatewayPort) + // Allow the server a brief moment to be fully ready in slower CI environments + await new Promise((resolve) => setTimeout(resolve, 150)) try { // Test with timeout (should use fallback) @@ -622,7 +628,7 @@ describe('Hooks E2E Tests', () => { selectiveServer.stop() selectiveFailingServer.stop() } - }) + }, 20000) test('should handle async fallback response generation', async () => { // Create a new failing server for this test @@ -688,7 +694,8 @@ describe('Hooks E2E Tests', () => { asyncGateway.addRoute(asyncRouteConfig) const asyncServer = await asyncGateway.listen(asyncGatewayPort) - await new Promise((resolve) => setTimeout(resolve, 50)) + // Allow the server a brief moment to be fully ready in slower CI environments + await new Promise((resolve) => setTimeout(resolve, 250)) try { const testRequestId = `test-${Date.now()}` @@ -715,5 +722,5 @@ describe('Hooks E2E Tests', () => { asyncServer.stop() asyncFailingServer.stop() } - }) + }, 20000) }) diff --git a/test/load-balancer/load-balancer.test.ts b/test/load-balancer/load-balancer.test.ts index 4ff6c2c..bb5e429 100644 --- a/test/load-balancer/load-balancer.test.ts +++ b/test/load-balancer/load-balancer.test.ts @@ -1205,8 +1205,8 @@ describe('HttpLoadBalancer', () => { const request = createMockRequest() const selections: string[] = [] - // Make requests to test distribution - for (let i = 0; i < 30; i++) { + // Make many requests to reduce randomness flakiness in CI + for (let i = 0; i < 300; i++) { const target = loadBalancer.selectTarget(request) if (target) { selections.push(target.url) @@ -1223,7 +1223,7 @@ describe('HttpLoadBalancer', () => { // Should distribute according to weights (1:2 ratio) expect(noWeightCount).toBeGreaterThan(0) expect(weightedCount).toBeGreaterThan(noWeightCount) - }) + }, 20000) test('handles session cleanup interval management', () => { const config: LoadBalancerConfig = { From ccc2aece38d4ec063eaa74d3348fe3cf93a476ad Mon Sep 17 00:00:00 2001 From: Rolando Santamaria Maso Date: Mon, 25 Aug 2025 21:52:53 +0200 Subject: [PATCH 4/6] fix: update random latency delay in echo server from 0-500ms to 0-200ms --- examples/echo-server-1.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/echo-server-1.ts b/examples/echo-server-1.ts index ca952df..a2343f9 100644 --- a/examples/echo-server-1.ts +++ b/examples/echo-server-1.ts @@ -19,7 +19,7 @@ const server = serve({ // Echo endpoint - return request details - // Add random latency delay (0-500ms) + // Add random latency delay (0-200ms) const delay = Math.floor(Math.random() * 200) await new Promise((resolve) => setTimeout(resolve, delay)) From b049875070067cc8d84ed9278ca8f4f52944f9b5 Mon Sep 17 00:00:00 2001 From: Rolando Santamaria Maso Date: Mon, 25 Aug 2025 21:53:00 +0200 Subject: [PATCH 5/6] feat: Add new load balancing strategies including Power of Two Choices, Latency-based, and Weighted Least Connections; update demo routes and logging --- examples/lb-example-all-options.ts | 133 +++++++++++++++++++++++++++-- 1 file changed, 126 insertions(+), 7 deletions(-) diff --git a/examples/lb-example-all-options.ts b/examples/lb-example-all-options.ts index 1ef3fb3..5ef4d17 100644 --- a/examples/lb-example-all-options.ts +++ b/examples/lb-example-all-options.ts @@ -42,7 +42,7 @@ const gateway = new BunGateway({ }) // ============================================================================= -// 1. ROUND ROBIN LOAD BALANCER WITH HEALTH CHECKS +// ROUND ROBIN LOAD BALANCER WITH HEALTH CHECKS // ============================================================================= console.log('\n1️⃣ Round Robin with Health Checks') @@ -83,7 +83,7 @@ gateway.addRoute({ }) // ============================================================================= -// 2. WEIGHTED LOAD BALANCER FOR HIGH-PERFORMANCE SCENARIOS +// WEIGHTED LOAD BALANCER FOR HIGH-PERFORMANCE SCENARIOS // ============================================================================= console.log('2️⃣ Weighted Load Balancer (Performance Optimized)') @@ -151,7 +151,7 @@ gateway.addRoute({ }) // ============================================================================= -// 4. IP HASH FOR SESSION AFFINITY +// IP HASH FOR SESSION AFFINITY // ============================================================================= console.log('4️⃣ IP Hash for Session Affinity') @@ -190,7 +190,7 @@ gateway.addRoute({ }) // ============================================================================= -// 5. RANDOM STRATEGY WITH ADVANCED ERROR HANDLING +// RANDOM STRATEGY WITH ADVANCED ERROR HANDLING // ============================================================================= console.log('5️⃣ Random Strategy with Advanced Error Handling') @@ -296,7 +296,118 @@ gateway.addRoute({ }) // ============================================================================= -// 7. MONITORING AND METRICS ENDPOINT +// POWER OF TWO CHOICES (P2C) STRATEGY +// ============================================================================= +console.log('6️⃣➕ Power of Two Choices (P2C) Strategy') + +gateway.addRoute({ + pattern: '/api/p2c/*', + loadBalancer: { + strategy: 'p2c', + targets: [ + { url: 'http://localhost:8080' }, + { url: 'http://localhost:8081' }, + ], + healthCheck: { + enabled: true, + interval: 10000, + timeout: 4000, + path: '/', + expectedStatus: 200, + }, + }, + proxy: { + pathRewrite: (path) => path.replace('/api/p2c', ''), + }, + hooks: { + beforeRequest: async (req) => { + logger.debug(`🎲 P2C routing for: ${req.url}`) + }, + }, +}) + +// Alias for P2C +gateway.addRoute({ + pattern: '/api/power-of-two/*', + loadBalancer: { + strategy: 'power-of-two-choices', + targets: [ + { url: 'http://localhost:8080' }, + { url: 'http://localhost:8081' }, + ], + healthCheck: { + enabled: true, + interval: 10000, + timeout: 4000, + path: '/', + expectedStatus: 200, + }, + }, + proxy: { + pathRewrite: (path) => path.replace('/api/power-of-two', ''), + }, +}) + +// ============================================================================= +// LATENCY-BASED STRATEGY +// ============================================================================= +console.log('6️⃣✅ Latency-based Strategy') + +gateway.addRoute({ + pattern: '/api/latency/*', + loadBalancer: { + strategy: 'latency', + targets: [ + { url: 'http://localhost:8080' }, + { url: 'http://localhost:8081' }, + ], + healthCheck: { + enabled: true, + interval: 8000, + timeout: 3000, + path: '/', + expectedStatus: 200, + }, + }, + proxy: { + pathRewrite: (path) => path.replace('/api/latency', ''), + timeout: 10000, + }, + hooks: { + afterResponse: async (req, res) => { + logger.info(`⏱️ Latency strategy served with ${res.status}`) + }, + }, +}) + +// ============================================================================= +// WEIGHTED LEAST CONNECTIONS +// ============================================================================= +console.log('6️⃣🔢 Weighted Least Connections Strategy') + +gateway.addRoute({ + pattern: '/api/wlc/*', + loadBalancer: { + strategy: 'weighted-least-connections', + targets: [ + { url: 'http://localhost:8080', weight: 3 }, + { url: 'http://localhost:8081', weight: 1 }, + ], + healthCheck: { + enabled: true, + interval: 12000, + timeout: 4000, + path: '/', + expectedStatus: 200, + }, + }, + proxy: { + pathRewrite: (path) => path.replace('/api/wlc', ''), + }, +}) + +// ============================================================================= +// MONITORING AND METRICS ENDPOINT // ============================================================================= console.log('7️⃣ Monitoring and Metrics') @@ -327,7 +438,7 @@ gateway.addRoute({ }) // ============================================================================= -// 8. HEALTH CHECK ENDPOINT +// HEALTH CHECK ENDPOINT // ============================================================================= gateway.addRoute({ pattern: '/health', @@ -348,7 +459,7 @@ gateway.addRoute({ }) // ============================================================================= -// 9. DEMO ENDPOINTS FOR TESTING +// DEMO ENDPOINTS FOR TESTING // ============================================================================= gateway.addRoute({ pattern: '/demo', @@ -359,6 +470,10 @@ gateway.addRoute({ 'Least Connections': '/api/least-connections/get', 'IP Hash': '/api/ip-hash/get', Random: '/api/random/get', + 'Power of Two Choices (P2C)': '/api/p2c/get', + 'P2C Alias (power-of-two-choices)': '/api/power-of-two/get', + 'Latency-based': '/api/latency/get', + 'Weighted Least Connections': '/api/wlc/get', 'Users Service': '/api/users/1', 'Posts Service': '/api/posts/1', Metrics: '/metrics', @@ -430,6 +545,10 @@ try { console.log(' • Least Connections: /api/least-connections/*') console.log(' • IP Hash: /api/ip-hash/*') console.log(' • Random: /api/random/*') + console.log(' • Power of Two Choices (P2C): /api/p2c/*') + console.log(' • P2C Alias (power-of-two-choices): /api/power-of-two/*') + console.log(' • Latency-based: /api/latency/*') + console.log(' • Weighted Least Connections: /api/wlc/*') console.log(' • Users Service: /api/users/*') console.log(' • Posts Service: /api/posts/*') From e3c400b95d35f28fea8cd248f529b9f22e671876 Mon Sep 17 00:00:00 2001 From: Rolando Santamaria Maso Date: Mon, 25 Aug 2025 21:53:03 +0200 Subject: [PATCH 6/6] feat: Enhance load balancer latency metrics tracking and support for additional strategies --- src/gateway/gateway.ts | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/src/gateway/gateway.ts b/src/gateway/gateway.ts index a92ccad..11a4d14 100644 --- a/src/gateway/gateway.ts +++ b/src/gateway/gateway.ts @@ -329,6 +329,8 @@ export class BunGateway implements Gateway { } } + // Measure end-to-end time to update latency metrics in the load balancer + const startedAt = Date.now() increaseTargetConnectionsIfLeastConnections( route.loadBalancer?.strategy, target, @@ -347,12 +349,22 @@ export class BunGateway implements Gateway { route.loadBalancer?.strategy, target, ) + // Update latency stats for strategies like 'latency' and as tie-breakers + try { + const duration = Date.now() - startedAt + loadBalancer.recordResponse(target.url, duration, false) + } catch {} }, onError: (req: Request, error: Error) => { decreaseTargetConnectionsIfLeastConnections( route.loadBalancer?.strategy, target, ) + // Record error with latency to penalize target appropriately + try { + const duration = Date.now() - startedAt + loadBalancer.recordResponse(target.url, duration, true) + } catch {} if (route.hooks?.onError) { route.hooks.onError!(req, error) } @@ -490,7 +502,13 @@ function increaseTargetConnectionsIfLeastConnections( strategy: string | undefined, target: any, ): void { - if (strategy === 'least-connections' && target.connections !== undefined) { + if ( + (strategy === 'least-connections' || + strategy === 'weighted-least-connections' || + strategy === 'p2c' || + strategy === 'power-of-two-choices') && + target.connections !== undefined + ) { target.connections++ } } @@ -499,7 +517,13 @@ function decreaseTargetConnectionsIfLeastConnections( strategy: string | undefined, target: any, ): void { - if (strategy === 'least-connections' && target.connections !== undefined) { + if ( + (strategy === 'least-connections' || + strategy === 'weighted-least-connections' || + strategy === 'p2c' || + strategy === 'power-of-two-choices') && + target.connections !== undefined + ) { target.connections-- } }