From 84310e96db2de0eea21d474a4a75f5bb141ac3b0 Mon Sep 17 00:00:00 2001 From: Hiro Tamada Date: Tue, 20 Jan 2026 15:05:36 -0500 Subject: [PATCH 1/2] fix(caddy): isolate routes by listen port to prevent wildcard conflicts Previously, all ingress routes were placed in a single Caddy server with multiple listen addresses. This caused conflicts when multiple wildcard ingresses matched the same hostname pattern on different ports (e.g., *.host.kernel.sh:443 and *.host.kernel.sh:3000), because Caddy evaluated routes without considering the request's listening port. This change creates separate Caddy servers for each unique listen port, ensuring that requests on port 3000 only see routes meant for port 3000, and requests on port 443 only see routes meant for port 443. Example before (broken): servers: ingress: listen: [":443", ":3000"] routes: - match: *.host.kernel.sh -> port 80 # Always wins - match: *.host.kernel.sh -> port 3000 # Never reached Example after (fixed): servers: ingress-443: listen: [":443"] routes: - match: *.host.kernel.sh -> port 80 ingress-3000: listen: [":3000"] routes: - match: *.host.kernel.sh -> port 3000 This enables apps to expose multiple ports (e.g., 3000, 8080) via wildcard ingresses without conflicts. --- lib/ingress/config.go | 124 ++++++++++++++++---------------- lib/ingress/config_test.go | 141 +++++++++++++++++++++++++++++++++---- 2 files changed, 186 insertions(+), 79 deletions(-) diff --git a/lib/ingress/config.go b/lib/ingress/config.go index db02ec8..bd5788d 100644 --- a/lib/ingress/config.go +++ b/lib/ingress/config.go @@ -7,7 +7,6 @@ import ( "log/slog" "os" "path/filepath" - "slices" "sort" "strings" @@ -195,19 +194,21 @@ func (g *CaddyConfigGenerator) GenerateConfig(ctx context.Context, ingresses []I } // buildConfig builds the complete Caddy configuration. +// Routes are grouped by listen port to prevent conflicts when multiple wildcard +// ingresses match the same hostname pattern on different ports. func (g *CaddyConfigGenerator) buildConfig(ctx context.Context, ingresses []Ingress) map[string]interface{} { log := logger.FromContext(ctx) - // Build routes from ingresses - routes := []interface{}{} - redirectRoutes := []interface{}{} + // Group routes by listen port to isolate them in separate Caddy servers. + // This prevents conflicts when multiple wildcard ingresses match the same + // hostname pattern on different ports (e.g., *.host.kernel.sh:443 and *.host.kernel.sh:3000). + routesByPort := map[int][]interface{}{} tlsHostnames := []string{} - listenPorts := map[int]bool{} + tlsPortsByHostname := map[string][]int{} // Track which ports need TLS for each hostname for _, ingress := range ingresses { for _, rule := range ingress.Rules { port := rule.Match.GetPort() - listenPorts[port] = true // Determine hostname pattern (wildcard or literal) and instance expression var hostnameMatch string @@ -256,23 +257,22 @@ func (g *CaddyConfigGenerator) buildConfig(ctx context.Context, ingresses []Ingr "host": []string{hostnameMatch}, }, }, - "handle": []interface{}{reverseProxy}, + "handle": []interface{}{reverseProxy}, + "terminal": true, } - // Add terminal to stop processing after this route matches - route["terminal"] = true - - routes = append(routes, route) + // Add route to port-specific group + routesByPort[port] = append(routesByPort[port], route) // Track TLS hostnames for automation policy // For patterns, use the wildcard for TLS (e.g., "*.example.com") if rule.TLS { tlsHostnames = append(tlsHostnames, hostnameMatch) + tlsPortsByHostname[hostnameMatch] = append(tlsPortsByHostname[hostnameMatch], port) // Add HTTP redirect route if requested - // Uses protocol matcher to only redirect HTTP, not HTTPS (which would cause redirect loop) + // These go to port 80 server if rule.RedirectHTTP { - listenPorts[80] = true redirectRoute := map[string]interface{}{ "match": []interface{}{ map[string]interface{}{ @@ -291,7 +291,7 @@ func (g *CaddyConfigGenerator) buildConfig(ctx context.Context, ingresses []Ingr }, "terminal": true, } - redirectRoutes = append(redirectRoutes, redirectRoute) + routesByPort[80] = append(routesByPort[80], redirectRoute) } } } @@ -299,8 +299,6 @@ func (g *CaddyConfigGenerator) buildConfig(ctx context.Context, ingresses []Ingr // Add API ingress route if configured // This routes requests to the API hostname directly to localhost (Hypeman API) - // IMPORTANT: API route must be prepended to routes so it takes precedence over - // wildcard patterns that might otherwise match the API hostname if g.apiIngress.IsEnabled() { log.InfoContext(ctx, "adding API ingress route", "hostname", g.apiIngress.Hostname, "port", g.apiIngress.Port) @@ -321,18 +319,16 @@ func (g *CaddyConfigGenerator) buildConfig(ctx context.Context, ingresses []Ingr "handle": []interface{}{apiReverseProxy}, "terminal": true, } - // Prepend API route so it takes precedence over wildcards - routes = append([]interface{}{apiRoute}, routes...) - // Add TLS configuration for API hostname + // Determine which port the API route goes to + apiListenPort := 80 if g.apiIngress.TLS { - listenPorts[443] = true + apiListenPort = 443 tlsHostnames = append(tlsHostnames, g.apiIngress.Hostname) + tlsPortsByHostname[g.apiIngress.Hostname] = append(tlsPortsByHostname[g.apiIngress.Hostname], 443) // Add HTTP to HTTPS redirect for API hostname - // Prepend so it takes precedence over wildcard redirects if g.apiIngress.RedirectHTTP { - listenPorts[80] = true apiRedirectRoute := map[string]interface{}{ "match": []interface{}{ map[string]interface{}{ @@ -351,22 +347,12 @@ func (g *CaddyConfigGenerator) buildConfig(ctx context.Context, ingresses []Ingr }, "terminal": true, } - redirectRoutes = append([]interface{}{apiRedirectRoute}, redirectRoutes...) + // Prepend so API redirect takes precedence + routesByPort[80] = append([]interface{}{apiRedirectRoute}, routesByPort[80]...) } - } else { - listenPorts[80] = true } - } - - // Build listen addresses (sorted for deterministic config output) - ports := make([]int, 0, len(listenPorts)) - for port := range listenPorts { - ports = append(ports, port) - } - sort.Ints(ports) - listenAddrs := make([]string, 0, len(ports)) - for _, port := range ports { - listenAddrs = append(listenAddrs, fmt.Sprintf("%s:%d", g.listenAddress, port)) + // Prepend API route so it takes precedence over wildcards + routesByPort[apiListenPort] = append([]interface{}{apiRoute}, routesByPort[apiListenPort]...) } // Build base config (admin API only) @@ -377,19 +363,20 @@ func (g *CaddyConfigGenerator) buildConfig(ctx context.Context, ingresses []Ingr }, } - // Only add HTTP server if we have listen addresses (i.e., ingresses exist) - if len(listenAddrs) > 0 { - // Build server configuration - server := map[string]interface{}{ - "listen": listenAddrs, - } + // Build servers map - one server per unique listen port + // This isolates routes by port, preventing conflicts when multiple wildcard + // ingresses match the same hostname pattern on different ports. + if len(routesByPort) > 0 { + servers := map[string]interface{}{} - // Combine redirect routes (for HTTP) and main routes - // Use slices.Concat to avoid modifying original slices - allRoutes := slices.Concat(redirectRoutes, routes) + // Get sorted list of ports for deterministic output + ports := make([]int, 0, len(routesByPort)) + for port := range routesByPort { + ports = append(ports, port) + } + sort.Ints(ports) - // Add catch-all route at the end to return 404 for unmatched hostnames - // This must be last since routes are evaluated in order + // Catch-all route returns 404 for unmatched hostnames catchAllRoute := map[string]interface{}{ "handle": []interface{}{ map[string]interface{}{ @@ -402,31 +389,40 @@ func (g *CaddyConfigGenerator) buildConfig(ctx context.Context, ingresses []Ingr }, }, } - allRoutes = append(allRoutes, catchAllRoute) - server["routes"] = allRoutes + for _, port := range ports { + routes := routesByPort[port] - // Configure automatic HTTPS settings - if len(tlsHostnames) > 0 { - // When we have TLS hostnames, disable only redirects - we handle them explicitly - server["automatic_https"] = map[string]interface{}{ - "disable_redirects": true, + // Add catch-all at the end of each server's routes + allRoutes := append(routes, catchAllRoute) + + server := map[string]interface{}{ + "listen": []string{fmt.Sprintf("%s:%d", g.listenAddress, port)}, + "routes": allRoutes, + "logs": map[string]interface{}{}, // Disable access logs } - } else { - // No TLS hostnames - disable automatic HTTPS completely - server["automatic_https"] = map[string]interface{}{ - "disable": true, + + // Configure automatic HTTPS settings based on port + if port == 80 { + // Port 80 is HTTP only - disable automatic HTTPS completely + server["automatic_https"] = map[string]interface{}{ + "disable": true, + } + } else { + // Other ports may have TLS - disable only redirects (we handle them explicitly) + server["automatic_https"] = map[string]interface{}{ + "disable_redirects": true, + } } - } - // Disable access logs (per-request logs) - we only want system logs - server["logs"] = map[string]interface{}{} + // Use descriptive server names + serverName := fmt.Sprintf("ingress-%d", port) + servers[serverName] = server + } config["apps"] = map[string]interface{}{ "http": map[string]interface{}{ - "servers": map[string]interface{}{ - "ingress": server, - }, + "servers": servers, }, } } diff --git a/lib/ingress/config_test.go b/lib/ingress/config_test.go index 67ce282..07273fd 100644 --- a/lib/ingress/config_test.go +++ b/lib/ingress/config_test.go @@ -295,7 +295,7 @@ func TestGenerateConfig_DeterministicOrder(t *testing.T) { } } - // Also verify the listen addresses are in sorted order (80, 443, 9000) + // Verify we have separate servers for each port (port-isolated architecture) var config map[string]interface{} err := json.Unmarshal(firstOutput, &config) require.NoError(t, err) @@ -303,13 +303,25 @@ func TestGenerateConfig_DeterministicOrder(t *testing.T) { apps := config["apps"].(map[string]interface{}) httpApp := apps["http"].(map[string]interface{}) servers := httpApp["servers"].(map[string]interface{}) - ingressServer := servers["ingress"].(map[string]interface{}) - listenAddrs := ingressServer["listen"].([]interface{}) - require.Len(t, listenAddrs, 3) - assert.Equal(t, "0.0.0.0:80", listenAddrs[0].(string)) - assert.Equal(t, "0.0.0.0:443", listenAddrs[1].(string)) - assert.Equal(t, "0.0.0.0:9000", listenAddrs[2].(string)) + // Should have 3 separate servers, one per port + require.Len(t, servers, 3) + + // Verify each server exists and has correct listen address + server80 := servers["ingress-80"].(map[string]interface{}) + listen80 := server80["listen"].([]interface{}) + require.Len(t, listen80, 1) + assert.Equal(t, "0.0.0.0:80", listen80[0].(string)) + + server443 := servers["ingress-443"].(map[string]interface{}) + listen443 := server443["listen"].([]interface{}) + require.Len(t, listen443, 1) + assert.Equal(t, "0.0.0.0:443", listen443[0].(string)) + + server9000 := servers["ingress-9000"].(map[string]interface{}) + listen9000 := server9000["listen"].([]interface{}) + require.Len(t, listen9000, 1) + assert.Equal(t, "0.0.0.0:9000", listen9000[0].(string)) } func TestGenerateConfig_DefaultPort(t *testing.T) { @@ -731,10 +743,11 @@ func TestGenerateConfig_MixedTLSAndNonTLS(t *testing.T) { // Verify HTTP redirect is present (for TLS rule with redirect_http) assert.Contains(t, configStr, "301") - // Verify automatic_https has disable_redirects (not fully disabled) - // because we have TLS hostnames + // With port-isolated servers: + // - Port 443 server has "disable_redirects": true (TLS enabled, manual redirects) + // - Port 80 server has "disable": true (HTTP only, no TLS) assert.Contains(t, configStr, `"disable_redirects"`) - assert.NotContains(t, configStr, `"disable": true`) + assert.Contains(t, configStr, `"disable": true`) // Port 80 server disables HTTPS completely } func TestHasTLSRules(t *testing.T) { @@ -891,11 +904,109 @@ func TestGenerateConfig_TLSHostnameDeduplication(t *testing.T) { assert.Len(t, subjects, 1, "TLS subjects should be deduplicated") assert.Equal(t, "*.example.com", subjects[0].(string)) - // Verify all three ports are in listen addresses - configStr := string(data) - assert.Contains(t, configStr, ":443") - assert.Contains(t, configStr, ":3000") - assert.Contains(t, configStr, ":8080") + // Verify all three ports have separate servers (port isolation) + httpApp := apps["http"].(map[string]interface{}) + servers := httpApp["servers"].(map[string]interface{}) + assert.Contains(t, servers, "ingress-443") + assert.Contains(t, servers, "ingress-3000") + assert.Contains(t, servers, "ingress-8080") +} + +func TestGenerateConfig_PortIsolation(t *testing.T) { + // Test that wildcard ingresses on different ports don't conflict + // by verifying each port gets its own isolated server with routes + tmpDir, err := os.MkdirTemp("", "ingress-config-port-isolation-test-*") + require.NoError(t, err) + defer os.RemoveAll(tmpDir) + + p := paths.New(tmpDir) + require.NoError(t, os.MkdirAll(p.CaddyDir(), 0755)) + require.NoError(t, os.MkdirAll(p.CaddyDataDir(), 0755)) + + // Create generator with ACME configured + acmeConfig := ACMEConfig{ + Email: "admin@example.com", + DNSProvider: DNSProviderCloudflare, + CloudflareAPIToken: "test-token", + AllowedDomains: "*.example.com", + } + generator := NewCaddyConfigGenerator(p, "0.0.0.0", "127.0.0.1", 2019, acmeConfig, APIIngressConfig{}, 5353) + + ctx := context.Background() + // Create wildcard ingresses on different ports that would conflict + // if they were in the same server (same hostname pattern, different target ports) + ingresses := []Ingress{ + { + ID: "ing-legacy", + Name: "wildcard-legacy", + Rules: []IngressRule{ + { + // Old style: HTTPS on 443 -> backend on 80 + Match: IngressMatch{Hostname: "{instance}.example.com", Port: 443}, + Target: IngressTarget{Instance: "{instance}", Port: 80}, + TLS: true, + }, + }, + }, + { + ID: "ing-app", + Name: "wildcard-app", + Rules: []IngressRule{ + { + // New style: HTTPS on 3000 -> backend on 3000 + Match: IngressMatch{Hostname: "{instance}.example.com", Port: 3000}, + Target: IngressTarget{Instance: "{instance}", Port: 3000}, + TLS: true, + }, + }, + }, + } + + data, err := generator.GenerateConfig(ctx, ingresses) + require.NoError(t, err) + + // Parse the config + var config map[string]interface{} + err = json.Unmarshal(data, &config) + require.NoError(t, err) + + apps := config["apps"].(map[string]interface{}) + httpApp := apps["http"].(map[string]interface{}) + servers := httpApp["servers"].(map[string]interface{}) + + // Should have 2 separate servers (one per port) + assert.Len(t, servers, 2) + + // Verify server for port 443 exists and routes to port 80 + server443 := servers["ingress-443"].(map[string]interface{}) + routes443 := server443["routes"].([]interface{}) + assert.GreaterOrEqual(t, len(routes443), 1) + + // First route should be the wildcard route to port 80 + route443 := routes443[0].(map[string]interface{}) + handle443 := route443["handle"].([]interface{})[0].(map[string]interface{}) + dynamicUpstreams443 := handle443["dynamic_upstreams"].(map[string]interface{}) + assert.Equal(t, "80", dynamicUpstreams443["port"]) + + // Verify server for port 3000 exists and routes to port 3000 + server3000 := servers["ingress-3000"].(map[string]interface{}) + routes3000 := server3000["routes"].([]interface{}) + assert.GreaterOrEqual(t, len(routes3000), 1) + + // First route should be the wildcard route to port 3000 + route3000 := routes3000[0].(map[string]interface{}) + handle3000 := route3000["handle"].([]interface{})[0].(map[string]interface{}) + dynamicUpstreams3000 := handle3000["dynamic_upstreams"].(map[string]interface{}) + assert.Equal(t, "3000", dynamicUpstreams3000["port"]) + + // Verify each server only listens on its own port + listen443 := server443["listen"].([]interface{}) + assert.Len(t, listen443, 1) + assert.Equal(t, "0.0.0.0:443", listen443[0]) + + listen3000 := server3000["listen"].([]interface{}) + assert.Len(t, listen3000, 1) + assert.Equal(t, "0.0.0.0:3000", listen3000[0]) } func TestDeduplicateStrings(t *testing.T) { From 76fd373740b985fc866bdd409d68fbc4bb7c8af9 Mon Sep 17 00:00:00 2001 From: Hiro Tamada Date: Tue, 20 Jan 2026 15:24:11 -0500 Subject: [PATCH 2/2] fix: disable automatic HTTPS for ports without TLS routes For non-TLS ingresses on non-standard ports, Caddy needs automatic HTTPS completely disabled, not just redirects disabled. Previously the code only disabled HTTPS completely for port 80, but any port without TLS routes should have it disabled. This fixes the integration tests that create non-TLS ingresses on random high ports. --- lib/ingress/config.go | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/lib/ingress/config.go b/lib/ingress/config.go index bd5788d..f7cb2d3 100644 --- a/lib/ingress/config.go +++ b/lib/ingress/config.go @@ -205,6 +205,7 @@ func (g *CaddyConfigGenerator) buildConfig(ctx context.Context, ingresses []Ingr routesByPort := map[int][]interface{}{} tlsHostnames := []string{} tlsPortsByHostname := map[string][]int{} // Track which ports need TLS for each hostname + tlsEnabledPorts := map[int]bool{} // Track which ports have at least one TLS route for _, ingress := range ingresses { for _, rule := range ingress.Rules { @@ -269,6 +270,7 @@ func (g *CaddyConfigGenerator) buildConfig(ctx context.Context, ingresses []Ingr if rule.TLS { tlsHostnames = append(tlsHostnames, hostnameMatch) tlsPortsByHostname[hostnameMatch] = append(tlsPortsByHostname[hostnameMatch], port) + tlsEnabledPorts[port] = true // Add HTTP redirect route if requested // These go to port 80 server @@ -326,6 +328,7 @@ func (g *CaddyConfigGenerator) buildConfig(ctx context.Context, ingresses []Ingr apiListenPort = 443 tlsHostnames = append(tlsHostnames, g.apiIngress.Hostname) tlsPortsByHostname[g.apiIngress.Hostname] = append(tlsPortsByHostname[g.apiIngress.Hostname], 443) + tlsEnabledPorts[443] = true // Add HTTP to HTTPS redirect for API hostname if g.apiIngress.RedirectHTTP { @@ -402,16 +405,16 @@ func (g *CaddyConfigGenerator) buildConfig(ctx context.Context, ingresses []Ingr "logs": map[string]interface{}{}, // Disable access logs } - // Configure automatic HTTPS settings based on port - if port == 80 { - // Port 80 is HTTP only - disable automatic HTTPS completely + // Configure automatic HTTPS settings based on whether this port has TLS routes + if tlsEnabledPorts[port] { + // This port has TLS routes - disable only automatic redirects (we handle them explicitly) server["automatic_https"] = map[string]interface{}{ - "disable": true, + "disable_redirects": true, } } else { - // Other ports may have TLS - disable only redirects (we handle them explicitly) + // No TLS routes on this port - disable automatic HTTPS completely server["automatic_https"] = map[string]interface{}{ - "disable_redirects": true, + "disable": true, } }