From 42ff11cd502ba285ceb3b5286fd5cd04c57d08be Mon Sep 17 00:00:00 2001 From: Zach Smith Date: Thu, 29 Jan 2026 22:04:36 -0800 Subject: [PATCH 01/24] feat: connector EnvoyPatchPolicy for HTTPProxy backends --- api/v1alpha/httpproxy_types.go | 7 + api/v1alpha1/connectorclass_types.go | 1 + cmd/main.go | 3 +- ...orking.datumapis.com_connectorclasses.yaml | 2 +- internal/controller/httpproxy_controller.go | 596 +++++++++++++++++- .../controller/httpproxy_controller_test.go | 325 +++++++++- internal/validation/httpproxy_validation.go | 7 +- .../validation/httpproxy_validation_test.go | 77 +++ 8 files changed, 989 insertions(+), 29 deletions(-) diff --git a/api/v1alpha/httpproxy_types.go b/api/v1alpha/httpproxy_types.go index 0fd2ed42..efbc9b62 100644 --- a/api/v1alpha/httpproxy_types.go +++ b/api/v1alpha/httpproxy_types.go @@ -190,6 +190,10 @@ const ( // This condition is present and true when a hostname defined in an HTTPProxy // is in use by another resource. HTTPProxyConditionHostnamesInUse = "HostnamesInUse" + + // This condition is true when connector tunnel metadata has been programmed + // via the downstream EnvoyPatchPolicy. + HTTPProxyConditionTunnelMetadataProgrammed = "TunnelMetadataProgrammed" ) const ( @@ -200,6 +204,9 @@ const ( // HTTPProxyReasonProgrammed indicates that the HTTP proxy has been programmed. HTTPProxyReasonProgrammed = "Programmed" + // HTTPProxyReasonTunnelMetadataApplied indicates tunnel metadata has been applied. + HTTPProxyReasonTunnelMetadataApplied = "TunnelMetadataApplied" + // HTTPProxyReasonConflict indicates that the HTTP proxy encountered a conflict // when being programmed. HTTPProxyReasonConflict = "Conflict" diff --git a/api/v1alpha1/connectorclass_types.go b/api/v1alpha1/connectorclass_types.go index 29b33cec..bf0a72c1 100644 --- a/api/v1alpha1/connectorclass_types.go +++ b/api/v1alpha1/connectorclass_types.go @@ -23,6 +23,7 @@ type ConnectorClassStatus struct { // +kubebuilder:object:root=true // +kubebuilder:subresource:status +// +kubebuilder:resource:scope=Cluster // ConnectorClass is the Schema for the connectorclasses API. type ConnectorClass struct { diff --git a/cmd/main.go b/cmd/main.go index 9edeaf0e..ea6ade85 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -218,7 +218,8 @@ func main() { } if err := (&controller.HTTPProxyReconciler{ - Config: serverConfig, + Config: serverConfig, + DownstreamCluster: downstreamCluster, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "HTTPProxy") os.Exit(1) diff --git a/config/crd/bases/networking.datumapis.com_connectorclasses.yaml b/config/crd/bases/networking.datumapis.com_connectorclasses.yaml index c507a5bb..ffffb3c6 100644 --- a/config/crd/bases/networking.datumapis.com_connectorclasses.yaml +++ b/config/crd/bases/networking.datumapis.com_connectorclasses.yaml @@ -12,7 +12,7 @@ spec: listKind: ConnectorClassList plural: connectorclasses singular: connectorclass - scope: Namespaced + scope: Cluster versions: - name: v1alpha1 schema: diff --git a/internal/controller/httpproxy_controller.go b/internal/controller/httpproxy_controller.go index 386fd478..5c2ced00 100644 --- a/internal/controller/httpproxy_controller.go +++ b/internal/controller/httpproxy_controller.go @@ -4,6 +4,7 @@ package controller import ( "context" + "encoding/json" "errors" "fmt" "net" @@ -12,8 +13,10 @@ import ( "strconv" "strings" + envoygatewayv1alpha1 "github.com/envoyproxy/gateway/api/v1alpha1" v1 "k8s.io/api/core/v1" discoveryv1 "k8s.io/api/discovery/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/api/equality" apierrors "k8s.io/apimachinery/pkg/api/errors" apimeta "k8s.io/apimachinery/pkg/api/meta" @@ -21,15 +24,21 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/cluster" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/log" gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" mcbuilder "sigs.k8s.io/multicluster-runtime/pkg/builder" mcmanager "sigs.k8s.io/multicluster-runtime/pkg/manager" mcreconcile "sigs.k8s.io/multicluster-runtime/pkg/reconcile" + mcsource "sigs.k8s.io/multicluster-runtime/pkg/source" networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" + networkingv1alpha1 "go.datum.net/network-services-operator/api/v1alpha1" "go.datum.net/network-services-operator/internal/config" + downstreamclient "go.datum.net/network-services-operator/internal/downstreamclient" conditionutil "go.datum.net/network-services-operator/internal/util/condition" gatewayutil "go.datum.net/network-services-operator/internal/util/gateway" ) @@ -38,6 +47,8 @@ import ( type HTTPProxyReconciler struct { mgr mcmanager.Manager Config config.NetworkServicesOperator + + DownstreamCluster cluster.Cluster } type desiredHTTPProxyResources struct { @@ -46,6 +57,8 @@ type desiredHTTPProxyResources struct { endpointSlices []*discoveryv1.EndpointSlice } +const httpProxyFinalizer = "networking.datumapis.com/httpproxy-cleanup" + const ( SchemeHTTP = "http" SchemeHTTPS = "https" @@ -57,6 +70,7 @@ const ( // +kubebuilder:rbac:groups=networking.datumapis.com,resources=httpproxies,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=networking.datumapis.com,resources=httpproxies/status,verbs=get;update;patch // +kubebuilder:rbac:groups=networking.datumapis.com,resources=httpproxies/finalizers,verbs=update +// +kubebuilder:rbac:groups=networking.datumapis.com,resources=connectors,verbs=get;list;watch func (r *HTTPProxyReconciler) Reconcile(ctx context.Context, req mcreconcile.Request) (_ ctrl.Result, err error) { logger := log.FromContext(ctx, "cluster", req.ClusterName) @@ -75,9 +89,37 @@ func (r *HTTPProxyReconciler) Reconcile(ctx context.Context, req mcreconcile.Req return ctrl.Result{}, err } + if !httpProxy.DeletionTimestamp.IsZero() { + if controllerutil.ContainsFinalizer(&httpProxy, httpProxyFinalizer) { + if err := r.cleanupConnectorEnvoyPatchPolicy(ctx, cl.GetClient(), req.ClusterName, &httpProxy); err != nil { + return ctrl.Result{}, err + } + controllerutil.RemoveFinalizer(&httpProxy, httpProxyFinalizer) + if err := cl.GetClient().Update(ctx, &httpProxy); err != nil { + return ctrl.Result{}, err + } + } + return ctrl.Result{}, nil + } + logger.Info("reconciling httpproxy") defer logger.Info("reconcile complete") + if updated := ensureConnectorNameAnnotation(&httpProxy); updated { + if err := cl.GetClient().Update(ctx, &httpProxy); err != nil { + return ctrl.Result{}, fmt.Errorf("failed updating httpproxy connector annotation: %w", err) + } + return ctrl.Result{}, nil + } + + if !controllerutil.ContainsFinalizer(&httpProxy, httpProxyFinalizer) { + controllerutil.AddFinalizer(&httpProxy, httpProxyFinalizer) + if err := cl.GetClient().Update(ctx, &httpProxy); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{}, nil + } + httpProxyCopy := httpProxy.DeepCopy() acceptedCondition := &metav1.Condition{ @@ -96,9 +138,23 @@ func (r *HTTPProxyReconciler) Reconcile(ctx context.Context, req mcreconcile.Req Message: "The HTTPProxy has not been programmed", } + tunnelMetadataCondition := &metav1.Condition{ + Type: networkingv1alpha.HTTPProxyConditionTunnelMetadataProgrammed, + Status: metav1.ConditionFalse, + Reason: networkingv1alpha.HTTPProxyReasonPending, + ObservedGeneration: httpProxy.Generation, + Message: "Waiting for downstream EnvoyPatchPolicy to be accepted and programmed", + } + setTunnelMetadataCondition := false + defer func() { apimeta.SetStatusCondition(&httpProxyCopy.Status.Conditions, *acceptedCondition) apimeta.SetStatusCondition(&httpProxyCopy.Status.Conditions, *programmedCondition) + if setTunnelMetadataCondition { + apimeta.SetStatusCondition(&httpProxyCopy.Status.Conditions, *tunnelMetadataCondition) + } else { + apimeta.RemoveStatusCondition(&httpProxyCopy.Status.Conditions, networkingv1alpha.HTTPProxyConditionTunnelMetadataProgrammed) + } if !equality.Semantic.DeepEqual(httpProxy.Status, httpProxyCopy.Status) { httpProxy.Status = httpProxyCopy.Status @@ -225,6 +281,20 @@ func (r *HTTPProxyReconciler) Reconcile(ctx context.Context, req mcreconcile.Req logger.Info("processed endpointslice", "result", result, "name", desiredEndpointSlice.Name) } + patchPolicy, hasConnectorBackends, err := r.reconcileConnectorEnvoyPatchPolicy( + ctx, + cl.GetClient(), + req.ClusterName, + &httpProxy, + desiredResources.gateway, + ) + if err != nil { + programmedCondition.Status = metav1.ConditionFalse + programmedCondition.Reason = networkingv1alpha.HTTPProxyReasonPending + programmedCondition.Message = err.Error() + return ctrl.Result{}, err + } + httpProxyCopy.Status.Addresses = gateway.Status.Addresses if c := apimeta.FindStatusCondition(gateway.Status.Conditions, string(gatewayv1.GatewayConditionAccepted)); c != nil { @@ -248,6 +318,32 @@ func (r *HTTPProxyReconciler) Reconcile(ctx context.Context, req mcreconcile.Req } } + if hasConnectorBackends { + connectorPolicyReady, connectorPolicyMessage := downstreamPatchPolicyReady( + patchPolicy, + r.Config.Gateway.DownstreamGatewayClassName, + ) + if !connectorPolicyReady { + programmedCondition.Status = metav1.ConditionFalse + programmedCondition.Reason = networkingv1alpha.HTTPProxyReasonPending + if connectorPolicyMessage == "" { + connectorPolicyMessage = "Waiting for downstream EnvoyPatchPolicy to be accepted and programmed" + } + programmedCondition.Message = connectorPolicyMessage + + tunnelMetadataCondition.Status = metav1.ConditionFalse + tunnelMetadataCondition.Reason = networkingv1alpha.HTTPProxyReasonPending + tunnelMetadataCondition.Message = connectorPolicyMessage + } else { + tunnelMetadataCondition.Status = metav1.ConditionTrue + tunnelMetadataCondition.Reason = networkingv1alpha.HTTPProxyReasonTunnelMetadataApplied + tunnelMetadataCondition.Message = "Connector tunnel metadata applied" + } + setTunnelMetadataCondition = true + } else { + apimeta.RemoveStatusCondition(&httpProxyCopy.Status.Conditions, networkingv1alpha.HTTPProxyConditionTunnelMetadataProgrammed) + } + r.reconcileHTTPProxyHostnameStatus(ctx, gateway, httpProxyCopy) return ctrl.Result{}, nil @@ -361,17 +457,70 @@ func (r *HTTPProxyReconciler) reconcileHTTPProxyHostnameStatus( } } +func ensureConnectorNameAnnotation(httpProxy *networkingv1alpha.HTTPProxy) bool { + var connectorName string + for _, rule := range httpProxy.Spec.Rules { + for _, backend := range rule.Backends { + if backend.Connector != nil && backend.Connector.Name != "" { + if connectorName == "" { + connectorName = backend.Connector.Name + } else if connectorName != backend.Connector.Name { + // Prefer first connector for annotation stability if multiple are present. + break + } + } + } + } + + annotations := httpProxy.GetAnnotations() + if connectorName == "" { + if annotations == nil { + return false + } + if _, ok := annotations[networkingv1alpha1.ConnectorNameAnnotation]; !ok { + return false + } + delete(annotations, networkingv1alpha1.ConnectorNameAnnotation) + if len(annotations) == 0 { + httpProxy.SetAnnotations(nil) + } else { + httpProxy.SetAnnotations(annotations) + } + return true + } + + if annotations == nil { + annotations = map[string]string{} + } + if annotations[networkingv1alpha1.ConnectorNameAnnotation] == connectorName { + return false + } + annotations[networkingv1alpha1.ConnectorNameAnnotation] = connectorName + httpProxy.SetAnnotations(annotations) + return true +} + // SetupWithManager sets up the controller with the Manager. func (r *HTTPProxyReconciler) SetupWithManager(mgr mcmanager.Manager) error { r.mgr = mgr - return mcbuilder.ControllerManagedBy(mgr). + builder := mcbuilder.ControllerManagedBy(mgr). For(&networkingv1alpha.HTTPProxy{}). Owns(&gatewayv1.Gateway{}). Owns(&gatewayv1.HTTPRoute{}). - Owns(&discoveryv1.EndpointSlice{}). - Named("httpproxy"). - Complete(r) + Owns(&discoveryv1.EndpointSlice{}) + + if r.DownstreamCluster != nil { + downstreamPolicySource := mcsource.TypedKind( + &envoygatewayv1alpha1.EnvoyPatchPolicy{}, + downstreamclient.TypedEnqueueRequestForUpstreamOwner[*envoygatewayv1alpha1.EnvoyPatchPolicy](&networkingv1alpha.HTTPProxy{}), + ) + + downstreamPolicyClusterSource, _ := downstreamPolicySource.ForCluster("", r.DownstreamCluster) + builder = builder.WatchesRawSource(downstreamPolicyClusterSource) + } + + return builder.Named("httpproxy").Complete(r) } func (r *HTTPProxyReconciler) collectDesiredResources( @@ -482,19 +631,23 @@ func (r *HTTPProxyReconciler) collectDesiredResources( // once we have one. This is in an effort to not block MVP goals. addressType := discoveryv1.AddressTypeFQDN - host := u.Hostname() - if ip := net.ParseIP(host); ip != nil { + targetHost := u.Hostname() + endpointHost := targetHost + if backend.Connector != nil { + // Connector backends don't rely on EndpointSlice addresses; use a safe placeholder. + endpointHost = "connector.invalid" + addressType = discoveryv1.AddressTypeFQDN + } else if ip := net.ParseIP(targetHost); ip != nil { if i := ip.To4(); i != nil && len(i) == net.IPv4len { addressType = discoveryv1.AddressTypeIPv4 } else { addressType = discoveryv1.AddressTypeIPv6 } } else { - hostnameRewriteFound := false for _, filter := range rule.Filters { if filter.Type == gatewayv1.HTTPRouteFilterURLRewrite { - filter.URLRewrite.Hostname = ptr.To(gatewayv1.PreciseHostname(host)) + filter.URLRewrite.Hostname = ptr.To(gatewayv1.PreciseHostname(targetHost)) hostnameRewriteFound = true break } @@ -504,7 +657,7 @@ func (r *HTTPProxyReconciler) collectDesiredResources( rule.Filters = append(rule.Filters, gatewayv1.HTTPRouteFilter{ Type: gatewayv1.HTTPRouteFilterURLRewrite, URLRewrite: &gatewayv1.HTTPURLRewriteFilter{ - Hostname: ptr.To(gatewayv1.PreciseHostname(host)), + Hostname: ptr.To(gatewayv1.PreciseHostname(targetHost)), }, }) } @@ -519,7 +672,7 @@ func (r *HTTPProxyReconciler) collectDesiredResources( Endpoints: []discoveryv1.Endpoint{ { Addresses: []string{ - host, + endpointHost, }, Conditions: discoveryv1.EndpointConditions{ Ready: ptr.To(true), @@ -577,3 +730,426 @@ func hasControllerConflict(obj, owner metav1.Object) bool { return controllerutil.HasControllerReference(obj) && !metav1.IsControlledBy(obj, owner) } + +type connectorBackendPatch struct { + // Gateway listener section name (default-https, etc.). + sectionName *gatewayv1.SectionName + + // Identify the HTTPRoute rule/match this backend applies to. + ruleIndex int + matchIndex int + + targetHost string + targetPort int + nodeID string +} + +func (r *HTTPProxyReconciler) reconcileConnectorEnvoyPatchPolicy( + ctx context.Context, + upstreamClient client.Client, + clusterName string, + httpProxy *networkingv1alpha.HTTPProxy, + gateway *gatewayv1.Gateway, +) (*envoygatewayv1alpha1.EnvoyPatchPolicy, bool, error) { + if r.DownstreamCluster == nil { + return nil, false, nil + } + + downstreamStrategy := downstreamclient.NewMappedNamespaceResourceStrategy( + clusterName, + upstreamClient, + r.DownstreamCluster.GetClient(), + ) + downstreamNamespaceName, err := downstreamStrategy.GetDownstreamNamespaceNameForUpstreamNamespace(ctx, httpProxy.Namespace) + if err != nil { + return nil, false, err + } + + connectorBackends, err := collectConnectorBackends(ctx, upstreamClient, httpProxy) + if err != nil { + return nil, false, err + } + + policyName := fmt.Sprintf("connector-%s", httpProxy.Name) + policyKey := client.ObjectKey{Namespace: downstreamNamespaceName, Name: policyName} + downstreamClient := downstreamStrategy.GetClient() + + if len(connectorBackends) == 0 { + var existing envoygatewayv1alpha1.EnvoyPatchPolicy + if err := downstreamClient.Get(ctx, policyKey, &existing); err != nil { + if apierrors.IsNotFound(err) { + if err := downstreamStrategy.DeleteAnchorForObject(ctx, httpProxy); err != nil { + return nil, false, err + } + return nil, false, nil + } + return nil, false, err + } + if err := downstreamClient.Delete(ctx, &existing); err != nil { + return nil, false, err + } + if err := downstreamStrategy.DeleteAnchorForObject(ctx, httpProxy); err != nil { + return nil, false, err + } + return nil, false, nil + } + + if r.Config.Gateway.DownstreamGatewayClassName == "" { + return nil, true, fmt.Errorf("downstreamGatewayClassName is required for connector patching") + } + + jsonPatches, err := buildConnectorEnvoyPatches( + downstreamNamespaceName, + gateway, + httpProxy, + connectorBackends, + ) + if err != nil { + return nil, true, err + } + + policy := envoygatewayv1alpha1.EnvoyPatchPolicy{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: downstreamNamespaceName, + Name: policyName, + }, + } + _, err = controllerutil.CreateOrUpdate(ctx, downstreamClient, &policy, func() error { + if err := downstreamStrategy.SetControllerReference(ctx, httpProxy, &policy); err != nil { + return err + } + policy.Spec = envoygatewayv1alpha1.EnvoyPatchPolicySpec{ + TargetRef: gatewayv1alpha2.LocalPolicyTargetReference{ + Group: gatewayv1.GroupName, + Kind: "GatewayClass", + Name: gatewayv1.ObjectName(r.Config.Gateway.DownstreamGatewayClassName), + }, + Type: envoygatewayv1alpha1.JSONPatchEnvoyPatchType, + JSONPatches: jsonPatches, + } + return nil + }) + if err != nil { + return nil, true, err + } + return &policy, true, nil +} + +func (r *HTTPProxyReconciler) cleanupConnectorEnvoyPatchPolicy( + ctx context.Context, + upstreamClient client.Client, + clusterName string, + httpProxy *networkingv1alpha.HTTPProxy, +) error { + if r.DownstreamCluster == nil { + return nil + } + + downstreamStrategy := downstreamclient.NewMappedNamespaceResourceStrategy( + clusterName, + upstreamClient, + r.DownstreamCluster.GetClient(), + ) + downstreamNamespaceName, err := downstreamStrategy.GetDownstreamNamespaceNameForUpstreamNamespace(ctx, httpProxy.Namespace) + if err != nil { + return err + } + + policyName := fmt.Sprintf("connector-%s", httpProxy.Name) + policyKey := client.ObjectKey{Namespace: downstreamNamespaceName, Name: policyName} + downstreamClient := downstreamStrategy.GetClient() + + var policy envoygatewayv1alpha1.EnvoyPatchPolicy + if err := downstreamClient.Get(ctx, policyKey, &policy); err != nil { + if apierrors.IsNotFound(err) { + return downstreamStrategy.DeleteAnchorForObject(ctx, httpProxy) + } + return err + } + if err := downstreamClient.Delete(ctx, &policy); err != nil { + return err + } + return downstreamStrategy.DeleteAnchorForObject(ctx, httpProxy) +} + +func downstreamPatchPolicyReady(policy *envoygatewayv1alpha1.EnvoyPatchPolicy, gatewayClassName string) (bool, string) { + if policy == nil { + return false, "Downstream EnvoyPatchPolicy not found" + } + + if len(policy.Status.Ancestors) == 0 { + return false, "Downstream EnvoyPatchPolicy has no status yet" + } + + for _, ancestor := range policy.Status.Ancestors { + if ptr.Deref(ancestor.AncestorRef.Kind, gatewayv1.Kind("")) != gatewayv1.Kind("GatewayClass") || + ancestor.AncestorRef.Name != gatewayv1.ObjectName(gatewayClassName) { + continue + } + + accepted := apimeta.FindStatusCondition(ancestor.Conditions, "Accepted") + if accepted == nil || accepted.Status != metav1.ConditionTrue { + return false, formatPolicyConditionMessage("Accepted", accepted) + } + + programmed := apimeta.FindStatusCondition(ancestor.Conditions, "Programmed") + if programmed == nil || programmed.Status != metav1.ConditionTrue { + return false, formatPolicyConditionMessage("Programmed", programmed) + } + + return true, "" + } + + return false, fmt.Sprintf("Downstream EnvoyPatchPolicy has no ancestor status for GatewayClass %q", gatewayClassName) +} + +func formatPolicyConditionMessage(conditionType string, condition *metav1.Condition) string { + if condition == nil { + return fmt.Sprintf("Downstream EnvoyPatchPolicy is missing %s condition", conditionType) + } + if condition.Message == "" { + return fmt.Sprintf("Downstream EnvoyPatchPolicy %s=%s (%s)", condition.Type, condition.Status, condition.Reason) + } + return fmt.Sprintf("Downstream EnvoyPatchPolicy %s=%s (%s): %s", condition.Type, condition.Status, condition.Reason, condition.Message) +} + +func collectConnectorBackends( + ctx context.Context, + cl client.Client, + httpProxy *networkingv1alpha.HTTPProxy, +) ([]connectorBackendPatch, error) { + connectorBackends := make([]connectorBackendPatch, 0) + for ruleIndex, rule := range httpProxy.Spec.Rules { + matchCount := len(rule.Matches) + if matchCount == 0 { + matchCount = 1 + } + for matchIndex := 0; matchIndex < matchCount; matchIndex++ { + for _, backend := range rule.Backends { + if backend.Connector == nil { + continue + } + + targetHost, targetPort, err := backendEndpointTarget(backend) + if err != nil { + return nil, err + } + + nodeID, err := connectorNodeID(ctx, cl, httpProxy.Namespace, backend.Connector.Name) + if err != nil { + return nil, err + } + + connectorBackends = append(connectorBackends, connectorBackendPatch{ + sectionName: nil, + ruleIndex: ruleIndex, + matchIndex: matchIndex, + targetHost: targetHost, + targetPort: targetPort, + nodeID: nodeID, + }) + } + } + } + return connectorBackends, nil +} + +func backendEndpointTarget(backend networkingv1alpha.HTTPProxyRuleBackend) (string, int, error) { + u, err := url.Parse(backend.Endpoint) + if err != nil { + return "", 0, fmt.Errorf("failed parsing backend endpoint: %w", err) + } + + targetHost := u.Hostname() + if targetHost == "" { + return "", 0, fmt.Errorf("backend endpoint host is required") + } + + targetPort := DefaultHTTPPort + if u.Scheme == SchemeHTTPS { + targetPort = DefaultHTTPSPort + } + if endpointPort := u.Port(); endpointPort != "" { + targetPort, err = strconv.Atoi(endpointPort) + if err != nil { + return "", 0, fmt.Errorf("invalid backend endpoint port: %w", err) + } + } + + return targetHost, targetPort, nil +} + +func connectorNodeID(ctx context.Context, cl client.Client, namespace, name string) (string, error) { + var connector networkingv1alpha1.Connector + if err := cl.Get(ctx, client.ObjectKey{Namespace: namespace, Name: name}, &connector); err != nil { + return "", err + } + + details := connector.Status.ConnectionDetails + if details == nil || details.Type != networkingv1alpha1.PublicKeyConnectorConnectionType || details.PublicKey == nil { + return "", fmt.Errorf("connector %q does not have public key connection details", name) + } + if details.PublicKey.Id == "" { + return "", fmt.Errorf("connector %q public key id is empty", name) + } + return details.PublicKey.Id, nil +} + +func buildConnectorEnvoyPatches( + downstreamNamespace string, + gateway *gatewayv1.Gateway, + httpProxy *networkingv1alpha.HTTPProxy, + backends []connectorBackendPatch, +) ([]envoygatewayv1alpha1.EnvoyJSONPatchConfig, error) { + routeConfigNames := connectorRouteConfigNames(downstreamNamespace, gateway) + patches := make([]envoygatewayv1alpha1.EnvoyJSONPatchConfig, 0) + // TODO: Make this idempotent + headersOp := envoygatewayv1alpha1.JSONPatchOperationType("add") + metadataOp := envoygatewayv1alpha1.JSONPatchOperationType("add") + + clusterJSON, err := json.Marshal("internal-tunnel-cluster") + if err != nil { + return nil, fmt.Errorf("failed to marshal cluster name: %w", err) + } + + for _, routeConfigName := range routeConfigNames { + for _, backend := range backends { + jsonPath := connectorRouteJSONPath( + downstreamNamespace, + gateway, + httpProxy.Name, + backend.sectionName, + backend.ruleIndex, + backend.matchIndex, + ) + + headersJSON, err := json.Marshal([]map[string]any{ + { + "header": map[string]any{ + "key": "x-tunnel-host", + "value": backend.targetHost, + }, + "append_action": "OVERWRITE_IF_EXISTS_OR_ADD", + }, + { + "header": map[string]any{ + "key": "x-tunnel-port", + "value": strconv.Itoa(backend.targetPort), + }, + "append_action": "OVERWRITE_IF_EXISTS_OR_ADD", + }, + { + "header": map[string]any{ + "key": "x-iroh-endpoint-id", + "value": backend.nodeID, + }, + "append_action": "OVERWRITE_IF_EXISTS_OR_ADD", + }, + }) + if err != nil { + return nil, fmt.Errorf("failed to marshal request headers: %w", err) + } + + // Per-route metadata used later by tcp_proxy tunneling_config via %DYNAMIC_METADATA(tunnel:...)%. + tunnelMetaJSON, err := json.Marshal(map[string]any{ + "host": backend.targetHost, + "port": backend.targetPort, + "endpoint_id": backend.nodeID, + }) + if err != nil { + return nil, fmt.Errorf("failed to marshal tunnel metadata: %w", err) + } + + patches = append(patches, + // 1) Write per-route metadata under metadata.filter_metadata.tunnel + // (do NOT replace /metadata as a whole, Envoy Gateway uses it too). + envoygatewayv1alpha1.EnvoyJSONPatchConfig{ + Type: "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + Name: routeConfigName, + Operation: envoygatewayv1alpha1.JSONPatchOperation{ + Op: metadataOp, + JSONPath: ptr.To(jsonPath), + Path: ptr.To("/metadata/filter_metadata/tunnel"), + Value: &apiextensionsv1.JSON{Raw: tunnelMetaJSON}, + }, + }, + // 2) Send matched requests to the internal tunnel listener. + envoygatewayv1alpha1.EnvoyJSONPatchConfig{ + Type: "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + Name: routeConfigName, + Operation: envoygatewayv1alpha1.JSONPatchOperation{ + Op: envoygatewayv1alpha1.JSONPatchOperationType("replace"), + JSONPath: ptr.To(jsonPath), + Path: ptr.To("/route/cluster"), + Value: &apiextensionsv1.JSON{Raw: clusterJSON}, + }, + }, + // 3) Inject internal control headers based on the selected route/backend. + // The client does not send these. + envoygatewayv1alpha1.EnvoyJSONPatchConfig{ + Type: "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + Name: routeConfigName, + Operation: envoygatewayv1alpha1.JSONPatchOperation{ + Op: headersOp, + JSONPath: ptr.To(jsonPath), + Path: ptr.To("/request_headers_to_add"), + Value: &apiextensionsv1.JSON{Raw: headersJSON}, + }, + }, + ) + } + } + + return patches, nil +} + +func connectorRouteConfigNames(downstreamNamespace string, gateway *gatewayv1.Gateway) []string { + routeConfigNames := []string{} + for _, listener := range gateway.Spec.Listeners { + if listener.Protocol != gatewayv1.HTTPSProtocolType { + continue + } + routeConfigNames = append(routeConfigNames, fmt.Sprintf("%s/%s/%s", downstreamNamespace, gateway.Name, listener.Name)) + } + return routeConfigNames +} + +func connectorRouteJSONPath( + downstreamNamespace string, + gateway *gatewayv1.Gateway, + httpRouteName string, + sectionName *gatewayv1.SectionName, + ruleIndex int, + matchIndex int, +) string { + // vhost matches the Gateway + optional sectionName + vhostConstraints := fmt.Sprintf( + `@.metadata.filter_metadata["envoy-gateway"].resources[0].kind=="%s" && @.metadata.filter_metadata["envoy-gateway"].resources[0].namespace=="%s" && @.metadata.filter_metadata["envoy-gateway"].resources[0].name=="%s"`, + KindGateway, + downstreamNamespace, + gateway.Name, + ) + + if sectionName != nil { + vhostConstraints += fmt.Sprintf( + ` && @.metadata.filter_metadata["envoy-gateway"].resources[0].sectionName=="%s"`, + string(*sectionName), + ) + } + + // routes match the HTTPRoute + rule/match + routeConstraints := fmt.Sprintf( + `@.metadata.filter_metadata["envoy-gateway"].resources[0].kind=="%s" && @.metadata.filter_metadata["envoy-gateway"].resources[0].namespace=="%s" && @.metadata.filter_metadata["envoy-gateway"].resources[0].name=="%s" && @.name =~ ".*?/rule/%d/match/%d/.*"`, + KindHTTPRoute, + downstreamNamespace, + httpRouteName, + ruleIndex, + matchIndex, + ) + + return sanitizeJSONPath(fmt.Sprintf( + `..virtual_hosts[?(%s)]..routes[?(!@.bogus && %s)]`, + vhostConstraints, + routeConstraints, + )) +} diff --git a/internal/controller/httpproxy_controller_test.go b/internal/controller/httpproxy_controller_test.go index 316ca99a..8f05a329 100644 --- a/internal/controller/httpproxy_controller_test.go +++ b/internal/controller/httpproxy_controller_test.go @@ -5,11 +5,15 @@ import ( "fmt" "testing" + envoygatewayv1alpha1 "github.com/envoyproxy/gateway/api/v1alpha1" "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" discoveryv1 "k8s.io/api/discovery/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/client-go/kubernetes/scheme" "k8s.io/utils/ptr" @@ -17,6 +21,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/client/interceptor" "sigs.k8s.io/controller-runtime/pkg/cluster" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -25,6 +30,7 @@ import ( mcreconcile "sigs.k8s.io/multicluster-runtime/pkg/reconcile" networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" + networkingv1alpha1 "go.datum.net/network-services-operator/api/v1alpha1" "go.datum.net/network-services-operator/internal/config" gatewayutil "go.datum.net/network-services-operator/internal/util/gateway" ) @@ -296,16 +302,19 @@ func TestHTTPProxyReconcile(t *testing.T) { testScheme := runtime.NewScheme() assert.NoError(t, scheme.AddToScheme(testScheme)) assert.NoError(t, gatewayv1.Install(testScheme)) + assert.NoError(t, envoygatewayv1alpha1.AddToScheme(testScheme)) assert.NoError(t, discoveryv1.AddToScheme(testScheme)) assert.NoError(t, networkingv1alpha.AddToScheme(testScheme)) + assert.NoError(t, networkingv1alpha1.AddToScheme(testScheme)) testConfig := config.NetworkServicesOperator{ HTTPProxy: config.HTTPProxyConfig{ GatewayClassName: "test-gateway-class", }, Gateway: config.GatewayConfig{ - ControllerName: gatewayv1.GatewayController("test-gateway-class"), - TargetDomain: "example.com", + ControllerName: gatewayv1.GatewayController("test-gateway-class"), + DownstreamGatewayClassName: "test-downstream-gateway-class", + TargetDomain: "example.com", ListenerTLSOptions: map[gatewayv1.AnnotationKey]gatewayv1.AnnotationValue{ gatewayv1.AnnotationKey("gateway.networking.datumapis.com/certificate-issuer"): gatewayv1.AnnotationValue("test-issuer"), }, @@ -314,19 +323,168 @@ func TestHTTPProxyReconcile(t *testing.T) { type testContext struct { *testing.T - reconciler *HTTPProxyReconciler - gateway *gatewayv1.Gateway + reconciler *HTTPProxyReconciler + gateway *gatewayv1.Gateway + downstreamClient client.Client + } + + connectorNamespaceUID := types.UID("11111111-1111-1111-1111-111111111111") + connectorDownstreamNamespace := fmt.Sprintf("ns-%s", connectorNamespaceUID) + connectorHTTPProxy := newHTTPProxy(func(h *networkingv1alpha.HTTPProxy) { + h.Spec.Rules[0].Backends[0].Connector = &networkingv1alpha.ConnectorReference{ + Name: "connector-1", + } + }) + connectorClearedHTTPProxy := newHTTPProxy(func(h *networkingv1alpha.HTTPProxy) { + h.Spec.Rules[0].Backends[0].Connector = &networkingv1alpha.ConnectorReference{ + Name: "connector-1", + } + }) + + connectorDownstreamObjects := func(proxy *networkingv1alpha.HTTPProxy) []client.Object { + return []client.Object{ + &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: connectorDownstreamNamespace}}, + &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("anchor-%s", proxy.UID), + Namespace: connectorDownstreamNamespace, + CreationTimestamp: metav1.Now(), + }}, + } } tests := []struct { name string httpProxy *networkingv1alpha.HTTPProxy existingObjects []client.Object + downstreamObjects []client.Object + namespaceUID string postCreateGatewayStatus func(*gatewayv1.Gateway) expectedError bool expectedConditions []metav1.Condition assert func(t *testContext, cl client.Client, httpProxy *networkingv1alpha.HTTPProxy) }{ + { + name: "connector backend creates envoy patch policy", + httpProxy: connectorHTTPProxy, + downstreamObjects: connectorDownstreamObjects(connectorHTTPProxy), + namespaceUID: string(connectorNamespaceUID), + existingObjects: []client.Object{ + &networkingv1alpha1.Connector{ + ObjectMeta: metav1.ObjectMeta{ + Name: "connector-1", + Namespace: "test", + }, + Status: networkingv1alpha1.ConnectorStatus{ + ConnectionDetails: &networkingv1alpha1.ConnectorConnectionDetails{ + Type: networkingv1alpha1.PublicKeyConnectorConnectionType, + PublicKey: &networkingv1alpha1.ConnectorConnectionDetailsPublicKey{ + Id: "node-123", + DiscoveryMode: networkingv1alpha1.DNSPublicKeyDiscoveryMode, + HomeRelay: "https://relay.example.test", + Addresses: []networkingv1alpha1.PublicKeyConnectorAddress{ + { + Address: "127.0.0.1", + Port: 80, + }, + }, + }, + }, + }, + }, + }, + expectedError: false, + expectedConditions: []metav1.Condition{ + { + Type: networkingv1alpha.HTTPProxyConditionAccepted, + Status: metav1.ConditionTrue, + Reason: networkingv1alpha.HTTPProxyReasonAccepted, + }, + { + Type: networkingv1alpha.HTTPProxyConditionProgrammed, + Status: metav1.ConditionFalse, + Reason: networkingv1alpha.HTTPProxyReasonPending, + }, + }, + assert: func(t *testContext, cl client.Client, httpProxy *networkingv1alpha.HTTPProxy) { + var patchList envoygatewayv1alpha1.EnvoyPatchPolicyList + err := t.downstreamClient.List(context.Background(), &patchList) + assert.NoError(t, err) + assert.Len(t, patchList.Items, 1) + assert.Equal(t, fmt.Sprintf("connector-%s", httpProxy.Name), patchList.Items[0].Name) + }, + }, + { + name: "connector patch policy removed when connector cleared", + httpProxy: connectorClearedHTTPProxy, + downstreamObjects: connectorDownstreamObjects(connectorClearedHTTPProxy), + namespaceUID: string(connectorNamespaceUID), + existingObjects: []client.Object{ + &networkingv1alpha1.Connector{ + ObjectMeta: metav1.ObjectMeta{ + Name: "connector-1", + Namespace: "test", + }, + Status: networkingv1alpha1.ConnectorStatus{ + ConnectionDetails: &networkingv1alpha1.ConnectorConnectionDetails{ + Type: networkingv1alpha1.PublicKeyConnectorConnectionType, + PublicKey: &networkingv1alpha1.ConnectorConnectionDetailsPublicKey{ + Id: "node-123", + DiscoveryMode: networkingv1alpha1.DNSPublicKeyDiscoveryMode, + HomeRelay: "https://relay.example.test", + Addresses: []networkingv1alpha1.PublicKeyConnectorAddress{ + { + Address: "127.0.0.1", + Port: 80, + }, + }, + }, + }, + }, + }, + }, + expectedError: false, + expectedConditions: []metav1.Condition{ + { + Type: networkingv1alpha.HTTPProxyConditionAccepted, + Status: metav1.ConditionTrue, + Reason: networkingv1alpha.HTTPProxyReasonAccepted, + }, + { + Type: networkingv1alpha.HTTPProxyConditionProgrammed, + Status: metav1.ConditionFalse, + Reason: networkingv1alpha.HTTPProxyReasonPending, + }, + }, + assert: func(t *testContext, cl client.Client, httpProxy *networkingv1alpha.HTTPProxy) { + ctx := context.Background() + + var patchList envoygatewayv1alpha1.EnvoyPatchPolicyList + err := t.downstreamClient.List(ctx, &patchList) + assert.NoError(t, err) + assert.Len(t, patchList.Items, 1) + + updatedProxy := &networkingv1alpha.HTTPProxy{} + assert.NoError(t, cl.Get(ctx, client.ObjectKeyFromObject(httpProxy), updatedProxy)) + updatedProxy.Spec.Rules[0].Backends[0].Connector = nil + assert.NoError(t, cl.Update(ctx, updatedProxy)) + + req := mcreconcile.Request{ + Request: reconcile.Request{ + NamespacedName: client.ObjectKeyFromObject(httpProxy), + }, + ClusterName: "test-cluster", + } + for i := 0; i < 2; i++ { + _, err = t.reconciler.Reconcile(ctx, req) + assert.NoError(t, err) + } + + patchList = envoygatewayv1alpha1.EnvoyPatchPolicyList{} + err = t.downstreamClient.List(ctx, &patchList) + assert.NoError(t, err) + assert.Len(t, patchList.Items, 0) + }, + }, { name: "basic reconcile - creates resources", httpProxy: newHTTPProxy(), @@ -706,6 +864,15 @@ func TestHTTPProxyReconcile(t *testing.T) { initialObjects = append(initialObjects, tt.httpProxy) initialObjects = append(initialObjects, tt.existingObjects...) + if tt.httpProxy.Namespace != "" { + namespace := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: tt.httpProxy.Namespace}} + if tt.namespaceUID != "" { + namespace.SetUID(types.UID(tt.namespaceUID)) + } else { + namespace.SetUID(uuid.NewUUID()) + } + initialObjects = append(initialObjects, namespace) + } fakeClientBuilder := fake.NewClientBuilder(). WithScheme(testScheme). @@ -724,14 +891,16 @@ func TestHTTPProxyReconcile(t *testing.T) { fakeDownstreamClient := fake.NewClientBuilder(). WithScheme(testScheme). + WithObjects(tt.downstreamObjects...). WithStatusSubresource(&gatewayv1.Gateway{}). Build() mgr := &fakeMockManager{cl: fakeClient} reconciler := &HTTPProxyReconciler{ - mgr: mgr, - Config: testConfig, + mgr: mgr, + Config: testConfig, + DownstreamCluster: &fakeCluster{cl: fakeDownstreamClient}, } gatewayReconciler := &GatewayReconciler{ @@ -750,12 +919,18 @@ func TestHTTPProxyReconcile(t *testing.T) { ctx := context.Background() ctx = log.IntoContext(ctx, logger) - _, err := reconciler.Reconcile(ctx, req) - - if tt.expectedError { - assert.Error(t, err) - } else { - assert.NoError(t, err) + var err error + for i := 0; i < 3; i++ { + _, err = reconciler.Reconcile(ctx, req) + if i < 2 { + assert.NoError(t, err) + continue + } + if tt.expectedError { + assert.Error(t, err) + } else { + assert.NoError(t, err) + } } _, err = gatewayReconciler.Reconcile(ctx, req) @@ -803,9 +978,10 @@ func TestHTTPProxyReconcile(t *testing.T) { if tt.assert != nil { testCtx := &testContext{ - T: t, - reconciler: reconciler, - gateway: &gateway, + T: t, + reconciler: reconciler, + gateway: &gateway, + downstreamClient: fakeDownstreamClient, } tt.assert(testCtx, fakeClient, &updatedProxy) } @@ -814,6 +990,125 @@ func TestHTTPProxyReconcile(t *testing.T) { } } +func TestConnectorRouteJSONPathTargetsRuleMatch(t *testing.T) { + path := connectorRouteJSONPath( + "ns-test", + &gatewayv1.Gateway{ObjectMeta: metav1.ObjectMeta{Name: "gw"}}, + "route-name", + ptr.To(gatewayv1.SectionName("default-https")), + 2, + 1, + ) + + assert.Contains(t, path, `sectionName=="default-https"`) + assert.Contains(t, path, `kind=="HTTPRoute"`) + assert.Contains(t, path, `name=="route-name"`) + assert.Contains(t, path, `/rule/2/match/1/`) +} + +func TestConnectorRouteJSONPathDistinctPerRuleMatch(t *testing.T) { + pathA := connectorRouteJSONPath( + "ns-test", + &gatewayv1.Gateway{ObjectMeta: metav1.ObjectMeta{Name: "gw"}}, + "route-name", + ptr.To(gatewayv1.SectionName("default-https")), + 0, + 0, + ) + pathB := connectorRouteJSONPath( + "ns-test", + &gatewayv1.Gateway{ObjectMeta: metav1.ObjectMeta{Name: "gw"}}, + "route-name", + ptr.To(gatewayv1.SectionName("default-https")), + 1, + 0, + ) + + assert.NotEqual(t, pathA, pathB) + assert.Contains(t, pathA, `/rule/0/match/0/`) + assert.Contains(t, pathB, `/rule/1/match/0/`) +} + +func TestHTTPProxyFinalizerCleanup(t *testing.T) { + logger := zap.New(zap.UseFlagOptions(&zap.Options{Development: true})) + ctx := log.IntoContext(context.Background(), logger) + + testScheme := runtime.NewScheme() + assert.NoError(t, scheme.AddToScheme(testScheme)) + assert.NoError(t, gatewayv1.Install(testScheme)) + assert.NoError(t, envoygatewayv1alpha1.AddToScheme(testScheme)) + assert.NoError(t, discoveryv1.AddToScheme(testScheme)) + assert.NoError(t, networkingv1alpha.AddToScheme(testScheme)) + assert.NoError(t, networkingv1alpha1.AddToScheme(testScheme)) + + testConfig := config.NetworkServicesOperator{ + HTTPProxy: config.HTTPProxyConfig{ + GatewayClassName: "test-gateway-class", + }, + Gateway: config.GatewayConfig{ + ControllerName: gatewayv1.GatewayController("test-gateway-class"), + DownstreamGatewayClassName: "test-downstream-gateway-class", + }, + } + + httpProxy := newHTTPProxy() + deletionTime := metav1.Now() + httpProxy.DeletionTimestamp = &deletionTime + httpProxy.Finalizers = append(httpProxy.Finalizers, httpProxyFinalizer) + + namespaceUID := types.UID("11111111-1111-1111-1111-111111111111") + upstreamNamespace := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: httpProxy.Namespace}} + upstreamNamespace.SetUID(namespaceUID) + + downstreamNamespaceName := fmt.Sprintf("ns-%s", namespaceUID) + downstreamNamespace := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: downstreamNamespaceName}} + downstreamPolicy := &envoygatewayv1alpha1.EnvoyPatchPolicy{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("connector-%s", httpProxy.Name), + Namespace: downstreamNamespaceName, + }, + } + + fakeClient := fake.NewClientBuilder(). + WithScheme(testScheme). + WithObjects(httpProxy, upstreamNamespace). + WithStatusSubresource(httpProxy). + Build() + + fakeDownstreamClient := fake.NewClientBuilder(). + WithScheme(testScheme). + WithObjects(downstreamNamespace, downstreamPolicy). + Build() + + reconciler := &HTTPProxyReconciler{ + mgr: &fakeMockManager{cl: fakeClient}, + Config: testConfig, + DownstreamCluster: &fakeCluster{cl: fakeDownstreamClient}, + } + + req := mcreconcile.Request{ + Request: reconcile.Request{ + NamespacedName: client.ObjectKeyFromObject(httpProxy), + }, + ClusterName: "test-cluster", + } + + _, err := reconciler.Reconcile(ctx, req) + assert.NoError(t, err) + + policyList := envoygatewayv1alpha1.EnvoyPatchPolicyList{} + assert.NoError(t, fakeDownstreamClient.List(ctx, &policyList)) + assert.Len(t, policyList.Items, 0) + + updatedProxy := &networkingv1alpha.HTTPProxy{} + err = fakeClient.Get(ctx, client.ObjectKeyFromObject(httpProxy), updatedProxy) + if err == nil { + assert.False(t, controllerutil.ContainsFinalizer(updatedProxy, httpProxyFinalizer)) + } else { + assert.True(t, apierrors.IsNotFound(err)) + } +} + func newHTTPProxy(opts ...func(*networkingv1alpha.HTTPProxy)) *networkingv1alpha.HTTPProxy { p := &networkingv1alpha.HTTPProxy{ ObjectMeta: metav1.ObjectMeta{ diff --git a/internal/validation/httpproxy_validation.go b/internal/validation/httpproxy_validation.go index 48cada21..1dede9a3 100644 --- a/internal/validation/httpproxy_validation.go +++ b/internal/validation/httpproxy_validation.go @@ -102,12 +102,13 @@ func validateHTTPProxyRuleBackend(backend networkingv1alpha.HTTPProxyRuleBackend // See: https://github.com/kubernetes/kubernetes/blob/d21da29c9ec486956b204050cdfaa46c686e29cc/pkg/apis/discovery/validation/validation.go#L115 hostFieldPath := endpointFieldPath.Key("host") host := u.Hostname() + hasConnector := backend.Connector != nil if ip := net.ParseIP(host); ip != nil { // Adapted from https://github.com/kubernetes/kubernetes/blob/d21da29c9ec486956b204050cdfaa46c686e29cc/pkg/apis/core/validation/validation.go#L7797 if ip.IsUnspecified() { allErrs = append(allErrs, field.Invalid(hostFieldPath, host, fmt.Sprintf("may not be unspecified (%v)", host))) } - if ip.IsLoopback() { + if ip.IsLoopback() && !hasConnector { allErrs = append(allErrs, field.Invalid(hostFieldPath, host, "may not be in the loopback range (127.0.0.0/8, ::1/128)")) } if ip.IsLinkLocalUnicast() { @@ -117,7 +118,9 @@ func validateHTTPProxyRuleBackend(backend networkingv1alpha.HTTPProxyRuleBackend allErrs = append(allErrs, field.Invalid(hostFieldPath, host, "may not be in the link-local multicast range (224.0.0.0/24, ff02::/10)")) } } else { - allErrs = append(allErrs, validation.IsFullyQualifiedDomainName(hostFieldPath, host)...) + if !hasConnector || host != "localhost" { + allErrs = append(allErrs, validation.IsFullyQualifiedDomainName(hostFieldPath, host)...) + } } if u.Path != "" { diff --git a/internal/validation/httpproxy_validation_test.go b/internal/validation/httpproxy_validation_test.go index 4801098a..8a57da95 100644 --- a/internal/validation/httpproxy_validation_test.go +++ b/internal/validation/httpproxy_validation_test.go @@ -19,6 +19,83 @@ func TestValidateHTTPProxy(t *testing.T) { proxy *networkingv1alpha.HTTPProxy expectedErrors field.ErrorList }{ + "loopback allowed with connector": { + proxy: &networkingv1alpha.HTTPProxy{ + Spec: networkingv1alpha.HTTPProxySpec{ + Rules: []networkingv1alpha.HTTPProxyRule{ + { + Backends: []networkingv1alpha.HTTPProxyRuleBackend{ + { + Endpoint: "http://127.0.0.1", + Connector: &networkingv1alpha.ConnectorReference{ + Name: "connector-1", + }, + }, + }, + }, + }, + }, + }, + expectedErrors: field.ErrorList{}, + }, + "localhost allowed with connector": { + proxy: &networkingv1alpha.HTTPProxy{ + Spec: networkingv1alpha.HTTPProxySpec{ + Rules: []networkingv1alpha.HTTPProxyRule{ + { + Backends: []networkingv1alpha.HTTPProxyRuleBackend{ + { + Endpoint: "http://localhost", + Connector: &networkingv1alpha.ConnectorReference{ + Name: "connector-1", + }, + }, + }, + }, + }, + }, + }, + expectedErrors: field.ErrorList{}, + }, + "localhost without connector invalid": { + proxy: &networkingv1alpha.HTTPProxy{ + Spec: networkingv1alpha.HTTPProxySpec{ + Rules: []networkingv1alpha.HTTPProxyRule{ + { + Backends: []networkingv1alpha.HTTPProxyRuleBackend{ + { + Endpoint: "http://localhost", + }, + }, + }, + }, + }, + }, + expectedErrors: field.ErrorList{ + field.Invalid(field.NewPath("spec", "rules").Index(0).Child("backends").Index(0).Child("endpoint").Key("host"), "Invalid", ""), + }, + }, + "loopback with empty connector name invalid": { + proxy: &networkingv1alpha.HTTPProxy{ + Spec: networkingv1alpha.HTTPProxySpec{ + Rules: []networkingv1alpha.HTTPProxyRule{ + { + Backends: []networkingv1alpha.HTTPProxyRuleBackend{ + { + Endpoint: "http://127.0.0.1", + Connector: &networkingv1alpha.ConnectorReference{ + Name: "", + }, + }, + }, + }, + }, + }, + }, + expectedErrors: field.ErrorList{ + field.Required(field.NewPath("spec", "rules").Index(0).Child("backends").Index(0).Child("connector", "name"), ""), + }, + }, "connector name required": { proxy: &networkingv1alpha.HTTPProxy{ Spec: networkingv1alpha.HTTPProxySpec{ From 882d7c7e6db650762cc791938600ae877e4c3caa Mon Sep 17 00:00:00 2001 From: Zach Smith Date: Fri, 30 Jan 2026 01:56:46 -0800 Subject: [PATCH 02/24] h2c architecture --- internal/controller/httpproxy_controller.go | 36 +++++---------------- 1 file changed, 8 insertions(+), 28 deletions(-) diff --git a/internal/controller/httpproxy_controller.go b/internal/controller/httpproxy_controller.go index 5c2ced00..914e349b 100644 --- a/internal/controller/httpproxy_controller.go +++ b/internal/controller/httpproxy_controller.go @@ -1005,9 +1005,11 @@ func buildConnectorEnvoyPatches( patches := make([]envoygatewayv1alpha1.EnvoyJSONPatchConfig, 0) // TODO: Make this idempotent headersOp := envoygatewayv1alpha1.JSONPatchOperationType("add") - metadataOp := envoygatewayv1alpha1.JSONPatchOperationType("add") - clusterJSON, err := json.Marshal("internal-tunnel-cluster") + // Connector traffic is routed to the local iroh-gateway instance (sidecar) + // via a bootstrap-defined cluster. The connector-specific tunnel destination + // is selected per request via headers injected below. + clusterJSON, err := json.Marshal("iroh-gateway") if err != nil { return nil, fmt.Errorf("failed to marshal cluster name: %w", err) } @@ -1026,14 +1028,14 @@ func buildConnectorEnvoyPatches( headersJSON, err := json.Marshal([]map[string]any{ { "header": map[string]any{ - "key": "x-tunnel-host", + "key": "x-datum-target-host", "value": backend.targetHost, }, "append_action": "OVERWRITE_IF_EXISTS_OR_ADD", }, { "header": map[string]any{ - "key": "x-tunnel-port", + "key": "x-datum-target-port", "value": strconv.Itoa(backend.targetPort), }, "append_action": "OVERWRITE_IF_EXISTS_OR_ADD", @@ -1050,30 +1052,8 @@ func buildConnectorEnvoyPatches( return nil, fmt.Errorf("failed to marshal request headers: %w", err) } - // Per-route metadata used later by tcp_proxy tunneling_config via %DYNAMIC_METADATA(tunnel:...)%. - tunnelMetaJSON, err := json.Marshal(map[string]any{ - "host": backend.targetHost, - "port": backend.targetPort, - "endpoint_id": backend.nodeID, - }) - if err != nil { - return nil, fmt.Errorf("failed to marshal tunnel metadata: %w", err) - } - patches = append(patches, - // 1) Write per-route metadata under metadata.filter_metadata.tunnel - // (do NOT replace /metadata as a whole, Envoy Gateway uses it too). - envoygatewayv1alpha1.EnvoyJSONPatchConfig{ - Type: "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", - Name: routeConfigName, - Operation: envoygatewayv1alpha1.JSONPatchOperation{ - Op: metadataOp, - JSONPath: ptr.To(jsonPath), - Path: ptr.To("/metadata/filter_metadata/tunnel"), - Value: &apiextensionsv1.JSON{Raw: tunnelMetaJSON}, - }, - }, - // 2) Send matched requests to the internal tunnel listener. + // 1) Send matched requests to the iroh-gateway cluster. envoygatewayv1alpha1.EnvoyJSONPatchConfig{ Type: "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", Name: routeConfigName, @@ -1084,7 +1064,7 @@ func buildConnectorEnvoyPatches( Value: &apiextensionsv1.JSON{Raw: clusterJSON}, }, }, - // 3) Inject internal control headers based on the selected route/backend. + // 2) Inject internal control headers based on the selected route/backend. // The client does not send these. envoygatewayv1alpha1.EnvoyJSONPatchConfig{ Type: "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", From b317a561c49a87a1e9065112b09c94ba786c3c8a Mon Sep 17 00:00:00 2001 From: Zach Smith Date: Fri, 30 Jan 2026 11:26:49 -0800 Subject: [PATCH 03/24] fix: envoy patch ordering --- internal/controller/httpproxy_controller.go | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/internal/controller/httpproxy_controller.go b/internal/controller/httpproxy_controller.go index 914e349b..9ec9ee84 100644 --- a/internal/controller/httpproxy_controller.go +++ b/internal/controller/httpproxy_controller.go @@ -286,7 +286,7 @@ func (r *HTTPProxyReconciler) Reconcile(ctx context.Context, req mcreconcile.Req cl.GetClient(), req.ClusterName, &httpProxy, - desiredResources.gateway, + gateway, ) if err != nil { programmedCondition.Status = metav1.ConditionFalse @@ -794,6 +794,18 @@ func (r *HTTPProxyReconciler) reconcileConnectorEnvoyPatchPolicy( return nil, false, nil } + // Wait for the Gateway to be Programmed before creating the EnvoyPatchPolicy. + // This ensures the RouteConfiguration exists in Envoy's xDS, so the patch + // can be applied immediately rather than waiting for Envoy Gateway to retry. + gatewayProgrammed := apimeta.IsStatusConditionTrue( + gateway.Status.Conditions, + string(gatewayv1.GatewayConditionProgrammed), + ) + if !gatewayProgrammed { + // Gateway not yet programmed; requeue will happen when Gateway status changes. + return nil, true, nil + } + if r.Config.Gateway.DownstreamGatewayClassName == "" { return nil, true, fmt.Errorf("downstreamGatewayClassName is required for connector patching") } From a5d4e479638bd63072899780d1eb182caf16856b Mon Sep 17 00:00:00 2001 From: Zach Smith Date: Fri, 30 Jan 2026 11:27:09 -0800 Subject: [PATCH 04/24] fix: envoy patch ordering --- .../controller/httpproxy_controller_test.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/internal/controller/httpproxy_controller_test.go b/internal/controller/httpproxy_controller_test.go index 8f05a329..f65dcd74 100644 --- a/internal/controller/httpproxy_controller_test.go +++ b/internal/controller/httpproxy_controller_test.go @@ -392,6 +392,15 @@ func TestHTTPProxyReconcile(t *testing.T) { }, }, }, + postCreateGatewayStatus: func(g *gatewayv1.Gateway) { + // EnvoyPatchPolicy is only created after Gateway is Programmed + apimeta.SetStatusCondition(&g.Status.Conditions, metav1.Condition{ + Type: string(gatewayv1.GatewayConditionProgrammed), + Status: metav1.ConditionTrue, + ObservedGeneration: g.Generation, + Reason: string(gatewayv1.GatewayReasonProgrammed), + }) + }, expectedError: false, expectedConditions: []metav1.Condition{ { @@ -442,6 +451,15 @@ func TestHTTPProxyReconcile(t *testing.T) { }, }, }, + postCreateGatewayStatus: func(g *gatewayv1.Gateway) { + // EnvoyPatchPolicy is only created after Gateway is Programmed + apimeta.SetStatusCondition(&g.Status.Conditions, metav1.Condition{ + Type: string(gatewayv1.GatewayConditionProgrammed), + Status: metav1.ConditionTrue, + ObservedGeneration: g.Generation, + Reason: string(gatewayv1.GatewayReasonProgrammed), + }) + }, expectedError: false, expectedConditions: []metav1.Condition{ { From f155b79e7e9a958e6679ec411fdb481892838e4d Mon Sep 17 00:00:00 2001 From: Zach Smith Date: Fri, 30 Jan 2026 17:14:35 -0800 Subject: [PATCH 05/24] fix: watch connectors and trigger httpproxy update --- internal/controller/httpproxy_controller.go | 61 ++++++++++++++++++++- 1 file changed, 60 insertions(+), 1 deletion(-) diff --git a/internal/controller/httpproxy_controller.go b/internal/controller/httpproxy_controller.go index 23a51a0d..75f4a220 100644 --- a/internal/controller/httpproxy_controller.go +++ b/internal/controller/httpproxy_controller.go @@ -27,6 +27,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/cluster" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/log" gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" @@ -508,7 +509,52 @@ func (r *HTTPProxyReconciler) SetupWithManager(mgr mcmanager.Manager) error { For(&networkingv1alpha.HTTPProxy{}). Owns(&gatewayv1.Gateway{}). Owns(&gatewayv1.HTTPRoute{}). - Owns(&discoveryv1.EndpointSlice{}) + Owns(&discoveryv1.EndpointSlice{}). + // Watch Connectors and reconcile HTTPProxies that reference them. + // This ensures EnvoyPatchPolicy headers are updated when a Connector's + // publicKey.id changes (e.g., after connector restart/reconnect). + Watches( + &networkingv1alpha1.Connector{}, + func(clusterName string, cl cluster.Cluster) handler.TypedEventHandler[client.Object, mcreconcile.Request] { + return handler.TypedEnqueueRequestsFromMapFunc(func(ctx context.Context, obj client.Object) []mcreconcile.Request { + logger := log.FromContext(ctx) + + connector, ok := obj.(*networkingv1alpha1.Connector) + if !ok { + return nil + } + + // List all HTTPProxies in the same namespace + var httpProxies networkingv1alpha.HTTPProxyList + if err := cl.GetClient().List(ctx, &httpProxies, client.InNamespace(connector.Namespace)); err != nil { + logger.Error(err, "failed to list HTTPProxies for Connector watch", "connector", connector.Name) + return nil + } + + var requests []mcreconcile.Request + for i := range httpProxies.Items { + httpProxy := &httpProxies.Items[i] + // Check if this HTTPProxy references the changed Connector + if httpProxyReferencesConnector(httpProxy, connector.Name) { + requests = append(requests, mcreconcile.Request{ + ClusterName: clusterName, + Request: ctrl.Request{ + NamespacedName: client.ObjectKeyFromObject(httpProxy), + }, + }) + } + } + + if len(requests) > 0 { + logger.Info("Connector changed, requeueing HTTPProxies", + "connector", connector.Name, + "httpProxyCount", len(requests)) + } + + return requests + }) + }, + ) if r.DownstreamCluster != nil { downstreamPolicySource := mcsource.TypedKind( @@ -523,6 +569,19 @@ func (r *HTTPProxyReconciler) SetupWithManager(mgr mcmanager.Manager) error { return builder.Named("httpproxy").Complete(r) } +// httpProxyReferencesConnector checks if an HTTPProxy has any backends +// that reference the given Connector name. +func httpProxyReferencesConnector(httpProxy *networkingv1alpha.HTTPProxy, connectorName string) bool { + for _, rule := range httpProxy.Spec.Rules { + for _, backend := range rule.Backends { + if backend.Connector != nil && backend.Connector.Name == connectorName { + return true + } + } + } + return false +} + func (r *HTTPProxyReconciler) collectDesiredResources( httpProxy *networkingv1alpha.HTTPProxy, ) (*desiredHTTPProxyResources, error) { From 6334f14dbdc5a9e9f60865e4318a46030032c6b6 Mon Sep 17 00:00:00 2001 From: Zach Smith Date: Wed, 4 Feb 2026 12:42:42 -0800 Subject: [PATCH 06/24] feat: gate connector on class exists --- internal/controller/connector_controller.go | 49 +++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/internal/controller/connector_controller.go b/internal/controller/connector_controller.go index 73033990..683e99fd 100644 --- a/internal/controller/connector_controller.go +++ b/internal/controller/connector_controller.go @@ -16,6 +16,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/cluster" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/log" @@ -105,6 +106,19 @@ func (r *ConnectorReconciler) Reconcile(ctx context.Context, req mcreconcile.Req apimeta.SetStatusCondition(&connector.Status.Conditions, *acceptedCondition) apimeta.SetStatusCondition(&connector.Status.Conditions, *readyCondition) + if acceptedCondition.Status != metav1.ConditionTrue { + readyCondition.Status = metav1.ConditionFalse + readyCondition.Reason = networkingv1alpha1.ConnectorReasonNotReady + readyCondition.Message = "Waiting for ConnectorClass to be resolved." + apimeta.SetStatusCondition(&connector.Status.Conditions, *readyCondition) + if !equality.Semantic.DeepEqual(*originalStatus, connector.Status) { + if statusErr := cl.GetClient().Status().Update(ctx, &connector); statusErr != nil { + return ctrl.Result{}, fmt.Errorf("failed updating connector status: %w", statusErr) + } + } + return ctrl.Result{}, nil + } + leaseDurationSeconds := r.connectorLeaseDurationSeconds() if connector.Status.LeaseRef == nil || connector.Status.LeaseRef.Name == "" { lease := &coordinationv1.Lease{ @@ -214,6 +228,41 @@ func (r *ConnectorReconciler) SetupWithManager(mgr mcmanager.Manager) error { return mcbuilder.ControllerManagedBy(mgr). For(&networkingv1alpha1.Connector{}). + Watches( + &networkingv1alpha1.ConnectorClass{}, + func(clusterName string, cl cluster.Cluster) handler.TypedEventHandler[client.Object, mcreconcile.Request] { + return handler.TypedEnqueueRequestsFromMapFunc(func(ctx context.Context, obj client.Object) []mcreconcile.Request { + logger := log.FromContext(ctx) + + connectorClass, ok := obj.(*networkingv1alpha1.ConnectorClass) + if !ok { + return nil + } + + var connectors networkingv1alpha1.ConnectorList + if err := cl.GetClient().List(ctx, &connectors); err != nil { + logger.Error(err, "failed to list Connectors for ConnectorClass watch", "connectorClass", connectorClass.Name) + return nil + } + + var requests []mcreconcile.Request + for i := range connectors.Items { + connector := &connectors.Items[i] + if connector.Spec.ConnectorClassName != connectorClass.Name { + continue + } + requests = append(requests, mcreconcile.Request{ + ClusterName: clusterName, + Request: ctrl.Request{ + NamespacedName: client.ObjectKeyFromObject(connector), + }, + }) + } + + return requests + }) + }, + ). Watches( &coordinationv1.Lease{}, mchandler.EnqueueRequestForOwner(&networkingv1alpha1.Connector{}, handler.OnlyControllerOwner()), From 47f5708a6f6108d5b33a889c9b730c0fdab3f5d5 Mon Sep 17 00:00:00 2001 From: Zach Smith Date: Wed, 4 Feb 2026 12:50:56 -0800 Subject: [PATCH 07/24] feat: tunnel not online static page --- internal/controller/httpproxy_controller.go | 92 ++++++++++++++++--- .../controller/httpproxy_controller_test.go | 62 +++++++++++++ 2 files changed, 142 insertions(+), 12 deletions(-) diff --git a/internal/controller/httpproxy_controller.go b/internal/controller/httpproxy_controller.go index 75f4a220..d78acc5a 100644 --- a/internal/controller/httpproxy_controller.go +++ b/internal/controller/httpproxy_controller.go @@ -829,6 +829,9 @@ type connectorBackendPatch struct { targetHost string targetPort int nodeID string + + // Whether the referenced Connector is ready to accept traffic. + connectorReady bool } func (r *HTTPProxyReconciler) reconcileConnectorEnvoyPatchPolicy( @@ -1034,18 +1037,19 @@ func collectConnectorBackends( return nil, err } - nodeID, err := connectorNodeID(ctx, cl, httpProxy.Namespace, backend.Connector.Name) + connectorReady, nodeID, err := connectorPatchDetails(ctx, cl, httpProxy.Namespace, backend.Connector.Name) if err != nil { return nil, err } connectorBackends = append(connectorBackends, connectorBackendPatch{ - sectionName: nil, - ruleIndex: ruleIndex, - matchIndex: matchIndex, - targetHost: targetHost, - targetPort: targetPort, - nodeID: nodeID, + sectionName: nil, + ruleIndex: ruleIndex, + matchIndex: matchIndex, + targetHost: targetHost, + targetPort: targetPort, + nodeID: nodeID, + connectorReady: connectorReady, }) } } @@ -1078,20 +1082,25 @@ func backendEndpointTarget(backend networkingv1alpha.HTTPProxyRuleBackend) (stri return targetHost, targetPort, nil } -func connectorNodeID(ctx context.Context, cl client.Client, namespace, name string) (string, error) { +func connectorPatchDetails(ctx context.Context, cl client.Client, namespace, name string) (bool, string, error) { var connector networkingv1alpha1.Connector if err := cl.Get(ctx, client.ObjectKey{Namespace: namespace, Name: name}, &connector); err != nil { - return "", err + return false, "", err + } + + ready := apimeta.IsStatusConditionTrue(connector.Status.Conditions, networkingv1alpha1.ConnectorConditionReady) + if !ready { + return false, "", nil } details := connector.Status.ConnectionDetails if details == nil || details.Type != networkingv1alpha1.PublicKeyConnectorConnectionType || details.PublicKey == nil { - return "", fmt.Errorf("connector %q does not have public key connection details", name) + return false, "", fmt.Errorf("connector %q does not have public key connection details", name) } if details.PublicKey.Id == "" { - return "", fmt.Errorf("connector %q public key id is empty", name) + return false, "", fmt.Errorf("connector %q public key id is empty", name) } - return details.PublicKey.Id, nil + return true, details.PublicKey.Id, nil } func buildConnectorEnvoyPatches( @@ -1113,6 +1122,29 @@ func buildConnectorEnvoyPatches( return nil, fmt.Errorf("failed to marshal cluster name: %w", err) } + directResponseJSON, err := json.Marshal(map[string]any{ + "status": 503, + "body": map[string]any{ + "inline_string": "Tunnel not online", + }, + }) + if err != nil { + return nil, fmt.Errorf("failed to marshal direct response body: %w", err) + } + + responseHeadersJSON, err := json.Marshal([]map[string]any{ + { + "header": map[string]any{ + "key": "content-type", + "value": "text/plain; charset=utf-8", + }, + "append_action": "OVERWRITE_IF_EXISTS_OR_ADD", + }, + }) + if err != nil { + return nil, fmt.Errorf("failed to marshal response headers: %w", err) + } + for _, routeConfigName := range routeConfigNames { for _, backend := range backends { jsonPath := connectorRouteJSONPath( @@ -1124,6 +1156,42 @@ func buildConnectorEnvoyPatches( backend.matchIndex, ) + if !backend.connectorReady { + patches = append(patches, + // Replace routing with a direct response when the connector is offline. + envoygatewayv1alpha1.EnvoyJSONPatchConfig{ + Type: "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + Name: routeConfigName, + Operation: envoygatewayv1alpha1.JSONPatchOperation{ + Op: envoygatewayv1alpha1.JSONPatchOperationType("remove"), + JSONPath: ptr.To(jsonPath), + Path: ptr.To("/route"), + }, + }, + envoygatewayv1alpha1.EnvoyJSONPatchConfig{ + Type: "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + Name: routeConfigName, + Operation: envoygatewayv1alpha1.JSONPatchOperation{ + Op: headersOp, + JSONPath: ptr.To(jsonPath), + Path: ptr.To("/direct_response"), + Value: &apiextensionsv1.JSON{Raw: directResponseJSON}, + }, + }, + envoygatewayv1alpha1.EnvoyJSONPatchConfig{ + Type: "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + Name: routeConfigName, + Operation: envoygatewayv1alpha1.JSONPatchOperation{ + Op: headersOp, + JSONPath: ptr.To(jsonPath), + Path: ptr.To("/response_headers_to_add"), + Value: &apiextensionsv1.JSON{Raw: responseHeadersJSON}, + }, + }, + ) + continue + } + headersJSON, err := json.Marshal([]map[string]any{ { "header": map[string]any{ diff --git a/internal/controller/httpproxy_controller_test.go b/internal/controller/httpproxy_controller_test.go index 0322ca82..8578eb81 100644 --- a/internal/controller/httpproxy_controller_test.go +++ b/internal/controller/httpproxy_controller_test.go @@ -1,6 +1,7 @@ package controller import ( + "bytes" "context" "fmt" "testing" @@ -477,6 +478,67 @@ func TestHTTPProxyReconcile(t *testing.T) { assert.Equal(t, fmt.Sprintf("connector-%s", httpProxy.Name), patchList.Items[0].Name) }, }, + { + name: "connector not ready uses direct response", + httpProxy: connectorHTTPProxy, + downstreamObjects: connectorDownstreamObjects(connectorHTTPProxy), + namespaceUID: string(connectorNamespaceUID), + existingObjects: []client.Object{ + &networkingv1alpha1.Connector{ + ObjectMeta: metav1.ObjectMeta{ + Name: "connector-1", + Namespace: "test", + }, + Status: networkingv1alpha1.ConnectorStatus{ + Conditions: []metav1.Condition{ + { + Type: networkingv1alpha1.ConnectorConditionReady, + Status: metav1.ConditionFalse, + Reason: networkingv1alpha1.ConnectorReasonNotReady, + }, + }, + }, + }, + }, + postCreateGatewayStatus: func(g *gatewayv1.Gateway) { + apimeta.SetStatusCondition(&g.Status.Conditions, metav1.Condition{ + Type: string(gatewayv1.GatewayConditionProgrammed), + Status: metav1.ConditionTrue, + ObservedGeneration: g.Generation, + Reason: string(gatewayv1.GatewayReasonProgrammed), + }) + }, + expectedError: false, + expectedConditions: []metav1.Condition{ + { + Type: networkingv1alpha.HTTPProxyConditionAccepted, + Status: metav1.ConditionTrue, + Reason: networkingv1alpha.HTTPProxyReasonAccepted, + }, + { + Type: networkingv1alpha.HTTPProxyConditionProgrammed, + Status: metav1.ConditionFalse, + Reason: networkingv1alpha.HTTPProxyReasonPending, + }, + }, + assert: func(t *testContext, cl client.Client, httpProxy *networkingv1alpha.HTTPProxy) { + var patchList envoygatewayv1alpha1.EnvoyPatchPolicyList + err := t.downstreamClient.List(context.Background(), &patchList) + assert.NoError(t, err) + if assert.Len(t, patchList.Items, 1) { + found := false + for _, patch := range patchList.Items[0].Spec.JSONPatches { + if ptr.Deref(patch.Operation.Path, "") == "/direct_response" && + patch.Operation.Value != nil && + bytes.Contains(patch.Operation.Value.Raw, []byte("Tunnel not online")) { + found = true + break + } + } + assert.True(t, found, "expected direct response patch for connector not ready") + } + }, + }, { name: "connector patch policy removed when connector cleared", httpProxy: connectorClearedHTTPProxy, From 85fc986c224f85a0005b8bac9b205fb451a1fa25 Mon Sep 17 00:00:00 2001 From: Zach Smith Date: Wed, 4 Feb 2026 13:17:31 -0800 Subject: [PATCH 08/24] fix: use httproutefilter instead for tunnel not online --- config/rbac/role.yaml | 12 + internal/controller/httpproxy_controller.go | 213 +++++++++++------- .../controller/httpproxy_controller_test.go | 49 +++- 3 files changed, 180 insertions(+), 94 deletions(-) diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 1d4f7bba..42039773 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -119,6 +119,18 @@ rules: - get - patch - update +- apiGroups: + - gateway.envoyproxy.io + resources: + - httproutefilters + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - gateway.networking.k8s.io resources: diff --git a/internal/controller/httpproxy_controller.go b/internal/controller/httpproxy_controller.go index d78acc5a..1587ca9e 100644 --- a/internal/controller/httpproxy_controller.go +++ b/internal/controller/httpproxy_controller.go @@ -8,6 +8,7 @@ import ( "errors" "fmt" "net" + "net/http" "net/url" "slices" "strconv" @@ -53,12 +54,14 @@ type HTTPProxyReconciler struct { } type desiredHTTPProxyResources struct { - gateway *gatewayv1.Gateway - httpRoute *gatewayv1.HTTPRoute - endpointSlices []*discoveryv1.EndpointSlice + gateway *gatewayv1.Gateway + httpRoute *gatewayv1.HTTPRoute + endpointSlices []*discoveryv1.EndpointSlice + httpRouteFilters []*envoygatewayv1alpha1.HTTPRouteFilter } const httpProxyFinalizer = "networking.datumapis.com/httpproxy-cleanup" +const connectorOfflineFilterPrefix = "connector-offline" const ( SchemeHTTP = "http" @@ -72,6 +75,7 @@ const ( // +kubebuilder:rbac:groups=networking.datumapis.com,resources=httpproxies/status,verbs=get;update;patch // +kubebuilder:rbac:groups=networking.datumapis.com,resources=httpproxies/finalizers,verbs=update // +kubebuilder:rbac:groups=networking.datumapis.com,resources=connectors,verbs=get;list;watch +// +kubebuilder:rbac:groups=gateway.envoyproxy.io,resources=httproutefilters,verbs=get;list;watch;create;update;patch;delete func (r *HTTPProxyReconciler) Reconcile(ctx context.Context, req mcreconcile.Request) (_ ctrl.Result, err error) { logger := log.FromContext(ctx, "cluster", req.ClusterName) @@ -166,7 +170,7 @@ func (r *HTTPProxyReconciler) Reconcile(ctx context.Context, req mcreconcile.Req } }() - desiredResources, err := r.collectDesiredResources(&httpProxy) + desiredResources, err := r.collectDesiredResources(ctx, cl.GetClient(), &httpProxy) if err != nil { return ctrl.Result{}, fmt.Errorf("failed to collect desired resources: %w", err) } @@ -219,6 +223,27 @@ func (r *HTTPProxyReconciler) Reconcile(ctx context.Context, req mcreconcile.Req // Maintain an HTTPRoute for all rules in the HTTPProxy + if len(desiredResources.httpRouteFilters) == 0 { + if err := cleanupConnectorOfflineHTTPRouteFilter(ctx, cl.GetClient(), &httpProxy); err != nil { + return ctrl.Result{}, err + } + } else { + for _, desiredFilter := range desiredResources.httpRouteFilters { + httpRouteFilter := desiredFilter.DeepCopy() + result, err := controllerutil.CreateOrUpdate(ctx, cl.GetClient(), httpRouteFilter, func() error { + if err := controllerutil.SetControllerReference(&httpProxy, httpRouteFilter, cl.GetScheme()); err != nil { + return fmt.Errorf("failed to set controller on HTTPRouteFilter: %w", err) + } + httpRouteFilter.Spec = desiredFilter.Spec + return nil + }) + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed updating httproutefilter resource: %w", err) + } + logger.Info("processed httproutefilter", "name", httpRouteFilter.Name, "result", result) + } + } + httpRoute := desiredResources.httpRoute.DeepCopy() result, err = controllerutil.CreateOrUpdate(ctx, cl.GetClient(), httpRoute, func() error { @@ -583,6 +608,8 @@ func httpProxyReferencesConnector(httpProxy *networkingv1alpha.HTTPProxy, connec } func (r *HTTPProxyReconciler) collectDesiredResources( + ctx context.Context, + cl client.Client, httpProxy *networkingv1alpha.HTTPProxy, ) (*desiredHTTPProxyResources, error) { @@ -649,10 +676,13 @@ func (r *HTTPProxyReconciler) collectDesiredResources( } var desiredEndpointSlices []*discoveryv1.EndpointSlice + var desiredRouteFilters []*envoygatewayv1alpha1.HTTPRouteFilter desiredRouteRules := make([]gatewayv1.HTTPRouteRule, len(httpProxy.Spec.Rules)) for ruleIndex, rule := range httpProxy.Spec.Rules { + ruleFilters := slices.Clone(rule.Filters) backendRefs := make([]gatewayv1.HTTPBackendRef, len(rule.Backends)) + offlineRuleSet := false // Validation will prevent this from occurring, unless the maximum items for // backends is adjusted. The following error has been placed here so that @@ -663,6 +693,35 @@ func (r *HTTPProxyReconciler) collectDesiredResources( } for backendIndex, backend := range rule.Backends { + if backend.Connector != nil { + ready, err := connectorReady(ctx, cl, httpProxy.Namespace, backend.Connector.Name) + if err != nil { + return nil, err + } + if !ready { + filterName := connectorOfflineFilterName(httpProxy) + ruleFilters = append(ruleFilters, gatewayv1.HTTPRouteFilter{ + Type: gatewayv1.HTTPRouteFilterExtensionRef, + ExtensionRef: &gatewayv1.LocalObjectReference{ + Group: envoygatewayv1alpha1.GroupName, + Kind: envoygatewayv1alpha1.KindHTTPRouteFilter, + Name: gatewayv1.ObjectName(filterName), + }, + }) + desiredRouteRules[ruleIndex] = gatewayv1.HTTPRouteRule{ + Name: rule.Name, + Matches: rule.Matches, + Filters: ruleFilters, + BackendRefs: nil, + } + if len(desiredRouteFilters) == 0 { + desiredRouteFilters = append(desiredRouteFilters, buildConnectorOfflineHTTPRouteFilter(httpProxy)) + } + offlineRuleSet = true + break + } + } + appProtocol := SchemeHTTP backendPort := DefaultHTTPPort @@ -714,15 +773,15 @@ func (r *HTTPProxyReconciler) collectDesiredResources( } // Use tls.hostname for the Host header rewrite hostnameRewriteFound := false - for i, filter := range rule.Filters { + for i, filter := range ruleFilters { if filter.Type == gatewayv1.HTTPRouteFilterURLRewrite { - rule.Filters[i].URLRewrite.Hostname = ptr.To(gatewayv1.PreciseHostname(*backend.TLS.Hostname)) + ruleFilters[i].URLRewrite.Hostname = ptr.To(gatewayv1.PreciseHostname(*backend.TLS.Hostname)) hostnameRewriteFound = true break } } if !hostnameRewriteFound { - rule.Filters = append(rule.Filters, gatewayv1.HTTPRouteFilter{ + ruleFilters = append(ruleFilters, gatewayv1.HTTPRouteFilter{ Type: gatewayv1.HTTPRouteFilterURLRewrite, URLRewrite: &gatewayv1.HTTPURLRewriteFilter{ Hostname: ptr.To(gatewayv1.PreciseHostname(*backend.TLS.Hostname)), @@ -732,16 +791,16 @@ func (r *HTTPProxyReconciler) collectDesiredResources( } else if !isIPAddress && backend.Connector == nil { // For FQDN endpoints, rewrite the Host header to match the backend hostname hostnameRewriteFound := false - for i, filter := range rule.Filters { + for i, filter := range ruleFilters { if filter.Type == gatewayv1.HTTPRouteFilterURLRewrite { - rule.Filters[i].URLRewrite.Hostname = ptr.To(gatewayv1.PreciseHostname(host)) + ruleFilters[i].URLRewrite.Hostname = ptr.To(gatewayv1.PreciseHostname(host)) hostnameRewriteFound = true break } } if !hostnameRewriteFound { - rule.Filters = append(rule.Filters, gatewayv1.HTTPRouteFilter{ + ruleFilters = append(ruleFilters, gatewayv1.HTTPRouteFilter{ Type: gatewayv1.HTTPRouteFilterURLRewrite, URLRewrite: &gatewayv1.HTTPURLRewriteFilter{ Hostname: ptr.To(gatewayv1.PreciseHostname(host)), @@ -793,10 +852,14 @@ func (r *HTTPProxyReconciler) collectDesiredResources( } } + if offlineRuleSet { + continue + } + desiredRouteRules[ruleIndex] = gatewayv1.HTTPRouteRule{ Name: rule.Name, Matches: rule.Matches, - Filters: rule.Filters, + Filters: ruleFilters, BackendRefs: backendRefs, } } @@ -804,9 +867,10 @@ func (r *HTTPProxyReconciler) collectDesiredResources( httpRoute.Spec.Rules = desiredRouteRules return &desiredHTTPProxyResources{ - gateway: gateway, - httpRoute: httpRoute, - endpointSlices: desiredEndpointSlices, + gateway: gateway, + httpRoute: httpRoute, + endpointSlices: desiredEndpointSlices, + httpRouteFilters: desiredRouteFilters, }, nil } @@ -829,9 +893,6 @@ type connectorBackendPatch struct { targetHost string targetPort int nodeID string - - // Whether the referenced Connector is ready to accept traffic. - connectorReady bool } func (r *HTTPProxyReconciler) reconcileConnectorEnvoyPatchPolicy( @@ -974,6 +1035,18 @@ func (r *HTTPProxyReconciler) cleanupConnectorEnvoyPatchPolicy( return downstreamStrategy.DeleteAnchorForObject(ctx, httpProxy) } +func cleanupConnectorOfflineHTTPRouteFilter(ctx context.Context, cl client.Client, httpProxy *networkingv1alpha.HTTPProxy) error { + filterKey := client.ObjectKey{Namespace: httpProxy.Namespace, Name: connectorOfflineFilterName(httpProxy)} + var filter envoygatewayv1alpha1.HTTPRouteFilter + if err := cl.Get(ctx, filterKey, &filter); err != nil { + if apierrors.IsNotFound(err) { + return nil + } + return err + } + return cl.Delete(ctx, &filter) +} + func downstreamPatchPolicyReady(policy *envoygatewayv1alpha1.EnvoyPatchPolicy, gatewayClassName string) (bool, string) { if policy == nil { return false, "Downstream EnvoyPatchPolicy not found" @@ -1041,15 +1114,17 @@ func collectConnectorBackends( if err != nil { return nil, err } + if !connectorReady { + continue + } connectorBackends = append(connectorBackends, connectorBackendPatch{ - sectionName: nil, - ruleIndex: ruleIndex, - matchIndex: matchIndex, - targetHost: targetHost, - targetPort: targetPort, - nodeID: nodeID, - connectorReady: connectorReady, + sectionName: nil, + ruleIndex: ruleIndex, + matchIndex: matchIndex, + targetHost: targetHost, + targetPort: targetPort, + nodeID: nodeID, }) } } @@ -1082,6 +1157,37 @@ func backendEndpointTarget(backend networkingv1alpha.HTTPProxyRuleBackend) (stri return targetHost, targetPort, nil } +func connectorOfflineFilterName(httpProxy *networkingv1alpha.HTTPProxy) string { + return fmt.Sprintf("%s-%s", connectorOfflineFilterPrefix, httpProxy.Name) +} + +func buildConnectorOfflineHTTPRouteFilter(httpProxy *networkingv1alpha.HTTPProxy) *envoygatewayv1alpha1.HTTPRouteFilter { + return &envoygatewayv1alpha1.HTTPRouteFilter{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: httpProxy.Namespace, + Name: connectorOfflineFilterName(httpProxy), + }, + Spec: envoygatewayv1alpha1.HTTPRouteFilterSpec{ + DirectResponse: &envoygatewayv1alpha1.HTTPDirectResponseFilter{ + ContentType: ptr.To("text/plain; charset=utf-8"), + StatusCode: ptr.To(http.StatusServiceUnavailable), + Body: &envoygatewayv1alpha1.CustomResponseBody{ + Type: ptr.To(envoygatewayv1alpha1.ResponseValueTypeInline), + Inline: ptr.To("Tunnel not online"), + }, + }, + }, + } +} + +func connectorReady(ctx context.Context, cl client.Client, namespace, name string) (bool, error) { + var connector networkingv1alpha1.Connector + if err := cl.Get(ctx, client.ObjectKey{Namespace: namespace, Name: name}, &connector); err != nil { + return false, err + } + return apimeta.IsStatusConditionTrue(connector.Status.Conditions, networkingv1alpha1.ConnectorConditionReady), nil +} + func connectorPatchDetails(ctx context.Context, cl client.Client, namespace, name string) (bool, string, error) { var connector networkingv1alpha1.Connector if err := cl.Get(ctx, client.ObjectKey{Namespace: namespace, Name: name}, &connector); err != nil { @@ -1122,29 +1228,6 @@ func buildConnectorEnvoyPatches( return nil, fmt.Errorf("failed to marshal cluster name: %w", err) } - directResponseJSON, err := json.Marshal(map[string]any{ - "status": 503, - "body": map[string]any{ - "inline_string": "Tunnel not online", - }, - }) - if err != nil { - return nil, fmt.Errorf("failed to marshal direct response body: %w", err) - } - - responseHeadersJSON, err := json.Marshal([]map[string]any{ - { - "header": map[string]any{ - "key": "content-type", - "value": "text/plain; charset=utf-8", - }, - "append_action": "OVERWRITE_IF_EXISTS_OR_ADD", - }, - }) - if err != nil { - return nil, fmt.Errorf("failed to marshal response headers: %w", err) - } - for _, routeConfigName := range routeConfigNames { for _, backend := range backends { jsonPath := connectorRouteJSONPath( @@ -1156,42 +1239,6 @@ func buildConnectorEnvoyPatches( backend.matchIndex, ) - if !backend.connectorReady { - patches = append(patches, - // Replace routing with a direct response when the connector is offline. - envoygatewayv1alpha1.EnvoyJSONPatchConfig{ - Type: "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", - Name: routeConfigName, - Operation: envoygatewayv1alpha1.JSONPatchOperation{ - Op: envoygatewayv1alpha1.JSONPatchOperationType("remove"), - JSONPath: ptr.To(jsonPath), - Path: ptr.To("/route"), - }, - }, - envoygatewayv1alpha1.EnvoyJSONPatchConfig{ - Type: "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", - Name: routeConfigName, - Operation: envoygatewayv1alpha1.JSONPatchOperation{ - Op: headersOp, - JSONPath: ptr.To(jsonPath), - Path: ptr.To("/direct_response"), - Value: &apiextensionsv1.JSON{Raw: directResponseJSON}, - }, - }, - envoygatewayv1alpha1.EnvoyJSONPatchConfig{ - Type: "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", - Name: routeConfigName, - Operation: envoygatewayv1alpha1.JSONPatchOperation{ - Op: headersOp, - JSONPath: ptr.To(jsonPath), - Path: ptr.To("/response_headers_to_add"), - Value: &apiextensionsv1.JSON{Raw: responseHeadersJSON}, - }, - }, - ) - continue - } - headersJSON, err := json.Marshal([]map[string]any{ { "header": map[string]any{ diff --git a/internal/controller/httpproxy_controller_test.go b/internal/controller/httpproxy_controller_test.go index 8578eb81..6b07aaee 100644 --- a/internal/controller/httpproxy_controller_test.go +++ b/internal/controller/httpproxy_controller_test.go @@ -1,7 +1,6 @@ package controller import ( - "bytes" "context" "fmt" "testing" @@ -296,7 +295,8 @@ func TestHTTPProxyCollectDesiredResources(t *testing.T) { t.Run(tt.name, func(t *testing.T) { reconciler := &HTTPProxyReconciler{Config: operatorConfig} - desiredResources, err := reconciler.collectDesiredResources(tt.httpProxy) + cl := fake.NewClientBuilder().WithScheme(scheme.Scheme).Build() + desiredResources, err := reconciler.collectDesiredResources(context.Background(), cl, tt.httpProxy) if tt.expectError != "" { assert.Error(t, err) @@ -431,6 +431,13 @@ func TestHTTPProxyReconcile(t *testing.T) { Namespace: "test", }, Status: networkingv1alpha1.ConnectorStatus{ + Conditions: []metav1.Condition{ + { + Type: networkingv1alpha1.ConnectorConditionReady, + Status: metav1.ConditionTrue, + Reason: networkingv1alpha1.ConnectorReasonReady, + }, + }, ConnectionDetails: &networkingv1alpha1.ConnectorConnectionDetails{ Type: networkingv1alpha1.PublicKeyConnectorConnectionType, PublicKey: &networkingv1alpha1.ConnectorConnectionDetailsPublicKey{ @@ -517,25 +524,38 @@ func TestHTTPProxyReconcile(t *testing.T) { }, { Type: networkingv1alpha.HTTPProxyConditionProgrammed, - Status: metav1.ConditionFalse, - Reason: networkingv1alpha.HTTPProxyReasonPending, + Status: metav1.ConditionTrue, + Reason: networkingv1alpha.HTTPProxyReasonProgrammed, }, }, assert: func(t *testContext, cl client.Client, httpProxy *networkingv1alpha.HTTPProxy) { + ctx := context.Background() + var patchList envoygatewayv1alpha1.EnvoyPatchPolicyList - err := t.downstreamClient.List(context.Background(), &patchList) + err := t.downstreamClient.List(ctx, &patchList) assert.NoError(t, err) - if assert.Len(t, patchList.Items, 1) { + assert.Len(t, patchList.Items, 0) + + httpRouteFilter := &envoygatewayv1alpha1.HTTPRouteFilter{} + filterKey := client.ObjectKey{Namespace: httpProxy.Namespace, Name: connectorOfflineFilterName(httpProxy)} + assert.NoError(t, cl.Get(ctx, filterKey, httpRouteFilter)) + assert.Equal(t, "Tunnel not online", ptr.Deref(httpRouteFilter.Spec.DirectResponse.Body.Inline, "")) + + httpRoute := &gatewayv1.HTTPRoute{} + assert.NoError(t, cl.Get(ctx, client.ObjectKeyFromObject(httpProxy), httpRoute)) + if assert.Len(t, httpRoute.Spec.Rules, 1) { + assert.Empty(t, httpRoute.Spec.Rules[0].BackendRefs) found := false - for _, patch := range patchList.Items[0].Spec.JSONPatches { - if ptr.Deref(patch.Operation.Path, "") == "/direct_response" && - patch.Operation.Value != nil && - bytes.Contains(patch.Operation.Value.Raw, []byte("Tunnel not online")) { + for _, filter := range httpRoute.Spec.Rules[0].Filters { + if filter.Type == gatewayv1.HTTPRouteFilterExtensionRef && + filter.ExtensionRef != nil && + filter.ExtensionRef.Kind == envoygatewayv1alpha1.KindHTTPRouteFilter && + filter.ExtensionRef.Name == gatewayv1.ObjectName(httpRouteFilter.Name) { found = true break } } - assert.True(t, found, "expected direct response patch for connector not ready") + assert.True(t, found, "expected HTTPRouteFilter extension ref on rule") } }, }, @@ -551,6 +571,13 @@ func TestHTTPProxyReconcile(t *testing.T) { Namespace: "test", }, Status: networkingv1alpha1.ConnectorStatus{ + Conditions: []metav1.Condition{ + { + Type: networkingv1alpha1.ConnectorConditionReady, + Status: metav1.ConditionTrue, + Reason: networkingv1alpha1.ConnectorReasonReady, + }, + }, ConnectionDetails: &networkingv1alpha1.ConnectorConnectionDetails{ Type: networkingv1alpha1.PublicKeyConnectorConnectionType, PublicKey: &networkingv1alpha1.ConnectorConnectionDetailsPublicKey{ From b709d477d0a6cb875769ec85ba1f2f0058289e34 Mon Sep 17 00:00:00 2001 From: Zach Smith Date: Mon, 9 Feb 2026 21:00:35 -0800 Subject: [PATCH 09/24] feat: allow CONNECT upgrade for route connectors --- internal/controller/httpproxy_controller.go | 22 +++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/internal/controller/httpproxy_controller.go b/internal/controller/httpproxy_controller.go index 1587ca9e..65a2398e 100644 --- a/internal/controller/httpproxy_controller.go +++ b/internal/controller/httpproxy_controller.go @@ -1228,6 +1228,28 @@ func buildConnectorEnvoyPatches( return nil, fmt.Errorf("failed to marshal cluster name: %w", err) } + // Enable CONNECT (and thus Extended CONNECT / WebSocket) on HTTPS listeners by + // patching the HCM upgrade_configs. Doing this via EnvoyPatchPolicy avoids + // gateway-level BackendTrafficPolicy, which can break route matching and cause + // 404s when combined with connector route patches (iroh-gateway cluster). + for _, listenerName := range routeConfigNames { + upgradeConfigsJSON, err := json.Marshal([]map[string]any{ + {"upgrade_type": "CONNECT"}, + }) + if err != nil { + return nil, fmt.Errorf("failed to marshal upgrade_configs: %w", err) + } + patches = append(patches, envoygatewayv1alpha1.EnvoyJSONPatchConfig{ + Type: "type.googleapis.com/envoy.config.listener.v3.Listener", + Name: listenerName, + Operation: envoygatewayv1alpha1.JSONPatchOperation{ + Op: envoygatewayv1alpha1.JSONPatchOperationType("add"), + Path: ptr.To("/default_filter_chain/filters/0/typed_config/upgrade_configs"), + Value: &apiextensionsv1.JSON{Raw: upgradeConfigsJSON}, + }, + }) + } + for _, routeConfigName := range routeConfigNames { for _, backend := range backends { jsonPath := connectorRouteJSONPath( From c532f3a5eed434aa3bcf616600d0aca79e1332a3 Mon Sep 17 00:00:00 2001 From: Zach Smith Date: Mon, 9 Feb 2026 21:03:52 -0800 Subject: [PATCH 10/24] fix: lint --- internal/controller/httpproxy_controller.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/controller/httpproxy_controller.go b/internal/controller/httpproxy_controller.go index 65a2398e..a09ef341 100644 --- a/internal/controller/httpproxy_controller.go +++ b/internal/controller/httpproxy_controller.go @@ -1243,8 +1243,8 @@ func buildConnectorEnvoyPatches( Type: "type.googleapis.com/envoy.config.listener.v3.Listener", Name: listenerName, Operation: envoygatewayv1alpha1.JSONPatchOperation{ - Op: envoygatewayv1alpha1.JSONPatchOperationType("add"), - Path: ptr.To("/default_filter_chain/filters/0/typed_config/upgrade_configs"), + Op: envoygatewayv1alpha1.JSONPatchOperationType("add"), + Path: ptr.To("/default_filter_chain/filters/0/typed_config/upgrade_configs"), Value: &apiextensionsv1.JSON{Raw: upgradeConfigsJSON}, }, }) From 0ec5fba1fd8db8db4e41096029c526f7b7f33ade Mon Sep 17 00:00:00 2001 From: Zach Smith Date: Tue, 10 Feb 2026 17:39:03 -0800 Subject: [PATCH 11/24] feat: patch route cluster lb metadata --- internal/controller/httpproxy_controller.go | 166 +++++++++++--------- 1 file changed, 90 insertions(+), 76 deletions(-) diff --git a/internal/controller/httpproxy_controller.go b/internal/controller/httpproxy_controller.go index a09ef341..db9e4129 100644 --- a/internal/controller/httpproxy_controller.go +++ b/internal/controller/httpproxy_controller.go @@ -1209,6 +1209,77 @@ func connectorPatchDetails(ctx context.Context, cl client.Client, namespace, nam return true, details.PublicKey.Id, nil } +// connectorInternalListenerName is the name of the static internal listener +// (added via EnvoyProxy bootstrap) that runs TcpProxy and sends CONNECT to +// iroh-gateway using dynamic metadata for hostname and x-iroh-endpoint-id. +const connectorInternalListenerName = "connector-tunnel" + +// connectorClusterName returns the cluster name Envoy Gateway assigns to the +// HTTPRoute backend for the given rule. Must match Envoy Gateway's naming so we +// can patch that cluster to point at the internal listener. +func connectorClusterName(downstreamNamespace, httpRouteName string, ruleIndex int) string { + return fmt.Sprintf("httproute/%s/%s/rule/%d", downstreamNamespace, httpRouteName, ruleIndex) +} + +// buildConnectorInternalListenerClusterJSON returns the Envoy cluster config +// JSON that points at the internal listener "connector-tunnel" with endpoint +// metadata (tunnel.address, tunnel.endpoint_id). InternalUpstreamTransport +// copies that metadata to the internal connection so TcpProxy can use +// %DYNAMIC_METADATA(tunnel:address)% and tunnel:endpoint_id for CONNECT. +func buildConnectorInternalListenerClusterJSON(clusterName string, backend connectorBackendPatch) ([]byte, error) { + tunnelAddress := fmt.Sprintf("%s:%d", backend.targetHost, backend.targetPort) + cluster := map[string]any{ + "name": clusterName, + "type": "STATIC", + "connect_timeout": "5s", + "load_assignment": map[string]any{ + "cluster_name": clusterName, + "endpoints": []map[string]any{ + { + "lb_endpoints": []map[string]any{ + { + "endpoint": map[string]any{ + "address": map[string]any{ + "envoy_internal_address": map[string]any{ + "server_listener_name": connectorInternalListenerName, + }, + }, + }, + "metadata": map[string]any{ + "filter_metadata": map[string]any{ + "tunnel": map[string]any{ + "address": tunnelAddress, + "endpoint_id": backend.nodeID, + }, + }, + }, + }, + }, + }, + }, + }, + "transport_socket": map[string]any{ + "name": "envoy.transport_sockets.internal_upstream", + "typed_config": map[string]any{ + "@type": "type.googleapis.com/envoy.extensions.transport_sockets.internal_upstream.v3.InternalUpstreamTransport", + "passthrough_metadata": []map[string]any{ + { + "kind": map[string]any{"host": map[string]any{}}, + "name": "tunnel", + }, + }, + "transport_socket": map[string]any{ + "name": "envoy.transport_sockets.raw_buffer", + "typed_config": map[string]any{ + "@type": "type.googleapis.com/envoy.extensions.transport_sockets.raw_buffer.v3.RawBuffer", + }, + }, + }, + }, + } + return json.Marshal(cluster) +} + func buildConnectorEnvoyPatches( downstreamNamespace string, gateway *gatewayv1.Gateway, @@ -1218,20 +1289,8 @@ func buildConnectorEnvoyPatches( routeConfigNames := connectorRouteConfigNames(downstreamNamespace, gateway) patches := make([]envoygatewayv1alpha1.EnvoyJSONPatchConfig, 0) // TODO: Make this idempotent - headersOp := envoygatewayv1alpha1.JSONPatchOperationType("add") - - // Connector traffic is routed to the local iroh-gateway instance (sidecar) - // via a bootstrap-defined cluster. The connector-specific tunnel destination - // is selected per request via headers injected below. - clusterJSON, err := json.Marshal("iroh-gateway") - if err != nil { - return nil, fmt.Errorf("failed to marshal cluster name: %w", err) - } - // Enable CONNECT (and thus Extended CONNECT / WebSocket) on HTTPS listeners by - // patching the HCM upgrade_configs. Doing this via EnvoyPatchPolicy avoids - // gateway-level BackendTrafficPolicy, which can break route matching and cause - // 404s when combined with connector route patches (iroh-gateway cluster). + // 1) Listener patch: enable CONNECT (and Extended CONNECT / WebSocket) on HTTPS listeners. for _, listenerName := range routeConfigNames { upgradeConfigsJSON, err := json.Marshal([]map[string]any{ {"upgrade_type": "CONNECT"}, @@ -1250,70 +1309,25 @@ func buildConnectorEnvoyPatches( }) } - for _, routeConfigName := range routeConfigNames { - for _, backend := range backends { - jsonPath := connectorRouteJSONPath( - downstreamNamespace, - gateway, - httpProxy.Name, - backend.sectionName, - backend.ruleIndex, - backend.matchIndex, - ) - - headersJSON, err := json.Marshal([]map[string]any{ - { - "header": map[string]any{ - "key": "x-datum-target-host", - "value": backend.targetHost, - }, - "append_action": "OVERWRITE_IF_EXISTS_OR_ADD", - }, - { - "header": map[string]any{ - "key": "x-datum-target-port", - "value": strconv.Itoa(backend.targetPort), - }, - "append_action": "OVERWRITE_IF_EXISTS_OR_ADD", - }, - { - "header": map[string]any{ - "key": "x-iroh-endpoint-id", - "value": backend.nodeID, - }, - "append_action": "OVERWRITE_IF_EXISTS_OR_ADD", - }, - }) - if err != nil { - return nil, fmt.Errorf("failed to marshal request headers: %w", err) - } - - patches = append(patches, - // 1) Send matched requests to the iroh-gateway cluster. - envoygatewayv1alpha1.EnvoyJSONPatchConfig{ - Type: "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", - Name: routeConfigName, - Operation: envoygatewayv1alpha1.JSONPatchOperation{ - Op: envoygatewayv1alpha1.JSONPatchOperationType("replace"), - JSONPath: ptr.To(jsonPath), - Path: ptr.To("/route/cluster"), - Value: &apiextensionsv1.JSON{Raw: clusterJSON}, - }, - }, - // 2) Inject internal control headers based on the selected route/backend. - // The client does not send these. - envoygatewayv1alpha1.EnvoyJSONPatchConfig{ - Type: "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", - Name: routeConfigName, - Operation: envoygatewayv1alpha1.JSONPatchOperation{ - Op: headersOp, - JSONPath: ptr.To(jsonPath), - Path: ptr.To("/request_headers_to_add"), - Value: &apiextensionsv1.JSON{Raw: headersJSON}, - }, - }, - ) + // 2) Cluster patch (per connector backend): point the route's cluster at the internal + // listener with endpoint metadata. Bootstrap must define the "connector-tunnel" internal + // listener (TcpProxy → iroh-gateway) and iroh-gateway cluster; InternalUpstreamTransport + // passes endpoint metadata so TcpProxy can send CONNECT with the right hostname and headers. + for _, backend := range backends { + clusterName := connectorClusterName(downstreamNamespace, httpProxy.Name, backend.ruleIndex) + clusterJSON, err := buildConnectorInternalListenerClusterJSON(clusterName, backend) + if err != nil { + return nil, fmt.Errorf("failed to build connector cluster JSON: %w", err) } + patches = append(patches, envoygatewayv1alpha1.EnvoyJSONPatchConfig{ + Type: "type.googleapis.com/envoy.config.cluster.v3.Cluster", + Name: clusterName, + Operation: envoygatewayv1alpha1.JSONPatchOperation{ + Op: envoygatewayv1alpha1.JSONPatchOperationType("replace"), + Path: ptr.To(""), + Value: &apiextensionsv1.JSON{Raw: clusterJSON}, + }, + }) } return patches, nil From c1cc22e055f34e5b96ee945da3ff7e05fcb0f744 Mon Sep 17 00:00:00 2001 From: Zach Smith Date: Tue, 10 Feb 2026 18:23:45 -0800 Subject: [PATCH 12/24] fix: remove client upgrade option from listener --- internal/controller/httpproxy_controller.go | 35 +-------------------- 1 file changed, 1 insertion(+), 34 deletions(-) diff --git a/internal/controller/httpproxy_controller.go b/internal/controller/httpproxy_controller.go index db9e4129..d98886ec 100644 --- a/internal/controller/httpproxy_controller.go +++ b/internal/controller/httpproxy_controller.go @@ -1286,30 +1286,8 @@ func buildConnectorEnvoyPatches( httpProxy *networkingv1alpha.HTTPProxy, backends []connectorBackendPatch, ) ([]envoygatewayv1alpha1.EnvoyJSONPatchConfig, error) { - routeConfigNames := connectorRouteConfigNames(downstreamNamespace, gateway) patches := make([]envoygatewayv1alpha1.EnvoyJSONPatchConfig, 0) - // TODO: Make this idempotent - - // 1) Listener patch: enable CONNECT (and Extended CONNECT / WebSocket) on HTTPS listeners. - for _, listenerName := range routeConfigNames { - upgradeConfigsJSON, err := json.Marshal([]map[string]any{ - {"upgrade_type": "CONNECT"}, - }) - if err != nil { - return nil, fmt.Errorf("failed to marshal upgrade_configs: %w", err) - } - patches = append(patches, envoygatewayv1alpha1.EnvoyJSONPatchConfig{ - Type: "type.googleapis.com/envoy.config.listener.v3.Listener", - Name: listenerName, - Operation: envoygatewayv1alpha1.JSONPatchOperation{ - Op: envoygatewayv1alpha1.JSONPatchOperationType("add"), - Path: ptr.To("/default_filter_chain/filters/0/typed_config/upgrade_configs"), - Value: &apiextensionsv1.JSON{Raw: upgradeConfigsJSON}, - }, - }) - } - - // 2) Cluster patch (per connector backend): point the route's cluster at the internal + // Cluster patch (per connector backend): point the route's cluster at the internal // listener with endpoint metadata. Bootstrap must define the "connector-tunnel" internal // listener (TcpProxy → iroh-gateway) and iroh-gateway cluster; InternalUpstreamTransport // passes endpoint metadata so TcpProxy can send CONNECT with the right hostname and headers. @@ -1333,17 +1311,6 @@ func buildConnectorEnvoyPatches( return patches, nil } -func connectorRouteConfigNames(downstreamNamespace string, gateway *gatewayv1.Gateway) []string { - routeConfigNames := []string{} - for _, listener := range gateway.Spec.Listeners { - if listener.Protocol != gatewayv1.HTTPSProtocolType { - continue - } - routeConfigNames = append(routeConfigNames, fmt.Sprintf("%s/%s/%s", downstreamNamespace, gateway.Name, listener.Name)) - } - return routeConfigNames -} - func connectorRouteJSONPath( downstreamNamespace string, gateway *gatewayv1.Gateway, From 5cc8265db7e8e1f31faa3615df5bbfb9c88494ec Mon Sep 17 00:00:00 2001 From: Zach Smith Date: Wed, 11 Feb 2026 14:01:25 -0800 Subject: [PATCH 13/24] feat: add vhost :authority target to http proxy connector tunnel --- internal/controller/httpproxy_controller.go | 26 +++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/internal/controller/httpproxy_controller.go b/internal/controller/httpproxy_controller.go index d98886ec..41d5de0b 100644 --- a/internal/controller/httpproxy_controller.go +++ b/internal/controller/httpproxy_controller.go @@ -1308,6 +1308,32 @@ func buildConnectorEnvoyPatches( }) } + // Add each unique tunnel target (host:port) to the RouteConfiguration's first vhost + // domains so CONNECT requests with :authority set to that target match the vhost. + // This is needed so CONNECT requests with :authority set to the tunnel target match the vhost. + routeConfigName := fmt.Sprintf("%s/%s/default-https", downstreamNamespace, gateway.Name) + seenDomains := make(map[string]struct{}) + for _, backend := range backends { + domain := fmt.Sprintf("%s:%d", backend.targetHost, backend.targetPort) + if _, ok := seenDomains[domain]; ok { + continue + } + seenDomains[domain] = struct{}{} + domainValue, err := json.Marshal(domain) + if err != nil { + return nil, fmt.Errorf("failed to marshal tunnel domain %q: %w", domain, err) + } + patches = append(patches, envoygatewayv1alpha1.EnvoyJSONPatchConfig{ + Type: "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + Name: routeConfigName, + Operation: envoygatewayv1alpha1.JSONPatchOperation{ + Op: envoygatewayv1alpha1.JSONPatchOperationType("add"), + Path: ptr.To("/virtual_hosts/0/domains/-"), + Value: &apiextensionsv1.JSON{Raw: domainValue}, + }, + }) + } + return patches, nil } From c6d0a19a53aae9583c78c9654bdb6a26fe770ce3 Mon Sep 17 00:00:00 2001 From: Zach Smith Date: Wed, 11 Feb 2026 14:03:05 -0800 Subject: [PATCH 14/24] fix: dupe comment --- internal/controller/httpproxy_controller.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/controller/httpproxy_controller.go b/internal/controller/httpproxy_controller.go index 41d5de0b..3b65b09a 100644 --- a/internal/controller/httpproxy_controller.go +++ b/internal/controller/httpproxy_controller.go @@ -1310,7 +1310,6 @@ func buildConnectorEnvoyPatches( // Add each unique tunnel target (host:port) to the RouteConfiguration's first vhost // domains so CONNECT requests with :authority set to that target match the vhost. - // This is needed so CONNECT requests with :authority set to the tunnel target match the vhost. routeConfigName := fmt.Sprintf("%s/%s/default-https", downstreamNamespace, gateway.Name) seenDomains := make(map[string]struct{}) for _, backend := range backends { From 9ecdab227053f9e39ba9e7207b2ddde7262bc69f Mon Sep 17 00:00:00 2001 From: Zach Smith Date: Wed, 11 Feb 2026 14:34:02 -0800 Subject: [PATCH 15/24] feat: add connect route --- internal/controller/httpproxy_controller.go | 37 +++++++++++++++++++-- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/internal/controller/httpproxy_controller.go b/internal/controller/httpproxy_controller.go index 3b65b09a..47ba5135 100644 --- a/internal/controller/httpproxy_controller.go +++ b/internal/controller/httpproxy_controller.go @@ -1288,9 +1288,7 @@ func buildConnectorEnvoyPatches( ) ([]envoygatewayv1alpha1.EnvoyJSONPatchConfig, error) { patches := make([]envoygatewayv1alpha1.EnvoyJSONPatchConfig, 0) // Cluster patch (per connector backend): point the route's cluster at the internal - // listener with endpoint metadata. Bootstrap must define the "connector-tunnel" internal - // listener (TcpProxy → iroh-gateway) and iroh-gateway cluster; InternalUpstreamTransport - // passes endpoint metadata so TcpProxy can send CONNECT with the right hostname and headers. + // listener with endpoint metadata. for _, backend := range backends { clusterName := connectorClusterName(downstreamNamespace, httpProxy.Name, backend.ruleIndex) clusterJSON, err := buildConnectorInternalListenerClusterJSON(clusterName, backend) @@ -1333,6 +1331,39 @@ func buildConnectorEnvoyPatches( }) } + // Add a CONNECT route (connect_matcher + route to connector cluster) so CONNECT requests + // are routed to the connector tunnel instead of 404. Insert at index 0 so CONNECT is + // matched before path-based routes. + if len(backends) > 0 { + first := backends[0] + connectorCluster := connectorClusterName(downstreamNamespace, httpProxy.Name, first.ruleIndex) + connectRoute := map[string]any{ + "name": fmt.Sprintf("connector-connect-%s", httpProxy.Name), + "match": map[string]any{ + "connect_matcher": map[string]any{}, + }, + "route": map[string]any{ + "cluster": connectorCluster, + "upgrade_configs": []map[string]any{ + {"upgrade_type": "CONNECT"}, + }, + }, + } + routeValue, err := json.Marshal(connectRoute) + if err != nil { + return nil, fmt.Errorf("failed to marshal CONNECT route: %w", err) + } + patches = append(patches, envoygatewayv1alpha1.EnvoyJSONPatchConfig{ + Type: "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + Name: routeConfigName, + Operation: envoygatewayv1alpha1.JSONPatchOperation{ + Op: envoygatewayv1alpha1.JSONPatchOperationType("add"), + Path: ptr.To("/virtual_hosts/0/routes/0"), + Value: &apiextensionsv1.JSON{Raw: routeValue}, + }, + }) + } + return patches, nil } From 4946da30b67747741f036f7245160b897079b064 Mon Sep 17 00:00:00 2001 From: Zach Smith Date: Wed, 11 Feb 2026 14:35:01 -0800 Subject: [PATCH 16/24] fix: lint --- internal/controller/httpproxy_controller.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/controller/httpproxy_controller.go b/internal/controller/httpproxy_controller.go index 47ba5135..23db53e2 100644 --- a/internal/controller/httpproxy_controller.go +++ b/internal/controller/httpproxy_controller.go @@ -1288,7 +1288,7 @@ func buildConnectorEnvoyPatches( ) ([]envoygatewayv1alpha1.EnvoyJSONPatchConfig, error) { patches := make([]envoygatewayv1alpha1.EnvoyJSONPatchConfig, 0) // Cluster patch (per connector backend): point the route's cluster at the internal - // listener with endpoint metadata. + // listener with endpoint metadata. for _, backend := range backends { clusterName := connectorClusterName(downstreamNamespace, httpProxy.Name, backend.ruleIndex) clusterJSON, err := buildConnectorInternalListenerClusterJSON(clusterName, backend) From 01fd442165a2f21b206b2fd951ca380880229c94 Mon Sep 17 00:00:00 2001 From: Zach Smith Date: Wed, 11 Feb 2026 15:16:43 -0800 Subject: [PATCH 17/24] fix: use host only for matching --- internal/controller/httpproxy_controller.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/controller/httpproxy_controller.go b/internal/controller/httpproxy_controller.go index 23db53e2..c2a7f35c 100644 --- a/internal/controller/httpproxy_controller.go +++ b/internal/controller/httpproxy_controller.go @@ -1306,12 +1306,12 @@ func buildConnectorEnvoyPatches( }) } - // Add each unique tunnel target (host:port) to the RouteConfiguration's first vhost + // Add each unique tunnel target (host only) to the RouteConfiguration's first vhost // domains so CONNECT requests with :authority set to that target match the vhost. routeConfigName := fmt.Sprintf("%s/%s/default-https", downstreamNamespace, gateway.Name) seenDomains := make(map[string]struct{}) for _, backend := range backends { - domain := fmt.Sprintf("%s:%d", backend.targetHost, backend.targetPort) + domain := backend.targetHost if _, ok := seenDomains[domain]; ok { continue } From 2981bdfec94e49ff5cc8a05e54b43a6e1ae5c94d Mon Sep 17 00:00:00 2001 From: Zach Smith Date: Wed, 11 Feb 2026 16:06:06 -0800 Subject: [PATCH 18/24] fix: terminate enovy connect --- internal/controller/httpproxy_controller.go | 31 ++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/internal/controller/httpproxy_controller.go b/internal/controller/httpproxy_controller.go index c2a7f35c..17812d28 100644 --- a/internal/controller/httpproxy_controller.go +++ b/internal/controller/httpproxy_controller.go @@ -100,6 +100,7 @@ func (r *HTTPProxyReconciler) Reconcile(ctx context.Context, req mcreconcile.Req return ctrl.Result{}, err } controllerutil.RemoveFinalizer(&httpProxy, httpProxyFinalizer) + normalizeHTTPProxyBackendEndpoints(&httpProxy) if err := cl.GetClient().Update(ctx, &httpProxy); err != nil { return ctrl.Result{}, err } @@ -111,6 +112,7 @@ func (r *HTTPProxyReconciler) Reconcile(ctx context.Context, req mcreconcile.Req defer logger.Info("reconcile complete") if updated := ensureConnectorNameAnnotation(&httpProxy); updated { + normalizeHTTPProxyBackendEndpoints(&httpProxy) if err := cl.GetClient().Update(ctx, &httpProxy); err != nil { return ctrl.Result{}, fmt.Errorf("failed updating httpproxy connector annotation: %w", err) } @@ -119,6 +121,7 @@ func (r *HTTPProxyReconciler) Reconcile(ctx context.Context, req mcreconcile.Req if !controllerutil.ContainsFinalizer(&httpProxy, httpProxyFinalizer) { controllerutil.AddFinalizer(&httpProxy, httpProxyFinalizer) + normalizeHTTPProxyBackendEndpoints(&httpProxy) if err := cl.GetClient().Update(ctx, &httpProxy); err != nil { return ctrl.Result{}, err } @@ -1132,6 +1135,29 @@ func collectConnectorBackends( return connectorBackends, nil } +// normalizeHTTPProxyBackendEndpoints strips path, query, and fragment from each +// rule backend endpoint URL so that specs like "http://example.com/" pass +// validation (endpoint must not have a path component). Mutates proxy in place. +func normalizeHTTPProxyBackendEndpoints(proxy *networkingv1alpha.HTTPProxy) { + for i := range proxy.Spec.Rules { + for j := range proxy.Spec.Rules[i].Backends { + ep := proxy.Spec.Rules[i].Backends[j].Endpoint + if ep == "" { + continue + } + u, err := url.Parse(ep) + if err != nil { + continue + } + u.Path = "" + u.RawPath = "" + u.RawQuery = "" + u.Fragment = "" + proxy.Spec.Rules[i].Backends[j].Endpoint = u.String() + } + } +} + func backendEndpointTarget(backend networkingv1alpha.HTTPProxyRuleBackend) (string, int, error) { u, err := url.Parse(backend.Endpoint) if err != nil { @@ -1345,7 +1371,10 @@ func buildConnectorEnvoyPatches( "route": map[string]any{ "cluster": connectorCluster, "upgrade_configs": []map[string]any{ - {"upgrade_type": "CONNECT"}, + { + "upgrade_type": "CONNECT", + "connect_config": map[string]any{}, + }, }, }, } From ccaa0e2f8072838d9fe4cd9772b712831611f835 Mon Sep 17 00:00:00 2001 From: Zach Smith Date: Fri, 13 Feb 2026 12:56:28 -0800 Subject: [PATCH 19/24] chore: refactor connector patch logic --- internal/config/config.go | 13 + internal/config/zz_generated.defaults.go | 3 + .../controller/connector_routing_compiler.go | 400 ++++++++++++++++++ internal/controller/httpproxy_controller.go | 217 +--------- .../controller/httpproxy_controller_test.go | 235 ++++++++++ 5 files changed, 652 insertions(+), 216 deletions(-) create mode 100644 internal/controller/connector_routing_compiler.go diff --git a/internal/config/config.go b/internal/config/config.go index 78e4b0d3..8174e77b 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -470,6 +470,12 @@ type GatewayConfig struct { // +default={"gateway.networking.datumapis.com/certificate-issuer": "auto"} ListenerTLSOptions map[gatewayv1.AnnotationKey]gatewayv1.AnnotationValue `json:"listenerTLSOptions"` + // ConnectorInternalListenerName is the Envoy internal listener name used by + // connector tunnel routing patches. + // + // +default="connector-tunnel" + ConnectorInternalListenerName string `json:"connectorInternalListenerName,omitempty"` + // Coraza specifies configuration for the Coraza WAF. Coraza CorazaConfig `json:"coraza,omitempty"` @@ -527,6 +533,13 @@ func (c *GatewayConfig) GatewayDNSAddress(gateway *gatewayv1.Gateway) string { return fmt.Sprintf("%s.%s", strings.ReplaceAll(string(gateway.UID), "-", ""), c.TargetDomain) } +func (c *GatewayConfig) ConnectorTunnelListenerName() string { + if c.ConnectorInternalListenerName == "" { + return "connector-tunnel" + } + return c.ConnectorInternalListenerName +} + // +k8s:deepcopy-gen=true type CorazaConfig struct { diff --git a/internal/config/zz_generated.defaults.go b/internal/config/zz_generated.defaults.go index 64e6dcd6..5fb1ec64 100644 --- a/internal/config/zz_generated.defaults.go +++ b/internal/config/zz_generated.defaults.go @@ -32,6 +32,9 @@ func SetObjectDefaults_NetworkServicesOperator(in *NetworkServicesOperator) { panic(err) } } + if in.Gateway.ConnectorInternalListenerName == "" { + in.Gateway.ConnectorInternalListenerName = "connector-tunnel" + } if in.Gateway.Coraza.LibraryID == "" { in.Gateway.Coraza.LibraryID = "coraza-waf" } diff --git a/internal/controller/connector_routing_compiler.go b/internal/controller/connector_routing_compiler.go new file mode 100644 index 00000000..4605c30f --- /dev/null +++ b/internal/controller/connector_routing_compiler.go @@ -0,0 +1,400 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package controller + +import ( + "encoding/json" + "fmt" + "net/http" + + envoygatewayv1alpha1 "github.com/envoyproxy/gateway/api/v1alpha1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/utils/ptr" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" +) + +// connectorBackendPatch captures route/backend data used to compile connector Envoy patches. +type connectorBackendPatch struct { + // Gateway listener section name (default-https, etc.). + sectionName *gatewayv1.SectionName + + // Identify the HTTPRoute rule/match this backend applies to. + ruleIndex int + matchIndex int + + targetHost string + targetPort int + nodeID string +} + +// connectorClusterName returns the cluster name Envoy Gateway assigns to the +// HTTPRoute backend for the given rule. Must match Envoy Gateway's naming so we +// can patch that cluster to point at the internal listener. +func connectorClusterName(downstreamNamespace, httpRouteName string, ruleIndex int) string { + return fmt.Sprintf("httproute/%s/%s/rule/%d", downstreamNamespace, httpRouteName, ruleIndex) +} + +// buildConnectorInternalListenerClusterJSON returns the Envoy cluster config +// JSON that points at the internal listener "connector-tunnel" with endpoint +// metadata (tunnel.address, tunnel.endpoint_id). InternalUpstreamTransport +// copies that metadata to the internal connection so TcpProxy can use +// %DYNAMIC_METADATA(tunnel:address)% and tunnel:endpoint_id for CONNECT. +func buildConnectorInternalListenerClusterJSON(clusterName, internalListenerName string, backend connectorBackendPatch) ([]byte, error) { + tunnelAddress := fmt.Sprintf("%s:%d", backend.targetHost, backend.targetPort) + cluster := map[string]any{ + "name": clusterName, + "type": "STATIC", + "connect_timeout": "5s", + "load_assignment": map[string]any{ + "cluster_name": clusterName, + "endpoints": []map[string]any{ + { + "lb_endpoints": []map[string]any{ + { + "endpoint": map[string]any{ + "address": map[string]any{ + "envoy_internal_address": map[string]any{ + "server_listener_name": internalListenerName, + }, + }, + }, + "metadata": map[string]any{ + "filter_metadata": map[string]any{ + "tunnel": map[string]any{ + "address": tunnelAddress, + "endpoint_id": backend.nodeID, + }, + }, + }, + }, + }, + }, + }, + }, + "transport_socket": map[string]any{ + "name": "envoy.transport_sockets.internal_upstream", + "typed_config": map[string]any{ + "@type": "type.googleapis.com/envoy.extensions.transport_sockets.internal_upstream.v3.InternalUpstreamTransport", + "passthrough_metadata": []map[string]any{ + { + "kind": map[string]any{"host": map[string]any{}}, + "name": "tunnel", + }, + }, + "transport_socket": map[string]any{ + "name": "envoy.transport_sockets.raw_buffer", + "typed_config": map[string]any{ + "@type": "type.googleapis.com/envoy.extensions.transport_sockets.raw_buffer.v3.RawBuffer", + }, + }, + }, + }, + } + return json.Marshal(cluster) +} + +func buildConnectorEnvoyPatches( + downstreamNamespace string, + internalListenerName string, + gateway *gatewayv1.Gateway, + httpProxy *networkingv1alpha.HTTPProxy, + backends []connectorBackendPatch, +) ([]envoygatewayv1alpha1.EnvoyJSONPatchConfig, error) { + patches := make([]envoygatewayv1alpha1.EnvoyJSONPatchConfig, 0) + // Cluster patch (per connector backend): point the route's cluster at the internal + // listener with endpoint metadata. + for _, backend := range backends { + clusterName := connectorClusterName(downstreamNamespace, httpProxy.Name, backend.ruleIndex) + clusterJSON, err := buildConnectorInternalListenerClusterJSON(clusterName, internalListenerName, backend) + if err != nil { + return nil, fmt.Errorf("failed to build connector cluster JSON: %w", err) + } + patches = append(patches, envoygatewayv1alpha1.EnvoyJSONPatchConfig{ + Type: "type.googleapis.com/envoy.config.cluster.v3.Cluster", + Name: clusterName, + Operation: envoygatewayv1alpha1.JSONPatchOperation{ + Op: envoygatewayv1alpha1.JSONPatchOperationType("replace"), + Path: ptr.To(""), + Value: &apiextensionsv1.JSON{Raw: clusterJSON}, + }, + }) + } + + // Add each unique tunnel target (host only) to HTTPS listener + // RouteConfiguration(s) so CONNECT requests with :authority set to that + // target match the vhost. + // + // Current behavior: HTTPProxy connector backends set sectionName=nil, so we + // patch all HTTPS listeners. + // + // Future extension: when sectionName is populated from route attachment + // context, patch only that specific HTTPS listener's RouteConfiguration. + allHTTPSRouteConfigNames := gatewayHTTPSRouteConfigNames(downstreamNamespace, gateway) + seenDomainRouteConfig := make(map[string]struct{}) + for _, backend := range backends { + domain := backend.targetHost + routeConfigNames := gatewayHTTPSRouteConfigNamesForSection( + downstreamNamespace, + gateway, + backend.sectionName, + allHTTPSRouteConfigNames, + ) + domainValue, err := json.Marshal(domain) + if err != nil { + return nil, fmt.Errorf("failed to marshal tunnel domain %q: %w", domain, err) + } + for _, routeConfigName := range routeConfigNames { + key := fmt.Sprintf("%s|%s", domain, routeConfigName) + if _, ok := seenDomainRouteConfig[key]; ok { + continue + } + seenDomainRouteConfig[key] = struct{}{} + patches = append(patches, envoygatewayv1alpha1.EnvoyJSONPatchConfig{ + Type: "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + Name: routeConfigName, + Operation: envoygatewayv1alpha1.JSONPatchOperation{ + Op: envoygatewayv1alpha1.JSONPatchOperationType("add"), + Path: ptr.To("/virtual_hosts/0/domains/-"), + Value: &apiextensionsv1.JSON{Raw: domainValue}, + }, + }) + } + } + + // Add CONNECT routes so CONNECT requests are routed to connector clusters. + // + // - Pure CONNECT fallback uses connect_matcher (no path semantics). + // - Extended CONNECT path-aware routes are generated when a rule explicitly + // matches method CONNECT with a non-root path. + // + // Insert at index 0 so CONNECT routes are matched before path-based routes. + connectRoutes, err := buildConnectorConnectRoutes(downstreamNamespace, httpProxy, backends) + if err != nil { + return nil, err + } + for _, connectRoute := range connectRoutes { + routeValue, err := json.Marshal(connectRoute.route) + if err != nil { + return nil, fmt.Errorf("failed to marshal CONNECT route: %w", err) + } + routeConfigNames := gatewayHTTPSRouteConfigNamesForSection( + downstreamNamespace, + gateway, + connectRoute.sectionName, + allHTTPSRouteConfigNames, + ) + for _, routeConfigName := range routeConfigNames { + patches = append(patches, envoygatewayv1alpha1.EnvoyJSONPatchConfig{ + Type: "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + Name: routeConfigName, + Operation: envoygatewayv1alpha1.JSONPatchOperation{ + Op: envoygatewayv1alpha1.JSONPatchOperationType("add"), + Path: ptr.To("/virtual_hosts/0/routes/0"), + Value: &apiextensionsv1.JSON{Raw: routeValue}, + }, + }) + } + } + + return patches, nil +} + +func gatewayHTTPSRouteConfigNames(downstreamNamespace string, gateway *gatewayv1.Gateway) []string { + names := make([]string, 0) + seen := make(map[string]struct{}) + + for _, listener := range gateway.Spec.Listeners { + if listener.Protocol != gatewayv1.HTTPSProtocolType { + continue + } + name := fmt.Sprintf("%s/%s/%s", downstreamNamespace, gateway.Name, listener.Name) + if _, ok := seen[name]; ok { + continue + } + seen[name] = struct{}{} + names = append(names, name) + } + + return names +} + +func gatewayHTTPSRouteConfigNamesForSection( + downstreamNamespace string, + gateway *gatewayv1.Gateway, + sectionName *gatewayv1.SectionName, + allHTTPSRouteConfigNames []string, +) []string { + if sectionName == nil { + return allHTTPSRouteConfigNames + } + + for _, listener := range gateway.Spec.Listeners { + if listener.Name != *sectionName || listener.Protocol != gatewayv1.HTTPSProtocolType { + continue + } + return []string{fmt.Sprintf("%s/%s/%s", downstreamNamespace, gateway.Name, listener.Name)} + } + + // If the provided section is not an HTTPS listener, no RouteConfiguration should be patched. + return nil +} + +type connectorConnectRoute struct { + sectionName *gatewayv1.SectionName + route map[string]any +} + +func buildConnectorConnectRoutes( + downstreamNamespace string, + httpProxy *networkingv1alpha.HTTPProxy, + backends []connectorBackendPatch, +) ([]connectorConnectRoute, error) { + if len(backends) == 0 { + return nil, nil + } + + clusterByRule := map[int]string{} + sectionByRule := map[int]*gatewayv1.SectionName{} + for _, backend := range backends { + if _, ok := clusterByRule[backend.ruleIndex]; ok { + continue + } + clusterByRule[backend.ruleIndex] = connectorClusterName(downstreamNamespace, httpProxy.Name, backend.ruleIndex) + sectionByRule[backend.ruleIndex] = backend.sectionName + } + + // Default pure CONNECT route falls back to the first connector backend. + fallbackCluster := connectorClusterName(downstreamNamespace, httpProxy.Name, backends[0].ruleIndex) + fallbackSection := backends[0].sectionName + connectRoutes := make([]connectorConnectRoute, 0) + + // Extended CONNECT path-aware routes are derived from explicit CONNECT method matches. + // We only create path-aware routes for non-root paths to avoid swallowing the pure CONNECT fallback. + for ruleIndex, rule := range httpProxy.Spec.Rules { + clusterName, ok := clusterByRule[ruleIndex] + if !ok { + continue + } + + for _, match := range rule.Matches { + if match.Method == nil || string(*match.Method) != http.MethodConnect { + continue + } + if match.Path == nil || match.Path.Value == nil { + continue + } + + pathValue := ptr.Deref(match.Path.Value, "") + if pathValue == "" || pathValue == "/" { + // Explicit CONNECT "/" acts as fallback target. + fallbackCluster = clusterName + fallbackSection = sectionByRule[ruleIndex] + continue + } + + connectMatch := map[string]any{ + "headers": []map[string]any{ + { + "name": ":method", + "string_match": map[string]any{ + "exact": http.MethodConnect, + }, + }, + }, + } + // TODO: Add optional :protocol matching for extended CONNECT routes + // when we need to distinguish routes beyond path semantics. + + switch ptr.Deref(match.Path.Type, gatewayv1.PathMatchPathPrefix) { + case gatewayv1.PathMatchPathPrefix: + connectMatch["prefix"] = pathValue + case gatewayv1.PathMatchExact: + connectMatch["path"] = pathValue + default: + // Regex path matching can be added later when needed. + continue + } + + connectRoutes = append(connectRoutes, connectorConnectRoute{ + sectionName: sectionByRule[ruleIndex], + route: map[string]any{ + "name": fmt.Sprintf("connector-connect-%s-rule-%d", httpProxy.Name, ruleIndex), + "match": connectMatch, + "route": map[string]any{ + "cluster": clusterName, + "upgrade_configs": []map[string]any{ + { + "upgrade_type": "CONNECT", + "connect_config": map[string]any{}, + }, + }, + }, + }, + }) + } + } + + // Always include a pure CONNECT fallback route. + connectRoutes = append(connectRoutes, connectorConnectRoute{ + sectionName: fallbackSection, + route: map[string]any{ + "name": fmt.Sprintf("connector-connect-%s", httpProxy.Name), + "match": map[string]any{ + "connect_matcher": map[string]any{}, + }, + "route": map[string]any{ + "cluster": fallbackCluster, + "upgrade_configs": []map[string]any{ + { + "upgrade_type": "CONNECT", + "connect_config": map[string]any{}, + }, + }, + }, + }, + }) + + return connectRoutes, nil +} + +func connectorRouteJSONPath( + downstreamNamespace string, + gateway *gatewayv1.Gateway, + httpRouteName string, + sectionName *gatewayv1.SectionName, + ruleIndex int, + matchIndex int, +) string { + // vhost matches the Gateway + optional sectionName + vhostConstraints := fmt.Sprintf( + `@.metadata.filter_metadata["envoy-gateway"].resources[0].kind=="%s" && @.metadata.filter_metadata["envoy-gateway"].resources[0].namespace=="%s" && @.metadata.filter_metadata["envoy-gateway"].resources[0].name=="%s"`, + KindGateway, + downstreamNamespace, + gateway.Name, + ) + + if sectionName != nil { + vhostConstraints += fmt.Sprintf( + ` && @.metadata.filter_metadata["envoy-gateway"].resources[0].sectionName=="%s"`, + string(*sectionName), + ) + } + + // routes match the HTTPRoute + rule/match + routeConstraints := fmt.Sprintf( + `@.metadata.filter_metadata["envoy-gateway"].resources[0].kind=="%s" && @.metadata.filter_metadata["envoy-gateway"].resources[0].namespace=="%s" && @.metadata.filter_metadata["envoy-gateway"].resources[0].name=="%s" && @.name =~ ".*?/rule/%d/match/%d/.*"`, + KindHTTPRoute, + downstreamNamespace, + httpRouteName, + ruleIndex, + matchIndex, + ) + + return sanitizeJSONPath(fmt.Sprintf( + `..virtual_hosts[?(%s)]..routes[?(!@.bogus && %s)]`, + vhostConstraints, + routeConstraints, + )) +} diff --git a/internal/controller/httpproxy_controller.go b/internal/controller/httpproxy_controller.go index 17812d28..5e0cd054 100644 --- a/internal/controller/httpproxy_controller.go +++ b/internal/controller/httpproxy_controller.go @@ -4,7 +4,6 @@ package controller import ( "context" - "encoding/json" "errors" "fmt" "net" @@ -17,7 +16,6 @@ import ( envoygatewayv1alpha1 "github.com/envoyproxy/gateway/api/v1alpha1" v1 "k8s.io/api/core/v1" discoveryv1 "k8s.io/api/discovery/v1" - apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/api/equality" apierrors "k8s.io/apimachinery/pkg/api/errors" apimeta "k8s.io/apimachinery/pkg/api/meta" @@ -885,19 +883,6 @@ func hasControllerConflict(obj, owner metav1.Object) bool { return controllerutil.HasControllerReference(obj) && !metav1.IsControlledBy(obj, owner) } -type connectorBackendPatch struct { - // Gateway listener section name (default-https, etc.). - sectionName *gatewayv1.SectionName - - // Identify the HTTPRoute rule/match this backend applies to. - ruleIndex int - matchIndex int - - targetHost string - targetPort int - nodeID string -} - func (r *HTTPProxyReconciler) reconcileConnectorEnvoyPatchPolicy( ctx context.Context, upstreamClient client.Client, @@ -966,6 +951,7 @@ func (r *HTTPProxyReconciler) reconcileConnectorEnvoyPatchPolicy( jsonPatches, err := buildConnectorEnvoyPatches( downstreamNamespaceName, + r.Config.Gateway.ConnectorTunnelListenerName(), gateway, httpProxy, connectorBackends, @@ -1234,204 +1220,3 @@ func connectorPatchDetails(ctx context.Context, cl client.Client, namespace, nam } return true, details.PublicKey.Id, nil } - -// connectorInternalListenerName is the name of the static internal listener -// (added via EnvoyProxy bootstrap) that runs TcpProxy and sends CONNECT to -// iroh-gateway using dynamic metadata for hostname and x-iroh-endpoint-id. -const connectorInternalListenerName = "connector-tunnel" - -// connectorClusterName returns the cluster name Envoy Gateway assigns to the -// HTTPRoute backend for the given rule. Must match Envoy Gateway's naming so we -// can patch that cluster to point at the internal listener. -func connectorClusterName(downstreamNamespace, httpRouteName string, ruleIndex int) string { - return fmt.Sprintf("httproute/%s/%s/rule/%d", downstreamNamespace, httpRouteName, ruleIndex) -} - -// buildConnectorInternalListenerClusterJSON returns the Envoy cluster config -// JSON that points at the internal listener "connector-tunnel" with endpoint -// metadata (tunnel.address, tunnel.endpoint_id). InternalUpstreamTransport -// copies that metadata to the internal connection so TcpProxy can use -// %DYNAMIC_METADATA(tunnel:address)% and tunnel:endpoint_id for CONNECT. -func buildConnectorInternalListenerClusterJSON(clusterName string, backend connectorBackendPatch) ([]byte, error) { - tunnelAddress := fmt.Sprintf("%s:%d", backend.targetHost, backend.targetPort) - cluster := map[string]any{ - "name": clusterName, - "type": "STATIC", - "connect_timeout": "5s", - "load_assignment": map[string]any{ - "cluster_name": clusterName, - "endpoints": []map[string]any{ - { - "lb_endpoints": []map[string]any{ - { - "endpoint": map[string]any{ - "address": map[string]any{ - "envoy_internal_address": map[string]any{ - "server_listener_name": connectorInternalListenerName, - }, - }, - }, - "metadata": map[string]any{ - "filter_metadata": map[string]any{ - "tunnel": map[string]any{ - "address": tunnelAddress, - "endpoint_id": backend.nodeID, - }, - }, - }, - }, - }, - }, - }, - }, - "transport_socket": map[string]any{ - "name": "envoy.transport_sockets.internal_upstream", - "typed_config": map[string]any{ - "@type": "type.googleapis.com/envoy.extensions.transport_sockets.internal_upstream.v3.InternalUpstreamTransport", - "passthrough_metadata": []map[string]any{ - { - "kind": map[string]any{"host": map[string]any{}}, - "name": "tunnel", - }, - }, - "transport_socket": map[string]any{ - "name": "envoy.transport_sockets.raw_buffer", - "typed_config": map[string]any{ - "@type": "type.googleapis.com/envoy.extensions.transport_sockets.raw_buffer.v3.RawBuffer", - }, - }, - }, - }, - } - return json.Marshal(cluster) -} - -func buildConnectorEnvoyPatches( - downstreamNamespace string, - gateway *gatewayv1.Gateway, - httpProxy *networkingv1alpha.HTTPProxy, - backends []connectorBackendPatch, -) ([]envoygatewayv1alpha1.EnvoyJSONPatchConfig, error) { - patches := make([]envoygatewayv1alpha1.EnvoyJSONPatchConfig, 0) - // Cluster patch (per connector backend): point the route's cluster at the internal - // listener with endpoint metadata. - for _, backend := range backends { - clusterName := connectorClusterName(downstreamNamespace, httpProxy.Name, backend.ruleIndex) - clusterJSON, err := buildConnectorInternalListenerClusterJSON(clusterName, backend) - if err != nil { - return nil, fmt.Errorf("failed to build connector cluster JSON: %w", err) - } - patches = append(patches, envoygatewayv1alpha1.EnvoyJSONPatchConfig{ - Type: "type.googleapis.com/envoy.config.cluster.v3.Cluster", - Name: clusterName, - Operation: envoygatewayv1alpha1.JSONPatchOperation{ - Op: envoygatewayv1alpha1.JSONPatchOperationType("replace"), - Path: ptr.To(""), - Value: &apiextensionsv1.JSON{Raw: clusterJSON}, - }, - }) - } - - // Add each unique tunnel target (host only) to the RouteConfiguration's first vhost - // domains so CONNECT requests with :authority set to that target match the vhost. - routeConfigName := fmt.Sprintf("%s/%s/default-https", downstreamNamespace, gateway.Name) - seenDomains := make(map[string]struct{}) - for _, backend := range backends { - domain := backend.targetHost - if _, ok := seenDomains[domain]; ok { - continue - } - seenDomains[domain] = struct{}{} - domainValue, err := json.Marshal(domain) - if err != nil { - return nil, fmt.Errorf("failed to marshal tunnel domain %q: %w", domain, err) - } - patches = append(patches, envoygatewayv1alpha1.EnvoyJSONPatchConfig{ - Type: "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", - Name: routeConfigName, - Operation: envoygatewayv1alpha1.JSONPatchOperation{ - Op: envoygatewayv1alpha1.JSONPatchOperationType("add"), - Path: ptr.To("/virtual_hosts/0/domains/-"), - Value: &apiextensionsv1.JSON{Raw: domainValue}, - }, - }) - } - - // Add a CONNECT route (connect_matcher + route to connector cluster) so CONNECT requests - // are routed to the connector tunnel instead of 404. Insert at index 0 so CONNECT is - // matched before path-based routes. - if len(backends) > 0 { - first := backends[0] - connectorCluster := connectorClusterName(downstreamNamespace, httpProxy.Name, first.ruleIndex) - connectRoute := map[string]any{ - "name": fmt.Sprintf("connector-connect-%s", httpProxy.Name), - "match": map[string]any{ - "connect_matcher": map[string]any{}, - }, - "route": map[string]any{ - "cluster": connectorCluster, - "upgrade_configs": []map[string]any{ - { - "upgrade_type": "CONNECT", - "connect_config": map[string]any{}, - }, - }, - }, - } - routeValue, err := json.Marshal(connectRoute) - if err != nil { - return nil, fmt.Errorf("failed to marshal CONNECT route: %w", err) - } - patches = append(patches, envoygatewayv1alpha1.EnvoyJSONPatchConfig{ - Type: "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", - Name: routeConfigName, - Operation: envoygatewayv1alpha1.JSONPatchOperation{ - Op: envoygatewayv1alpha1.JSONPatchOperationType("add"), - Path: ptr.To("/virtual_hosts/0/routes/0"), - Value: &apiextensionsv1.JSON{Raw: routeValue}, - }, - }) - } - - return patches, nil -} - -func connectorRouteJSONPath( - downstreamNamespace string, - gateway *gatewayv1.Gateway, - httpRouteName string, - sectionName *gatewayv1.SectionName, - ruleIndex int, - matchIndex int, -) string { - // vhost matches the Gateway + optional sectionName - vhostConstraints := fmt.Sprintf( - `@.metadata.filter_metadata["envoy-gateway"].resources[0].kind=="%s" && @.metadata.filter_metadata["envoy-gateway"].resources[0].namespace=="%s" && @.metadata.filter_metadata["envoy-gateway"].resources[0].name=="%s"`, - KindGateway, - downstreamNamespace, - gateway.Name, - ) - - if sectionName != nil { - vhostConstraints += fmt.Sprintf( - ` && @.metadata.filter_metadata["envoy-gateway"].resources[0].sectionName=="%s"`, - string(*sectionName), - ) - } - - // routes match the HTTPRoute + rule/match - routeConstraints := fmt.Sprintf( - `@.metadata.filter_metadata["envoy-gateway"].resources[0].kind=="%s" && @.metadata.filter_metadata["envoy-gateway"].resources[0].namespace=="%s" && @.metadata.filter_metadata["envoy-gateway"].resources[0].name=="%s" && @.name =~ ".*?/rule/%d/match/%d/.*"`, - KindHTTPRoute, - downstreamNamespace, - httpRouteName, - ruleIndex, - matchIndex, - ) - - return sanitizeJSONPath(fmt.Sprintf( - `..virtual_hosts[?(%s)]..routes[?(!@.bogus && %s)]`, - vhostConstraints, - routeConstraints, - )) -} diff --git a/internal/controller/httpproxy_controller_test.go b/internal/controller/httpproxy_controller_test.go index 6b07aaee..3ba119e6 100644 --- a/internal/controller/httpproxy_controller_test.go +++ b/internal/controller/httpproxy_controller_test.go @@ -2,7 +2,9 @@ package controller import ( "context" + "encoding/json" "fmt" + "net/http" "testing" envoygatewayv1alpha1 "github.com/envoyproxy/gateway/api/v1alpha1" @@ -1191,6 +1193,239 @@ func TestConnectorRouteJSONPathDistinctPerRuleMatch(t *testing.T) { assert.Contains(t, pathB, `/rule/1/match/0/`) } +func TestBuildConnectorEnvoyPatchesTargetsAllHTTPSListeners(t *testing.T) { + gateway := &gatewayv1.Gateway{ + ObjectMeta: metav1.ObjectMeta{Name: "gw"}, + Spec: gatewayv1.GatewaySpec{ + Listeners: []gatewayv1.Listener{ + { + Name: gatewayv1.SectionName("default-http"), + Protocol: gatewayv1.HTTPProtocolType, + }, + { + Name: gatewayv1.SectionName("default-https"), + Protocol: gatewayv1.HTTPSProtocolType, + }, + { + Name: gatewayv1.SectionName("https-hostname-0"), + Protocol: gatewayv1.HTTPSProtocolType, + }, + }, + }, + } + + backends := []connectorBackendPatch{ + { + ruleIndex: 0, + matchIndex: 0, + targetHost: "127.0.0.1", + targetPort: 5432, + nodeID: "node-123", + }, + } + + patches, err := buildConnectorEnvoyPatches( + "ns-test", + "connector-tunnel", + gateway, + newHTTPProxy(), + backends, + ) + assert.NoError(t, err) + + routeConfigPatchCounts := map[string]int{} + for _, patch := range patches { + if patch.Type != "type.googleapis.com/envoy.config.route.v3.RouteConfiguration" { + continue + } + routeConfigPatchCounts[patch.Name]++ + } + + assert.Equal(t, 2, routeConfigPatchCounts["ns-test/gw/default-https"]) + assert.Equal(t, 2, routeConfigPatchCounts["ns-test/gw/https-hostname-0"]) + assert.NotContains(t, routeConfigPatchCounts, "ns-test/gw/default-http") +} + +func TestBuildConnectorEnvoyPatchesAddsExtendedConnectPathRouteAndFallback(t *testing.T) { + gateway := &gatewayv1.Gateway{ + ObjectMeta: metav1.ObjectMeta{Name: "gw"}, + Spec: gatewayv1.GatewaySpec{ + Listeners: []gatewayv1.Listener{ + { + Name: gatewayv1.SectionName("default-https"), + Protocol: gatewayv1.HTTPSProtocolType, + }, + }, + }, + } + + httpProxy := newHTTPProxy(func(h *networkingv1alpha.HTTPProxy) { + h.Spec.Rules = []networkingv1alpha.HTTPProxyRule{ + { + Matches: []gatewayv1.HTTPRouteMatch{ + { + Method: ptr.To(gatewayv1.HTTPMethod(http.MethodConnect)), + Path: &gatewayv1.HTTPPathMatch{ + Type: ptr.To(gatewayv1.PathMatchPathPrefix), + Value: ptr.To("/"), + }, + }, + }, + Backends: []networkingv1alpha.HTTPProxyRuleBackend{ + { + Endpoint: "http://www.example.com", + Connector: &networkingv1alpha.ConnectorReference{ + Name: "connector-a", + }, + }, + }, + }, + { + Matches: []gatewayv1.HTTPRouteMatch{ + { + Method: ptr.To(gatewayv1.HTTPMethod(http.MethodConnect)), + Path: &gatewayv1.HTTPPathMatch{ + Type: ptr.To(gatewayv1.PathMatchPathPrefix), + Value: ptr.To("/ws"), + }, + }, + }, + Backends: []networkingv1alpha.HTTPProxyRuleBackend{ + { + Endpoint: "http://www.example.com", + Connector: &networkingv1alpha.ConnectorReference{ + Name: "connector-b", + }, + }, + }, + }, + } + }) + + backends := []connectorBackendPatch{ + { + ruleIndex: 0, + matchIndex: 0, + targetHost: "127.0.0.1", + targetPort: 5432, + nodeID: "node-1", + }, + { + ruleIndex: 1, + matchIndex: 0, + targetHost: "127.0.0.2", + targetPort: 8443, + nodeID: "node-2", + }, + } + + patches, err := buildConnectorEnvoyPatches( + "ns-test", + "connector-tunnel", + gateway, + httpProxy, + backends, + ) + assert.NoError(t, err) + + type routeDoc struct { + Name string `json:"name"` + Match map[string]interface{} `json:"match"` + Route struct { + Cluster string `json:"cluster"` + } `json:"route"` + } + + var routes []routeDoc + for _, patch := range patches { + if patch.Type != "type.googleapis.com/envoy.config.route.v3.RouteConfiguration" { + continue + } + if ptr.Deref(patch.Operation.Path, "") != "/virtual_hosts/0/routes/0" { + continue + } + var parsed routeDoc + assert.NoError(t, json.Unmarshal(patch.Operation.Value.Raw, &parsed)) + routes = append(routes, parsed) + } + + assert.Len(t, routes, 2) + + // One path-aware extended CONNECT route should target rule 1 cluster (/ws). + foundExtended := false + for _, r := range routes { + if r.Name != "connector-connect-test-rule-1" { + continue + } + foundExtended = true + assert.Equal(t, "httproute/ns-test/test/rule/1", r.Route.Cluster) + assert.Equal(t, "/ws", r.Match["prefix"]) + } + assert.True(t, foundExtended, "expected extended CONNECT path-aware route") + + // One pure CONNECT fallback route should target fallback (rule 0, path "/"). + foundFallback := false + for _, r := range routes { + if r.Name != "connector-connect-test" { + continue + } + foundFallback = true + assert.Equal(t, "httproute/ns-test/test/rule/0", r.Route.Cluster) + _, hasConnectMatcher := r.Match["connect_matcher"] + assert.True(t, hasConnectMatcher) + } + assert.True(t, foundFallback, "expected pure CONNECT fallback route") +} + +func TestBuildConnectorEnvoyPatchesScopesRouteConfigBySectionName(t *testing.T) { + gateway := &gatewayv1.Gateway{ + ObjectMeta: metav1.ObjectMeta{Name: "gw"}, + Spec: gatewayv1.GatewaySpec{ + Listeners: []gatewayv1.Listener{ + { + Name: gatewayv1.SectionName("default-https"), + Protocol: gatewayv1.HTTPSProtocolType, + }, + { + Name: gatewayv1.SectionName("https-hostname-0"), + Protocol: gatewayv1.HTTPSProtocolType, + }, + }, + }, + } + + backends := []connectorBackendPatch{ + { + sectionName: ptr.To(gatewayv1.SectionName("https-hostname-0")), + ruleIndex: 0, + matchIndex: 0, + targetHost: "127.0.0.1", + targetPort: 5432, + nodeID: "node-123", + }, + } + + patches, err := buildConnectorEnvoyPatches( + "ns-test", + "connector-tunnel", + gateway, + newHTTPProxy(), + backends, + ) + assert.NoError(t, err) + + routeConfigPatchCounts := map[string]int{} + for _, patch := range patches { + if patch.Type != "type.googleapis.com/envoy.config.route.v3.RouteConfiguration" { + continue + } + routeConfigPatchCounts[patch.Name]++ + } + + assert.Equal(t, 2, routeConfigPatchCounts["ns-test/gw/https-hostname-0"]) + assert.NotContains(t, routeConfigPatchCounts, "ns-test/gw/default-https") +} + func TestHTTPProxyFinalizerCleanup(t *testing.T) { logger := zap.New(zap.UseFlagOptions(&zap.Options{Development: true})) ctx := log.IntoContext(context.Background(), logger) From 92de1f1e6529a31795d32f2d32f62444977d3b20 Mon Sep 17 00:00:00 2001 From: Zach Smith Date: Fri, 13 Feb 2026 13:01:51 -0800 Subject: [PATCH 20/24] chore: fix status naming --- api/v1alpha/httpproxy_types.go | 8 ++++---- internal/controller/httpproxy_controller.go | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/api/v1alpha/httpproxy_types.go b/api/v1alpha/httpproxy_types.go index 48bfd261..6a4782e9 100644 --- a/api/v1alpha/httpproxy_types.go +++ b/api/v1alpha/httpproxy_types.go @@ -219,9 +219,9 @@ const ( // is in use by another resource. HTTPProxyConditionHostnamesInUse = "HostnamesInUse" - // This condition is true when connector tunnel metadata has been programmed + // This condition is true when connector metadata has been programmed // via the downstream EnvoyPatchPolicy. - HTTPProxyConditionTunnelMetadataProgrammed = "TunnelMetadataProgrammed" + HTTPProxyConditionConnectorMetadataProgrammed = "ConnectorMetadataProgrammed" ) const ( @@ -232,8 +232,8 @@ const ( // HTTPProxyReasonProgrammed indicates that the HTTP proxy has been programmed. HTTPProxyReasonProgrammed = "Programmed" - // HTTPProxyReasonTunnelMetadataApplied indicates tunnel metadata has been applied. - HTTPProxyReasonTunnelMetadataApplied = "TunnelMetadataApplied" + // HTTPProxyReasonConnectorMetadataApplied indicates connector metadata has been applied. + HTTPProxyReasonConnectorMetadataApplied = "ConnectorMetadataApplied" // HTTPProxyReasonConflict indicates that the HTTP proxy encountered a conflict // when being programmed. diff --git a/internal/controller/httpproxy_controller.go b/internal/controller/httpproxy_controller.go index 5e0cd054..d18906cc 100644 --- a/internal/controller/httpproxy_controller.go +++ b/internal/controller/httpproxy_controller.go @@ -145,7 +145,7 @@ func (r *HTTPProxyReconciler) Reconcile(ctx context.Context, req mcreconcile.Req } tunnelMetadataCondition := &metav1.Condition{ - Type: networkingv1alpha.HTTPProxyConditionTunnelMetadataProgrammed, + Type: networkingv1alpha.HTTPProxyConditionConnectorMetadataProgrammed, Status: metav1.ConditionFalse, Reason: networkingv1alpha.HTTPProxyReasonPending, ObservedGeneration: httpProxy.Generation, @@ -159,7 +159,7 @@ func (r *HTTPProxyReconciler) Reconcile(ctx context.Context, req mcreconcile.Req if setTunnelMetadataCondition { apimeta.SetStatusCondition(&httpProxyCopy.Status.Conditions, *tunnelMetadataCondition) } else { - apimeta.RemoveStatusCondition(&httpProxyCopy.Status.Conditions, networkingv1alpha.HTTPProxyConditionTunnelMetadataProgrammed) + apimeta.RemoveStatusCondition(&httpProxyCopy.Status.Conditions, networkingv1alpha.HTTPProxyConditionConnectorMetadataProgrammed) } if !equality.Semantic.DeepEqual(httpProxy.Status, httpProxyCopy.Status) { @@ -363,12 +363,12 @@ func (r *HTTPProxyReconciler) Reconcile(ctx context.Context, req mcreconcile.Req tunnelMetadataCondition.Message = connectorPolicyMessage } else { tunnelMetadataCondition.Status = metav1.ConditionTrue - tunnelMetadataCondition.Reason = networkingv1alpha.HTTPProxyReasonTunnelMetadataApplied + tunnelMetadataCondition.Reason = networkingv1alpha.HTTPProxyReasonConnectorMetadataApplied tunnelMetadataCondition.Message = "Connector tunnel metadata applied" } setTunnelMetadataCondition = true } else { - apimeta.RemoveStatusCondition(&httpProxyCopy.Status.Conditions, networkingv1alpha.HTTPProxyConditionTunnelMetadataProgrammed) + apimeta.RemoveStatusCondition(&httpProxyCopy.Status.Conditions, networkingv1alpha.HTTPProxyConditionConnectorMetadataProgrammed) } r.reconcileHTTPProxyHostnameStatus(ctx, gateway, httpProxyCopy) From 8e59fbbb20e11518bebd12fe59a8bc73a72e27ec Mon Sep 17 00:00:00 2001 From: Zach Smith Date: Fri, 13 Feb 2026 13:19:19 -0800 Subject: [PATCH 21/24] fix: lint --- .../controller/connector_routing_compiler.go | 11 +++---- internal/controller/httpproxy_controller.go | 29 ++----------------- .../controller/httpproxy_controller_test.go | 10 ++++--- 3 files changed, 13 insertions(+), 37 deletions(-) diff --git a/internal/controller/connector_routing_compiler.go b/internal/controller/connector_routing_compiler.go index 4605c30f..ee68a4fb 100644 --- a/internal/controller/connector_routing_compiler.go +++ b/internal/controller/connector_routing_compiler.go @@ -170,10 +170,7 @@ func buildConnectorEnvoyPatches( // matches method CONNECT with a non-root path. // // Insert at index 0 so CONNECT routes are matched before path-based routes. - connectRoutes, err := buildConnectorConnectRoutes(downstreamNamespace, httpProxy, backends) - if err != nil { - return nil, err - } + connectRoutes := buildConnectorConnectRoutes(downstreamNamespace, httpProxy, backends) for _, connectRoute := range connectRoutes { routeValue, err := json.Marshal(connectRoute.route) if err != nil { @@ -250,9 +247,9 @@ func buildConnectorConnectRoutes( downstreamNamespace string, httpProxy *networkingv1alpha.HTTPProxy, backends []connectorBackendPatch, -) ([]connectorConnectRoute, error) { +) []connectorConnectRoute { if len(backends) == 0 { - return nil, nil + return nil } clusterByRule := map[int]string{} @@ -356,7 +353,7 @@ func buildConnectorConnectRoutes( }, }) - return connectRoutes, nil + return connectRoutes } func connectorRouteJSONPath( diff --git a/internal/controller/httpproxy_controller.go b/internal/controller/httpproxy_controller.go index d18906cc..0375238f 100644 --- a/internal/controller/httpproxy_controller.go +++ b/internal/controller/httpproxy_controller.go @@ -98,7 +98,6 @@ func (r *HTTPProxyReconciler) Reconcile(ctx context.Context, req mcreconcile.Req return ctrl.Result{}, err } controllerutil.RemoveFinalizer(&httpProxy, httpProxyFinalizer) - normalizeHTTPProxyBackendEndpoints(&httpProxy) if err := cl.GetClient().Update(ctx, &httpProxy); err != nil { return ctrl.Result{}, err } @@ -110,7 +109,6 @@ func (r *HTTPProxyReconciler) Reconcile(ctx context.Context, req mcreconcile.Req defer logger.Info("reconcile complete") if updated := ensureConnectorNameAnnotation(&httpProxy); updated { - normalizeHTTPProxyBackendEndpoints(&httpProxy) if err := cl.GetClient().Update(ctx, &httpProxy); err != nil { return ctrl.Result{}, fmt.Errorf("failed updating httpproxy connector annotation: %w", err) } @@ -119,7 +117,6 @@ func (r *HTTPProxyReconciler) Reconcile(ctx context.Context, req mcreconcile.Req if !controllerutil.ContainsFinalizer(&httpProxy, httpProxyFinalizer) { controllerutil.AddFinalizer(&httpProxy, httpProxyFinalizer) - normalizeHTTPProxyBackendEndpoints(&httpProxy) if err := cl.GetClient().Update(ctx, &httpProxy); err != nil { return ctrl.Result{}, err } @@ -484,6 +481,9 @@ func (r *HTTPProxyReconciler) reconcileHTTPProxyHostnameStatus( } } +// Today we store only a single connector name for filtering stability +// because selector fields on nested arrays (rules[].backends[].connector) are +// not supported in a way that lets us index and watch those references directly. func ensureConnectorNameAnnotation(httpProxy *networkingv1alpha.HTTPProxy) bool { var connectorName string for _, rule := range httpProxy.Spec.Rules { @@ -1121,29 +1121,6 @@ func collectConnectorBackends( return connectorBackends, nil } -// normalizeHTTPProxyBackendEndpoints strips path, query, and fragment from each -// rule backend endpoint URL so that specs like "http://example.com/" pass -// validation (endpoint must not have a path component). Mutates proxy in place. -func normalizeHTTPProxyBackendEndpoints(proxy *networkingv1alpha.HTTPProxy) { - for i := range proxy.Spec.Rules { - for j := range proxy.Spec.Rules[i].Backends { - ep := proxy.Spec.Rules[i].Backends[j].Endpoint - if ep == "" { - continue - } - u, err := url.Parse(ep) - if err != nil { - continue - } - u.Path = "" - u.RawPath = "" - u.RawQuery = "" - u.Fragment = "" - proxy.Spec.Rules[i].Backends[j].Endpoint = u.String() - } - } -} - func backendEndpointTarget(backend networkingv1alpha.HTTPProxyRuleBackend) (string, int, error) { u, err := url.Parse(backend.Endpoint) if err != nil { diff --git a/internal/controller/httpproxy_controller_test.go b/internal/controller/httpproxy_controller_test.go index 3ba119e6..3c52efa0 100644 --- a/internal/controller/httpproxy_controller_test.go +++ b/internal/controller/httpproxy_controller_test.go @@ -37,6 +37,8 @@ import ( gatewayutil "go.datum.net/network-services-operator/internal/util/gateway" ) +const routeConfigurationTypeURL = "type.googleapis.com/envoy.config.route.v3.RouteConfiguration" + //nolint:gocyclo func TestHTTPProxyCollectDesiredResources(t *testing.T) { @@ -1235,7 +1237,7 @@ func TestBuildConnectorEnvoyPatchesTargetsAllHTTPSListeners(t *testing.T) { routeConfigPatchCounts := map[string]int{} for _, patch := range patches { - if patch.Type != "type.googleapis.com/envoy.config.route.v3.RouteConfiguration" { + if patch.Type != routeConfigurationTypeURL { continue } routeConfigPatchCounts[patch.Name]++ @@ -1336,9 +1338,9 @@ func TestBuildConnectorEnvoyPatchesAddsExtendedConnectPathRouteAndFallback(t *te } `json:"route"` } - var routes []routeDoc + routes := make([]routeDoc, 0, len(patches)) for _, patch := range patches { - if patch.Type != "type.googleapis.com/envoy.config.route.v3.RouteConfiguration" { + if patch.Type != routeConfigurationTypeURL { continue } if ptr.Deref(patch.Operation.Path, "") != "/virtual_hosts/0/routes/0" { @@ -1416,7 +1418,7 @@ func TestBuildConnectorEnvoyPatchesScopesRouteConfigBySectionName(t *testing.T) routeConfigPatchCounts := map[string]int{} for _, patch := range patches { - if patch.Type != "type.googleapis.com/envoy.config.route.v3.RouteConfiguration" { + if patch.Type != routeConfigurationTypeURL { continue } routeConfigPatchCounts[patch.Name]++ From fffb2a35ff226678093570ccbc96dae281998ea1 Mon Sep 17 00:00:00 2001 From: Zach Smith Date: Fri, 13 Feb 2026 13:24:48 -0800 Subject: [PATCH 22/24] chore: refactor synthetic connector endpoint slice name --- internal/controller/httpproxy_controller.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/controller/httpproxy_controller.go b/internal/controller/httpproxy_controller.go index 0375238f..58cefad4 100644 --- a/internal/controller/httpproxy_controller.go +++ b/internal/controller/httpproxy_controller.go @@ -755,7 +755,7 @@ func (r *HTTPProxyReconciler) collectDesiredResources( isIPAddress := false if backend.Connector != nil { // Connector backends don't rely on EndpointSlice addresses; use a safe placeholder. - endpointHost = "connector.invalid" + endpointHost = "connector.local" addressType = discoveryv1.AddressTypeFQDN } else if ip := net.ParseIP(host); ip != nil { isIPAddress = true From 83ad5937cf388af79a03230db6265439312eba7c Mon Sep 17 00:00:00 2001 From: Zach Smith Date: Wed, 18 Feb 2026 11:18:50 -0800 Subject: [PATCH 23/24] chore: adjust status message --- internal/controller/httpproxy_controller.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/controller/httpproxy_controller.go b/internal/controller/httpproxy_controller.go index 58cefad4..9e115e00 100644 --- a/internal/controller/httpproxy_controller.go +++ b/internal/controller/httpproxy_controller.go @@ -146,7 +146,7 @@ func (r *HTTPProxyReconciler) Reconcile(ctx context.Context, req mcreconcile.Req Status: metav1.ConditionFalse, Reason: networkingv1alpha.HTTPProxyReasonPending, ObservedGeneration: httpProxy.Generation, - Message: "Waiting for downstream EnvoyPatchPolicy to be accepted and programmed", + Message: "Waiting for envoy to be configured", } setTunnelMetadataCondition := false From a29d2c255ab52ffca7eca2de8c9be524102d30c1 Mon Sep 17 00:00:00 2001 From: Zach Smith Date: Wed, 18 Feb 2026 11:34:55 -0800 Subject: [PATCH 24/24] fix: remove annotation for connector --- internal/controller/httpproxy_controller.go | 53 --------------------- 1 file changed, 53 deletions(-) diff --git a/internal/controller/httpproxy_controller.go b/internal/controller/httpproxy_controller.go index 9e115e00..680c57ac 100644 --- a/internal/controller/httpproxy_controller.go +++ b/internal/controller/httpproxy_controller.go @@ -108,13 +108,6 @@ func (r *HTTPProxyReconciler) Reconcile(ctx context.Context, req mcreconcile.Req logger.Info("reconciling httpproxy") defer logger.Info("reconcile complete") - if updated := ensureConnectorNameAnnotation(&httpProxy); updated { - if err := cl.GetClient().Update(ctx, &httpProxy); err != nil { - return ctrl.Result{}, fmt.Errorf("failed updating httpproxy connector annotation: %w", err) - } - return ctrl.Result{}, nil - } - if !controllerutil.ContainsFinalizer(&httpProxy, httpProxyFinalizer) { controllerutil.AddFinalizer(&httpProxy, httpProxyFinalizer) if err := cl.GetClient().Update(ctx, &httpProxy); err != nil { @@ -481,52 +474,6 @@ func (r *HTTPProxyReconciler) reconcileHTTPProxyHostnameStatus( } } -// Today we store only a single connector name for filtering stability -// because selector fields on nested arrays (rules[].backends[].connector) are -// not supported in a way that lets us index and watch those references directly. -func ensureConnectorNameAnnotation(httpProxy *networkingv1alpha.HTTPProxy) bool { - var connectorName string - for _, rule := range httpProxy.Spec.Rules { - for _, backend := range rule.Backends { - if backend.Connector != nil && backend.Connector.Name != "" { - if connectorName == "" { - connectorName = backend.Connector.Name - } else if connectorName != backend.Connector.Name { - // Prefer first connector for annotation stability if multiple are present. - break - } - } - } - } - - annotations := httpProxy.GetAnnotations() - if connectorName == "" { - if annotations == nil { - return false - } - if _, ok := annotations[networkingv1alpha1.ConnectorNameAnnotation]; !ok { - return false - } - delete(annotations, networkingv1alpha1.ConnectorNameAnnotation) - if len(annotations) == 0 { - httpProxy.SetAnnotations(nil) - } else { - httpProxy.SetAnnotations(annotations) - } - return true - } - - if annotations == nil { - annotations = map[string]string{} - } - if annotations[networkingv1alpha1.ConnectorNameAnnotation] == connectorName { - return false - } - annotations[networkingv1alpha1.ConnectorNameAnnotation] = connectorName - httpProxy.SetAnnotations(annotations) - return true -} - // SetupWithManager sets up the controller with the Manager. func (r *HTTPProxyReconciler) SetupWithManager(mgr mcmanager.Manager) error { r.mgr = mgr