From 147cd8452958396bfcf310f4ec53fa84b4a59223 Mon Sep 17 00:00:00 2001 From: Anna Williamson Date: Thu, 29 Jan 2026 11:45:12 -0800 Subject: [PATCH 1/3] Add toleration for prepull daemonset --- internal/controller/core/site_controller_pre_pull.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/internal/controller/core/site_controller_pre_pull.go b/internal/controller/core/site_controller_pre_pull.go index c67d65cc..14d62298 100644 --- a/internal/controller/core/site_controller_pre_pull.go +++ b/internal/controller/core/site_controller_pre_pull.go @@ -104,6 +104,13 @@ func deployPrePullDaemonset(ctx context.Context, r *SiteReconciler, req controll }, } + // Add universal toleration to run on all nodes regardless of taints + prePullDaemonset.Spec.Template.Spec.Tolerations = []v1.Toleration{ + { + Operator: v1.TolerationOpExists, + }, + } + if len(site.Spec.Workbench.Tolerations) > 0 { // add the tolerations to the daemonset for _, t := range site.Spec.Workbench.Tolerations { From e60bc4660e776bf1e19ae64876ccf6304d04b812 Mon Sep 17 00:00:00 2001 From: Anna Williamson Date: Thu, 29 Jan 2026 15:47:23 -0800 Subject: [PATCH 2/3] Improved scheduling of prepull daemonset --- api/core/v1beta1/site_types.go | 5 +++++ api/core/v1beta1/zz_generated.deepcopy.go | 5 +++++ .../core/v1beta1/sitespec.go | 11 ++++++++++ config/crd/bases/core.posit.team_sites.yaml | 8 +++++++ .../core/site_controller_pre_pull.go | 21 +++++++++++++++++++ 5 files changed, 50 insertions(+) diff --git a/api/core/v1beta1/site_types.go b/api/core/v1beta1/site_types.go index da24cada..229dc069 100644 --- a/api/core/v1beta1/site_types.go +++ b/api/core/v1beta1/site_types.go @@ -95,6 +95,11 @@ type SiteSpec struct { DisablePrePullImages bool `json:"disablePrePullImages,omitempty"` + // PrepullNodePools is a list of Karpenter NodePool names to target for image prepulling. + // When set, the prepull DaemonSet will use node affinity to only run on nodes from these pools. + // This is typically populated automatically for node pools with session_taints enabled. + PrepullNodePools []string `json:"prepullNodePools,omitempty"` + // MainDatabaseCredentialSecret configures the secret used for storing the main database credentials MainDatabaseCredentialSecret SecretConfig `json:"mainDatabaseCredentialSecret,omitempty"` diff --git a/api/core/v1beta1/zz_generated.deepcopy.go b/api/core/v1beta1/zz_generated.deepcopy.go index 9471a100..296ecfe1 100644 --- a/api/core/v1beta1/zz_generated.deepcopy.go +++ b/api/core/v1beta1/zz_generated.deepcopy.go @@ -2135,6 +2135,11 @@ func (in *SiteSpec) DeepCopyInto(out *SiteSpec) { } out.Secret = in.Secret out.WorkloadSecret = in.WorkloadSecret + if in.PrepullNodePools != nil { + in, out := &in.PrepullNodePools, &out.PrepullNodePools + *out = make([]string, len(*in)) + copy(*out, *in) + } out.MainDatabaseCredentialSecret = in.MainDatabaseCredentialSecret if in.EnableFQDNHealthChecks != nil { in, out := &in.EnableFQDNHealthChecks, &out.EnableFQDNHealthChecks diff --git a/client-go/applyconfiguration/core/v1beta1/sitespec.go b/client-go/applyconfiguration/core/v1beta1/sitespec.go index b13749bf..f3bcf3ee 100644 --- a/client-go/applyconfiguration/core/v1beta1/sitespec.go +++ b/client-go/applyconfiguration/core/v1beta1/sitespec.go @@ -34,6 +34,7 @@ type SiteSpecApplyConfiguration struct { Secret *SecretConfigApplyConfiguration `json:"secret,omitempty"` WorkloadSecret *SecretConfigApplyConfiguration `json:"workloadSecret,omitempty"` DisablePrePullImages *bool `json:"disablePrePullImages,omitempty"` + PrepullNodePools []string `json:"prepullNodePools,omitempty"` MainDatabaseCredentialSecret *SecretConfigApplyConfiguration `json:"mainDatabaseCredentialSecret,omitempty"` DropDatabaseOnTeardown *bool `json:"dropDatabaseOnTearDown,omitempty"` Debug *bool `json:"debug,omitempty"` @@ -232,6 +233,16 @@ func (b *SiteSpecApplyConfiguration) WithDisablePrePullImages(value bool) *SiteS return b } +// WithPrepullNodePools adds the given value to the PrepullNodePools field in the declarative configuration +// and returns the receiver, so that objects can be build by chaining "With" function invocations. +// If called multiple times, values provided by each call will be appended to the PrepullNodePools field. +func (b *SiteSpecApplyConfiguration) WithPrepullNodePools(values ...string) *SiteSpecApplyConfiguration { + for i := range values { + b.PrepullNodePools = append(b.PrepullNodePools, values[i]) + } + return b +} + // WithMainDatabaseCredentialSecret sets the MainDatabaseCredentialSecret field in the declarative configuration to the given value // and returns the receiver, so that objects can be built by chaining "With" function invocations. // If called multiple times, the MainDatabaseCredentialSecret field is set to the value of the last call. diff --git a/config/crd/bases/core.posit.team_sites.yaml b/config/crd/bases/core.posit.team_sites.yaml index 0524fcb6..8d30a5c4 100644 --- a/config/crd/bases/core.posit.team_sites.yaml +++ b/config/crd/bases/core.posit.team_sites.yaml @@ -737,6 +737,14 @@ spec: PackageManagerUrl specifies the Package Manager URL for Workbench to use If empty, Workbench will use the local Package Manager URL by default type: string + prepullNodePools: + description: |- + PrepullNodePools is a list of Karpenter NodePool names to target for image prepulling. + When set, the prepull DaemonSet will use node affinity to only run on nodes from these pools. + This is typically populated automatically for node pools with session_taints enabled. + items: + type: string + type: array secret: description: Secret configures the secret management for this Site properties: diff --git a/internal/controller/core/site_controller_pre_pull.go b/internal/controller/core/site_controller_pre_pull.go index 14d62298..3afcb975 100644 --- a/internal/controller/core/site_controller_pre_pull.go +++ b/internal/controller/core/site_controller_pre_pull.go @@ -111,6 +111,27 @@ func deployPrePullDaemonset(ctx context.Context, r *SiteReconciler, req controll }, } + // Add node affinity to target specific node pools if specified + if len(site.Spec.PrepullNodePools) > 0 { + prePullDaemonset.Spec.Template.Spec.Affinity = &v1.Affinity{ + NodeAffinity: &v1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: "karpenter.sh/nodepool", + Operator: v1.NodeSelectorOpIn, + Values: site.Spec.PrepullNodePools, + }, + }, + }, + }, + }, + }, + } + } + if len(site.Spec.Workbench.Tolerations) > 0 { // add the tolerations to the daemonset for _, t := range site.Spec.Workbench.Tolerations { From 6de896e7dbc77e1308208be4079654221b66919a Mon Sep 17 00:00:00 2001 From: Anna Williamson Date: Fri, 30 Jan 2026 15:04:15 -0800 Subject: [PATCH 3/3] Revert prepullnode plan, implement anti-affinity for system nodes --- api/core/v1beta1/site_types.go | 5 -- api/core/v1beta1/zz_generated.deepcopy.go | 5 -- .../core/v1beta1/sitespec.go | 11 --- config/crd/bases/core.posit.team_sites.yaml | 8 -- .../templates/crd/core.posit.team_sites.yaml | 73 +++++++++++++------ .../core/site_controller_pre_pull.go | 35 +++------ 6 files changed, 62 insertions(+), 75 deletions(-) diff --git a/api/core/v1beta1/site_types.go b/api/core/v1beta1/site_types.go index 229dc069..da24cada 100644 --- a/api/core/v1beta1/site_types.go +++ b/api/core/v1beta1/site_types.go @@ -95,11 +95,6 @@ type SiteSpec struct { DisablePrePullImages bool `json:"disablePrePullImages,omitempty"` - // PrepullNodePools is a list of Karpenter NodePool names to target for image prepulling. - // When set, the prepull DaemonSet will use node affinity to only run on nodes from these pools. - // This is typically populated automatically for node pools with session_taints enabled. - PrepullNodePools []string `json:"prepullNodePools,omitempty"` - // MainDatabaseCredentialSecret configures the secret used for storing the main database credentials MainDatabaseCredentialSecret SecretConfig `json:"mainDatabaseCredentialSecret,omitempty"` diff --git a/api/core/v1beta1/zz_generated.deepcopy.go b/api/core/v1beta1/zz_generated.deepcopy.go index 296ecfe1..9471a100 100644 --- a/api/core/v1beta1/zz_generated.deepcopy.go +++ b/api/core/v1beta1/zz_generated.deepcopy.go @@ -2135,11 +2135,6 @@ func (in *SiteSpec) DeepCopyInto(out *SiteSpec) { } out.Secret = in.Secret out.WorkloadSecret = in.WorkloadSecret - if in.PrepullNodePools != nil { - in, out := &in.PrepullNodePools, &out.PrepullNodePools - *out = make([]string, len(*in)) - copy(*out, *in) - } out.MainDatabaseCredentialSecret = in.MainDatabaseCredentialSecret if in.EnableFQDNHealthChecks != nil { in, out := &in.EnableFQDNHealthChecks, &out.EnableFQDNHealthChecks diff --git a/client-go/applyconfiguration/core/v1beta1/sitespec.go b/client-go/applyconfiguration/core/v1beta1/sitespec.go index f3bcf3ee..b13749bf 100644 --- a/client-go/applyconfiguration/core/v1beta1/sitespec.go +++ b/client-go/applyconfiguration/core/v1beta1/sitespec.go @@ -34,7 +34,6 @@ type SiteSpecApplyConfiguration struct { Secret *SecretConfigApplyConfiguration `json:"secret,omitempty"` WorkloadSecret *SecretConfigApplyConfiguration `json:"workloadSecret,omitempty"` DisablePrePullImages *bool `json:"disablePrePullImages,omitempty"` - PrepullNodePools []string `json:"prepullNodePools,omitempty"` MainDatabaseCredentialSecret *SecretConfigApplyConfiguration `json:"mainDatabaseCredentialSecret,omitempty"` DropDatabaseOnTeardown *bool `json:"dropDatabaseOnTearDown,omitempty"` Debug *bool `json:"debug,omitempty"` @@ -233,16 +232,6 @@ func (b *SiteSpecApplyConfiguration) WithDisablePrePullImages(value bool) *SiteS return b } -// WithPrepullNodePools adds the given value to the PrepullNodePools field in the declarative configuration -// and returns the receiver, so that objects can be build by chaining "With" function invocations. -// If called multiple times, values provided by each call will be appended to the PrepullNodePools field. -func (b *SiteSpecApplyConfiguration) WithPrepullNodePools(values ...string) *SiteSpecApplyConfiguration { - for i := range values { - b.PrepullNodePools = append(b.PrepullNodePools, values[i]) - } - return b -} - // WithMainDatabaseCredentialSecret sets the MainDatabaseCredentialSecret field in the declarative configuration to the given value // and returns the receiver, so that objects can be built by chaining "With" function invocations. // If called multiple times, the MainDatabaseCredentialSecret field is set to the value of the last call. diff --git a/config/crd/bases/core.posit.team_sites.yaml b/config/crd/bases/core.posit.team_sites.yaml index 8d30a5c4..0524fcb6 100644 --- a/config/crd/bases/core.posit.team_sites.yaml +++ b/config/crd/bases/core.posit.team_sites.yaml @@ -737,14 +737,6 @@ spec: PackageManagerUrl specifies the Package Manager URL for Workbench to use If empty, Workbench will use the local Package Manager URL by default type: string - prepullNodePools: - description: |- - PrepullNodePools is a list of Karpenter NodePool names to target for image prepulling. - When set, the prepull DaemonSet will use node affinity to only run on nodes from these pools. - This is typically populated automatically for node pools with session_taints enabled. - items: - type: string - type: array secret: description: Secret configures the secret management for this Site properties: diff --git a/dist/chart/templates/crd/core.posit.team_sites.yaml b/dist/chart/templates/crd/core.posit.team_sites.yaml index 8664d14d..0524fcb6 100755 --- a/dist/chart/templates/crd/core.posit.team_sites.yaml +++ b/dist/chart/templates/crd/core.posit.team_sites.yaml @@ -1,32 +1,11 @@ -{{- if .Values.crd.enable }} --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: - labels: - {{- include "chart.labels" . | nindent 4 }} annotations: - {{- if .Values.certmanager.enable }} - cert-manager.io/inject-ca-from: "{{ .Release.Namespace }}/serving-cert" - {{- end }} - {{- if .Values.crd.keep }} - "helm.sh/resource-policy": keep - {{- end }} controller-gen.kubebuilder.io/version: v0.17.0 name: sites.core.posit.team spec: - {{- if .Values.webhook.enable }} - conversion: - strategy: Webhook - webhook: - clientConfig: - service: - namespace: {{ .Release.Namespace }} - name: webhook-service - path: /convert - conversionReviewVersions: - - v1 - {{- end }} group: core.posit.team names: kind: Site @@ -486,6 +465,11 @@ spec: flightdeck: description: Flightdeck contains Flightdeck configuration properties: + enabled: + description: |- + Enabled controls whether Flightdeck is deployed. Defaults to true if not specified. + Set to false to explicitly disable Flightdeck deployment. + type: boolean featureEnabler: description: FeatureEnabler controls which features are enabled in Flightdeck @@ -500,7 +484,11 @@ spec: type: boolean type: object image: - description: Image is the container image for Flightdeck + description: |- + Image is the container image for Flightdeck. + Can be a tag (e.g., "v1.2.3") which will be combined with the default registry, + or a full image path (e.g., "my-registry.io/flightdeck:v1.0.0"). + Defaults to "docker.io/posit/ptd-flightdeck:latest" if not specified. type: string imagePullPolicy: description: ImagePullPolicy controls when the kubelet pulls the @@ -1328,6 +1316,46 @@ spec: description: SessionInitContainerImageTag specifies the init container image tag for Workbench sessions type: string + sessionTolerations: + description: SessionTolerations are tolerations applied only to + session pods (not the main workbench server) + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array snowflake: properties: accountId: @@ -1459,4 +1487,3 @@ spec: storage: true subresources: status: {} -{{- end -}} diff --git a/internal/controller/core/site_controller_pre_pull.go b/internal/controller/core/site_controller_pre_pull.go index 3afcb975..b3e86ffd 100644 --- a/internal/controller/core/site_controller_pre_pull.go +++ b/internal/controller/core/site_controller_pre_pull.go @@ -111,34 +111,23 @@ func deployPrePullDaemonset(ctx context.Context, r *SiteReconciler, req controll }, } - // Add node affinity to target specific node pools if specified - if len(site.Spec.PrepullNodePools) > 0 { - prePullDaemonset.Spec.Template.Spec.Affinity = &v1.Affinity{ - NodeAffinity: &v1.NodeAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ - NodeSelectorTerms: []v1.NodeSelectorTerm{ - { - MatchExpressions: []v1.NodeSelectorRequirement{ - { - Key: "karpenter.sh/nodepool", - Operator: v1.NodeSelectorOpIn, - Values: site.Spec.PrepullNodePools, - }, + // Add anti-affinity to avoid scheduling on system nodes (nodes labeled with posit.team/node-role: system) + prePullDaemonset.Spec.Template.Spec.Affinity = &v1.Affinity{ + NodeAffinity: &v1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: "posit.team/node-role", + Operator: v1.NodeSelectorOpNotIn, + Values: []string{"system"}, }, }, }, }, }, - } - } - - if len(site.Spec.Workbench.Tolerations) > 0 { - // add the tolerations to the daemonset - for _, t := range site.Spec.Workbench.Tolerations { - prePullDaemonset.Spec.Template.Spec.Tolerations = append(prePullDaemonset.Spec.Template.Spec.Tolerations, *t.DeepCopy()) - } - - // TODO: should also use the workbench node selectors...? But could differ from Connect... + }, } return nil }); err != nil {