From 2e5a14acea8938bda2e2ff9648ecf48507991558 Mon Sep 17 00:00:00 2001 From: Aaron Date: Sat, 29 Aug 2026 21:34:39 +0300 Subject: [PATCH 1/2] feat: add affinity and nodeSelector to common daemonset config Expose spec.daemonsets.affinity and spec.daemonsets.nodeSelector on ClusterPolicy and wire them through Helm. User nodeSelector labels are merged with existing operand nodeSelector entries. Use a schemaless CRD field for affinity so we do not embed the full corev1.Affinity OpenAPI schema in the ClusterPolicy CRD. Signed-off-by: amarkdotdev # Conflicts: # controllers/transforms_test.go Signed-off-by: Aaron Signed-off-by: Aaron --- api/nvidia/v1/clusterpolicy_types.go | 15 +++++ api/nvidia/v1/zz_generated.deepcopy.go | 12 ++++ .../manifests/nvidia.com_clusterpolicies.yaml | 10 +++ .../crd/bases/nvidia.com_clusterpolicies.yaml | 10 +++ controllers/object_controls.go | 13 ++++ controllers/transforms_test.go | 65 +++++++++++++++++++ .../crds/nvidia.com_clusterpolicies.yaml | 10 +++ .../gpu-operator/templates/clusterpolicy.yaml | 6 ++ deployments/gpu-operator/values.yaml | 2 + 9 files changed, 143 insertions(+) diff --git a/api/nvidia/v1/clusterpolicy_types.go b/api/nvidia/v1/clusterpolicy_types.go index 5c3316b582..296aa437d9 100644 --- a/api/nvidia/v1/clusterpolicy_types.go +++ b/api/nvidia/v1/clusterpolicy_types.go @@ -309,6 +309,21 @@ type DaemonsetsSpec struct { // +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.x-descriptors="urn:alm:descriptor:com.tectonic.ui:advanced,urn:alm:descriptor:io.kubernetes:Tolerations" Tolerations []corev1.Toleration `json:"tolerations,omitempty"` + // Optional: Set nodeSelector merged with operand nodeSelector labels + // +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors=true + // +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.displayName="NodeSelector for all Daemonsets" + // +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.x-descriptors="urn:alm:descriptor:com.tectonic.ui:advanced,urn:alm:descriptor:com.tectonic.ui:nodeSelector" + NodeSelector map[string]string `json:"nodeSelector,omitempty"` + + // Optional: Set affinity for all Daemonsets + // +kubebuilder:validation:Type=object + // +kubebuilder:validation:Schemaless + // +kubebuilder:pruning:PreserveUnknownFields + // +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors=true + // +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.displayName="Affinity for all Daemonsets" + // +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.x-descriptors="urn:alm:descriptor:com.tectonic.ui:advanced,urn:alm:descriptor:com.tectonic.ui:affinity" + Affinity *corev1.Affinity `json:"affinity,omitempty"` + // +kubebuilder:validation:Optional // +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors=true // +operator-sdk:gen-csv:customresourcedefinitions.specDescriptors.displayName="PriorityClassName" diff --git a/api/nvidia/v1/zz_generated.deepcopy.go b/api/nvidia/v1/zz_generated.deepcopy.go index 9e936de60d..8a01aa39d7 100644 --- a/api/nvidia/v1/zz_generated.deepcopy.go +++ b/api/nvidia/v1/zz_generated.deepcopy.go @@ -525,6 +525,18 @@ func (in *DaemonsetsSpec) DeepCopyInto(out *DaemonsetsSpec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.NodeSelector != nil { + in, out := &in.NodeSelector, &out.NodeSelector + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Affinity != nil { + in, out := &in.Affinity, &out.Affinity + *out = new(corev1.Affinity) + (*in).DeepCopyInto(*out) + } if in.RollingUpdate != nil { in, out := &in.RollingUpdate, &out.RollingUpdate *out = new(RollingUpdateSpec) diff --git a/bundle/manifests/nvidia.com_clusterpolicies.yaml b/bundle/manifests/nvidia.com_clusterpolicies.yaml index 6257b44d5a..fc97c5aeec 100644 --- a/bundle/manifests/nvidia.com_clusterpolicies.yaml +++ b/bundle/manifests/nvidia.com_clusterpolicies.yaml @@ -159,6 +159,10 @@ spec: daemonsets: description: Daemonset defines common configuration for all Daemonsets properties: + affinity: + description: 'Optional: Set affinity for all Daemonsets' + type: object + x-kubernetes-preserve-unknown-fields: true annotations: additionalProperties: type: string @@ -175,6 +179,12 @@ spec: (scope and select) objects. May match selectors of replication controllers and services. type: object + nodeSelector: + additionalProperties: + type: string + description: 'Optional: Set nodeSelector merged with operand nodeSelector + labels' + type: object podSecurityContext: description: 'Optional: Set pod-level security context for all DaemonSet pods (applies as defaults to all containers)' diff --git a/config/crd/bases/nvidia.com_clusterpolicies.yaml b/config/crd/bases/nvidia.com_clusterpolicies.yaml index 6257b44d5a..fc97c5aeec 100644 --- a/config/crd/bases/nvidia.com_clusterpolicies.yaml +++ b/config/crd/bases/nvidia.com_clusterpolicies.yaml @@ -159,6 +159,10 @@ spec: daemonsets: description: Daemonset defines common configuration for all Daemonsets properties: + affinity: + description: 'Optional: Set affinity for all Daemonsets' + type: object + x-kubernetes-preserve-unknown-fields: true annotations: additionalProperties: type: string @@ -175,6 +179,12 @@ spec: (scope and select) objects. May match selectors of replication controllers and services. type: object + nodeSelector: + additionalProperties: + type: string + description: 'Optional: Set nodeSelector merged with operand nodeSelector + labels' + type: object podSecurityContext: description: 'Optional: Set pod-level security context for all DaemonSet pods (applies as defaults to all containers)' diff --git a/controllers/object_controls.go b/controllers/object_controls.go index a4afe67c81..9e698d6c3a 100644 --- a/controllers/object_controls.go +++ b/controllers/object_controls.go @@ -829,6 +829,19 @@ func applyCommonDaemonsetConfig(obj *appsv1.DaemonSet, config *gpuv1.ClusterPoli obj.Spec.Template.Spec.Tolerations = config.Daemonsets.Tolerations } + if len(config.Daemonsets.NodeSelector) > 0 { + if obj.Spec.Template.Spec.NodeSelector == nil { + obj.Spec.Template.Spec.NodeSelector = make(map[string]string) + } + for key, value := range config.Daemonsets.NodeSelector { + obj.Spec.Template.Spec.NodeSelector[key] = value + } + } + + if config.Daemonsets.Affinity != nil { + obj.Spec.Template.Spec.Affinity = config.Daemonsets.Affinity + } + // set pod-level security context if specified (applies as defaults to all containers in the pod) if config.Daemonsets.PodSecurityContext != nil { obj.Spec.Template.Spec.SecurityContext = config.Daemonsets.PodSecurityContext diff --git a/controllers/transforms_test.go b/controllers/transforms_test.go index ff831e7667..82d21d11ca 100644 --- a/controllers/transforms_test.go +++ b/controllers/transforms_test.go @@ -149,6 +149,16 @@ func (d Daemonset) WithTolerations(tolerations []corev1.Toleration) Daemonset { return d } +func (d Daemonset) WithNodeSelector(nodeSelector map[string]string) Daemonset { + d.Spec.Template.Spec.NodeSelector = nodeSelector + return d +} + +func (d Daemonset) WithAffinity(affinity *corev1.Affinity) Daemonset { + d.Spec.Template.Spec.Affinity = affinity + return d +} + func (d Daemonset) WithPodSecurityContext(psc *corev1.PodSecurityContext) Daemonset { d.Spec.Template.Spec.SecurityContext = psc return d @@ -707,6 +717,61 @@ func TestApplyCommonDaemonSetConfig(t *testing.T) { }, }), }, + { + description: "nodeSelector configured", + ds: NewDaemonset().WithNodeSelector(map[string]string{ + "nvidia.com/gpu.deploy.device-plugin": "true", + }), + dsSpec: gpuv1.DaemonsetsSpec{ + NodeSelector: map[string]string{ + "karpenter.sh/nodepool": "gpu", + }, + }, + expectedDs: NewDaemonset().WithNodeSelector(map[string]string{ + "nvidia.com/gpu.deploy.device-plugin": "true", + "karpenter.sh/nodepool": "gpu", + }), + }, + { + description: "affinity configured", + ds: NewDaemonset(), + dsSpec: gpuv1.DaemonsetsSpec{ + Affinity: &corev1.Affinity{ + NodeAffinity: &corev1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ + NodeSelectorTerms: []corev1.NodeSelectorTerm{ + { + MatchExpressions: []corev1.NodeSelectorRequirement{ + { + Key: "karpenter.sh/nodepool", + Operator: corev1.NodeSelectorOpIn, + Values: []string{"gpu"}, + }, + }, + }, + }, + }, + }, + }, + }, + expectedDs: NewDaemonset().WithAffinity(&corev1.Affinity{ + NodeAffinity: &corev1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ + NodeSelectorTerms: []corev1.NodeSelectorTerm{ + { + MatchExpressions: []corev1.NodeSelectorRequirement{ + { + Key: "karpenter.sh/nodepool", + Operator: corev1.NodeSelectorOpIn, + Values: []string{"gpu"}, + }, + }, + }, + }, + }, + }, + }) + }, { description: "invalid updatestrategy configured", ds: NewDaemonset(), diff --git a/deployments/gpu-operator/crds/nvidia.com_clusterpolicies.yaml b/deployments/gpu-operator/crds/nvidia.com_clusterpolicies.yaml index 6257b44d5a..fc97c5aeec 100644 --- a/deployments/gpu-operator/crds/nvidia.com_clusterpolicies.yaml +++ b/deployments/gpu-operator/crds/nvidia.com_clusterpolicies.yaml @@ -159,6 +159,10 @@ spec: daemonsets: description: Daemonset defines common configuration for all Daemonsets properties: + affinity: + description: 'Optional: Set affinity for all Daemonsets' + type: object + x-kubernetes-preserve-unknown-fields: true annotations: additionalProperties: type: string @@ -175,6 +179,12 @@ spec: (scope and select) objects. May match selectors of replication controllers and services. type: object + nodeSelector: + additionalProperties: + type: string + description: 'Optional: Set nodeSelector merged with operand nodeSelector + labels' + type: object podSecurityContext: description: 'Optional: Set pod-level security context for all DaemonSet pods (applies as defaults to all containers)' diff --git a/deployments/gpu-operator/templates/clusterpolicy.yaml b/deployments/gpu-operator/templates/clusterpolicy.yaml index e156f5b7a9..6add2c6521 100644 --- a/deployments/gpu-operator/templates/clusterpolicy.yaml +++ b/deployments/gpu-operator/templates/clusterpolicy.yaml @@ -44,6 +44,12 @@ spec: {{- if .Values.daemonsets.tolerations }} tolerations: {{ toYaml .Values.daemonsets.tolerations | nindent 6 }} {{- end }} + {{- if .Values.daemonsets.nodeSelector }} + nodeSelector: {{ toYaml .Values.daemonsets.nodeSelector | nindent 6 }} + {{- end }} + {{- if .Values.daemonsets.affinity }} + affinity: {{ toYaml .Values.daemonsets.affinity | nindent 6 }} + {{- end }} {{- if .Values.daemonsets.priorityClassName }} priorityClassName: {{ .Values.daemonsets.priorityClassName }} {{- end }} diff --git a/deployments/gpu-operator/values.yaml b/deployments/gpu-operator/values.yaml index 45947b26cc..92ec92048d 100644 --- a/deployments/gpu-operator/values.yaml +++ b/deployments/gpu-operator/values.yaml @@ -41,6 +41,8 @@ hostPaths: daemonsets: labels: {} annotations: {} + nodeSelector: {} + affinity: {} priorityClassName: system-node-critical tolerations: - key: nvidia.com/gpu From aebd56c3949a7f5cd65a401ceb6cbf70c418034d Mon Sep 17 00:00:00 2001 From: Aaron Date: Sat, 29 Aug 2026 21:34:40 +0300 Subject: [PATCH 2/2] chore: regenerate GPUCluster CRDs Regenerate published GPUCluster CRDs for the common daemonset affinity and nodeSelector API fields. Signed-off-by: amarkdotdev Signed-off-by: Aaron --- bundle/manifests/nvidia.com_gpuclusters.yaml | 10 ++++++++++ config/crd/bases/nvidia.com_gpuclusters.yaml | 10 ++++++++++ .../gpu-operator/crds/nvidia.com_gpuclusters.yaml | 10 ++++++++++ 3 files changed, 30 insertions(+) diff --git a/bundle/manifests/nvidia.com_gpuclusters.yaml b/bundle/manifests/nvidia.com_gpuclusters.yaml index d0666b36b0..a97a0a9c02 100644 --- a/bundle/manifests/nvidia.com_gpuclusters.yaml +++ b/bundle/manifests/nvidia.com_gpuclusters.yaml @@ -57,6 +57,10 @@ spec: Daemonsets defines the common configuration applied to all DaemonSets deployed by the GPUCluster controller. properties: + affinity: + description: 'Optional: Set affinity for all Daemonsets' + type: object + x-kubernetes-preserve-unknown-fields: true annotations: additionalProperties: type: string @@ -73,6 +77,12 @@ spec: (scope and select) objects. May match selectors of replication controllers and services. type: object + nodeSelector: + additionalProperties: + type: string + description: 'Optional: Set nodeSelector merged with operand nodeSelector + labels' + type: object podSecurityContext: description: 'Optional: Set pod-level security context for all DaemonSet pods (applies as defaults to all containers)' diff --git a/config/crd/bases/nvidia.com_gpuclusters.yaml b/config/crd/bases/nvidia.com_gpuclusters.yaml index d0666b36b0..a97a0a9c02 100644 --- a/config/crd/bases/nvidia.com_gpuclusters.yaml +++ b/config/crd/bases/nvidia.com_gpuclusters.yaml @@ -57,6 +57,10 @@ spec: Daemonsets defines the common configuration applied to all DaemonSets deployed by the GPUCluster controller. properties: + affinity: + description: 'Optional: Set affinity for all Daemonsets' + type: object + x-kubernetes-preserve-unknown-fields: true annotations: additionalProperties: type: string @@ -73,6 +77,12 @@ spec: (scope and select) objects. May match selectors of replication controllers and services. type: object + nodeSelector: + additionalProperties: + type: string + description: 'Optional: Set nodeSelector merged with operand nodeSelector + labels' + type: object podSecurityContext: description: 'Optional: Set pod-level security context for all DaemonSet pods (applies as defaults to all containers)' diff --git a/deployments/gpu-operator/crds/nvidia.com_gpuclusters.yaml b/deployments/gpu-operator/crds/nvidia.com_gpuclusters.yaml index d0666b36b0..a97a0a9c02 100644 --- a/deployments/gpu-operator/crds/nvidia.com_gpuclusters.yaml +++ b/deployments/gpu-operator/crds/nvidia.com_gpuclusters.yaml @@ -57,6 +57,10 @@ spec: Daemonsets defines the common configuration applied to all DaemonSets deployed by the GPUCluster controller. properties: + affinity: + description: 'Optional: Set affinity for all Daemonsets' + type: object + x-kubernetes-preserve-unknown-fields: true annotations: additionalProperties: type: string @@ -73,6 +77,12 @@ spec: (scope and select) objects. May match selectors of replication controllers and services. type: object + nodeSelector: + additionalProperties: + type: string + description: 'Optional: Set nodeSelector merged with operand nodeSelector + labels' + type: object podSecurityContext: description: 'Optional: Set pod-level security context for all DaemonSet pods (applies as defaults to all containers)'