Skip to content

feat: add --service-dns-domain flag for configurable member FQDN #445

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
silentred wants to merge 2 commits into
base: customized-controller
Choose a base branch
from
Open

feat: add --service-dns-domain flag for configurable member FQDN #445

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a933d3c
feat: add --service-dns-domain flag for configurable member FQDN
silentred Aug 1, 2026
2314af2
update manager deploymant in TestAutoProvider
silentred Aug 11, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 10 additions & 3 deletions cmd/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ func main() {
var probeAddr string
var secureMetrics bool
var enableHTTP2 bool
var serviceDNSDomain string
var tlsOpts []func(*tls.Config)
flag.StringVar(&imageRegistry, "image-registry", controller.DefaultImageRegistry,
Comment thread
silentred marked this conversation as resolved.
"The container registry to pull etcd images from. Defaults to "+controller.DefaultImageRegistry+".")
Expand All @@ -77,6 +78,11 @@ func main() {
"If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.")
flag.BoolVar(&enableHTTP2, "enable-http2", false,
"If set, HTTP/2 will be enabled for the metrics and webhook servers")
flag.StringVar(&serviceDNSDomain, "service-dns-domain", "",
"Kubernetes service DNS suffix used to build member FQDNs. "+
"Empty (the default) uses the short form <pod>.<svc>.<ns>.svc, "+
"resolvable via the pod's DNS search path. "+
"Set this to the kubelet's --cluster-domain value to use the full FQDN form <pod>.<svc>.<ns>.svc.<domain>.")
opts := zap.Options{
Development: true,
}
Expand Down Expand Up @@ -151,9 +157,10 @@ func main() {
}

if err = (&controller.EtcdClusterReconciler{
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
ImageRegistry: imageRegistry,
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
ImageRegistry: imageRegistry,
ServiceDNSDomain: serviceDNSDomain,
}).SetupWithManager(mgr); err != nil {
setupLog.Error(err, "unable to create controller", "controller", "EtcdCluster")
os.Exit(1)
Expand Down
4 changes: 2 additions & 2 deletions internal/controller/etcdcluster.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,13 +13,13 @@ const (
hashLength = 12
)

func EtcdClusterHash(ec *v1alpha1.EtcdCluster) string {
func EtcdClusterHash(ec *v1alpha1.EtcdCluster, dnsDomain string) string {
clusterSpec := ec.DeepCopy().Spec
// size is not used for calculating the hash
clusterSpec.Size = 0
// version is handled separately
clusterSpec.Version = ""
clusterSpec.EtcdOptions = createArgs(ec.Name, ec.Spec.EtcdOptions, clusterTLSEnabled(ec))
clusterSpec.EtcdOptions = createArgs(ec.Name, ec.Spec.EtcdOptions, dnsDomain, clusterTLSEnabled(ec))
// we don't want to roll etcd pods when metadata is changed.
// instead, we'd do in-place update for them.
if clusterSpec.PodTemplate != nil {
Expand Down
22 changes: 13 additions & 9 deletions internal/controller/etcdcluster_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,10 @@ type EtcdClusterReconciler struct {
Scheme *runtime.Scheme
Recorder events.EventRecorder
ImageRegistry string
// ServiceDNSDomain is the Kubernetes service DNS suffix used to build
// member FQDNs and default cert SANs. See the --service-dns-domain flag.
// Empty means the short form `<pod>.<svc>.<ns>.svc` is used.
ServiceDNSDomain string
}

// reconcileState holds all transient data for a single reconciliation loop.
Expand Down Expand Up @@ -262,13 +266,13 @@ func (r *EtcdClusterReconciler) bootstrapCluster(ctx context.Context, s *reconci
logger := log.FromContext(ctx)

if s.cluster.Spec.TLS != nil {
if err := createClientCertificate(ctx, s.cluster, r.Client); err != nil {
if err := createClientCertificate(ctx, s.cluster, r.ServiceDNSDomain, r.Client); err != nil {
logger.Error(err, "Failed to create Client Certificate.")
}
// Server/peer certs must exist before buildReconcileClientTLS below reads
// the server cert Secret. createMemberPod also calls this (idempotently)
// once the first pod is created.
if err := applyEtcdMemberCerts(ctx, s.cluster, r.Client); err != nil {
if err := applyEtcdMemberCerts(ctx, s.cluster, r.ServiceDNSDomain, r.Client); err != nil {
logger.Error(err, "Failed to create server/peer certificates.")
}
} else {
Expand Down Expand Up @@ -297,7 +301,7 @@ func (r *EtcdClusterReconciler) bootstrapCluster(ctx context.Context, s *reconci
}

logger.Info("No member pods found, creating first member pod", "expectedSize", s.cluster.Spec.Size)
if err := createMemberPod(ctx, logger, r.Client, s.cluster, 0, r.Scheme); err != nil {
if err := createMemberPod(ctx, logger, r.Client, s.cluster, r.ServiceDNSDomain, 0, r.Scheme); err != nil {
return ctrl.Result{}, err
}
return ctrl.Result{RequeueAfter: requeueDuration}, nil
Expand All @@ -317,7 +321,7 @@ func (r *EtcdClusterReconciler) healthCheckAndFix(ctx context.Context, s *reconc
logger.Info("Now checking health of the cluster members")

var err error
s.memberListResp, s.memberHealth, err = healthCheck(s.cluster.Name, s.cluster.Namespace, s.pods, clusterTLSEnabled(s.cluster), s.tlsConfig, logger)
s.memberListResp, s.memberHealth, err = healthCheck(s.cluster.Name, s.cluster.Namespace, r.ServiceDNSDomain, s.pods, clusterTLSEnabled(s.cluster), s.tlsConfig, logger)
if err != nil {
return ctrl.Result{}, fmt.Errorf("health check failed: %w", err)
}
Expand Down Expand Up @@ -361,7 +365,7 @@ func (r *EtcdClusterReconciler) reconcileExceptions(ctx context.Context, s *reco
}
next := nextPodOrdinal(ordinals, s.cluster.Spec.Size)
logger.Info("Creating a new member pod", "cluster", s.cluster.Name, "ordinal", next)
if err := createMemberPod(ctx, logger, r.Client, s.cluster, next, r.Scheme); err != nil {
if err := createMemberPod(ctx, logger, r.Client, s.cluster, r.ServiceDNSDomain, next, r.Scheme); err != nil {
return ctrl.Result{}, err
}
} else {
Expand Down Expand Up @@ -404,7 +408,7 @@ func (r *EtcdClusterReconciler) promoteLearner(ctx context.Context, s *reconcile
}

logger.Info("Learner is ready to be promoted to voting member", "learnerID", learner)
eps := clientEndpointsFromPods(s.cluster.Name, s.cluster.Namespace, s.pods, clusterTLSEnabled(s.cluster))
eps := clientEndpointsFromPods(s.cluster.Name, s.cluster.Namespace, r.ServiceDNSDomain, s.pods, clusterTLSEnabled(s.cluster))
// Exclude the learner (last ordinal) from the endpoint list used for promotion.
eps = eps[:(len(eps) - 1)]
if err := etcdutils.PromoteLearner(etcdutils.ClientConfig{Endpoints: eps, TLS: s.tlsConfig}, learner); err != nil {
Expand Down Expand Up @@ -444,12 +448,12 @@ func (r *EtcdClusterReconciler) scaleCluster(ctx context.Context, s *reconcileSt
return ctrl.Result{}, nil
}

eps := clientEndpointsFromPods(s.cluster.Name, s.cluster.Namespace, s.pods, clusterTLSEnabled(s.cluster))
eps := clientEndpointsFromPods(s.cluster.Name, s.cluster.Namespace, r.ServiceDNSDomain, s.pods, clusterTLSEnabled(s.cluster))

if currentPodCount < desiredSize {
// Scale out: add a new learner member to etcd, then create its pod.
nextOrdinal := int(currentPodCount)
_, peerURL := peerEndpointForOrdinalIndex(s.cluster, nextOrdinal)
_, peerURL := peerEndpointForOrdinalIndex(s.cluster, r.ServiceDNSDomain, nextOrdinal)
logger.Info("[Scale out] adding a new learner member to etcd cluster", "peerURL", peerURL)
if _, err := etcdutils.AddMember(etcdutils.ClientConfig{Endpoints: eps, TLS: s.tlsConfig}, []string{peerURL}, true); err != nil {
return ctrl.Result{}, err
Expand All @@ -458,7 +462,7 @@ func (r *EtcdClusterReconciler) scaleCluster(ctx context.Context, s *reconcileSt

// gofail: var exceptionAfterMemberAdd struct{}

if err := createMemberPod(ctx, logger, r.Client, s.cluster, nextOrdinal, r.Scheme); err != nil {
if err := createMemberPod(ctx, logger, r.Client, s.cluster, r.ServiceDNSDomain, nextOrdinal, r.Scheme); err != nil {
return ctrl.Result{}, err
}
return ctrl.Result{RequeueAfter: requeueDuration}, nil
Expand Down
43 changes: 43 additions & 0 deletions internal/controller/etcdcluster_controller_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -373,3 +373,46 @@ func TestBootstrapCluster(t *testing.T) {
assert.Equal(t, "None", svc.Spec.ClusterIP)
})
}

// TestServiceDNSDomainPlumbsThroughBuildMemberPod verifies that the
// value held in EtcdClusterReconciler.ServiceDNSDomain is what ends up in
// the Pod's --advertise-* URLs. This is the controller-level wiring check
// that guarantees the flag actually flows into the Pod template, separate
// from the unit test on buildMemberPod alone.
func TestServiceDNSDomainPlumbsThroughBuildMemberPod(t *testing.T) {
tests := []struct {
name string
domain string
wantPeer string
wantClient string
}{
{
name: "empty domain uses short form",
domain: "",
wantPeer: "http://$(POD_NAME).example.$(POD_NAMESPACE).svc:2380",
wantClient: "http://$(POD_NAME).example.$(POD_NAMESPACE).svc:2379",
},
{
name: "custom domain uses full form",
domain: "corp.local",
wantPeer: "http://$(POD_NAME).example.$(POD_NAMESPACE).svc.corp.local:2380",
wantClient: "http://$(POD_NAME).example.$(POD_NAMESPACE).svc.corp.local:2379",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
r := &EtcdClusterReconciler{ServiceDNSDomain: tt.domain}
ec := &ecv1alpha1.EtcdCluster{
ObjectMeta: metav1.ObjectMeta{Name: "example", Namespace: "default", UID: "abc"},
Spec: ecv1alpha1.EtcdClusterSpec{Size: 3, Version: "3.5.17"},
}
pod := buildMemberPod(ec, "example-0", r.ServiceDNSDomain, etcdClusterStateNew, "ignored")
args := ""
for _, a := range pod.Spec.Containers[0].Args {
args += a + "\n"
}
assert.Contains(t, args, "--initial-advertise-peer-urls="+tt.wantPeer)
assert.Contains(t, args, "--advertise-client-urls="+tt.wantClient)
})
}
}
4 changes: 2 additions & 2 deletions internal/controller/etcdcluster_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ func TestEtcdClusterHash(t *testing.T) {
}

// Calculate base hash once to use as a benchmark in assertions
baseHash := EtcdClusterHash(baseCluster)
baseHash := EtcdClusterHash(baseCluster, "")

// Helper inline functions to generate mutated clusters cleanly
withSize := func(size int) *v1alpha1.EtcdCluster {
Expand Down Expand Up @@ -118,7 +118,7 @@ func TestEtcdClusterHash(t *testing.T) {
// Loop through the table executing each row in isolation
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
currentHash := EtcdClusterHash(tt.cluster)
currentHash := EtcdClusterHash(tt.cluster, "")

if tt.checkLength {
assert.Len(t, currentHash, hashLength, "Hash must respect the %d-character limit constraint", hashLength)
Expand Down
51 changes: 32 additions & 19 deletions internal/controller/pods.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -134,28 +134,40 @@ func waitForPodReady(ctx context.Context, logger logr.Logger, c client.Client, p
return nil
}

// memberDNSSuffix returns the trailing segment appended to "<pod>.<sts>.<ns>."
// when building an etcd member FQDN. When dnsDomain is empty (the operator's
// default) the suffix is the short form "svc"; otherwise it is "svc.<dnsDomain>".
// Kept in sync with test/e2e/helpers_test.go's mirror copy.
func memberDNSSuffix(dnsDomain string) string {
if dnsDomain == "" {
return "svc"
}
return "svc." + dnsDomain
}

// clientEndpointsFromPods builds the client endpoint URL for every pod in the
// slice, in the same order. The DNS form is:
//
// {scheme}://{podName}.{clusterName}.{namespace}.svc.cluster.local:2379
// {scheme}://{podName}.{clusterName}.{namespace}.{dnsSuffix}:2379
//
// where scheme reflects the cluster's TLS configuration.
func clientEndpointsFromPods(clusterName, namespace string, pods []*corev1.Pod, tlsEnabled bool) []string {
// where scheme reflects the cluster's TLS configuration and dnsSuffix is
// produced by memberDNSSuffix from the operator's --service-dns-domain flag.
func clientEndpointsFromPods(clusterName, namespace, dnsDomain string, pods []*corev1.Pod, tlsEnabled bool) []string {
if len(pods) == 0 {
return nil
}
eps := make([]string, 0, len(pods))
for _, pod := range pods {
eps = append(eps, clientEndpointForOrdinal(clusterName, namespace, podOrdinal(pod.Name, clusterName), tlsEnabled))
eps = append(eps, clientEndpointForOrdinal(clusterName, namespace, dnsDomain, podOrdinal(pod.Name, clusterName), tlsEnabled))
}
return eps
}

// clientEndpointForOrdinal returns the client endpoint URL for a member at the
// given ordinal index. The URL scheme is https when tlsEnabled is true, otherwise http.
func clientEndpointForOrdinal(clusterName, namespace string, ordinal int, tlsEnabled bool) string {
return fmt.Sprintf("%s://%s-%d.%s.%s.svc.cluster.local:2379",
clusterScheme(tlsEnabled), clusterName, ordinal, clusterName, namespace)
func clientEndpointForOrdinal(clusterName, namespace, dnsDomain string, ordinal int, tlsEnabled bool) string {
return fmt.Sprintf("%s://%s-%d.%s.%s.%s:2379",
clusterScheme(tlsEnabled), clusterName, ordinal, clusterName, namespace, memberDNSSuffix(dnsDomain))
}

// clusterScheme returns the URL scheme to use for cluster endpoints: "https"
Expand Down Expand Up @@ -186,12 +198,12 @@ func areAllMembersHealthy(memberHealth []etcdutils.EpHealth) bool {

// healthCheck returns a MemberListResponse and per-endpoint health information
// for the etcd cluster reachable through the given pods.
func healthCheck(clusterName, namespace string, pods []*corev1.Pod, tlsEnabled bool, tlsConfig *tls.Config, lg klog.Logger) (*clientv3.MemberListResponse, []etcdutils.EpHealth, error) {
func healthCheck(clusterName, namespace, dnsDomain string, pods []*corev1.Pod, tlsEnabled bool, tlsConfig *tls.Config, lg klog.Logger) (*clientv3.MemberListResponse, []etcdutils.EpHealth, error) {
if len(pods) == 0 {
return nil, nil, nil
}

endpoints := clientEndpointsFromPods(clusterName, namespace, pods, tlsEnabled)
endpoints := clientEndpointsFromPods(clusterName, namespace, dnsDomain, pods, tlsEnabled)

memberlistResp, err := etcdutils.MemberList(etcdutils.ClientConfig{Endpoints: endpoints, TLS: tlsConfig})
if err != nil {
Expand Down Expand Up @@ -240,15 +252,16 @@ const (
peerTrustedCAFile = peerCertMountDir + "/ca.crt"
)

func defaultArgs(name string, tlsEnabled bool) []string {
func defaultArgs(name, dnsDomain string, tlsEnabled bool) []string {
scheme := clusterScheme(tlsEnabled)
suffix := memberDNSSuffix(dnsDomain)
args := make([]string, 0, 13)
args = append(args,
"--name=$(POD_NAME)",
fmt.Sprintf("--listen-peer-urls=%s://0.0.0.0:2380", scheme),
fmt.Sprintf("--listen-client-urls=%s://0.0.0.0:2379", scheme),
fmt.Sprintf("--initial-advertise-peer-urls=%s://$(POD_NAME).%s.$(POD_NAMESPACE).svc.cluster.local:2380", scheme, name),
fmt.Sprintf("--advertise-client-urls=%s://$(POD_NAME).%s.$(POD_NAMESPACE).svc.cluster.local:2379", scheme, name),
fmt.Sprintf("--initial-advertise-peer-urls=%s://$(POD_NAME).%s.$(POD_NAMESPACE).%s:2380", scheme, name, suffix),
fmt.Sprintf("--advertise-client-urls=%s://$(POD_NAME).%s.$(POD_NAMESPACE).%s:2379", scheme, name, suffix),
)
if !tlsEnabled {
return args
Expand All @@ -270,7 +283,7 @@ const (
)

// buildMemberPod constructs the Pod object for a single etcd member.
func buildMemberPod(ec *ecv1alpha1.EtcdCluster, podName string, initialClusterState etcdClusterState, initialCluster string) *corev1.Pod {
func buildMemberPod(ec *ecv1alpha1.EtcdCluster, podName, dnsDomain string, initialClusterState etcdClusterState, initialCluster string) *corev1.Pod {
// Start with custom labels then overwrite with the mandatory defaults so
// that the headless-service selector is always satisfied.
labels := make(map[string]string)
Expand All @@ -285,7 +298,7 @@ func buildMemberPod(ec *ecv1alpha1.EtcdCluster, podName string, initialClusterSt
len(ec.Spec.PodTemplate.Metadata.Annotations) > 0 {
maps.Copy(annotations, ec.Spec.PodTemplate.Metadata.Annotations)
}
annotations[HashMetadataKey] = EtcdClusterHash(ec)
annotations[HashMetadataKey] = EtcdClusterHash(ec, dnsDomain)

envVars := []corev1.EnvVar{
{
Expand All @@ -309,7 +322,7 @@ func buildMemberPod(ec *ecv1alpha1.EtcdCluster, podName string, initialClusterSt
Name: "etcd",
Image: fmt.Sprintf("%s:%s", ec.Spec.ImageRegistry, ec.Spec.Version),
Command: []string{"/usr/local/bin/etcd"},
Args: createArgs(ec.Name, ec.Spec.EtcdOptions, clusterTLSEnabled(ec)),
Args: createArgs(ec.Name, ec.Spec.EtcdOptions, dnsDomain, clusterTLSEnabled(ec)),
Env: envVars,
Ports: []corev1.ContainerPort{
{Name: "client", ContainerPort: 2379},
Expand Down Expand Up @@ -407,11 +420,11 @@ func buildMemberPod(ec *ecv1alpha1.EtcdCluster, podName string, initialClusterSt
// createMemberPod creates a single etcd member Pod (and, if needed, its PVC)
// for the given ordinal index. It does not wait for the pod to become ready;
// the caller is responsible for requeueing until the pod is healthy.
func createMemberPod(ctx context.Context, logger logr.Logger, c client.Client, ec *ecv1alpha1.EtcdCluster, ordinal int, scheme *runtime.Scheme) error {
func createMemberPod(ctx context.Context, logger logr.Logger, c client.Client, ec *ecv1alpha1.EtcdCluster, dnsDomain string, ordinal int, scheme *runtime.Scheme) error {
podName := memberPodName(ec.Name, ordinal)

// Ensure TLS certificates exist before the pod mounts them.
if err := applyEtcdMemberCerts(ctx, ec, c); err != nil {
if err := applyEtcdMemberCerts(ctx, ec, dnsDomain, c); err != nil {
return err
}

Expand All @@ -430,11 +443,11 @@ func createMemberPod(ctx context.Context, logger logr.Logger, c client.Client, e
// Build the initial-cluster value: all peers from ordinal 0 to this one.
var clusterParts []string
for i := range ordinal + 1 {
name, peerURL := peerEndpointForOrdinalIndex(ec, i)
name, peerURL := peerEndpointForOrdinalIndex(ec, dnsDomain, i)
clusterParts = append(clusterParts, fmt.Sprintf("%s=%s", name, peerURL))
}

pod := buildMemberPod(ec, podName, state, strings.Join(clusterParts, ","))
pod := buildMemberPod(ec, podName, dnsDomain, state, strings.Join(clusterParts, ","))
if err := controllerutil.SetControllerReference(ec, pod, scheme); err != nil {
return err
}
Expand Down
Loading