feat(ha): Add leader election to prevent duplicate ACM imports

.gitignore

@@ -1,2 +1,3 @@
 .env
 .DS_Store
+cert-manager-sync

cmd/cert-manager-sync/main.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"cmp"
+	"context"
 	"os"
 	"time"
 
@@ -11,8 +12,11 @@ import (
 	log "github.com/sirupsen/logrus"
 	_ "golang.org/x/crypto/x509roots/fallback" // Embeds x509root certificates into the binary
 	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/informers"
 	"k8s.io/client-go/tools/cache"
+	"k8s.io/client-go/tools/leaderelection"
+	"k8s.io/client-go/tools/leaderelection/resourcelock"
 )
 
 func init() {
@@ -36,23 +40,14 @@ func init() {
 	}
 }
 
-func main() {
-	l := log.WithFields(
-		log.Fields{
-			"fn": "main",
-		},
-	)
-	l.Info("starting cert-manager-sync")
-	if os.Getenv("ENABLE_METRICS") != "false" {
-		go metrics.Serve()
-	}
+func runController(ctx context.Context) {
+	l := log.WithFields(log.Fields{"fn": "runController"})
+	l.Info("starting informers as leader")
+
 	factory := informers.NewSharedInformerFactory(state.KubeClient, 30*time.Second)
 	secretInformer := factory.Core().V1().Secrets().Informer()
 
-	stopper := make(chan struct{})
-	defer close(stopper)
-
-	secretInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{
+	_, _ = secretInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{
 		AddFunc: func(obj interface{}) {
 			s := obj.(*v1.Secret)
 			if !state.SecretWatched(s) {
@@ -73,13 +68,62 @@ func main() {
 		},
 	})
 
-	factory.Start(stopper)
+	factory.Start(ctx.Done())
 
-	// Wait for the caches to sync
-	if !cache.WaitForCacheSync(stopper, secretInformer.HasSynced) {
-		panic("Timed out waiting for caches to sync")
+	if !cache.WaitForCacheSync(ctx.Done(), secretInformer.HasSynced) {
+		l.Error("timed out waiting for caches to sync")
+		return
 	}
 
-	// Run the informer
-	<-stopper
+	<-ctx.Done()
+	l.Info("leader lost, stopping informers")
+}
+
+func main() {
+	l := log.WithFields(log.Fields{"fn": "main"})
+	l.Info("starting cert-manager-sync")
+
+	if os.Getenv("ENABLE_METRICS") != "false" {
+		go metrics.Serve()
+	}
+
+	if os.Getenv("LEADER_ELECTION_ENABLED") == "false" {
+		l.Info("leader election disabled, running directly")
+		runController(context.Background())
+		return
+	}
+
+	id, _ := os.Hostname()
+	ns := cmp.Or(os.Getenv("LEADER_ELECTION_NAMESPACE"), "cert-manager-sync")
+	lockName := cmp.Or(os.Getenv("LEADER_ELECTION_LOCK_NAME"), "cert-manager-sync-leader")
+
+	lock := &resourcelock.LeaseLock{
+		LeaseMeta: metav1.ObjectMeta{Name: lockName, Namespace: ns},
+		Client:    state.KubeClient.CoordinationV1(),
+		LockConfig: resourcelock.ResourceLockConfig{
+			Identity: id,
+		},
+	}
+
+	ctx := context.Background()
+
+	leaderelection.RunOrDie(ctx, leaderelection.LeaderElectionConfig{
+		Lock:            lock,
+		ReleaseOnCancel: true,
+		LeaseDuration:   15 * time.Second,
+		RenewDeadline:   10 * time.Second,
+		RetryPeriod:     2 * time.Second,
+		Callbacks: leaderelection.LeaderCallbacks{
+			OnStartedLeading: runController,
+			OnStoppedLeading: func() {
+				l.Info("leader election lost")
+			},
+			OnNewLeader: func(identity string) {
+				if identity == id {
+					return
+				}
+				l.Infof("new leader elected: %s", identity)
+			},
+		},
+	})
 }

cmd/cert-manager-sync/main_test.go

@@ -0,0 +1,39 @@
+package main
+
+import (
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestLeaderElectionDefaults(t *testing.T) {
+	os.Unsetenv("LEADER_ELECTION_ENABLED")
+	os.Unsetenv("LEADER_ELECTION_LOCK_NAME")
+	os.Unsetenv("LEADER_ELECTION_NAMESPACE")
+
+	// Default: leader election enabled
+	assert.NotEqual(t, "false", os.Getenv("LEADER_ELECTION_ENABLED"),
+		"leader election should be enabled by default")
+}
+
+func TestLeaderElectionDisabled(t *testing.T) {
+	os.Setenv("LEADER_ELECTION_ENABLED", "false")
+	defer os.Unsetenv("LEADER_ELECTION_ENABLED")
+
+	assert.Equal(t, "false", os.Getenv("LEADER_ELECTION_ENABLED"))
+}
+
+func TestLeaderElectionCustomLockName(t *testing.T) {
+	os.Setenv("LEADER_ELECTION_LOCK_NAME", "custom-lock")
+	defer os.Unsetenv("LEADER_ELECTION_LOCK_NAME")
+
+	assert.Equal(t, "custom-lock", os.Getenv("LEADER_ELECTION_LOCK_NAME"))
+}
+
+func TestLeaderElectionCustomNamespace(t *testing.T) {
+	os.Setenv("LEADER_ELECTION_NAMESPACE", "kube-system")
+	defer os.Unsetenv("LEADER_ELECTION_NAMESPACE")
+
+	assert.Equal(t, "kube-system", os.Getenv("LEADER_ELECTION_NAMESPACE"))
+}

deploy/cert-manager-sync/templates/deployment.yaml

@@ -65,6 +65,14 @@ spec:
             value: "{{ if and .Values.metrics .Values.metrics.enabled }}{{ .Values.metrics.enabled }}{{ else }}false{{ end }}"
           - name: METRICS_PORT
             value: "{{ if and .Values.metrics .Values.metrics.port }}{{ .Values.metrics.port }}{{ else }}9090{{ end }}"
+          - name: LEADER_ELECTION_ENABLED
+            value: "{{ .Values.leaderElection.enabled }}"
+          - name: LEADER_ELECTION_NAMESPACE
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.namespace
+          - name: LEADER_ELECTION_LOCK_NAME
+            value: "{{ .Values.leaderElection.lockName }}"
           {{- with .Values.env }}
           {{- toYaml . | nindent 10 }}
           {{- end }}

deploy/cert-manager-sync/templates/leader-election-rbac.yaml

@@ -0,0 +1,28 @@
+{{- if and .Values.clusterRole.create .Values.leaderElection.enabled -}}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: {{ include "cert-manager-sync.fullname" . }}-leader-election
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "cert-manager-sync.labels" . | nindent 4 }}
+rules:
+- apiGroups: ["coordination.k8s.io"]
+  resources: ["leases"]
+  verbs: ["get", "create", "update"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: {{ include "cert-manager-sync.fullname" . }}-leader-election
+  namespace: {{ .Release.Namespace }}
+subjects:
+- kind: ServiceAccount
+  name: {{ include "cert-manager-sync.serviceAccountName" . }}
+  namespace: {{ .Release.Namespace }}
+roleRef:
+  kind: Role
+  name: {{ include "cert-manager-sync.fullname" . }}-leader-election
+  apiGroup: rbac.authorization.k8s.io
+{{- end }}

deploy/cert-manager-sync/values.yaml

@@ -4,6 +4,10 @@
 
 replicaCount: 1
 
+leaderElection:
+  enabled: true
+  lockName: cert-manager-sync-leader
+
 image:
   repository: robertlestak/cert-manager-sync
   pullPolicy: IfNotPresent

go.mod

@@ -38,7 +38,7 @@ require (
 	github.com/emicklei/go-restful/v3 v3.12.2 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/fxamacker/cbor/v2 v2.9.0 // indirect
-	github.com/go-jose/go-jose/v4 v4.1.3 // indirect
+	github.com/go-jose/go-jose/v4 v4.1.4 // indirect
 	github.com/go-logr/logr v1.4.3 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-openapi/jsonpointer v0.21.0 // indirect
@@ -97,7 +97,7 @@ require (
 	golang.org/x/time v0.14.0 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20260217215200-42d3e9bedb6d // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20260217215200-42d3e9bedb6d // indirect
-	google.golang.org/grpc v1.79.1 // indirect
+	google.golang.org/grpc v1.79.3 // indirect
 	google.golang.org/protobuf v1.36.11 // indirect
 	gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect

go.sum

@@ -45,8 +45,8 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2
 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM=
 github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ=
-github.com/go-jose/go-jose/v4 v4.1.3 h1:CVLmWDhDVRa6Mi/IgCgaopNosCaHz7zrMeF9MlZRkrs=
-github.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08=
+github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA=
+github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08=
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
 github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
@@ -254,8 +254,8 @@ google.golang.org/genproto/googleapis/api v0.0.0-20260217215200-42d3e9bedb6d h1:
 google.golang.org/genproto/googleapis/api v0.0.0-20260217215200-42d3e9bedb6d/go.mod h1:48U2I+QQUYhsFrg2SY6r+nJzeOtjey7j//WBESw+qyQ=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20260217215200-42d3e9bedb6d h1:t/LOSXPJ9R0B6fnZNyALBRfZBH0Uy0gT+uR+SJ6syqQ=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20260217215200-42d3e9bedb6d/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
-google.golang.org/grpc v1.79.1 h1:zGhSi45ODB9/p3VAawt9a+O/MULLl9dpizzNNpq7flY=
-google.golang.org/grpc v1.79.1/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ=
+google.golang.org/grpc v1.79.3 h1:sybAEdRIEtvcD68Gx7dmnwjZKlyfuc61Dyo9pGXXkKE=
+google.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ=
 google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
 google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

internal/metrics/metrics.go

@@ -4,6 +4,7 @@ import (
 	"cmp"
 	"net/http"
 	"os"
+	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
@@ -44,7 +45,13 @@ func Serve() {
 		w.WriteHeader(http.StatusOK)
 	})
 	http.Handle("/metrics", promhttp.Handler())
-	if err := http.ListenAndServe(":"+port, nil); err != nil {
+	srv := &http.Server{
+		Addr:         ":" + port,
+		ReadTimeout:  5 * time.Second,
+		WriteTimeout: 10 * time.Second,
+		IdleTimeout:  120 * time.Second,
+	}
+	if err := srv.ListenAndServe(); err != nil {
 		l.WithError(err).Error("error starting http server")
 		os.Exit(1)
 	}

pkg/certmanagersync/certmanagersync.go

@@ -165,7 +165,7 @@ func calculateNextRetryTime(secret *corev1.Secret) time.Time {
 	// Calculate the delay using binary exponential backoff
 	var delay time.Duration
 	if retries < 31 {
-		delay = time.Duration(1<<uint(retries)) * time.Minute
+		delay = time.Duration(1<<uint(retries)) * time.Minute // #nosec G115 -- guarded by retries < 31
 	} else {
 		delay = 32 * time.Hour
 	}

pkg/state/certmanagersync.go

@@ -23,6 +23,7 @@ import (
 var (
 	OperatorName  = "cert-manager-sync.lestak.sh"
 	KubeClient    *kubernetes.Clientset
+	KubeConfig    *rest.Config
 	EventRecorder record.EventRecorder
 )
 
@@ -171,7 +172,7 @@ func CreateKubeClient() error {
 	}
 	var config *rest.Config
 	// naïvely assume if no kubeconfig file that we are running in cluster
-	if _, err := os.Stat(kubeconfig); os.IsNotExist(err) {
+	if _, err := os.Stat(kubeconfig); os.IsNotExist(err) { // #nosec G703 -- standard k8s client KUBECONFIG pattern
 		config, err = rest.InClusterConfig()
 		if err != nil {
 			l.Debugf("res.InClusterConfig error=%v", err)
@@ -189,6 +190,7 @@ func CreateKubeClient() error {
 		l.Debugf("kubernetes.NewForConfig error=%v", err)
 		return err
 	}
+	KubeConfig = config
 	// Create broadcaster
 	broadcaster := record.NewBroadcaster()
 	broadcaster.StartRecordingToSink(&typedcorev1.EventSinkImpl{Interface: KubeClient.CoreV1().Events("")})

pkg/state/certmanagersync_test.go

@@ -346,3 +346,9 @@ func TestNamespaceEnabledEnabledSingleSecretsNamespace(t *testing.T) {
 		})
 	}
 }
+
+func TestKubeConfigExported(t *testing.T) {
+	// KubeConfig should be nil before CreateKubeClient is called
+	// but the variable itself must be accessible (exported)
+	assert.Nil(t, KubeConfig, "KubeConfig should be nil before initialization")
+}

stores/filepath/filepath.go

@@ -69,16 +69,16 @@ func (s *FilepathStore) Sync(c *tlssecret.Certificate) (map[string]string, error
 	l = l.WithFields(log.Fields{
 		"id": certPath,
 	})
-	if err := os.WriteFile(certPath, c.Certificate, 0644); err != nil {
+	if err := os.WriteFile(certPath, c.Certificate, 0644); err != nil { // #nosec G306 -- certs are public
 		l.WithError(err).Errorf("sync error")
 		return nil, fmt.Errorf("failed to write certificate file to %s: %w", certPath, err)
 	}
-	if err := os.WriteFile(keyPath, c.Key, 0644); err != nil {
+	if err := os.WriteFile(keyPath, c.Key, 0600); err != nil {
 		l.WithError(err).Errorf("sync error")
 		return nil, fmt.Errorf("failed to write key file to %s: %w", keyPath, err)
 	}
 	if len(c.Ca) > 0 {
-		if err := os.WriteFile(caPath, c.Ca, 0644); err != nil {
+		if err := os.WriteFile(caPath, c.Ca, 0644); err != nil { // #nosec G306 -- CA certs are public
 			l.WithError(err).Errorf("sync error")
 			return nil, fmt.Errorf("failed to write CA file to %s: %w", caPath, err)
 		}

stores/imperva/imperva.go

@@ -96,7 +96,7 @@ func (s *ImpervaStore) UploadImpervaCert(cert *tlssecret.Certificate) error {
 		PrivateKey:  bKey,
 		AuthType:    cmp.Or(s.AuthType, "RSA"),
 	}
-	jd, err := json.Marshal(up)
+	jd, err := json.Marshal(up) // #nosec G117 -- private key required by Imperva API
 	if err != nil {
 		l.WithError(err).Errorf("json.Marshal error")
 		return fmt.Errorf("failed to marshal Imperva certificate upload request: %w", err)

stores/threatx/threatx.go

@@ -104,7 +104,7 @@ func (s *ThreatXStore) ThreatxLogin(ctx context.Context) error {
 	}
 	r.Command = "login"
 	r.APIToken = s.APIToken
-	jd, jerr := json.Marshal(r)
+	jd, jerr := json.Marshal(r) // #nosec G117 -- API token required by ThreatX API
 	if jerr != nil {
 		l.Error(jerr)
 		return fmt.Errorf("failed to marshal ThreatX login request: %w", jerr)