From 1c9527b5fbcd3ee7877fd4d6d4b1d3a8066f93ef Mon Sep 17 00:00:00 2001 From: Zach Smith Date: Fri, 13 Feb 2026 00:23:41 -0800 Subject: [PATCH 1/5] chore: refactor out functionality now that apiserver is more correct --- .github/workflows/test-e2e.yml | 2 +- README.md | 2 +- .../controller/controlplane_controller.go | 159 +----------------- test/e2e/e2e_suite_test.go | 9 +- 4 files changed, 13 insertions(+), 159 deletions(-) diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index 6bf59de..5d20645 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -29,4 +29,4 @@ jobs: - name: Running Test e2e run: | go mod tidy - APISERVER_IMAGE=kplanedev/apiserver:v0.0.2 make test-e2e + APISERVER_IMAGE=kplanedev/apiserver:v0.0.3 make test-e2e diff --git a/README.md b/README.md index 1294054..cb288c6 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ make uninstall The e2e suite uses a Kind cluster and a shared apiserver image. ```sh -APISERVER_IMAGE=kplanedev/apiserver:v0.0.2 make test-e2e +APISERVER_IMAGE=kplanedev/apiserver:v0.0.3 make test-e2e ``` Optional overrides: diff --git a/internal/controller/controlplane_controller.go b/internal/controller/controlplane_controller.go index 1c83ebe..a98ccd7 100644 --- a/internal/controller/controlplane_controller.go +++ b/internal/controller/controlplane_controller.go @@ -504,22 +504,8 @@ func (r *ControlPlaneReconciler) clusterClientForEndpoint(endpoint string) (*kub func (r *ControlPlaneReconciler) bootstrapVirtualCluster(ctx context.Context, clientset *kubernetes.Clientset, endpoint string, caData []byte) error { adminNS := r.virtualAdminNamespace() - requiredNamespaces := []string{ - "default", - "kube-system", - "kube-public", - "kube-node-lease", - adminNS, - } - seen := make(map[string]struct{}, len(requiredNamespaces)) - for _, ns := range requiredNamespaces { - if _, ok := seen[ns]; ok { - continue - } - seen[ns] = struct{}{} - if err := ensureNamespace(ctx, clientset, ns); err != nil { - return err - } + if err := ensureNamespace(ctx, clientset, adminNS); err != nil { + return err } if _, err := clientset.CoreV1().ServiceAccounts(adminNS).Create(ctx, &corev1.ServiceAccount{ @@ -945,53 +931,9 @@ func ensureBootstrapToken(ctx context.Context, clientset *kubernetes.Clientset) } func ensureBootstrapRBAC(ctx context.Context, clientset *kubernetes.Clientset) error { - if err := ensurePublicInfoViewer(ctx, clientset); err != nil { - return err - } if err := ensureKubeadmBootstrapperRole(ctx, clientset); err != nil { return err } - if err := ensureNodeRBAC(ctx, clientset); err != nil { - return err - } - - clusterRole := &rbacv1.ClusterRole{ - ObjectMeta: metav1.ObjectMeta{Name: "system:node-bootstrapper"}, - Rules: []rbacv1.PolicyRule{ - { - APIGroups: []string{"certificates.k8s.io"}, - Resources: []string{"certificatesigningrequests", "certificatesigningrequests/nodeclient"}, - Verbs: []string{"create", "get", "list", "watch"}, - }, - }, - } - _, err := clientset.RbacV1().ClusterRoles().Create(ctx, clusterRole, metav1.CreateOptions{}) - if err != nil && !apierrors.IsAlreadyExists(err) { - return err - } - - clusterRoleBinding := &rbacv1.ClusterRoleBinding{ - ObjectMeta: metav1.ObjectMeta{Name: "system:node-bootstrapper"}, - RoleRef: rbacv1.RoleRef{ - APIGroup: "rbac.authorization.k8s.io", - Kind: "ClusterRole", - Name: "system:node-bootstrapper", - }, - Subjects: []rbacv1.Subject{ - { - Kind: "Group", - Name: "system:bootstrappers", - }, - { - Kind: "Group", - Name: "system:bootstrappers:kubeadm:default-node-token", - }, - }, - } - _, err = clientset.RbacV1().ClusterRoleBindings().Create(ctx, clusterRoleBinding, metav1.CreateOptions{}) - if err != nil && !apierrors.IsAlreadyExists(err) { - return err - } roleBinding := &rbacv1.RoleBinding{ ObjectMeta: metav1.ObjectMeta{ @@ -1014,25 +956,7 @@ func ensureBootstrapRBAC(ctx context.Context, clientset *kubernetes.Clientset) e }, }, } - _, err = clientset.RbacV1().RoleBindings("kube-public").Create(ctx, roleBinding, metav1.CreateOptions{}) - if err != nil && !apierrors.IsAlreadyExists(err) { - return err - } - return nil -} - -func ensurePublicInfoViewer(ctx context.Context, clientset *kubernetes.Clientset) error { - clusterRole := &rbacv1.ClusterRole{ - ObjectMeta: metav1.ObjectMeta{Name: "system:public-info-viewer"}, - Rules: []rbacv1.PolicyRule{ - { - APIGroups: []string{""}, - Resources: []string{"configmaps"}, - Verbs: []string{"get", "list", "watch"}, - }, - }, - } - _, err := clientset.RbacV1().ClusterRoles().Create(ctx, clusterRole, metav1.CreateOptions{}) + _, err := clientset.RbacV1().RoleBindings("kube-public").Create(ctx, roleBinding, metav1.CreateOptions{}) if err != nil && !apierrors.IsAlreadyExists(err) { return err } @@ -1086,83 +1010,6 @@ func ensureKubeadmBootstrapperRole(ctx context.Context, clientset *kubernetes.Cl return nil } -func ensureNodeRBAC(ctx context.Context, clientset *kubernetes.Clientset) error { - clusterRole := &rbacv1.ClusterRole{ - ObjectMeta: metav1.ObjectMeta{Name: "system:node"}, - Rules: []rbacv1.PolicyRule{ - { - APIGroups: []string{""}, - Resources: []string{"nodes"}, - Verbs: []string{"create", "get", "list", "watch", "update", "patch"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"nodes/status"}, - Verbs: []string{"update", "patch"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"pods"}, - Verbs: []string{"get", "list", "watch"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"pods/status"}, - Verbs: []string{"update", "patch"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"services", "endpoints", "configmaps", "secrets"}, - Verbs: []string{"get", "list", "watch"}, - }, - { - APIGroups: []string{"coordination.k8s.io"}, - Resources: []string{"leases"}, - Verbs: []string{"get", "list", "watch", "create", "update", "patch"}, - }, - { - APIGroups: []string{"storage.k8s.io"}, - Resources: []string{"csidrivers"}, - Verbs: []string{"get", "list", "watch"}, - }, - { - APIGroups: []string{"node.k8s.io"}, - Resources: []string{"runtimeclasses"}, - Verbs: []string{"get", "list", "watch"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"events"}, - Verbs: []string{"create", "patch", "update"}, - }, - }, - } - _, err := clientset.RbacV1().ClusterRoles().Create(ctx, clusterRole, metav1.CreateOptions{}) - if err != nil && !apierrors.IsAlreadyExists(err) { - return err - } - - clusterRoleBinding := &rbacv1.ClusterRoleBinding{ - ObjectMeta: metav1.ObjectMeta{Name: "system:nodes"}, - RoleRef: rbacv1.RoleRef{ - APIGroup: "rbac.authorization.k8s.io", - Kind: "ClusterRole", - Name: "system:node", - }, - Subjects: []rbacv1.Subject{ - { - Kind: "Group", - Name: "system:nodes", - }, - }, - } - _, err = clientset.RbacV1().ClusterRoleBindings().Create(ctx, clusterRoleBinding, metav1.CreateOptions{}) - if err != nil && !apierrors.IsAlreadyExists(err) { - return err - } - return nil -} - func encodeBase64(data []byte) string { if len(data) == 0 { return "" diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 0b827b4..fed1039 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -45,7 +45,7 @@ var ( // with the code source changes to be tested. projectImage = "example.com/controlplane-operator:v0.0.1" - apiserverImage = "kplanedev/apiserver:v0.0.2" + apiserverImage = "kplanedev/apiserver:v0.0.3" apiserverRepoDir = "" ) @@ -86,6 +86,8 @@ var _ = BeforeSuite(func() { if output, err := cmd.CombinedOutput(); err != nil { ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to build apiserver image: %s", string(output)) } + } else if isLocalImagePresent(apiserverImage) { + By("using the local apiserver image") } else { By("pulling the apiserver image") cmd = exec.Command("docker", "pull", apiserverImage) @@ -125,6 +127,11 @@ var _ = BeforeSuite(func() { } }) +func isLocalImagePresent(image string) bool { + cmd := exec.Command("docker", "image", "inspect", image) + return cmd.Run() == nil +} + var _ = AfterSuite(func() { // Teardown CertManager after the suite if not skipped and if it was not already installed if !skipCertManagerInstall && !isCertManagerAlreadyInstalled { From 75dc1d21ec22361dbf9e54ac1102079b80a8f693 Mon Sep 17 00:00:00 2001 From: Zach Smith Date: Fri, 13 Feb 2026 00:33:14 -0800 Subject: [PATCH 2/5] fix: e2e test image --- test/e2e/e2e_suite_test.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index fed1039..f082deb 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -64,14 +64,14 @@ var _ = BeforeSuite(func() { if image := os.Getenv("APISERVER_IMAGE"); image != "" { apiserverImage = image } + apiserverArch := runtime.GOARCH + if v := os.Getenv("APISERVER_ARCH"); v != "" { + apiserverArch = v + } var cmd *exec.Cmd if _, err := os.Stat(apiserverRepoDir); err == nil { By("building the apiserver binary for Kind") - apiserverArch := runtime.GOARCH - if v := os.Getenv("APISERVER_ARCH"); v != "" { - apiserverArch = v - } apiserverBinary := fmt.Sprintf(".dev/bin/apiserver-linux-%s", apiserverArch) cmd := exec.Command("go", "build", "-o", apiserverBinary, "./cmd/apiserver") cmd.Env = append(os.Environ(), "GOOS=linux", fmt.Sprintf("GOARCH=%s", apiserverArch)) @@ -90,7 +90,7 @@ var _ = BeforeSuite(func() { By("using the local apiserver image") } else { By("pulling the apiserver image") - cmd = exec.Command("docker", "pull", apiserverImage) + cmd = exec.Command("docker", "pull", "--platform", fmt.Sprintf("linux/%s", apiserverArch), apiserverImage) if output, err := cmd.CombinedOutput(); err != nil { ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to pull apiserver image: %s", string(output)) } From 94cb1d17beda4b0080c7daa6842e5a20781e0608 Mon Sep 17 00:00:00 2001 From: Zach Smith Date: Fri, 13 Feb 2026 00:37:40 -0800 Subject: [PATCH 3/5] fix: single arch --- test/utils/utils.go | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/test/utils/utils.go b/test/utils/utils.go index de9ebf1..0215efd 100644 --- a/test/utils/utils.go +++ b/test/utils/utils.go @@ -139,13 +139,34 @@ func LoadImageToKindClusterWithName(name string) error { if v, ok := os.LookupEnv("KIND_CLUSTER"); ok { cluster = v } - kindOptions := []string{"load", "docker-image", name, "--name", cluster} kindBinary := defaultKindBinary if v, ok := os.LookupEnv("KIND"); ok { kindBinary = v } + + tmpFile, err := os.CreateTemp("", "kplane-kind-image-*.tar") + if err != nil { + return err + } + tmpPath := tmpFile.Name() + if closeErr := tmpFile.Close(); closeErr != nil && err == nil { + err = closeErr + } + defer os.Remove(tmpPath) + + if err == nil { + cmd := exec.Command("docker", "save", "-o", tmpPath, name) + if _, err = Run(cmd); err == nil { + cmd = exec.Command(kindBinary, "load", "image-archive", "--name", cluster, tmpPath) + if _, err = Run(cmd); err == nil { + return nil + } + } + } + + kindOptions := []string{"load", "docker-image", name, "--name", cluster} cmd := exec.Command(kindBinary, kindOptions...) - _, err := Run(cmd) + _, err = Run(cmd) return err } From bd1884a6f99aa59f11bb1ac59100e80f275e978d Mon Sep 17 00:00:00 2001 From: Zach Smith Date: Fri, 13 Feb 2026 00:39:24 -0800 Subject: [PATCH 4/5] fix: lint --- test/utils/utils.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/utils/utils.go b/test/utils/utils.go index 0215efd..31a2320 100644 --- a/test/utils/utils.go +++ b/test/utils/utils.go @@ -152,7 +152,11 @@ func LoadImageToKindClusterWithName(name string) error { if closeErr := tmpFile.Close(); closeErr != nil && err == nil { err = closeErr } - defer os.Remove(tmpPath) + defer func() { + if err := os.Remove(tmpPath); err != nil { + warnError(err) + } + }() if err == nil { cmd := exec.Command("docker", "save", "-o", tmpPath, name) From 968daf08ff1861576215b49aea0ea6daa3581e3f Mon Sep 17 00:00:00 2001 From: Zach Smith Date: Fri, 13 Feb 2026 00:39:46 -0800 Subject: [PATCH 5/5] fix: lint --- test/utils/utils.go | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/test/utils/utils.go b/test/utils/utils.go index 31a2320..2254b9b 100644 --- a/test/utils/utils.go +++ b/test/utils/utils.go @@ -149,8 +149,8 @@ func LoadImageToKindClusterWithName(name string) error { return err } tmpPath := tmpFile.Name() - if closeErr := tmpFile.Close(); closeErr != nil && err == nil { - err = closeErr + if closeErr := tmpFile.Close(); closeErr != nil { + warnError(closeErr) } defer func() { if err := os.Remove(tmpPath); err != nil { @@ -158,18 +158,16 @@ func LoadImageToKindClusterWithName(name string) error { } }() - if err == nil { - cmd := exec.Command("docker", "save", "-o", tmpPath, name) + cmd := exec.Command("docker", "save", "-o", tmpPath, name) + if _, err = Run(cmd); err == nil { + cmd = exec.Command(kindBinary, "load", "image-archive", "--name", cluster, tmpPath) if _, err = Run(cmd); err == nil { - cmd = exec.Command(kindBinary, "load", "image-archive", "--name", cluster, tmpPath) - if _, err = Run(cmd); err == nil { - return nil - } + return nil } } kindOptions := []string{"load", "docker-image", name, "--name", cluster} - cmd := exec.Command(kindBinary, kindOptions...) + cmd = exec.Command(kindBinary, kindOptions...) _, err = Run(cmd) return err }