diff --git a/.github/workflows/pipeline.yaml b/.github/workflows/pipeline.yaml index a67b2176..1b6edc20 100644 --- a/.github/workflows/pipeline.yaml +++ b/.github/workflows/pipeline.yaml @@ -36,6 +36,7 @@ jobs: shell: bash run: | go install github.com/go-task/task/v3/cmd/task@latest + go install github.com/k3d-io/k3d/v5@latest go install sigs.k8s.io/kind@latest task test @@ -43,7 +44,7 @@ jobs: - name: Dump kubectl logs if: failure() run: | - kubectl config use-context kind-atc-test + kubectl config use-context k3d-atc-test kubectl cluster-info dump > k8s_dump.out - name: Upload k8s dump diff --git a/Taskfile.yml b/Taskfile.yml index 70e2839e..2415c1af 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -20,6 +20,7 @@ tasks: update-tools: cmds: - go install sigs.k8s.io/kind@latest + - go install github.com/k3d-io/k3d/v5@latest update-deps: cmds: diff --git a/cmd/atc-installer/installer/run.go b/cmd/atc-installer/installer/run.go index b518816b..935271c5 100644 --- a/cmd/atc-installer/installer/run.go +++ b/cmd/atc-installer/installer/run.go @@ -14,6 +14,7 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" @@ -295,12 +296,56 @@ func Run(cfg Config) (flight.Stages, error) { }, } + networkPolicy := &networkingv1.NetworkPolicy{ + TypeMeta: metav1.TypeMeta{ + Kind: "NetworkPolicy", + APIVersion: networkingv1.SchemeGroupVersion.Identifier(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: flight.Release() + "-atc", + Namespace: flight.Namespace(), + }, + Spec: networkingv1.NetworkPolicySpec{ + PodSelector: metav1.LabelSelector{ + MatchLabels: selector, + }, + PolicyTypes: []networkingv1.PolicyType{ + networkingv1.PolicyTypeIngress, + }, + Ingress: []networkingv1.NetworkPolicyIngressRule{ + { + From: []networkingv1.NetworkPolicyPeer{ + { + NamespaceSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "kubernetes.io/metadata.name": "kube-system", + }, + }, + PodSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "component": "kube-apiserver", + }, + }, + }, + }, + Ports: []networkingv1.NetworkPolicyPort{ + { + Protocol: ptr.To(corev1.ProtocolTCP), + Port: ptr.To(intstr.FromInt(cfg.Port)), + }, + }, + }, + }, + }, + } + return flight.Stages{ { svc, tlsSecret, account, binding, + networkPolicy, }, { // By moving the deployment deletion into a later stage, this means we will also delete it first diff --git a/cmd/atc/main_test.go b/cmd/atc/main_test.go index 25f061d4..af9b65fa 100644 --- a/cmd/atc/main_test.go +++ b/cmd/atc/main_test.go @@ -5,6 +5,7 @@ import ( "encoding/base64" "encoding/json" "fmt" + "io" "os" "slices" "strconv" @@ -59,19 +60,19 @@ func TestMain(m *testing.M) { must(os.RemoveAll("./test_output")) must(os.MkdirAll("./test_output", 0o755)) - must(x.X("kind delete cluster --name=atc-test")) + must(x.X("k3d cluster delete atc-test")) - must(x.X("kind create cluster --name=atc-test --config -", x.Input(strings.NewReader(` - kind: Cluster - apiVersion: kind.x-k8s.io/v1alpha4 - nodes: - - role: control-plane - extraPortMappings: - - containerPort: 30000 - hostPort: 80 - listenAddress: "127.0.0.1" - protocol: TCP - `)))) + must(x.X("k3d cluster create --config - -p 80:30000@loadbalancer", x.Input(strings.NewReader(` +apiVersion: k3d.io/v1alpha5 +kind: Simple +metadata: + name: atc-test +image: rancher/k3s:v1.31.4-k3s1 +servers: 1 +agents: 0 +`)))) + + must(x.X("kubectl rollout status deployment/coredns -n kube-system --timeout=120s")) if ci, _ := strconv.ParseBool(os.Getenv("CI")); !ci { must(x.X("kubectl apply -f https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml")) @@ -91,20 +92,20 @@ func TestMain(m *testing.M) { "docker build -t yokecd/atc:test -f Dockerfile.atc .", x.Dir("../.."), )) - must(x.X("kind load --name=atc-test docker-image yokecd/atc:test")) + must(x.X("k3d image import yokecd/atc:test --cluster atc-test")) must(x.X("docker build -t yokecd/wasmcache:test -f ./internal/testing/Dockerfile.wasmcache ../..")) - must(x.X("kind load --name=atc-test docker-image yokecd/wasmcache:test")) + must(x.X("k3d image import yokecd/wasmcache:test --cluster atc-test")) must(x.X("docker build -t yokecd/c4ts:test -f ./internal/testing/Dockerfile.c4ts ./internal/testing")) - must(x.X("kind load --name=atc-test docker-image yokecd/c4ts:test")) + must(x.X("k3d image import yokecd/c4ts:test --cluster atc-test")) client := internal.Must2(k8s.NewClientFromKubeConfig(home.Kubeconfig)) commander := yoke.FromK8Client(client) ctx := internal.WithDebugFlag(context.Background(), new(true)) - must(commander.Takeoff(ctx, yoke.TakeoffParams{ + if takeoffErr := commander.Takeoff(ctx, yoke.TakeoffParams{ Release: "atc", Namespace: "atc", Flight: yoke.FlightParams{ @@ -118,10 +119,29 @@ func TestMain(m *testing.M) { Args: []string{"--skip-version-check"}, }, CreateNamespace: true, - Wait: 120 * time.Second, + Wait: 2 * time.Minute, Poll: time.Second, - })) + }); takeoffErr != nil { + fmt.Println("[[DEBUG ATC LOG STREAM]]") + pods, err := client.Clientset.CoreV1().Pods("atc").List(ctx, metav1.ListOptions{LabelSelector: "yoke.cd/app=atc"}) + if err != nil { + panic(err) + } + if len(pods.Items) != 1 { + panic(fmt.Errorf("expected 1 atc pod but got: %d", len(pods.Items))) + } + + req := client.Clientset.CoreV1().Pods("atc").GetLogs(pods.Items[0].Name, &corev1.PodLogOptions{}) + rc, err := req.Stream(ctx) + if err != nil { + panic(fmt.Errorf("failed to get atc log stream: %v", err)) + } + if _, err := io.Copy(os.Stdout, rc); err != nil { + panic(fmt.Errorf("failed to copy atc log stream to stdout: %v", err)) + } + panic(takeoffErr) + } must(commander.Takeoff(ctx, yoke.TakeoffParams{ Release: "wasmcache", Namespace: "atc", @@ -146,6 +166,110 @@ func TestMain(m *testing.M) { os.Exit(m.Run()) } +func TestAdmissionExclusiveToKubeSystem(t *testing.T) { + client, err := k8s.NewClientFromKubeConfig(home.Kubeconfig) + require.NoError(t, err) + + ctx := context.Background() + commander := yoke.FromK8Client(client) + + // Create a test namespace + testNamespace := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-admission-exclusive", + }, + } + // Create the namespace via yoke + require.NoError(t, client.EnsureNamespace(ctx, testNamespace.Name)) + + defer func() { + // Clean up the test namespace + require.NoError(t, client.Clientset.CoreV1().Namespaces().Delete(ctx, testNamespace.Name, metav1.DeleteOptions{})) + }() + + releaseName := "test-admission-curl" + // Deploy the curl pod as a yoke release + require.NoError(t, commander.Takeoff(ctx, yoke.TakeoffParams{ + Release: releaseName, + Namespace: testNamespace.Name, + Flight: yoke.FlightParams{ + Input: internal.JSONReader(corev1.Pod{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Pod", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: releaseName, + Namespace: testNamespace.Name, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "curl", + Image: "curlimages/curl:latest", + Command: []string{ + "sh", "-c", + "curl -v http://atc-atc.atc:80/live && sleep 3600", + }, + }, + }, + RestartPolicy: corev1.RestartPolicyNever, + }, + }), + }, + })) + + // Mayday to ensure cleanup of the release and its resources. + defer func() { + require.NoError(t, commander.Mayday(ctx, yoke.MaydayParams{ + Release: releaseName, + Namespace: testNamespace.Name, + })) + }() + + // Poll pod state and logs using client-go + testutils.EventuallyNoErrorf( + t, + func() error { + pod, err := client.Clientset.CoreV1().Pods(testNamespace.Name).Get(ctx, releaseName, metav1.GetOptions{}) + if err != nil { + return err + } + switch pod.Status.Phase { + case corev1.PodPending, corev1.PodRunning: + return fmt.Errorf("pod is in %s state, waiting", pod.Status.Phase) + case corev1.PodSucceeded: + // curl exited 0 — it connected to ATC, which should not be allowed + return fmt.Errorf("pod curl succeeded unexpectedly: ATC should be inaccessible from non-kube-system namespace") + } + // PodFailed: curl exited non-zero — expected; fall through to log check + logs, err := client.Clientset.CoreV1().Pods(testNamespace.Name). + GetLogs(releaseName, &corev1.PodLogOptions{}).Stream(ctx) + if err != nil { + return fmt.Errorf("failed to get pod logs: %w", err) + } + defer func(logs io.ReadCloser) { + err := logs.Close() + if err != nil { + fmt.Printf("failed to close logs stream: %v", err) + } + }(logs) + logData, err := io.ReadAll(logs) + if err != nil { + return fmt.Errorf("failed to read pod logs: %w", err) + } + logStr := string(logData) + if strings.Contains(logStr, "Connected to") || strings.Contains(logStr, "200 OK") { + return fmt.Errorf("pod should not access ATC, but logs show success: %s", logStr) + } + return nil + }, + time.Second, + 2*time.Minute, + "pod curl to ATC should fail, but it did not", + ) +} + func TestAirTrafficController(t *testing.T) { client, err := k8s.NewClientFromKubeConfig(home.Kubeconfig) require.NoError(t, err)