diff --git a/docs/advanced/runners/kubernetes.md b/docs/advanced/runners/kubernetes.md index e5a1582..db956ba 100644 --- a/docs/advanced/runners/kubernetes.md +++ b/docs/advanced/runners/kubernetes.md @@ -11,7 +11,7 @@ runners: default: kubernetes config: kubernetes: - # Context to use. Optional: By default, zoe uses the current context set in the kube config file. + # Context to use. Optional: By default, zoe uses the current context set in the kube config file. context: mu-kube-context # Namespace to use. Optional: By default, zoe uses the 'default' namespace. namespace: env-staging @@ -27,4 +27,41 @@ runners: annotations: key1: value1 key2: value2 + # Service Account name. Optional: By default, pods use the 'default' service account. + # Useful for AWS IRSA (IAM Roles for Service Accounts) or other pod identity mechanisms. + serviceAccountName: zoe-service-account ``` + +## AWS IRSA Support + +Zoe supports [AWS IAM Roles for Service Accounts (IRSA)](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html) on EKS clusters. This allows Zoe pods to authenticate to AWS services (like MSK) without requiring AWS credentials to be stored in configuration files or environment variables. + +To use IRSA with Zoe: + +1. **Create an IAM role** with the necessary permissions (e.g., MSK access) +2. **Set up the OIDC provider** for your EKS cluster (usually done during cluster creation) +3. **Create a Kubernetes Service Account** with the `eks.amazonaws.com/role-arn` annotation: + +```yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: zoe-service-account + namespace: zoe-namespace + annotations: + eks.amazonaws.com/role-arn: arn:aws:iam::123456789012:role/ZoeKafkaAccess +``` + +4. **Configure Zoe** to use the service account: + +```yaml +runners: + default: kubernetes + config: + kubernetes: + namespace: zoe-namespace + serviceAccountName: zoe-service-account + # ... other configuration +``` + +When configured this way, Zoe pods will automatically receive temporary AWS credentials and can access AWS services according to the IAM role's permissions. diff --git a/docs/configuration/reference.md b/docs/configuration/reference.md index 41279d7..b5b15d2 100644 --- a/docs/configuration/reference.md +++ b/docs/configuration/reference.md @@ -66,10 +66,12 @@ runners: memory: "512M" # Timeout of the commands timeoutMs: 300000 - # Annotations to attach to the pods + # Annotations to attach to the pods annotations: key1: value1 key2: value2 + # Service Account name (optional). Useful for AWS IRSA or other pod identity mechanisms + serviceAccountName: zoe-service-account # The lambda runner configuration lambda: diff --git a/docs/guides/kubernetes/guide.md b/docs/guides/kubernetes/guide.md index 8ef91a0..f5687af 100644 --- a/docs/guides/kubernetes/guide.md +++ b/docs/guides/kubernetes/guide.md @@ -119,7 +119,27 @@ Ensure zoe is aware about our new configuration: └─────────┴─────────────┴──────────┴────────┴────────┘ ``` -Notice our use of `-e k8s` in the above command. Zoe supports having multiple configuration files inside its config directory representing different environments. To point to a specific environment, we use the `-e ` (`` is the name of the configuration file without the extension). When no environment is specified, zoe uses the environment called `default`. +Notice our use of `-e k8s` in the above command. Zoe supports having multiple configuration files inside its config directory representing different environments. To point to a specific environment, we use the `-e ` (`` is the name of the configuration file without the extension). When no environment is specified, zoe uses the environment called `default`. + +### AWS IRSA (Optional) + +If you're running on AWS EKS and need to access AWS services like MSK (Managed Streaming for Kafka), you can configure Zoe to use [IAM Roles for Service Accounts (IRSA)](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html). This allows Zoe pods to authenticate to AWS without storing credentials. + +To enable IRSA: + +1. Create a Kubernetes Service Account with the appropriate IAM role annotation +2. Add `serviceAccountName` to your Zoe configuration: + +```yaml +runners: + default: "kubernetes" + config: + kubernetes: + namespace: env-staging + serviceAccountName: zoe-service-account # Your service account name +``` + +For more details, see the [Kubernetes runner documentation](https://adevinta.github.io/zoe/advanced/runners/kubernetes/#aws-irsa-support). Zoe is now ready to be used against our cluster! diff --git a/examples/config/kubernetes/service-account-example.yml b/examples/config/kubernetes/service-account-example.yml new file mode 100644 index 0000000..65c9ec2 --- /dev/null +++ b/examples/config/kubernetes/service-account-example.yml @@ -0,0 +1,31 @@ +# Copyright (c) 2020 Adevinta. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +--- +clusters: + default: + props: + bootstrap.servers: "broker:9092" + key.deserializer: "org.apache.kafka.common.serialization.StringDeserializer" + value.deserializer: "org.apache.kafka.common.serialization.StringDeserializer" + key.serializer: "org.apache.kafka.common.serialization.StringSerializer" + value.serializer: "org.apache.kafka.common.serialization.ByteArraySerializer" + +runners: + default: "kubernetes" + config: + kubernetes: + namespace: "zoe-namespace" + serviceAccountName: "zoe-service-account" # Service Account for IRSA or other Pod identity mechanisms + cpu: "1" + memory: "512M" + timeoutMs: 300000 + deletePodAfterCompletion: true + annotations: + app: "zoe" + environment: "production" diff --git a/zoe-cli/src/commands/main.kt b/zoe-cli/src/commands/main.kt index b23ad12..6fd0255 100644 --- a/zoe-cli/src/commands/main.kt +++ b/zoe-cli/src/commands/main.kt @@ -237,6 +237,7 @@ fun mainModule(context: CliContext) = module { deletePodsAfterCompletion = kubeConfig.deletePodAfterCompletion, timeoutMs = kubeConfig.timeoutMs, annotations = kubeConfig.annotations, + serviceAccountName = kubeConfig.serviceAccountName ), executor = ioPool, namespace = kubeConfig.namespace, diff --git a/zoe-cli/src/config/config.kt b/zoe-cli/src/config/config.kt index 58885d3..a4117d0 100644 --- a/zoe-cli/src/config/config.kt +++ b/zoe-cli/src/config/config.kt @@ -121,6 +121,7 @@ data class KubernetesRunnerConfig( val timeoutMs: Long = 300000, val image: DockerImageConfig = DockerImageConfig(), val annotations: Map = emptyMap(), + val serviceAccountName: String? = null ) data class DockerImageConfig( diff --git a/zoe-service/build.gradle.kts b/zoe-service/build.gradle.kts index bec9797..3a30c99 100644 --- a/zoe-service/build.gradle.kts +++ b/zoe-service/build.gradle.kts @@ -25,6 +25,7 @@ dependencies { testImplementation(group = "junit", name = "junit", version = "4.12") testImplementation("org.testcontainers:testcontainers:1.20.3") testImplementation("org.testcontainers:kafka:1.20.3") + testImplementation("io.fabric8:kubernetes-server-mock:4.10.1") testImplementation("org.spekframework.spek2:spek-dsl-jvm:2.0.10") testRuntimeOnly("org.spekframework.spek2:spek-runner-junit5:2.0.10") @@ -44,6 +45,12 @@ tasks { compileTestKotlin { kotlinOptions.jvmTarget = "21" } + + test { + useJUnitPlatform { + includeEngines("spek2") + } + } } sourceSets { diff --git a/zoe-service/resources/pod.template.json b/zoe-service/resources/pod.template.json index 6f8029a..4ff8728 100644 --- a/zoe-service/resources/pod.template.json +++ b/zoe-service/resources/pod.template.json @@ -6,6 +6,14 @@ "labels": {} }, "spec": { + "securityContext": { + "runAsNonRoot": true, + "runAsUser": 1000, + "fsGroup": 1000, + "seccompProfile": { + "type": "RuntimeDefault" + } + }, "volumes": [ { "name": "output-volume", @@ -25,7 +33,18 @@ "mountPath": "/output", "name": "output-volume" } - ] + ], + "securityContext": { + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": ["ALL"] + }, + "runAsNonRoot": true, + "runAsUser": 1000, + "seccompProfile": { + "type": "RuntimeDefault" + } + } } ], "containers": [ @@ -45,7 +64,18 @@ "mountPath": "/output", "name": "output-volume" } - ] + ], + "securityContext": { + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": ["ALL"] + }, + "runAsNonRoot": true, + "runAsUser": 1000, + "seccompProfile": { + "type": "RuntimeDefault" + } + } }, { "name": "tailer", @@ -66,7 +96,18 @@ "mountPath": "/output", "name": "output-volume" } - ] + ], + "securityContext": { + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": ["ALL"] + }, + "runAsNonRoot": true, + "runAsUser": 1000, + "seccompProfile": { + "type": "RuntimeDefault" + } + } } ], "restartPolicy": "Never" diff --git a/zoe-service/src/runners/kubernetes.kt b/zoe-service/src/runners/kubernetes.kt index 81a656f..4c49e09 100644 --- a/zoe-service/src/runners/kubernetes.kt +++ b/zoe-service/src/runners/kubernetes.kt @@ -68,7 +68,8 @@ class KubernetesRunner( val cpu: String, val memory: String, val timeoutMs: Long?, - val annotations: Map + val annotations: Map, + val serviceAccountName: String? ) private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Default) @@ -212,6 +213,11 @@ class KubernetesRunner( metadata.labels = labels metadata.annotations = annotations + // Set service account name if provided + configuration.serviceAccountName?.let { + spec.serviceAccountName = it + } + spec.containers.find { it.name == "zoe" }?.apply { resources.requests = mapOf( "cpu" to Quantity.parse(configuration.cpu), @@ -266,4 +272,4 @@ fun Watchable>.watchContinuously(): Flow = flo while (true) { emitAll(watchAsFlow()) } -} \ No newline at end of file +} diff --git a/zoe-service/test/runners/KubernetesRunnerTest.kt b/zoe-service/test/runners/KubernetesRunnerTest.kt new file mode 100644 index 0000000..083b3d5 --- /dev/null +++ b/zoe-service/test/runners/KubernetesRunnerTest.kt @@ -0,0 +1,128 @@ +// Copyright (c) 2020 Adevinta. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +package com.adevinta.oss.zoe.service.runners + +import io.fabric8.kubernetes.api.model.Pod +import io.fabric8.kubernetes.client.DefaultKubernetesClient +import io.fabric8.kubernetes.client.NamespacedKubernetesClient +import io.fabric8.kubernetes.client.server.mock.KubernetesServer +import org.junit.Assert +import org.spekframework.spek2.Spek +import org.spekframework.spek2.style.specification.describe +import java.util.concurrent.Executors + +object KubernetesRunnerTest : Spek({ + + describe("KubernetesRunner pod generation") { + + val executor = Executors.newSingleThreadExecutor() + val server = KubernetesServer(false, true) + + beforeGroup { + server.before() + } + + afterGroup { + server.after() + executor.shutdown() + } + + it("should set serviceAccountName when provided in configuration") { + val client: NamespacedKubernetesClient = server.client.inNamespace("test") + val serviceAccountName = "my-service-account" + + val config = KubernetesRunner.Config( + deletePodsAfterCompletion = false, + zoeImage = "wlezzar/zoe:test", + cpu = "1", + memory = "512M", + timeoutMs = 60000L, + annotations = emptyMap(), + serviceAccountName = serviceAccountName + ) + + val runner = KubernetesRunner( + name = "test-runner", + client = client, + executor = executor, + closeClientAtShutdown = false, + configuration = config + ) + + // Use reflection to access the private generatePodObject method + val method = KubernetesRunner::class.java.getDeclaredMethod( + "generatePodObject", + String::class.java, + Map::class.java, + List::class.java + ) + method.isAccessible = true + + val pod = method.invoke( + runner, + "wlezzar/zoe:test", + emptyMap(), + listOf("arg1", "arg2") + ) as Pod + + Assert.assertEquals( + "serviceAccountName should be set in pod spec", + serviceAccountName, + pod.spec.serviceAccountName + ) + + runner.close() + } + + it("should not set serviceAccountName when not provided in configuration") { + val client: NamespacedKubernetesClient = server.client.inNamespace("test") + + val config = KubernetesRunner.Config( + deletePodsAfterCompletion = false, + zoeImage = "wlezzar/zoe:test", + cpu = "1", + memory = "512M", + timeoutMs = 60000L, + annotations = emptyMap(), + serviceAccountName = null + ) + + val runner = KubernetesRunner( + name = "test-runner", + client = client, + executor = executor, + closeClientAtShutdown = false, + configuration = config + ) + + // Use reflection to access the private generatePodObject method + val method = KubernetesRunner::class.java.getDeclaredMethod( + "generatePodObject", + String::class.java, + Map::class.java, + List::class.java + ) + method.isAccessible = true + + val pod = method.invoke( + runner, + "wlezzar/zoe:test", + emptyMap(), + listOf("arg1", "arg2") + ) as Pod + + Assert.assertNull( + "serviceAccountName should be null when not configured", + pod.spec.serviceAccountName + ) + + runner.close() + } + } +})