From 11030e99d1b268e823bf4392e2acbee95842c7db Mon Sep 17 00:00:00 2001 From: kpoliwka Date: Fri, 15 Mar 2024 15:13:26 +0100 Subject: [PATCH 01/17] add eks --- .scalafmt.conf | 44 +++------- infra/Pulumi.yaml | 7 -- infra/eks/Main.scala | 28 +++++++ infra/eks/Pulumi.yaml | 5 ++ infra/eks/project.scala | 6 ++ infra/k8s/Pulumi.yaml | 11 +++ infra/{ => k8s}/project.scala | 0 infra/{ => k8s}/src/main/scala/Jaeger.scala | 12 ++- infra/{ => k8s}/src/main/scala/Kafka.scala | 16 ++-- infra/k8s/src/main/scala/Main.scala | 84 +++++++++++++++++++ infra/{ => k8s}/src/main/scala/Ops.scala | 0 infra/{ => k8s}/src/main/scala/Postgres.scala | 29 +++++-- infra/{ => k8s}/src/main/scala/VSS.scala | 18 +++- .../{ => k8s}/src/main/scala/Zookeeper.scala | 14 ++-- infra/src/main/scala/Main.scala | 42 ---------- project/build.properties | 2 +- 16 files changed, 208 insertions(+), 110 deletions(-) delete mode 100644 infra/Pulumi.yaml create mode 100644 infra/eks/Main.scala create mode 100644 infra/eks/Pulumi.yaml create mode 100644 infra/eks/project.scala create mode 100644 infra/k8s/Pulumi.yaml rename infra/{ => k8s}/project.scala (100%) rename infra/{ => k8s}/src/main/scala/Jaeger.scala (87%) rename infra/{ => k8s}/src/main/scala/Kafka.scala (84%) create mode 100644 infra/k8s/src/main/scala/Main.scala rename infra/{ => k8s}/src/main/scala/Ops.scala (100%) rename infra/{ => k8s}/src/main/scala/Postgres.scala (85%) rename infra/{ => k8s}/src/main/scala/VSS.scala (87%) rename infra/{ => k8s}/src/main/scala/Zookeeper.scala (80%) delete mode 100644 infra/src/main/scala/Main.scala diff --git a/.scalafmt.conf b/.scalafmt.conf index bfb16ed..75662e5 100644 --- a/.scalafmt.conf +++ b/.scalafmt.conf @@ -1,34 +1,12 @@ -version = 3.7.1 -maxColumn = 120 -align.preset = more +version = 3.5.2 runner.dialect = scala3 - -continuationIndent.callSite = 2 -continuationIndent.defnSite = 2 -continuationIndent.extendSite = 2 -danglingParentheses.preset = true - -docstrings.blankFirstLine = true - -newlines { - sometimesBeforeColonInMethodReturnType = false -} - -align { - arrowEnumeratorGenerator = false - ifWhileOpenParen = false - openParenCallSite = false - openParenDefnSite = false - tokens = [{code = "=>", owner = "Case"}, "<-", "%", "%%", "="] - preset = "some" -} - -rewrite { - rules = [SortImports, RedundantBraces, redundantParens] - redundantBraces.maxLines = 1 -} - -spaces { - beforeContextBoundColon = true - inImportCurlyBraces = false -} +project.excludePaths = [ + "glob:**/src/main/scala/besom/rpc/**.scala" +] +project.git = true +align = most +align.openParenCallSite = false +align.openParenDefnSite = false +align.tokens = [{code = "=>", owner = "Case"}, "<-", "%", "%%", "="] +indent.defnSite = 2 +maxColumn = 120 \ No newline at end of file diff --git a/infra/Pulumi.yaml b/infra/Pulumi.yaml deleted file mode 100644 index 4267c05..0000000 --- a/infra/Pulumi.yaml +++ /dev/null @@ -1,7 +0,0 @@ -name: vss -description: VSS infra -runtime: scala -config: - vss:localRegistry: "localhost:5001" - vss:imageName: "vss-zio" - vss:imageTag: "0.1.0-SNAPSHOT" \ No newline at end of file diff --git a/infra/eks/Main.scala b/infra/eks/Main.scala new file mode 100644 index 0000000..891cec7 --- /dev/null +++ b/infra/eks/Main.scala @@ -0,0 +1,28 @@ +import besom.* +import besom.api.{awsx, eks} + +@main def main = Pulumi.run { + val appName = "vss" + val vpc = awsx.ec2.Vpc( + name = s"$appName-vpc", + awsx.ec2.VpcArgs( + cidrBlock = "10.1.0.0/16", + enableDnsHostnames = true, + enableDnsSupport = true + ) + ) + + val cluster = eks.Cluster( + name = s"$appName-cluster", + eks.ClusterArgs( + vpcId = vpc.vpcId, + subnetIds = vpc.publicSubnetIds, + desiredCapacity = 2, + minSize = 1, + maxSize = 2, + storageClasses = "gp2" + ) + ) + + Stack(cluster).exports(kubeconfig = cluster.kubeconfigJson) +} diff --git a/infra/eks/Pulumi.yaml b/infra/eks/Pulumi.yaml new file mode 100644 index 0000000..1e54620 --- /dev/null +++ b/infra/eks/Pulumi.yaml @@ -0,0 +1,5 @@ +name: vss-aws-eks +runtime: scala +description: VSS AWS EKS infra +config: + aws:region: us-west-2 \ No newline at end of file diff --git a/infra/eks/project.scala b/infra/eks/project.scala new file mode 100644 index 0000000..bb5f9a0 --- /dev/null +++ b/infra/eks/project.scala @@ -0,0 +1,6 @@ +//> using scala "3.3.3" +//> using options -Werror -Wunused:all -Wvalue-discard -Wnonunit-statement +//> using plugin "org.virtuslab::besom-compiler-plugin:0.2.2" +//> using dep "org.virtuslab::besom-core:0.2.2" +//> using dep "org.virtuslab::besom-awsx:2.5.0-core.0.2" +//> using dep "org.virtuslab::besom-eks:2.2.1-core.0.2" diff --git a/infra/k8s/Pulumi.yaml b/infra/k8s/Pulumi.yaml new file mode 100644 index 0000000..a340182 --- /dev/null +++ b/infra/k8s/Pulumi.yaml @@ -0,0 +1,11 @@ +name: vss +description: VSS infra +runtime: scala +config: + vss:localRegistry: "localhost:5001" + vss:imageName: "vss-cats" + vss:imageTag: "0.1.0-SNAPSHOT" + vss:cluster: "local" + vss:cluster-org: "organization" + vss:cluster-project: "vss-aws-eks" + vss:cluster-stack: "dev" \ No newline at end of file diff --git a/infra/project.scala b/infra/k8s/project.scala similarity index 100% rename from infra/project.scala rename to infra/k8s/project.scala diff --git a/infra/src/main/scala/Jaeger.scala b/infra/k8s/src/main/scala/Jaeger.scala similarity index 87% rename from infra/src/main/scala/Jaeger.scala rename to infra/k8s/src/main/scala/Jaeger.scala index 513614b..a5a1a47 100644 --- a/infra/src/main/scala/Jaeger.scala +++ b/infra/k8s/src/main/scala/Jaeger.scala @@ -25,7 +25,7 @@ object Jaeger { "zipkin-collector" -> (None, 9411) ) - def deploy(using Context)(namespace: Output[Namespace]) = Deployment( + def deploy(using Context)(namespace: Output[Namespace], k8sProvider: Output[k8s.Provider]) = Deployment( appName, DeploymentArgs( spec = DeploymentSpecArgs( @@ -64,10 +64,13 @@ object Jaeger { name = s"$appName-deployment", namespace = namespace.metadata.name ) - ) + ), + opts(provider = k8sProvider) ) - def deployService(using Context)(namespace: Output[Namespace]) = Service( + def deployService(using + Context + )(namespace: Output[Namespace], jaegerDeployment: Output[Deployment], k8sProvider: Output[k8s.Provider]) = Service( appName, ServiceArgs( spec = ServiceSpecArgs( @@ -80,7 +83,8 @@ object Jaeger { name = s"$appName-service", namespace = namespace.metadata.name ) - ) + ), + opts(dependsOn = jaegerDeployment, provider = k8sProvider) ) } diff --git a/infra/src/main/scala/Kafka.scala b/infra/k8s/src/main/scala/Kafka.scala similarity index 84% rename from infra/src/main/scala/Kafka.scala rename to infra/k8s/src/main/scala/Kafka.scala index 0c74b59..b74f009 100644 --- a/infra/src/main/scala/Kafka.scala +++ b/infra/k8s/src/main/scala/Kafka.scala @@ -2,7 +2,7 @@ import besom.* import besom.util.* import besom.api.kubernetes as k8s import k8s.core.v1.inputs.* -import k8s.core.v1.{ConfigMap, Namespace, Service, ConfigMapArgs, ServiceArgs} +import k8s.core.v1.{ConfigMap, ConfigMapArgs, Namespace, Service, ServiceArgs} import k8s.apps.v1.inputs.* import k8s.apps.v1.{Deployment, DeploymentArgs} import k8s.meta.v1.inputs.* @@ -14,7 +14,9 @@ object Kafka { val kafkaServiceName = s"$appName-service" val port = 9092 - def deploy(using Context)(namespace: Output[Namespace], zookeeperService: Output[Service]) = Deployment( + def deploy(using + Context + )(namespace: Output[Namespace], zookeeperService: Output[Service], k8sProvider: Output[k8s.Provider]) = Deployment( appName, DeploymentArgs( spec = DeploymentSpecArgs( @@ -61,10 +63,13 @@ object Kafka { name = "kafka-deployment", namespace = namespace.metadata.name ) - ) + ), + opts(provider = k8sProvider) ) - def deployService(using Context)(namespace: Output[Namespace]) = Service( + def deployService(using + Context + )(namespace: Output[Namespace], kafkaDeployment: Output[Deployment], k8sProvider: Output[k8s.Provider]) = Service( appName, ServiceArgs( spec = ServiceSpecArgs( @@ -77,7 +82,8 @@ object Kafka { name = kafkaServiceName, namespace = namespace.metadata.name ) - ) + ), + opts(dependsOn = kafkaDeployment, provider = k8sProvider) ) } diff --git a/infra/k8s/src/main/scala/Main.scala b/infra/k8s/src/main/scala/Main.scala new file mode 100644 index 0000000..884c22c --- /dev/null +++ b/infra/k8s/src/main/scala/Main.scala @@ -0,0 +1,84 @@ +import besom.* +import besom.api.kubernetes as k8s +import k8s.core.v1.{Namespace, Service} +import k8s.core.v1.enums.ServiceSpecType +import besom.internal.{Config, Output} +import besom.json.DefaultJsonProtocol.StringJsonFormat + +@main def main = Pulumi.run { + + val serviceType = config + .requireString("cluster") + .map: + case "remote" => + ServiceSpecType.LoadBalancer + case _ => + ServiceSpecType.ClusterIP + + val k8sProvider = config + .requireString("cluster") + .flatMap: + case "local" => + k8s.Provider(name = "vss-local-provider") + case "remote" => + for + k8sOrgName <- config.getString("cluster-org").getOrElse("organization") + k8sProjName <- config.requireString("cluster-project") + k8sStackName <- config.requireString("cluster-stack") + stack <- StackReference(name = s"$k8sOrgName/$k8sProjName/$k8sStackName") + kubeconfigJson <- stack.requireOutput("kubeconfig") + provider <- k8s.Provider( + name = "vss-remote-provider", + k8s.ProviderArgs(kubeconfig = kubeconfigJson.convertTo[String]) + ) + yield provider + case str => + throw Exception( + s"$str value not allowed. Available values are local or remote. Change vss:cluster configuration" + ) + + val appNamespace = Namespace(name = "vss", opts = opts(provider = k8sProvider)) + + // zookeeper + val zooDeployment = Zookeeper.deploy(appNamespace, k8sProvider) + val zooService = Zookeeper.deployService(appNamespace, zooDeployment, k8sProvider) + + // kafka + val kafkaDeployment = Kafka.deploy(appNamespace, zooService, k8sProvider) + val kafkaService = Kafka.deployService(appNamespace, kafkaDeployment, k8sProvider) + + // postgres + val postgresDeployment = Postgres.deploy(appNamespace, k8sProvider) + val postgresService = Postgres.deployService(appNamespace, postgresDeployment, k8sProvider) + + // jaeger + val jaegerDeployment = Jaeger.deploy(appNamespace, k8sProvider) + val jaegerService = Jaeger.deployService(appNamespace, jaegerDeployment, k8sProvider) + + // vss + val vssDeployment = VSS.deploy(config, appNamespace, postgresService, kafkaService, jaegerService, k8sProvider) + val vssService = VSS.deployService(serviceType, appNamespace, vssDeployment, k8sProvider) + + val vssServiceUrl = + vssService.status.loadBalancer.ingress + .map( + _.flatMap(_.headOption.flatMap(_.hostname)) + .map(host => p"http://$host:${VSS.ports("main-http")._2}/docs") + .getOrElse("Host not find. Probably vss:cluster is set to local") + ) + + Stack.exports( + serviceUrl = vssServiceUrl, + namespaceName = appNamespace.metadata.name, + zookeeperDeploymentName = zooDeployment.metadata.name, + zookeeperServiceName = zooService.metadata.name, + kafkaDeploymentName = kafkaDeployment.metadata.name, + kafkaServiceName = kafkaService.metadata.name, + postgresDeploymentName = postgresDeployment.metadata.name, + postgresServiceName = postgresService.metadata.name, + jaegerDeploymentName = jaegerDeployment.metadata.name, + jaegerServiceName = jaegerService.metadata.name, + vssDeploymentName = vssDeployment.metadata.name, + vssServiceName = vssService.metadata.name + ) +} diff --git a/infra/src/main/scala/Ops.scala b/infra/k8s/src/main/scala/Ops.scala similarity index 100% rename from infra/src/main/scala/Ops.scala rename to infra/k8s/src/main/scala/Ops.scala diff --git a/infra/src/main/scala/Postgres.scala b/infra/k8s/src/main/scala/Postgres.scala similarity index 85% rename from infra/src/main/scala/Postgres.scala rename to infra/k8s/src/main/scala/Postgres.scala index 5b480d2..c37c4d0 100644 --- a/infra/src/main/scala/Postgres.scala +++ b/infra/k8s/src/main/scala/Postgres.scala @@ -14,18 +14,20 @@ object Postgres { val labels = Map("app" -> "postgres") val port = 5432 - def deploy(using Context)(namespace: Output[Namespace]) = { + def deploy(using Context)(namespace: Output[Namespace], k8sProvider: Output[k8s.Provider]) = { val postgresPV = PersistentVolume( appName, k8s.core.v1.PersistentVolumeArgs( metadata = ObjectMetaArgs(name = s"$appName-pv", namespace = namespace.metadata.name), spec = PersistentVolumeSpecArgs( + storageClassName = "standard", capacity = Map("storage" -> "8Gi"), accessModes = List("ReadWriteMany"), hostPath = HostPathVolumeSourceArgs("/data/db") ) - ) + ), + opts(provider = k8sProvider) ) val postgresPVC = PersistentVolumeClaim( @@ -33,12 +35,15 @@ object Postgres { k8s.core.v1.PersistentVolumeClaimArgs( metadata = ObjectMetaArgs(name = s"$appName-pvc", namespace = namespace.metadata.name), spec = PersistentVolumeClaimSpecArgs( + storageClassName = "standard", + volumeName = postgresPV.metadata.name, accessModes = List("ReadWriteMany"), resources = VolumeResourceRequirementsArgs( requests = Map("storage" -> "8Gi") ) ) - ) + ), + opts(provider = k8sProvider) ) val postgresConfigMap = ConfigMap( @@ -51,7 +56,8 @@ object Postgres { "POSTGRES_USER" -> "postgres", "POSTGRES_PASSWORD" -> "postgres" ) - ) + ), + opts(provider = k8sProvider) ) val initConfigMap = ConfigMap( @@ -59,8 +65,9 @@ object Postgres { ConfigMapArgs( metadata = ObjectMetaArgs(name = s"$appName-init-config-map", namespace = namespace.metadata.name), // path starts from besom/.scala-build - data = Ops.readFileIntoConfigMap("../commons/src/main/resources/tables.sql", Some("init.sql")) - ) + data = Ops.readFileIntoConfigMap("../../commons/src/main/resources/tables.sql", Some("init.sql")) + ), + opts(provider = k8sProvider) ) Deployment( @@ -120,11 +127,14 @@ object Postgres { name = s"$appName-deployment", namespace = namespace.metadata.name ) - ) + ), + opts(provider = k8sProvider) ) } - def deployService(using Context)(namespace: Output[Namespace]) = Service( + def deployService(using + Context + )(namespace: Output[Namespace], postgresDeployment: Output[Deployment], k8sProvider: Output[k8s.Provider]) = Service( appName, ServiceArgs( spec = ServiceSpecArgs( @@ -137,6 +147,7 @@ object Postgres { name = s"$appName-service", namespace = namespace.metadata.name ) - ) + ), + opts(dependsOn = postgresDeployment, provider = k8sProvider) ) } diff --git a/infra/src/main/scala/VSS.scala b/infra/k8s/src/main/scala/VSS.scala similarity index 87% rename from infra/src/main/scala/VSS.scala rename to infra/k8s/src/main/scala/VSS.scala index 9e298c8..4cf8418 100644 --- a/infra/src/main/scala/VSS.scala +++ b/infra/k8s/src/main/scala/VSS.scala @@ -3,6 +3,7 @@ import besom.util.* import besom.api.kubernetes as k8s import k8s.core.v1.inputs.* import k8s.core.v1.{ConfigMap, ConfigMapArgs, Namespace, Service, ServiceArgs} +import k8s.core.v1.enums.ServiceSpecType import k8s.apps.v1.inputs.* import k8s.apps.v1.{Deployment, DeploymentArgs} import k8s.meta.v1.inputs.* @@ -24,7 +25,8 @@ object VSS { namespace: Output[Namespace], postgresService: Output[Service], kafkaService: Output[Service], - jaegerService: Output[Service] + jaegerService: Output[Service], + k8sProvider: Output[k8s.Provider] ) = { val localRegistry = config.requireString("localRegistry") val imageName = config.requireString("imageName") @@ -74,14 +76,21 @@ object VSS { name = s"$appName-deployment", namespace = namespace.metadata.name ) - ) + ), + opts(provider = k8sProvider) ) } - def deployService(using Context)(namespace: Output[Namespace]) = Service( + def deployService(using Context)( + serviceType: Output[ServiceSpecType], + namespace: Output[Namespace], + vssDeployment: Output[Deployment], + k8sProvider: Output[k8s.Provider] + ) = Service( appName, ServiceArgs( spec = ServiceSpecArgs( + `type` = serviceType, selector = labels, ports = ports.map { case (name, (protocol, port)) => ServicePortArgs(name = name, port = port, targetPort = port, protocol = protocol) @@ -91,7 +100,8 @@ object VSS { name = s"$appName-service", namespace = namespace.metadata.name ) - ) + ), + opts(dependsOn = vssDeployment, provider = k8sProvider) ) } diff --git a/infra/src/main/scala/Zookeeper.scala b/infra/k8s/src/main/scala/Zookeeper.scala similarity index 80% rename from infra/src/main/scala/Zookeeper.scala rename to infra/k8s/src/main/scala/Zookeeper.scala index c386a44..785c6d7 100644 --- a/infra/src/main/scala/Zookeeper.scala +++ b/infra/k8s/src/main/scala/Zookeeper.scala @@ -2,7 +2,7 @@ import besom.* import besom.util.NonEmptyString import besom.api.kubernetes as k8s import k8s.core.v1.inputs.* -import k8s.core.v1.{ConfigMap, Namespace, Service, ConfigMapArgs, ServiceArgs} +import k8s.core.v1.{ConfigMap, ConfigMapArgs, Namespace, Service, ServiceArgs} import k8s.apps.v1.inputs.* import k8s.apps.v1.{Deployment, DeploymentArgs} import k8s.meta.v1.inputs.* @@ -13,7 +13,7 @@ object Zookeeper { val appName: NonEmptyString = "zookeeper" // todo fix inference in NonEmptyString val labels = Map("app" -> "zookeeper") - def deploy(using Context)(namespace: Output[Namespace]) = Deployment( + def deploy(using Context)(namespace: Output[Namespace], k8sProvider: Output[k8s.Provider]) = Deployment( appName, DeploymentArgs( spec = DeploymentSpecArgs( @@ -46,10 +46,13 @@ object Zookeeper { name = s"$appName-deployment", namespace = namespace.metadata.name ) - ) + ), + opts(provider = k8sProvider) ) - def deployService(using Context)(namespace: Output[Namespace]) = Service( + def deployService(using + Context + )(namespace: Output[Namespace], zooDeployment: Output[Deployment], k8sProvider: Output[k8s.Provider]) = Service( appName, ServiceArgs( spec = ServiceSpecArgs( @@ -62,7 +65,8 @@ object Zookeeper { name = s"$appName-service", namespace = namespace.metadata.name ) - ) + ), + opts(dependsOn = zooDeployment, provider = k8sProvider) ) } diff --git a/infra/src/main/scala/Main.scala b/infra/src/main/scala/Main.scala deleted file mode 100644 index 922385c..0000000 --- a/infra/src/main/scala/Main.scala +++ /dev/null @@ -1,42 +0,0 @@ -import besom.* -import besom.api.kubernetes.core.v1.{Namespace, Service} -import besom.internal.{Config, Output} - -@main def main = Pulumi.run { - - val appNamespace = Namespace(name = "vss") - - // zookeeper - val zooDeployment = Zookeeper.deploy(appNamespace) - val zooService = Zookeeper.deployService(appNamespace) - - // kafka - val kafkaDeployment = Kafka.deploy(appNamespace, zooService) - val kafkaService = Kafka.deployService(appNamespace) - - // postgres - val postgresDeployment = Postgres.deploy(appNamespace) - val postgresService = Postgres.deployService(appNamespace) - - // jaeger - val jaegerDeployment = Jaeger.deploy(appNamespace) - val jaegerService = Jaeger.deployService(appNamespace) - - // vss - val vssDeployment = VSS.deploy(config, appNamespace, postgresService, kafkaService, jaegerService) - val vssService = VSS.deployService(appNamespace) - - Stack.exports( - namespaceName = appNamespace.metadata.name, - zookeeperDeploymentName = zooDeployment.metadata.name, - zookeeperServiceName = zooService.metadata.name, - kafkaDeploymentName = kafkaDeployment.metadata.name, - kafkaServiceName = kafkaService.metadata.name, - postgresDeploymentName = postgresDeployment.metadata.name, - postgresServiceName = postgresService.metadata.name, - jaegerDeploymentName = jaegerDeployment.metadata.name, - jaegerServiceName = jaegerService.metadata.name, - vssDeploymentName = vssDeployment.metadata.name, - vssServiceName = vssService.metadata.name - ) -} diff --git a/project/build.properties b/project/build.properties index f344c14..b19d4e1 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version = 1.8.2 +sbt.version = 1.9.7 From eea36dd10a1e5ebecb6af12b62e194be92b81cad Mon Sep 17 00:00:00 2001 From: kpoliwka Date: Fri, 15 Mar 2024 15:21:00 +0100 Subject: [PATCH 02/17] rollback changes --- .scalafmt.conf | 44 +++++++++++++++++++++++++++++++++----------- 1 file changed, 33 insertions(+), 11 deletions(-) diff --git a/.scalafmt.conf b/.scalafmt.conf index 75662e5..bfb16ed 100644 --- a/.scalafmt.conf +++ b/.scalafmt.conf @@ -1,12 +1,34 @@ -version = 3.5.2 +version = 3.7.1 +maxColumn = 120 +align.preset = more runner.dialect = scala3 -project.excludePaths = [ - "glob:**/src/main/scala/besom/rpc/**.scala" -] -project.git = true -align = most -align.openParenCallSite = false -align.openParenDefnSite = false -align.tokens = [{code = "=>", owner = "Case"}, "<-", "%", "%%", "="] -indent.defnSite = 2 -maxColumn = 120 \ No newline at end of file + +continuationIndent.callSite = 2 +continuationIndent.defnSite = 2 +continuationIndent.extendSite = 2 +danglingParentheses.preset = true + +docstrings.blankFirstLine = true + +newlines { + sometimesBeforeColonInMethodReturnType = false +} + +align { + arrowEnumeratorGenerator = false + ifWhileOpenParen = false + openParenCallSite = false + openParenDefnSite = false + tokens = [{code = "=>", owner = "Case"}, "<-", "%", "%%", "="] + preset = "some" +} + +rewrite { + rules = [SortImports, RedundantBraces, redundantParens] + redundantBraces.maxLines = 1 +} + +spaces { + beforeContextBoundColon = true + inImportCurlyBraces = false +} From 7c8bf7cdf6a56263709864e45a70c415c7775cf1 Mon Sep 17 00:00:00 2001 From: kpoliwka Date: Wed, 20 Mar 2024 07:21:41 +0100 Subject: [PATCH 03/17] add Grafana, Loki and Promtail --- infra/k8s/src/main/scala/Grafana.scala | 137 ++++++++++++++++ infra/k8s/src/main/scala/Loki.scala | 150 +++++++++++++++++ infra/k8s/src/main/scala/Main.scala | 16 ++ infra/k8s/src/main/scala/Promtail.scala | 204 ++++++++++++++++++++++++ 4 files changed, 507 insertions(+) create mode 100644 infra/k8s/src/main/scala/Grafana.scala create mode 100644 infra/k8s/src/main/scala/Loki.scala create mode 100644 infra/k8s/src/main/scala/Promtail.scala diff --git a/infra/k8s/src/main/scala/Grafana.scala b/infra/k8s/src/main/scala/Grafana.scala new file mode 100644 index 0000000..83b206a --- /dev/null +++ b/infra/k8s/src/main/scala/Grafana.scala @@ -0,0 +1,137 @@ +import besom.api.kubernetes.core.v1.Namespace +import besom.* +import besom.aliases.NonEmptyString +import besom.api.kubernetes as k8s +import k8s.apps.v1.inputs.* +import k8s.apps.v1.{Deployment, DeploymentArgs} +import k8s.core.v1.inputs.* +import k8s.core.v1.{ConfigMapArgs, ServiceArgs, *} +import k8s.meta.v1.inputs.* +import besom.internal.{Context, Output} +import besom.util.NonEmptyString +import besom.aliases.NonEmptyString + +object Grafana: + val appName: NonEmptyString = "grafana" + val labels = Map("app" -> "grafana") + val port = 3000 + + def deploy(using Context)(namespace: Output[Namespace], k8sProvider: Output[k8s.Provider]) = { + val grafanaPV = PersistentVolume( + appName, + k8s.core.v1.PersistentVolumeArgs( + metadata = ObjectMetaArgs(name = s"$appName-pv", namespace = namespace.metadata.name), + spec = PersistentVolumeSpecArgs( + storageClassName = "standard", + capacity = Map("storage" -> "1Gi"), + accessModes = List("ReadWriteOnce"), + hostPath = HostPathVolumeSourceArgs("/data/grafana") + ) + ), + opts(provider = k8sProvider) + ) + val grafanaPVC = PersistentVolumeClaim( + appName, + k8s.core.v1.PersistentVolumeClaimArgs( + metadata = ObjectMetaArgs(name = s"$appName-pvc", namespace = namespace.metadata.name), + spec = PersistentVolumeClaimSpecArgs( + storageClassName = "standard", + volumeName = grafanaPV.metadata.name, + accessModes = List("ReadWriteOnce"), + resources = VolumeResourceRequirementsArgs( + requests = Map("storage" -> "1Gi") + ) + ) + ), + opts(provider = k8sProvider) + ) + + Deployment( + appName, + DeploymentArgs( + spec = DeploymentSpecArgs( + selector = LabelSelectorArgs(matchLabels = labels), + replicas = 1, + template = PodTemplateSpecArgs( + metadata = ObjectMetaArgs( + name = s"$appName-deployment", + labels = labels, + namespace = namespace.metadata.name + ), + spec = PodSpecArgs( + securityContext = PodSecurityContextArgs(runAsUser = 0, fsGroup = 0, runAsGroup = 0), + containers = List( + ContainerArgs( + name = appName, + image = "grafana/grafana:latest", + ports = List( + ContainerPortArgs(containerPort = port) + ), + readinessProbe = ProbeArgs( + failureThreshold = 3, + httpGet = HttpGetActionArgs( + path = "/robots.txt", + port = port, + scheme = "HTTP" + ), + initialDelaySeconds = 10, + periodSeconds = 30, + successThreshold = 1, + timeoutSeconds = 2 + ), + livenessProbe = ProbeArgs( + failureThreshold = 3, + tcpSocket = TcpSocketActionArgs(port = port), + initialDelaySeconds = 30, + periodSeconds = 10, + successThreshold = 1, + timeoutSeconds = 1 + ), + resources = ResourceRequirementsArgs( + requests = Map("cpu" -> "250m", "memory" -> "750Mi") + ), + volumeMounts = List( + VolumeMountArgs(mountPath = "/var/lib/grafana", name = s"$appName-pv") + ) + ) + ), + volumes = List( + VolumeArgs( + name = s"$appName-pv", + persistentVolumeClaim = PersistentVolumeClaimVolumeSourceArgs( + claimName = grafanaPVC.metadata.name.map(_.get) + ) + ) + ) + ) + ) + ), + metadata = ObjectMetaArgs( + name = s"$appName-deployment", + namespace = namespace.metadata.name + ) + ), + opts(provider = k8sProvider) + ) + } + + def deployService(using + Context + )(namespace: Output[Namespace], grafanaDeployment: Output[Deployment], k8sProvider: Output[k8s.Provider]) = Service( + appName, + ServiceArgs( + spec = ServiceSpecArgs( + selector = labels, + sessionAffinity = "None", + `type` = k8s.core.v1.enums.ServiceSpecType.ClusterIP, + ports = List( + ServicePortArgs(port = port, targetPort = port) + ) + ), + metadata = ObjectMetaArgs( + name = s"$appName-service", + namespace = namespace.metadata.name + ) + ), + opts(dependsOn = grafanaDeployment, provider = k8sProvider) + ) diff --git a/infra/k8s/src/main/scala/Loki.scala b/infra/k8s/src/main/scala/Loki.scala new file mode 100644 index 0000000..419667d --- /dev/null +++ b/infra/k8s/src/main/scala/Loki.scala @@ -0,0 +1,150 @@ +import besom.api.kubernetes.core.v1.Namespace +import besom.* +import besom.aliases.NonEmptyString +import besom.api.kubernetes as k8s +import k8s.apps.v1.inputs.* +import k8s.apps.v1.{Deployment, DeploymentArgs} +import k8s.core.v1.inputs.* +import k8s.core.v1.{ConfigMapArgs, ServiceArgs, *} +import k8s.meta.v1.inputs.* +import besom.internal.{Context, Output} +import besom.util.NonEmptyString +import besom.aliases.NonEmptyString + +object Loki: + val appName: NonEmptyString = "loki" + val labels = Map("app" -> "loki") + val port = 3100 + private val configFileName = "loki.yaml" + + def deploy(using Context)(namespace: Output[Namespace], k8sProvider: Output[k8s.Provider]) = + val configMap = ConfigMap( + s"$appName-config", + ConfigMapArgs( + metadata = ObjectMetaArgs(name = s"$appName-config", namespace = namespace.metadata.name), + data = Map( + configFileName -> + s"""auth_enabled: false + | + |server: + | http_listen_port: $port + | + |ingester: + | wal: + | dir: /tmp/wal + | lifecycler: + | address: 127.0.0.1 + | ring: + | kvstore: + | store: inmemory + | replication_factor: 1 + | chunk_idle_period: 15m + | chunk_retain_period: 30s + | + |schema_config: + | configs: + | - from: 2020-10-24 + | store: boltdb-shipper + | object_store: filesystem + | schema: v11 + | index: + | prefix: index_ + | period: 24h + | + |storage_config: + | boltdb_shipper: + | active_index_directory: /tmp/loki/index + | cache_location: /tmp/loki/cache + | cache_ttl: 24h + | shared_store: filesystem + | filesystem: + | directory: /tmp/loki/chunks + | + |compactor: + | working_directory: /tmp/loki/compactor + | shared_store: filesystem + | + |limits_config: + | reject_old_samples: true + | reject_old_samples_max_age: 168h + | + |chunk_store_config: + | max_look_back_period: 0s + | + |table_manager: + | retention_deletes_enabled: false + | retention_period: 0s + """.stripMargin + ) + ), + opts(provider = k8sProvider) + ) + + Deployment( + appName, + DeploymentArgs( + spec = DeploymentSpecArgs( + selector = LabelSelectorArgs(matchLabels = labels), + replicas = 1, + template = PodTemplateSpecArgs( + metadata = ObjectMetaArgs( + name = s"$appName-deployment", + labels = labels, + namespace = namespace.metadata.name + ), + spec = PodSpecArgs( + containers = List( + ContainerArgs( + name = appName, + image = "grafana/loki:latest", + ports = List( + ContainerPortArgs(containerPort = port) + ), + args = List(s"-config.file=/etc/loki/$configFileName"), + volumeMounts = List( + VolumeMountArgs( + mountPath = "/etc/loki", + readOnly = true, + name = s"$appName-config-volume" + ) + ) + ) + ), + volumes = List( + VolumeArgs( + name = s"$appName-config-volume", + configMap = ConfigMapVolumeSourceArgs( + name = configMap.metadata.name + ) + ) + ) + ) + ) + ), + metadata = ObjectMetaArgs( + name = s"$appName-deployment", + namespace = namespace.metadata.name + ) + ), + opts(provider = k8sProvider) + ) + + def deployService(using + Context + )(namespace: Output[Namespace], lokiDeployment: Output[Deployment], k8sProvider: Output[k8s.Provider]) = Service( + appName, + ServiceArgs( + spec = ServiceSpecArgs( + selector = labels, + `type` = k8s.core.v1.enums.ServiceSpecType.ClusterIP, + ports = List( + ServicePortArgs(port = port, targetPort = port) + ) + ), + metadata = ObjectMetaArgs( + name = s"$appName-service", + namespace = namespace.metadata.name + ) + ), + opts(dependsOn = lokiDeployment, provider = k8sProvider) + ) diff --git a/infra/k8s/src/main/scala/Main.scala b/infra/k8s/src/main/scala/Main.scala index 884c22c..655fdc7 100644 --- a/infra/k8s/src/main/scala/Main.scala +++ b/infra/k8s/src/main/scala/Main.scala @@ -39,6 +39,17 @@ import besom.json.DefaultJsonProtocol.StringJsonFormat val appNamespace = Namespace(name = "vss", opts = opts(provider = k8sProvider)) + // loki + val lokiDeployment = Loki.deploy(appNamespace, k8sProvider) + val lokiService = Loki.deployService(appNamespace, lokiDeployment, k8sProvider) + + // promtail + val promtailDaemonSet = Promtail.deploy(lokiService, appNamespace, k8sProvider) + + // grafana + val grafanaDeployment = Grafana.deploy(appNamespace, k8sProvider) + val grafanaService = Grafana.deployService(appNamespace, grafanaDeployment, k8sProvider) + // zookeeper val zooDeployment = Zookeeper.deploy(appNamespace, k8sProvider) val zooService = Zookeeper.deployService(appNamespace, zooDeployment, k8sProvider) @@ -70,6 +81,11 @@ import besom.json.DefaultJsonProtocol.StringJsonFormat Stack.exports( serviceUrl = vssServiceUrl, namespaceName = appNamespace.metadata.name, + lokiDeploymentName = lokiDeployment.metadata.name, + lokiServiceName = lokiService.metadata.name, + grafanaDeploymentName = grafanaDeployment.metadata.name, + grafanaServiceName = grafanaService.metadata.name, + promtailDaemonSetName = promtailDaemonSet.metadata.name, zookeeperDeploymentName = zooDeployment.metadata.name, zookeeperServiceName = zooService.metadata.name, kafkaDeploymentName = kafkaDeployment.metadata.name, diff --git a/infra/k8s/src/main/scala/Promtail.scala b/infra/k8s/src/main/scala/Promtail.scala new file mode 100644 index 0000000..c51dc26 --- /dev/null +++ b/infra/k8s/src/main/scala/Promtail.scala @@ -0,0 +1,204 @@ +import besom.api.kubernetes.core.v1.Namespace +import besom.* +import besom.aliases.NonEmptyString +import besom.api.kubernetes as k8s +import k8s.apps.v1.inputs.* +import k8s.apps.v1.{DaemonSet, DaemonSetArgs, Deployment, DeploymentArgs} +import k8s.core.v1.inputs.* +import k8s.core.v1.{ConfigMapArgs, ServiceAccountArgs, ServiceArgs, *} +import k8s.meta.v1.inputs.* +import besom.internal.{Context, Output} +import besom.util.NonEmptyString +import besom.aliases.NonEmptyString +import besom.api.kubernetes.rbac.v1.inputs.{PolicyRuleArgs, RoleRefArgs, SubjectArgs} +import besom.api.kubernetes.rbac.v1.{ClusterRole, ClusterRoleArgs, ClusterRoleBinding, ClusterRoleBindingArgs} +import besom.api.kubernetes.scheduling.v1.{PriorityClass, PriorityClassArgs} + +object Promtail: + val appName: NonEmptyString = "promtail" + val labels = Map("app" -> "promtail") + val port = 9080 + private val configFileName = "promtail.yaml" + + def deploy(using + Context + )(lokiService: Output[Service], namespace: Output[Namespace], k8sProvider: Output[k8s.Provider]) = + val configMap = ConfigMap( + s"$appName-config", + ConfigMapArgs( + metadata = ObjectMetaArgs(name = s"$appName-config", namespace = namespace.metadata.name), + data = Map( + configFileName -> + p"""server: + | http_listen_port: $port + | grpc_listen_port: 0 + |clients: + |- url: http://${lokiService.metadata.name.map(_.get)}:3100/loki/api/v1/push + |positions: + | filename: /tmp/positions.yaml + |target_config: + | sync_period: 10s + |scrape_configs: + |- job_name: pod-logs + | kubernetes_sd_configs: + | - role: pod + | pipeline_stages: + | - docker: {} + | relabel_configs: + | - source_labels: + | - __meta_kubernetes_pod_node_name + | target_label: __host__ + | - action: labelmap + | regex: __meta_kubernetes_pod_label_(.+) + | - action: replace + | replacement: $$1 + | separator: / + | source_labels: + | - __meta_kubernetes_namespace + | - __meta_kubernetes_pod_name + | target_label: job + | - action: replace + | source_labels: + | - __meta_kubernetes_namespace + | target_label: namespace + | - action: replace + | source_labels: + | - __meta_kubernetes_pod_name + | target_label: pod + | - action: replace + | source_labels: + | - __meta_kubernetes_pod_container_name + | target_label: container + | - replacement: /var/log/pods/*$$1/*.log + | separator: / + | source_labels: + | - __meta_kubernetes_pod_uid + | - __meta_kubernetes_pod_container_name + | target_label: __path__ + |""".stripMargin + ) + ), + opts(provider = k8sProvider) + ) + + val serviceAccount = ServiceAccount( + s"$appName-serviceaccount", + ServiceAccountArgs( + metadata = ObjectMetaArgs(name = s"$appName-serviceaccount", namespace = namespace.metadata.name) + ) + ) + + val clusterRole = ClusterRole( + s"$appName-clusterrole", + ClusterRoleArgs( + metadata = ObjectMetaArgs(name = s"$appName-clusterrole", namespace = namespace.metadata.name), + rules = List( + PolicyRuleArgs( + apiGroups = List(""), + resources = List("nodes", "services", "pods"), + verbs = List("get", "watch", "list") + ) + ) + ) + ) + + val clusterRoleBinding = ClusterRoleBinding( + s"$appName-clusterrolebinding", + ClusterRoleBindingArgs( + metadata = ObjectMetaArgs(name = s"$appName-clusterrolebinding", namespace = namespace.metadata.name), + subjects = List( + SubjectArgs( + kind = "ServiceAccount", + name = serviceAccount.metadata.name.map(_.get), + namespace = namespace.metadata.name + ) + ), + roleRef = RoleRefArgs( + kind = "ClusterRole", + name = clusterRole.metadata.name.map(_.get), + apiGroup = "rbac.authorization.k8s.io" + ) + ), + opts = opts(dependsOn = clusterRole, serviceAccount) + ) + + val priorityClass = PriorityClass( + s"$appName-priorityclass", + PriorityClassArgs( + metadata = ObjectMetaArgs(name = s"$appName-priorityclass", namespace = namespace.metadata.name), + value = 1000000, + globalDefault = false, + description = "This priority class should be used for log service pods only." + ) + ) + + DaemonSet( + s"$appName-daemonset", + DaemonSetArgs( + metadata = ObjectMetaArgs(name = s"$appName-daemonset", namespace = namespace.metadata.name), + spec = DaemonSetSpecArgs( + selector = LabelSelectorArgs(matchLabels = labels), + template = PodTemplateSpecArgs( + metadata = ObjectMetaArgs( + name = s"$appName-deployment", + labels = labels, + namespace = namespace.metadata.name + ), + spec = PodSpecArgs( + priorityClassName = priorityClass.metadata.name, + serviceAccount = serviceAccount.metadata.name, + containers = List( + ContainerArgs( + name = appName, + image = "grafana/promtail:latest", + ports = List( + ContainerPortArgs(containerPort = port) + ), + args = List(s"-config.file=/etc/promtail/$configFileName"), + env = List( + EnvVarArgs( + name = "HOSTNAME", + valueFrom = EnvVarSourceArgs( + fieldRef = ObjectFieldSelectorArgs(fieldPath = "spec.nodeName") + ) + ) + ), + volumeMounts = List( + VolumeMountArgs( + mountPath = "/var/log", + name = "logs" + ), + VolumeMountArgs( + mountPath = "/etc/promtail", + name = s"$appName-config" + ), + VolumeMountArgs( + mountPath = "/var/lib/docker/containers", + readOnly = true, + name = "varlibdockercontainers" + ) + ) + ) + ), + volumes = List( + VolumeArgs( + name = "logs", + hostPath = HostPathVolumeSourceArgs(path = "/var/log") + ), + VolumeArgs( + name = "varlibdockercontainers", + hostPath = HostPathVolumeSourceArgs(path = "/var/lib/docker/containers") + ), + VolumeArgs( + name = s"$appName-config", + configMap = ConfigMapVolumeSourceArgs( + name = configMap.metadata.name + ) + ) + ) + ) + ) + ) + ), + opts = opts(dependsOn = clusterRoleBinding) + ) From 4e798c30fc8960f41e0343a0df2f7a4ef777b790 Mon Sep 17 00:00:00 2001 From: kpoliwka Date: Wed, 20 Mar 2024 10:16:38 +0100 Subject: [PATCH 04/17] small fixes --- infra/k8s/src/main/scala/Grafana.scala | 13 ++++++--- infra/k8s/src/main/scala/Loki.scala | 1 - infra/k8s/src/main/scala/Main.scala | 13 +++++++-- infra/k8s/src/main/scala/Promtail.scala | 37 ++++++++----------------- 4 files changed, 32 insertions(+), 32 deletions(-) diff --git a/infra/k8s/src/main/scala/Grafana.scala b/infra/k8s/src/main/scala/Grafana.scala index 83b206a..2d7a149 100644 --- a/infra/k8s/src/main/scala/Grafana.scala +++ b/infra/k8s/src/main/scala/Grafana.scala @@ -1,4 +1,3 @@ -import besom.api.kubernetes.core.v1.Namespace import besom.* import besom.aliases.NonEmptyString import besom.api.kubernetes as k8s @@ -6,6 +5,7 @@ import k8s.apps.v1.inputs.* import k8s.apps.v1.{Deployment, DeploymentArgs} import k8s.core.v1.inputs.* import k8s.core.v1.{ConfigMapArgs, ServiceArgs, *} +import k8s.core.v1.enums.ServiceSpecType import k8s.meta.v1.inputs.* import besom.internal.{Context, Output} import besom.util.NonEmptyString @@ -59,7 +59,7 @@ object Grafana: namespace = namespace.metadata.name ), spec = PodSpecArgs( - securityContext = PodSecurityContextArgs(runAsUser = 0, fsGroup = 0, runAsGroup = 0), + securityContext = PodSecurityContextArgs(runAsUser = 0, fsGroup = 0), containers = List( ContainerArgs( name = appName, @@ -117,13 +117,18 @@ object Grafana: def deployService(using Context - )(namespace: Output[Namespace], grafanaDeployment: Output[Deployment], k8sProvider: Output[k8s.Provider]) = Service( + )( + serviceType: Output[ServiceSpecType], + namespace: Output[Namespace], + grafanaDeployment: Output[Deployment], + k8sProvider: Output[k8s.Provider] + ) = Service( appName, ServiceArgs( spec = ServiceSpecArgs( selector = labels, sessionAffinity = "None", - `type` = k8s.core.v1.enums.ServiceSpecType.ClusterIP, + `type` = serviceType, ports = List( ServicePortArgs(port = port, targetPort = port) ) diff --git a/infra/k8s/src/main/scala/Loki.scala b/infra/k8s/src/main/scala/Loki.scala index 419667d..440cc9e 100644 --- a/infra/k8s/src/main/scala/Loki.scala +++ b/infra/k8s/src/main/scala/Loki.scala @@ -1,4 +1,3 @@ -import besom.api.kubernetes.core.v1.Namespace import besom.* import besom.aliases.NonEmptyString import besom.api.kubernetes as k8s diff --git a/infra/k8s/src/main/scala/Main.scala b/infra/k8s/src/main/scala/Main.scala index 655fdc7..fe15a1f 100644 --- a/infra/k8s/src/main/scala/Main.scala +++ b/infra/k8s/src/main/scala/Main.scala @@ -48,7 +48,7 @@ import besom.json.DefaultJsonProtocol.StringJsonFormat // grafana val grafanaDeployment = Grafana.deploy(appNamespace, k8sProvider) - val grafanaService = Grafana.deployService(appNamespace, grafanaDeployment, k8sProvider) + val grafanaService = Grafana.deployService(serviceType, appNamespace, grafanaDeployment, k8sProvider) // zookeeper val zooDeployment = Zookeeper.deploy(appNamespace, k8sProvider) @@ -70,6 +70,14 @@ import besom.json.DefaultJsonProtocol.StringJsonFormat val vssDeployment = VSS.deploy(config, appNamespace, postgresService, kafkaService, jaegerService, k8sProvider) val vssService = VSS.deployService(serviceType, appNamespace, vssDeployment, k8sProvider) + val grafanaServiceUrl = + grafanaService.status.loadBalancer.ingress + .map( + _.flatMap(_.headOption.flatMap(_.hostname)) + .map(host => p"http://$host:${Grafana.port}") + .getOrElse("Host not find. Probably vss:cluster is set to local") + ) + val vssServiceUrl = vssService.status.loadBalancer.ingress .map( @@ -79,7 +87,8 @@ import besom.json.DefaultJsonProtocol.StringJsonFormat ) Stack.exports( - serviceUrl = vssServiceUrl, + grafanaServiceUrl = grafanaServiceUrl, + vssServiceUrl = vssServiceUrl, namespaceName = appNamespace.metadata.name, lokiDeploymentName = lokiDeployment.metadata.name, lokiServiceName = lokiService.metadata.name, diff --git a/infra/k8s/src/main/scala/Promtail.scala b/infra/k8s/src/main/scala/Promtail.scala index c51dc26..7c9c406 100644 --- a/infra/k8s/src/main/scala/Promtail.scala +++ b/infra/k8s/src/main/scala/Promtail.scala @@ -1,18 +1,16 @@ -import besom.api.kubernetes.core.v1.Namespace import besom.* import besom.aliases.NonEmptyString import besom.api.kubernetes as k8s import k8s.apps.v1.inputs.* import k8s.apps.v1.{DaemonSet, DaemonSetArgs, Deployment, DeploymentArgs} import k8s.core.v1.inputs.* -import k8s.core.v1.{ConfigMapArgs, ServiceAccountArgs, ServiceArgs, *} +import k8s.core.v1.{ConfigMapArgs, ServiceAccountArgs, *} import k8s.meta.v1.inputs.* +import k8s.rbac.v1.inputs.{PolicyRuleArgs, RoleRefArgs, SubjectArgs} +import k8s.rbac.v1.{ClusterRole, ClusterRoleArgs, ClusterRoleBinding, ClusterRoleBindingArgs} import besom.internal.{Context, Output} import besom.util.NonEmptyString import besom.aliases.NonEmptyString -import besom.api.kubernetes.rbac.v1.inputs.{PolicyRuleArgs, RoleRefArgs, SubjectArgs} -import besom.api.kubernetes.rbac.v1.{ClusterRole, ClusterRoleArgs, ClusterRoleBinding, ClusterRoleBindingArgs} -import besom.api.kubernetes.scheduling.v1.{PriorityClass, PriorityClassArgs} object Promtail: val appName: NonEmptyString = "promtail" @@ -82,16 +80,16 @@ object Promtail: ) val serviceAccount = ServiceAccount( - s"$appName-serviceaccount", + s"$appName-service-account", ServiceAccountArgs( - metadata = ObjectMetaArgs(name = s"$appName-serviceaccount", namespace = namespace.metadata.name) + metadata = ObjectMetaArgs(name = s"$appName-service-account", namespace = namespace.metadata.name) ) ) val clusterRole = ClusterRole( - s"$appName-clusterrole", + s"$appName-cluster-role", ClusterRoleArgs( - metadata = ObjectMetaArgs(name = s"$appName-clusterrole", namespace = namespace.metadata.name), + metadata = ObjectMetaArgs(name = s"$appName-cluster-role", namespace = namespace.metadata.name), rules = List( PolicyRuleArgs( apiGroups = List(""), @@ -103,9 +101,9 @@ object Promtail: ) val clusterRoleBinding = ClusterRoleBinding( - s"$appName-clusterrolebinding", + s"$appName-cluster-role-binding", ClusterRoleBindingArgs( - metadata = ObjectMetaArgs(name = s"$appName-clusterrolebinding", namespace = namespace.metadata.name), + metadata = ObjectMetaArgs(name = s"$appName-cluster-role-binding", namespace = namespace.metadata.name), subjects = List( SubjectArgs( kind = "ServiceAccount", @@ -119,23 +117,13 @@ object Promtail: apiGroup = "rbac.authorization.k8s.io" ) ), - opts = opts(dependsOn = clusterRole, serviceAccount) - ) - - val priorityClass = PriorityClass( - s"$appName-priorityclass", - PriorityClassArgs( - metadata = ObjectMetaArgs(name = s"$appName-priorityclass", namespace = namespace.metadata.name), - value = 1000000, - globalDefault = false, - description = "This priority class should be used for log service pods only." - ) + opts = opts(retainOnDelete = false, dependsOn = List(clusterRole, serviceAccount)) ) DaemonSet( - s"$appName-daemonset", + s"$appName-daemon-set", DaemonSetArgs( - metadata = ObjectMetaArgs(name = s"$appName-daemonset", namespace = namespace.metadata.name), + metadata = ObjectMetaArgs(name = s"$appName-daemon-set", namespace = namespace.metadata.name), spec = DaemonSetSpecArgs( selector = LabelSelectorArgs(matchLabels = labels), template = PodTemplateSpecArgs( @@ -145,7 +133,6 @@ object Promtail: namespace = namespace.metadata.name ), spec = PodSpecArgs( - priorityClassName = priorityClass.metadata.name, serviceAccount = serviceAccount.metadata.name, containers = List( ContainerArgs( From 10eb412c288158f53d8239af60b4ebfafa14b474 Mon Sep 17 00:00:00 2001 From: kpoliwka Date: Wed, 20 Mar 2024 18:06:29 +0100 Subject: [PATCH 05/17] small fixes --- infra/k8s/src/main/scala/Main.scala | 11 +++++++++-- infra/k8s/src/main/scala/Promtail.scala | 12 +++++++----- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/infra/k8s/src/main/scala/Main.scala b/infra/k8s/src/main/scala/Main.scala index fe15a1f..a027aca 100644 --- a/infra/k8s/src/main/scala/Main.scala +++ b/infra/k8s/src/main/scala/Main.scala @@ -1,6 +1,9 @@ import besom.* import besom.api.kubernetes as k8s -import k8s.core.v1.{Namespace, Service} +import besom.api.kubernetes.meta.v1.inputs.ObjectMetaArgs +import besom.api.kubernetes.rbac.v1.inputs.PolicyRuleArgs +import besom.api.kubernetes.rbac.v1.{ClusterRole, ClusterRoleArgs} +import k8s.core.v1.{Namespace, NamespaceArgs, Service, ServiceAccount, ServiceAccountArgs} import k8s.core.v1.enums.ServiceSpecType import besom.internal.{Config, Output} import besom.json.DefaultJsonProtocol.StringJsonFormat @@ -37,7 +40,11 @@ import besom.json.DefaultJsonProtocol.StringJsonFormat s"$str value not allowed. Available values are local or remote. Change vss:cluster configuration" ) - val appNamespace = Namespace(name = "vss", opts = opts(provider = k8sProvider)) + val appNamespace = Namespace( + name = "default", + NamespaceArgs(metadata = ObjectMetaArgs(name = "default")), + opts = opts(provider = k8sProvider, retainOnDelete = true) + ) // loki val lokiDeployment = Loki.deploy(appNamespace, k8sProvider) diff --git a/infra/k8s/src/main/scala/Promtail.scala b/infra/k8s/src/main/scala/Promtail.scala index 7c9c406..cde88e5 100644 --- a/infra/k8s/src/main/scala/Promtail.scala +++ b/infra/k8s/src/main/scala/Promtail.scala @@ -82,8 +82,9 @@ object Promtail: val serviceAccount = ServiceAccount( s"$appName-service-account", ServiceAccountArgs( - metadata = ObjectMetaArgs(name = s"$appName-service-account", namespace = namespace.metadata.name) - ) + metadata = ObjectMetaArgs(name = s"$appName-service-account" /*, namespace = namespace.metadata.name*/ ) + ), + opts(provider = k8sProvider) ) val clusterRole = ClusterRole( @@ -97,7 +98,8 @@ object Promtail: verbs = List("get", "watch", "list") ) ) - ) + ), + opts(provider = k8sProvider) ) val clusterRoleBinding = ClusterRoleBinding( @@ -117,7 +119,7 @@ object Promtail: apiGroup = "rbac.authorization.k8s.io" ) ), - opts = opts(retainOnDelete = false, dependsOn = List(clusterRole, serviceAccount)) + opts = opts(provider = k8sProvider, retainOnDelete = false, dependsOn = List(clusterRole, serviceAccount)) ) DaemonSet( @@ -187,5 +189,5 @@ object Promtail: ) ) ), - opts = opts(dependsOn = clusterRoleBinding) + opts = opts(provider = k8sProvider, dependsOn = clusterRoleBinding) ) From 2f6d84f81da4c73519671a772f2997150ee086fc Mon Sep 17 00:00:00 2001 From: kpoliwka Date: Thu, 21 Mar 2024 07:47:35 +0100 Subject: [PATCH 06/17] fix namespace --- infra/k8s/src/main/scala/Main.scala | 6 +++--- infra/k8s/src/main/scala/Promtail.scala | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/infra/k8s/src/main/scala/Main.scala b/infra/k8s/src/main/scala/Main.scala index a027aca..796679a 100644 --- a/infra/k8s/src/main/scala/Main.scala +++ b/infra/k8s/src/main/scala/Main.scala @@ -41,9 +41,9 @@ import besom.json.DefaultJsonProtocol.StringJsonFormat ) val appNamespace = Namespace( - name = "default", - NamespaceArgs(metadata = ObjectMetaArgs(name = "default")), - opts = opts(provider = k8sProvider, retainOnDelete = true) + name = "vss", + NamespaceArgs(metadata = ObjectMetaArgs(name = "vss")), + opts = opts(provider = k8sProvider) ) // loki diff --git a/infra/k8s/src/main/scala/Promtail.scala b/infra/k8s/src/main/scala/Promtail.scala index cde88e5..908ed9a 100644 --- a/infra/k8s/src/main/scala/Promtail.scala +++ b/infra/k8s/src/main/scala/Promtail.scala @@ -82,7 +82,7 @@ object Promtail: val serviceAccount = ServiceAccount( s"$appName-service-account", ServiceAccountArgs( - metadata = ObjectMetaArgs(name = s"$appName-service-account" /*, namespace = namespace.metadata.name*/ ) + metadata = ObjectMetaArgs(name = s"$appName-service-account", namespace = namespace.metadata.name) ), opts(provider = k8sProvider) ) From c1a9732d505513136e7fc6dad0d87aa34b08c897 Mon Sep 17 00:00:00 2001 From: kpoliwka Date: Thu, 21 Mar 2024 17:00:48 +0100 Subject: [PATCH 07/17] fix --- build.sbt | 13 +++++ infra/eks/Main.scala | 21 ++++++- infra/k8s/Pulumi.yaml | 5 +- infra/k8s/project.scala | 1 + infra/k8s/src/main/scala/Main.scala | 85 ++++++++++++++++++++++++++++- infra/k8s/src/main/scala/VSS.scala | 12 ++-- project/plugins.sbt | 2 +- 7 files changed, 123 insertions(+), 16 deletions(-) diff --git a/build.sbt b/build.sbt index b740f33..a8218e0 100644 --- a/build.sbt +++ b/build.sbt @@ -137,6 +137,19 @@ def setupCommonDockerImageConfig(project: Project): Project = project .settings( dockerRepository := Some("localhost:5001"), + dockerBuildCommand := { + if (sys.props("os.arch") != "amd64") { + // use buildx with platform to build supported amd64 images on other CPU architectures + // this may require that you have first run 'docker buildx create' to set docker buildx up + dockerExecCommand.value ++ Seq( + "buildx", + "build", + "--platform=linux/amd64", + //"--platform=linux/arm64/v8", + "--load" + ) ++ dockerBuildOptions.value :+ "." + } else dockerBuildCommand.value + }, dockerBaseImage := "eclipse-temurin:11.0.16.1_1-jdk-focal", Docker / aggregate := false, Compile / packageDoc / publishArtifact := false diff --git a/infra/eks/Main.scala b/infra/eks/Main.scala index 891cec7..7f08427 100644 --- a/infra/eks/Main.scala +++ b/infra/eks/Main.scala @@ -1,5 +1,5 @@ import besom.* -import besom.api.{awsx, eks} +import besom.api.{aws, awsx, eks} @main def main = Pulumi.run { val appName = "vss" @@ -24,5 +24,22 @@ import besom.api.{awsx, eks} ) ) - Stack(cluster).exports(kubeconfig = cluster.kubeconfigJson) + val repo = aws.ecr.Repository( + s"$appName-repository", + aws.ecr.RepositoryArgs( + imageTagMutability = "MUTABLE" + ) + ) + + val authorizationToken = + aws.ecr.getAuthorizationToken(aws.ecr.GetAuthorizationTokenArgs(registryId = repo.registryId)) + + Stack(cluster).exports( + registryEndpoint = authorizationToken.proxyEndpoint, + repositoryUrl = repo.repositoryUrl, + accessKeyId = authorizationToken.map(_.userName).asSecret, + secretAccessKey = authorizationToken.map(_.password).asSecret, + kubeconfig = cluster.kubeconfigJson, + organization = pulumiOrganization + ) } diff --git a/infra/k8s/Pulumi.yaml b/infra/k8s/Pulumi.yaml index a340182..e7ef7a9 100644 --- a/infra/k8s/Pulumi.yaml +++ b/infra/k8s/Pulumi.yaml @@ -2,10 +2,9 @@ name: vss description: VSS infra runtime: scala config: - vss:localRegistry: "localhost:5001" - vss:imageName: "vss-cats" + vss:localRepository: "localhost:5001/vss-zio" vss:imageTag: "0.1.0-SNAPSHOT" - vss:cluster: "local" + vss:cluster: "remote" vss:cluster-org: "organization" vss:cluster-project: "vss-aws-eks" vss:cluster-stack: "dev" \ No newline at end of file diff --git a/infra/k8s/project.scala b/infra/k8s/project.scala index fa8f045..d444f87 100644 --- a/infra/k8s/project.scala +++ b/infra/k8s/project.scala @@ -1,3 +1,4 @@ //> using scala 3.3.3 //> using dep org.virtuslab::besom-core:0.2.2 //> using dep org.virtuslab::besom-kubernetes::4.8.0-core.0.2 +//> using dep "org.virtuslab::besom-docker:4.5.1-core.0.2" diff --git a/infra/k8s/src/main/scala/Main.scala b/infra/k8s/src/main/scala/Main.scala index 796679a..770bf5f 100644 --- a/infra/k8s/src/main/scala/Main.scala +++ b/infra/k8s/src/main/scala/Main.scala @@ -1,5 +1,6 @@ +import VSS.appName import besom.* -import besom.api.kubernetes as k8s +import besom.api.{docker, kubernetes as k8s} import besom.api.kubernetes.meta.v1.inputs.ObjectMetaArgs import besom.api.kubernetes.rbac.v1.inputs.PolicyRuleArgs import besom.api.kubernetes.rbac.v1.{ClusterRole, ClusterRoleArgs} @@ -46,6 +47,81 @@ import besom.json.DefaultJsonProtocol.StringJsonFormat opts = opts(provider = k8sProvider) ) + val imageTag = config.requireString("imageTag") + val localRepository = config.requireString("localRepository") + + val out = config + .requireString("cluster") + .flatMap: + case "local" => + Output(None, p"$localRepository:$imageTag") + case "remote" => + for + k8sOrgName <- config.getString("cluster-org").getOrElse("organization") + k8sProjName <- config.requireString("cluster-project") + k8sStackName <- config.requireString("cluster-stack") + stack <- StackReference(name = s"$k8sOrgName/$k8sProjName/$k8sStackName") + registryEndpoint <- stack.requireOutput("registryEndpoint").map(_.convertTo[String]) + repositoryUrl <- stack.requireOutput("repositoryUrl").map(_.convertTo[String]) + secretAccessKey <- stack.requireOutput("secretAccessKey").map(_.convertTo[String]) + accessKeyId <- stack.requireOutput("accessKeyId").map(_.convertTo[String]) + dockerProvider <- docker.Provider( + s"$appName-docker-provider", + docker.ProviderArgs( + registryAuth = List( + docker.inputs.ProviderRegistryAuthArgs( + address = registryEndpoint, + username = accessKeyId, + password = secretAccessKey + ) + ) + ) + ) + tag <- docker.Tag( + s"$appName-tag", + docker.TagArgs( + sourceImage = p"$localRepository:$imageTag", + targetImage = p"$repositoryUrl:$imageTag" + ), + opts = opts(provider = dockerProvider) + ) + image <- docker.RegistryImage( + s"$appName-image", + docker.RegistryImageArgs(name = tag.targetImage), + opts = opts(provider = dockerProvider) + ) + k8sRegistry <- k8s.core.v1.Secret( + s"$appName-registry-secret", + k8s.core.v1.SecretArgs( + metadata = ObjectMetaArgs( + name = s"$appName-registry-secret", + namespace = appNamespace.metadata.name + ), + `type` = "kubernetes.io/dockerconfigjson", + data = Map( + ".dockerconfigjson" -> + p"""{ + | "auths": { + | "$repositoryUrl": { + | "username": "$accessKeyId", + | "password": "$secretAccessKey", + | "auth": "${p"$accessKeyId:$secretAccessKey".map(base64)}" + | } + | } + |}""".stripMargin.map(base64) + ) + ), + opts = opts(provider = k8sProvider, dependsOn = image) + ) + yield (Some(k8sRegistry), tag.targetImage) + case str => + throw Exception( + s"$str value not allowed. Available values are local or remote. Change vss:cluster configuration" + ) + + val k8sRegistrySecret = out.map(_._1) + val image = out.flatMap(_._2) + // loki val lokiDeployment = Loki.deploy(appNamespace, k8sProvider) val lokiService = Loki.deployService(appNamespace, lokiDeployment, k8sProvider) @@ -74,8 +150,9 @@ import besom.json.DefaultJsonProtocol.StringJsonFormat val jaegerService = Jaeger.deployService(appNamespace, jaegerDeployment, k8sProvider) // vss - val vssDeployment = VSS.deploy(config, appNamespace, postgresService, kafkaService, jaegerService, k8sProvider) - val vssService = VSS.deployService(serviceType, appNamespace, vssDeployment, k8sProvider) + val vssDeployment = + VSS.deploy(k8sRegistrySecret, image, appNamespace, postgresService, kafkaService, jaegerService, k8sProvider) + val vssService = VSS.deployService(serviceType, appNamespace, vssDeployment, k8sProvider) val grafanaServiceUrl = grafanaService.status.loadBalancer.ingress @@ -114,3 +191,5 @@ import besom.json.DefaultJsonProtocol.StringJsonFormat vssServiceName = vssService.metadata.name ) } + +private def base64: String => String = v => java.util.Base64.getEncoder.encodeToString(v.getBytes) diff --git a/infra/k8s/src/main/scala/VSS.scala b/infra/k8s/src/main/scala/VSS.scala index 4cf8418..56a4b9c 100644 --- a/infra/k8s/src/main/scala/VSS.scala +++ b/infra/k8s/src/main/scala/VSS.scala @@ -21,18 +21,14 @@ object VSS { ) def deploy(using Context)( - config: Config, + k8sRegistrySecret: Output[Option[k8s.core.v1.Secret]], + image: Output[String], namespace: Output[Namespace], postgresService: Output[Service], kafkaService: Output[Service], jaegerService: Output[Service], k8sProvider: Output[k8s.Provider] ) = { - val localRegistry = config.requireString("localRegistry") - val imageName = config.requireString("imageName") - val imageTag = config.requireString("imageTag") - val image = pulumi"$localRegistry/$imageName:$imageTag" - Deployment( appName, DeploymentArgs( @@ -46,11 +42,13 @@ object VSS { namespace = namespace.metadata.name ), spec = PodSpecArgs( + imagePullSecrets = + k8sRegistrySecret.map(_.map(secret => LocalObjectReferenceArgs(name = secret.metadata.name)).toList), containers = List( ContainerArgs( name = appName, image = image, - imagePullPolicy = "IfNotPresent", + imagePullPolicy = "Always", ports = ports.map { case (name, (protocol, port)) => ContainerPortArgs(containerPort = port, protocol) }.toList, diff --git a/project/plugins.sbt b/project/plugins.sbt index 22ae078..20637c1 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,4 +1,4 @@ addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.4.6") libraryDependencies ++= Seq("com.thesamet.scalapb.zio-grpc" %% "zio-grpc-codegen" % "0.6.0-rc4") addSbtPlugin("org.typelevel" % "sbt-fs2-grpc" % "2.5.11") -addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % "1.8.0") +addSbtPlugin("com.github.sbt" %% "sbt-native-packager" % "1.9.16") From 25eb5822ded28f7facc69df1d3a49e12f2ec74f0 Mon Sep 17 00:00:00 2001 From: kpoliwka Date: Mon, 25 Mar 2024 10:55:10 +0100 Subject: [PATCH 08/17] refactor --- infra/k8s/src/main/scala/Cluster.scala | 11 ++ infra/k8s/src/main/scala/Main.scala | 205 +++++++++++++------------ 2 files changed, 121 insertions(+), 95 deletions(-) create mode 100644 infra/k8s/src/main/scala/Cluster.scala diff --git a/infra/k8s/src/main/scala/Cluster.scala b/infra/k8s/src/main/scala/Cluster.scala new file mode 100644 index 0000000..40cac46 --- /dev/null +++ b/infra/k8s/src/main/scala/Cluster.scala @@ -0,0 +1,11 @@ +enum Cluster(val name: String): + case Local extends Cluster("local") + case Remote extends Cluster("remote") + +object Cluster: + val parseName = Cluster.values + .map(c => c.name -> c) + .toMap + .withDefault(str => + throw Exception(s"$str value not allowed. Available values are local or remote. Change vss:cluster configuration") + ) diff --git a/infra/k8s/src/main/scala/Main.scala b/infra/k8s/src/main/scala/Main.scala index 770bf5f..710cd04 100644 --- a/infra/k8s/src/main/scala/Main.scala +++ b/infra/k8s/src/main/scala/Main.scala @@ -1,4 +1,3 @@ -import VSS.appName import besom.* import besom.api.{docker, kubernetes as k8s} import besom.api.kubernetes.meta.v1.inputs.ObjectMetaArgs @@ -8,116 +7,107 @@ import k8s.core.v1.{Namespace, NamespaceArgs, Service, ServiceAccount, ServiceAc import k8s.core.v1.enums.ServiceSpecType import besom.internal.{Config, Output} import besom.json.DefaultJsonProtocol.StringJsonFormat +import besom.json.{JsField, JsObject, JsString, JsValue, JsonReader} @main def main = Pulumi.run { - - val serviceType = config - .requireString("cluster") - .map: - case "remote" => - ServiceSpecType.LoadBalancer - case _ => - ServiceSpecType.ClusterIP - - val k8sProvider = config + val appName: NonEmptyString = "vss" + val clusterConfig = config .requireString("cluster") - .flatMap: - case "local" => - k8s.Provider(name = "vss-local-provider") - case "remote" => - for - k8sOrgName <- config.getString("cluster-org").getOrElse("organization") - k8sProjName <- config.requireString("cluster-project") - k8sStackName <- config.requireString("cluster-stack") - stack <- StackReference(name = s"$k8sOrgName/$k8sProjName/$k8sStackName") - kubeconfigJson <- stack.requireOutput("kubeconfig") - provider <- k8s.Provider( - name = "vss-remote-provider", - k8s.ProviderArgs(kubeconfig = kubeconfigJson.convertTo[String]) - ) - yield provider - case str => - throw Exception( - s"$str value not allowed. Available values are local or remote. Change vss:cluster configuration" - ) + .map(Cluster.parseName) + + val serviceType = clusterConfig.map: + case Cluster.Remote => + ServiceSpecType.LoadBalancer + case _ => + ServiceSpecType.ClusterIP + + val clusterStack = + for + orgName <- config.getString("cluster-org").getOrElse("organization") + projName <- config.requireString("cluster-project") + stackName <- config.requireString("cluster-stack") + stack <- StackReference(name = s"$orgName/$projName/$stackName") + yield stack + + val k8sProvider = clusterConfig.flatMap: + case Cluster.Local => + k8s.Provider(name = s"$appName-local-provider") + case Cluster.Remote => + k8s.Provider( + name = s"$appName-remote-provider", + k8s.ProviderArgs(kubeconfig = clusterStack.requireOutput("kubeconfig").convertTo[String]) + ) val appNamespace = Namespace( - name = "vss", - NamespaceArgs(metadata = ObjectMetaArgs(name = "vss")), + name = appName, + NamespaceArgs(metadata = ObjectMetaArgs(name = appName)), opts = opts(provider = k8sProvider) ) val imageTag = config.requireString("imageTag") val localRepository = config.requireString("localRepository") - val out = config - .requireString("cluster") - .flatMap: - case "local" => - Output(None, p"$localRepository:$imageTag") - case "remote" => - for - k8sOrgName <- config.getString("cluster-org").getOrElse("organization") - k8sProjName <- config.requireString("cluster-project") - k8sStackName <- config.requireString("cluster-stack") - stack <- StackReference(name = s"$k8sOrgName/$k8sProjName/$k8sStackName") - registryEndpoint <- stack.requireOutput("registryEndpoint").map(_.convertTo[String]) - repositoryUrl <- stack.requireOutput("repositoryUrl").map(_.convertTo[String]) - secretAccessKey <- stack.requireOutput("secretAccessKey").map(_.convertTo[String]) - accessKeyId <- stack.requireOutput("accessKeyId").map(_.convertTo[String]) - dockerProvider <- docker.Provider( - s"$appName-docker-provider", - docker.ProviderArgs( - registryAuth = List( - docker.inputs.ProviderRegistryAuthArgs( - address = registryEndpoint, - username = accessKeyId, - password = secretAccessKey - ) + val registryEndpoint = clusterStack.requireOutput("registryEndpoint").convertTo[String] + val repositoryUrl = clusterStack.requireOutput("repositoryUrl").convertTo[String] + val secretAccessKeyJsValue = clusterStack.requireOutput("secretAccessKey") + val accessKeyIdJsValue = clusterStack.requireOutput("accessKeyId") + val secretAccessKey = secretAccessKeyJsValue.convertTo[String] + val accessKeyId = accessKeyIdJsValue.convertTo[String] + + val out = clusterConfig.flatMap: + case Cluster.Local => + Output(None, p"$localRepository:$imageTag") + case Cluster.Remote => + for + dockerProvider <- docker.Provider( + name = s"$appName-docker-provider", + docker.ProviderArgs( + registryAuth = List( + docker.inputs.ProviderRegistryAuthArgs( + address = registryEndpoint, + username = accessKeyId, + password = secretAccessKey ) ) ) - tag <- docker.Tag( - s"$appName-tag", - docker.TagArgs( - sourceImage = p"$localRepository:$imageTag", - targetImage = p"$repositoryUrl:$imageTag" - ), - opts = opts(provider = dockerProvider) - ) - image <- docker.RegistryImage( - s"$appName-image", - docker.RegistryImageArgs(name = tag.targetImage), - opts = opts(provider = dockerProvider) - ) - k8sRegistry <- k8s.core.v1.Secret( - s"$appName-registry-secret", - k8s.core.v1.SecretArgs( - metadata = ObjectMetaArgs( - name = s"$appName-registry-secret", - namespace = appNamespace.metadata.name - ), - `type` = "kubernetes.io/dockerconfigjson", - data = Map( - ".dockerconfigjson" -> - p"""{ - | "auths": { - | "$repositoryUrl": { - | "username": "$accessKeyId", - | "password": "$secretAccessKey", - | "auth": "${p"$accessKeyId:$secretAccessKey".map(base64)}" - | } - | } - |}""".stripMargin.map(base64) - ) + ) + tag <- docker.Tag( + s"$appName-tag", + docker.TagArgs( + sourceImage = p"$localRepository:$imageTag", + targetImage = p"$repositoryUrl:$imageTag" + ), + opts = opts(provider = dockerProvider) + ) + image <- docker.RegistryImage( + s"$appName-image", + docker.RegistryImageArgs(name = tag.targetImage), + opts = opts(provider = dockerProvider) + ) + k8sRegistry <- k8s.core.v1.Secret( + s"$appName-registry-secret", + k8s.core.v1.SecretArgs( + metadata = ObjectMetaArgs( + name = s"$appName-registry-secret", + namespace = appNamespace.metadata.name ), - opts = opts(provider = k8sProvider, dependsOn = image) - ) - yield (Some(k8sRegistry), tag.targetImage) - case str => - throw Exception( - s"$str value not allowed. Available values are local or remote. Change vss:cluster configuration" + `type` = "kubernetes.io/dockerconfigjson", + stringData = Map( + ".dockerconfigjson" -> + jsObjectOutput( + "auths" -> jsObjectOutput( + repositoryUrl -> jsObjectOutput( + "username" -> accessKeyIdJsValue, + "password" -> secretAccessKeyJsValue, + "auth" -> p"$accessKeyId:$secretAccessKey".map(base64).map(JsString(_)) + ) + ) + ).prettyPrint + ) + ), + opts = opts(provider = k8sProvider, dependsOn = image) ) + yield (Some(k8sRegistry), tag.targetImage) val k8sRegistrySecret = out.map(_._1) val image = out.flatMap(_._2) @@ -193,3 +183,28 @@ import besom.json.DefaultJsonProtocol.StringJsonFormat } private def base64: String => String = v => java.util.Base64.getEncoder.encodeToString(v.getBytes) + +private def jsObjectOutput( + members: (String | Output[String], JsValue | Output[JsValue])* +)(using Context): Output[JsObject] = + Output + .sequence( + members.toSeq.map { + case (k: String, v: JsValue) => + Output(k, v) + case (k: String, ov: Output[JsValue]) => + ov.map(v => (k, v)) + case (ok: Output[String], v: JsValue) => + ok.map(k => (k, v)) + case (ok: Output[String], ov: Output[JsValue]) => + ok.zip(ov) + } + ) + .map(o => JsObject.apply(o*)) + +extension (o: Output[StackReference]) + def requireOutput(name: NonEmptyString)(using Context): Output[JsValue] = o.flatMap(_.requireOutput(name)) + +extension (o: Output[JsValue]) + def convertTo[T : JsonReader]: Output[T] = o.map(_.convertTo[T]) + def prettyPrint: Output[String] = o.map(_.prettyPrint) From 6d10030cf0d45ec92eebf42b32e1a66e425e53b4 Mon Sep 17 00:00:00 2001 From: kpoliwka Date: Mon, 25 Mar 2024 11:36:04 +0100 Subject: [PATCH 09/17] refactor --- infra/k8s/src/main/scala/Grafana.scala | 4 - infra/k8s/src/main/scala/Main.scala | 115 ++++++++++++------------ infra/k8s/src/main/scala/Promtail.scala | 4 - infra/k8s/src/main/scala/VSS.scala | 15 ++-- 4 files changed, 63 insertions(+), 75 deletions(-) diff --git a/infra/k8s/src/main/scala/Grafana.scala b/infra/k8s/src/main/scala/Grafana.scala index 2d7a149..b4c3f21 100644 --- a/infra/k8s/src/main/scala/Grafana.scala +++ b/infra/k8s/src/main/scala/Grafana.scala @@ -1,5 +1,4 @@ import besom.* -import besom.aliases.NonEmptyString import besom.api.kubernetes as k8s import k8s.apps.v1.inputs.* import k8s.apps.v1.{Deployment, DeploymentArgs} @@ -7,9 +6,6 @@ import k8s.core.v1.inputs.* import k8s.core.v1.{ConfigMapArgs, ServiceArgs, *} import k8s.core.v1.enums.ServiceSpecType import k8s.meta.v1.inputs.* -import besom.internal.{Context, Output} -import besom.util.NonEmptyString -import besom.aliases.NonEmptyString object Grafana: val appName: NonEmptyString = "grafana" diff --git a/infra/k8s/src/main/scala/Main.scala b/infra/k8s/src/main/scala/Main.scala index 710cd04..73bf016 100644 --- a/infra/k8s/src/main/scala/Main.scala +++ b/infra/k8s/src/main/scala/Main.scala @@ -1,13 +1,10 @@ import besom.* -import besom.api.{docker, kubernetes as k8s} +import besom.api.kubernetes.core.v1.enums.ServiceSpecType +import besom.api.kubernetes.core.v1.{Namespace, NamespaceArgs} import besom.api.kubernetes.meta.v1.inputs.ObjectMetaArgs -import besom.api.kubernetes.rbac.v1.inputs.PolicyRuleArgs -import besom.api.kubernetes.rbac.v1.{ClusterRole, ClusterRoleArgs} -import k8s.core.v1.{Namespace, NamespaceArgs, Service, ServiceAccount, ServiceAccountArgs} -import k8s.core.v1.enums.ServiceSpecType -import besom.internal.{Config, Output} +import besom.api.{docker, kubernetes as k8s} +import besom.json.* import besom.json.DefaultJsonProtocol.StringJsonFormat -import besom.json.{JsField, JsObject, JsString, JsValue, JsonReader} @main def main = Pulumi.run { val appName: NonEmptyString = "vss" @@ -54,63 +51,65 @@ import besom.json.{JsField, JsObject, JsString, JsValue, JsonReader} val secretAccessKey = secretAccessKeyJsValue.convertTo[String] val accessKeyId = accessKeyIdJsValue.convertTo[String] - val out = clusterConfig.flatMap: + val appImage = clusterConfig.flatMap: case Cluster.Local => - Output(None, p"$localRepository:$imageTag") + p"$localRepository:$imageTag" case Cluster.Remote => - for - dockerProvider <- docker.Provider( - name = s"$appName-docker-provider", - docker.ProviderArgs( - registryAuth = List( - docker.inputs.ProviderRegistryAuthArgs( - address = registryEndpoint, - username = accessKeyId, - password = secretAccessKey - ) + val dockerProvider = docker.Provider( + name = s"$appName-docker-provider", + docker.ProviderArgs( + registryAuth = List( + docker.inputs.ProviderRegistryAuthArgs( + address = registryEndpoint, + username = accessKeyId, + password = secretAccessKey ) ) ) - tag <- docker.Tag( - s"$appName-tag", - docker.TagArgs( - sourceImage = p"$localRepository:$imageTag", - targetImage = p"$repositoryUrl:$imageTag" + ) + val tag = docker.Tag( + name = s"$appName-tag", + docker.TagArgs( + sourceImage = p"$localRepository:$imageTag", + targetImage = p"$repositoryUrl:$imageTag" + ), + opts = opts(provider = dockerProvider) + ) + val image = docker.RegistryImage( + name = s"$appName-image", + docker.RegistryImageArgs(name = tag.targetImage), + opts = opts(provider = dockerProvider) + ) + image.flatMap(_ => tag.targetImage) + + val appImagePullSecret = clusterConfig.flatMap: + case Cluster.Local => + Output(None) + case Cluster.Remote => + val secret = k8s.core.v1.Secret( + s"$appName-registry-secret", + k8s.core.v1.SecretArgs( + metadata = ObjectMetaArgs( + name = s"$appName-registry-secret", + namespace = appNamespace.metadata.name ), - opts = opts(provider = dockerProvider) - ) - image <- docker.RegistryImage( - s"$appName-image", - docker.RegistryImageArgs(name = tag.targetImage), - opts = opts(provider = dockerProvider) - ) - k8sRegistry <- k8s.core.v1.Secret( - s"$appName-registry-secret", - k8s.core.v1.SecretArgs( - metadata = ObjectMetaArgs( - name = s"$appName-registry-secret", - namespace = appNamespace.metadata.name - ), - `type` = "kubernetes.io/dockerconfigjson", - stringData = Map( - ".dockerconfigjson" -> - jsObjectOutput( - "auths" -> jsObjectOutput( - repositoryUrl -> jsObjectOutput( - "username" -> accessKeyIdJsValue, - "password" -> secretAccessKeyJsValue, - "auth" -> p"$accessKeyId:$secretAccessKey".map(base64).map(JsString(_)) - ) + `type` = "kubernetes.io/dockerconfigjson", + stringData = Map( + ".dockerconfigjson" -> + jsObjectOutput( + "auths" -> jsObjectOutput( + repositoryUrl -> jsObjectOutput( + "username" -> accessKeyIdJsValue, + "password" -> secretAccessKeyJsValue, + "auth" -> p"$accessKeyId:$secretAccessKey".map(base64).map(JsString(_)) ) - ).prettyPrint - ) - ), - opts = opts(provider = k8sProvider, dependsOn = image) - ) - yield (Some(k8sRegistry), tag.targetImage) - - val k8sRegistrySecret = out.map(_._1) - val image = out.flatMap(_._2) + ) + ).prettyPrint + ) + ), + opts = opts(provider = k8sProvider) + ) + secret.map(Some(_)) // loki val lokiDeployment = Loki.deploy(appNamespace, k8sProvider) @@ -141,7 +140,7 @@ import besom.json.{JsField, JsObject, JsString, JsValue, JsonReader} // vss val vssDeployment = - VSS.deploy(k8sRegistrySecret, image, appNamespace, postgresService, kafkaService, jaegerService, k8sProvider) + VSS.deploy(appImagePullSecret, appImage, appNamespace, postgresService, kafkaService, jaegerService, k8sProvider) val vssService = VSS.deployService(serviceType, appNamespace, vssDeployment, k8sProvider) val grafanaServiceUrl = diff --git a/infra/k8s/src/main/scala/Promtail.scala b/infra/k8s/src/main/scala/Promtail.scala index 908ed9a..6bb2032 100644 --- a/infra/k8s/src/main/scala/Promtail.scala +++ b/infra/k8s/src/main/scala/Promtail.scala @@ -1,5 +1,4 @@ import besom.* -import besom.aliases.NonEmptyString import besom.api.kubernetes as k8s import k8s.apps.v1.inputs.* import k8s.apps.v1.{DaemonSet, DaemonSetArgs, Deployment, DeploymentArgs} @@ -8,9 +7,6 @@ import k8s.core.v1.{ConfigMapArgs, ServiceAccountArgs, *} import k8s.meta.v1.inputs.* import k8s.rbac.v1.inputs.{PolicyRuleArgs, RoleRefArgs, SubjectArgs} import k8s.rbac.v1.{ClusterRole, ClusterRoleArgs, ClusterRoleBinding, ClusterRoleBindingArgs} -import besom.internal.{Context, Output} -import besom.util.NonEmptyString -import besom.aliases.NonEmptyString object Promtail: val appName: NonEmptyString = "promtail" diff --git a/infra/k8s/src/main/scala/VSS.scala b/infra/k8s/src/main/scala/VSS.scala index 56a4b9c..adc6534 100644 --- a/infra/k8s/src/main/scala/VSS.scala +++ b/infra/k8s/src/main/scala/VSS.scala @@ -1,14 +1,11 @@ import besom.* -import besom.util.* import besom.api.kubernetes as k8s -import k8s.core.v1.inputs.* -import k8s.core.v1.{ConfigMap, ConfigMapArgs, Namespace, Service, ServiceArgs} -import k8s.core.v1.enums.ServiceSpecType -import k8s.apps.v1.inputs.* -import k8s.apps.v1.{Deployment, DeploymentArgs} -import k8s.meta.v1.inputs.* -import besom.internal.{Context, Output} -import besom.internal.Config +import besom.api.kubernetes.apps.v1.inputs.* +import besom.api.kubernetes.apps.v1.{Deployment, DeploymentArgs} +import besom.api.kubernetes.core.v1.enums.ServiceSpecType +import besom.api.kubernetes.core.v1.inputs.* +import besom.api.kubernetes.core.v1.{Namespace, Service, ServiceArgs} +import besom.api.kubernetes.meta.v1.inputs.* object VSS { val appName: NonEmptyString = "vss-app" // todo fix inference in NonEmptyString From dbe78bbf65655000e8fca206333619b426459c5a Mon Sep 17 00:00:00 2001 From: kpoliwka Date: Tue, 26 Mar 2024 10:32:07 +0100 Subject: [PATCH 10/17] add besom-grafana to project --- infra/k8s/Pulumi.yaml | 3 +- infra/k8s/project.scala | 3 +- infra/k8s/src/main/scala/Grafana.scala | 78 ++++++++++++++++++++----- infra/k8s/src/main/scala/Main.scala | 26 ++++----- infra/k8s/src/main/scala/Promtail.scala | 4 +- 5 files changed, 79 insertions(+), 35 deletions(-) diff --git a/infra/k8s/Pulumi.yaml b/infra/k8s/Pulumi.yaml index e7ef7a9..e608923 100644 --- a/infra/k8s/Pulumi.yaml +++ b/infra/k8s/Pulumi.yaml @@ -7,4 +7,5 @@ config: vss:cluster: "remote" vss:cluster-org: "organization" vss:cluster-project: "vss-aws-eks" - vss:cluster-stack: "dev" \ No newline at end of file + vss:cluster-stack: "dev" + grafana:auth: "admin:admin" \ No newline at end of file diff --git a/infra/k8s/project.scala b/infra/k8s/project.scala index d444f87..a5d1f3c 100644 --- a/infra/k8s/project.scala +++ b/infra/k8s/project.scala @@ -1,4 +1,5 @@ //> using scala 3.3.3 //> using dep org.virtuslab::besom-core:0.2.2 //> using dep org.virtuslab::besom-kubernetes::4.8.0-core.0.2 -//> using dep "org.virtuslab::besom-docker:4.5.1-core.0.2" +//> using dep org.virtuslab::besom-docker:4.5.1-core.0.2 +//> using dep org.virtuslab::besom-grafana:0.2.0-core.0.2 diff --git a/infra/k8s/src/main/scala/Grafana.scala b/infra/k8s/src/main/scala/Grafana.scala index b4c3f21..9d5a606 100644 --- a/infra/k8s/src/main/scala/Grafana.scala +++ b/infra/k8s/src/main/scala/Grafana.scala @@ -1,5 +1,6 @@ import besom.* import besom.api.kubernetes as k8s +import besom.api.grafana import k8s.apps.v1.inputs.* import k8s.apps.v1.{Deployment, DeploymentArgs} import k8s.core.v1.inputs.* @@ -114,25 +115,72 @@ object Grafana: def deployService(using Context )( + lokiUrl: Output[String], + jaegerUrl: Output[String], serviceType: Output[ServiceSpecType], namespace: Output[Namespace], grafanaDeployment: Output[Deployment], k8sProvider: Output[k8s.Provider] - ) = Service( - appName, - ServiceArgs( - spec = ServiceSpecArgs( - selector = labels, - sessionAffinity = "None", - `type` = serviceType, - ports = List( - ServicePortArgs(port = port, targetPort = port) + ) = + val service = Service( + appName, + ServiceArgs( + spec = ServiceSpecArgs( + selector = labels, + sessionAffinity = "None", + `type` = serviceType, + ports = List( + ServicePortArgs(port = port, targetPort = port) + ) + ), + metadata = ObjectMetaArgs( + name = s"$appName-service", + namespace = namespace.metadata.name ) ), - metadata = ObjectMetaArgs( - name = s"$appName-service", - namespace = namespace.metadata.name + opts(dependsOn = grafanaDeployment, provider = k8sProvider) + ) + + val serviceUrl = + service.status.loadBalancer.ingress + .map( + _.flatMap(_.headOption.flatMap(_.hostname)) + .getOrElse(p"localhost") + ) + .flatMap(host => p"http://$host:$port") + + val grafanaProvider = grafana.Provider( + name = s"$appName-provider", + grafana.ProviderArgs( + retryWait = 20, // seconds + retries = 6, + auth = config.requireString("grafana:auth"), + url = serviceUrl ) - ), - opts(dependsOn = grafanaDeployment, provider = k8sProvider) - ) + ) + + val lokiDataSource = grafana.DataSource( + name = s"$appName-loki-data-source", + grafana.DataSourceArgs( + url = lokiUrl, + `type` = "loki", + basicAuthEnabled = false, + isDefault = true + ), + opts = opts(provider = grafanaProvider, dependsOn = service) + ) + + val jaegerDataSource = grafana.DataSource( + name = s"$appName-jaeger-data-source", + grafana.DataSourceArgs( + url = jaegerUrl, + `type` = "jaeger", + basicAuthEnabled = false + ), + opts = opts(provider = grafanaProvider, dependsOn = service) + ) + for + _ <- lokiDataSource + _ <- jaegerDataSource + yield serviceUrl + end deployService diff --git a/infra/k8s/src/main/scala/Main.scala b/infra/k8s/src/main/scala/Main.scala index 73bf016..c2d661a 100644 --- a/infra/k8s/src/main/scala/Main.scala +++ b/infra/k8s/src/main/scala/Main.scala @@ -114,13 +114,10 @@ import besom.json.DefaultJsonProtocol.StringJsonFormat // loki val lokiDeployment = Loki.deploy(appNamespace, k8sProvider) val lokiService = Loki.deployService(appNamespace, lokiDeployment, k8sProvider) + val lokiUrl = p"http://${lokiService.metadata.name.map(_.get)}:${Loki.port}" // promtail - val promtailDaemonSet = Promtail.deploy(lokiService, appNamespace, k8sProvider) - - // grafana - val grafanaDeployment = Grafana.deploy(appNamespace, k8sProvider) - val grafanaService = Grafana.deployService(serviceType, appNamespace, grafanaDeployment, k8sProvider) + val promtailDaemonSet = Promtail.deploy(lokiUrl, appNamespace, k8sProvider) // zookeeper val zooDeployment = Zookeeper.deploy(appNamespace, k8sProvider) @@ -137,27 +134,25 @@ import besom.json.DefaultJsonProtocol.StringJsonFormat // jaeger val jaegerDeployment = Jaeger.deploy(appNamespace, k8sProvider) val jaegerService = Jaeger.deployService(appNamespace, jaegerDeployment, k8sProvider) + val jaegerUrl = p"http://${jaegerService.metadata.name.map(_.get)}:${Jaeger.ports("frontend")._2}" + + // grafana + val grafanaDeployment = Grafana.deploy(appNamespace, k8sProvider) + val grafanaServiceUrl = + Grafana.deployService(lokiUrl, jaegerUrl, serviceType, appNamespace, grafanaDeployment, k8sProvider) // vss val vssDeployment = VSS.deploy(appImagePullSecret, appImage, appNamespace, postgresService, kafkaService, jaegerService, k8sProvider) val vssService = VSS.deployService(serviceType, appNamespace, vssDeployment, k8sProvider) - val grafanaServiceUrl = - grafanaService.status.loadBalancer.ingress - .map( - _.flatMap(_.headOption.flatMap(_.hostname)) - .map(host => p"http://$host:${Grafana.port}") - .getOrElse("Host not find. Probably vss:cluster is set to local") - ) - val vssServiceUrl = vssService.status.loadBalancer.ingress .map( _.flatMap(_.headOption.flatMap(_.hostname)) - .map(host => p"http://$host:${VSS.ports("main-http")._2}/docs") - .getOrElse("Host not find. Probably vss:cluster is set to local") + .getOrElse(p"localhost") ) + .flatMap(host => p"http://$host:${VSS.ports("main-http")._2}/docs") Stack.exports( grafanaServiceUrl = grafanaServiceUrl, @@ -166,7 +161,6 @@ import besom.json.DefaultJsonProtocol.StringJsonFormat lokiDeploymentName = lokiDeployment.metadata.name, lokiServiceName = lokiService.metadata.name, grafanaDeploymentName = grafanaDeployment.metadata.name, - grafanaServiceName = grafanaService.metadata.name, promtailDaemonSetName = promtailDaemonSet.metadata.name, zookeeperDeploymentName = zooDeployment.metadata.name, zookeeperServiceName = zooService.metadata.name, diff --git a/infra/k8s/src/main/scala/Promtail.scala b/infra/k8s/src/main/scala/Promtail.scala index 6bb2032..717a441 100644 --- a/infra/k8s/src/main/scala/Promtail.scala +++ b/infra/k8s/src/main/scala/Promtail.scala @@ -16,7 +16,7 @@ object Promtail: def deploy(using Context - )(lokiService: Output[Service], namespace: Output[Namespace], k8sProvider: Output[k8s.Provider]) = + )(lokiUrl: Output[String], namespace: Output[Namespace], k8sProvider: Output[k8s.Provider]) = val configMap = ConfigMap( s"$appName-config", ConfigMapArgs( @@ -27,7 +27,7 @@ object Promtail: | http_listen_port: $port | grpc_listen_port: 0 |clients: - |- url: http://${lokiService.metadata.name.map(_.get)}:3100/loki/api/v1/push + |- url: $lokiUrl/loki/api/v1/push |positions: | filename: /tmp/positions.yaml |target_config: From efe1f103d4536c3cb9ab67dfa833856868b104b7 Mon Sep 17 00:00:00 2001 From: kpoliwka Date: Tue, 26 Mar 2024 10:35:45 +0100 Subject: [PATCH 11/17] refactor --- infra/k8s/src/main/scala/Main.scala | 13 ++------- infra/k8s/src/main/scala/VSS.scala | 42 ++++++++++++++++------------- 2 files changed, 26 insertions(+), 29 deletions(-) diff --git a/infra/k8s/src/main/scala/Main.scala b/infra/k8s/src/main/scala/Main.scala index c2d661a..0b53bd5 100644 --- a/infra/k8s/src/main/scala/Main.scala +++ b/infra/k8s/src/main/scala/Main.scala @@ -144,15 +144,7 @@ import besom.json.DefaultJsonProtocol.StringJsonFormat // vss val vssDeployment = VSS.deploy(appImagePullSecret, appImage, appNamespace, postgresService, kafkaService, jaegerService, k8sProvider) - val vssService = VSS.deployService(serviceType, appNamespace, vssDeployment, k8sProvider) - - val vssServiceUrl = - vssService.status.loadBalancer.ingress - .map( - _.flatMap(_.headOption.flatMap(_.hostname)) - .getOrElse(p"localhost") - ) - .flatMap(host => p"http://$host:${VSS.ports("main-http")._2}/docs") + val vssServiceUrl = VSS.deployService(serviceType, appNamespace, vssDeployment, k8sProvider) Stack.exports( grafanaServiceUrl = grafanaServiceUrl, @@ -170,8 +162,7 @@ import besom.json.DefaultJsonProtocol.StringJsonFormat postgresServiceName = postgresService.metadata.name, jaegerDeploymentName = jaegerDeployment.metadata.name, jaegerServiceName = jaegerService.metadata.name, - vssDeploymentName = vssDeployment.metadata.name, - vssServiceName = vssService.metadata.name + vssDeploymentName = vssDeployment.metadata.name ) } diff --git a/infra/k8s/src/main/scala/VSS.scala b/infra/k8s/src/main/scala/VSS.scala index adc6534..635eac5 100644 --- a/infra/k8s/src/main/scala/VSS.scala +++ b/infra/k8s/src/main/scala/VSS.scala @@ -7,7 +7,7 @@ import besom.api.kubernetes.core.v1.inputs.* import besom.api.kubernetes.core.v1.{Namespace, Service, ServiceArgs} import besom.api.kubernetes.meta.v1.inputs.* -object VSS { +object VSS: val appName: NonEmptyString = "vss-app" // todo fix inference in NonEmptyString val labels = Map("app" -> "vss-app") val ports = Map( @@ -81,22 +81,28 @@ object VSS { namespace: Output[Namespace], vssDeployment: Output[Deployment], k8sProvider: Output[k8s.Provider] - ) = Service( - appName, - ServiceArgs( - spec = ServiceSpecArgs( - `type` = serviceType, - selector = labels, - ports = ports.map { case (name, (protocol, port)) => - ServicePortArgs(name = name, port = port, targetPort = port, protocol = protocol) - }.toList + ) = + val service = Service( + appName, + ServiceArgs( + spec = ServiceSpecArgs( + `type` = serviceType, + selector = labels, + ports = ports.map { case (name, (protocol, port)) => + ServicePortArgs(name = name, port = port, targetPort = port, protocol = protocol) + }.toList + ), + metadata = ObjectMetaArgs( + name = s"$appName-service", + namespace = namespace.metadata.name + ) ), - metadata = ObjectMetaArgs( - name = s"$appName-service", - namespace = namespace.metadata.name - ) - ), - opts(dependsOn = vssDeployment, provider = k8sProvider) - ) + opts(dependsOn = vssDeployment, provider = k8sProvider) + ) -} + service.status.loadBalancer.ingress + .map( + _.flatMap(_.headOption.flatMap(_.hostname)) + .getOrElse(p"localhost") + ) + .flatMap(host => p"http://$host:${ports("main-http")._2}/docs") From 035415a90d642f893b406f4111a7cfc2299e457a Mon Sep 17 00:00:00 2001 From: kpoliwka Date: Tue, 26 Mar 2024 12:21:54 +0100 Subject: [PATCH 12/17] refactor --- infra/k8s/src/main/scala/Grafana.scala | 2 +- infra/k8s/src/main/scala/Main.scala | 14 ++++++++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/infra/k8s/src/main/scala/Grafana.scala b/infra/k8s/src/main/scala/Grafana.scala index 9d5a606..02121ea 100644 --- a/infra/k8s/src/main/scala/Grafana.scala +++ b/infra/k8s/src/main/scala/Grafana.scala @@ -153,7 +153,7 @@ object Grafana: name = s"$appName-provider", grafana.ProviderArgs( retryWait = 20, // seconds - retries = 6, + retries = 8, auth = config.requireString("grafana:auth"), url = serviceUrl ) diff --git a/infra/k8s/src/main/scala/Main.scala b/infra/k8s/src/main/scala/Main.scala index 0b53bd5..df70d2a 100644 --- a/infra/k8s/src/main/scala/Main.scala +++ b/infra/k8s/src/main/scala/Main.scala @@ -75,12 +75,13 @@ import besom.json.DefaultJsonProtocol.StringJsonFormat ), opts = opts(provider = dockerProvider) ) - val image = docker.RegistryImage( - name = s"$appName-image", - docker.RegistryImageArgs(name = tag.targetImage), - opts = opts(provider = dockerProvider) - ) - image.flatMap(_ => tag.targetImage) + docker + .RegistryImage( + name = s"$appName-image", + docker.RegistryImageArgs(name = tag.targetImage), + opts = opts(provider = dockerProvider) + ) + .name val appImagePullSecret = clusterConfig.flatMap: case Cluster.Local => @@ -147,6 +148,7 @@ import besom.json.DefaultJsonProtocol.StringJsonFormat val vssServiceUrl = VSS.deployService(serviceType, appNamespace, vssDeployment, k8sProvider) Stack.exports( + appImage = appImage, grafanaServiceUrl = grafanaServiceUrl, vssServiceUrl = vssServiceUrl, namespaceName = appNamespace.metadata.name, From 84b4970a119caf33b57c5111422a9e4dfcab8acb Mon Sep 17 00:00:00 2001 From: kpoliwka Date: Thu, 28 Mar 2024 10:15:38 +0100 Subject: [PATCH 13/17] refactor --- infra/k8s/Pulumi.yaml | 6 +++--- infra/k8s/src/main/scala/Grafana.scala | 14 ++++++++------ infra/k8s/src/main/scala/Main.scala | 6 +++--- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/infra/k8s/Pulumi.yaml b/infra/k8s/Pulumi.yaml index e608923..51d3c83 100644 --- a/infra/k8s/Pulumi.yaml +++ b/infra/k8s/Pulumi.yaml @@ -5,7 +5,7 @@ config: vss:localRepository: "localhost:5001/vss-zio" vss:imageTag: "0.1.0-SNAPSHOT" vss:cluster: "remote" - vss:cluster-org: "organization" - vss:cluster-project: "vss-aws-eks" - vss:cluster-stack: "dev" + vss:clusterOrg: "organization" + vss:clusterProject: "vss-aws-eks" + vss:clusterStack: "dev" grafana:auth: "admin:admin" \ No newline at end of file diff --git a/infra/k8s/src/main/scala/Grafana.scala b/infra/k8s/src/main/scala/Grafana.scala index 02121ea..97523c8 100644 --- a/infra/k8s/src/main/scala/Grafana.scala +++ b/infra/k8s/src/main/scala/Grafana.scala @@ -154,9 +154,10 @@ object Grafana: grafana.ProviderArgs( retryWait = 20, // seconds retries = 8, - auth = config.requireString("grafana:auth"), - url = serviceUrl - ) + url = serviceUrl, + auth = config.requireString("grafana:auth"), //grafana.config.getAuth + ), + opts = opts(dependsOn = grafanaDeployment, deletedWith = grafanaDeployment) ) val lokiDataSource = grafana.DataSource( @@ -167,7 +168,7 @@ object Grafana: basicAuthEnabled = false, isDefault = true ), - opts = opts(provider = grafanaProvider, dependsOn = service) + opts = opts(provider = grafanaProvider, dependsOn = service, deletedWith = grafanaDeployment) ) val jaegerDataSource = grafana.DataSource( @@ -175,9 +176,10 @@ object Grafana: grafana.DataSourceArgs( url = jaegerUrl, `type` = "jaeger", - basicAuthEnabled = false + basicAuthEnabled = false, + isDefault = false ), - opts = opts(provider = grafanaProvider, dependsOn = service) + opts = opts(provider = grafanaProvider, dependsOn = service, deletedWith = grafanaDeployment) ) for _ <- lokiDataSource diff --git a/infra/k8s/src/main/scala/Main.scala b/infra/k8s/src/main/scala/Main.scala index df70d2a..02cef8c 100644 --- a/infra/k8s/src/main/scala/Main.scala +++ b/infra/k8s/src/main/scala/Main.scala @@ -20,9 +20,9 @@ import besom.json.DefaultJsonProtocol.StringJsonFormat val clusterStack = for - orgName <- config.getString("cluster-org").getOrElse("organization") - projName <- config.requireString("cluster-project") - stackName <- config.requireString("cluster-stack") + orgName <- config.getString("clusterOrg").getOrElse("organization") + projName <- config.requireString("clusterProject") + stackName <- config.requireString("clusterStack") stack <- StackReference(name = s"$orgName/$projName/$stackName") yield stack From 5930ea7f0f5f3d6e902553839691751c16cbf558 Mon Sep 17 00:00:00 2001 From: kpoliwka Date: Fri, 5 Apr 2024 13:31:43 +0200 Subject: [PATCH 14/17] add gcp gke example --- infra/eks/Main.scala | 3 +- infra/gke/Main.scala | 117 ++++++++++++++++++++++++ infra/gke/Pulumi.yaml | 6 ++ infra/gke/project.scala | 5 + infra/k8s/src/main/scala/Grafana.scala | 67 +++++++------- infra/k8s/src/main/scala/Main.scala | 4 +- infra/k8s/src/main/scala/Postgres.scala | 2 +- infra/k8s/src/main/scala/VSS.scala | 9 +- 8 files changed, 175 insertions(+), 38 deletions(-) create mode 100644 infra/gke/Main.scala create mode 100644 infra/gke/Pulumi.yaml create mode 100644 infra/gke/project.scala diff --git a/infra/eks/Main.scala b/infra/eks/Main.scala index 7f08427..ea48a33 100644 --- a/infra/eks/Main.scala +++ b/infra/eks/Main.scala @@ -39,7 +39,6 @@ import besom.api.{aws, awsx, eks} repositoryUrl = repo.repositoryUrl, accessKeyId = authorizationToken.map(_.userName).asSecret, secretAccessKey = authorizationToken.map(_.password).asSecret, - kubeconfig = cluster.kubeconfigJson, - organization = pulumiOrganization + kubeconfig = cluster.kubeconfigJson ) } diff --git a/infra/gke/Main.scala b/infra/gke/Main.scala new file mode 100644 index 0000000..239f1e1 --- /dev/null +++ b/infra/gke/Main.scala @@ -0,0 +1,117 @@ +import besom.* +import besom.api.gcp + +enum GCPService(val name: String): + case KubernetesEngine extends GCPService("container.googleapis.com") + case ArtifactRegistry extends GCPService("artifactregistry.googleapis.com") + +@main def main = Pulumi.run { + val appName = "vss" + + // Enable GCP service(s) for the current project + val enableServices: Map[GCPService, Output[gcp.projects.Service]] = + GCPService.values + .map(api => api -> projectService(api)) + .toMap + + val k8sCluster = gcp.container.Cluster( + name = s"$appName-cluster", + gcp.container.ClusterArgs( + deletionProtection = false, + initialNodeCount = 1, + minMasterVersion = "1.29.1-gke.1589018", + nodeVersion = "1.29.1-gke.1589018", + nodeConfig = gcp.container.inputs.ClusterNodeConfigArgs( + machineType = "n1-standard-1", + oauthScopes = List( + "https://www.googleapis.com/auth/compute", + "https://www.googleapis.com/auth/devstorage.read_only", + "https://www.googleapis.com/auth/logging.write", + "https://www.googleapis.com/auth/monitoring" + ) + ) + ), + opts = opts(dependsOn = enableServices(GCPService.KubernetesEngine)) + ) + + val context = p"${k8sCluster.project}_${k8sCluster.location}_${k8sCluster.name}" + val kubeconfig = + p"""apiVersion: v1 + |clusters: + |- cluster: + | certificate-authority-data: ${k8sCluster.masterAuth.clusterCaCertificate.map(_.get).asPlaintext} + | server: https://${k8sCluster.endpoint} + | name: $context + |contexts: + |- context: + | cluster: $context + | user: $context + | name: $context + |current-context: $context + |kind: Config + |preferences: {} + |users: + |- name: $context + | user: + | exec: + | apiVersion: client.authentication.k8s.io/v1beta1 + | command: gke-gcloud-auth-plugin + | installHint: Install gke-gcloud-auth-plugin for use with kubectl by following + | https://cloud.google.com/blog/products/containers-kubernetes/kubectl-auth-changes-in-gke + | provideClusterInfo: true + |""".stripMargin + + val repository = gcp.artifactregistry.Repository( + name = s"$appName-repository", + gcp.artifactregistry.RepositoryArgs( + repositoryId = s"$appName-repository", + format = "DOCKER" + ), + opts = opts(dependsOn = enableServices(GCPService.ArtifactRegistry)) + ) + + val serviceAccount = gcp.serviceaccount.Account( + name = s"$appName-service-account", + gcp.serviceaccount.AccountArgs( + accountId = s"$appName-service-account", + displayName = "Service Account for Artifact Registry" + ) + ) + + val serviceAccountKey = gcp.serviceaccount.Key( + name = s"$appName-service-account-key", + gcp.serviceaccount.KeyArgs( + serviceAccountId = serviceAccount.accountId, + privateKeyType = "TYPE_GOOGLE_CREDENTIALS_FILE" + ) + ) + + val repositoryIamBinding = gcp.artifactregistry.RepositoryIamMember( + name = s"$appName-service-iam-everyone", + gcp.artifactregistry.RepositoryIamMemberArgs( + repository = repository.repositoryId, + role = "roles/artifactregistry.repoAdmin", + member = "allUsers" + ) + ) + + Stack(repositoryIamBinding).exports( + registryEndpoint = p"https://${k8sCluster.location}-docker.pkg.dev", + repositoryUrl = p"${k8sCluster.location}-docker.pkg.dev/${k8sCluster.project}/${repository.repositoryId}/$appName", + accessKeyId = "_json_key_base64", + secretAccessKey = serviceAccountKey.privateKey, + kubeconfig = kubeconfig + ) +} + +private def projectService(api: GCPService)(using Context): Output[gcp.projects.Service] = + gcp.projects.Service( + name = s"enable-${api.name.replace(".", "-")}", + gcp.projects.ServiceArgs( + service = api.name, + /* if true - at every destroy this will disable the dependent services for the whole project */ + disableDependentServices = true, + /* if true - at every destroy this will disable the service for the whole project */ + disableOnDestroy = true + ) + ) diff --git a/infra/gke/Pulumi.yaml b/infra/gke/Pulumi.yaml new file mode 100644 index 0000000..723cc07 --- /dev/null +++ b/infra/gke/Pulumi.yaml @@ -0,0 +1,6 @@ +name: vss-gcp-gke +runtime: scala +description: VSS GCP GKE infra +config: + gcp:project: besom-413811 + gcp:region: us-west1 \ No newline at end of file diff --git a/infra/gke/project.scala b/infra/gke/project.scala new file mode 100644 index 0000000..ce416ef --- /dev/null +++ b/infra/gke/project.scala @@ -0,0 +1,5 @@ +//> using scala "3.3.3" +//> using options -Werror -Wunused:all -Wvalue-discard -Wnonunit-statement +//> using plugin "org.virtuslab::besom-compiler-plugin:0.2.2" +//> using dep "org.virtuslab::besom-core:0.2.2" +//> using dep "org.virtuslab::besom-gcp:7.9.0-core.0.2" diff --git a/infra/k8s/src/main/scala/Grafana.scala b/infra/k8s/src/main/scala/Grafana.scala index 97523c8..8b2ea86 100644 --- a/infra/k8s/src/main/scala/Grafana.scala +++ b/infra/k8s/src/main/scala/Grafana.scala @@ -13,34 +13,26 @@ object Grafana: val labels = Map("app" -> "grafana") val port = 3000 - def deploy(using Context)(namespace: Output[Namespace], k8sProvider: Output[k8s.Provider]) = { - val grafanaPV = PersistentVolume( - appName, - k8s.core.v1.PersistentVolumeArgs( - metadata = ObjectMetaArgs(name = s"$appName-pv", namespace = namespace.metadata.name), - spec = PersistentVolumeSpecArgs( - storageClassName = "standard", - capacity = Map("storage" -> "1Gi"), - accessModes = List("ReadWriteOnce"), - hostPath = HostPathVolumeSourceArgs("/data/grafana") - ) - ), - opts(provider = k8sProvider) - ) - val grafanaPVC = PersistentVolumeClaim( - appName, - k8s.core.v1.PersistentVolumeClaimArgs( - metadata = ObjectMetaArgs(name = s"$appName-pvc", namespace = namespace.metadata.name), - spec = PersistentVolumeClaimSpecArgs( - storageClassName = "standard", - volumeName = grafanaPV.metadata.name, - accessModes = List("ReadWriteOnce"), - resources = VolumeResourceRequirementsArgs( - requests = Map("storage" -> "1Gi") - ) + def deploy(using + Context + )(namespace: Output[Namespace], postgresService: Output[Service], k8sProvider: Output[k8s.Provider]) = { + val grafanaConfigMap = ConfigMap( + s"$appName-config", + ConfigMapArgs( + metadata = ObjectMetaArgs(name = s"$appName-init-config-map", namespace = namespace.metadata.name), + data = Map( + "grafana.ini" -> + p""" + |[database] + |type = postgres + |host = ${postgresService.metadata.name.map(_.get)}:${Postgres.port} + |name = vss + |user = postgres + |password = postgres + """.stripMargin ) ), - opts(provider = k8sProvider) + opts(provider = k8sProvider, deleteBeforeReplace = true) ) Deployment( @@ -88,15 +80,19 @@ object Grafana: requests = Map("cpu" -> "250m", "memory" -> "750Mi") ), volumeMounts = List( - VolumeMountArgs(mountPath = "/var/lib/grafana", name = s"$appName-pv") + VolumeMountArgs( + mountPath = "/etc/grafana/grafana.ini", + subPath = "grafana.ini", + name = s"$appName-config" + ) ) ) ), volumes = List( VolumeArgs( - name = s"$appName-pv", - persistentVolumeClaim = PersistentVolumeClaimVolumeSourceArgs( - claimName = grafanaPVC.metadata.name.map(_.get) + name = s"$appName-config", + configMap = ConfigMapVolumeSourceArgs( + name = grafanaConfigMap.metadata.name ) ) ) @@ -144,7 +140,14 @@ object Grafana: val serviceUrl = service.status.loadBalancer.ingress .map( - _.flatMap(_.headOption.flatMap(_.hostname)) + _.flatMap(_.headOption) + .flatMap(ingress => + ingress.ip match + case None => + ingress.hostname + case other => + other + ) .getOrElse(p"localhost") ) .flatMap(host => p"http://$host:$port") @@ -155,7 +158,7 @@ object Grafana: retryWait = 20, // seconds retries = 8, url = serviceUrl, - auth = config.requireString("grafana:auth"), //grafana.config.getAuth + auth = config.requireString("grafana:auth") // grafana.config.getAuth ), opts = opts(dependsOn = grafanaDeployment, deletedWith = grafanaDeployment) ) diff --git a/infra/k8s/src/main/scala/Main.scala b/infra/k8s/src/main/scala/Main.scala index 02cef8c..ea7fd23 100644 --- a/infra/k8s/src/main/scala/Main.scala +++ b/infra/k8s/src/main/scala/Main.scala @@ -108,7 +108,7 @@ import besom.json.DefaultJsonProtocol.StringJsonFormat ).prettyPrint ) ), - opts = opts(provider = k8sProvider) + opts = opts(provider = k8sProvider, deleteBeforeReplace = true) ) secret.map(Some(_)) @@ -138,7 +138,7 @@ import besom.json.DefaultJsonProtocol.StringJsonFormat val jaegerUrl = p"http://${jaegerService.metadata.name.map(_.get)}:${Jaeger.ports("frontend")._2}" // grafana - val grafanaDeployment = Grafana.deploy(appNamespace, k8sProvider) + val grafanaDeployment = Grafana.deploy(appNamespace, postgresService, k8sProvider) val grafanaServiceUrl = Grafana.deployService(lokiUrl, jaegerUrl, serviceType, appNamespace, grafanaDeployment, k8sProvider) diff --git a/infra/k8s/src/main/scala/Postgres.scala b/infra/k8s/src/main/scala/Postgres.scala index c37c4d0..19808fd 100644 --- a/infra/k8s/src/main/scala/Postgres.scala +++ b/infra/k8s/src/main/scala/Postgres.scala @@ -24,7 +24,7 @@ object Postgres { storageClassName = "standard", capacity = Map("storage" -> "8Gi"), accessModes = List("ReadWriteMany"), - hostPath = HostPathVolumeSourceArgs("/data/db") + hostPath = HostPathVolumeSourceArgs("/mnt/disks/share/data/db") ) ), opts(provider = k8sProvider) diff --git a/infra/k8s/src/main/scala/VSS.scala b/infra/k8s/src/main/scala/VSS.scala index 635eac5..e5eba03 100644 --- a/infra/k8s/src/main/scala/VSS.scala +++ b/infra/k8s/src/main/scala/VSS.scala @@ -102,7 +102,14 @@ object VSS: service.status.loadBalancer.ingress .map( - _.flatMap(_.headOption.flatMap(_.hostname)) + _.flatMap(_.headOption) + .flatMap(ingress => + ingress.ip match + case None => + ingress.hostname + case other => + other + ) .getOrElse(p"localhost") ) .flatMap(host => p"http://$host:${ports("main-http")._2}/docs") From 3585f7f3c7977f7106b87f679baacffa5debf883 Mon Sep 17 00:00:00 2001 From: kpoliwka Date: Tue, 9 Apr 2024 09:58:03 +0200 Subject: [PATCH 15/17] add azure aks example --- infra/azure-aks/Main.scala | 76 +++++++++++++++++++++++++++++++++++ infra/azure-aks/Pulumi.yaml | 5 +++ infra/azure-aks/project.scala | 5 +++ 3 files changed, 86 insertions(+) create mode 100644 infra/azure-aks/Main.scala create mode 100644 infra/azure-aks/Pulumi.yaml create mode 100644 infra/azure-aks/project.scala diff --git a/infra/azure-aks/Main.scala b/infra/azure-aks/Main.scala new file mode 100644 index 0000000..c7f2c79 --- /dev/null +++ b/infra/azure-aks/Main.scala @@ -0,0 +1,76 @@ +import besom.* +import besom.api.azurenative + +@main def main = Pulumi.run { + val appName = "vss" + + val resourceGroup = azurenative.resources.ResourceGroup(s"$appName-resource-group") + + val identity = azurenative.managedidentity.UserAssignedIdentity( + name = s"$appName-identity", + azurenative.managedidentity.UserAssignedIdentityArgs(resourceGroupName = resourceGroup.name) + ) + + val k8sCluster = azurenative.containerservice.ManagedCluster( + name = s"$appName-cluster", + azurenative.containerservice.ManagedClusterArgs( + resourceGroupName = resourceGroup.name, + dnsPrefix = appName, + identity = azurenative.containerservice.inputs.ManagedClusterIdentityArgs( + `type` = azurenative.containerservice.enums.ResourceIdentityType.UserAssigned, + userAssignedIdentities = List(identity.id) + ), + agentPoolProfiles = List( + azurenative.containerservice.inputs.ManagedClusterAgentPoolProfileArgs( + count = 1, + vmSize = "Standard_DS2_v2", + mode = azurenative.containerservice.enums.AgentPoolMode.System, + name = "agentpool", + osType = azurenative.containerservice.enums.OsType.Linux + ) + ) + ) + ) + + val kubeconfig = azurenative.containerservice + .listManagedClusterUserCredentials( + azurenative.containerservice.ListManagedClusterUserCredentialsArgs( + resourceName = k8sCluster.name, + resourceGroupName = resourceGroup.name + ) + ) + .kubeconfigs + .map(_.head.value) + + val registry = azurenative.containerregistry.Registry( + name = s"$appName-registry", + azurenative.containerregistry.RegistryArgs( + resourceGroupName = resourceGroup.name, + registryName = s"${appName}Registry", + sku = azurenative.containerregistry.inputs.SkuArgs( + name = azurenative.containerregistry.enums.SkuName.Basic + ), + adminUserEnabled = true + ), + opts = opts(deleteBeforeReplace = true) + ) + + val registryCredentials = azurenative.containerregistry.listRegistryCredentials( + azurenative.containerregistry.ListRegistryCredentialsArgs( + registryName = registry.name, + resourceGroupName = resourceGroup.name + ) + ) + + Stack.exports( + registryEndpoint = registry.loginServer, + repositoryUrl = p"${registry.loginServer}/$appName", + accessKeyId = registryCredentials.username, + // TODO make it secret + secretAccessKey = registryCredentials.passwords.map(_.head.head).value, + // TODO make it secret + kubeconfig = kubeconfig.map(base64Decoder) + ) +} + +private def base64Decoder: String => String = v => new String(java.util.Base64.getDecoder.decode(v.getBytes)) diff --git a/infra/azure-aks/Pulumi.yaml b/infra/azure-aks/Pulumi.yaml new file mode 100644 index 0000000..aa5ac04 --- /dev/null +++ b/infra/azure-aks/Pulumi.yaml @@ -0,0 +1,5 @@ +name: vss-azure-aks +runtime: scala +description: VSS Azure AKS infra +config: + azure-native:location: East US \ No newline at end of file diff --git a/infra/azure-aks/project.scala b/infra/azure-aks/project.scala new file mode 100644 index 0000000..5824a5d --- /dev/null +++ b/infra/azure-aks/project.scala @@ -0,0 +1,5 @@ +//> using scala "3.3.3" +//> using options -Werror -Wunused:all -Wvalue-discard -Wnonunit-statement +//> using plugin "org.virtuslab::besom-compiler-plugin:0.2.2" +//> using dep "org.virtuslab::besom-core:0.2.2" +//> using dep "org.virtuslab::besom-azure-native:2.29.0-core.0.2" From 2aa0f6002e9b1973303caeef4cad2e885b7f5e86 Mon Sep 17 00:00:00 2001 From: kpoliwka Date: Tue, 9 Apr 2024 10:00:50 +0200 Subject: [PATCH 16/17] rename folders --- infra/{eks => aws-eks}/Main.scala | 0 infra/{eks => aws-eks}/Pulumi.yaml | 0 infra/{eks => aws-eks}/project.scala | 0 infra/{gke => gcp-gke}/Main.scala | 0 infra/{gke => gcp-gke}/Pulumi.yaml | 0 infra/{gke => gcp-gke}/project.scala | 0 6 files changed, 0 insertions(+), 0 deletions(-) rename infra/{eks => aws-eks}/Main.scala (100%) rename infra/{eks => aws-eks}/Pulumi.yaml (100%) rename infra/{eks => aws-eks}/project.scala (100%) rename infra/{gke => gcp-gke}/Main.scala (100%) rename infra/{gke => gcp-gke}/Pulumi.yaml (100%) rename infra/{gke => gcp-gke}/project.scala (100%) diff --git a/infra/eks/Main.scala b/infra/aws-eks/Main.scala similarity index 100% rename from infra/eks/Main.scala rename to infra/aws-eks/Main.scala diff --git a/infra/eks/Pulumi.yaml b/infra/aws-eks/Pulumi.yaml similarity index 100% rename from infra/eks/Pulumi.yaml rename to infra/aws-eks/Pulumi.yaml diff --git a/infra/eks/project.scala b/infra/aws-eks/project.scala similarity index 100% rename from infra/eks/project.scala rename to infra/aws-eks/project.scala diff --git a/infra/gke/Main.scala b/infra/gcp-gke/Main.scala similarity index 100% rename from infra/gke/Main.scala rename to infra/gcp-gke/Main.scala diff --git a/infra/gke/Pulumi.yaml b/infra/gcp-gke/Pulumi.yaml similarity index 100% rename from infra/gke/Pulumi.yaml rename to infra/gcp-gke/Pulumi.yaml diff --git a/infra/gke/project.scala b/infra/gcp-gke/project.scala similarity index 100% rename from infra/gke/project.scala rename to infra/gcp-gke/project.scala From abdc6772c408244bdd2e732494d3655e635e8478 Mon Sep 17 00:00:00 2001 From: kpoliwka Date: Tue, 9 Apr 2024 14:05:31 +0200 Subject: [PATCH 17/17] add image tags --- infra/gcp-gke/Main.scala | 2 +- infra/k8s/src/main/scala/Grafana.scala | 3 ++- infra/k8s/src/main/scala/Jaeger.scala | 3 ++- infra/k8s/src/main/scala/Kafka.scala | 3 ++- infra/k8s/src/main/scala/Loki.scala | 3 ++- infra/k8s/src/main/scala/Postgres.scala | 3 ++- infra/k8s/src/main/scala/Promtail.scala | 3 ++- infra/k8s/src/main/scala/Zookeeper.scala | 3 ++- 8 files changed, 15 insertions(+), 8 deletions(-) diff --git a/infra/gcp-gke/Main.scala b/infra/gcp-gke/Main.scala index 239f1e1..edbd9f1 100644 --- a/infra/gcp-gke/Main.scala +++ b/infra/gcp-gke/Main.scala @@ -96,7 +96,7 @@ enum GCPService(val name: String): ) Stack(repositoryIamBinding).exports( - registryEndpoint = p"https://${k8sCluster.location}-docker.pkg.dev", + registryEndpoint = p"${k8sCluster.location}-docker.pkg.dev", repositoryUrl = p"${k8sCluster.location}-docker.pkg.dev/${k8sCluster.project}/${repository.repositoryId}/$appName", accessKeyId = "_json_key_base64", secretAccessKey = serviceAccountKey.privateKey, diff --git a/infra/k8s/src/main/scala/Grafana.scala b/infra/k8s/src/main/scala/Grafana.scala index 8b2ea86..17c6026 100644 --- a/infra/k8s/src/main/scala/Grafana.scala +++ b/infra/k8s/src/main/scala/Grafana.scala @@ -12,6 +12,7 @@ object Grafana: val appName: NonEmptyString = "grafana" val labels = Map("app" -> "grafana") val port = 3000 + private val imageTag = "10.1.9" def deploy(using Context @@ -52,7 +53,7 @@ object Grafana: containers = List( ContainerArgs( name = appName, - image = "grafana/grafana:latest", + image = s"grafana/grafana:$imageTag", ports = List( ContainerPortArgs(containerPort = port) ), diff --git a/infra/k8s/src/main/scala/Jaeger.scala b/infra/k8s/src/main/scala/Jaeger.scala index a5a1a47..d648d5d 100644 --- a/infra/k8s/src/main/scala/Jaeger.scala +++ b/infra/k8s/src/main/scala/Jaeger.scala @@ -13,6 +13,7 @@ import besom.aliases.NonEmptyString object Jaeger { val appName: NonEmptyString = "jaeger" // todo fix inference in NonEmptyString val labels = Map("app" -> "jaeger") + private val imageTag = "1.36" // https://www.jaegertracing.io/docs/1.6/getting-started/#all-in-one-docker-image - port descriptions val ports = Map( @@ -41,7 +42,7 @@ object Jaeger { containers = List( ContainerArgs( name = appName, - image = "jaegertracing/all-in-one:1.36", + image = s"jaegertracing/all-in-one:$imageTag", ports = ports.map { case (name, (protocol, port)) => ContainerPortArgs(containerPort = port, protocol) }.toList, diff --git a/infra/k8s/src/main/scala/Kafka.scala b/infra/k8s/src/main/scala/Kafka.scala index b74f009..454dacb 100644 --- a/infra/k8s/src/main/scala/Kafka.scala +++ b/infra/k8s/src/main/scala/Kafka.scala @@ -13,6 +13,7 @@ object Kafka { val labels = Map("app" -> "kafka") val kafkaServiceName = s"$appName-service" val port = 9092 + private val imageTag = "7.0.1" def deploy(using Context @@ -32,7 +33,7 @@ object Kafka { containers = List( ContainerArgs( name = "kafka-broker", - image = "confluentinc/cp-kafka:7.0.1", + image = s"confluentinc/cp-kafka:$imageTag", ports = List( ContainerPortArgs(containerPort = port) ), diff --git a/infra/k8s/src/main/scala/Loki.scala b/infra/k8s/src/main/scala/Loki.scala index 440cc9e..022755a 100644 --- a/infra/k8s/src/main/scala/Loki.scala +++ b/infra/k8s/src/main/scala/Loki.scala @@ -14,6 +14,7 @@ object Loki: val appName: NonEmptyString = "loki" val labels = Map("app" -> "loki") val port = 3100 + private val imageTag = "2.8.11" private val configFileName = "loki.yaml" def deploy(using Context)(namespace: Output[Namespace], k8sProvider: Output[k8s.Provider]) = @@ -95,7 +96,7 @@ object Loki: containers = List( ContainerArgs( name = appName, - image = "grafana/loki:latest", + image = s"grafana/loki:$imageTag", ports = List( ContainerPortArgs(containerPort = port) ), diff --git a/infra/k8s/src/main/scala/Postgres.scala b/infra/k8s/src/main/scala/Postgres.scala index 19808fd..40471a8 100644 --- a/infra/k8s/src/main/scala/Postgres.scala +++ b/infra/k8s/src/main/scala/Postgres.scala @@ -13,6 +13,7 @@ object Postgres { val appName: NonEmptyString = "postgres" // todo fix inference in NonEmptyString val labels = Map("app" -> "postgres") val port = 5432 + private val imageTag = "14.1-alpine" def deploy(using Context)(namespace: Output[Namespace], k8sProvider: Output[k8s.Provider]) = { @@ -86,7 +87,7 @@ object Postgres { containers = List( ContainerArgs( name = appName, - image = "postgres:14.1-alpine", + image = s"postgres:$imageTag", ports = List( ContainerPortArgs(containerPort = port) ), diff --git a/infra/k8s/src/main/scala/Promtail.scala b/infra/k8s/src/main/scala/Promtail.scala index 717a441..ea22f99 100644 --- a/infra/k8s/src/main/scala/Promtail.scala +++ b/infra/k8s/src/main/scala/Promtail.scala @@ -12,6 +12,7 @@ object Promtail: val appName: NonEmptyString = "promtail" val labels = Map("app" -> "promtail") val port = 9080 + private val imageTag = "2.8.11" private val configFileName = "promtail.yaml" def deploy(using @@ -135,7 +136,7 @@ object Promtail: containers = List( ContainerArgs( name = appName, - image = "grafana/promtail:latest", + image = s"grafana/promtail:$imageTag", ports = List( ContainerPortArgs(containerPort = port) ), diff --git a/infra/k8s/src/main/scala/Zookeeper.scala b/infra/k8s/src/main/scala/Zookeeper.scala index 785c6d7..f795d13 100644 --- a/infra/k8s/src/main/scala/Zookeeper.scala +++ b/infra/k8s/src/main/scala/Zookeeper.scala @@ -12,6 +12,7 @@ import besom.aliases.NonEmptyString object Zookeeper { val appName: NonEmptyString = "zookeeper" // todo fix inference in NonEmptyString val labels = Map("app" -> "zookeeper") + private val imageTag = "7.0.1" def deploy(using Context)(namespace: Output[Namespace], k8sProvider: Output[k8s.Provider]) = Deployment( appName, @@ -29,7 +30,7 @@ object Zookeeper { containers = List( ContainerArgs( name = appName, - image = "confluentinc/cp-zookeeper:7.0.1", + image = s"confluentinc/cp-zookeeper:$imageTag", ports = List( ContainerPortArgs(containerPort = 2181) ),