Dev to Main

.github/workflows/linter-self-hosted.yaml

@@ -9,7 +9,8 @@ name: Lint Code Base Self Hosted
 # Start the job on all push #
 #############################
 on:
-#  push:
+  push:
+    branches: [dev, master, main]
 #    branches-ignore: [master, main]
 #    # Remove the line above to run when pushing to master or main
   pull_request:

.github/workflows/linter.yaml

@@ -9,7 +9,8 @@ name: Lint Code Base
 # Start the job on all push #
 #############################
 on:
-#  push:
+  push:
+    branches: [dev, master, main]
 #    branches-ignore: [master, main]
 #    # Remove the line above to run when pushing to master or main
   pull_request:
@@ -63,7 +64,7 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           VALIDATE_GO: false
           IGNORE_GITIGNORED_FILES: true
-          FILTER_REGEX_EXCLUDE: ".*(sandbox|couchbase|templates).*"
+          FILTER_REGEX_EXCLUDE: ".*(sandbox|couchbase|templates|devops/otel/local).*"
           BASH_SEVERITY: "error"
           VALIDATE_NATURAL_LANGUAGE: false
           VALIDATE_DOCKERFILE_HADOLINT: false

.gitignore

@@ -67,3 +67,9 @@ core.*
 *.core
 
 secrets.txt
+src/item-service/var/log/shoppinglist/item-service.log
+src/item-service/__debug_bin*
+
+*.log
+
+devops/otel/local/grafana/grafana.ini

.vscode/launch.json

@@ -0,0 +1,22 @@
+{
+    "version": "0.2.0",
+    "configurations": [
+      {
+        "name": "Launch Package",
+        "type": "go",
+        "request": "launch",
+        "mode": "debug",
+        "program": "${workspaceFolder}/src/item-service",
+        "env": {
+          "COUCHBASE_CONNECTION_STRING":"couchbase://localhost/",
+          "COUCHBASE_USERNAME":"Administrator",
+          "COUCHBASE_PASSWORD":"Qwerty123",
+          "COUCHBASE_BUCKET":"default",
+          "OTEL_COLLECTOR_HOST":"localhost",
+          "SERVICE_NAME": "item-service",
+          "PORT":"81",
+          "COUCHBASE_RAM_QUOTA_MB": "100"
+        }
+      }
+    ]
+  }

Makefile

@@ -9,9 +9,9 @@ build-bastion:
 	docker build -t oltur/bastion:latest devops/docker -f devops/docker/bastion.Dockerfile
 	docker push oltur/bastion:latest
 
-.PHONY:install-bastion
-install-bastion: #build-bastion
-	helm install bastion-host --values devops/bastion-host/values.yaml  ./devops/bastion-host
+.PHONY: install-bastion
+install-bastion: # build-bastion
+	helm upgrade --install bastion-host --values devops/bastion-host/values.yaml  ./devops/bastion-host
 
 .PHONY:uninstall-bastion
 uninstall-bastion: #build-bastion
@@ -49,7 +49,7 @@ build-user-service-debug:
 #	$(eval COUCHBASE_PASSWORD=$(shell helm status couchbase --namespace couchbase | sed -n -e 's/^.*password: //p'))
 #	$(eval SERVICE_VERSION=$(SERVICE_VERSION))
 #	$(eval DEBUG=false)
-#	helm install user-service --values devops/user-service/values.yaml --set SERVICE_NAME=user-service --set DEBUG=$(DEBUG) --set COUCHBASE_PASSWORD=$(COUCHBASE_PASSWORD) --set SERVICE_VERSION=$(SERVICE_VERSION) devops/service
+#	helm upgrade --install user-service --values devops/user-service/values.yaml --set SERVICE_NAME=user-service --set DEBUG=$(DEBUG) --set COUCHBASE_PASSWORD=$(COUCHBASE_PASSWORD) --set SERVICE_VERSION=$(SERVICE_VERSION) devops/service
 
 .PHONY: upgrade-user-service
 upgrade-user-service: # build-user-service
@@ -65,7 +65,7 @@ upgrade-user-service: # build-user-service
 #	$(eval COUCHBASE_PASSWORD=$(shell helm status couchbase --namespace couchbase | sed -n -e 's/^.*password: //p'))
 #	$(eval SERVICE_VERSION=$(SERVICE_VERSION))
 #	$(eval DEBUG=true)
-#	helm install user-service --values devops/user-service/values.yaml --set SERVICE_NAME=user-service --set DEBUG=$(DEBUG) --set COUCHBASE_PASSWORD=$(COUCHBASE_PASSWORD) --set SERVICE_VERSION=$(SERVICE_VERSION) devops/service
+#	helm upgrade --install user-service --values devops/user-service/values.yaml --set SERVICE_NAME=user-service --set DEBUG=$(DEBUG) --set COUCHBASE_PASSWORD=$(COUCHBASE_PASSWORD) --set SERVICE_VERSION=$(SERVICE_VERSION) devops/service
 
 .PHONY: upgrade-user-service-debug
 upgrade-user-service-debug: # build-user-service-debug
@@ -97,15 +97,15 @@ build-item-service-debug:
 #	$(eval COUCHBASE_PASSWORD=$(shell helm status couchbase --namespace couchbase | sed -n -e 's/^.*password: //p'))
 #	$(eval SERVICE_VERSION=$(SERVICE_VERSION))
 #	$(eval DEBUG=false)
-#	helm install item-service --values devops/item-service/values.yaml --set DEBUG=$(DEBUG) --set COUCHBASE_PASSWORD=$(COUCHBASE_PASSWORD) --set SERVICE_VERSION=$(SERVICE_VERSION) devops/service
+#	helm upgrade --install item-service --values devops/item-service/values.yaml --set DEBUG=$(DEBUG) --set COUCHBASE_PASSWORD=$(COUCHBASE_PASSWORD) --set SERVICE_VERSION=$(SERVICE_VERSION) devops/service
 
 #.PHONY: install-item-service-debug
 #install-item-service-debug: # build-item-service-debug
 #	# kind load docker-image item-service:
 #	$(eval COUCHBASE_PASSWORD=$(shell helm status couchbase --namespace couchbase | sed -n -e 's/^.*password: //p'))
 #	$(eval SERVICE_VERSION=$(SERVICE_VERSION))
 #	$(eval DEBUG=true)
-#	helm install item-service --values devops/item-service/values.yaml --set DEBUG=$(DEBUG) --set COUCHBASE_PASSWORD=$(COUCHBASE_PASSWORD) --set SERVICE_VERSION=$(SERVICE_VERSION) devops/service
+#	helm upgrade --install item-service --values devops/item-service/values.yaml --set DEBUG=$(DEBUG) --set COUCHBASE_PASSWORD=$(COUCHBASE_PASSWORD) --set SERVICE_VERSION=$(SERVICE_VERSION) devops/service
 
 .PHONY: upgrade-item-service
 upgrade-item-service: # build-item-service
@@ -126,36 +126,47 @@ upgrade-item-service-debug: # build-item-service-debug
 .PHONY: uninstall-item-service
 uninstall-item-service:
 	helm uninstall item-service
+
 #====================================
-.PHONY: install-monitoring
-install-monitoring:
-	helm install kube-prometheus-stack prometheus-community/kube-prometheus-stack --namespace monitoring --create-namespace
 
-.PHONY: uninstall-monitoring
-uninstall-monitoring:
+.PHONY: install-kube-prometheus-stack
+install-kube-prometheus-stack:
+	helm upgrade --install kube-prometheus-stack prometheus-community/kube-prometheus-stack --namespace monitoring --create-namespace --values devops/kube-prometheus-stack/values.yaml
+
+.PHONY: uninstall-kube-prometheus-stack
+uninstall-kube-prometheus-stack:
 	helm uninstall kube-prometheus-stack --namespace monitoring
 
-.PHONY: expose-prometheus
-expose-prometheus:
+.PHONY: expose-kube-prometheus-stack-prometheus
+expose-kube-prometheus-stack-prometheus:
 	kubectl --namespace monitoring port-forward svc/kube-prometheus-stack 9090
 	# Then access via http://localhost:9090
 
-.PHONY: expose-grafana
-expose-grafana:
+.PHONY: expose-kube-prometheus-stack-grafana
+expose-kube-prometheus-stack-grafana:
 	kubectl --namespace monitoring port-forward svc/kube-prometheus-stack-grafana 3000:80
 	# Then access via http://localhost:3000
 
-.PHONY: expose-alertmanager
-expose-alertmanager:
+.PHONY: expose-kube-prometheus-stack-alertmanager
+expose-kube-prometheus-stack-alertmanager:
 	kubectl --namespace monitoring port-forward svc/alertmanager-main 9093
 	# Then access via http://localhost:9093
 
+.PHONY: expose-loki
+expose-loki:
+	kubectl port-forward svc/loki 3000:80
+	# Then access via http://localhost:3000
+
+.PHONY: expose-zipkin
+expose-zipkin:
+	kubectl port-forward svc/zipkin 9411:9411
+	# Then access via http://localhost:9411
 #==================================================================================================
 
 .PHONY: install-traefik
 install-traefik:
 	helm repo add traefik https://traefik.github.io/charts
-	helm install traefik traefik/traefik --values devops/k8s/traefik/values.yaml
+	helm upgrade --install traefik traefik/traefik --values devops/k8s/traefik/values.yaml
 
 .PHONY: uninstall-traefik
 uninstall-traefik:
@@ -169,15 +180,16 @@ expose-traefik-dashboard:
 
 .PHONY: install-couchbase-local
 install-couchbase-local:
-	docker run -d --name couchbase -p 8091-8094:8091-8094 -p 11210:11210 couchbase
+	# docker run -d --name couchbase -p 8091-8094:8091-8094 -p 11210:11210 couchbase
+	docker run -d --name db -p 8091-8096:8091-8096 -p 11210-11211:11210-11211 couchbase
 
 .PHONY: install-couchbase
 install-couchbase:
-#	helm repo add couchbase https://couchbase-partners.github.io/helm-charts/
-#	helm repo update
-#	helm install couchbase --set cluster.name=couchbase --values devops/couchbase/values.yaml --set tls.generate=true couchbase/couchbase-operator
-#	helm install couchbase --set cluster.name=couchbase --values devops/couchbase/values.yaml --set tls.generate=true ./devops/couchbase/couchbase-operator
-	helm upgrade --install couchbase --set cluster.name=couchbase --values devops/couchbase/values.yaml --set tls.generate=true --namespace couchbase --create-namespace ./devops/couchbase/couchbase-operator
+	helm repo add couchbase https://couchbase-partners.github.io/helm-charts/
+	helm repo update
+#	helm upgrade --install couchbase --set cluster.name=couchbase --values devops/couchbase/values.yaml --set tls.generate=true couchbase/couchbase-operator
+#	helm upgrade --install couchbase --set cluster.name=couchbase --values devops/couchbase/values.yaml --set tls.generate=true ./devops/couchbase/couchbase-operator
+	helm upgrade --install couchbase --set cluster.name=couchbase --values devops/couchbase/values.yaml --namespace couchbase --create-namespace ./devops/couchbase/couchbase-operator
 
 .PHONY: uninstall-couchbase
 uninstall-couchbase:
@@ -189,7 +201,7 @@ status-couchbase:
 
 .PHONY: expose-couchbase-ui
 expose-couchbase-ui:
-	kubectl port-forward --namespace couchbase couchbase-0001 38091:8091
+	kubectl port-forward --namespace couchbase service/couchbase 38091:8091
 
 .PHONY: expose-couchbase-api
 expose-couchbase-api:
@@ -199,15 +211,15 @@ expose-couchbase-api:
 
 .PHONY: install-test-service
 install-test-service:
-	helm install test-service --values devops/test-service/values.yaml  ./devops/test-service # --namespace test
+	helm upgrade --install test-service --values devops/test-service/values.yaml  ./devops/test-service # --namespace test
 
 .PHONY: uninstall-test-service
 uninstall-test-service:
 	helm uninstall test-service
 
 .PHONY: install-ingress-example
 install-ingress-example:
-	helm install ingress-example ./devops/kind/ingress-example
+	helm upgrade --install ingress-example ./devops/kind/ingress-example
 
 .PHONY: uninstall-ingress-example
 uninstall-ingress-example:
@@ -216,7 +228,7 @@ uninstall-ingress-example:
 #==================================================================================================
 .PHONY: lint
 lint:
-	$(eval FILTER_REGEX_EXCLUDE=".*(sandbox|couchbase|templates).*")
+	$(eval FILTER_REGEX_EXCLUDE=".*(sandbox|couchbase|templates|devops/otel/local).*")
 	echo $(FILTER_REGEX_EXCLUDE)
 	docker run \
       -e RUN_LOCAL=true \
@@ -241,11 +253,75 @@ lint:
 install-self-hosted-github-runners:
 	kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.13.3/cert-manager.crds.yaml
 	helm repo add jetstack https://charts.jetstack.io
-	helm install cert-manager --namespace cert-manager --version v1.13.3 jetstack/cert-manager --create-namespace
-	kubectl create ns actions-runner-system
+	helm upgrade --install cert-manager --namespace cert-manager --version v1.13.3 jetstack/cert-manager --create-namespace
+	kubectl create namespace actions-runner-system --dry-run=client -o yaml | kubectl apply -f -
 	kubectl create secret generic controller-manager -n actions-runner-system --from-literal=github_token=ghp_?????????????????? #self-hosted-runners-cluster-prod
 	helm repo add actions-runner-controller https://actions-runner-controller.github.io/actions-runner-controller
 	helm repo update
 	helm upgrade --install --namespace actions-runner-system --create-namespace --wait actions-runner-controller actions-runner-controller/actions-runner-controller --set syncPeriod=1m
 	kubectl create -f ./devops/self-hosted-runners/runner.yaml
-	kubectl apply -f ./devops/self-hosted-runners/horizontal_runner_autoscaler.yaml
+	kubectl apply -f ./devops/self-hosted-runners/horizontal_runner_autoscaler.yaml
+
+.PHONY: install-otel-collector
+install-otel-collector:
+	helm repo add open-telemetry https://open-telemetry.github.io/opentelemetry-helm-charts
+	helm upgrade --install my-opentelemetry-collector open-telemetry/opentelemetry-collector --set mode=deployment --values devops/otel/opentelemetry-collector/values.yaml
+	# --set mode=<daemonset|deployment|statefulset>
+
+.PHONY: install-loki
+install-loki:
+	helm repo add grafana https://grafana.github.io/helm-charts
+	helm repo update
+
+	k apply -f devops/loki/sc.yaml   
+	kubectl patch storageclass local-storage -p '{"metadata": {"annotations":{"storageclass.kubernetes.io/is-default-class":"true"}}}'
+	k apply -f devops/loki/pv.yaml 
+	helm upgrade --install loki grafana/loki --values devops/loki/values.yaml 
+
+.PHONY: uninstall-loki
+uninstall-loki:
+	helm uninstall loki
+
+.PHONY: install-zipkin
+install-zipkin:
+	# helm repo add zipkin-helm https://financial-times.github.io/zipkin-helm/docs
+	# helm upgrade --install zipkin-helm zipkin-helm/zipkin-helm
+	k apply -f devops/zipkin/zipkin.yaml
+
+.PHONY: uninstall-zipkin
+uninstall-zipkin:
+	# helm uninstall zipkin-helm 
+	k delete -f devops/zipkin/zipkin.yaml
+
+# .PHONY: install-otel-collector
+# install-otel-collector:
+# 	# kubectl apply -f ./devops/otel/collector.yaml
+# 	kubectl apply -f ./devops/otel/collector/otel-config.yaml
+
+
+# now using one above
+# .PHONY: uninstall-otel-collector
+# uninstall-otel-collector:
+# 	# kubectl delete -f ./devops/otel/collector.yaml
+# 	kubectl delete -f ./devops/otel/collector/otel-config.yaml
+
+.PHONY: install-local-otel-collector
+install-local-otel-collector:
+	docker pull otel/opentelemetry-collector:0.93.0
+	docker run -d --name otel-collector -p 127.0.0.1:4317:4317 -p 127.0.0.1:55679:55679 otel/opentelemetry-collector:0.93.0
+
+.PHONY: install-metrics-server
+install-metrics-server:
+	kubectl apply -f ./devops/metrics-server/components.yaml
+
+# .PHONY: install-k8s-otel-collector
+# install-k8s-otel-collector:
+# 	helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
+# 	helm repo update
+# 	helm upgrade --install ksm prometheus-community/kube-state-metrics -n "default"
+# 	helm upgrade --install nodeexporter prometheus-community/prometheus-node-exporter -n "default"
+# 	TBD
+
+# .PHONY: install-prometheus
+# install-prometheus:
+# 	helm repo add prometheus-community https://prometheus-community.github.io/helm-charts

devops/couchbase/couchbase-operator/values-all.yaml

@@ -656,7 +656,7 @@ buckets:
     # than or equal to 100Mi.  More info:
     # https://kubernetes.io/docs/concepts/configuration/manage-resources-
     # containers/#resource-units-in-kubernetes
-    memoryQuota: 100Mi
+    memoryQuota: 500Mi
     # -- MiniumumDurability defines how durable a document write is by default,
     # and can be made more durable by the client.  This feature enables ACID
     # transactions. When none, Couchbase server will respond when the document
@@ -1147,7 +1147,7 @@ cluster:
     # resource scheduling. More info:
     # https://kubernetes.io/docs/concepts/configuration/manage-resources-
     # containers/#resource-units-in-kubernetes
-    queryServiceMemoryQuota:
+    queryServiceMemoryQuota: 256Mi
     # -- SearchServiceMemQuota is the amount of memory that should be allocated
     # to the search service. This value is per-pod, and only applicable to pods
     # belonging to server classes running the search service.  This field must

devops/couchbase/couchbase-operator/values.yaml

@@ -646,7 +646,7 @@ buckets:
     # than or equal to 100Mi.  More info:
     # https://kubernetes.io/docs/concepts/configuration/manage-resources-
     # containers/#resource-units-in-kubernetes
-    memoryQuota: 100Mi
+    memoryQuota: 200Mi
     # -- Replicas defines how many copies of documents Couchbase server
     # maintains.  This directly affects how fault tolerant a Couchbase cluster
     # is.  With a single replica, the cluster can tolerate one data pod going