diff --git a/.github/workflows/images.yaml b/.github/workflows/images.yaml index 78df6c1a..0cedecbe 100644 --- a/.github/workflows/images.yaml +++ b/.github/workflows/images.yaml @@ -56,7 +56,7 @@ jobs: id: imageRegistryTag uses: mikefarah/yq@v4.44.1 with: - cmd: yq '.images[0].newTag | sub("(.*)@.*", "${1}")' registry/generic/kustomization.yaml + cmd: yq '.images[0].newTag | sub("(.*)@.*", "${1}")' registry/images/kustomization.yaml - name: Mirror registry image from hub run: | diff --git a/bin/y-build b/bin/y-build index 37c9f7dd..124f3997 100755 --- a/bin/y-build +++ b/bin/y-build @@ -81,7 +81,7 @@ DEFAULT_REGISTRY=builds-registry.ystack.svc.cluster.local [ -z "$PUSH_REGISTRY" ] && PUSH_REGISTRY=$DEFAULT_REGISTRY if [ -z "$BUILDKIT_HOST" ]; then BUILDKIT_HOSTNAME=buildkitd.ystack.svc.cluster.local - # Prefer port 80 (Gateway API) for local clusters, fall back to 8547 (direct service) + # Prefer port 80 (Gateway API) for local clusters, fall back to 8547 (y-kubefwd for remote clusters) if BUILDKIT_HOST=tcp://$BUILDKIT_HOSTNAME:80 y-buildctl --timeout 2 debug workers >/dev/null 2>&1; then BUILDKIT_HOST=tcp://$BUILDKIT_HOSTNAME:80 else diff --git a/bin/y-chrome-devtools-mcp b/bin/y-chrome-devtools-mcp index 71c9cc40..50524805 100755 --- a/bin/y-chrome-devtools-mcp +++ b/bin/y-chrome-devtools-mcp @@ -45,4 +45,5 @@ elif ! echo "$@" | grep -q "\-\-isolated"; then [ "DEVTOOLS_MCP_ALLOW_NON_ISOLATED" = "1" ] && echo "Overriding due to DEVTOOLS_MCP_ALLOW_NON_ISOLATED=$DEVTOOLS_MCP_ALLOW_NON_ISOLATED" 1>&2 || exit 1 fi -/usr/bin/env node $BIN_DIR/build/src/index.js "$@" +BIN_ENTRY=$(jq -r '.bin["chrome-devtools-mcp"]' "$BIN_DIR/package.json") +/usr/bin/env node "$BIN_DIR/$BIN_ENTRY" "$@" diff --git a/bin/y-cluster-blobs-ls b/bin/y-cluster-blobs-ls index 2be551d8..c7949d4d 100755 --- a/bin/y-cluster-blobs-ls +++ b/bin/y-cluster-blobs-ls @@ -9,7 +9,7 @@ case $ctx in *) echo "Initial arg must be --context=local" && exit 1 ;; esac -[ -z "$NAMESPACE" ] && NAMESPACE=ystack +[ -z "$NAMESPACE" ] && NAMESPACE=blobs BUCKET="${1:-}" diff --git a/bin/y-cluster-converge-ystack b/bin/y-cluster-converge-ystack index dfadb4e7..47e06c3c 100755 --- a/bin/y-cluster-converge-ystack +++ b/bin/y-cluster-converge-ystack @@ -5,17 +5,19 @@ set -eo pipefail YSTACK_HOME="$(cd "$(dirname "$0")/.." && pwd)" CONTEXT="" -SERVER_SIDE="" +EXCLUDE="" +OVERRIDE_IP="" while [ $# -gt 0 ]; do case "$1" in --context=*) CONTEXT="${1#*=}"; shift ;; - --server-side) SERVER_SIDE="--server-side=true --force-conflicts"; shift ;; + --exclude=*) EXCLUDE="${1#*=}"; shift ;; + --override-ip=*) OVERRIDE_IP="${1#*=}"; shift ;; *) echo "Unknown flag: $1" >&2; exit 1 ;; esac done -[ -z "$CONTEXT" ] && echo "Usage: y-cluster-converge-ystack --context= [--server-side]" && exit 1 +[ -z "$CONTEXT" ] && echo "Usage: y-cluster-converge-ystack --context= [--exclude=SUBSTRING] [--override-ip=IP]" && exit 1 k() { kubectl --context="$CONTEXT" "$@" @@ -23,80 +25,106 @@ k() { apply_base() { local base="$1" - local basepath="$YSTACK_HOME/k3s/$base/" - echo "# Applying $basepath ..." - k apply $SERVER_SIDE -k "$basepath" + local output + output=$(k apply -k "$YSTACK_HOME/k3s/$base/" 2>&1) || { + echo "$output" >&2 + return 1 + } + [ -n "$output" ] && echo "$output" } -# 1. Namespace -apply_base 00-ystack-namespace -k get ns ystack -echo "# Validated: namespace ystack exists" - -# 2. Gateway API CRDs (managed by k3s traefik-crd HelmChart) + traefik Gateway provider config -echo "# Waiting for Gateway API CRDs (from k3s traefik-crd) ..." -until k get crd gateways.gateway.networking.k8s.io >/dev/null 2>&1; do sleep 2; done -apply_base 05-gateway-api -echo "# Validated: Gateway API CRDs installed" - -# 3. Gateway (wait for CRD to be served before applying) -echo "# Waiting for Gateway resource type to be served ..." -until k api-resources --api-group=gateway.networking.k8s.io 2>/dev/null | grep -q gateways; do sleep 2; done -apply_base 06-gateway +# List bases in order, filter out -disabled suffix +echo "[y-cluster-converge-ystack] Listing bases" +BASES=() +for dir in "$YSTACK_HOME"/k3s/[0-9][0-9]-*/; do + base=$(basename "$dir") + if [[ "$base" == *-disabled ]]; then + echo "[y-cluster-converge-ystack] Skipping disabled: $base" + continue + fi + if [ -n "$EXCLUDE" ] && [[ "$base" == *"$EXCLUDE"* ]]; then + echo "[y-cluster-converge-ystack] Skipping excluded (--exclude=$EXCLUDE): $base" + continue + fi + BASES+=("$base") +done +echo "[y-cluster-converge-ystack] Bases: ${BASES[*]}" + +prev_digit="" +for base in "${BASES[@]}"; do + digit="${base:0:1}" + + # Between digit groups, wait for readiness + if [ -n "$prev_digit" ] && [ "$digit" != "$prev_digit" ]; then + echo "[y-cluster-converge-ystack] Waiting for rollouts after ${prev_digit}* bases" + + # After CRDs (1*), wait for all of them to be established + if [ "$prev_digit" = "1" ]; then + echo "[y-cluster-converge-ystack] Waiting for all CRDs to be established" + k wait --for=condition=Established crd --all --timeout=60s + fi + + # Wait for all deployments that exist in any namespace + for ns in $(k get deploy --all-namespaces --no-headers -o custom-columns=NS:.metadata.namespace 2>/dev/null | sort -u); do + echo "[y-cluster-converge-ystack] Waiting for deployments in $ns" + k -n "$ns" rollout status deploy --timeout=120s + done + + # After 2* (gateway + y-kustomize), update /etc/hosts so curl can reach services + if [ "$prev_digit" = "2" ]; then + if [ -n "$OVERRIDE_IP" ]; then + echo "[y-cluster-converge-ystack] Annotating gateway with yolean.se/override-ip=$OVERRIDE_IP" + k -n ystack annotate gateway ystack yolean.se/override-ip="$OVERRIDE_IP" --overwrite + fi + if ! "$YSTACK_HOME/bin/y-k8s-ingress-hosts" --context="$CONTEXT" --ensure; then + echo "[y-cluster-converge-ystack] WARNING: /etc/hosts update failed (may need manual sudo)" >&2 + fi + fi + + # After 4* (kafka secrets updated), restart y-kustomize so volume mounts refresh + # without waiting for kubelet sync (can take 60-120s) + if [ "$prev_digit" = "4" ]; then + echo "[y-cluster-converge-ystack] Restarting y-kustomize to pick up updated secrets" + k -n ystack rollout restart deploy/y-kustomize + k -n ystack rollout status deploy/y-kustomize --timeout=60s + fi + + # Before 6* bases, verify y-kustomize serves real content + if [ "$digit" = "6" ]; then + echo "[y-cluster-converge-ystack] Verifying y-kustomize API" + curl -sf --retry 20 --retry-delay 2 --retry-all-errors --connect-timeout 2 http://y-kustomize.ystack.svc.cluster.local/health + echo "[y-cluster-converge-ystack] y-kustomize health ok" + curl -sf --retry 20 --retry-delay 2 --retry-all-errors --connect-timeout 2 http://y-kustomize.ystack.svc.cluster.local/v1/blobs/setup-bucket-job/base-for-annotations.yaml >/dev/null + echo "[y-cluster-converge-ystack] y-kustomize serving blobs bases" + curl -sf --retry 20 --retry-delay 2 --retry-all-errors --connect-timeout 2 http://y-kustomize.ystack.svc.cluster.local/v1/kafka/setup-topic-job/base-for-annotations.yaml >/dev/null + echo "[y-cluster-converge-ystack] y-kustomize serving kafka bases" + fi + fi + + echo "[y-cluster-converge-ystack] Applying $base" + if [[ "$base" == 1* ]]; then + k apply -k "$YSTACK_HOME/k3s/$base/" --server-side=true --force-conflicts + else + apply_base "$base" + fi + + prev_digit="$digit" +done + +# Update /etc/hosts now that all routes exist +if ! "$YSTACK_HOME/bin/y-k8s-ingress-hosts" --context="$CONTEXT" --ensure; then + echo "[y-cluster-converge-ystack] WARNING: /etc/hosts update failed (may need manual sudo)" >&2 +fi + +# Validation +echo "[y-cluster-converge-ystack] Validation" k -n ystack get gateway ystack -echo "# Validated: gateway ystack exists" - -# 4. VersityGW (S3-compatible blob store) -apply_base 10-versitygw -k -n ystack rollout status deploy/versitygw --timeout=120s -echo "# Validated: versitygw rollout complete" - -# 5. Builds registry -apply_base 20-builds-registry-versitygw -k -n ystack get svc builds-registry -CLUSTER_IP=$(k -n ystack get svc builds-registry -o=jsonpath='{.spec.clusterIP}') -if [ "$CLUSTER_IP" != "10.43.0.50" ]; then - echo "ERROR: builds-registry clusterIP is $CLUSTER_IP, expected 10.43.0.50" >&2 - exit 1 +k -n ystack get deploy y-kustomize +k -n blobs get svc y-s3-api +k -n kafka get statefulset redpanda +CLUSTER_IP=$(k -n ystack get svc builds-registry -o=jsonpath='{.spec.clusterIP}' 2>/dev/null || echo "") +if [ -n "$CLUSTER_IP" ] && [ "$CLUSTER_IP" != "10.43.0.50" ]; then + echo "[y-cluster-converge-ystack] WARNING: builds-registry clusterIP is $CLUSTER_IP, expected 10.43.0.50" >&2 fi -echo "# Validated: builds-registry clusterIP=10.43.0.50" - -# 6. Builds registry HTTPRoute -apply_base 07-builds-registry-httproute -k -n ystack get httproute builds-registry -echo "# Validated: httproute builds-registry exists" - -# 6.5 Buildkitd GRPCRoute -apply_base 08-buildkitd-grpcroute -k -n ystack get grpcroute buildkitd -echo "# Validated: grpcroute buildkitd exists" - -# 7. Monitoring operator + CRDs -apply_base 30-monitoring-operator -echo "# Waiting for prometheus-operator CRDs to register ..." -until k get crd prometheuses.monitoring.coreos.com >/dev/null 2>&1; do sleep 2; done -until k get crd alertmanagers.monitoring.coreos.com >/dev/null 2>&1; do sleep 2; done -until k get crd servicemonitors.monitoring.coreos.com >/dev/null 2>&1; do sleep 2; done -echo "# Validated: prometheus-operator CRDs registered" - -# 8. Monitoring CRs (Prometheus, Alertmanager, exporters) -apply_base 31-monitoring -k -n monitoring get prometheus now -echo "# Validated: monitoring stack exists" - -# 6.8 Prometheus HTTPRoute -apply_base 09-prometheus-httproute -k -n monitoring get httproute prometheus-now -echo "# Validated: httproute prometheus-now exists" - -# 7. Prod registry -apply_base 21-prod-registry -k -n ystack get svc prod-registry -echo "# Validated: prod-registry service exists" - -# 8. Buildkit -apply_base 40-buildkit -k -n ystack get statefulset buildkitd -echo "# Validated: buildkitd statefulset exists" - -echo "# y-cluster-converge-ystack: all steps completed successfully" + +echo "[y-cluster-converge-ystack] Completed. To verify use: y-cluster-validate-ystack --context=$CONTEXT" diff --git a/bin/y-cluster-local-detect b/bin/y-cluster-local-detect index 681d286c..4bdd7fe5 100755 --- a/bin/y-cluster-local-detect +++ b/bin/y-cluster-local-detect @@ -8,6 +8,7 @@ case "$CLUSTER" in ystack-k3d) PROVISIONER=k3d ;; ystack-multipass) PROVISIONER=multipass ;; ystack-lima) PROVISIONER=lima ;; + ystack-qemu) PROVISIONER=qemu ;; *) echo "No recognized ystack cluster at --context=local (cluster name: '$CLUSTER')" >&2 exit 1 diff --git a/bin/y-cluster-provision b/bin/y-cluster-provision new file mode 100755 index 00000000..5b5936dc --- /dev/null +++ b/bin/y-cluster-provision @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +[ -z "$DEBUG" ] || set -x +set -eo pipefail + +TEARDOWN=false +[ "$1" = "--teardown" ] && TEARDOWN=true + +if [ "$TEARDOWN" = "true" ]; then + YSTACK_PROVISIONER=$(y-cluster-local-detect) + echo "[y-cluster-provision] Tearing down $YSTACK_PROVISIONER cluster ..." + y-cluster-provision-$YSTACK_PROVISIONER --teardown + exit $? +fi + +if [ -n "$YSTACK_PROVISIONER" ]; then + true +elif command -v qemu-system-x86_64 >/dev/null 2>&1 && command -v qemu-img >/dev/null 2>&1 && command -v cloud-localds >/dev/null 2>&1 && [ -e /dev/kvm ]; then + YSTACK_PROVISIONER=qemu +elif command -v multipass >/dev/null 2>&1; then + YSTACK_PROVISIONER=multipass +elif command -v docker >/dev/null 2>&1; then + YSTACK_PROVISIONER=k3d +else + echo "No provisioner found. Set the YSTACK_PROVISIONER env." && exit 1 +fi + +echo "[y-cluster-provision] Provisioning using y-cluster-provision-$YSTACK_PROVISIONER ..." + +exec y-cluster-provision-$YSTACK_PROVISIONER "$@" diff --git a/bin/y-cluster-provision-k3d b/bin/y-cluster-provision-k3d index 34292107..9baa6595 100755 --- a/bin/y-cluster-provision-k3d +++ b/bin/y-cluster-provision-k3d @@ -9,10 +9,12 @@ YSTACK_HOME="$(cd "$(dirname "$0")/.." && pwd)" CTX=local K3D_NAME=ystack YSTACK_HOST=ystack.local -K3D_MEMORY="6G" +K3D_MEMORY="8G" K3D_AGENTS="0" K3D_DOCKER_UPDATE="--cpuset-cpus=3 --cpus=3" SKIP_CONVERGE=false +SKIP_IMAGE_LOAD=false +EXCLUDE=monitoring while [ $# -gt 0 ]; do case "$1" in @@ -22,11 +24,13 @@ Usage: y-cluster-provision-k3d [flags] Flags: --context=NAME kubeconfig context name (default: local) - --memory=SIZE server node memory limit (default: 6G) + --memory=SIZE server node memory limit (default: 8G) --agents=N number of agent nodes (default: 0) --docker-update=ARGS docker update flags for the server container (default: --cpuset-cpus=3 --cpus=3) --host=HOSTNAME hostname for ingress (default: ystack.local) + --exclude=SUBSTRING exclude k3s bases matching substring (default: monitoring) --skip-converge skip converge, validate, and post-provision steps + --skip-image-load skip image cache and load into containerd --teardown delete existing cluster and exit -h, --help show this help EOF @@ -36,7 +40,9 @@ EOF --agents=*) K3D_AGENTS="${1#*=}"; shift ;; --docker-update=*) K3D_DOCKER_UPDATE="${1#*=}"; shift ;; --host=*) YSTACK_HOST="${1#*=}"; shift ;; + --exclude=*) EXCLUDE="${1#*=}"; shift ;; --skip-converge) SKIP_CONVERGE=true; shift ;; + --skip-image-load) SKIP_IMAGE_LOAD=true; shift ;; --teardown) TEARDOWN=true; shift ;; *) echo "Unknown flag: $1" >&2; exit 1 ;; esac @@ -59,7 +65,7 @@ fi # Check for existing cluster if y-k3d cluster list 2>/dev/null | grep -q "^$K3D_NAME "; then - echo "ERROR: k3d cluster '$K3D_NAME' already exists. Delete it first with: y-k3d cluster delete $K3D_NAME" >&2 + echo "ERROR: k3d cluster '$K3D_NAME' already exists. Delete it first with: y-cluster-provision-k3d --teardown" >&2 exit 1 fi @@ -110,27 +116,28 @@ sed -e 's/name: k3d-ystack/name: ystack-k3d/g' \ -e 's/user: admin@k3d-ystack/user: ystack-k3d/g' "$KUBECONFIG" > "$KUBECONFIG.tmp" \ && mv "$KUBECONFIG.tmp" "$KUBECONFIG" +echo "# Waiting for API server to be ready ..." +until kubectl --context=$CTX get nodes >/dev/null 2>&1; do sleep 2; done + if [ "$SKIP_CONVERGE" = "true" ]; then echo "# --skip-converge: skipping converge, validate, and post-provision steps" exit 0 fi -echo "# Saving ystack images to local cache ..." -y-image-cache-ystack > /etc/hosts" docker exec k3d-ystack-server-0 sh -cex "echo '$PROD_REGISTRY_IP prod-registry.ystack.svc.cluster.local' >> /etc/hosts" - -echo "# Checking /etc/hosts ..." -if ! y-k8s-ingress-hosts --context=$CTX -check -override-ip "${YSTACK_PORTS_IP:-127.0.0.1}"; then - echo "# Updating /etc/hosts (requires sudo) ..." - y-k8s-ingress-hosts --context=$CTX -write -override-ip "${YSTACK_PORTS_IP:-127.0.0.1}" -fi diff --git a/bin/y-cluster-provision-lima b/bin/y-cluster-provision-lima index 38f45662..9e02a154 100755 --- a/bin/y-cluster-provision-lima +++ b/bin/y-cluster-provision-lima @@ -8,6 +8,8 @@ YSTACK_HOME="$(cd "$(dirname "$0")/.." && pwd)" CTX=local SKIP_CONVERGE=false +SKIP_IMAGE_LOAD=false +EXCLUDE=monitoring while [ $# -gt 0 ]; do case "$1" in -h|--help) @@ -16,13 +18,19 @@ Usage: y-cluster-provision-lima [flags] Flags: --context=NAME kubeconfig context name (default: local) + --exclude=SUBSTRING exclude k3s bases matching substring (default: monitoring) --skip-converge skip converge, validate, and post-provision steps + --skip-image-load skip image cache and load into containerd --teardown delete existing VM and exit + --teardown-prune also remove cached OS images (limactl prune) -h, --help show this help EOF exit 0 ;; --context=*) CTX="${1#*=}"; shift ;; + --exclude=*) EXCLUDE="${1#*=}"; shift ;; --skip-converge) SKIP_CONVERGE=true; shift ;; + --skip-image-load) SKIP_IMAGE_LOAD=true; shift ;; + --teardown-prune) TEARDOWN=true; TEARDOWN_PRUNE=true; shift ;; --teardown) TEARDOWN=true; shift ;; *) echo "Unknown flag: $1" >&2; exit 1 ;; esac @@ -35,10 +43,10 @@ command -v limactl >/dev/null 2>&1 || { echo "ERROR: limactl not found in PATH" if [ "$TEARDOWN" = "true" ]; then if limactl list 2>/dev/null | grep -q "^ystack "; then limactl delete -f ystack - limactl prune + [ "$TEARDOWN_PRUNE" = "true" ] && limactl prune kubectl config delete-context $CTX 2>/dev/null || true else - echo "# No Lima VM 'ystack' found" + echo "[y-cluster-provision-lima] No Lima VM 'ystack' found" fi exit 0 fi @@ -63,7 +71,7 @@ TOPOLOGY_ZONE="local" # Place airgap tarball before k3s starts AIRGAP_TAR=$(y-k3s-airgap-download) if [ -f "$AIRGAP_TAR" ]; then - echo "# Placing airgap tarball into VM ..." + echo "[y-cluster-provision-lima] Placing airgap tarball into VM" limactl shell ystack sudo mkdir -p /var/lib/rancher/k3s/agent/images limactl shell ystack sudo cp "$AIRGAP_TAR" /var/lib/rancher/k3s/agent/images/ fi @@ -84,46 +92,43 @@ k() { } until k -n kube-system get pods 2>/dev/null; do - echo "==> Waiting for the cluster respond ..." + echo "[y-cluster-provision-lima] Waiting for the cluster to respond" sleep 1 done until k -n kube-system get serviceaccount default 2>/dev/null; do - echo "==> Waiting for the default service account to exist ..." + echo "[y-cluster-provision-lima] Waiting for the default service account to exist" sleep 1 done if [ "$SKIP_CONVERGE" = "true" ]; then - echo "# --skip-converge: skipping converge, validate, and post-provision steps" + echo "[y-cluster-provision-lima] --skip-converge: skipping converge, validate, and post-provision steps" y-kubeconfig-import "$KUBECONFIG.tmp" exit 0 fi -echo "==> Testing amd64 compatibility ..." -k run amd64test --image=gcr.io/google_containers/pause-amd64:3.2@sha256:4a1c4b21597c1b4415bdbecb28a3296c6b5e23ca4f9feeb599860a1dac6a0108 -while k get pod amd64test -o=jsonpath='{.status.containerStatuses[0]}' | grep -v '"started":true'; do sleep 3; done -k delete --wait=false pod amd64test +# echo "==> Testing amd64 compatibility ..." +# k run amd64test --image=gcr.io/google_containers/pause-amd64:3.2@sha256:4a1c4b21597c1b4415bdbecb28a3296c6b5e23ca4f9feeb599860a1dac6a0108 +# while k get pod amd64test -o=jsonpath='{.status.containerStatuses[0]}' | grep -v '"started":true'; do sleep 3; done +# k delete --wait=false pod amd64test # Import kubeconfig before cache-load and converge (y-kubeconfig-import moves the .tmp file) y-kubeconfig-import "$KUBECONFIG.tmp" -echo "# Saving ystack images to local cache ..." -y-image-cache-ystack > /etc/hosts" limactl shell ystack sudo sh -c "echo '$PROD_REGISTRY_IP prod-registry.ystack.svc.cluster.local' >> /etc/hosts" - -echo "# Checking /etc/hosts ..." -if ! y-k8s-ingress-hosts --context=$CTX -check -override-ip 127.0.0.1; then - echo "# Updating /etc/hosts (requires sudo) ..." - y-k8s-ingress-hosts --context=$CTX -write -override-ip 127.0.0.1 -fi diff --git a/bin/y-cluster-provision-multipass b/bin/y-cluster-provision-multipass index bf9add02..9e93dcac 100755 --- a/bin/y-cluster-provision-multipass +++ b/bin/y-cluster-provision-multipass @@ -10,6 +10,8 @@ CTX=local VM_NAME="ystack-master" VM_RESOURCES="-m 8G -d 40G -c 4" SKIP_CONVERGE=false +SKIP_IMAGE_LOAD=false +EXCLUDE=monitoring while [ $# -gt 0 ]; do case "$1" in -h|--help) @@ -20,7 +22,9 @@ Flags: --context=NAME kubeconfig context name (default: local) --vm-name=NAME multipass VM name (default: ystack-master) --vm-resources=ARGS multipass launch resource flags (default: -m 8G -d 40G -c 4) + --exclude=SUBSTRING exclude k3s bases matching substring (default: monitoring) --skip-converge skip converge, validate, and post-provision steps + --skip-image-load skip image cache and load into containerd --teardown delete existing VM and exit -h, --help show this help EOF @@ -28,7 +32,9 @@ EOF --context=*) CTX="${1#*=}"; shift ;; --vm-name=*) VM_NAME="${1#*=}"; shift ;; --vm-resources=*) VM_RESOURCES="${1#*=}"; shift ;; + --exclude=*) EXCLUDE="${1#*=}"; shift ;; --skip-converge) SKIP_CONVERGE=true; shift ;; + --skip-image-load) SKIP_IMAGE_LOAD=true; shift ;; --teardown) TEARDOWN=true; shift ;; *) echo "Unknown flag: $1" >&2; exit 1 ;; esac @@ -102,14 +108,17 @@ if [ "$SKIP_CONVERGE" = "true" ]; then exit 0 fi -echo "# Saving ystack images to local cache ..." -y-image-cache-ystack > /etc/hosts" multipass exec "$VM_NAME" -- sudo sh -c "echo '$PROD_REGISTRY_IP prod-registry.ystack.svc.cluster.local' >> /etc/hosts" -echo "# Updating /etc/hosts (requires sudo) ..." -y-k8s-ingress-hosts --context=$CTX -write - echo "# Done. Master IP: $K3S_NODEIP_MASTER" diff --git a/bin/y-cluster-provision-qemu b/bin/y-cluster-provision-qemu new file mode 100755 index 00000000..1d60dc9b --- /dev/null +++ b/bin/y-cluster-provision-qemu @@ -0,0 +1,252 @@ +#!/usr/bin/env bash +[ -z "$DEBUG" ] || set -x +set -eo pipefail + +YSTACK_HOME="$(cd "$(dirname "$0")/.." && pwd)" + +[ -z "$KUBECONFIG" ] && echo "Provision requires an explicit KUBECONFIG env" && exit 1 + +CTX=local +VM_NAME="ystack-qemu" +VM_DISK="$HOME/.cache/ystack-qemu/$VM_NAME.qcow2" +VM_DISK_SIZE="40G" +VM_MEMORY="8192" +VM_CPUS="4" +VM_SSH_PORT="2222" +SKIP_CONVERGE=false +SKIP_IMAGE_LOAD=false +EXCLUDE=monitoring + +while [ $# -gt 0 ]; do + case "$1" in + -h|--help) + cat >&2 <&2; exit 1 ;; + esac +done + +VM_DIR="$(dirname "$VM_DISK")" +VM_PIDFILE="$VM_DIR/$VM_NAME.pid" +VM_SEED="$VM_DIR/$VM_NAME-seed.img" + +ssh_vm() { + ssh -i "$VM_SSH_KEY" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + -o LogLevel=ERROR -p "$VM_SSH_PORT" ystack@localhost "$@" +} + +scp_to_vm() { + scp -i "$VM_SSH_KEY" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + -o LogLevel=ERROR -P "$VM_SSH_PORT" "$1" "ystack@localhost:$2" +} + +# Verify prerequisites +MISSING="" +command -v qemu-system-x86_64 >/dev/null 2>&1 || MISSING="$MISSING qemu-system-x86" +command -v qemu-img >/dev/null 2>&1 || MISSING="$MISSING qemu-utils" +command -v cloud-localds >/dev/null 2>&1 || MISSING="$MISSING cloud-image-utils" +if [ -n "$MISSING" ]; then + echo "Missing packages:$MISSING" >&2 + echo "" >&2 + echo " sudo apt install qemu-system-x86 qemu-utils cloud-image-utils" >&2 + exit 1 +fi +if [ ! -e /dev/kvm ]; then + echo "ERROR: /dev/kvm not found — KVM not available on this machine" >&2 + exit 1 +fi +if ! id -nG | grep -qw kvm; then + echo "ERROR: $USER is not in the kvm group" >&2 + echo "" >&2 + echo " sudo usermod -aG kvm $USER" >&2 + echo " # then log out and back in, or: newgrp kvm" >&2 + exit 1 +fi + +# Export mode +if [ -n "$EXPORT_VMDK" ]; then + [ -f "$VM_DISK" ] || { echo "ERROR: VM disk $VM_DISK not found" >&2; exit 1; } + echo "[y-cluster-provision-qemu] Exporting $VM_DISK to $EXPORT_VMDK ..." + qemu-img convert -f qcow2 -O vmdk -o subformat=streamOptimized "$VM_DISK" "$EXPORT_VMDK" + echo "[y-cluster-provision-qemu] Exported: $EXPORT_VMDK" + exit 0 +fi + +# Teardown mode +if [ "$TEARDOWN" = "true" ]; then + if [ -f "$VM_PIDFILE" ]; then + PID=$(cat "$VM_PIDFILE") + if kill -0 "$PID" 2>/dev/null; then + echo "[y-cluster-provision-qemu] Stopping VM (pid $PID) ..." + kill "$PID" + sleep 2 + fi + rm -f "$VM_PIDFILE" + fi + kubectl config delete-context $CTX 2>/dev/null || true + echo "[y-cluster-provision-qemu] Teardown complete. Disk preserved at $VM_DISK" + exit 0 +fi + +# Check for running VM +if [ -f "$VM_PIDFILE" ] && kill -0 "$(cat "$VM_PIDFILE")" 2>/dev/null; then + echo "ERROR: VM already running (pid $(cat "$VM_PIDFILE")). Use --teardown first." >&2 + exit 1 +fi + +mkdir -p "$VM_DIR" + +# Download Ubuntu cloud image if not cached +UBUNTU_VERSION="noble" +CLOUD_IMG="$VM_DIR/ubuntu-${UBUNTU_VERSION}-server-cloudimg-amd64.img" +if [ ! -f "$CLOUD_IMG" ]; then + echo "[y-cluster-provision-qemu] Downloading Ubuntu $UBUNTU_VERSION cloud image ..." + curl -fSL -o "$CLOUD_IMG" \ + "https://cloud-images.ubuntu.com/${UBUNTU_VERSION}/current/${UBUNTU_VERSION}-server-cloudimg-amd64.img" +fi + +# Create VM disk from cloud image +if [ ! -f "$VM_DISK" ]; then + echo "[y-cluster-provision-qemu] Creating VM disk ($VM_DISK_SIZE) ..." + qemu-img create -f qcow2 -b "$CLOUD_IMG" -F qcow2 "$VM_DISK" "$VM_DISK_SIZE" +fi + +# Generate SSH key for VM access +VM_SSH_KEY="$VM_DIR/$VM_NAME-ssh" +if [ ! -f "$VM_SSH_KEY" ]; then + ssh-keygen -t ed25519 -f "$VM_SSH_KEY" -N "" -q +fi + +# Create cloud-init seed +SSH_PUB=$(cat "$VM_SSH_KEY.pub") +CLOUD_INIT="$VM_DIR/cloud-init.yaml" +cat > "$CLOUD_INIT" </dev/null && break + sleep 2 +done +ssh_vm true || { echo "ERROR: SSH not available after 120s" >&2; exit 1; } + +echo "[y-cluster-provision-qemu] VM ready, installing k3s ..." + +# Disable swap +ssh_vm "sudo swapoff -a" + +# Transfer and configure registry mirrors +REGISTRY_TMP=$(mktemp) +YSTACK_PROD_REGISTRY=$YSTACK_PROD_REGISTRY YSTACK_PROD_REGISTRY_REWRITE=$YSTACK_PROD_REGISTRY_REWRITE y-registry-config k3s-yaml > "$REGISTRY_TMP" +scp_to_vm "$REGISTRY_TMP" /tmp/registries.yaml +rm -f "$REGISTRY_TMP" +ssh_vm "sudo mkdir -p /etc/rancher/k3s && sudo mv /tmp/registries.yaml /etc/rancher/k3s/" + +# Transfer airgap images if available +AIRGAP_TAR=$(y-k3s-airgap-download) +if [ -f "$AIRGAP_TAR" ]; then + echo "[y-cluster-provision-qemu] Transferring airgap tarball ..." + scp_to_vm "$AIRGAP_TAR" /tmp/k3s-airgap.tar.zst + ssh_vm "sudo mkdir -p /var/lib/rancher/k3s/agent/images && sudo mv /tmp/k3s-airgap.tar.zst /var/lib/rancher/k3s/agent/images/" +fi + +# Install k3s +ssh_vm "sudo bash -cex '$(cat $YSTACK_HOME/bin/y-k3s-install)'" + +# Extract kubeconfig +ssh_vm "sudo cat /etc/rancher/k3s/k3s.yaml" \ + | sed "s|127.0.0.1|127.0.0.1|" \ + > "$KUBECONFIG.tmp" + +KUBECONFIG="$KUBECONFIG.tmp" kubectl config rename-context default $CTX + +# Set cluster name for y-cluster-local-detect +sed -i 's/name: default/name: ystack-qemu/g; s/cluster: default/cluster: ystack-qemu/g; s/user: default/user: ystack-qemu/g' "$KUBECONFIG.tmp" + +y-kubeconfig-import "$KUBECONFIG.tmp" + +if [ "$SKIP_CONVERGE" = "true" ]; then + echo "[y-cluster-provision-qemu] --skip-converge: done" + exit 0 +fi + +if [ "$SKIP_IMAGE_LOAD" = "true" ]; then + echo "[y-cluster-provision-qemu] --skip-image-load: skipping" +else + echo "[y-cluster-provision-qemu] Loading images ..." + y-image-cache-ystack > /etc/hosts'" +ssh_vm "sudo sh -c 'echo \"$PROD_REGISTRY_IP prod-registry.ystack.svc.cluster.local\" >> /etc/hosts'" + +echo "[y-cluster-provision-qemu] Done. SSH: ssh -p $VM_SSH_PORT -i $VM_SSH_KEY ystack@localhost" +echo "[y-cluster-provision-qemu] Export: y-cluster-provision-qemu --export-vmdk=appliance.vmdk" diff --git a/bin/y-cluster-sudoers b/bin/y-cluster-sudoers index 47696e62..29df93de 100755 --- a/bin/y-cluster-sudoers +++ b/bin/y-cluster-sudoers @@ -9,59 +9,78 @@ SUDOERS_FILE="/etc/sudoers.d/ystack-cluster" usage() { cat >&2 <&2; exit 1 ;; esac +done +[ ${#USERS[@]} -eq 0 ] && USERS=("${SUDO_USER:-$USER}") rules() { - cat <&1 | grep -q "sudo-rs" && [ -x /usr/bin/sudo.ws ]; then - echo "" - echo "WARNING: sudo-rs is the default sudo on this system." - echo "NOPASSWD rules may not work. To fix, switch to the original sudo:" - echo "" - echo " sudo update-alternatives --set sudo /usr/bin/sudo.ws" - echo "" + # sudo-rs (default on Ubuntu 25.10+) has a bug where NOPASSWD rules from + # /etc/sudoers.d/ are listed by "sudo -l" but not applied during execution. + # The original sudo (sudo.ws) handles these rules correctly. + if sudo --version 2>&1 | grep -q "sudo-rs"; then + echo "" >&2 + echo "WARNING: sudo-rs detected. NOPASSWD rules may not work." >&2 + echo "sudo-rs has a known bug where rules in /etc/sudoers.d/ are parsed" >&2 + echo "but not applied. Switch to the original sudo to fix:" >&2 + echo " sudo update-alternatives --set sudo /usr/bin/sudo.ws" >&2 fi } -if [ "$WRITE" = "true" ]; then +if [ "$MODE" = "write" ]; then if [ $(id -u) -ne 0 ]; then - echo "Writing sudoers rules requires root. Re-running with sudo ..." - exec sudo "$0" --write + if sudo -n true 2>/dev/null; then + echo "Writing sudoers rules requires root. Re-running with sudo ..." + exec sudo "$0" --write "${USERS[@]}" + else + echo "Can't sudo as $USER. A user with sudo rights can run for example:" >&2 + echo " sudo -u $USER $0 ${USERS[*]} | sudo tee $SUDOERS_FILE" >&2 + exit 1 + fi fi TMPFILE=$(mktemp) rules > "$TMPFILE" - # Validate before installing if visudo -cf "$TMPFILE"; then cp "$TMPFILE" "$SUDOERS_FILE" chmod 0440 "$SUDOERS_FILE" @@ -74,9 +93,6 @@ if [ "$WRITE" = "true" ]; then exit 1 fi else - echo "# Sudoers rules that would be written to $SUDOERS_FILE" - echo "# Run 'y-cluster-sudoers --write' to install them." - echo "" rules sudo_rs_check fi diff --git a/bin/y-cluster-validate-ystack b/bin/y-cluster-validate-ystack index ecff4692..df276b19 100755 --- a/bin/y-cluster-validate-ystack +++ b/bin/y-cluster-validate-ystack @@ -26,37 +26,31 @@ report() { local result="$2" if [ "$result" = "ok" ]; then PASS=$((PASS + 1)) - echo " PASS $name" + echo "[y-cluster-validate-ystack] PASS $name" else FAIL=$((FAIL + 1)) - echo " FAIL $name - $result" + echo "[y-cluster-validate-ystack] FAIL $name - $result" fi } -echo "=== Dev Cluster Validation: context=$CONTEXT ===" -echo "" +echo "[y-cluster-validate-ystack] Dev cluster validation: context=$CONTEXT" -# 1. Namespace k get ns ystack >/dev/null 2>&1 \ && report "namespace ystack" "ok" \ || report "namespace ystack" "not found" -# 2. Gateway API CRDs k get crd gateways.gateway.networking.k8s.io >/dev/null 2>&1 \ && report "gateway-api CRDs" "ok" \ || report "gateway-api CRDs" "not installed" -# 3. Gateway k -n ystack get gateway ystack >/dev/null 2>&1 \ && report "gateway ystack" "ok" \ || report "gateway ystack" "not found" -# 4. versitygw -ROLLOUT=$(k -n ystack rollout status deploy/versitygw --timeout=5s 2>&1) \ +ROLLOUT=$(k -n blobs rollout status deploy/versitygw --timeout=5s 2>&1) \ && report "versitygw rollout" "ok" \ || report "versitygw rollout" "$ROLLOUT" -# 5. builds-registry service + clusterIP CLUSTER_IP=$(k -n ystack get svc builds-registry -o=jsonpath='{.spec.clusterIP}' 2>/dev/null) if [ "$CLUSTER_IP" = "10.43.0.50" ]; then report "builds-registry clusterIP" "ok" @@ -64,12 +58,10 @@ else report "builds-registry clusterIP" "got '$CLUSTER_IP', expected 10.43.0.50" fi -# 6. HTTPRoute k -n ystack get httproute builds-registry >/dev/null 2>&1 \ && report "httproute builds-registry" "ok" \ || report "httproute builds-registry" "not found" -# 7. prod-registry service PROD_IP=$(k -n ystack get svc prod-registry -o=jsonpath='{.spec.clusterIP}' 2>/dev/null) if [ "$PROD_IP" = "10.43.0.51" ]; then report "prod-registry clusterIP" "ok" @@ -77,87 +69,51 @@ else report "prod-registry clusterIP" "got '$PROD_IP', expected 10.43.0.51" fi -# 7.5 Buildkitd GRPCRoute k -n ystack get grpcroute buildkitd >/dev/null 2>&1 \ && report "grpcroute buildkitd" "ok" \ || report "grpcroute buildkitd" "not found" -# 7.6 Monitoring stack k -n monitoring get prometheus now >/dev/null 2>&1 \ && report "prometheus now" "ok" \ || report "prometheus now" "not found" -# 7.7 Prometheus HTTPRoute k -n monitoring get httproute prometheus-now >/dev/null 2>&1 \ && report "httproute prometheus-now" "ok" \ || report "httproute prometheus-now" "not found" -# 8. buildkitd statefulset k -n ystack get statefulset buildkitd >/dev/null 2>&1 \ && report "buildkitd statefulset" "ok" \ || report "buildkitd statefulset" "not found" -# 9. Registry pod running (wait for rollout) -echo " .... waiting for registry rollout (up to 10s)" +echo "[y-cluster-validate-ystack] Waiting for registry rollout (up to 10s)" ROLLOUT_REG=$(k -n ystack rollout status deploy/registry --timeout=10s 2>&1) \ && report "registry rollout" "ok" \ || report "registry rollout" "$ROLLOUT_REG" -# 10. bucket-create job completed -echo " .... waiting for bucket-create job (up to 10s)" -k -n ystack wait --for=condition=complete job/bucket-create-ystack-builds --timeout=10s >/dev/null 2>&1 \ - && report "bucket-create job" "ok" \ - || report "bucket-create job" "not complete within 10s" - -# 11. In-cluster registry access via crane port-forward -echo "" -echo "--- Registry access (in-cluster curl) ---" -REGISTRY_POD=$(k -n ystack get pod -l ystack-builds-registry=http -o=jsonpath='{.items[0].metadata.name}' 2>/dev/null) -if [ -n "$REGISTRY_POD" ]; then - CATALOG=$(k -n ystack exec "$REGISTRY_POD" -- wget -q -O- http://localhost/v2/_catalog 2>/dev/null) - if echo "$CATALOG" | grep -q "repositories"; then - report "in-cluster registry v2 API" "ok" - else - report "in-cluster registry v2 API" "no response" - fi -else - report "in-cluster registry v2 API" "no registry pod" -fi +REGISTRY_HOST="builds-registry.ystack.svc.cluster.local" +echo "[y-cluster-validate-ystack] Registry access" +CATALOG=$(curl -sf --retry 20 --retry-delay 2 --retry-all-errors --connect-timeout 2 http://$REGISTRY_HOST/v2/_catalog 2>/dev/null) && \ + echo "$CATALOG" | grep -q "repositories" \ + && report "registry v2 API" "ok" \ + || report "registry v2 API" "no response" -# 12. Build and deploy example app -echo "" -echo "--- Example app build + deploy (y-skaffold run) ---" -EXAMPLE_DIR="$YSTACK_HOME/examples/basic-dev-inner-loop" -EXAMPLE_NS=default +echo "[y-cluster-validate-ystack] Build + deploy (y-build)" +EXAMPLE_DIR="$YSTACK_HOME/examples/y-build" +VALIDATE_IMAGE="$REGISTRY_HOST/ystack-validate/y-build-test:latest" y-buildkitd-available --context="$CONTEXT" 2>&1 || true -echo " .... building and deploying example app" -if (cd "$EXAMPLE_DIR" && SKAFFOLD_DEFAULT_REPO= y-skaffold run --cache-artifacts=false --kube-context="$CONTEXT" -n "$EXAMPLE_NS"); then - report "y-skaffold run" "ok" - - # Apply gateway route for the example and wait for traefik to reconcile - k -n "$EXAMPLE_NS" apply -k "$EXAMPLE_DIR/k8s/gatewayapi/" >/dev/null 2>&1 - echo " .... waiting for gateway route to propagate" - for i in 1 2 3 4 5; do - RESPONSE=$(curl -s --connect-timeout 10 -H "Host: node-backend.ystack.svc.cluster.local" "http://builds-registry.ystack.svc.cluster.local/" 2>/dev/null) - [ "$RESPONSE" = "Hello World!" ] && break - sleep 2 - done - if [ "$RESPONSE" = "Hello World!" ]; then - report "example app response" "ok" - else - report "example app response" "got '$RESPONSE'" - fi - - # Clean up - echo " .... cleaning up example app" - k -n "$EXAMPLE_NS" delete -k "$EXAMPLE_DIR/k8s/gatewayapi/" --ignore-not-found >/dev/null 2>&1 - (cd "$EXAMPLE_DIR" && y-skaffold delete --kube-context="$CONTEXT" -n "$EXAMPLE_NS") >/dev/null 2>&1 +echo "[y-cluster-validate-ystack] Building example image" +if BUILD_CONTEXT="$EXAMPLE_DIR" IMAGE="$VALIDATE_IMAGE" IMPORT_CACHE=false EXPORT_CACHE=false y-build; then + report "y-build" "ok" + + curl -sSf --retry 20 --retry-delay 2 --retry-all-errors --connect-timeout 2 \ + "http://$REGISTRY_HOST/v2/ystack-validate/y-build-test/tags/list" | grep -q '"latest"' \ + && report "y-build-test pushed" "ok" \ + || report "y-build-test pushed" "image not found in registry" else - report "y-skaffold run" "build or deploy failed" + report "y-build" "build failed" fi -echo "" -echo "=== Results: $PASS passed, $FAIL failed ===" +echo "[y-cluster-validate-ystack] Results: $PASS passed, $FAIL failed" if [ "$FAIL" -gt 0 ]; then exit 1 diff --git a/bin/y-image-list-ystack b/bin/y-image-list-ystack index e837f185..c15a13fa 100755 --- a/bin/y-image-list-ystack +++ b/bin/y-image-list-ystack @@ -4,8 +4,8 @@ set -eo pipefail YSTACK_HOME="$(cd "$(dirname "$0")/.." && pwd)" -# Converge bases from y-cluster-converge-ystack -BASES=$(grep 'apply_base ' "$YSTACK_HOME/bin/y-cluster-converge-ystack" | sed 's/apply_base //') +# Converge bases from y-cluster-converge-ystack BASES array +BASES=$(sed -n '/^BASES=(/,/^)/{ /^BASES=(/d; /^)/d; s/^[[:space:]]*//; p; }' "$YSTACK_HOME/bin/y-cluster-converge-ystack") for base in $BASES; do kubectl kustomize "$YSTACK_HOME/k3s/$base/" 2>/dev/null \ | grep -oE 'image:\s*\S+' \ diff --git a/bin/y-k8s-ingress-hosts b/bin/y-k8s-ingress-hosts index 54f61181..b10529cc 100755 --- a/bin/y-k8s-ingress-hosts +++ b/bin/y-k8s-ingress-hosts @@ -7,6 +7,8 @@ YBIN="$(dirname $0)" CTX="" CHECK=false +ENSURE=false +EXPLICIT_OVERRIDE_IP="" PASSTHROUGH=() while [ $# -gt 0 ]; do @@ -19,12 +21,20 @@ Flags: --context=NAME kubeconfig context name (required) -write rewrite host file -check|--check check if /etc/hosts includes required entries (no sudo) - -override-ip=IP use this IP for all entries + --ensure check, then write if needed (combines -check and -write) + -override-ip=IP use this IP for all entries (overrides gateway annotation) + -override-ip IP use this IP for all entries (overrides gateway annotation) -h, --help show this help + +If no -override-ip is given, reads yolean.se/override-ip annotation from +the ystack gateway in ystack namespace. EOF exit 0 ;; --context=*) CTX="${1#*=}"; shift ;; -check|--check) CHECK=true; shift ;; + --ensure) ENSURE=true; shift ;; + -override-ip=*) EXPLICIT_OVERRIDE_IP="${1#*=}"; shift ;; + -override-ip) EXPLICIT_OVERRIDE_IP="$2"; shift; shift ;; *) PASSTHROUGH+=("$1"); shift ;; esac done @@ -33,29 +43,52 @@ done CONTEXT_KUBECONFIG=$(mktemp) trap "rm -f $CONTEXT_KUBECONFIG" EXIT -kubectl config view --raw --minify --context="$CTX" > "$CONTEXT_KUBECONFIG" +kubectl config view --raw --minify --context="$CTX" --request-timeout=5s > "$CONTEXT_KUBECONFIG" + +# Resolve override IP: explicit flag > gateway annotation +OVERRIDE_IP="$EXPLICIT_OVERRIDE_IP" +if [ -z "$OVERRIDE_IP" ]; then + OVERRIDE_IP=$(kubectl --context="$CTX" --request-timeout=5s -n ystack get gateway ystack \ + -o jsonpath='{.metadata.annotations.yolean\.se/override-ip}' 2>/dev/null || true) + if [ -n "$OVERRIDE_IP" ]; then + echo "# Using override-ip=$OVERRIDE_IP from gateway annotation" + fi +fi +if [ -n "$OVERRIDE_IP" ]; then + PASSTHROUGH+=("-override-ip" "$OVERRIDE_IP") +fi version=$(y-bin-download $YBIN/y-bin.optional.yaml k8s-ingress-hosts) -if $CHECK; then +if $CHECK || $ENSURE; then NEEDED=$($YBIN/y-k8s-ingress-hosts-v${version}-bin -kubeconfig "$CONTEXT_KUBECONFIG" "${PASSTHROUGH[@]}" 2>/dev/null | grep -v '^#') - MISSING=0 + STALE=0 while IFS= read -r line; do [ -z "$line" ] && continue + EXPECTED_IP=$(echo "$line" | awk '{print $1}') HOST=$(echo "$line" | awk '{print $2}') - if ! grep -q "$HOST" /etc/hosts; then + ACTUAL=$(grep -E "^[^#]*[[:space:]]$HOST([[:space:]]|$)" /etc/hosts 2>/dev/null || true) + if [ -z "$ACTUAL" ]; then echo "Missing: $line" - MISSING=1 + STALE=1 + elif ! echo "$ACTUAL" | grep -qE "^[[:space:]]*$EXPECTED_IP[[:space:]]"; then + ACTUAL_IP=$(echo "$ACTUAL" | awk '{print $1}') + echo "Stale: $HOST has $ACTUAL_IP, expected $EXPECTED_IP" + STALE=1 fi done <<< "$NEEDED" - if [ $MISSING -eq 1 ]; then - echo "# /etc/hosts needs updating. Run with -write to fix." + if [ $STALE -eq 0 ]; then + echo "# /etc/hosts is up to date" + exit 0 + fi + if ! $ENSURE; then + echo "# /etc/hosts needs updating. Run with -write or --ensure to fix." exit 1 fi - echo "# /etc/hosts is up to date" - exit 0 + echo "# /etc/hosts needs updating, writing ..." + PASSTHROUGH+=("-write") fi -[ $(id -u) -ne 0 ] && exec sudo -E $YBIN/y-k8s-ingress-hosts-v${version}-bin -kubeconfig "$CONTEXT_KUBECONFIG" "${PASSTHROUGH[@]}" +[ $(id -u) -ne 0 ] && exec sudo $YBIN/y-k8s-ingress-hosts-v${version}-bin -kubeconfig "$CONTEXT_KUBECONFIG" "${PASSTHROUGH[@]}" $YBIN/y-k8s-ingress-hosts-v${version}-bin -kubeconfig "$CONTEXT_KUBECONFIG" "${PASSTHROUGH[@]}" || exit $? diff --git a/bin/y-kubefwd b/bin/y-kubefwd index 535f7c1d..14091734 100755 --- a/bin/y-kubefwd +++ b/bin/y-kubefwd @@ -9,6 +9,8 @@ case $ctx in *) echo "Initial arg must be --context=" && exit 1 ;; esac +CONTEXT_NAME="${ctx#--context=}" + [ "$YSTACK_BUILDKIT_REQUIRE" != "true" ] || [ $(id -u) -eq 0 ] || [ -z "$ctx" ] || y-buildkitd-available $ctx version=$(y-bin-download $YBIN/y-bin.optional.yaml kubefwd) @@ -27,5 +29,6 @@ fi addargs="$ctx" [[ "$*" == *-l* ]] || addargs="$addargs -l ystack-kubefwd!=never" +[[ "$CONTEXT_NAME" == "local" ]] || [[ "$*" == *--domain* ]] || addargs="$addargs --domain=$CONTEXT_NAME" $YBIN/y-kubefwd-v${version}-bin $addargs "$@" || exit $? diff --git a/bin/y-localhost b/bin/y-localhost index 2692eccb..26c599a4 100755 --- a/bin/y-localhost +++ b/bin/y-localhost @@ -23,13 +23,13 @@ case "$platform" in Darwin) ifconfig lo0 | grep $ip > /dev/null && exit 0 echo "Loopback alias for $ip $hostname not found. Will try to create ..." - [ $(id -u) -eq 0 ] || exec sudo -E $0 "$@" + [ $(id -u) -eq 0 ] || exec sudo $0 "$@" ifconfig lo0 alias $ip up ;; Linux) if [ -z "$(ip addr show dev lo label lo:$num)" ]; then echo "Adding missing interface lo:$num ..." - [ $(id -u) -eq 0 ] || exec sudo -E $0 "$@" + [ $(id -u) -eq 0 ] || exec sudo $0 "$@" ip addr add $ip/32 dev lo label lo:$num fi ;; diff --git a/blobs-minio/common/kustomization.yaml b/blobs-minio/common/kustomization.yaml new file mode 100644 index 00000000..ffddbcb6 --- /dev/null +++ b/blobs-minio/common/kustomization.yaml @@ -0,0 +1,3 @@ +# yaml-language-server: $schema=https://json.schemastore.org/kustomization.json +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization diff --git a/minio/defaultsecret/kustomization.yaml b/blobs-minio/defaultsecret/kustomization.yaml similarity index 51% rename from minio/defaultsecret/kustomization.yaml rename to blobs-minio/defaultsecret/kustomization.yaml index 114baaaa..b4544d0e 100644 --- a/minio/defaultsecret/kustomization.yaml +++ b/blobs-minio/defaultsecret/kustomization.yaml @@ -1,3 +1,7 @@ +# yaml-language-server: $schema=https://json.schemastore.org/kustomization.json +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: blobs generatorOptions: disableNameSuffixHash: true secretGenerator: diff --git a/blobs-minio/standalone,defaultsecret/kustomization.yaml b/blobs-minio/standalone,defaultsecret/kustomization.yaml new file mode 100644 index 00000000..40a968a5 --- /dev/null +++ b/blobs-minio/standalone,defaultsecret/kustomization.yaml @@ -0,0 +1,6 @@ +# yaml-language-server: $schema=https://json.schemastore.org/kustomization.json +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- ../defaultsecret +- ../standalone diff --git a/minio/standalone/deployment.yaml b/blobs-minio/standalone/deployment.yaml similarity index 100% rename from minio/standalone/deployment.yaml rename to blobs-minio/standalone/deployment.yaml diff --git a/blobs-minio/standalone/kustomization.yaml b/blobs-minio/standalone/kustomization.yaml new file mode 100644 index 00000000..9d5c044b --- /dev/null +++ b/blobs-minio/standalone/kustomization.yaml @@ -0,0 +1,7 @@ +# yaml-language-server: $schema=https://json.schemastore.org/kustomization.json +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: blobs +resources: +- deployment.yaml +- pvc.yaml diff --git a/minio/standalone/pvc.yaml b/blobs-minio/standalone/pvc.yaml similarity index 100% rename from minio/standalone/pvc.yaml rename to blobs-minio/standalone/pvc.yaml diff --git a/blobs-versitygw/defaultsecret/kustomization.yaml b/blobs-versitygw/defaultsecret/kustomization.yaml new file mode 100644 index 00000000..02ea822d --- /dev/null +++ b/blobs-versitygw/defaultsecret/kustomization.yaml @@ -0,0 +1,11 @@ +# yaml-language-server: $schema=https://json.schemastore.org/kustomization.json +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: blobs +secretGenerator: +- name: versitygw-server + options: + disableNameSuffixHash: true + literals: + - root-accesskey=YstackEXAMPLEKEY + - root-secretkey=github.com/Yolean/ystack-EXAMPLE diff --git a/blobs-versitygw/standalone,defaultsecret/kustomization.yaml b/blobs-versitygw/standalone,defaultsecret/kustomization.yaml new file mode 100644 index 00000000..40a968a5 --- /dev/null +++ b/blobs-versitygw/standalone,defaultsecret/kustomization.yaml @@ -0,0 +1,6 @@ +# yaml-language-server: $schema=https://json.schemastore.org/kustomization.json +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- ../defaultsecret +- ../standalone diff --git a/versitygw/standalone/deployment.yaml b/blobs-versitygw/standalone/deployment.yaml similarity index 87% rename from versitygw/standalone/deployment.yaml rename to blobs-versitygw/standalone/deployment.yaml index 44d3dcd1..96b0d9cc 100644 --- a/versitygw/standalone/deployment.yaml +++ b/blobs-versitygw/standalone/deployment.yaml @@ -24,13 +24,13 @@ spec: - name: ROOT_ACCESS_KEY_ID valueFrom: secretKeyRef: - name: minio - key: accesskey + name: versitygw-server + key: root-accesskey - name: ROOT_SECRET_ACCESS_KEY valueFrom: secretKeyRef: - name: minio - key: secretkey + name: versitygw-server + key: root-secretkey ports: - containerPort: 7070 name: s3 diff --git a/blobs-versitygw/standalone/kustomization.yaml b/blobs-versitygw/standalone/kustomization.yaml new file mode 100644 index 00000000..9d5c044b --- /dev/null +++ b/blobs-versitygw/standalone/kustomization.yaml @@ -0,0 +1,7 @@ +# yaml-language-server: $schema=https://json.schemastore.org/kustomization.json +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: blobs +resources: +- deployment.yaml +- pvc.yaml diff --git a/versitygw/standalone/pvc.yaml b/blobs-versitygw/standalone/pvc.yaml similarity index 100% rename from versitygw/standalone/pvc.yaml rename to blobs-versitygw/standalone/pvc.yaml diff --git a/blobs-versitygw/y-kustomize/kustomization.yaml b/blobs-versitygw/y-kustomize/kustomization.yaml new file mode 100644 index 00000000..95aff7f4 --- /dev/null +++ b/blobs-versitygw/y-kustomize/kustomization.yaml @@ -0,0 +1,12 @@ +# yaml-language-server: $schema=https://json.schemastore.org/kustomization.json +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: ystack +secretGenerator: +- name: y-kustomize.blobs.setup-bucket-job + options: + disableNameSuffixHash: true + labels: + yolean.se/module-part: config + files: + - base-for-annotations.yaml=y-kustomize-bases/blobs/setup-bucket-job/base-for-annotations.yaml diff --git a/blobs-versitygw/y-kustomize/y-kustomize-bases/blobs/setup-bucket-job/base-for-annotations.yaml b/blobs-versitygw/y-kustomize/y-kustomize-bases/blobs/setup-bucket-job/base-for-annotations.yaml new file mode 100644 index 00000000..8735fb90 --- /dev/null +++ b/blobs-versitygw/y-kustomize/y-kustomize-bases/blobs/setup-bucket-job/base-for-annotations.yaml @@ -0,0 +1,51 @@ +apiVersion: v1 +kind: Secret +metadata: + name: bucket +stringData: + endpoint: http://y-s3-api.blobs.svc.cluster.local + accesskey: YstackEXAMPLEKEY + secretkey: github.com/Yolean/ystack-EXAMPLE +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: setup-bucket + labels: + yolean.se/converge-mode: replace +spec: + template: + metadata: + annotations: + yolean.se/bucket-name: "" + spec: + containers: + - name: mc + image: minio/mc:RELEASE.2025-08-13T08-35-41Z + env: + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: versitygw-server + key: root-accesskey + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: versitygw-server + key: root-secretkey + - name: BUCKET_NAME + valueFrom: + fieldRef: + fieldPath: metadata.annotations['yolean.se/bucket-name'] + - name: S3_ENDPOINT + value: http://y-s3-api.blobs.svc.cluster.local + command: + - sh + - -ce + - | + until mc alias set s3 $S3_ENDPOINT $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY 2>/dev/null; do + sleep 2 + done + mc mb --ignore-existing s3/$BUCKET_NAME + restartPolicy: Never + backoffLimit: 10 diff --git a/blobs/minio/kustomization.yaml b/blobs/minio/kustomization.yaml new file mode 100644 index 00000000..93940ff1 --- /dev/null +++ b/blobs/minio/kustomization.yaml @@ -0,0 +1,8 @@ +# yaml-language-server: $schema=https://json.schemastore.org/kustomization.json +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: blobs + +resources: +- y-s3-api-service.yaml diff --git a/minio/common/blobs-minio-service.yaml b/blobs/minio/y-s3-api-service.yaml similarity index 74% rename from minio/common/blobs-minio-service.yaml rename to blobs/minio/y-s3-api-service.yaml index 55388f44..01f01d25 100644 --- a/minio/common/blobs-minio-service.yaml +++ b/blobs/minio/y-s3-api-service.yaml @@ -1,9 +1,7 @@ apiVersion: v1 kind: Service metadata: - name: blobs-minio - labels: - app: minio + name: y-s3-api spec: selector: app: minio diff --git a/blobs/versitygw/kustomization.yaml b/blobs/versitygw/kustomization.yaml new file mode 100644 index 00000000..93940ff1 --- /dev/null +++ b/blobs/versitygw/kustomization.yaml @@ -0,0 +1,8 @@ +# yaml-language-server: $schema=https://json.schemastore.org/kustomization.json +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: blobs + +resources: +- y-s3-api-service.yaml diff --git a/versitygw/common/blobs-versitygw-service.yaml b/blobs/versitygw/y-s3-api-service.yaml similarity index 71% rename from versitygw/common/blobs-versitygw-service.yaml rename to blobs/versitygw/y-s3-api-service.yaml index 71e6468f..ed5e7148 100644 --- a/versitygw/common/blobs-versitygw-service.yaml +++ b/blobs/versitygw/y-s3-api-service.yaml @@ -1,9 +1,7 @@ apiVersion: v1 kind: Service metadata: - name: blobs-versitygw - labels: - app: versitygw + name: y-s3-api spec: selector: app: versitygw diff --git a/buildkit/gateway-proxy/deployment.yaml b/buildkit/gateway-proxy/deployment.yaml deleted file mode 100644 index aa5fb975..00000000 --- a/buildkit/gateway-proxy/deployment.yaml +++ /dev/null @@ -1,29 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: buildkitd-gateway-proxy -spec: - replicas: 1 - selector: - matchLabels: - app: buildkitd-gateway-proxy - template: - metadata: - labels: - app: buildkitd-gateway-proxy - spec: - containers: - - name: envoy - image: ghcr.io/yolean/envoy:distroless-v1.37.0 - args: - - -c - - /etc/envoy/envoy.yaml - ports: - - containerPort: 8547 - volumeMounts: - - name: config - mountPath: /etc/envoy - volumes: - - name: config - configMap: - name: buildkitd-gateway-proxy diff --git a/buildkit/gateway-proxy/envoy.yaml b/buildkit/gateway-proxy/envoy.yaml deleted file mode 100644 index 61d37901..00000000 --- a/buildkit/gateway-proxy/envoy.yaml +++ /dev/null @@ -1,54 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: buildkitd-gateway-proxy -data: - envoy.yaml: | - static_resources: - listeners: - - name: buildkitd - address: - socket_address: - address: 0.0.0.0 - port_value: 8547 - filter_chains: - - filters: - - name: envoy.filters.network.http_connection_manager - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager - codec_type: HTTP2 - stat_prefix: buildkitd - route_config: - name: local_route - virtual_hosts: - - name: gateway - domains: - - "*" - routes: - - match: - prefix: "/" - route: - cluster: gateway - timeout: 3600s - http_filters: - - name: envoy.filters.http.router - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router - clusters: - - name: gateway - type: STRICT_DNS - lb_policy: ROUND_ROBIN - typed_extension_protocol_options: - envoy.extensions.upstreams.http.v3.HttpProtocolOptions: - "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions - explicit_http_config: - http2_protocol_options: {} - load_assignment: - cluster_name: gateway - endpoints: - - lb_endpoints: - - endpoint: - address: - socket_address: - address: traefik.kube-system.svc.cluster.local - port_value: 8000 diff --git a/buildkit/gateway-proxy/kustomization.yaml b/buildkit/gateway-proxy/kustomization.yaml deleted file mode 100644 index 3cb271a0..00000000 --- a/buildkit/gateway-proxy/kustomization.yaml +++ /dev/null @@ -1,4 +0,0 @@ -resources: -- envoy.yaml -- deployment.yaml -- service.yaml diff --git a/buildkit/gateway-proxy/service.yaml b/buildkit/gateway-proxy/service.yaml deleted file mode 100644 index ccd9568c..00000000 --- a/buildkit/gateway-proxy/service.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: buildkitd-gateway -spec: - ports: - - port: 8547 - protocol: TCP - selector: - app: buildkitd-gateway-proxy diff --git a/buildkit/grpcroute/grpcroute.yaml b/buildkit/gateway/grpcroute.yaml similarity index 84% rename from buildkit/grpcroute/grpcroute.yaml rename to buildkit/gateway/grpcroute.yaml index 9471bb17..83df9b71 100644 --- a/buildkit/grpcroute/grpcroute.yaml +++ b/buildkit/gateway/grpcroute.yaml @@ -2,6 +2,8 @@ apiVersion: gateway.networking.k8s.io/v1 kind: GRPCRoute metadata: name: buildkitd + labels: + yolean.se/module-part: gateway spec: parentRefs: - name: ystack diff --git a/buildkit/gateway/kustomization.yaml b/buildkit/gateway/kustomization.yaml new file mode 100644 index 00000000..71ac98d8 --- /dev/null +++ b/buildkit/gateway/kustomization.yaml @@ -0,0 +1,6 @@ +# yaml-language-server: $schema=https://json.schemastore.org/kustomization.json +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: ystack +resources: +- grpcroute.yaml diff --git a/buildkit/grpcroute/kustomization.yaml b/buildkit/grpcroute/kustomization.yaml deleted file mode 100644 index f61f72e7..00000000 --- a/buildkit/grpcroute/kustomization.yaml +++ /dev/null @@ -1,2 +0,0 @@ -resources: -- grpcroute.yaml diff --git a/buildkit/kustomization.yaml b/buildkit/kustomization.yaml index 606baa65..295d6b18 100644 --- a/buildkit/kustomization.yaml +++ b/buildkit/kustomization.yaml @@ -1,5 +1,7 @@ +# yaml-language-server: $schema=https://json.schemastore.org/kustomization.json apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization +namespace: ystack images: - name: moby/buildkit:rootless diff --git a/docker/kustomization.yaml b/docker/kustomization.yaml index f5cc8cf4..6168e84e 100644 --- a/docker/kustomization.yaml +++ b/docker/kustomization.yaml @@ -1,3 +1,4 @@ +# yaml-language-server: $schema=https://json.schemastore.org/kustomization.json apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization diff --git a/examples/basic-dev-inner-loop/k8s/gatewayapi/kustomization.yaml b/examples/basic-dev-inner-loop/k8s/gatewayapi/kustomization.yaml index 0164dfb9..25c7af17 100644 --- a/examples/basic-dev-inner-loop/k8s/gatewayapi/kustomization.yaml +++ b/examples/basic-dev-inner-loop/k8s/gatewayapi/kustomization.yaml @@ -1,3 +1,4 @@ +# yaml-language-server: $schema=https://json.schemastore.org/kustomization.json apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization diff --git a/examples/y-build/Dockerfile b/examples/y-build/Dockerfile new file mode 100644 index 00000000..e242868c --- /dev/null +++ b/examples/y-build/Dockerfile @@ -0,0 +1,2 @@ +FROM ghcr.io/yolean/static-web-server:2.41.0 +COPY index.html /public/index.html diff --git a/examples/y-build/index.html b/examples/y-build/index.html new file mode 100644 index 00000000..13109c02 --- /dev/null +++ b/examples/y-build/index.html @@ -0,0 +1 @@ +{"status":"ok"} diff --git a/k3s/06-gateway/gateway.yaml b/gateway/gateway.yaml similarity index 87% rename from k3s/06-gateway/gateway.yaml rename to gateway/gateway.yaml index d22ed284..f924565e 100644 --- a/k3s/06-gateway/gateway.yaml +++ b/gateway/gateway.yaml @@ -2,6 +2,8 @@ apiVersion: gateway.networking.k8s.io/v1 kind: Gateway metadata: name: ystack + labels: + yolean.se/module-part: gateway spec: gatewayClassName: traefik listeners: diff --git a/gateway/kustomization.yaml b/gateway/kustomization.yaml new file mode 100644 index 00000000..3a50ffbf --- /dev/null +++ b/gateway/kustomization.yaml @@ -0,0 +1,6 @@ +# yaml-language-server: $schema=https://json.schemastore.org/kustomization.json +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: ystack +resources: +- gateway.yaml diff --git a/git-source/base/kustomization.yaml b/git-source/base/kustomization.yaml index 283d6ef5..eb59ff20 100644 --- a/git-source/base/kustomization.yaml +++ b/git-source/base/kustomization.yaml @@ -1,3 +1,4 @@ +# yaml-language-server: $schema=https://json.schemastore.org/kustomization.json apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization namespace: ystack # install's app_url depends on namespace diff --git a/k3s/00-namespace-ystack/kustomization.yaml b/k3s/00-namespace-ystack/kustomization.yaml new file mode 100644 index 00000000..f94b4ff2 --- /dev/null +++ b/k3s/00-namespace-ystack/kustomization.yaml @@ -0,0 +1,5 @@ +# yaml-language-server: $schema=https://json.schemastore.org/kustomization.json +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- namespace.yaml diff --git a/k3s/00-ystack-namespace/ystack-namespace.yaml b/k3s/00-namespace-ystack/namespace.yaml similarity index 100% rename from k3s/00-ystack-namespace/ystack-namespace.yaml rename to k3s/00-namespace-ystack/namespace.yaml diff --git a/k3s/00-ystack-namespace/kustomization.yaml b/k3s/00-ystack-namespace/kustomization.yaml deleted file mode 100644 index 6c95cac6..00000000 --- a/k3s/00-ystack-namespace/kustomization.yaml +++ /dev/null @@ -1,2 +0,0 @@ -resources: -- ystack-namespace.yaml diff --git a/k3s/01-namespace-blobs/kustomization.yaml b/k3s/01-namespace-blobs/kustomization.yaml new file mode 100644 index 00000000..f94b4ff2 --- /dev/null +++ b/k3s/01-namespace-blobs/kustomization.yaml @@ -0,0 +1,5 @@ +# yaml-language-server: $schema=https://json.schemastore.org/kustomization.json +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- namespace.yaml diff --git a/k3s/01-namespace-blobs/namespace.yaml b/k3s/01-namespace-blobs/namespace.yaml new file mode 100644 index 00000000..6c24b094 --- /dev/null +++ b/k3s/01-namespace-blobs/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: blobs diff --git a/k3s/02-namespace-kafka/kustomization.yaml b/k3s/02-namespace-kafka/kustomization.yaml new file mode 100644 index 00000000..f94b4ff2 --- /dev/null +++ b/k3s/02-namespace-kafka/kustomization.yaml @@ -0,0 +1,5 @@ +# yaml-language-server: $schema=https://json.schemastore.org/kustomization.json +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- namespace.yaml diff --git a/k3s/02-namespace-kafka/namespace.yaml b/k3s/02-namespace-kafka/namespace.yaml new file mode 100644 index 00000000..f92e7e85 --- /dev/null +++ b/k3s/02-namespace-kafka/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: kafka diff --git a/k3s/03-namespace-monitoring/kustomization.yaml b/k3s/03-namespace-monitoring/kustomization.yaml new file mode 100644 index 00000000..f94b4ff2 --- /dev/null +++ b/k3s/03-namespace-monitoring/kustomization.yaml @@ -0,0 +1,5 @@ +# yaml-language-server: $schema=https://json.schemastore.org/kustomization.json +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- namespace.yaml diff --git a/monitoring/namespace/monitoring-namespace.yaml b/k3s/03-namespace-monitoring/namespace.yaml similarity index 100% rename from monitoring/namespace/monitoring-namespace.yaml rename to k3s/03-namespace-monitoring/namespace.yaml diff --git a/k3s/05-gateway-api/kustomization.yaml b/k3s/05-gateway-api/kustomization.yaml deleted file mode 100644 index 03470fca..00000000 --- a/k3s/05-gateway-api/kustomization.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -# Gateway API CRDs are managed by k3s via the traefik-crd HelmChart. -# We only configure traefik to enable the Gateway provider. -resources: -- traefik-gateway-provider.yaml diff --git a/k3s/06-gateway/kustomization.yaml b/k3s/06-gateway/kustomization.yaml deleted file mode 100644 index 7f96db99..00000000 --- a/k3s/06-gateway/kustomization.yaml +++ /dev/null @@ -1,3 +0,0 @@ -namespace: ystack -resources: -- gateway.yaml diff --git a/k3s/07-builds-registry-httproute/kustomization.yaml b/k3s/07-builds-registry-httproute/kustomization.yaml deleted file mode 100644 index a9b9d433..00000000 --- a/k3s/07-builds-registry-httproute/kustomization.yaml +++ /dev/null @@ -1,3 +0,0 @@ -namespace: ystack -bases: -- ../../registry/httproute diff --git a/k3s/08-buildkitd-grpcroute/kustomization.yaml b/k3s/08-buildkitd-grpcroute/kustomization.yaml deleted file mode 100644 index de7a8fa7..00000000 --- a/k3s/08-buildkitd-grpcroute/kustomization.yaml +++ /dev/null @@ -1,3 +0,0 @@ -namespace: ystack -bases: -- ../../buildkit/grpcroute diff --git a/k3s/09-prometheus-httproute/kustomization.yaml b/k3s/09-prometheus-httproute/kustomization.yaml deleted file mode 100644 index 90e65331..00000000 --- a/k3s/09-prometheus-httproute/kustomization.yaml +++ /dev/null @@ -1,3 +0,0 @@ -namespace: monitoring -bases: -- ../../monitoring/httproute diff --git a/k3s/09-y-kustomize-secrets-init/kustomization.yaml b/k3s/09-y-kustomize-secrets-init/kustomization.yaml new file mode 100644 index 00000000..74657401 --- /dev/null +++ b/k3s/09-y-kustomize-secrets-init/kustomization.yaml @@ -0,0 +1,7 @@ +# yaml-language-server: $schema=https://json.schemastore.org/kustomization.json +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: ystack +resources: +- y-kustomize.blobs.setup-bucket-job.yaml +- y-kustomize.kafka.setup-topic-job.yaml diff --git a/k3s/09-y-kustomize-secrets-init/y-kustomize.blobs.setup-bucket-job.yaml b/k3s/09-y-kustomize-secrets-init/y-kustomize.blobs.setup-bucket-job.yaml new file mode 100644 index 00000000..364012e9 --- /dev/null +++ b/k3s/09-y-kustomize-secrets-init/y-kustomize.blobs.setup-bucket-job.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: Secret +metadata: + name: y-kustomize.blobs.setup-bucket-job +type: Opaque diff --git a/k3s/09-y-kustomize-secrets-init/y-kustomize.kafka.setup-topic-job.yaml b/k3s/09-y-kustomize-secrets-init/y-kustomize.kafka.setup-topic-job.yaml new file mode 100644 index 00000000..66ab2c42 --- /dev/null +++ b/k3s/09-y-kustomize-secrets-init/y-kustomize.kafka.setup-topic-job.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: Secret +metadata: + name: y-kustomize.kafka.setup-topic-job +type: Opaque diff --git a/k3s/10-gateway-api/kustomization.yaml b/k3s/10-gateway-api/kustomization.yaml new file mode 100644 index 00000000..195509f2 --- /dev/null +++ b/k3s/10-gateway-api/kustomization.yaml @@ -0,0 +1,5 @@ +# yaml-language-server: $schema=https://json.schemastore.org/kustomization.json +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- traefik-gateway-provider.yaml diff --git a/k3s/05-gateway-api/traefik-gateway-provider.yaml b/k3s/10-gateway-api/traefik-gateway-provider.yaml similarity index 100% rename from k3s/05-gateway-api/traefik-gateway-provider.yaml rename to k3s/10-gateway-api/traefik-gateway-provider.yaml diff --git a/k3s/10-minio/kustomization.yaml b/k3s/10-minio/kustomization.yaml deleted file mode 100644 index a19d3220..00000000 --- a/k3s/10-minio/kustomization.yaml +++ /dev/null @@ -1,3 +0,0 @@ -namespace: ystack -bases: -- ../../minio/standalone,defaultsecret diff --git a/k3s/10-versitygw/kustomization.yaml b/k3s/10-versitygw/kustomization.yaml deleted file mode 100644 index 059ac73d..00000000 --- a/k3s/10-versitygw/kustomization.yaml +++ /dev/null @@ -1,3 +0,0 @@ -namespace: ystack -bases: -- ../../versitygw/standalone,defaultsecret diff --git a/k3s/11-monitoring-operator/kustomization.yaml b/k3s/11-monitoring-operator/kustomization.yaml new file mode 100644 index 00000000..fe1e4dfd --- /dev/null +++ b/k3s/11-monitoring-operator/kustomization.yaml @@ -0,0 +1,5 @@ +# yaml-language-server: $schema=https://json.schemastore.org/kustomization.json +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- ../../monitoring/prometheus-operator diff --git a/k3s/20-builds-registry-versitygw/kustomization.yaml b/k3s/20-builds-registry-versitygw/kustomization.yaml deleted file mode 100644 index d7ffb6e9..00000000 --- a/k3s/20-builds-registry-versitygw/kustomization.yaml +++ /dev/null @@ -1,7 +0,0 @@ -namespace: ystack -bases: -- ../../registry/builds-service -- ../../registry/generic,versitygw -patchesStrategicMerge: -- builds-registry-magic-numbers.yaml -- builds-registry-replicas-1.yaml diff --git a/k3s/20-builds-registry/builds-registry-magic-numbers.yaml b/k3s/20-builds-registry/builds-registry-magic-numbers.yaml deleted file mode 100644 index 4cbb91b5..00000000 --- a/k3s/20-builds-registry/builds-registry-magic-numbers.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: builds-registry -spec: - type: NodePort - clusterIP: 10.43.0.50 diff --git a/k3s/20-builds-registry/builds-registry-replicas-1.yaml b/k3s/20-builds-registry/builds-registry-replicas-1.yaml deleted file mode 100644 index 21ec56b7..00000000 --- a/k3s/20-builds-registry/builds-registry-replicas-1.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: registry -spec: - replicas: 1 diff --git a/k3s/20-builds-registry/kustomization.yaml b/k3s/20-builds-registry/kustomization.yaml deleted file mode 100644 index 1f72dce7..00000000 --- a/k3s/20-builds-registry/kustomization.yaml +++ /dev/null @@ -1,7 +0,0 @@ -namespace: ystack -bases: -- ../../registry/builds-service -- ../../registry/generic,minio -patchesStrategicMerge: -- builds-registry-magic-numbers.yaml -- builds-registry-replicas-1.yaml diff --git a/k3s/20-gateway/kustomization.yaml b/k3s/20-gateway/kustomization.yaml new file mode 100644 index 00000000..9e7eef0a --- /dev/null +++ b/k3s/20-gateway/kustomization.yaml @@ -0,0 +1,5 @@ +# yaml-language-server: $schema=https://json.schemastore.org/kustomization.json +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- ../../gateway diff --git a/k3s/21-prod-registry/kustomization.yaml b/k3s/21-prod-registry/kustomization.yaml deleted file mode 100644 index 57af4b55..00000000 --- a/k3s/21-prod-registry/kustomization.yaml +++ /dev/null @@ -1,5 +0,0 @@ -namespace: ystack -bases: -- ../../registry/prod-service -patchesStrategicMerge: -- prod-registry-magic-numbers.yaml diff --git a/k3s/29-y-kustomize/kustomization.yaml b/k3s/29-y-kustomize/kustomization.yaml new file mode 100644 index 00000000..f8acb768 --- /dev/null +++ b/k3s/29-y-kustomize/kustomization.yaml @@ -0,0 +1,5 @@ +# yaml-language-server: $schema=https://json.schemastore.org/kustomization.json +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- ../../y-kustomize diff --git a/k3s/30-blobs-minio-disabled/kustomization.yaml b/k3s/30-blobs-minio-disabled/kustomization.yaml new file mode 100644 index 00000000..11b88a3c --- /dev/null +++ b/k3s/30-blobs-minio-disabled/kustomization.yaml @@ -0,0 +1,6 @@ +# yaml-language-server: $schema=https://json.schemastore.org/kustomization.json +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- ../../blobs/minio +- ../../blobs-minio/standalone,defaultsecret diff --git a/k3s/30-blobs-ystack/kustomization.yaml b/k3s/30-blobs-ystack/kustomization.yaml new file mode 100644 index 00000000..ac86556c --- /dev/null +++ b/k3s/30-blobs-ystack/kustomization.yaml @@ -0,0 +1,5 @@ +# yaml-language-server: $schema=https://json.schemastore.org/kustomization.json +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- ../../blobs-versitygw/y-kustomize diff --git a/k3s/30-blobs/kustomization.yaml b/k3s/30-blobs/kustomization.yaml new file mode 100644 index 00000000..9b0cd8e5 --- /dev/null +++ b/k3s/30-blobs/kustomization.yaml @@ -0,0 +1,6 @@ +# yaml-language-server: $schema=https://json.schemastore.org/kustomization.json +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- ../../blobs/versitygw +- ../../blobs-versitygw/standalone,defaultsecret diff --git a/k3s/30-monitoring-operator/kustomization.yaml b/k3s/30-monitoring-operator/kustomization.yaml deleted file mode 100644 index 0290e37b..00000000 --- a/k3s/30-monitoring-operator/kustomization.yaml +++ /dev/null @@ -1,3 +0,0 @@ -resources: -- ../../monitoring/namespace -- ../../monitoring/prometheus-operator diff --git a/k3s/31-monitoring/kustomization.yaml b/k3s/31-monitoring/kustomization.yaml deleted file mode 100644 index 14c81cb0..00000000 --- a/k3s/31-monitoring/kustomization.yaml +++ /dev/null @@ -1,6 +0,0 @@ -resources: -- ../../monitoring/namespace -- ../../monitoring/prometheus-now -- ../../monitoring/alertmanager-main -- ../../monitoring/kube-state-metrics-now -- ../../monitoring/node-exporter-now diff --git a/k3s/40-buildkit/kustomization.yaml b/k3s/40-buildkit/kustomization.yaml deleted file mode 100644 index 0197be56..00000000 --- a/k3s/40-buildkit/kustomization.yaml +++ /dev/null @@ -1,9 +0,0 @@ -namespace: ystack -bases: -- ../../buildkit -- ../../buildkit/gateway-proxy -- ../../versitygw/standalone,defaultsecret -resources: -- buildkitd-nodeport-service.yaml -patchesStrategicMerge: -- buildkitd-replicas-0.yaml diff --git a/k3s/40-kafka-ystack/kustomization.yaml b/k3s/40-kafka-ystack/kustomization.yaml new file mode 100644 index 00000000..163632b8 --- /dev/null +++ b/k3s/40-kafka-ystack/kustomization.yaml @@ -0,0 +1,5 @@ +# yaml-language-server: $schema=https://json.schemastore.org/kustomization.json +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- ../../kafka/y-kustomize diff --git a/k3s/40-kafka/kustomization.yaml b/k3s/40-kafka/kustomization.yaml new file mode 100644 index 00000000..10195997 --- /dev/null +++ b/k3s/40-kafka/kustomization.yaml @@ -0,0 +1,7 @@ +# yaml-language-server: $schema=https://json.schemastore.org/kustomization.json +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- ../../kafka/base +components: +- ../../kafka/redpanda-image diff --git a/k3s/50-monitoring/kustomization.yaml b/k3s/50-monitoring/kustomization.yaml new file mode 100644 index 00000000..46125267 --- /dev/null +++ b/k3s/50-monitoring/kustomization.yaml @@ -0,0 +1,9 @@ +# yaml-language-server: $schema=https://json.schemastore.org/kustomization.json +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- ../../monitoring/prometheus-now +- ../../monitoring/alertmanager-main +- ../../monitoring/kube-state-metrics-now +- ../../monitoring/node-exporter-now +- ../../monitoring/gateway diff --git a/k3s/20-builds-registry-versitygw/builds-registry-magic-numbers.yaml b/k3s/60-builds-registry/builds-registry-magic-numbers.yaml similarity index 100% rename from k3s/20-builds-registry-versitygw/builds-registry-magic-numbers.yaml rename to k3s/60-builds-registry/builds-registry-magic-numbers.yaml diff --git a/k3s/20-builds-registry-versitygw/builds-registry-replicas-1.yaml b/k3s/60-builds-registry/builds-registry-replicas-1.yaml similarity index 100% rename from k3s/20-builds-registry-versitygw/builds-registry-replicas-1.yaml rename to k3s/60-builds-registry/builds-registry-replicas-1.yaml diff --git a/k3s/60-builds-registry/deployment-s3.yaml b/k3s/60-builds-registry/deployment-s3.yaml new file mode 100644 index 00000000..d69daae9 --- /dev/null +++ b/k3s/60-builds-registry/deployment-s3.yaml @@ -0,0 +1,35 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: registry +spec: + template: + spec: + containers: + - name: docker-v2 + env: + - name: REGISTRY_STORAGE + value: s3 + - name: REGISTRY_STORAGE_S3_ACCESSKEY + valueFrom: + secretKeyRef: + name: builds-registry-bucket + key: accesskey + - name: REGISTRY_STORAGE_S3_SECRETKEY + valueFrom: + secretKeyRef: + name: builds-registry-bucket + key: secretkey + - name: REGISTRY_STORAGE_S3_REGIONENDPOINT + valueFrom: + secretKeyRef: + name: builds-registry-bucket + key: endpoint + - name: REGISTRY_STORAGE_S3_REGION + value: us-east-1 + - name: REGISTRY_STORAGE_S3_BUCKET + value: ystack-builds-registry + - name: REGISTRY_STORAGE_S3_FORCEPATHSTYLE + value: "true" + - name: REGISTRY_STORAGE_REDIRECT_DISABLE + value: "true" diff --git a/k3s/60-builds-registry/kustomization.yaml b/k3s/60-builds-registry/kustomization.yaml new file mode 100644 index 00000000..d54ad388 --- /dev/null +++ b/k3s/60-builds-registry/kustomization.yaml @@ -0,0 +1,20 @@ +# yaml-language-server: $schema=https://json.schemastore.org/kustomization.json +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +# namespace override: blobs-versitygw/defaultsecret targets blobs, +# but builds-registry needs the secret in ystack +namespace: ystack + +resources: +- ../../registry/builds-service +- ../../registry/generic +- ../../registry/gateway +- ../../blobs-versitygw/defaultsecret +- ../../registry/builds-bucket +- ../../registry/builds-topic + +patches: +- path: builds-registry-magic-numbers.yaml +- path: builds-registry-replicas-1.yaml +- path: deployment-s3.yaml diff --git a/k3s/61-prod-registry/kustomization.yaml b/k3s/61-prod-registry/kustomization.yaml new file mode 100644 index 00000000..e29d054f --- /dev/null +++ b/k3s/61-prod-registry/kustomization.yaml @@ -0,0 +1,7 @@ +# yaml-language-server: $schema=https://json.schemastore.org/kustomization.json +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- ../../registry/prod-service +patches: +- path: prod-registry-magic-numbers.yaml diff --git a/k3s/21-prod-registry/prod-registry-magic-numbers.yaml b/k3s/61-prod-registry/prod-registry-magic-numbers.yaml similarity index 100% rename from k3s/21-prod-registry/prod-registry-magic-numbers.yaml rename to k3s/61-prod-registry/prod-registry-magic-numbers.yaml diff --git a/k3s/40-buildkit/buildkitd-nodeport-service.yaml b/k3s/62-buildkit/buildkitd-nodeport-service.yaml similarity index 100% rename from k3s/40-buildkit/buildkitd-nodeport-service.yaml rename to k3s/62-buildkit/buildkitd-nodeport-service.yaml diff --git a/k3s/40-buildkit/buildkitd-replicas-0.yaml b/k3s/62-buildkit/buildkitd-replicas-0.yaml similarity index 100% rename from k3s/40-buildkit/buildkitd-replicas-0.yaml rename to k3s/62-buildkit/buildkitd-replicas-0.yaml diff --git a/k3s/62-buildkit/kustomization.yaml b/k3s/62-buildkit/kustomization.yaml new file mode 100644 index 00000000..627f0606 --- /dev/null +++ b/k3s/62-buildkit/kustomization.yaml @@ -0,0 +1,9 @@ +# yaml-language-server: $schema=https://json.schemastore.org/kustomization.json +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- ../../buildkit +- ../../buildkit/gateway +- buildkitd-nodeport-service.yaml +patches: +- path: buildkitd-replicas-0.yaml diff --git a/k3s/README.md b/k3s/README.md new file mode 100644 index 00000000..64c67132 --- /dev/null +++ b/k3s/README.md @@ -0,0 +1,25 @@ + +This structure is the configuration for [y-cluster-converge-ystack](../bin/y-cluster-converge-ystack). + +Converge principles: + +- List the bases in order. + Filter out any name that ends with `-disabled`. +- Single pass: apply each base with `kubectl apply -k`. + `1*` bases use `--server-side=true --force-conflicts` (required for large CRDs). +- Between digit groups (0→1, 1→2, etc.), wait for all deployment rollouts. +- After `1*`, validate that CRDs are registered and served. +- Before `6*`, verify [y-kustomize api](../y-kustomize/openapi/openapi.yaml) serves real content + (secrets from `3*` and `4*` need time to propagate to mounted volumes). + +Each base is applied with `kubectl apply -k` — no label selectors, no multi-pass. + +Bases: + +- 0*: namespaces + y-kustomize empty secret init (never deleted) +- 1*: Gateway API, CRDs +- 2*: y-kustomize deployment, gateway +- 3*: blobs (real y-kustomize blobs secret) +- 4*: kafka (real y-kustomize kafka secret) +- 5*: monitoring +- 6*: registries, buildkit (depend on y-kustomize HTTP for remote bases) diff --git a/kafka/base/kustomization.yaml b/kafka/base/kustomization.yaml index d9df7472..d9efd0a9 100644 --- a/kafka/base/kustomization.yaml +++ b/kafka/base/kustomization.yaml @@ -6,7 +6,8 @@ kind: Kustomization namespace: kafka resources: -- ../redpanda/kafka +- ../redpanda +- y-bootstrap-service.yaml patches: - path: ./redpanda-resources.yaml diff --git a/kafka/base/y-bootstrap-service.yaml b/kafka/base/y-bootstrap-service.yaml new file mode 100644 index 00000000..3e0ce84f --- /dev/null +++ b/kafka/base/y-bootstrap-service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: y-bootstrap + namespace: kafka +spec: + type: ClusterIP + selector: + app.kubernetes.io/name: redpanda + app.kubernetes.io/instance: redpanda + app.kubernetes.io/component: redpanda-statefulset + ports: + - name: kafka + port: 9092 + targetPort: 9092 diff --git a/kafka/redpanda-image/kustomization.yaml b/kafka/redpanda-image/kustomization.yaml index d286caca..3997aacb 100644 --- a/kafka/redpanda-image/kustomization.yaml +++ b/kafka/redpanda-image/kustomization.yaml @@ -2,6 +2,6 @@ apiVersion: kustomize.config.k8s.io/v1alpha1 kind: Component images: -- name: redpandadata/redpanda +- name: docker.redpanda.com/redpandadata/redpanda newName: ghcr.io/yolean/redpanda - newTag: v24.2.22@sha256:5132085d4fe35b0fd6ddedc7f0fe3d3ba7be12c5e3829e1a2b986cd41b1d3538 + newTag: v24.2.14@sha256:a91cddd8a93181b85107a3cde0beebb5fcdc765d10b010af398e0dcad18d4dbf diff --git a/kafka/redpanda/kafka/kustomization.yaml b/kafka/redpanda/kafka/kustomization.yaml deleted file mode 100644 index 7b232000..00000000 --- a/kafka/redpanda/kafka/kustomization.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: - -- ./redpanda/templates/configmap.yaml -- ./redpanda/templates/poddisruptionbudget.yaml -- ./redpanda/templates/rbac.yaml -- ./redpanda/templates/secrets.yaml -- ./redpanda/templates/service.internal.yaml -- ./redpanda/templates/statefulset.yaml -# - ./redpanda/templates/tests/test-api-status.yaml -# - ./redpanda/templates/tests/test-kafka-nodelete.yaml -# - ./redpanda/templates/tests/test-kafka-produce-consume.yaml -# - ./redpanda/templates/tests/test-lifecycle-scripts.yaml -# - ./redpanda/templates/tests/test-pandaproxy-status.yaml -# - ./redpanda/templates/tests/test-rack-awareness.yaml -# - ./redpanda/templates/tests/test-schemaregistry-status.yaml diff --git a/kafka/redpanda/kafka/redpanda/templates/statefulset.yaml b/kafka/redpanda/kafka/redpanda/templates/statefulset.yaml index 47d1379b..64ae3908 100644 --- a/kafka/redpanda/kafka/redpanda/templates/statefulset.yaml +++ b/kafka/redpanda/kafka/redpanda/templates/statefulset.yaml @@ -179,7 +179,6 @@ spec: mountPath: /var/lib/redpanda/data resources: limits: - cpu: 250m memory: 1171Mi volumes: diff --git a/kafka/redpanda/kustomization.yaml b/kafka/redpanda/kustomization.yaml new file mode 100644 index 00000000..338222fc --- /dev/null +++ b/kafka/redpanda/kustomization.yaml @@ -0,0 +1,18 @@ +# yaml-language-server: $schema=https://json.schemastore.org/kustomization.json +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + +- ./kafka/redpanda/templates/configmap.yaml +- ./kafka/redpanda/templates/poddisruptionbudget.yaml +- ./kafka/redpanda/templates/rbac.yaml +- ./kafka/redpanda/templates/secrets.yaml +- ./kafka/redpanda/templates/service.internal.yaml +- ./kafka/redpanda/templates/statefulset.yaml +# - ./kafka/redpanda/templates/tests/test-api-status.yaml +# - ./kafka/redpanda/templates/tests/test-kafka-nodelete.yaml +# - ./kafka/redpanda/templates/tests/test-kafka-produce-consume.yaml +# - ./kafka/redpanda/templates/tests/test-lifecycle-scripts.yaml +# - ./kafka/redpanda/templates/tests/test-pandaproxy-status.yaml +# - ./kafka/redpanda/templates/tests/test-rack-awareness.yaml +# - ./kafka/redpanda/templates/tests/test-schemaregistry-status.yaml diff --git a/kafka/topic-job/kafka-topic-job.yaml b/kafka/topic-job/kafka-topic-job.yaml index 9edc0348..0084f378 100644 --- a/kafka/topic-job/kafka-topic-job.yaml +++ b/kafka/topic-job/kafka-topic-job.yaml @@ -21,7 +21,7 @@ spec: activeDeadlineSeconds: 3600 containers: - name: topic - image: redpandadata/redpanda + image: docker.redpanda.com/redpandadata/redpanda args: - | [ -n "$KAFKA_BOOTSTRAP" ] || exit 1 @@ -67,5 +67,4 @@ spec: cpu: 250m memory: 100Mi limits: - cpu: 250m memory: 100Mi diff --git a/kafka/topic-job/kustomization.yaml b/kafka/topic-job/kustomization.yaml index f745302c..95351b2d 100644 --- a/kafka/topic-job/kustomization.yaml +++ b/kafka/topic-job/kustomization.yaml @@ -1,3 +1,4 @@ +# yaml-language-server: $schema=https://json.schemastore.org/kustomization.json apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization diff --git a/kafka/y-kustomize/kustomization.yaml b/kafka/y-kustomize/kustomization.yaml new file mode 100644 index 00000000..36b5dd24 --- /dev/null +++ b/kafka/y-kustomize/kustomization.yaml @@ -0,0 +1,13 @@ +# yaml-language-server: $schema=https://json.schemastore.org/kustomization.json +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: ystack + +secretGenerator: +- name: y-kustomize.kafka.setup-topic-job + options: + disableNameSuffixHash: true + labels: + yolean.se/module-part: config + files: + - base-for-annotations.yaml=y-kustomize-bases/kafka/setup-topic-job/setup-topic-job.yaml diff --git a/kafka/y-kustomize/y-kustomize-bases/kafka/setup-topic-job/setup-topic-job.yaml b/kafka/y-kustomize/y-kustomize-bases/kafka/setup-topic-job/setup-topic-job.yaml new file mode 100644 index 00000000..9306ebb1 --- /dev/null +++ b/kafka/y-kustomize/y-kustomize-bases/kafka/setup-topic-job/setup-topic-job.yaml @@ -0,0 +1,78 @@ +apiVersion: v1 +kind: Secret +metadata: + name: kafka-bootstrap +stringData: + broker: y-bootstrap.kafka.svc.cluster.local:9092 +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: setup-topic + labels: + yolean.se/converge-mode: replace +spec: + template: + metadata: + annotations: + yolean.se/kafka-bootstrap: y-bootstrap.kafka.svc.cluster.local:9092 + yolean.se/kafka-topic-name: "" + yolean.se/kafka-topic-config: >- + max.message.bytes=524288 + retention.bytes=-1 + retention.ms=-1 + yolean.se/kafka-topic-partitions: "1" + yolean.se/kafka-topic-replicas: "-1" + spec: + restartPolicy: Never + activeDeadlineSeconds: 3600 + containers: + - name: topic + image: ghcr.io/yolean/redpanda:v24.2.14@sha256:a91cddd8a93181b85107a3cde0beebb5fcdc765d10b010af398e0dcad18d4dbf + args: + - | + [ -n "$KAFKA_BOOTSTRAP" ] || exit 1 + [ -n "$TOPIC_NAME" ] || exit 1 + [ -n "$TOPIC_CONFIG" ] || exit 1 + function config_args { + FLAG=$1 + for C in $TOPIC_CONFIG; do echo -n " $FLAG $C"; done + echo '' + } + until rpk cluster --brokers $KAFKA_BOOTSTRAP info -b -c; do sleep 1; done; + if rpk topic --brokers $KAFKA_BOOTSTRAP describe "$TOPIC_NAME"; then + rpk topic --brokers $KAFKA_BOOTSTRAP alter-config "$TOPIC_NAME" $(config_args --set) | grep OK + else + rpk topic --brokers $KAFKA_BOOTSTRAP create "$TOPIC_NAME" --partitions "$TOPIC_PARTITIONS" --replicas "$TOPIC_REPLICAS" $(config_args --topic-config) + fi + command: + - /bin/bash + - -cex + env: + - name: KAFKA_BOOTSTRAP + valueFrom: + fieldRef: + fieldPath: metadata.annotations['yolean.se/kafka-bootstrap'] + - name: TOPIC_NAME + valueFrom: + fieldRef: + fieldPath: metadata.annotations['yolean.se/kafka-topic-name'] + - name: TOPIC_CONFIG + valueFrom: + fieldRef: + fieldPath: metadata.annotations['yolean.se/kafka-topic-config'] + - name: TOPIC_PARTITIONS + valueFrom: + fieldRef: + fieldPath: metadata.annotations['yolean.se/kafka-topic-partitions'] + - name: TOPIC_REPLICAS + valueFrom: + fieldRef: + fieldPath: metadata.annotations['yolean.se/kafka-topic-replicas'] + resources: + requests: + cpu: 250m + memory: 100Mi + limits: + memory: 100Mi + backoffLimit: 10 diff --git a/minio/common/kustomization.yaml b/minio/common/kustomization.yaml deleted file mode 100644 index 18f445c8..00000000 --- a/minio/common/kustomization.yaml +++ /dev/null @@ -1,2 +0,0 @@ -resources: -- blobs-minio-service.yaml diff --git a/minio/standalone,defaultsecret/kustomization.yaml b/minio/standalone,defaultsecret/kustomization.yaml deleted file mode 100644 index 49e1b2ea..00000000 --- a/minio/standalone,defaultsecret/kustomization.yaml +++ /dev/null @@ -1,3 +0,0 @@ -bases: -- ../defaultsecret -- ../standalone diff --git a/minio/standalone/kustomization.yaml b/minio/standalone/kustomization.yaml deleted file mode 100644 index 6bd8040e..00000000 --- a/minio/standalone/kustomization.yaml +++ /dev/null @@ -1,4 +0,0 @@ -resources: -- ../common -- deployment.yaml -- pvc.yaml diff --git a/monitoring/alertmanager-main/kustomization.yaml b/monitoring/alertmanager-main/kustomization.yaml index d0466b78..ffb8dafa 100644 --- a/monitoring/alertmanager-main/kustomization.yaml +++ b/monitoring/alertmanager-main/kustomization.yaml @@ -1,3 +1,4 @@ +# yaml-language-server: $schema=https://json.schemastore.org/kustomization.json apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization diff --git a/monitoring/httproute/httproute.yaml b/monitoring/gateway/httproute.yaml similarity index 86% rename from monitoring/httproute/httproute.yaml rename to monitoring/gateway/httproute.yaml index a456a7c8..48649130 100644 --- a/monitoring/httproute/httproute.yaml +++ b/monitoring/gateway/httproute.yaml @@ -2,6 +2,8 @@ apiVersion: gateway.networking.k8s.io/v1 kind: HTTPRoute metadata: name: prometheus-now + labels: + yolean.se/module-part: gateway spec: parentRefs: - name: ystack diff --git a/monitoring/gateway/kustomization.yaml b/monitoring/gateway/kustomization.yaml new file mode 100644 index 00000000..b54bbbbb --- /dev/null +++ b/monitoring/gateway/kustomization.yaml @@ -0,0 +1,6 @@ +# yaml-language-server: $schema=https://json.schemastore.org/kustomization.json +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: monitoring +resources: +- httproute.yaml diff --git a/monitoring/grafana/grafana-deployment.yaml b/monitoring/grafana/grafana-deployment.yaml index 3a251aab..fa02a592 100644 --- a/monitoring/grafana/grafana-deployment.yaml +++ b/monitoring/grafana/grafana-deployment.yaml @@ -123,7 +123,6 @@ spec: value: "2147483647" resources: limits: - cpu: 10m memory: 20Mi volumeMounts: - mountPath: /dashboards diff --git a/monitoring/grafana/kustomization.yaml b/monitoring/grafana/kustomization.yaml index 7ca0e584..bdbe1071 100644 --- a/monitoring/grafana/kustomization.yaml +++ b/monitoring/grafana/kustomization.yaml @@ -1,3 +1,4 @@ +# yaml-language-server: $schema=https://json.schemastore.org/kustomization.json apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization diff --git a/monitoring/httproute/kustomization.yaml b/monitoring/httproute/kustomization.yaml deleted file mode 100644 index dbc3f2d6..00000000 --- a/monitoring/httproute/kustomization.yaml +++ /dev/null @@ -1,2 +0,0 @@ -resources: -- httproute.yaml diff --git a/monitoring/kube-state-metrics-now/kustomization.yaml b/monitoring/kube-state-metrics-now/kustomization.yaml index d1b51d82..1725753f 100644 --- a/monitoring/kube-state-metrics-now/kustomization.yaml +++ b/monitoring/kube-state-metrics-now/kustomization.yaml @@ -1,3 +1,4 @@ +# yaml-language-server: $schema=https://json.schemastore.org/kustomization.json apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization diff --git a/monitoring/kube-state-metrics/kustomization.yaml b/monitoring/kube-state-metrics/kustomization.yaml index 63b9ca3e..59b95bd9 100644 --- a/monitoring/kube-state-metrics/kustomization.yaml +++ b/monitoring/kube-state-metrics/kustomization.yaml @@ -1,3 +1,4 @@ +# yaml-language-server: $schema=https://json.schemastore.org/kustomization.json apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization diff --git a/monitoring/namespace/kustomization.yaml b/monitoring/namespace/kustomization.yaml deleted file mode 100644 index b33245b9..00000000 --- a/monitoring/namespace/kustomization.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -resources: -- monitoring-namespace.yaml diff --git a/monitoring/node-exporter-now/kustomization.yaml b/monitoring/node-exporter-now/kustomization.yaml index 19ef18db..9f8da82c 100644 --- a/monitoring/node-exporter-now/kustomization.yaml +++ b/monitoring/node-exporter-now/kustomization.yaml @@ -1,3 +1,4 @@ +# yaml-language-server: $schema=https://json.schemastore.org/kustomization.json apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization diff --git a/monitoring/node-exporter/kustomization.yaml b/monitoring/node-exporter/kustomization.yaml index 923ec99a..157dee39 100644 --- a/monitoring/node-exporter/kustomization.yaml +++ b/monitoring/node-exporter/kustomization.yaml @@ -1,3 +1,7 @@ +# yaml-language-server: $schema=https://json.schemastore.org/kustomization.json +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + resources: - node-exporter-serviceAccount.yaml - node-exporter-clusterRole.yaml diff --git a/monitoring/node-exporter/node-exporter-daemonset.yaml b/monitoring/node-exporter/node-exporter-daemonset.yaml index d6c658a5..5bf889e9 100644 --- a/monitoring/node-exporter/node-exporter-daemonset.yaml +++ b/monitoring/node-exporter/node-exporter-daemonset.yaml @@ -41,7 +41,6 @@ spec: containerPort: 9100 resources: limits: - cpu: 1000m memory: 30Mi requests: cpu: 20m diff --git a/monitoring/prometheus-now/kustomization.yaml b/monitoring/prometheus-now/kustomization.yaml index b63efad0..51960fd4 100644 --- a/monitoring/prometheus-now/kustomization.yaml +++ b/monitoring/prometheus-now/kustomization.yaml @@ -1,3 +1,4 @@ +# yaml-language-server: $schema=https://json.schemastore.org/kustomization.json apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization diff --git a/monitoring/prometheus-operator/kustomization.yaml b/monitoring/prometheus-operator/kustomization.yaml index 262f0e0c..932b773b 100644 --- a/monitoring/prometheus-operator/kustomization.yaml +++ b/monitoring/prometheus-operator/kustomization.yaml @@ -1,3 +1,4 @@ +# yaml-language-server: $schema=https://json.schemastore.org/kustomization.json apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization diff --git a/monitoring/rbac-prometheus/kustomization.yaml b/monitoring/rbac-prometheus/kustomization.yaml index 5bced713..079cc035 100644 --- a/monitoring/rbac-prometheus/kustomization.yaml +++ b/monitoring/rbac-prometheus/kustomization.yaml @@ -1,3 +1,7 @@ +# yaml-language-server: $schema=https://json.schemastore.org/kustomization.json +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + # https://github.com/coreos/prometheus-operator/commit/ee40763ad45839982ca6e09578cdc0eb25b0e836 resources: - prometheus-service-account.yaml diff --git a/registry/builds-bucket/kustomization.yaml b/registry/builds-bucket/kustomization.yaml new file mode 100644 index 00000000..287618bb --- /dev/null +++ b/registry/builds-bucket/kustomization.yaml @@ -0,0 +1,9 @@ +# yaml-language-server: $schema=https://json.schemastore.org/kustomization.json +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: ystack +namePrefix: builds-registry- +resources: +- http://y-kustomize.ystack.svc.cluster.local/v1/blobs/setup-bucket-job/base-for-annotations.yaml +commonAnnotations: + yolean.se/bucket-name: ystack-builds-registry diff --git a/registry/builds-service/kustomization.yaml b/registry/builds-service/kustomization.yaml index 92102610..edb67401 100644 --- a/registry/builds-service/kustomization.yaml +++ b/registry/builds-service/kustomization.yaml @@ -1,2 +1,6 @@ +# yaml-language-server: $schema=https://json.schemastore.org/kustomization.json +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: ystack resources: - builds-registry-service.yaml diff --git a/registry/builds-topic/kustomization.yaml b/registry/builds-topic/kustomization.yaml new file mode 100644 index 00000000..11322b9c --- /dev/null +++ b/registry/builds-topic/kustomization.yaml @@ -0,0 +1,8 @@ +# yaml-language-server: $schema=https://json.schemastore.org/kustomization.json +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: ystack +resources: +- http://y-kustomize.ystack.svc.cluster.local/v1/kafka/setup-topic-job/base-for-annotations.yaml +commonAnnotations: + yolean.se/kafka-topic-name: ystack.builds-registry.stream.json diff --git a/registry/httproute/httproute.yaml b/registry/gateway/httproute.yaml similarity index 85% rename from registry/httproute/httproute.yaml rename to registry/gateway/httproute.yaml index c9eda95b..a083abfc 100644 --- a/registry/httproute/httproute.yaml +++ b/registry/gateway/httproute.yaml @@ -2,6 +2,8 @@ apiVersion: gateway.networking.k8s.io/v1 kind: HTTPRoute metadata: name: builds-registry + labels: + yolean.se/module-part: gateway spec: parentRefs: - name: ystack diff --git a/registry/gateway/kustomization.yaml b/registry/gateway/kustomization.yaml new file mode 100644 index 00000000..0f6ec012 --- /dev/null +++ b/registry/gateway/kustomization.yaml @@ -0,0 +1,6 @@ +# yaml-language-server: $schema=https://json.schemastore.org/kustomization.json +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: ystack +resources: +- httproute.yaml diff --git a/registry/generic,kafka/config.yml b/registry/generic,kafka/config.yml deleted file mode 100644 index 6ffc7018..00000000 --- a/registry/generic,kafka/config.yml +++ /dev/null @@ -1,25 +0,0 @@ -version: 0.1 -log: - fields: - service: registry -storage: - cache: - blobdescriptor: inmemory -http: - headers: - X-Content-Type-Options: [nosniff] -notifications: - endpoints: - - name: pixy - disabled: false - url: http://pixy/topics/ystack.builds-registry.stream.json/messages - timeout: 10s - threshold: 1 - backoff: 1s - ignoredmediatypes: - - application/octet-stream -health: - storagedriver: - enabled: true - interval: 10s - threshold: 3 diff --git a/registry/generic,kafka/kustomization.yaml b/registry/generic,kafka/kustomization.yaml deleted file mode 100644 index 965886d1..00000000 --- a/registry/generic,kafka/kustomization.yaml +++ /dev/null @@ -1,9 +0,0 @@ -bases: -- ../generic -resources: -- topic-create.yaml -configMapGenerator: -- name: registry-config - behavior: replace - files: - - config.yml diff --git a/registry/generic,kafka/topic-create.yaml b/registry/generic,kafka/topic-create.yaml deleted file mode 100644 index 36ad7557..00000000 --- a/registry/generic,kafka/topic-create.yaml +++ /dev/null @@ -1,35 +0,0 @@ -apiVersion: batch/v1 -kind: Job -metadata: - name: ystack-builds-registry-topic-create -spec: - template: - spec: - containers: - - name: topic-create - image: solsson/kafka-cli@sha256:9fa3306e9f5d18283d10e01f7c115d8321eedc682f262aff784bd0126e1f2221 - env: - - name: TOPIC_NAME - value: ystack.builds-registry.stream.json - - name: PARTITIONS - value: "1" - - name: REPLICATION_FACTOR - value: "1" - command: - - ./bin/kafka-topics.sh - - --zookeeper - - zookeeper.kafka.svc.cluster.local:2181 - - --create - - --if-not-exists - - --topic - - $(TOPIC_NAME) - - --partitions - - $(PARTITIONS) - - --replication-factor - - $(REPLICATION_FACTOR) - resources: - limits: - cpu: 100m - memory: 20Mi - restartPolicy: Never - backoffLimit: 20 diff --git a/registry/generic,minio/bucket-create-ystack-builds.yaml b/registry/generic,minio/bucket-create-ystack-builds.yaml index 6adb1947..6bc05182 100644 --- a/registry/generic,minio/bucket-create-ystack-builds.yaml +++ b/registry/generic,minio/bucket-create-ystack-builds.yaml @@ -20,7 +20,7 @@ spec: name: minio key: secretkey - name: MINIO_HOST - value: http://blobs-minio + value: http://y-s3-api.blobs.svc.cluster.local - name: MINIO_REGION value: us-east-1 - name: BUCKET_NAME diff --git a/registry/generic,minio/deployment.yaml b/registry/generic,minio/deployment.yaml index 664c85d4..efb0f21b 100644 --- a/registry/generic,minio/deployment.yaml +++ b/registry/generic,minio/deployment.yaml @@ -21,7 +21,7 @@ spec: name: minio key: secretkey - name: REGISTRY_STORAGE_S3_REGIONENDPOINT - value: http://blobs-minio.ystack.svc.cluster.local + value: http://y-s3-api.blobs.svc.cluster.local - name: REGISTRY_STORAGE_S3_REGION value: us-east-1 - name: REGISTRY_STORAGE_S3_BUCKET diff --git a/registry/generic,minio/kustomization.yaml b/registry/generic,minio/kustomization.yaml index e4884414..f3a321a3 100644 --- a/registry/generic,minio/kustomization.yaml +++ b/registry/generic,minio/kustomization.yaml @@ -1,5 +1,9 @@ +# yaml-language-server: $schema=https://json.schemastore.org/kustomization.json +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + bases: -- ../../minio/defaultsecret +- ../../blobs-minio/defaultsecret - ../generic resources: - bucket-create-ystack-builds.yaml diff --git a/registry/generic,versitygw/bucket-create-ystack-builds.yaml b/registry/generic,versitygw/bucket-create-ystack-builds.yaml index 0359d160..e6f5845b 100644 --- a/registry/generic,versitygw/bucket-create-ystack-builds.yaml +++ b/registry/generic,versitygw/bucket-create-ystack-builds.yaml @@ -22,7 +22,7 @@ spec: - name: BUCKET_NAME value: ystack-builds-registry - name: S3_ENDPOINT - value: http://blobs-versitygw + value: http://y-s3-api.blobs.svc.cluster.local command: - sh - -ce diff --git a/registry/generic,versitygw/deployment.yaml b/registry/generic,versitygw/deployment.yaml index 4f124986..efb0f21b 100644 --- a/registry/generic,versitygw/deployment.yaml +++ b/registry/generic,versitygw/deployment.yaml @@ -21,7 +21,7 @@ spec: name: minio key: secretkey - name: REGISTRY_STORAGE_S3_REGIONENDPOINT - value: http://blobs-versitygw.ystack.svc.cluster.local + value: http://y-s3-api.blobs.svc.cluster.local - name: REGISTRY_STORAGE_S3_REGION value: us-east-1 - name: REGISTRY_STORAGE_S3_BUCKET diff --git a/registry/generic,versitygw/kustomization.yaml b/registry/generic,versitygw/kustomization.yaml index 7a5ace39..9e1ec712 100644 --- a/registry/generic,versitygw/kustomization.yaml +++ b/registry/generic,versitygw/kustomization.yaml @@ -1,5 +1,9 @@ +# yaml-language-server: $schema=https://json.schemastore.org/kustomization.json +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + bases: -- ../../versitygw/defaultsecret +- ../../blobs-versitygw/defaultsecret - ../generic resources: - bucket-create-ystack-builds.yaml diff --git a/registry/generic/deployment.yaml b/registry/generic/deployment.yaml index dec3344d..44af2581 100644 --- a/registry/generic/deployment.yaml +++ b/registry/generic/deployment.yaml @@ -26,7 +26,6 @@ spec: cpu: 10m memory: 16Mi limits: - cpu: 500m memory: 800Mi ports: - containerPort: 80 diff --git a/registry/generic/kustomization.yaml b/registry/generic/kustomization.yaml index e77f20ad..4f3a312a 100644 --- a/registry/generic/kustomization.yaml +++ b/registry/generic/kustomization.yaml @@ -1,11 +1,10 @@ # yaml-language-server: $schema=https://json.schemastore.org/kustomization.json apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization +namespace: ystack -images: -- name: registry - newName: ghcr.io/yolean/registry - newTag: 3.0.0@sha256:6c5666b861f3505b116bb9aa9b25175e71210414bd010d92035ff64018f9457e +components: +- ../images resources: - deployment.yaml diff --git a/registry/httproute/kustomization.yaml b/registry/httproute/kustomization.yaml deleted file mode 100644 index dbc3f2d6..00000000 --- a/registry/httproute/kustomization.yaml +++ /dev/null @@ -1,2 +0,0 @@ -resources: -- httproute.yaml diff --git a/registry/images/kustomization.yaml b/registry/images/kustomization.yaml new file mode 100644 index 00000000..c5c9061f --- /dev/null +++ b/registry/images/kustomization.yaml @@ -0,0 +1,7 @@ +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component + +images: +- name: registry + newName: ghcr.io/yolean/registry + newTag: 3.0.0@sha256:6c5666b861f3505b116bb9aa9b25175e71210414bd010d92035ff64018f9457e diff --git a/registry/prod-service/kustomization.yaml b/registry/prod-service/kustomization.yaml index d7f1a8c3..a25da685 100644 --- a/registry/prod-service/kustomization.yaml +++ b/registry/prod-service/kustomization.yaml @@ -1,2 +1,6 @@ +# yaml-language-server: $schema=https://json.schemastore.org/kustomization.json +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: ystack resources: - prod-registry-service-placeholder.yaml diff --git a/registry/tls/add-tls-container.yaml b/registry/tls/add-tls-container.yaml index b3b52959..d5cd0e61 100644 --- a/registry/tls/add-tls-container.yaml +++ b/registry/tls/add-tls-container.yaml @@ -7,13 +7,12 @@ spec: spec: containers: - name: docker-v2-tls - image: registry:2.8.3@sha256:12a6ddd56d2de5611ff0d9735ac0ac1d1e44073c7d042477329e589c46867e4e + image: registry resources: requests: cpu: 10m memory: 16Mi limits: - cpu: 100m memory: 200Mi ports: - containerPort: 443 diff --git a/registry/tls/kustomization.yaml b/registry/tls/kustomization.yaml index 2ad91aa3..42ed0927 100644 --- a/registry/tls/kustomization.yaml +++ b/registry/tls/kustomization.yaml @@ -1,5 +1,7 @@ bases: - ../generic +components: +- ../images resources: - rbac.yaml - job.yaml diff --git a/tekton/dashboard-release/kustomization.yaml b/tekton/dashboard-release/kustomization.yaml index 65adeceb..890c5dc0 100644 --- a/tekton/dashboard-release/kustomization.yaml +++ b/tekton/dashboard-release/kustomization.yaml @@ -1,3 +1,4 @@ +# yaml-language-server: $schema=https://json.schemastore.org/kustomization.json apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization diff --git a/tekton/release-resolvers/kustomization.yaml b/tekton/release-resolvers/kustomization.yaml index f694a778..12ecd23c 100644 --- a/tekton/release-resolvers/kustomization.yaml +++ b/tekton/release-resolvers/kustomization.yaml @@ -1,3 +1,4 @@ +# yaml-language-server: $schema=https://json.schemastore.org/kustomization.json apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization diff --git a/tekton/release/kustomization.yaml b/tekton/release/kustomization.yaml index 49722c3f..aeb60d22 100644 --- a/tekton/release/kustomization.yaml +++ b/tekton/release/kustomization.yaml @@ -1,3 +1,4 @@ +# yaml-language-server: $schema=https://json.schemastore.org/kustomization.json apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization diff --git a/tekton/triggers-release/kustomization.yaml b/tekton/triggers-release/kustomization.yaml index 993fdb74..fc226f04 100644 --- a/tekton/triggers-release/kustomization.yaml +++ b/tekton/triggers-release/kustomization.yaml @@ -1,3 +1,4 @@ +# yaml-language-server: $schema=https://json.schemastore.org/kustomization.json apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization diff --git a/versitygw/common/kustomization.yaml b/versitygw/common/kustomization.yaml deleted file mode 100644 index b1433941..00000000 --- a/versitygw/common/kustomization.yaml +++ /dev/null @@ -1,2 +0,0 @@ -resources: -- blobs-versitygw-service.yaml diff --git a/versitygw/defaultsecret/kustomization.yaml b/versitygw/defaultsecret/kustomization.yaml deleted file mode 100644 index 114baaaa..00000000 --- a/versitygw/defaultsecret/kustomization.yaml +++ /dev/null @@ -1,7 +0,0 @@ -generatorOptions: - disableNameSuffixHash: true -secretGenerator: -- name: minio - literals: - - accesskey=YstackEXAMPLEKEY - - secretkey=github.com/Yolean/ystack-EXAMPLE diff --git a/versitygw/standalone,defaultsecret/kustomization.yaml b/versitygw/standalone,defaultsecret/kustomization.yaml deleted file mode 100644 index 49e1b2ea..00000000 --- a/versitygw/standalone,defaultsecret/kustomization.yaml +++ /dev/null @@ -1,3 +0,0 @@ -bases: -- ../defaultsecret -- ../standalone diff --git a/versitygw/standalone/kustomization.yaml b/versitygw/standalone/kustomization.yaml deleted file mode 100644 index 2ca2b092..00000000 --- a/versitygw/standalone/kustomization.yaml +++ /dev/null @@ -1,5 +0,0 @@ -bases: -- ../common -resources: -- deployment.yaml -- pvc.yaml diff --git a/y-kustomize/TODO_VALIDATE.md b/y-kustomize/TODO_VALIDATE.md new file mode 100644 index 00000000..3e5523a1 --- /dev/null +++ b/y-kustomize/TODO_VALIDATE.md @@ -0,0 +1,63 @@ +# y-kustomize validation + +## Design + +The `y-kustomize/openapi/` directory is a kustomize base that produces: + +1. A Secret `y-kustomize-openapi` containing: + - `openapi.yaml` — the OpenAPI 3.1 spec + - `validate.sh` — a test script + +2. A Job `y-kustomize-openapitest` using + `ghcr.io/yolean/curl-yq:387f24cd8a6098c1dafcdb4e5fd368b13af65ca3` + that runs `validate.sh`. + +## SWS hosting + +The `y-kustomize-openapi` secret is mounted as an optional volume in the +SWS deployment, serving the spec at a discovery path such as +`/openapi.yaml`. + +## Validation script + +The script: + +1. Waits for the openapi spec to be available at the discovery URL, + confirming y-kustomize is serving and the spec secret is mounted. +2. Parses the spec with `yq` to extract all paths. +3. For each `get` endpoint in the spec: + - Fetches the URL and asserts HTTP 200. + - For `base-for-annotations.yaml` endpoints, validates that the + response parses as YAML and contains expected resource kinds + (Secret, Job). +4. Reports pass/fail per endpoint. + +Endpoints backed by optional secrets that are not yet created (e.g. +`/v1/kafka/setup-topic-job/base-for-annotations.yaml` before kafka is +installed) are expected to return 404 and should not fail the test. + +## Converge integration + +Add after the `09-y-kustomize` step in `y-cluster-converge-ystack`: + +```bash +apply_base 09-y-kustomize-openapitest +k -n ystack wait job/y-kustomize-openapitest --for=condition=complete --timeout=60s +echo "# Validated: y-kustomize API spec test passed" +``` + +This runs before any consumer (like `10-versitygw` or +`20-builds-registry-versitygw`) depends on y-kustomize. + +After `10-versitygw` creates the blobs secret and y-kustomize picks it +up, the test could optionally run again to validate the newly available +endpoint. This is not yet designed. + +## TODO + +- [ ] Create `y-kustomize/openapi/validate.sh` +- [ ] Create `y-kustomize/openapi/kustomization.yaml` with secretGenerator + and Job resource +- [ ] Add `y-kustomize-openapi` volume mount to `y-kustomize/deployment.yaml` +- [ ] Add `k3s/09-y-kustomize-openapitest/` referencing the openapi base +- [ ] Add the converge step diff --git a/y-kustomize/deployment.yaml b/y-kustomize/deployment.yaml new file mode 100644 index 00000000..cfab2dc3 --- /dev/null +++ b/y-kustomize/deployment.yaml @@ -0,0 +1,54 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: y-kustomize + labels: + app: y-kustomize + yolean.se/module-part: gateway +spec: + replicas: 1 + selector: + matchLabels: + app: y-kustomize + template: + metadata: + labels: + app: y-kustomize + spec: + containers: + - name: sws + image: ghcr.io/yolean/static-web-server:2.41.0 + args: + - --port=8787 + - --root=/srv + - --directory-listing=false + - --health + - --log-level=info + - --log-remote-address + - --ignore-hidden-files=false + - --disable-symlinks=false + ports: + - containerPort: 8787 + name: http + readinessProbe: + httpGet: + path: /health + port: 8787 + resources: + requests: + cpu: 5m + memory: 8Mi + limits: + memory: 32Mi + volumeMounts: + - name: base-blobs-setup-bucket-job + mountPath: /srv/v1/blobs/setup-bucket-job + - name: base-kafka-setup-topic-job + mountPath: /srv/v1/kafka/setup-topic-job + volumes: + - name: base-blobs-setup-bucket-job + secret: + secretName: y-kustomize.blobs.setup-bucket-job + - name: base-kafka-setup-topic-job + secret: + secretName: y-kustomize.kafka.setup-topic-job diff --git a/y-kustomize/httproute.yaml b/y-kustomize/httproute.yaml new file mode 100644 index 00000000..5e8e3318 --- /dev/null +++ b/y-kustomize/httproute.yaml @@ -0,0 +1,15 @@ +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: y-kustomize + labels: + yolean.se/module-part: gateway +spec: + parentRefs: + - name: ystack + hostnames: + - y-kustomize.ystack.svc.cluster.local + rules: + - backendRefs: + - name: y-kustomize + port: 80 diff --git a/y-kustomize/kustomization.yaml b/y-kustomize/kustomization.yaml new file mode 100644 index 00000000..f029df14 --- /dev/null +++ b/y-kustomize/kustomization.yaml @@ -0,0 +1,8 @@ +# yaml-language-server: $schema=https://json.schemastore.org/kustomization.json +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: ystack +resources: +- deployment.yaml +- service.yaml +- httproute.yaml diff --git a/y-kustomize/openapi/openapi.yaml b/y-kustomize/openapi/openapi.yaml new file mode 100644 index 00000000..b1691714 --- /dev/null +++ b/y-kustomize/openapi/openapi.yaml @@ -0,0 +1,84 @@ +openapi: 3.1.0 +info: + title: y-kustomize + version: v1 + description: | + In-cluster HTTP server providing kustomize base resources for + infrastructure setup jobs. Consumers reference these URLs in their + kustomization.yaml `resources` field. + + Each base-for-annotations.yaml is a multi-document YAML file containing: + 1. A Secret with consumer credentials and endpoint URL + 2. A Job that creates/configures the resource and is idempotent + + Consumers customize via kustomize namePrefix (which prefixes the + Secret name) and JSON patches (to set resource-specific values + like bucket name or topic name via annotations). + + The Secret uses stable names (no hash suffix) so workloads in the + namespace can reference it after the setup job completes. + + Implementations may serve different content — for example a + production implementation might return a CRD-based resource that + provisions per-namespace credentials, while a dev implementation + returns a Job with shared credentials. + +servers: +- url: http://y-kustomize.ystack.svc.cluster.local + +paths: + /v1/blobs/setup-bucket-job/base-for-annotations.yaml: + get: + operationId: getBlobsSetupBucketJob + summary: Kustomize base for S3 bucket setup + description: | + Returns a multi-document YAML containing: + - A Secret named `bucket` with keys `endpoint`, `accesskey`, `secretkey` + - A Job named `setup-bucket` that creates a bucket at the S3 endpoint + + The Job expects these values to be patched by the consumer: + - `BUCKET_NAME` env var (default: `default`) + + The Secret provides consumer-facing credentials for accessing the + bucket after setup. These may differ from the admin credentials + the Job uses to create the bucket. + responses: + "200": + description: Multi-document YAML (Secret + Job) + content: + application/yaml: + schema: + type: string + + /v1/kafka/setup-topic-job/base-for-annotations.yaml: + get: + operationId: getKafkaSetupTopicJob + summary: Kustomize base for Kafka topic setup + description: | + Returns a multi-document YAML containing: + - A Secret named `topic` with keys `bootstrap` and any credentials + - A Job named `setup-topic` that creates and configures a topic + + The Job is configured via annotations: + - `yolean.se/kafka-topic-name` (required) + - `yolean.se/kafka-topic-config` (key=value pairs) + - `yolean.se/kafka-topic-partitions` (default: "1") + - `yolean.se/kafka-topic-replicas` (default: "-1") + + The Secret provides consumer-facing connection details for + producing to or consuming from the topic after setup. + responses: + "200": + description: Multi-document YAML (Secret + Job) + content: + application/yaml: + schema: + type: string + + /health: + get: + operationId: getHealth + summary: Health check + responses: + "200": + description: Server is healthy diff --git a/y-kustomize/service.yaml b/y-kustomize/service.yaml new file mode 100644 index 00000000..7ea2d39d --- /dev/null +++ b/y-kustomize/service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: y-kustomize + labels: + app: y-kustomize + yolean.se/module-part: gateway +spec: + selector: + app: y-kustomize + ports: + - name: http + port: 80 + targetPort: 8787