diff --git a/.env.test b/.env.test index 60ffa53..5ee845a 100644 --- a/.env.test +++ b/.env.test @@ -1,4 +1,4 @@ -RUNBOAT_REPOS='[{"repo": "^oca/.*", "branch": "^15.0$", "builds": [{"image": "ghcr.io/oca/oca-ci/py3.8-odoo15.0:latest"}]}, {"repo": "^oca/.*", "branch": "^16.0$", "builds": [{"image": "ghcr.io/oca/oca-ci/py3.10-odoo16.0:latest", "kubefiles_path": "/tmp"}]}]' +RUNBOAT_REPOS='[{"repo": "^oca/.*", "branch": "^15.0$", "builds": [{"image": "ghcr.io/oca/oca-ci/py3.8-odoo15.0:latest"}]}, {"repo": "^oca/.*", "branch": "^16.0$", "builds": [{"image": "ghcr.io/oca/oca-ci/py3.10-odoo16.0:latest", "kubefiles_path": "/tmp"}]}, {"repo": "^kencove/.*", "branch": "^16.0.*$", "builds": [{"image": "ghcr.io/oca/oca-ci/py3.10-odoo16.0:latest"}], "platform": "gitlab", "project_id": "19358266"}]' RUNBOAT_API_ADMIN_USER="admin" RUNBOAT_API_ADMIN_PASSWD="admin" RUNBOAT_BUILD_NAMESPACE=runboat-builds @@ -8,4 +8,7 @@ RUNBOAT_BUILD_SECRET_ENV='{"PGPASSWORD": "thepgpassword"}' RUNBOAT_BUILD_TEMPLATE_VARS='{"storageClassName": "my-storage-class"}' RUNBOAT_GITHUB_TOKEN= RUNBOAT_GITHUB_WEBHOOK_SECRET= +RUNBOAT_GITLAB_TOKEN= +RUNBOAT_GITLAB_WEBHOOK_TOKEN= +RUNBOAT_GITLAB_URL=https://gitlab.com RUNBOAT_LOG_CONFIG=log-config.yaml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..9414a8d --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,93 @@ +name: Build, Test & Push + +on: + push: + branches: [main, "feat/**"] + pull_request: + branches: [main] + +env: + REGISTRY: us-central1-docker.pkg.dev + IMAGE: us-central1-docker.pkg.dev/kencove-prod/kencove-docker-repo/runboat + CHART_REPO: oci://us-central1-docker.pkg.dev/kencove-prod/kencove-docker-repo + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.13" + - name: Install dependencies + run: pip install -e ".[test]" + - name: Run tests + run: pytest tests/ -v + + build-image: + needs: test + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write + steps: + - uses: actions/checkout@v4 + + - id: auth + uses: google-github-actions/auth@v2 + with: + workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ secrets.GCP_SERVICE_ACCOUNT }} + + - uses: google-github-actions/setup-gcloud@v2 + + - name: Configure Docker for GCP Artifact Registry + run: gcloud auth configure-docker us-central1-docker.pkg.dev --quiet + + - name: Set image tags + id: tags + run: | + SHA_SHORT=$(echo "${{ github.sha }}" | cut -c1-12) + BRANCH=$(echo "${{ github.ref_name }}" | sed 's/[^a-zA-Z0-9._-]/-/g') + echo "sha_tag=${SHA_SHORT}" >> "$GITHUB_OUTPUT" + echo "branch_tag=${BRANCH}" >> "$GITHUB_OUTPUT" + + - name: Build Docker image + run: | + docker build \ + -t ${{ env.IMAGE }}:${{ steps.tags.outputs.sha_tag }} \ + -t ${{ env.IMAGE }}:${{ steps.tags.outputs.branch_tag }} \ + ${{ github.ref_name == 'main' && format('-t {0}:latest', env.IMAGE) || '' }} \ + . + + - name: Push Docker image + run: | + docker push ${{ env.IMAGE }}:${{ steps.tags.outputs.sha_tag }} + docker push ${{ env.IMAGE }}:${{ steps.tags.outputs.branch_tag }} + ${{ github.ref_name == 'main' && format('docker push {0}:latest', env.IMAGE) || 'true' }} + + push-chart: + needs: test + runs-on: ubuntu-latest + if: github.event_name == 'push' && github.ref_name == 'main' + permissions: + contents: read + id-token: write + steps: + - uses: actions/checkout@v4 + + - id: auth + uses: google-github-actions/auth@v2 + with: + workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ secrets.GCP_SERVICE_ACCOUNT }} + + - uses: google-github-actions/setup-gcloud@v2 + + - name: Configure Helm for GCP Artifact Registry + run: gcloud auth print-access-token | helm registry login us-central1-docker.pkg.dev -u oauth2accesstoken --password-stdin + + - name: Package and push Helm chart + run: | + helm package chart/ + helm push runboat-*.tgz ${{ env.CHART_REPO }} diff --git a/Dockerfile b/Dockerfile index dbf4a5a..add698b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,6 +22,9 @@ ENV RUNBOAT_BUILD_TEMPLATE_VARS='{}' ENV RUNBOAT_BUILD_DEFAULT_KUBEFILES_PATH= ENV RUNBOAT_GITHUB_TOKEN= ENV RUNBOAT_GITHUB_WEBHOOK_SECRET= +ENV RUNBOAT_GITLAB_TOKEN= +ENV RUNBOAT_GITLAB_WEBHOOK_TOKEN= +ENV RUNBOAT_GITLAB_URL=https://gitlab.com ENV RUNBOAT_BASE_URL=https://runboat.example.com ENV RUNBOAT_ADDITIONAL_FOOTER_HTML='' diff --git a/README.md b/README.md index 2e81c14..e31038f 100644 --- a/README.md +++ b/README.md @@ -2,26 +2,29 @@ A simple Odoo runbot lookalike on kubernetes. Main goal is replacing the OCA runbot. -[![pre-commit.ci status](https://results.pre-commit.ci/badge/github/sbidoul/runboat/main.svg)](https://results.pre-commit.ci/latest/github/sbidoul/runboat/main) +This fork adds **GitLab support** (webhooks, API, commit statuses) alongside the +existing GitHub support, plus a **Helm chart** for deployment. ## Principle of operation This program is a Kubernetes operator that manages Odoo instances with pre-installed -addons. The addons come from commits on branches and pull requests in GitHub -repositories. A deployment of a given commit of a given branch or pull request of a -given repository is known as a build. +addons. The addons come from commits on branches and pull/merge requests in GitHub +or GitLab repositories. A deployment of a given commit of a given branch or +pull/merge request of a given repository is known as a build. Runboat has the following main components: - An in-memory database of deployed builds, with their current status. - A REST API to list builds and trigger new deployments as well as start, stop, redeploy or undeploy builds. -- A GitHub webhook to automatically trigger new builds on pushes to branches and pull - requests of supported repositories and branches (configured via regular expressions). +- **GitHub webhook** (`/webhooks/github`) to automatically trigger builds on pushes and + pull requests. +- **GitLab webhook** (`/webhooks/gitlab`) to automatically trigger builds on pushes and + merge requests. - A controller that performs the following tasks: - - monitor deployments in a kubernetes namespaces to maintain the in-memory database; - - on new deployments, trigger an initialization job to check out the GitHub repo, + - monitor deployments in a kubernetes namespace to maintain the in-memory database; + - on new deployments, trigger an initialization job to check out the repo, install dependencies, create the corresponding postgres database and install the addons in it; - initialization jobs are started concurrently up to a configured limit; @@ -49,6 +52,40 @@ knowledge about *what* is deployed is in the a specific contract to be managed by the runboat controller. This contract is described in the [Kubernetes resources](#kubernetes-resources) section below. +## What's new in this fork + +### GitLab support + +GitLab support works alongside GitHub — both can be active simultaneously: + +| Feature | GitHub | GitLab | +|---------|--------|--------| +| Webhook endpoint | `/webhooks/github` | `/webhooks/gitlab` | +| Webhook auth | HMAC-SHA256 signature | `X-Gitlab-Token` header | +| Push events | `push` | `Push Hook` | +| PR/MR events | `pull_request` | `Merge Request Hook` | +| Commit statuses | `POST /repos/{owner}/{repo}/statuses/{sha}` | `POST /api/v4/projects/{id}/statuses/{sha}` | +| Repo clone | GitHub tarball URL | GitLab archive API (supports private repos) | +| API trigger | `POST /api/v1/builds/trigger/pr` | `POST /api/v1/builds/trigger/mr` | + +Configuration per repo (`RUNBOAT_REPOS`): +```json +[ + { + "repo": "^myorg/myrepo$", + "branch": "^16\\.0$", + "platform": "gitlab", + "project_id": "12345678", + "builds": [{"image": "ghcr.io/oca/oca-ci/py3.10-odoo16.0:latest"}] + } +] +``` + +### Helm chart + +A Helm chart is included in `chart/` for deploying Runboat and its PostgreSQL +dependency to Kubernetes. See [Helm deployment](#helm-deployment) below. + ## Requirements For running the builds: @@ -72,7 +109,104 @@ For running the controller (runboat itself): The controller can be run outside the kubernetes cluster or deployed inside it, or even in a different cluster. -## Deployment quickstart +## Helm deployment + +The included Helm chart (`chart/`) deploys Runboat with all dependencies: + +### Prerequisites + +- Kubernetes 1.24+ +- Helm 3.x +- nginx-ingress controller +- cert-manager (for TLS) +- CNPG operator (for managed PostgreSQL, or use external PG) +- Wildcard DNS pointing to your ingress load balancer + +### Quick start + +```bash +# 1. Create namespaces +kubectl create namespace runboat +kubectl create namespace runboat-builds + +# 2. Create your values override +cat > my-values.yaml < Webhooks > Add webhook +# URL: https://runboat.example.com/webhooks/github +# Secret: +# Events: Pushes, Pull requests +# +# GitLab: Settings > Webhooks > Add new webhook +# URL: https://runboat.example.com/webhooks/gitlab +# Secret token: +# Triggers: Push events, Merge request events +``` + +### Upgrading + +```bash +helm upgrade runboat ./chart -n runboat -f my-values.yaml +``` + +### Configuration reference + +See [`chart/values.yaml`](./chart/values.yaml) for all available settings with +inline documentation. + +## Deployment quickstart (docker-compose) A typical deployment looks like this. @@ -134,7 +268,7 @@ actually deploy. It expects the following to hold true: - `runboat/pr`: the pull request number if this build is for a pull request; - `runboat/git-commit`: the commit sha. -- the home page of a running build is exposed at `http://{build_slug}.{build_domain}`. +- the home page of a running build is exposed at `https://{build_slug}.{build_domain}`. During the lifecycle of a build, the controller does the following on the deployed resources: @@ -195,10 +329,12 @@ See environment variables examples in [Dockerfile](./Dockerfile), ## Credits -Authored by Stéphane Bidoul (@sbidoul) and +Authored by Stephane Bidoul (@sbidoul) and [contributors](https://github.com/sbidoul/runboat/graphs/contributors) with support of [ACSONE](https://acsone.eu). +GitLab support and Helm chart by [Kencove](https://kencove.com). + Contributions welcome. Do not hesitate to reach out for help on how to get started. diff --git a/chart/Chart.yaml b/chart/Chart.yaml new file mode 100644 index 0000000..eb23bf9 --- /dev/null +++ b/chart/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: runboat +description: Runboat - on-demand Odoo review environments on Kubernetes +type: application +version: 0.1.0 +appVersion: "0.2" diff --git a/chart/templates/_helpers.tpl b/chart/templates/_helpers.tpl new file mode 100644 index 0000000..f235f0f --- /dev/null +++ b/chart/templates/_helpers.tpl @@ -0,0 +1,82 @@ +{{/* +Chart name +*/}} +{{- define "runboat.name" -}} +{{- .Chart.Name | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Fullname +*/}} +{{- define "runboat.fullname" -}} +{{- printf "%s" .Release.Name | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "runboat.labels" -}} +helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }} +app.kubernetes.io/name: {{ include "runboat.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "runboat.selectorLabels" -}} +app.kubernetes.io/name: {{ include "runboat.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +PostgreSQL host — auto-derive from CNPG or use explicit override +*/}} +{{- define "runboat.dbHost" -}} +{{- if .Values.postgresql.host }} +{{- .Values.postgresql.host }} +{{- else }} +{{- printf "runboat-db-rw.%s.svc.cluster.local" .Values.namespace }} +{{- end }} +{{- end }} + +{{/* +Build PGHOST env for builds +*/}} +{{- define "runboat.buildEnv" -}} +{{- $env := dict "PGHOST" (include "runboat.dbHost" .) "PGPORT" .Values.postgresql.port "PGUSER" .Values.postgresql.user }} +{{- range $k, $v := .Values.config.buildEnv }} +{{- $_ := set $env $k $v }} +{{- end }} +{{- $env | toJson }} +{{- end }} + +{{/* +Build secret env for builds +*/}} +{{- define "runboat.buildSecretEnv" -}} +{{- $env := dict "PGPASSWORD" .Values.postgresql.password }} +{{- range $k, $v := .Values.config.buildSecretEnv }} +{{- $_ := set $env $k $v }} +{{- end }} +{{- $env | toJson }} +{{- end }} + +{{/* +Build template vars +*/}} +{{- define "runboat.buildTemplateVars" -}} +{{- $vars := dict "storageClassName" .Values.buildStorage.storageClassName }} +{{- range $k, $v := .Values.config.buildTemplateVars }} +{{- $_ := set $vars $k $v }} +{{- end }} +{{- $vars | toJson }} +{{- end }} + +{{/* +Repos JSON +*/}} +{{- define "runboat.repos" -}} +{{- .Values.repos | toJson }} +{{- end }} diff --git a/chart/templates/cnpg-cluster.yaml b/chart/templates/cnpg-cluster.yaml new file mode 100644 index 0000000..15ccf90 --- /dev/null +++ b/chart/templates/cnpg-cluster.yaml @@ -0,0 +1,66 @@ +{{- if .Values.postgresql.cnpg.enabled }} +apiVersion: v1 +kind: Secret +metadata: + name: runboat-db-superuser + namespace: {{ .Values.namespace }} + labels: + {{- include "runboat.labels" . | nindent 4 }} +type: kubernetes.io/basic-auth +stringData: + username: postgres + password: {{ .Values.postgresql.cnpg.superuserPassword | quote }} +--- +## App user secret — CNPG uses this for the DB owner instead of auto-generating +apiVersion: v1 +kind: Secret +metadata: + name: runboat-db-app + namespace: {{ .Values.namespace }} + labels: + {{- include "runboat.labels" . | nindent 4 }} + cnpg.io/reload: "true" +type: kubernetes.io/basic-auth +stringData: + username: {{ .Values.postgresql.user | quote }} + password: {{ .Values.postgresql.password | quote }} + dbname: runboat + host: {{ include "runboat.dbHost" . }} + port: {{ .Values.postgresql.port | quote }} +--- +apiVersion: postgresql.cnpg.io/v1 +kind: Cluster +metadata: + name: runboat-db + namespace: {{ .Values.namespace }} + labels: + {{- include "runboat.labels" . | nindent 4 }} +spec: + imageName: {{ .Values.postgresql.cnpg.image | quote }} + {{- with .Values.postgresql.cnpg.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 4 }} + {{- end }} + instances: {{ .Values.postgresql.cnpg.instances }} + storage: + size: {{ .Values.postgresql.cnpg.storage.size }} + {{- if .Values.postgresql.cnpg.storage.storageClass }} + storageClass: {{ .Values.postgresql.cnpg.storage.storageClass }} + {{- end }} + postgresql: + parameters: + {{- toYaml .Values.postgresql.cnpg.parameters | nindent 6 }} + enableSuperuserAccess: true + superuserSecret: + name: runboat-db-superuser + bootstrap: + initdb: + database: runboat + owner: {{ .Values.postgresql.user }} + secret: + name: runboat-db-app + postInitApplicationSQL: + - ALTER ROLE {{ .Values.postgresql.user }} CREATEDB + resources: + {{- toYaml .Values.postgresql.cnpg.resources | nindent 4 }} +{{- end }} diff --git a/chart/templates/configmap.yaml b/chart/templates/configmap.yaml new file mode 100644 index 0000000..e8e1193 --- /dev/null +++ b/chart/templates/configmap.yaml @@ -0,0 +1,21 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "runboat.fullname" . }}-config + namespace: {{ .Values.namespace }} + labels: + {{- include "runboat.labels" . | nindent 4 }} +data: + RUNBOAT_BUILD_NAMESPACE: {{ .Values.buildNamespace | quote }} + RUNBOAT_BUILD_DOMAIN: {{ .Values.config.buildDomain | quote }} + RUNBOAT_BASE_URL: {{ .Values.config.baseUrl | quote }} + RUNBOAT_GITLAB_URL: {{ .Values.gitlab.url | quote }} + RUNBOAT_MAX_INITIALIZING: {{ .Values.config.maxInitializing | quote }} + RUNBOAT_MAX_STARTED: {{ .Values.config.maxStarted | quote }} + RUNBOAT_MAX_DEPLOYED: {{ .Values.config.maxDeployed | quote }} + RUNBOAT_NO_CLEANUP_JOB: {{ .Values.config.noCleanupJob | quote }} + RUNBOAT_DISABLE_COMMIT_STATUSES: {{ .Values.config.disableCommitStatuses | quote }} + RUNBOAT_ADDITIONAL_FOOTER_HTML: {{ .Values.config.additionalFooterHtml | quote }} + RUNBOAT_BUILD_ENV: {{ include "runboat.buildEnv" . | squote }} + RUNBOAT_BUILD_TEMPLATE_VARS: {{ include "runboat.buildTemplateVars" . | squote }} + RUNBOAT_REPOS: {{ include "runboat.repos" . | squote }} diff --git a/chart/templates/deployment.yaml b/chart/templates/deployment.yaml new file mode 100644 index 0000000..f3dc9d4 --- /dev/null +++ b/chart/templates/deployment.yaml @@ -0,0 +1,52 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "runboat.fullname" . }} + namespace: {{ .Values.namespace }} + labels: + {{- include "runboat.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "runboat.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "runboat.selectorLabels" . | nindent 8 }} + annotations: + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + checksum/secret: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }} + spec: + serviceAccountName: {{ .Values.serviceAccount.name }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: runboat + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - name: http + containerPort: 8000 + protocol: TCP + envFrom: + - secretRef: + name: {{ include "runboat.fullname" . }}-env + - configMapRef: + name: {{ include "runboat.fullname" . }}-config + resources: + {{- toYaml .Values.resources | nindent 12 }} + livenessProbe: + httpGet: + path: /api/v1/status + port: http + initialDelaySeconds: 10 + periodSeconds: 30 + readinessProbe: + httpGet: + path: /api/v1/status + port: http + initialDelaySeconds: 5 + periodSeconds: 10 diff --git a/chart/templates/ingress.yaml b/chart/templates/ingress.yaml new file mode 100644 index 0000000..cbe5f07 --- /dev/null +++ b/chart/templates/ingress.yaml @@ -0,0 +1,34 @@ +{{- if .Values.ingress.enabled }} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ include "runboat.fullname" . }} + namespace: {{ .Values.namespace }} + labels: + {{- include "runboat.labels" . | nindent 4 }} + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if .Values.ingress.className }} + ingressClassName: {{ .Values.ingress.className }} + {{- end }} + {{- if .Values.ingress.tls.enabled }} + tls: + - hosts: + - {{ .Values.ingress.host }} + secretName: {{ .Values.ingress.tls.secretName }} + {{- end }} + rules: + - host: {{ .Values.ingress.host }} + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: {{ include "runboat.fullname" . }} + port: + name: http +{{- end }} diff --git a/chart/templates/rbac.yaml b/chart/templates/rbac.yaml new file mode 100644 index 0000000..5eb5ad1 --- /dev/null +++ b/chart/templates/rbac.yaml @@ -0,0 +1,39 @@ +{{- if .Values.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "runboat.fullname" . }} + labels: + {{- include "runboat.labels" . | nindent 4 }} +rules: + - apiGroups: ["apps"] + resources: ["deployments"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + - apiGroups: ["batch"] + resources: ["jobs"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete", "deletecollection"] + - apiGroups: [""] + resources: ["services", "persistentvolumeclaims", "configmaps", "secrets", "pods"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete", "deletecollection"] + - apiGroups: [""] + resources: ["pods/log"] + verbs: ["get"] + - apiGroups: ["networking.k8s.io"] + resources: ["ingresses"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "runboat.fullname" . }} + labels: + {{- include "runboat.labels" . | nindent 4 }} +subjects: + - kind: ServiceAccount + name: {{ .Values.serviceAccount.name }} + namespace: {{ .Values.namespace }} +roleRef: + kind: ClusterRole + name: {{ include "runboat.fullname" . }} + apiGroup: rbac.authorization.k8s.io +{{- end }} diff --git a/chart/templates/secret.yaml b/chart/templates/secret.yaml new file mode 100644 index 0000000..2dc6240 --- /dev/null +++ b/chart/templates/secret.yaml @@ -0,0 +1,24 @@ +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "runboat.fullname" . }}-env + namespace: {{ .Values.namespace }} + labels: + {{- include "runboat.labels" . | nindent 4 }} +type: Opaque +stringData: + RUNBOAT_API_ADMIN_USER: {{ .Values.config.apiAdminUser | quote }} + RUNBOAT_API_ADMIN_PASSWD: {{ .Values.config.apiAdminPassword | quote }} + {{- if .Values.github.token }} + RUNBOAT_GITHUB_TOKEN: {{ .Values.github.token | quote }} + {{- end }} + {{- if .Values.github.webhookSecret }} + RUNBOAT_GITHUB_WEBHOOK_SECRET: {{ .Values.github.webhookSecret | quote }} + {{- end }} + {{- if .Values.gitlab.token }} + RUNBOAT_GITLAB_TOKEN: {{ .Values.gitlab.token | quote }} + {{- end }} + {{- if .Values.gitlab.webhookToken }} + RUNBOAT_GITLAB_WEBHOOK_TOKEN: {{ .Values.gitlab.webhookToken | quote }} + {{- end }} + RUNBOAT_BUILD_SECRET_ENV: {{ include "runboat.buildSecretEnv" . | squote }} diff --git a/chart/templates/service.yaml b/chart/templates/service.yaml new file mode 100644 index 0000000..3dfe63e --- /dev/null +++ b/chart/templates/service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "runboat.fullname" . }} + namespace: {{ .Values.namespace }} + labels: + {{- include "runboat.labels" . | nindent 4 }} +spec: + type: ClusterIP + ports: + - port: 80 + targetPort: http + protocol: TCP + name: http + selector: + {{- include "runboat.selectorLabels" . | nindent 4 }} diff --git a/chart/templates/serviceaccount.yaml b/chart/templates/serviceaccount.yaml new file mode 100644 index 0000000..bc3aaff --- /dev/null +++ b/chart/templates/serviceaccount.yaml @@ -0,0 +1,9 @@ +{{- if .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.serviceAccount.name }} + namespace: {{ .Values.namespace }} + labels: + {{- include "runboat.labels" . | nindent 4 }} +{{- end }} diff --git a/chart/values.yaml b/chart/values.yaml new file mode 100644 index 0000000..f59b60f --- /dev/null +++ b/chart/values.yaml @@ -0,0 +1,244 @@ +## --------------------------------------------------------------- +## Runboat Helm Chart — values.yaml +## --------------------------------------------------------------- +## Deploy Runboat (K8s operator for Odoo review environments) with +## support for GitHub and/or GitLab webhooks. +## --------------------------------------------------------------- + +## --------------------------------------------------------------- +## Controller image +## --------------------------------------------------------------- +image: + repository: ghcr.io/your-org/runboat + tag: latest + pullPolicy: Always + +## If your image is in a private registry, list pull secrets here. +## The secret must exist in the runboat namespace. +imagePullSecrets: [] + # - name: my-registry-secret + +replicaCount: 1 + +resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi + +## --------------------------------------------------------------- +## Namespaces +## --------------------------------------------------------------- +## namespace: where Runboat controller runs +## buildNamespace: where per-build K8s resources are created +namespace: runboat +buildNamespace: runboat-builds + +## --------------------------------------------------------------- +## Ingress for the Runboat UI / API +## --------------------------------------------------------------- +ingress: + enabled: true + className: nginx + host: runboat.example.com + annotations: + ## cert-manager will auto-provision a TLS certificate: + cert-manager.io/cluster-issuer: letsencrypt-prod + nginx.ingress.kubernetes.io/proxy-body-size: 10m + tls: + enabled: true + secretName: runboat-tls + +## --------------------------------------------------------------- +## TLS for per-build ingresses (wildcard certificate) +## --------------------------------------------------------------- +## Builds get URLs like .runboat.example.com. +## Create a wildcard Certificate CR in the builds namespace so all +## build ingresses share one cert: +## +## apiVersion: cert-manager.io/v1 +## kind: Certificate +## metadata: +## name: runboat-builds-wildcard +## namespace: runboat-builds # must match buildNamespace +## spec: +## secretName: runboat-builds-wildcard-tls +## dnsNames: +## - "*.runboat.example.com" +## issuerRef: +## kind: ClusterIssuer +## name: letsencrypt-prod # must support DNS-01 +## +## The kubefiles reference this secret automatically: +## spec.tls[].secretName: runboat-builds-wildcard-tls +## +## NOTE: Wildcard certs require a DNS-01 solver (e.g. Cloudflare, +## Route53, DigitalOcean). HTTP-01 cannot issue wildcards. +## --------------------------------------------------------------- + +## --------------------------------------------------------------- +## RBAC +## --------------------------------------------------------------- +## Runboat needs permissions to create/delete Deployments, Jobs, +## Services, Ingresses, PVCs, Secrets, ConfigMaps in the builds +## namespace. +rbac: + create: true + +serviceAccount: + create: true + name: runboat + +## --------------------------------------------------------------- +## PostgreSQL for builds (shared instance) +## --------------------------------------------------------------- +## Runboat creates/drops a database per build on this shared PG. +## Options: +## 1. cnpg.enabled: true — deploy a CNPG Cluster CR (requires +## the CNPG operator: kubectl apply -f https://github.com/ +## cloudnative-pg/cloudnative-pg/releases/latest) +## 2. cnpg.enabled: false — use an external PostgreSQL +## --------------------------------------------------------------- +postgresql: + cnpg: + enabled: true + instances: 1 + image: "ghcr.io/cloudnative-pg/postgresql:16" + imagePullSecrets: [] + # - name: my-registry-secret + storage: + size: 20Gi + storageClass: "" # "" = cluster default; e.g. "do-block-storage" + resources: + requests: + cpu: 500m + memory: 512Mi + limits: + cpu: "2" + memory: 2Gi + parameters: + archive_mode: "off" + shared_buffers: 256MB + max_connections: "200" + ## Superuser password — CHANGE THIS in your values override + superuserPassword: CHANGE_ME + + ## Connection details (auto-derived from CNPG when cnpg.enabled) + ## Override these when using an external PostgreSQL. + host: "" # auto: runboat-db-rw..svc.cluster.local + port: "5432" + user: runboat + password: CHANGE_ME + +## --------------------------------------------------------------- +## Build storage — each build gets a PVC for addons, venv, filestore +## --------------------------------------------------------------- +buildStorage: + storageClassName: "" # "" = cluster default; e.g. "do-block-storage" + +## --------------------------------------------------------------- +## Runboat configuration +## --------------------------------------------------------------- +config: + ## API admin credentials (used for trigger endpoints) + apiAdminUser: admin + apiAdminPassword: CHANGE_ME + + ## Build concurrency limits + maxInitializing: 2 # max concurrent init jobs + maxStarted: 4 # max running Odoo instances (others scale to 0) + maxDeployed: 20 # max total builds (oldest stopped get deleted) + + ## Domain for build ingresses: . + buildDomain: runboat.example.com + + ## Runboat UI base URL (for backlinks in commit statuses) + baseUrl: https://runboat.example.com + + ## Skip cleanup jobs — just delete K8s resources on undeploy + noCleanupJob: false + + ## Disable posting commit statuses to GitHub/GitLab + disableCommitStatuses: false + + ## Additional footer HTML for the web UI + additionalFooterHtml: "" + + ## Extra env vars injected into every build container/job. + ## These are non-secret and stored in a ConfigMap. + buildEnv: {} + # PGHOST: runboat-db-rw.runboat.svc.cluster.local + # PGPORT: "5432" + # PGUSER: runboat + + ## Secret env vars injected into every build container/job. + ## Stored in a K8s Secret. Use for DB password, API tokens, etc. + buildSecretEnv: {} + # PGPASSWORD: my-db-password + # RUNBOAT_GITLAB_TOKEN: glpat-xxxxx + + ## Template vars for kubefiles Jinja rendering + buildTemplateVars: {} + # storageClassName: do-block-storage + +## --------------------------------------------------------------- +## GitHub configuration +## --------------------------------------------------------------- +## token: Personal Access Token with repo + statuses scope. +## Used to fetch branch/PR info and post commit statuses. +## webhookSecret: HMAC secret for verifying GitHub webhook payloads. +## Set the same value in the GitHub repo webhook config. +github: + token: "" + webhookSecret: "" + +## --------------------------------------------------------------- +## GitLab configuration (new — works alongside GitHub) +## --------------------------------------------------------------- +## token: Project Access Token or PAT with api + read_repository scope. +## Used to fetch branch/MR info, post commit statuses, and +## download repo archives for private repos. +## webhookToken: Plain token for X-Gitlab-Token header verification. +## Set the same value in the GitLab project webhook config. +## url: Base URL of your GitLab instance. Defaults to gitlab.com. +## Change for self-hosted: https://gitlab.mycompany.com +gitlab: + token: "" + webhookToken: "" + url: https://gitlab.com + +## --------------------------------------------------------------- +## Repository configuration +## --------------------------------------------------------------- +## Each entry defines a repo pattern, branch pattern, build image, +## and optionally platform + project_id for GitLab repos. +## +## Fields: +## repo: Regex matched against incoming webhook repo path +## branch: Regex matched against target branch +## platform: "github" (default) or "gitlab" +## project_id: GitLab numeric project ID (required for gitlab) +## builds: List with exactly one build config: +## image: Container image for the build +## env: Extra env vars (extends config.buildEnv) +## secret_env: Extra secret env vars +## template_vars: Extra template vars +## kubefiles_path: Override kubefiles directory +## --------------------------------------------------------------- +repos: [] + ## GitHub example — OCA repos: + # - repo: "^oca/.*" + # branch: "^16\\.0$" + # platform: github + # builds: + # - image: "ghcr.io/oca/oca-ci/py3.10-odoo16.0:latest" + + ## GitLab example — private Odoo addon repo: + # - repo: "^myorg/odoo/addons/myrepo$" + # branch: "^16\\.0.*$" + # platform: gitlab + # project_id: "12345678" + # builds: + # - image: "ghcr.io/oca/oca-ci/py3.10-odoo16.0:latest" diff --git a/src/runboat/api.py b/src/runboat/api.py index 8e141c0..91897a4 100644 --- a/src/runboat/api.py +++ b/src/runboat/api.py @@ -9,10 +9,11 @@ from sse_starlette.sse import EventSourceResponse from starlette.status import HTTP_404_NOT_FOUND -from . import github, models +from . import github, gitlab, models from .controller import Controller, controller from .db import SortOrder from .deps import authenticated +from .settings import settings router = APIRouter() @@ -108,7 +109,16 @@ async def undeploy_builds( ) async def trigger_branch(repo: str, branch: str) -> None: """Trigger build for a branch.""" - commit_info = await github.get_branch_info(repo, branch) + try: + repo_settings = settings.get_repo_settings(repo, branch) + except Exception: + repo_settings = None + if repo_settings and repo_settings.platform == "gitlab": + commit_info = await gitlab.get_branch_info( + repo, branch, project_id=repo_settings.project_id + ) + else: + commit_info = await github.get_branch_info(repo, branch) await controller.deploy_commit(commit_info) @@ -122,6 +132,18 @@ async def trigger_pull(repo: str, pr: int) -> None: await controller.deploy_commit(commit_info) +@router.post( + "/builds/trigger/mr", + dependencies=[Depends(authenticated)], +) +async def trigger_merge_request( + repo: str, mr: int, project_id: str | None = None +) -> None: + """Trigger build for a GitLab merge request.""" + commit_info = await gitlab.get_merge_request_info(repo, mr, project_id=project_id) + await controller.deploy_commit(commit_info) + + async def _build_by_name(name: str) -> models.Build: build = await controller.get_build(name) if build is None: diff --git a/src/runboat/db.py b/src/runboat/db.py index bc984e8..23c47a1 100644 --- a/src/runboat/db.py +++ b/src/runboat/db.py @@ -41,7 +41,9 @@ def register_listener(self, listener: BuildListener) -> None: @classmethod def _build_from_row(cls, row: "sqlite3.Row") -> Build: - commit_info_fields = {"repo", "target_branch", "pr", "git_commit"} + commit_info_fields = { + "repo", "target_branch", "pr", "git_commit", "platform", "project_id", + } commit_info = CommitInfo(**{k: row[k] for k in commit_info_fields}) return Build( commit_info=commit_info, @@ -59,6 +61,8 @@ def reset(self) -> None: " target_branch TEXT NOT NULL, " " pr INTEGER, " " git_commit TEXT NOT NULL, " + " platform TEXT NOT NULL DEFAULT 'github', " + " project_id TEXT, " " desired_replicas INTEGER NOT NULL," " status TEXT NOT NULL, " " init_status TEXT NOT NULL, " @@ -117,13 +121,15 @@ def add(self, build: Build) -> None: " target_branch," " pr," " git_commit," + " platform," + " project_id," " desired_replicas," " status," " init_status, " " last_scaled, " " created" ") " - "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", ( build.name, build.deployment_name, @@ -131,6 +137,8 @@ def add(self, build: Build) -> None: build.commit_info.target_branch, build.commit_info.pr, build.commit_info.git_commit, + build.commit_info.platform, + build.commit_info.project_id, build.desired_replicas, build.status, build.init_status, diff --git a/src/runboat/exceptions.py b/src/runboat/exceptions.py index 6956f35..52a03b4 100644 --- a/src/runboat/exceptions.py +++ b/src/runboat/exceptions.py @@ -14,5 +14,9 @@ class NotFoundOnGitHub(ClientError): pass +class NotFoundOnGitLab(ClientError): + pass + + class RepoOrBranchNotSupported(ClientError): pass diff --git a/src/runboat/github.py b/src/runboat/github.py index 63ed7ed..f0d3d2e 100644 --- a/src/runboat/github.py +++ b/src/runboat/github.py @@ -31,6 +31,8 @@ class CommitInfo(BaseModel): target_branch: str pr: int | None git_commit: str + platform: str = "github" # "github" or "gitlab" + project_id: str | None = None # GitLab numeric project ID @field_validator("repo") def validate_repo(cls, v: str) -> str: diff --git a/src/runboat/gitlab.py b/src/runboat/gitlab.py new file mode 100644 index 0000000..36ba2ad --- /dev/null +++ b/src/runboat/gitlab.py @@ -0,0 +1,106 @@ +import logging +from enum import Enum +from typing import Any +from urllib.parse import quote + +import httpx + +from .exceptions import NotFoundOnGitLab +from .github import CommitInfo +from .settings import settings + +_logger = logging.getLogger(__name__) + + +async def _gitlab_request(method: str, url: str, json: Any = None) -> Any: + async with httpx.AsyncClient() as client: + full_url = f"{settings.gitlab_url}/api/v4{url}" + headers: dict[str, str] = {} + if settings.gitlab_token: + # Support both OAuth/Bearer tokens and personal access tokens + headers["Authorization"] = f"Bearer {settings.gitlab_token}" + response = await client.request(method, full_url, headers=headers, json=json) + if response.status_code == 404: + raise NotFoundOnGitLab(f"GitLab URL not found: {full_url}.") + response.raise_for_status() + return response.json() + + +async def get_branch_info( + repo: str, branch: str, project_id: str | None = None +) -> CommitInfo: + if project_id: + branch_data = await _gitlab_request( + "GET", + f"/projects/{project_id}/repository/branches/{quote(branch, safe='')}", + ) + else: + encoded_repo = quote(repo, safe="") + branch_data = await _gitlab_request( + "GET", + f"/projects/{encoded_repo}/repository/branches/{quote(branch, safe='')}", + ) + return CommitInfo( + repo=repo, + target_branch=branch, + pr=None, + git_commit=branch_data["commit"]["id"], + platform="gitlab", + project_id=project_id, + ) + + +async def get_merge_request_info( + repo: str, mr_iid: int, project_id: str | None = None +) -> CommitInfo: + pid = project_id or quote(repo, safe="") + mr_data = await _gitlab_request( + "GET", + f"/projects/{pid}/merge_requests/{mr_iid}", + ) + return CommitInfo( + repo=repo, + target_branch=mr_data["target_branch"], + pr=mr_iid, + git_commit=mr_data["sha"], + platform="gitlab", + project_id=project_id, + ) + + +class GitLabStatusState(str, Enum): + pending = "pending" + running = "running" + success = "success" + failed = "failed" + canceled = "canceled" + + +async def notify_status( + repo: str, + sha: str, + state: GitLabStatusState, + target_url: str | None, + project_id: str | None = None, +) -> None: + if settings.disable_commit_statuses: + return + pid = project_id or quote(repo, safe="") + json_payload: dict[str, Any] = { + "state": state.value, + "context": "runboat/build", + "name": "runboat/build", + } + if target_url: + json_payload["target_url"] = target_url + try: + await _gitlab_request( + "POST", + f"/projects/{pid}/statuses/{sha}", + json=json_payload, + ) + except httpx.HTTPStatusError as e: + _logger.error( + f"Failed to post GitLab commit status (code {e.response.status_code}):\n" + f"{e.response.text}" + ) diff --git a/src/runboat/k8s.py b/src/runboat/k8s.py index af946aa..0e9f70c 100644 --- a/src/runboat/k8s.py +++ b/src/runboat/k8s.py @@ -154,6 +154,7 @@ class DeploymentVars(BaseModel): build_env: dict[str, str] build_secret_env: dict[str, str] build_template_vars: dict[str, str] + gitlab_url: str = "https://gitlab.com" def make_deployment_vars( @@ -176,6 +177,7 @@ def make_deployment_vars( build_env=settings.build_env | build_settings.env, build_secret_env=settings.build_secret_env | build_settings.secret_env, build_template_vars=settings.build_template_vars | build_settings.template_vars, + gitlab_url=settings.gitlab_url, ) diff --git a/src/runboat/kubefiles/ingress_mailhog.yaml b/src/runboat/kubefiles/ingress_mailhog.yaml index fd6442f..b0ddc1c 100644 --- a/src/runboat/kubefiles/ingress_mailhog.yaml +++ b/src/runboat/kubefiles/ingress_mailhog.yaml @@ -3,6 +3,9 @@ kind: Ingress metadata: name: mailhog spec: + ingressClassName: nginx + tls: + - secretName: runboat-builds-wildcard-tls rules: - http: paths: diff --git a/src/runboat/kubefiles/ingress_odoo.yaml b/src/runboat/kubefiles/ingress_odoo.yaml index b15e579..773eb75 100644 --- a/src/runboat/kubefiles/ingress_odoo.yaml +++ b/src/runboat/kubefiles/ingress_odoo.yaml @@ -3,6 +3,9 @@ kind: Ingress metadata: name: odoo spec: + ingressClassName: nginx + tls: + - secretName: runboat-builds-wildcard-tls rules: - http: paths: diff --git a/src/runboat/kubefiles/initialize.yaml b/src/runboat/kubefiles/initialize.yaml index 5efa327..798f950 100644 --- a/src/runboat/kubefiles/initialize.yaml +++ b/src/runboat/kubefiles/initialize.yaml @@ -27,8 +27,8 @@ spec: args: ["bash", "/runboat/runboat-initialize.sh"] resources: limits: - cpu: 1000m - memory: 1Gi + cpu: 2000m + memory: 2Gi requests: cpu: 1000m memory: 1Gi @@ -42,4 +42,4 @@ spec: restartPolicy: Never completions: 1 backoffLimit: 0 - activeDeadlineSeconds: 1200 + activeDeadlineSeconds: 3600 diff --git a/src/runboat/kubefiles/kustomization.yaml.jinja b/src/runboat/kubefiles/kustomization.yaml.jinja index 0cdac10..b3614a2 100644 --- a/src/runboat/kubefiles/kustomization.yaml.jinja +++ b/src/runboat/kubefiles/kustomization.yaml.jinja @@ -27,6 +27,8 @@ commonAnnotations: runboat/target-branch: "{{ commit_info.target_branch }}" runboat/pr: "{{ commit_info.pr if commit_info.pr else '' }}" runboat/git-commit: "{{ commit_info.git_commit }}" + runboat/platform: "{{ commit_info.platform }}" + runboat/project-id: "{{ commit_info.project_id if commit_info.project_id else '' }}" images: - name: odoo @@ -50,6 +52,9 @@ configMapGenerator: - ADDONS_DIR=/mnt/data/odoo-addons-dir - RUNBOAT_GIT_REPO={{ commit_info.repo }} - RUNBOAT_GIT_REF={{ commit_info.git_commit }} + - RUNBOAT_PLATFORM={{ commit_info.platform }} + - RUNBOAT_PROJECT_ID={{ commit_info.project_id if commit_info.project_id else '' }} + - RUNBOAT_GITLAB_URL={{ gitlab_url }} {%- for key, value in build_env.items() %} - {{ key }}={{ value }} {%- endfor %} diff --git a/src/runboat/kubefiles/runboat-clone-and-install.sh b/src/runboat/kubefiles/runboat-clone-and-install.sh index d29f4d0..ea288cb 100755 --- a/src/runboat/kubefiles/runboat-clone-and-install.sh +++ b/src/runboat/kubefiles/runboat-clone-and-install.sh @@ -13,7 +13,26 @@ rm -fr $ADDONS_DIR # which exceeded the default pod memory limit. mkdir -p $ADDONS_DIR cd $ADDONS_DIR -curl -sSL https://github.com/${RUNBOAT_GIT_REPO}/tarball/${RUNBOAT_GIT_REF} | tar zxf - --strip-components=1 + +if [[ "${RUNBOAT_PLATFORM}" == "gitlab" ]]; then + # GitLab archive API: /projects/:id/repository/archive.tar.gz?sha=:ref + # Use project_id if available, otherwise URL-encode the repo path + if [[ -n "${RUNBOAT_PROJECT_ID}" ]]; then + ARCHIVE_URL="${RUNBOAT_GITLAB_URL}/api/v4/projects/${RUNBOAT_PROJECT_ID}/repository/archive.tar.gz?sha=${RUNBOAT_GIT_REF}" + else + ENCODED_REPO=$(echo "${RUNBOAT_GIT_REPO}" | sed 's|/|%2F|g') + ARCHIVE_URL="${RUNBOAT_GITLAB_URL}/api/v4/projects/${ENCODED_REPO}/repository/archive.tar.gz?sha=${RUNBOAT_GIT_REF}" + fi + # Use Bearer auth if RUNBOAT_GITLAB_TOKEN is set (for private repos) + if [[ -n "${RUNBOAT_GITLAB_TOKEN}" ]]; then + curl -sSL -H "Authorization: Bearer ${RUNBOAT_GITLAB_TOKEN}" "${ARCHIVE_URL}" | tar zxf - --strip-components=1 + else + curl -sSL "${ARCHIVE_URL}" | tar zxf - --strip-components=1 + fi +else + # GitHub tarball URL + curl -sSL https://github.com/${RUNBOAT_GIT_REPO}/tarball/${RUNBOAT_GIT_REF} | tar zxf - --strip-components=1 +fi # Install. INSTALL_METHOD=${INSTALL_METHOD:-oca_install_addons} diff --git a/src/runboat/kubefiles/runboat-start.sh b/src/runboat/kubefiles/runboat-start.sh index ef67bbd..5721cac 100755 --- a/src/runboat/kubefiles/runboat-start.sh +++ b/src/runboat/kubefiles/runboat-start.sh @@ -38,7 +38,7 @@ oca_wait_for_postgres # --db_user is necessary for Odoo <= 10 unbuffer $(which odoo || which openerp-server) \ --data-dir=/mnt/data/odoo-data-dir \ - --db-filter=^${PGDATABASE} \ + --db-filter='^${PGDATABASE}$$' \ --db_user=${PGUSER} \ --smtp=localhost \ --smtp-port=1025 diff --git a/src/runboat/models.py b/src/runboat/models.py index ac17dba..126db42 100644 --- a/src/runboat/models.py +++ b/src/runboat/models.py @@ -7,13 +7,46 @@ from kubernetes.client.models.v1_deployment import V1Deployment from pydantic import BaseModel, ConfigDict -from . import github, k8s +from . import github, gitlab, k8s from .github import CommitInfo, GitHubStatusState +from .gitlab import GitLabStatusState from .settings import settings from .utils import slugify _logger = logging.getLogger(__name__) +# Mapping from GitHub status states to GitLab equivalents +_GITHUB_TO_GITLAB_STATUS = { + GitHubStatusState.pending: GitLabStatusState.pending, + GitHubStatusState.success: GitLabStatusState.success, + GitHubStatusState.failure: GitLabStatusState.failed, + GitHubStatusState.error: GitLabStatusState.failed, +} + + +async def _notify_status( + commit_info: CommitInfo, + state: GitHubStatusState, + target_url: str | None, +) -> None: + """Platform-aware status notification dispatch.""" + if commit_info.platform == "gitlab": + gitlab_state = _GITHUB_TO_GITLAB_STATUS[state] + await gitlab.notify_status( + commit_info.repo, + commit_info.git_commit, + gitlab_state, + target_url, + project_id=commit_info.project_id, + ) + else: + await github.notify_status( + commit_info.repo, + commit_info.git_commit, + state, + target_url, + ) + class BuildEvent(str, Enum): modified = "upd" @@ -83,6 +116,11 @@ def from_deployment(cls, deployment: V1Deployment) -> "Build": target_branch=deployment.metadata.annotations["runboat/target-branch"], pr=deployment.metadata.annotations.get("runboat/pr") or None, git_commit=deployment.metadata.annotations["runboat/git-commit"], + platform=deployment.metadata.annotations.get( + "runboat/platform", "github" + ), + project_id=deployment.metadata.annotations.get("runboat/project-id") + or None, ), init_status=deployment.metadata.annotations["runboat/init-status"], status=cls._status_from_deployment(deployment), @@ -122,7 +160,8 @@ def make_slug( ) -> str: slug = f"{slugify(commit_info.repo)}-{slugify(commit_info.target_branch)}" if commit_info.pr: - slug = f"{slug}-pr{slugify(commit_info.pr)}" + prefix = "mr" if commit_info.platform == "gitlab" else "pr" + slug = f"{slug}-{prefix}{slugify(commit_info.pr)}" slug = f"{slug}-{commit_info.git_commit[:12]}" return slug @@ -132,16 +171,27 @@ def slug(self) -> str: @property def deploy_link(self) -> str: - return f"http://{self.slug}.{settings.build_domain}" + return f"https://{self.slug}.{settings.build_domain}" @property def deploy_link_mailhog(self) -> str: - return f"http://{self.slug}.mail.{settings.build_domain}" + return f"https://{self.slug}.mail.{settings.build_domain}" + + @property + def _repo_base_url(self) -> str: + if self.commit_info.platform == "gitlab": + return f"{settings.gitlab_url}/{self.commit_info.repo}" + return f"https://github.com/{self.commit_info.repo}" @property def repo_target_branch_link(self) -> str: + if self.commit_info.platform == "gitlab": + return ( + f"{self._repo_base_url}" + f"/-/tree/{self.commit_info.target_branch}" + ) return ( - f"https://github.com/{self.commit_info.repo}" + f"{self._repo_base_url}" f"/tree/{self.commit_info.target_branch}" ) @@ -149,18 +199,23 @@ def repo_target_branch_link(self) -> str: def repo_pr_link(self) -> str | None: if not self.commit_info.pr: return None - return f"https://github.com/{self.commit_info.repo}/pull/{self.commit_info.pr}" + if self.commit_info.platform == "gitlab": + return ( + f"{self._repo_base_url}" + f"/-/merge_requests/{self.commit_info.pr}" + ) + return f"{self._repo_base_url}/pull/{self.commit_info.pr}" @property def repo_commit_link(self) -> str: - link = f"https://github.com/{self.commit_info.repo}" + if self.commit_info.platform == "gitlab": + return f"{self._repo_base_url}/-/commit/{self.commit_info.git_commit}" if self.commit_info.pr: return ( - f"{link}/pull/{self.commit_info.pr}" + f"{self._repo_base_url}/pull/{self.commit_info.pr}" f"/commits/{self.commit_info.git_commit}" ) - else: - return f"{link}/commit/{self.commit_info.git_commit}" + return f"{self._repo_base_url}/commit/{self.commit_info.git_commit}" @property def webui_link(self) -> str: @@ -205,12 +260,7 @@ async def deploy(cls, commit_info: CommitInfo) -> None: await cls._deploy( commit_info, name, slug, job_kind=k8s.DeploymentMode.deployment ) - await github.notify_status( - commit_info.repo, - commit_info.git_commit, - GitHubStatusState.pending, - target_url=None, - ) + await _notify_status(commit_info, GitHubStatusState.pending, target_url=None) async def start(self) -> None: """Start build if init succeeded, or reinitialize if failed.""" @@ -305,11 +355,8 @@ async def on_initialize_started(self) -> None: return _logger.info(f"Initialization job started for {self}.") if await self._patch(init_status=BuildInitStatus.started, desired_replicas=0): - await github.notify_status( - self.commit_info.repo, - self.commit_info.git_commit, - GitHubStatusState.pending, - target_url=self.live_link, + await _notify_status( + self.commit_info, GitHubStatusState.pending, self.live_link ) async def on_initialize_succeeded(self) -> None: @@ -325,11 +372,8 @@ async def on_initialize_succeeded(self) -> None: job_kind=k8s.DeploymentMode.stop, ) if await self._patch(init_status=BuildInitStatus.succeeded): - await github.notify_status( - self.commit_info.repo, - self.commit_info.git_commit, - GitHubStatusState.success, - target_url=self.live_link, + await _notify_status( + self.commit_info, GitHubStatusState.success, self.live_link ) async def on_initialize_failed(self) -> None: @@ -345,11 +389,8 @@ async def on_initialize_failed(self) -> None: job_kind=k8s.DeploymentMode.stop, ) if await self._patch(init_status=BuildInitStatus.failed, desired_replicas=0): - await github.notify_status( - self.commit_info.repo, - self.commit_info.git_commit, - GitHubStatusState.failure, - target_url=self.live_link, + await _notify_status( + self.commit_info, GitHubStatusState.failure, self.live_link ) async def on_cleanup_started(self) -> None: @@ -415,7 +456,10 @@ class Repo(BaseModel): model_config = ConfigDict(from_attributes=True) name: str + platform: str = "github" @property def link(self) -> str: + if self.platform == "gitlab": + return f"{settings.gitlab_url}/{self.name}" return f"https://github.com/{self.name}" diff --git a/src/runboat/settings.py b/src/runboat/settings.py index e915afc..dfddc1c 100644 --- a/src/runboat/settings.py +++ b/src/runboat/settings.py @@ -30,6 +30,8 @@ class RepoSettings(BaseModel): repo: str # regex branch: str # regex builds: list[BuildSettings] + platform: str = "github" # "github" or "gitlab" + project_id: str | None = None # GitLab numeric project ID @field_validator("builds") def validate_builds(cls, v: list[BuildSettings]) -> list[BuildSettings]: @@ -75,6 +77,12 @@ class Settings(BaseSettings): github_token: str | None = None # The secret used to verify GitHub webhook signatures github_webhook_secret: bytes | None = None + # The token to use for GitLab API calls (PRIVATE-TOKEN). + gitlab_token: str | None = None + # Plain token for GitLab webhook verification (X-Gitlab-Token header). + gitlab_webhook_token: str | None = None + # Base URL for GitLab instance (supports self-hosted). + gitlab_url: str = "https://gitlab.com" # The file with the python logging configuration to use for the runboat controller. log_config: str | None = None # The base url where the runboat UI and API is exposed on internet. @@ -112,5 +120,16 @@ def is_repo_and_branch_supported(self, repo: str, target_branch: str) -> bool: else: return True + def get_repo_settings(self, repo: str, target_branch: str) -> RepoSettings: + for repo_settings in self.repos: + if not re.match(repo_settings.repo, repo, re.IGNORECASE): + continue + if not re.match(repo_settings.branch, target_branch): + continue + return repo_settings + raise RepoOrBranchNotSupported( + f"Branch {target_branch} of {repo} not supported." + ) + settings = Settings() diff --git a/src/runboat/webhooks.py b/src/runboat/webhooks.py index 61c8f25..642c9dc 100644 --- a/src/runboat/webhooks.py +++ b/src/runboat/webhooks.py @@ -2,12 +2,15 @@ import logging from typing import Annotated -from fastapi import APIRouter, BackgroundTasks, Header, Request +from fastapi import APIRouter, BackgroundTasks, Header, HTTPException, Request from .controller import controller from .github import CommitInfo from .settings import settings +_MR_DEPLOY_ACTIONS = {"open", "reopen", "update"} +_MR_UNDEPLOY_ACTIONS = {"close", "merge"} + _logger = logging.getLogger(__name__) router = APIRouter() @@ -88,3 +91,87 @@ async def receive_payload( git_commit=payload["after"], ), ) + + +def _verify_gitlab_token(x_gitlab_token: str | None) -> bool: + if not settings.gitlab_webhook_token: + return True + if not x_gitlab_token: + _logger.warning("Got GitLab payload without X-Gitlab-Token") + return False + if not hmac.compare_digest(x_gitlab_token, settings.gitlab_webhook_token): + _logger.warning("Got GitLab payload with invalid X-Gitlab-Token") + return False + return True + + +def _gitlab_repo_info(payload: dict) -> tuple[str, str | None]: + """Extract repo path and project_id from a GitLab webhook payload.""" + repo = payload["project"]["path_with_namespace"] + project_id = str(payload["project"]["id"]) + return repo, project_id + + +@router.post("/webhooks/gitlab") +async def receive_gitlab_payload( + background_tasks: BackgroundTasks, + request: Request, + x_gitlab_event: Annotated[str, Header(...)], + x_gitlab_token: Annotated[str | None, Header(...)] = None, +) -> None: + if not _verify_gitlab_token(x_gitlab_token): + raise HTTPException(status_code=403, detail="Invalid X-Gitlab-Token") + payload = await request.json() + if x_gitlab_event == "Merge Request Hook": + repo, project_id = _gitlab_repo_info(payload) + attrs = payload["object_attributes"] + action = attrs["action"] + target_branch = attrs["target_branch"] + if not settings.is_repo_and_branch_supported(repo, target_branch): + _logger.debug( + "Ignoring GitLab MR payload for unsupported repo %s " + "or target branch %s", + repo, + target_branch, + ) + return + if action in _MR_DEPLOY_ACTIONS: + background_tasks.add_task( + controller.deploy_commit, + CommitInfo( + repo=repo, + target_branch=target_branch, + pr=attrs["iid"], + git_commit=attrs["last_commit"]["id"], + platform="gitlab", + project_id=project_id, + ), + ) + elif action in _MR_UNDEPLOY_ACTIONS: + background_tasks.add_task( + controller.undeploy_builds, + repo=repo, + pr=attrs["iid"], + ) + elif x_gitlab_event == "Push Hook": + repo, project_id = _gitlab_repo_info(payload) + target_branch = payload["ref"].split("/")[-1] + if not settings.is_repo_and_branch_supported(repo, target_branch): + _logger.debug( + "Ignoring GitLab push payload for unsupported repo %s " + "or target branch %s", + repo, + target_branch, + ) + return + background_tasks.add_task( + controller.deploy_commit, + CommitInfo( + repo=repo, + target_branch=target_branch, + pr=None, + git_commit=payload["after"], + platform="gitlab", + project_id=project_id, + ), + ) diff --git a/tests/test_render_kubefiles.py b/tests/test_render_kubefiles.py index 07e90a1..10c805b 100644 --- a/tests/test_render_kubefiles.py +++ b/tests/test_render_kubefiles.py @@ -25,6 +25,8 @@ runboat/target-branch: "15.0" runboat/pr: "" runboat/git-commit: "abcdef123456789" + runboat/platform: "github" + runboat/project-id: "" images: - name: odoo @@ -46,6 +48,9 @@ - ADDONS_DIR=/mnt/data/odoo-addons-dir - RUNBOAT_GIT_REPO=oca/mis-builder - RUNBOAT_GIT_REF=abcdef123456789 + - RUNBOAT_PLATFORM=github + - RUNBOAT_PROJECT_ID= + - RUNBOAT_GITLAB_URL=https://gitlab.com - name: runboat-scripts files: - runboat-clone-and-install.sh diff --git a/tests/test_webhook.py b/tests/test_webhook.py index fd29c93..e6ae517 100644 --- a/tests/test_webhook.py +++ b/tests/test_webhook.py @@ -5,7 +5,7 @@ from runboat.app import app from runboat.controller import controller from runboat.github import CommitInfo -from runboat.webhooks import _verify_github_signature +from runboat.webhooks import _verify_github_signature, _verify_gitlab_token client = TestClient(app) @@ -153,3 +153,156 @@ def test_verify_github_signature() -> None: b"secret", b"body", ) + + +# --- GitLab webhook tests --- + + +def test_webhook_gitlab_push(mocker: MockerFixture) -> None: + mock = mocker.patch("fastapi.BackgroundTasks.add_task") + response = client.post( + "/webhooks/gitlab", + headers={ + "X-Gitlab-Event": "Push Hook", + }, + json={ + "project": { + "path_with_namespace": "kencove/odoo/addons/ken", + "id": 19358266, + }, + "ref": "refs/heads/16.0", + "after": "abc123def", + }, + ) + response.raise_for_status() + mock.assert_called_with( + controller.deploy_commit, + CommitInfo( + repo="kencove/odoo/addons/ken", + target_branch="16.0", + pr=None, + git_commit="abc123def", + platform="gitlab", + project_id="19358266", + ), + ) + + +def test_webhook_gitlab_push_unsupported_repo(mocker: MockerFixture) -> None: + mock = mocker.patch("fastapi.BackgroundTasks.add_task") + response = client.post( + "/webhooks/gitlab", + headers={ + "X-Gitlab-Event": "Push Hook", + }, + json={ + "project": { + "path_with_namespace": "other/unsupported-repo", + "id": 99999, + }, + "ref": "refs/heads/16.0", + "after": "abc123def", + }, + ) + response.raise_for_status() + mock.assert_not_called() + + +@pytest.mark.parametrize("action", ["open", "update"]) +def test_webhook_gitlab_mr(action: str, mocker: MockerFixture) -> None: + mock = mocker.patch("fastapi.BackgroundTasks.add_task") + response = client.post( + "/webhooks/gitlab", + headers={ + "X-Gitlab-Event": "Merge Request Hook", + }, + json={ + "project": { + "path_with_namespace": "kencove/odoo/addons/ken", + "id": 19358266, + }, + "object_attributes": { + "action": action, + "iid": 42, + "target_branch": "16.0", + "source_branch": "16.0-feature-xyz", + "last_commit": {"id": "def456abc"}, + }, + }, + ) + response.raise_for_status() + mock.assert_called_with( + controller.deploy_commit, + CommitInfo( + repo="kencove/odoo/addons/ken", + target_branch="16.0", + pr=42, + git_commit="def456abc", + platform="gitlab", + project_id="19358266", + ), + ) + + +@pytest.mark.parametrize("action", ["close", "merge"]) +def test_webhook_gitlab_mr_close(action: str, mocker: MockerFixture) -> None: + mock = mocker.patch("fastapi.BackgroundTasks.add_task") + response = client.post( + "/webhooks/gitlab", + headers={ + "X-Gitlab-Event": "Merge Request Hook", + }, + json={ + "project": { + "path_with_namespace": "kencove/odoo/addons/ken", + "id": 19358266, + }, + "object_attributes": { + "action": action, + "iid": 42, + "target_branch": "16.0", + "source_branch": "16.0-feature-xyz", + "last_commit": {"id": "def456abc"}, + }, + }, + ) + response.raise_for_status() + mock.assert_called_with( + controller.undeploy_builds, + repo="kencove/odoo/addons/ken", + pr=42, + ) + + +@pytest.mark.parametrize("action", ["open", "update", "close", "merge"]) +def test_webhook_gitlab_mr_unsupported_branch( + action: str, mocker: MockerFixture +) -> None: + mock = mocker.patch("fastapi.BackgroundTasks.add_task") + response = client.post( + "/webhooks/gitlab", + headers={ + "X-Gitlab-Event": "Merge Request Hook", + }, + json={ + "project": { + "path_with_namespace": "kencove/odoo/addons/ken", + "id": 19358266, + }, + "object_attributes": { + "action": action, + "iid": 42, + "target_branch": "14.0", # branch 14.0 not declared in .env.test + "source_branch": "14.0-feature-xyz", + "last_commit": {"id": "def456abc"}, + }, + }, + ) + response.raise_for_status() + mock.assert_not_called() + + +def test_verify_gitlab_token() -> None: + # No token configured (settings.gitlab_webhook_token is empty/None) → always pass + assert _verify_gitlab_token(None) + assert _verify_gitlab_token("any-token")