theo-exp

.github/workflows/integration-tests.yml

@@ -18,6 +18,7 @@ on:
       - "keycloak/**"
       - "kubernetes/**"
       - "preprocessing/**"
+      - "sequlus/**"
       - "website/**"
       - "deploy.py"
       - "integration-tests/**"
@@ -77,6 +78,20 @@ jobs:
       - name: Create k3d cluster
         run: |
           ./deploy.py --verbose cluster --bind-all
+      - name: Wait for sequlus Docker image
+        run: |
+          echo "Waiting for ghcr.io/loculus-project/sequlus:commit-${{ env.sha }} to be available..."
+          for i in $(seq 1 120); do
+            if docker pull ghcr.io/loculus-project/sequlus:commit-${{ env.sha }} > /dev/null 2>&1; then
+              echo "Image available after $((i * 10)) seconds"
+              break
+            fi
+            if [ $i -eq 120 ]; then
+              echo "Timed out waiting for sequlus image after 20 minutes"
+              exit 1
+            fi
+            sleep 10
+          done
       - name: Deploy with Helm
         run: |
           ./deploy.py --verbose helm \

.github/workflows/sequlus-image.yml

@@ -0,0 +1,73 @@
+name: sequlus-image
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+    inputs:
+      build_arm:
+        type: boolean
+        description: "Build for ARM as well"
+        default: false
+        required: true
+  workflow_call:
+    inputs:
+      build_arm:
+        type: string
+        description: "Build for ARM as well"
+        default: "false"
+        required: true
+env:
+  DOCKER_IMAGE_NAME: ghcr.io/loculus-project/sequlus
+  BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
+  BUILD_ARM: ${{ github.event.inputs.build_arm || inputs.build_arm || github.ref == 'refs/heads/main' }}
+  sha: ${{ github.event.pull_request.head.sha || github.sha }}
+defaults:
+  run:
+    working-directory: ./sequlus
+concurrency:
+  group: ci-${{ github.ref == 'refs/heads/main' && github.run_id || github.ref }}-sequlus-${{github.event.inputs.build_arm}}
+  cancel-in-progress: true
+jobs:
+  sequlus-image:
+    name: Build sequlus Docker Image
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    permissions:
+      contents: read
+      packages: write
+      checks: read
+    steps:
+      - uses: actions/checkout@v6
+      - name: Shorten sha
+        run: echo "sha=${sha::7}" >> $GITHUB_ENV
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Docker metadata
+        id: dockerMetadata
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.DOCKER_IMAGE_NAME }}
+          tags: |
+            type=raw,value=${{ env.BRANCH_NAME }}
+            type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }}
+            type=raw,value=commit-${{ env.sha }}
+            type=raw,value=${{ env.BRANCH_NAME }}-arm,enable=${{ env.BUILD_ARM }}
+      - name: Build and push image
+        uses: docker/build-push-action@v6
+        with:
+          context: ./sequlus
+          push: true
+          tags: ${{ steps.dockerMetadata.outputs.tags }}
+          cache-from: |
+            type=gha,scope=sequlus-${{ github.ref }}
+            type=gha,scope=sequlus-refs/heads/main
+          cache-to: type=gha,mode=max,scope=sequlus-${{ github.ref }}
+          platforms: ${{ env.BUILD_ARM == 'true' && 'linux/amd64,linux/arm64' || 'linux/amd64' }}

kubernetes/loculus/templates/_urls.tpl

@@ -55,7 +55,11 @@
 {{ define "loculus.generateInternalLapisUrls" }}
   {{ range $_, $item := (include "loculus.enabledOrganisms" . | fromJson).organisms }}
 {{- $key := $item.key }}
+    {{- if $.Values.useSequlus }}
+    "{{ $key }}": "http://loculus-sequlus-service:8090/{{ $key }}"
+    {{- else }}
     "{{ $key }}": "{{ if not $.Values.disableWebsite }}http://{{ template "loculus.lapisServiceName" $key }}:8080{{ else -}}http://{{ $.Values.localHost }}:8080/{{ $key }}{{ end }}"
+    {{- end }}
   {{ end }}
 {{ end }}
 

kubernetes/loculus/templates/lapis-deployment.yaml

@@ -1,3 +1,4 @@
+{{- if not .Values.useSequlus }}
 {{- $dockerTag := include "loculus.dockerTag" .Values }}
 
 {{- range $_, $item := (include "loculus.enabledOrganisms" . | fromJson).organisms }}
@@ -68,3 +69,4 @@ spec:
       volumes:
         {{- include "loculus.configVolume" (dict "name" "lapis-silo-database-config" "configmap" (printf "lapis-silo-database-config-%s" $key)) | nindent 8 }}
 {{- end }}
+{{- end }}

kubernetes/loculus/templates/lapis-ingress.yaml

@@ -27,7 +27,11 @@ kind: Ingress
 metadata:
   name: lapis-ingress
   annotations:
+    {{- if $.Values.useSequlus }}
+    traefik.ingress.kubernetes.io/router.middlewares: "{{ $.Release.Namespace }}-cors-all-origins@kubernetescrd"
+    {{- else }}
     traefik.ingress.kubernetes.io/router.middlewares: "{{ $.Release.Namespace }}-cors-all-origins@kubernetescrd,{{- $first := true }}{{- range $key := $organismKeys }}{{ if $first }}{{ $first = false }}{{ else }},{{ end }}{{ $.Release.Namespace }}-strip-{{ $key }}-prefix@kubernetescrd{{- end }}"
+    {{- end }}
 spec:
   rules:
   - host: {{ if eq $.Values.environment "server" }}{{ $lapisHost }}{{ end }}
@@ -38,16 +42,23 @@ spec:
         pathType: {{ ternary "ImplementationSpecific" "Prefix" (eq $.Values.environment "server") }}
         backend:
           service:
+            {{- if $.Values.useSequlus }}
+            name: loculus-sequlus-service
+            port:
+              number: 8090
+            {{- else }}
             name: {{ template "loculus.lapisServiceName" $key }}
             port:
               number: 8080
+            {{- end }}
       {{- end }}
   {{- if eq $.Values.environment "server" }}
   tls:
   - hosts:
     - {{ $lapisHost }}
   {{- end }}
 
+{{- if not $.Values.useSequlus }}
 {{- range $key := $organismKeys }}
 ---
 apiVersion: traefik.containo.us/v1alpha1
@@ -59,6 +70,7 @@ spec:
     prefixes:
       - /{{ $key }}/
 {{- end }}
+{{- end }}
 ---
 apiVersion: traefik.containo.us/v1alpha1
 kind: Middleware
@@ -98,7 +110,13 @@ spec:
           # the request is forwarded, we can point all paths to the first
           # organism's service as a placeholder.
           service:
+            {{- if $.Values.useSequlus }}
+            name: loculus-sequlus-service
+            port:
+              number: 8090
+            {{- else }}
             name: {{ template "loculus.lapisServiceName" (first $organismKeys) }}
             port:
               number: 8080
+            {{- end }}
       {{- end }}

kubernetes/loculus/templates/lapis-service.yaml

@@ -1,3 +1,4 @@
+{{- if not .Values.useSequlus }}
 {{- range $_, $item := (include "loculus.enabledOrganisms" . | fromJson).organisms }}
 {{- $key := $item.key }}
 ---
@@ -16,3 +17,4 @@ spec:
       protocol: TCP
       name: http
 {{- end }}
+{{- end }}

kubernetes/loculus/templates/sequlus-deployment.yaml

@@ -0,0 +1,139 @@
+{{- if .Values.useSequlus }}
+{{- $enabledOrganismsResult := (include "loculus.enabledOrganisms" . | fromJson) }}
+{{- $enabledOrganismsList := $enabledOrganismsResult.organisms }}
+{{- $lineageConfig := dict }}
+{{- range $_, $item := $enabledOrganismsList }}
+  {{- $key := $item.key }}
+  {{- $organismContent := $item.contents }}
+  {{- $schema := $organismContent.schema | include "loculus.patchMetadataSchema" | fromYaml }}
+  {{- range $entry := $schema.metadata }}
+    {{- if hasKey $entry "lineageSystem" }}
+      {{- $lineageSystem := $entry.lineageSystem }}
+      {{- $configKey := printf "%s/%s" $key $entry.name }}
+      {{- if hasKey $.Values.lineageSystemDefinitions $lineageSystem }}
+        {{- $_ := set $lineageConfig $configKey (index $.Values.lineageSystemDefinitions $lineageSystem) }}
+      {{- end }}
+    {{- end }}
+  {{- end }}
+{{- end }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: loculus-sequlus
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: loculus
+      component: sequlus
+  template:
+    metadata:
+      annotations:
+        timestamp: {{ now | quote }}
+      labels:
+        app: loculus
+        component: sequlus
+    spec:
+      {{- include "possiblePriorityClassName" $ | nindent 6 }}
+      initContainers:
+        - name: wait-for-backend
+          image: busybox
+          command: ["sh", "-c", "until wget -q -O- http://loculus-backend-service:8079/actuator/health > /dev/null 2>&1; do echo 'Waiting for backend...'; sleep 5; done; echo 'Backend is ready'"]
+        {{- range $_, $item := $enabledOrganismsList }}
+        {{- $key := $item.key }}
+        - name: process-ref-genomes-{{ $key }}
+          image: ghcr.io/loculus-project/config-processor:{{ include "loculus.dockerTag" $.Values }}
+          imagePullPolicy: {{ $.Values.imagePullPolicy }}
+          command: ["sh", "-c", "python3 /app/config-processor.py /input /output && cp /output/reference_genomes.json /reference_genomes/{{ $key }}.json"]
+          volumeMounts:
+            - name: configmap-{{ $key }}
+              mountPath: /input
+            - name: configmap-{{ $key }}-processed
+              mountPath: /output
+            - name: reference-genomes
+              mountPath: /reference_genomes
+        {{- end }}
+      containers:
+        - name: sequlus
+          image: "{{ .Values.images.sequlus.repository }}:{{ .Values.images.sequlus.tag | default (include "loculus.dockerTag" .Values) }}"
+          imagePullPolicy: "{{ .Values.images.sequlus.pullPolicy | default .Values.imagePullPolicy }}"
+          ports:
+            - containerPort: 8090
+          args:
+            - "--port"
+            - "8090"
+            - "--backend-url"
+            {{- if .Values.disableBackend }}
+            - "http://host.k3d.internal:8079"
+            {{- else }}
+            - "http://loculus-backend-service:8079"
+            {{- end }}
+            - "--database-url"
+            - "$(DATABASE_URL)"
+            - "--reference-genomes-dir"
+            - "/reference_genomes"
+            - "--data-dir"
+            - "/data"
+            - "--refresh-interval-secs"
+            - "10"
+          env:
+            - name: DB_USERNAME
+              valueFrom:
+                secretKeyRef:
+                  name: database
+                  key: username
+            - name: DB_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: database
+                  key: password
+            - name: DATABASE_URL
+              value: "postgres://$(DB_USERNAME):$(DB_PASSWORD)@loculus-database-service:5432/loculus"
+            {{- if $lineageConfig }}
+            - name: LINEAGE_CONFIG
+              value: {{ $lineageConfig | toJson | quote }}
+            {{- end }}
+          volumeMounts:
+            - name: reference-genomes
+              mountPath: /reference_genomes
+            - name: sequlus-data
+              mountPath: /data
+          startupProbe:
+            httpGet:
+              path: /health
+              port: 8090
+            initialDelaySeconds: 5
+            periodSeconds: 5
+            failureThreshold: 60
+            timeoutSeconds: 3
+          readinessProbe:
+            httpGet:
+              path: /health
+              port: 8090
+            periodSeconds: 10
+            failureThreshold: 6
+            timeoutSeconds: 5
+          livenessProbe:
+            httpGet:
+              path: /health
+              port: 8090
+            periodSeconds: 10
+            failureThreshold: 6
+            timeoutSeconds: 5
+      volumes:
+        - name: reference-genomes
+          emptyDir: {}
+        - name: sequlus-data
+          emptyDir: {}
+        {{- range $_, $item := $enabledOrganismsList }}
+        {{- $key := $item.key }}
+        - name: configmap-{{ $key }}
+          configMap:
+            name: lapis-silo-database-config-{{ $key }}
+            items:
+              - key: reference_genomes.json
+                path: reference_genomes.json
+        - name: configmap-{{ $key }}-processed
+          emptyDir: {}
+        {{- end }}
+{{- end }}

kubernetes/loculus/templates/sequlus-service.yaml

@@ -0,0 +1,16 @@
+{{- if .Values.useSequlus }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: loculus-sequlus-service
+spec:
+  type: ClusterIP
+  selector:
+    app: loculus
+    component: sequlus
+  ports:
+    - port: 8090
+      targetPort: 8090
+      protocol: TCP
+      name: http
+{{- end }}

kubernetes/loculus/templates/silo-deployment.yaml

@@ -1,3 +1,4 @@
+{{- if not .Values.useSequlus }}
 {{- $dockerTag := include "loculus.dockerTag" .Values }}
 {{- $keycloakTokenUrl := "http://loculus-keycloak-service:8083/realms/loculus/protocol/openid-connect/token" }}
 
@@ -124,3 +125,4 @@ spec:
         - name: lapis-silo-input-data-cache
           emptyDir: {}
 {{- end }}
+{{- end }}

kubernetes/loculus/templates/silo-service.yaml

@@ -1,3 +1,4 @@
+{{- if not .Values.useSequlus }}
 {{- range $_, $item := (include "loculus.enabledOrganisms" . | fromJson).organisms }}
 {{- $key := $item.key }}
 ---
@@ -15,4 +16,5 @@ spec:
       targetPort: 8081
       protocol: TCP
       name: http
+{{- end }}
 {{- end }}

kubernetes/loculus/values.schema.json

@@ -1420,6 +1420,12 @@
       "default": false,
       "description": "If true, disables ENA (European Nucleotide Archive) submission service."
     },
+    "useSequlus": {
+      "groups": ["services"],
+      "type": "boolean",
+      "default": false,
+      "description": "If true, deploys sequlus (Rust-based LAPIS replacement) instead of per-organism LAPIS+SILO pairs."
+    },
     "organisms": {
       "groups": ["organism-conf"],
       "type": "object",
@@ -1639,11 +1645,12 @@
     "images": {
       "groups": ["general"],
       "type": "object",
-      "description": "Which docker images to use. You can specify an [image spec (type)](#image-spec-type) for `lapis`, `lapisSilo`, `loculusSilo`, `website` and `backend.`",
+      "description": "Which docker images to use. You can specify an [image spec (type)](#image-spec-type) for `lapis`, `lapisSilo`, `loculusSilo`, `sequlus`, `website` and `backend.`",
       "properties": {
         "lapis": { "$ref": "#/definitions/imageSpec" },
         "lapisSilo": { "$ref": "#/definitions/imageSpec" },
         "loculusSilo": { "$ref": "#/definitions/imageSpec" },
+        "sequlus": { "$ref": "#/definitions/imageSpec" },
         "website": { "$ref": "#/definitions/imageSpec" },
         "backend": { "$ref": "#/definitions/imageSpec" }
       },