From 7e7c3916de4404f192f9f43129cc250372ed4991 Mon Sep 17 00:00:00 2001 From: Ludovic Tane Date: Fri, 6 Sep 2024 17:07:54 +0200 Subject: [PATCH 1/4] Kubernetes deployment for the Geocoder --- components/geocoder/configmap.yaml | 608 +++++++++++++++++++++ components/geocoder/deployment.yaml | 49 ++ components/geocoder/kustomization.yaml | 6 + components/geocoder/service.yaml | 15 + components/nominatim-db/kustomization.yaml | 8 + components/nominatim-db/pv.yaml | 13 + components/nominatim-db/secret.yaml | 8 + components/nominatim-db/service.yaml | 17 + components/nominatim-db/statefulset.yaml | 59 ++ components/nominatim/configmap.yaml | 10 + components/nominatim/deployment.yaml | 85 +++ components/nominatim/kustomization.yaml | 10 + components/nominatim/pv.yaml | 13 + components/nominatim/pvc.yaml | 12 + components/nominatim/secret.yaml | 7 + components/nominatim/service.yaml | 14 + 16 files changed, 934 insertions(+) create mode 100644 components/geocoder/configmap.yaml create mode 100644 components/geocoder/deployment.yaml create mode 100644 components/geocoder/kustomization.yaml create mode 100644 components/geocoder/service.yaml create mode 100644 components/nominatim-db/kustomization.yaml create mode 100644 components/nominatim-db/pv.yaml create mode 100644 components/nominatim-db/secret.yaml create mode 100644 components/nominatim-db/service.yaml create mode 100644 components/nominatim-db/statefulset.yaml create mode 100644 components/nominatim/configmap.yaml create mode 100644 components/nominatim/deployment.yaml create mode 100644 components/nominatim/kustomization.yaml create mode 100644 components/nominatim/pv.yaml create mode 100644 components/nominatim/pvc.yaml create mode 100644 components/nominatim/secret.yaml create mode 100644 components/nominatim/service.yaml diff --git a/components/geocoder/configmap.yaml b/components/geocoder/configmap.yaml new file mode 100644 index 0000000..8371d12 --- /dev/null +++ b/components/geocoder/configmap.yaml @@ -0,0 +1,608 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: geocode-proxy +data: + openapi.yaml: | + openapi: 3.0.1 + info: + title: KomMonitor Geocoder Proxy + description: "KomMonitor Geocoder Proxy that defines a common query and response\ + \ interface for arbitrary geocoding services like nominatim, photon (datasource\ + \ is OSM)" + termsOfService: http://swagger.io/terms/ + contact: + email: christian.danowski-buhren@hs-bochum.de + license: + name: Apache 2.0 + url: http://www.apache.org/licenses/LICENSE-2.0.html + version: 0.0.1 + externalDocs: + description: Find out more about Swagger + url: http://swagger.io + servers: + - url: http://localhost:8092/ + - url: http://localhost/kommonitor/api/geocoderproxy/ + tags: + - name: geocode + description: geocode locations or reverse geocode coordinates + - name: reverse-geocode + description: "find address details for a given longitude, latitude point" + paths: + /geocoder/geocode/query-string: + get: + tags: + - geocode + summary: proxies search request to underlying geocoding service in order to + find coordinates for entered location + description: proxies search request to underlying geocoding service in order to find coordinates for entered location + operationId: geocodeByQueryString + parameters: + - name: q + in: query + description: "the query string (e.g. 'Müllerstr. 3, Essen' or '3, Müllerstraß\ + e, 45147')" + required: true + style: form + explode: true + schema: + type: string + example: "3, Müllerstraße, 45147" + - name: lon + in: query + description: optional longitude to prioritize results + required: false + style: form + explode: true + schema: + type: string + example: "7.1234" + - name: lat + in: query + description: optional latitude to prioritize results + required: false + style: form + explode: true + schema: + type: string + example: "51.4321" + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/GeocodingOutputType' + "401": + description: Unauthorized + headers: + WWW_Authenticate: + style: simple + explode: false + schema: + type: string + content: {} + "405": + description: Invalid input + content: {} + "500": + description: Internal Server Error + content: {} + x-swagger-router-controller: Geocode + /geocoder/geocode/query-string/batch: + post: + tags: + - geocode + summary: proxies search batch request to underlying geocoding service in order + to find coordinates for each entered location + description: proxies search batch request to underlying geocoding service in + order to find coordinates for each entered location + operationId: geocodeByQueryStringBatch + parameters: + - name: lon + in: query + description: optional longitude to prioritize results + required: false + style: form + explode: true + schema: + type: string + example: "7.1234" + - name: lat + in: query + description: optional latitude to prioritize results + required: false + style: form + explode: true + schema: + type: string + example: "51.4321" + requestBody: + description: "list of query string (e.g. 'Müllerstr. 3, Essen' or '3, Müllerstraß\ + e, 45147')" + content: + application/json: + schema: + type: array + items: + type: string + example: "3, Müllerstraße, 45147" + required: true + responses: + "200": + description: OK + content: + application/json: + schema: + type: array + items: + $ref: '#/components/schemas/GeocodingOutputType' + x-content-type: application/json + "401": + description: Unauthorized + headers: + WWW_Authenticate: + style: simple + explode: false + schema: + type: string + content: {} + "405": + description: Invalid input + content: {} + "500": + description: Internal Server Error + content: {} + x-swagger-router-controller: Geocode + /geocoder/geocode/query-structured: + get: + tags: + - geocode + summary: "proxies search request to underlying geocoding service in order to\ + \ find coordinates for entered structured data. No parameter is required.\ + \ Instead any combination of structrured query elements may respond with 0\ + \ ... n result records in the form of a GeoJSON FeatureCollection, where each\ + \ feature has more or less details with regard to address details" + description: "proxies search request to underlying geocoding service in order\ + \ to find coordinates for entered structured data. No parameter is required.\ + \ Instead any combination of structrured query elements may respond with 0\ + \ ... n result records in the form of a GeoJSON FeatureCollection, where each\ + \ feature has more or less details with regard to address details" + operationId: geocodeByStructuredQuery + parameters: + - name: country + in: query + description: optional country name + required: false + style: form + explode: true + schema: + type: string + example: Deutschland + - name: state + in: query + description: optional state name + required: false + style: form + explode: true + schema: + type: string + example: Nordrhein-Westfalen + - name: city + in: query + description: optional city name + required: false + style: form + explode: true + schema: + type: string + example: Essen + - name: district + in: query + description: optional district name + required: false + style: form + explode: true + schema: + type: string + example: Holsterhausen + - name: postcode + in: query + description: optional postcode name + required: false + style: form + explode: true + schema: + type: string + example: "45147" + - name: street + in: query + description: required street name + required: false + style: form + explode: true + schema: + type: string + example: Müllerstraße + - name: housenumber + in: query + description: optional housenumber name + required: false + style: form + explode: true + schema: + type: string + example: 13c + - name: lon + in: query + description: optional longitude to prioritize results + required: false + style: form + explode: true + schema: + type: string + example: "7.1234" + - name: lat + in: query + description: optional latitude to prioritize results + required: false + style: form + explode: true + schema: + type: string + example: "51.4321" + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/GeocodingOutputType' + "401": + description: Unauthorized + headers: + WWW_Authenticate: + style: simple + explode: false + schema: + type: string + content: {} + "405": + description: Invalid input + content: {} + "500": + description: Internal Server Error + content: {} + x-swagger-router-controller: Geocode + /geocoder/geocode/query-structured/batch: + post: + tags: + - geocode + summary: "proxies search batch request to underlying geocoding service in order\ + \ to find coordinates for entered structured data. No parameter is required.\ + \ Instead any combination of structrured query elements may respond with 0\ + \ ... n result records in the form of a GeoJSON FeatureCollection, where each\ + \ feature has more or less details with regard to address details" + description: "proxies search batch request to underlying geocoding service in\ + \ order to find coordinates for entered structured data. No parameter is required.\ + \ Instead any combination of structrured query elements may respond with 0\ + \ ... n result records in the form of a GeoJSON FeatureCollection, where each\ + \ feature has more or less details with regard to address details" + operationId: geocodeByStructuredQueryBatch + parameters: + - name: lon + in: query + description: optional longitude to prioritize results + required: false + style: form + explode: true + schema: + type: string + example: "7.1234" + - name: lat + in: query + description: optional latitude to prioritize results + required: false + style: form + explode: true + schema: + type: string + example: "51.4321" + requestBody: + description: list of structured geocoding inputs + content: + application/json: + schema: + type: array + items: + $ref: '#/components/schemas/GeocodingStructuredBatchInputType' + required: true + responses: + "200": + description: OK + content: + application/json: + schema: + type: array + items: + $ref: '#/components/schemas/GeocodingOutputType' + x-content-type: application/json + "401": + description: Unauthorized + headers: + WWW_Authenticate: + style: simple + explode: false + schema: + type: string + content: {} + "405": + description: Invalid input + content: {} + "500": + description: Internal Server Error + content: {} + x-swagger-router-controller: Geocode + /geocoder/reverse: + get: + tags: + - reverse-geocode + summary: "proxies reverse-geocode request to underlying geocoding service in\ + \ order to find address details for given longitude, latitude coordinates" + description: "proxies reverse-geocode request to underlying geocoding service\ + \ in order to find address details for given longitude, latitude coordinates" + operationId: reverseGeocode + parameters: + - name: lon + in: query + description: required longitude + required: true + style: form + explode: true + schema: + type: string + example: "7.1234" + - name: lat + in: query + description: required latitude + required: true + style: form + explode: true + schema: + type: string + example: "51.4321" + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/GeocodingOutputType' + "401": + description: Unauthorized + headers: + WWW_Authenticate: + style: simple + explode: false + schema: + type: string + content: {} + "405": + description: Invalid input + content: {} + "500": + description: Internal Server Error + content: {} + x-swagger-router-controller: ReverseGeocode + components: + schemas: + GeocodingStructuredBatchInputType: + type: object + properties: + country: + type: string + description: optional country name + example: Deutschland + state: + type: string + description: optional state name + example: Nordrhein-Westfalen + city: + type: string + description: optional city name + example: Essen + district: + type: string + description: optional district name + example: Holsterhausen + postcode: + type: string + description: optional postcode name + example: "45147" + street: + type: string + description: optional street name + example: Müllerstraße + housenumber: + type: string + description: optional housenumber name + example: 13 C + GeocodingOutputType: + type: object + properties: + type: + type: string + default: FeatureCollection + features: + type: array + items: + $ref: '#/components/schemas/GeocodingFeatureType' + description: output generated by a geocoding request. 0 or more records in the + form of a GeoJSON FeatureCollection (if emtpy then the returned FeatureCollection + has an empty features array) + example: + features: + - geometry: + coordinates: + - 7.1234 + - 51.4321 + type: Point + properties: + country: Deutschland + city: Essen + housenumber: 13c + street: Müllerstraße + district: Holsterhausen + postcode: "45147" + state: Nordrhein-Westfalen + category: building + type: apartments + display_name: "3, Müllerstraße, Holsterhausen, Essen, Nordrhein-Westfalen,\ + \ 45147, Deutschland" + - geometry: + coordinates: + - 7.1234 + - 51.4321 + type: Point + properties: + country: Deutschland + city: Essen + housenumber: 13c + street: Müllerstraße + district: Holsterhausen + postcode: "45147" + state: Nordrhein-Westfalen + category: building + type: apartments + display_name: "3, Müllerstraße, Holsterhausen, Essen, Nordrhein-Westfalen,\ + \ 45147, Deutschland" + type: FeatureCollection + GeocodingFeatureType: + type: object + properties: + geometry: + $ref: '#/components/schemas/GeocodingGeometryType' + properties: + $ref: '#/components/schemas/GeocodingPropertiesType' + example: + geometry: + coordinates: + - 7.1234 + - 51.4321 + type: Point + properties: + country: Deutschland + city: Essen + housenumber: 13c + street: Müllerstraße + district: Holsterhausen + postcode: "45147" + state: Nordrhein-Westfalen + category: building + type: apartments + display_name: "3, Müllerstraße, Holsterhausen, Essen, Nordrhein-Westfalen,\ + \ 45147, Deutschland" + GeocodingGeometryType: + type: object + properties: + type: + type: string + description: the geometry type of the feature - here in geocoding context + it is always point + default: Point + coordinates: + maximum: 2 + minimum: 2 + type: array + description: "coordinate array of feature point geometry using longitude,\ + \ latitude" + example: + - 7.1234 + - 51.4321 + items: + type: number + format: float + example: + coordinates: + - 7.1234 + - 51.4321 + type: Point + GeocodingPropertiesType: + type: object + properties: + category: + type: string + description: category of response (osm_key) + example: building + type: + type: string + description: type within category (osm_value) + example: apartments + display_name: + type: string + description: a descriptive string combining several address details within + one string (i.e. for display popup purposes) + example: "3, Müllerstraße, Holsterhausen, Essen, Nordrhein-Westfalen, 45147,\ + \ Deutschland" + country: + type: string + description: name of the country + example: Deutschland + state: + type: string + description: name of the state + example: Nordrhein-Westfalen + postcode: + type: string + description: postal code + example: "45147" + city: + type: string + description: name of the city + example: Essen + district: + type: string + description: name of the city district + example: Holsterhausen + street: + type: string + description: name of the street + example: Müllerstraße + housenumber: + type: string + description: house number (including any suffix) if available + example: 13c + geocoderank: + type: string + description: evaluation of geocoding result (unresolved = least accurate cause false street and/or city/postcode; low_accuracy = more accurate due to same street and same city/postcode but no housenumber; high_accuracy = most accurate due to same street and same city/postcode and same housenumber) + enum: + - "unresolved" + - "low_accuracy" + - "high_accuracy" + geocodedesc: + type: string + description: descriptive evaluation of geocoding result + example: + country: Deutschland + city: Essen + housenumber: 13c + street: Müllerstraße + geocoderank: "high_accuracy" + district: Holsterhausen + postcode: "45147" + state: Nordrhein-Westfalen + category: building + type: apartments + display_name: "3, Müllerstraße, Holsterhausen, Essen, Nordrhein-Westfalen,\ + \ 45147, Deutschland" + responses: + UnauthorizedError: + description: API key is missing or invalid + headers: + WWW_Authenticate: + style: simple + explode: false + schema: + type: string + content: {} + diff --git a/components/geocoder/deployment.yaml b/components/geocoder/deployment.yaml new file mode 100644 index 0000000..f9fb794 --- /dev/null +++ b/components/geocoder/deployment.yaml @@ -0,0 +1,49 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: geocode-proxy + labels: + app: geocode-proxy +spec: + replicas: 1 + selector: + matchLabels: + app: geocode-proxy + template: + metadata: + labels: + app: geocode-proxy + spec: + containers: + - name: geocode-proxy + image: kommonitor/geocoder-proxy + imagePullPolicy: Always + resources: + requests: + memory: "16M" + cpu: "1m" + limits: + memory: "1024M" + cpu: "1000m" + ports: + - containerPort: 8092 + env: + - name: NOMINATIM_URL + value: "http://nominatim-service:8080" + - name: PORT + value: "8092" + + envFrom: + - configMapRef: + name: geocode-proxy + volumeMounts: + - name: openapi-config-volume + mountPath: /code/api/openapi.yaml + subPath: openapi.yaml + volumes: + - name: openapi-config-volume + configMap: + name: geocode-proxy + items: + - key: openapi.yaml + path: openapi.yaml \ No newline at end of file diff --git a/components/geocoder/kustomization.yaml b/components/geocoder/kustomization.yaml new file mode 100644 index 0000000..8d19601 --- /dev/null +++ b/components/geocoder/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - deployment.yaml + - service.yaml + - configmap.yaml diff --git a/components/geocoder/service.yaml b/components/geocoder/service.yaml new file mode 100644 index 0000000..43de8b1 --- /dev/null +++ b/components/geocoder/service.yaml @@ -0,0 +1,15 @@ + apiVersion: v1 + kind: Service + metadata: + name: geocode-proxy + labels: + app: geocode-proxy + spec: + selector: + app: geocode-proxy + ports: + - name: http + port: 8092 + targetPort: 8092 + type: NodePort + diff --git a/components/nominatim-db/kustomization.yaml b/components/nominatim-db/kustomization.yaml new file mode 100644 index 0000000..2884660 --- /dev/null +++ b/components/nominatim-db/kustomization.yaml @@ -0,0 +1,8 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - statefulset.yaml + - service.yaml + - secret.yaml + - pv.yaml \ No newline at end of file diff --git a/components/nominatim-db/pv.yaml b/components/nominatim-db/pv.yaml new file mode 100644 index 0000000..7e7c11f --- /dev/null +++ b/components/nominatim-db/pv.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: nominatim-db-data +spec: + capacity: + storage: 10Gi + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Retain + storageClassName: retained-ssd + hostPath: + path: "/data/nominatim-db" diff --git a/components/nominatim-db/secret.yaml b/components/nominatim-db/secret.yaml new file mode 100644 index 0000000..ca13354 --- /dev/null +++ b/components/nominatim-db/secret.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Secret +metadata: + name: nominatim-db +type: Opaque +data: + POSTGRES_USER: YWRtaW4= + POSTGRES_PASSWORD: YWRtaW4= diff --git a/components/nominatim-db/service.yaml b/components/nominatim-db/service.yaml new file mode 100644 index 0000000..d402657 --- /dev/null +++ b/components/nominatim-db/service.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + name: nominatim-db + labels: + app: nominatim-db +spec: + type: NodePort + ports: + - name: tcp + port: 5432 + targetPort: 5432 + #clusterIP: None + selector: + app: nominatim-db + + diff --git a/components/nominatim-db/statefulset.yaml b/components/nominatim-db/statefulset.yaml new file mode 100644 index 0000000..6a24308 --- /dev/null +++ b/components/nominatim-db/statefulset.yaml @@ -0,0 +1,59 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: nominatim-db +spec: + serviceName: nominatim-db + selector: + matchLabels: + app: nominatim-db + replicas: 1 + template: + metadata: + labels: + app: nominatim-db + spec: + containers: + - name: nominatim-db + image: postgres:15 + imagePullPolicy: "Always" + resources: + requests: + memory: "1Gi" + cpu: "500m" + limits: + memory: "2Gi" + cpu: "1000m" + ports: + - containerPort: 5432 + env: + - name: ALLOW_IP_RANGE + value: 0.0.0.0/0 + - name: POSTGRES_USER + valueFrom: + secretKeyRef: + name: nominatim-db + key: POSTGRES_USER + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: nominatim-db + key: POSTGRES_PASSWORD + - name: POSTGRES_DB + value: nominatim-data #wie in keykock-db definiert + - name: PGDATA + value: /var/lib/postgresql/data/pgdata + volumeMounts: + - name: nominatim-db-data + mountPath: /var/lib/postgresql/data + volumeClaimTemplates: + - metadata: + name: nominatim-db-data + spec: + storageClassName: retained-ssd + resources: + requests: + storage: 10Gi + volumeMode: Filesystem + accessModes: + - "ReadWriteOnce" diff --git a/components/nominatim/configmap.yaml b/components/nominatim/configmap.yaml new file mode 100644 index 0000000..db383e3 --- /dev/null +++ b/components/nominatim/configmap.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: nominatim +data: + PBF_URL: "https://download.geofabrik.de/europe/germany/nordrhein-westfalen/duesseldorf-regbez-latest.osm.pbf" + REPLICATION_URL: "https://download.geofabrik.de/europe/germany/nordrhein-westfalen/duesseldorf-regbez-updates/" + REPLICATION_UPDATE_INTERVAL: "86400" + UPDATE_MODE: "once" + REPLICATION_RECHECK_INTERVAL: "300" \ No newline at end of file diff --git a/components/nominatim/deployment.yaml b/components/nominatim/deployment.yaml new file mode 100644 index 0000000..412a91e --- /dev/null +++ b/components/nominatim/deployment.yaml @@ -0,0 +1,85 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nominatim + labels: + app: nominatim +spec: + replicas: 1 + selector: + matchLabels: + app: nominatim + template: + metadata: + labels: + app: nominatim + spec: + # initContainers: + # - name: init-myservice #wait until service nominatim-db is ready + # image: busybox:1.31.1 + # command: ['sh', '-c', 'until nslookup nominatim-db; do echo waiting for nominatim-db; sleep 2; done;'] + + containers: + - name: nominatim + image: mediagis/nominatim:4.4 + imagePullPolicy: Always + resources: + requests: + memory: "16M" + cpu: "1m" + limits: + memory: "2048M" + cpu: "1000m" + ports: + - containerPort: 8080 + env: + - name: PBF_URL + valueFrom: + configMapKeyRef: + name: nominatim + key: PBF_URL + - name: REPLICATION_URL + valueFrom: + configMapKeyRef: + name: nominatim + key: REPLICATION_URL + - name: REPLICATION_UPDATE_INTERVAL + valueFrom: + configMapKeyRef: + name: nominatim + key: REPLICATION_UPDATE_INTERVAL + - name: UPDATE_MODE + valueFrom: + configMapKeyRef: + name: nominatim + key: UPDATE_MODE + - name: REPLICATION_RECHECK_INTERVAL + valueFrom: + configMapKeyRef: + name: nominatim + key: REPLICATION_RECHECK_INTERVAL + - name: NOMINATIM_TOKENIZER + value: "icu" + - name: NOMINATIM_DATABASE_DSN + value: "pgsql:dbname=nominatim-data;hostaddr=10.103.126.75;user=admin;password=admin" + - name: PGHOSTADDR + value: "10.103.126.75" + - name: PGDATABASE + value: "nominatim-data" + - name: PGUSER + valueFrom: + secretKeyRef: + name: nominatim-db + key: POSTGRES_USER + - name: PGPASSWORD + valueFrom: + secretKeyRef: + name: nominatim-db + key: POSTGRES_PASSWORD + volumeMounts: + - name: nominatim-storage + mountPath: /var/lib/nominatim + volumes: + - name: nominatim-storage + persistentVolumeClaim: + claimName: nominatim-storage diff --git a/components/nominatim/kustomization.yaml b/components/nominatim/kustomization.yaml new file mode 100644 index 0000000..c6c2940 --- /dev/null +++ b/components/nominatim/kustomization.yaml @@ -0,0 +1,10 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - configmap.yaml + - secret.yaml + - service.yaml + - deployment.yaml + - pv.yaml + - pvc.yaml diff --git a/components/nominatim/pv.yaml b/components/nominatim/pv.yaml new file mode 100644 index 0000000..608f137 --- /dev/null +++ b/components/nominatim/pv.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: nominatim-storage +spec: + capacity: + storage: 10Gi + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Retain + storageClassName: retained-ssd + hostPath: + path: "/data/nominatim-storage" diff --git a/components/nominatim/pvc.yaml b/components/nominatim/pvc.yaml new file mode 100644 index 0000000..54cd609 --- /dev/null +++ b/components/nominatim/pvc.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: nominatim-storage +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + storageClassName: retained-ssd + diff --git a/components/nominatim/secret.yaml b/components/nominatim/secret.yaml new file mode 100644 index 0000000..0d9b3a0 --- /dev/null +++ b/components/nominatim/secret.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Secret +metadata: + name: nominatim +type: Opaque +data: + NOMINATIM_PASSWORD: YWRtaW4= diff --git a/components/nominatim/service.yaml b/components/nominatim/service.yaml new file mode 100644 index 0000000..397f8ee --- /dev/null +++ b/components/nominatim/service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: nominatim + labels: + app: nominatim +spec: + selector: + app: nominatim + ports: + - name: http1 + port: 8080 + targetPort: 8080 + type: NodePort \ No newline at end of file From 0cd0280e6a6727c4a6d44cebbfd9cfefdd0c57c7 Mon Sep 17 00:00:00 2001 From: Ludovic Tane Date: Tue, 24 Sep 2024 09:17:27 +0200 Subject: [PATCH 2/4] configuration with an internal rather than an external database --- components/geocoder/deployment.yaml | 2 +- components/geocoder/service.yaml | 2 +- components/nominatim-db/kustomization.yaml | 8 --- components/nominatim-db/pv.yaml | 13 ----- components/nominatim-db/secret.yaml | 8 --- components/nominatim-db/service.yaml | 17 ------- components/nominatim-db/statefulset.yaml | 59 ---------------------- components/nominatim/deployment.yaml | 37 ++++---------- components/nominatim/pvc.yaml | 3 +- components/nominatim/service.yaml | 4 +- 10 files changed, 16 insertions(+), 137 deletions(-) delete mode 100644 components/nominatim-db/kustomization.yaml delete mode 100644 components/nominatim-db/pv.yaml delete mode 100644 components/nominatim-db/secret.yaml delete mode 100644 components/nominatim-db/service.yaml delete mode 100644 components/nominatim-db/statefulset.yaml diff --git a/components/geocoder/deployment.yaml b/components/geocoder/deployment.yaml index f9fb794..2915ee6 100644 --- a/components/geocoder/deployment.yaml +++ b/components/geocoder/deployment.yaml @@ -29,7 +29,7 @@ spec: - containerPort: 8092 env: - name: NOMINATIM_URL - value: "http://nominatim-service:8080" + value: "http://nominatim:8080" - name: PORT value: "8092" diff --git a/components/geocoder/service.yaml b/components/geocoder/service.yaml index 43de8b1..0f50d2e 100644 --- a/components/geocoder/service.yaml +++ b/components/geocoder/service.yaml @@ -11,5 +11,5 @@ - name: http port: 8092 targetPort: 8092 + nodePort: 30920 type: NodePort - diff --git a/components/nominatim-db/kustomization.yaml b/components/nominatim-db/kustomization.yaml deleted file mode 100644 index 2884660..0000000 --- a/components/nominatim-db/kustomization.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -resources: - - statefulset.yaml - - service.yaml - - secret.yaml - - pv.yaml \ No newline at end of file diff --git a/components/nominatim-db/pv.yaml b/components/nominatim-db/pv.yaml deleted file mode 100644 index 7e7c11f..0000000 --- a/components/nominatim-db/pv.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: v1 -kind: PersistentVolume -metadata: - name: nominatim-db-data -spec: - capacity: - storage: 10Gi - accessModes: - - ReadWriteOnce - persistentVolumeReclaimPolicy: Retain - storageClassName: retained-ssd - hostPath: - path: "/data/nominatim-db" diff --git a/components/nominatim-db/secret.yaml b/components/nominatim-db/secret.yaml deleted file mode 100644 index ca13354..0000000 --- a/components/nominatim-db/secret.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: nominatim-db -type: Opaque -data: - POSTGRES_USER: YWRtaW4= - POSTGRES_PASSWORD: YWRtaW4= diff --git a/components/nominatim-db/service.yaml b/components/nominatim-db/service.yaml deleted file mode 100644 index d402657..0000000 --- a/components/nominatim-db/service.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: nominatim-db - labels: - app: nominatim-db -spec: - type: NodePort - ports: - - name: tcp - port: 5432 - targetPort: 5432 - #clusterIP: None - selector: - app: nominatim-db - - diff --git a/components/nominatim-db/statefulset.yaml b/components/nominatim-db/statefulset.yaml deleted file mode 100644 index 6a24308..0000000 --- a/components/nominatim-db/statefulset.yaml +++ /dev/null @@ -1,59 +0,0 @@ -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: nominatim-db -spec: - serviceName: nominatim-db - selector: - matchLabels: - app: nominatim-db - replicas: 1 - template: - metadata: - labels: - app: nominatim-db - spec: - containers: - - name: nominatim-db - image: postgres:15 - imagePullPolicy: "Always" - resources: - requests: - memory: "1Gi" - cpu: "500m" - limits: - memory: "2Gi" - cpu: "1000m" - ports: - - containerPort: 5432 - env: - - name: ALLOW_IP_RANGE - value: 0.0.0.0/0 - - name: POSTGRES_USER - valueFrom: - secretKeyRef: - name: nominatim-db - key: POSTGRES_USER - - name: POSTGRES_PASSWORD - valueFrom: - secretKeyRef: - name: nominatim-db - key: POSTGRES_PASSWORD - - name: POSTGRES_DB - value: nominatim-data #wie in keykock-db definiert - - name: PGDATA - value: /var/lib/postgresql/data/pgdata - volumeMounts: - - name: nominatim-db-data - mountPath: /var/lib/postgresql/data - volumeClaimTemplates: - - metadata: - name: nominatim-db-data - spec: - storageClassName: retained-ssd - resources: - requests: - storage: 10Gi - volumeMode: Filesystem - accessModes: - - "ReadWriteOnce" diff --git a/components/nominatim/deployment.yaml b/components/nominatim/deployment.yaml index 412a91e..ce5a007 100644 --- a/components/nominatim/deployment.yaml +++ b/components/nominatim/deployment.yaml @@ -14,22 +14,17 @@ spec: labels: app: nominatim spec: - # initContainers: - # - name: init-myservice #wait until service nominatim-db is ready - # image: busybox:1.31.1 - # command: ['sh', '-c', 'until nslookup nominatim-db; do echo waiting for nominatim-db; sleep 2; done;'] - containers: - name: nominatim image: mediagis/nominatim:4.4 imagePullPolicy: Always resources: requests: - memory: "16M" - cpu: "1m" - limits: - memory: "2048M" + memory: "4Gi" cpu: "1000m" + limits: + memory: "8Gi" + cpu: "2000m" ports: - containerPort: 8080 env: @@ -58,28 +53,16 @@ spec: configMapKeyRef: name: nominatim key: REPLICATION_RECHECK_INTERVAL - - name: NOMINATIM_TOKENIZER - value: "icu" - - name: NOMINATIM_DATABASE_DSN - value: "pgsql:dbname=nominatim-data;hostaddr=10.103.126.75;user=admin;password=admin" - - name: PGHOSTADDR - value: "10.103.126.75" - - name: PGDATABASE - value: "nominatim-data" - - name: PGUSER - valueFrom: - secretKeyRef: - name: nominatim-db - key: POSTGRES_USER - - name: PGPASSWORD - valueFrom: - secretKeyRef: - name: nominatim-db - key: POSTGRES_PASSWORD volumeMounts: - name: nominatim-storage mountPath: /var/lib/nominatim + - name: nominatim-db-data + mountPath: /var/lib/postgresql/14/main volumes: - name: nominatim-storage persistentVolumeClaim: claimName: nominatim-storage + - name: nominatim-db-data + persistentVolumeClaim: + claimName: nominatim-db-data + diff --git a/components/nominatim/pvc.yaml b/components/nominatim/pvc.yaml index 54cd609..39e928e 100644 --- a/components/nominatim/pvc.yaml +++ b/components/nominatim/pvc.yaml @@ -8,5 +8,4 @@ spec: resources: requests: storage: 10Gi - storageClassName: retained-ssd - + storageClassName: retained-ssd \ No newline at end of file diff --git a/components/nominatim/service.yaml b/components/nominatim/service.yaml index 397f8ee..38dc2e2 100644 --- a/components/nominatim/service.yaml +++ b/components/nominatim/service.yaml @@ -11,4 +11,6 @@ spec: - name: http1 port: 8080 targetPort: 8080 - type: NodePort \ No newline at end of file + nodePort: 30922 + type: NodePort + From 678c2e1f39e744cac62a9c599d809e115ab297f7 Mon Sep 17 00:00:00 2001 From: Ludovic Tane Date: Wed, 25 Sep 2024 12:03:51 +0200 Subject: [PATCH 3/4] ready for aws deployment --- components/geocoder/ingress.yaml | 14 ++++++++++++++ components/geocoder/kustomization.yaml | 3 +++ components/geocoder/namespace.yaml | 4 ++++ components/geocoder/rolebinding.yaml | 11 +++++++++++ components/nominatim/deployment.yaml | 10 +++++----- components/nominatim/ingress.yaml | 14 ++++++++++++++ components/nominatim/kustomization.yaml | 11 +++++++---- components/nominatim/namespace.yaml | 4 ++++ components/nominatim/rolebinding.yaml | 11 +++++++++++ 9 files changed, 73 insertions(+), 9 deletions(-) create mode 100644 components/geocoder/ingress.yaml create mode 100644 components/geocoder/namespace.yaml create mode 100644 components/geocoder/rolebinding.yaml create mode 100644 components/nominatim/ingress.yaml create mode 100644 components/nominatim/namespace.yaml create mode 100644 components/nominatim/rolebinding.yaml diff --git a/components/geocoder/ingress.yaml b/components/geocoder/ingress.yaml new file mode 100644 index 0000000..4e2b48d --- /dev/null +++ b/components/geocoder/ingress.yaml @@ -0,0 +1,14 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + annotations: + kubernetes.io/ingress.class: "nginx" + cert-manager.io/issuer: letsencrypt + name: de-kommonitor-geocoder +spec: + tls: + - hosts: + - demo.kommonitor-geocoder.de.52north.org + secretName: de-kommonitor-geocoder-tls + rules: + - host: demo.kommonitor-geocoder.de.52north.org \ No newline at end of file diff --git a/components/geocoder/kustomization.yaml b/components/geocoder/kustomization.yaml index 8d19601..1253836 100644 --- a/components/geocoder/kustomization.yaml +++ b/components/geocoder/kustomization.yaml @@ -4,3 +4,6 @@ resources: - deployment.yaml - service.yaml - configmap.yaml + - namespace.yaml + - rolebinding.yaml + - ingress.yaml diff --git a/components/geocoder/namespace.yaml b/components/geocoder/namespace.yaml new file mode 100644 index 0000000..f771577 --- /dev/null +++ b/components/geocoder/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: de-kommonitor-geocoder \ No newline at end of file diff --git a/components/geocoder/rolebinding.yaml b/components/geocoder/rolebinding.yaml new file mode 100644 index 0000000..918f3b4 --- /dev/null +++ b/components/geocoder/rolebinding.yaml @@ -0,0 +1,11 @@ +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: de-kommonitor-geocoder-admin-access +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: admin +subjects: +- kind: Group + name: de-kommonitor-geocoder-admin # This is the link between k8s and the IAM role \ No newline at end of file diff --git a/components/nominatim/deployment.yaml b/components/nominatim/deployment.yaml index ce5a007..5f3b9ea 100644 --- a/components/nominatim/deployment.yaml +++ b/components/nominatim/deployment.yaml @@ -56,13 +56,13 @@ spec: volumeMounts: - name: nominatim-storage mountPath: /var/lib/nominatim - - name: nominatim-db-data - mountPath: /var/lib/postgresql/14/main + # - name: nominatim-db-data + # mountPath: /var/lib/postgresql/14/main volumes: - name: nominatim-storage persistentVolumeClaim: claimName: nominatim-storage - - name: nominatim-db-data - persistentVolumeClaim: - claimName: nominatim-db-data + # - name: nominatim-db-data + # persistentVolumeClaim: + # claimName: nominatim-db-data diff --git a/components/nominatim/ingress.yaml b/components/nominatim/ingress.yaml new file mode 100644 index 0000000..4e2b48d --- /dev/null +++ b/components/nominatim/ingress.yaml @@ -0,0 +1,14 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + annotations: + kubernetes.io/ingress.class: "nginx" + cert-manager.io/issuer: letsencrypt + name: de-kommonitor-geocoder +spec: + tls: + - hosts: + - demo.kommonitor-geocoder.de.52north.org + secretName: de-kommonitor-geocoder-tls + rules: + - host: demo.kommonitor-geocoder.de.52north.org \ No newline at end of file diff --git a/components/nominatim/kustomization.yaml b/components/nominatim/kustomization.yaml index c6c2940..07073ee 100644 --- a/components/nominatim/kustomization.yaml +++ b/components/nominatim/kustomization.yaml @@ -1,10 +1,13 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization - resources: - - configmap.yaml - - secret.yaml - - service.yaml - deployment.yaml + - service.yaml + - configmap.yaml + - namespace.yaml + - rolebinding.yaml + - ingress.yaml - pv.yaml - pvc.yaml + - secret.yaml + diff --git a/components/nominatim/namespace.yaml b/components/nominatim/namespace.yaml new file mode 100644 index 0000000..f771577 --- /dev/null +++ b/components/nominatim/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: de-kommonitor-geocoder \ No newline at end of file diff --git a/components/nominatim/rolebinding.yaml b/components/nominatim/rolebinding.yaml new file mode 100644 index 0000000..918f3b4 --- /dev/null +++ b/components/nominatim/rolebinding.yaml @@ -0,0 +1,11 @@ +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: de-kommonitor-geocoder-admin-access +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: admin +subjects: +- kind: Group + name: de-kommonitor-geocoder-admin # This is the link between k8s and the IAM role \ No newline at end of file From 520ae88d472d2f11c83033ae5f358377dab08b3f Mon Sep 17 00:00:00 2001 From: Ludovic Tane Date: Tue, 1 Oct 2024 14:21:43 +0200 Subject: [PATCH 4/4] persistens + ressources requests/limits --- components/geocoder/deployment.yaml | 4 +- components/kubegres/my-postgres-secret.yaml | 9 ++++ components/kubegres/my-postgres.yaml | 26 ++++++++++++ components/nominatim/deployment.yaml | 46 +++++++++++++++++---- components/nominatim/kustomization.yaml | 2 - components/nominatim/pv.yaml | 13 ------ components/nominatim/pvc.yaml | 19 ++++++++- components/nominatim/secret.yaml | 7 ---- 8 files changed, 92 insertions(+), 34 deletions(-) create mode 100644 components/kubegres/my-postgres-secret.yaml create mode 100644 components/kubegres/my-postgres.yaml delete mode 100644 components/nominatim/pv.yaml delete mode 100644 components/nominatim/secret.yaml diff --git a/components/geocoder/deployment.yaml b/components/geocoder/deployment.yaml index 2915ee6..0a9e22c 100644 --- a/components/geocoder/deployment.yaml +++ b/components/geocoder/deployment.yaml @@ -23,8 +23,8 @@ spec: memory: "16M" cpu: "1m" limits: - memory: "1024M" - cpu: "1000m" + memory: "500M" + cpu: "500m" ports: - containerPort: 8092 env: diff --git a/components/kubegres/my-postgres-secret.yaml b/components/kubegres/my-postgres-secret.yaml new file mode 100644 index 0000000..937cba3 --- /dev/null +++ b/components/kubegres/my-postgres-secret.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Secret +metadata: + name: mypostgres-secret + namespace: default +type: Opaque +stringData: + superUserPassword: postgresSuperUserPsw + replicationUserPassword: postgresReplicaPsw \ No newline at end of file diff --git a/components/kubegres/my-postgres.yaml b/components/kubegres/my-postgres.yaml new file mode 100644 index 0000000..68f26b9 --- /dev/null +++ b/components/kubegres/my-postgres.yaml @@ -0,0 +1,26 @@ +apiVersion: kubegres.reactive-tech.io/v1 +kind: Kubegres +metadata: + name: mypostgres + namespace: default + +spec: + + replicas: 3 + image: postgres:16.1 + + database: + size: 200Mi + + env: + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: mypostgres-secret + key: superUserPassword + + - name: POSTGRES_REPLICATION_PASSWORD + valueFrom: + secretKeyRef: + name: mypostgres-secret + key: replicationUserPassword \ No newline at end of file diff --git a/components/nominatim/deployment.yaml b/components/nominatim/deployment.yaml index 5f3b9ea..1d5e032 100644 --- a/components/nominatim/deployment.yaml +++ b/components/nominatim/deployment.yaml @@ -14,17 +14,45 @@ spec: labels: app: nominatim spec: + initContainers: + - name: init + image: mediagis/nominatim:4.4 + resources: + limits: + memory: "3Gi" + cpu: "1000m" + command: + - bash + args: + - '-c' + - 'mkdir -p /var/lib/postgresql/14/main && sudo chown postgres:postgres /var/lib/postgresql/14/main && bash /app/init.sh' + env: + - name: PBF_URL + valueFrom: + configMapKeyRef: + name: nominatim + key: PBF_URL + - name: REPLICATION_URL + valueFrom: + configMapKeyRef: + name: nominatim + key: REPLICATION_URL + volumeMounts: + - name: nominatim-storage + mountPath: /var/lib/nominatim + - name: nominatim-db-data + mountPath: /var/lib/postgresql/14/ containers: - name: nominatim image: mediagis/nominatim:4.4 imagePullPolicy: Always resources: requests: - memory: "4Gi" - cpu: "1000m" + memory: "1Gi" + cpu: "100m" limits: - memory: "8Gi" - cpu: "2000m" + memory: "3Gi" + cpu: "1000m" ports: - containerPort: 8080 env: @@ -56,13 +84,13 @@ spec: volumeMounts: - name: nominatim-storage mountPath: /var/lib/nominatim - # - name: nominatim-db-data - # mountPath: /var/lib/postgresql/14/main + - name: nominatim-db-data + mountPath: /var/lib/postgresql/14/ volumes: - name: nominatim-storage persistentVolumeClaim: claimName: nominatim-storage - # - name: nominatim-db-data - # persistentVolumeClaim: - # claimName: nominatim-db-data + - name: nominatim-db-data + persistentVolumeClaim: + claimName: nominatim-db-data diff --git a/components/nominatim/kustomization.yaml b/components/nominatim/kustomization.yaml index 07073ee..2201839 100644 --- a/components/nominatim/kustomization.yaml +++ b/components/nominatim/kustomization.yaml @@ -7,7 +7,5 @@ resources: - namespace.yaml - rolebinding.yaml - ingress.yaml - - pv.yaml - pvc.yaml - - secret.yaml diff --git a/components/nominatim/pv.yaml b/components/nominatim/pv.yaml deleted file mode 100644 index 608f137..0000000 --- a/components/nominatim/pv.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: v1 -kind: PersistentVolume -metadata: - name: nominatim-storage -spec: - capacity: - storage: 10Gi - accessModes: - - ReadWriteOnce - persistentVolumeReclaimPolicy: Retain - storageClassName: retained-ssd - hostPath: - path: "/data/nominatim-storage" diff --git a/components/nominatim/pvc.yaml b/components/nominatim/pvc.yaml index 39e928e..9520e7b 100644 --- a/components/nominatim/pvc.yaml +++ b/components/nominatim/pvc.yaml @@ -2,10 +2,27 @@ apiVersion: v1 kind: PersistentVolumeClaim metadata: name: nominatim-storage +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 2Gi + storageClassName: retained-ssd-gp3 + volumeMode: Filesystem + +--- + +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: nominatim-db-data spec: accessModes: - ReadWriteOnce resources: requests: storage: 10Gi - storageClassName: retained-ssd \ No newline at end of file + storageClassName: retained-ssd-gp3 + volumeMode: Filesystem + \ No newline at end of file diff --git a/components/nominatim/secret.yaml b/components/nominatim/secret.yaml deleted file mode 100644 index 0d9b3a0..0000000 --- a/components/nominatim/secret.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: nominatim -type: Opaque -data: - NOMINATIM_PASSWORD: YWRtaW4=