From 24275f2afa90cb2b24edd7e76f945ebddeebea7a Mon Sep 17 00:00:00 2001 From: Kashyap Ramakur Date: Thu, 8 May 2025 11:49:18 -0400 Subject: [PATCH 01/24] resource limit updates feature env --- charts/data-processing-service/values-feature.yaml | 8 ++++---- charts/dataingestion-service/values-feature.yaml | 8 +++++++- charts/elasticsearch-efs/values-feature.yaml | 8 ++++++-- .../investigation-reporting-service/values-feature.yaml | 8 +++++++- charts/kafka-connect-sink/values-feature.yaml | 8 +++++++- charts/ldfdata-reporting-service/values-feature.yaml | 8 +++++++- charts/liquibase/values-feature.yaml | 8 +++++++- charts/modernization-api/values-feature.yaml | 6 +++++- charts/nbs-gateway/values-feature.yaml | 9 +++++++-- charts/nifi-efs/values-feature.yaml | 8 ++++++-- charts/nnd-service/values-feature.yaml | 9 +++++++-- charts/observation-reporting-service/values-feature.yaml | 8 +++++++- .../organization-reporting-service/values-feature.yaml | 8 +++++++- charts/page-builder-api/values-feature.yaml | 6 +++++- charts/person-reporting-service/values-feature.yaml | 8 +++++++- .../values-feature.yaml | 8 +++++++- charts/srte-data-service/values-feature.yaml | 8 ++++---- 17 files changed, 107 insertions(+), 27 deletions(-) diff --git a/charts/data-processing-service/values-feature.yaml b/charts/data-processing-service/values-feature.yaml index 4556ad0e6..97b0ad13e 100644 --- a/charts/data-processing-service/values-feature.yaml +++ b/charts/data-processing-service/values-feature.yaml @@ -41,11 +41,11 @@ ingress: # Kubernetes resource limits resources: limits: - memory: "800Mi" - cpu: "1000m" + memory: "1Gi" + cpu: "500m" requests: - memory: "600Mi" - cpu: "500m" + memory: "512Mi" + cpu: "250m" autoscaling: enabled: false diff --git a/charts/dataingestion-service/values-feature.yaml b/charts/dataingestion-service/values-feature.yaml index eccd128af..d2e630527 100644 --- a/charts/dataingestion-service/values-feature.yaml +++ b/charts/dataingestion-service/values-feature.yaml @@ -61,7 +61,13 @@ srtedataService: # Kubernetes resource limits -resources: {} +resources: + limits: + memory: "1Gi" + cpu: "500m" + requests: + memory: "512Mi" + cpu: "250m" autoscaling: enabled: false diff --git a/charts/elasticsearch-efs/values-feature.yaml b/charts/elasticsearch-efs/values-feature.yaml index 67a877252..ba0b42be1 100644 --- a/charts/elasticsearch-efs/values-feature.yaml +++ b/charts/elasticsearch-efs/values-feature.yaml @@ -33,9 +33,13 @@ ingress: annotations: {} tls: [] -resources: +resources: limits: - memory: "3Gi" + memory: "1Gi" + cpu: "500m" + requests: + memory: "512Mi" + cpu: "250m" autoscaling: enabled: false diff --git a/charts/investigation-reporting-service/values-feature.yaml b/charts/investigation-reporting-service/values-feature.yaml index f3514e7b3..1a11f89da 100644 --- a/charts/investigation-reporting-service/values-feature.yaml +++ b/charts/investigation-reporting-service/values-feature.yaml @@ -40,7 +40,13 @@ ingress: - app.feature.nbspreview.com # Kubernetes resource limits -resources: {} +resources: + limits: + memory: "1Gi" + cpu: "500m" + requests: + memory: "512Mi" + cpu: "250m" autoscaling: enabled: false diff --git a/charts/kafka-connect-sink/values-feature.yaml b/charts/kafka-connect-sink/values-feature.yaml index 48cf4ed47..7d58cca1f 100644 --- a/charts/kafka-connect-sink/values-feature.yaml +++ b/charts/kafka-connect-sink/values-feature.yaml @@ -81,7 +81,13 @@ customEnv: { } SQL_SERVER_CONNECTOR_CONFIG: /etc/sqlServerConnector.json JDBC_CONNECTOR_CONFIG: /etc/jdbcConnector.sh -resources: { } +resources: + limits: + memory: "1Gi" + cpu: "500m" + requests: + memory: "512Mi" + cpu: "250m" # We usually recommend not to specify default resources and to leave this as a conscious # choice for the user. This also increases chances charts run on environments with little # resources, such as Minikube. If you do want to specify resources, uncomment the following diff --git a/charts/ldfdata-reporting-service/values-feature.yaml b/charts/ldfdata-reporting-service/values-feature.yaml index f47b882a6..1a65f350e 100644 --- a/charts/ldfdata-reporting-service/values-feature.yaml +++ b/charts/ldfdata-reporting-service/values-feature.yaml @@ -40,7 +40,13 @@ ingress: - app.feature.nbspreview.com # Kubernetes resource limits -resources: {} +resources: + limits: + memory: "1Gi" + cpu: "500m" + requests: + memory: "512Mi" + cpu: "250m" autoscaling: enabled: false diff --git a/charts/liquibase/values-feature.yaml b/charts/liquibase/values-feature.yaml index 016e0da2f..81e0e0451 100644 --- a/charts/liquibase/values-feature.yaml +++ b/charts/liquibase/values-feature.yaml @@ -40,7 +40,13 @@ ingress: - app.feature.nbspreview.com # Kubernetes resource limits -resources: {} +resources: + limits: + memory: "1Gi" + cpu: "500m" + requests: + memory: "512Mi" + cpu: "250m" autoscaling: enabled: false diff --git a/charts/modernization-api/values-feature.yaml b/charts/modernization-api/values-feature.yaml index 3d422a942..7711decd5 100644 --- a/charts/modernization-api/values-feature.yaml +++ b/charts/modernization-api/values-feature.yaml @@ -63,7 +63,11 @@ nbsExternalName: app-classic.feature.nbspreview.com resources: limits: - memory: "2Gi" + memory: "1Gi" + cpu: "500m" + requests: + memory: "512Mi" + cpu: "250m" autoscaling: enabled: false diff --git a/charts/nbs-gateway/values-feature.yaml b/charts/nbs-gateway/values-feature.yaml index b88a6991a..01b8cc338 100644 --- a/charts/nbs-gateway/values-feature.yaml +++ b/charts/nbs-gateway/values-feature.yaml @@ -35,8 +35,13 @@ pageBuilder: enabled: "true" edit: enabled: "true" - -resources: {} +resources: + limits: + memory: "1Gi" + cpu: "500m" + requests: + memory: "512Mi" + cpu: "250m" nbsExternalName: app-classic.feature.nbspreview.com diff --git a/charts/nifi-efs/values-feature.yaml b/charts/nifi-efs/values-feature.yaml index b3dd03c76..af2df287a 100644 --- a/charts/nifi-efs/values-feature.yaml +++ b/charts/nifi-efs/values-feature.yaml @@ -48,9 +48,13 @@ ingress: port: number: 8443 -resources: +resources: limits: - memory: "3Gi" + memory: "2Gi" + cpu: "500m" + requests: + memory: "1Gi" + cpu: "250m" autoscaling: enabled: false diff --git a/charts/nnd-service/values-feature.yaml b/charts/nnd-service/values-feature.yaml index 15ed8d97e..7eb84d957 100644 --- a/charts/nnd-service/values-feature.yaml +++ b/charts/nnd-service/values-feature.yaml @@ -30,8 +30,13 @@ service: port: 8081 httpsPort: 443 -# Kubernetes resource limits -resources: {} +resources: + limits: + memory: "1Gi" + cpu: "500m" + requests: + memory: "512Mi" + cpu: "250m" autoscaling: enabled: false diff --git a/charts/observation-reporting-service/values-feature.yaml b/charts/observation-reporting-service/values-feature.yaml index bb3144ae2..761932532 100644 --- a/charts/observation-reporting-service/values-feature.yaml +++ b/charts/observation-reporting-service/values-feature.yaml @@ -40,7 +40,13 @@ ingress: - app.feature.nbspreview.com # Kubernetes resource limits -resources: {} +resources: + limits: + memory: "1Gi" + cpu: "500m" + requests: + memory: "512Mi" + cpu: "250m" autoscaling: enabled: false diff --git a/charts/organization-reporting-service/values-feature.yaml b/charts/organization-reporting-service/values-feature.yaml index 01ed8021f..a37fc5f06 100644 --- a/charts/organization-reporting-service/values-feature.yaml +++ b/charts/organization-reporting-service/values-feature.yaml @@ -40,7 +40,13 @@ ingress: - app.feature.nbspreview.com # Kubernetes resource limits -resources: {} +resources: + limits: + memory: "1Gi" + cpu: "500m" + requests: + memory: "512Mi" + cpu: "250m" autoscaling: enabled: false diff --git a/charts/page-builder-api/values-feature.yaml b/charts/page-builder-api/values-feature.yaml index 9afaeeb7d..962e7180f 100644 --- a/charts/page-builder-api/values-feature.yaml +++ b/charts/page-builder-api/values-feature.yaml @@ -60,7 +60,11 @@ istioGatewayIngress: # Kubernetes resource limits resources: limits: - memory: "2Gi" + memory: "1Gi" + cpu: "500m" + requests: + memory: "512Mi" + cpu: "250m" autoscaling: enabled: false diff --git a/charts/person-reporting-service/values-feature.yaml b/charts/person-reporting-service/values-feature.yaml index 0f7d3b071..90d19b4ce 100644 --- a/charts/person-reporting-service/values-feature.yaml +++ b/charts/person-reporting-service/values-feature.yaml @@ -40,7 +40,13 @@ ingress: - app.feature.nbspreview.com # Kubernetes resource limits -resources: {} +resources: + limits: + memory: "1Gi" + cpu: "500m" + requests: + memory: "512Mi" + cpu: "250m" autoscaling: enabled: false diff --git a/charts/post-processing-reporting-service/values-feature.yaml b/charts/post-processing-reporting-service/values-feature.yaml index 8f57c6860..bc8932ab0 100644 --- a/charts/post-processing-reporting-service/values-feature.yaml +++ b/charts/post-processing-reporting-service/values-feature.yaml @@ -43,7 +43,13 @@ ingress: - app.feature.nbspreview.com # Kubernetes resource limits -resources: {} +resources: + limits: + memory: "1Gi" + cpu: "500m" + requests: + memory: "512Mi" + cpu: "250m" autoscaling: enabled: false diff --git a/charts/srte-data-service/values-feature.yaml b/charts/srte-data-service/values-feature.yaml index c9e0ad287..cb4e20db7 100644 --- a/charts/srte-data-service/values-feature.yaml +++ b/charts/srte-data-service/values-feature.yaml @@ -33,11 +33,11 @@ service: # Kubernetes resource limits resources: limits: - memory: "2Gi" - cpu: "1000m" + memory: "1Gi" + cpu: "500m" requests: - memory: "1Gi" - cpu: "500m" + memory: "512Mi" + cpu: "250m" autoscaling: enabled: false From 4cfe7e8e4502910867ef3b0542502e4715fdaf5c Mon Sep 17 00:00:00 2001 From: Duc Nguyen Date: Tue, 13 May 2025 15:30:45 -0700 Subject: [PATCH 02/24] update image --- charts/data-processing-service/values-dts1.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/data-processing-service/values-dts1.yaml b/charts/data-processing-service/values-dts1.yaml index 1291e7bfb..7b1f5e1e4 100644 --- a/charts/data-processing-service/values-dts1.yaml +++ b/charts/data-processing-service/values-dts1.yaml @@ -5,7 +5,7 @@ env: "dts" image: repository: "" pullPolicy: IfNotPresent - tag: 1.0.1-SNAPSHOT.5723def + tag: 1.0.1-SNAPSHOT-PERF-3-5-13-2025-1.fda2445 imagePullSecrets: [] nameOverride: "" From e440431f8d291e47ff5c873d14de91f9efdb1d98 Mon Sep 17 00:00:00 2001 From: Duc Nguyen Date: Tue, 13 May 2025 15:31:54 -0700 Subject: [PATCH 03/24] update image --- charts/data-processing-service/values-dts1.yaml | 2 +- charts/data-processing-service/values-feature.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/data-processing-service/values-dts1.yaml b/charts/data-processing-service/values-dts1.yaml index 7b1f5e1e4..1291e7bfb 100644 --- a/charts/data-processing-service/values-dts1.yaml +++ b/charts/data-processing-service/values-dts1.yaml @@ -5,7 +5,7 @@ env: "dts" image: repository: "" pullPolicy: IfNotPresent - tag: 1.0.1-SNAPSHOT-PERF-3-5-13-2025-1.fda2445 + tag: 1.0.1-SNAPSHOT.5723def imagePullSecrets: [] nameOverride: "" diff --git a/charts/data-processing-service/values-feature.yaml b/charts/data-processing-service/values-feature.yaml index 97b0ad13e..dfd08428f 100644 --- a/charts/data-processing-service/values-feature.yaml +++ b/charts/data-processing-service/values-feature.yaml @@ -5,7 +5,7 @@ env: "feature" image: repository: "" pullPolicy: IfNotPresent - tag: 1.0.1-SNAPSHOT.3998b71 + tag: 1.0.1-SNAPSHOT-PERF-3-5-13-2025-1.fda2445 imagePullSecrets: [] nameOverride: "" From 0a0e652f561cecaab488bb23a266645e24505050 Mon Sep 17 00:00:00 2001 From: Duc Nguyen Date: Tue, 13 May 2025 15:33:47 -0700 Subject: [PATCH 04/24] updat --- charts/data-processing-service/values-feature.yaml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/charts/data-processing-service/values-feature.yaml b/charts/data-processing-service/values-feature.yaml index dfd08428f..f2eb38136 100644 --- a/charts/data-processing-service/values-feature.yaml +++ b/charts/data-processing-service/values-feature.yaml @@ -40,12 +40,15 @@ ingress: # Kubernetes resource limits resources: - limits: - memory: "1Gi" - cpu: "500m" requests: - memory: "512Mi" - cpu: "250m" + memory: "1Gi" + cpu: "1000m" + # limits: + # memory: "1Gi" + # cpu: "500m" + # requests: + # memory: "512Mi" + # cpu: "250m" autoscaling: enabled: false From 4771954d81ad5634b015d6fe7c3e4550a9dd7b35 Mon Sep 17 00:00:00 2001 From: Duc Nguyen Date: Tue, 13 May 2025 15:56:52 -0700 Subject: [PATCH 05/24] test affinity --- charts/data-processing-service/templates/deployment.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/charts/data-processing-service/templates/deployment.yaml b/charts/data-processing-service/templates/deployment.yaml index fb57869b0..826873451 100644 --- a/charts/data-processing-service/templates/deployment.yaml +++ b/charts/data-processing-service/templates/deployment.yaml @@ -20,6 +20,13 @@ spec: labels: {{- include "data-processing-service.selectorLabels" . | nindent 8 }} spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app.kubernetes.io/name: {{ include "data-processing-service.name" . }} + topologyKey: "kubernetes.io/hostname" {{- with .Values.imagePullSecrets }} imagePullSecrets: {{- toYaml . | nindent 8 }} From 073b71e824f05c722d63f17c8c9f953b5ac81b56 Mon Sep 17 00:00:00 2001 From: Duc Nguyen Date: Tue, 13 May 2025 16:04:34 -0700 Subject: [PATCH 06/24] update --- charts/data-processing-service/templates/deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/data-processing-service/templates/deployment.yaml b/charts/data-processing-service/templates/deployment.yaml index 826873451..a19e0ac2e 100644 --- a/charts/data-processing-service/templates/deployment.yaml +++ b/charts/data-processing-service/templates/deployment.yaml @@ -22,7 +22,7 @@ spec: spec: affinity: podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: + preferredDuringSchedulingIgnoredDuringExecution: - labelSelector: matchLabels: app.kubernetes.io/name: {{ include "data-processing-service.name" . }} From 2385ecadaf69803bc76286d2eca315760172a2e0 Mon Sep 17 00:00:00 2001 From: Duc Nguyen Date: Tue, 13 May 2025 16:05:58 -0700 Subject: [PATCH 07/24] update --- .../data-processing-service/templates/deployment.yaml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/charts/data-processing-service/templates/deployment.yaml b/charts/data-processing-service/templates/deployment.yaml index a19e0ac2e..bfda49557 100644 --- a/charts/data-processing-service/templates/deployment.yaml +++ b/charts/data-processing-service/templates/deployment.yaml @@ -23,10 +23,12 @@ spec: affinity: podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchLabels: - app.kubernetes.io/name: {{ include "data-processing-service.name" . }} - topologyKey: "kubernetes.io/hostname" + - weight: 100 + podAffinityTerm: + labelSelector: + matchLabels: + app.kubernetes.io/name: {{ include "data-processing-service.name" . }} + topologyKey: "kubernetes.io/hostname" {{- with .Values.imagePullSecrets }} imagePullSecrets: {{- toYaml . | nindent 8 }} From d50ab5fefb9ed2e3044f64902e1b0aea26370ba2 Mon Sep 17 00:00:00 2001 From: Duc Nguyen Date: Tue, 13 May 2025 16:07:47 -0700 Subject: [PATCH 08/24] update --- .../templates/deployment.yaml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/charts/data-processing-service/templates/deployment.yaml b/charts/data-processing-service/templates/deployment.yaml index bfda49557..f313d5fd3 100644 --- a/charts/data-processing-service/templates/deployment.yaml +++ b/charts/data-processing-service/templates/deployment.yaml @@ -20,15 +20,15 @@ spec: labels: {{- include "data-processing-service.selectorLabels" . | nindent 8 }} spec: - affinity: - podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 100 - podAffinityTerm: - labelSelector: - matchLabels: - app.kubernetes.io/name: {{ include "data-processing-service.name" . }} - topologyKey: "kubernetes.io/hostname" + # affinity: + # podAntiAffinity: + # preferredDuringSchedulingIgnoredDuringExecution: + # - weight: 100 + # podAffinityTerm: + # labelSelector: + # matchLabels: + # app.kubernetes.io/name: {{ include "data-processing-service.name" . }} + # topologyKey: "kubernetes.io/hostname" {{- with .Values.imagePullSecrets }} imagePullSecrets: {{- toYaml . | nindent 8 }} From 9681904cd95828fa4f21740e05e66db9fb06b943 Mon Sep 17 00:00:00 2001 From: Duc Nguyen Date: Tue, 13 May 2025 16:09:01 -0700 Subject: [PATCH 09/24] update --- .../templates/deployment.yaml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/charts/data-processing-service/templates/deployment.yaml b/charts/data-processing-service/templates/deployment.yaml index f313d5fd3..bfda49557 100644 --- a/charts/data-processing-service/templates/deployment.yaml +++ b/charts/data-processing-service/templates/deployment.yaml @@ -20,15 +20,15 @@ spec: labels: {{- include "data-processing-service.selectorLabels" . | nindent 8 }} spec: - # affinity: - # podAntiAffinity: - # preferredDuringSchedulingIgnoredDuringExecution: - # - weight: 100 - # podAffinityTerm: - # labelSelector: - # matchLabels: - # app.kubernetes.io/name: {{ include "data-processing-service.name" . }} - # topologyKey: "kubernetes.io/hostname" + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchLabels: + app.kubernetes.io/name: {{ include "data-processing-service.name" . }} + topologyKey: "kubernetes.io/hostname" {{- with .Values.imagePullSecrets }} imagePullSecrets: {{- toYaml . | nindent 8 }} From 709fb697c8b5b61740f65b4e15b73006b7b0179f Mon Sep 17 00:00:00 2001 From: Duc Nguyen Date: Tue, 13 May 2025 16:38:27 -0700 Subject: [PATCH 10/24] env --- .../templates/deployment.yaml | 4 ++++ charts/data-processing-service/values-feature.yaml | 12 ++++++++---- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/charts/data-processing-service/templates/deployment.yaml b/charts/data-processing-service/templates/deployment.yaml index bfda49557..6d6574935 100644 --- a/charts/data-processing-service/templates/deployment.yaml +++ b/charts/data-processing-service/templates/deployment.yaml @@ -100,6 +100,10 @@ spec: value: {{ .Values.features.modernizedMatching.enabled | default false | quote }} - name: MODERNIZED_MATCHING_URL value: {{ .Values.features.modernizedMatching.url | default "" }} + - name: THREAD_FLAG + value: {{ .Values.thread.enabled | default false | quote }} + - name: THREAD_POOL_SIZE + value: "{{ .Values.thread.size }}"" resources: {{- toYaml .Values.resources | nindent 12 }} {{- with .Values.nodeSelector }} diff --git a/charts/data-processing-service/values-feature.yaml b/charts/data-processing-service/values-feature.yaml index f2eb38136..b70cbceb9 100644 --- a/charts/data-processing-service/values-feature.yaml +++ b/charts/data-processing-service/values-feature.yaml @@ -42,10 +42,10 @@ ingress: resources: requests: memory: "1Gi" - cpu: "1000m" - # limits: - # memory: "1Gi" - # cpu: "500m" + cpu: "800m" + limits: + memory: "1Gi" + cpu: "1500m" # requests: # memory: "512Mi" # cpu: "250m" @@ -101,3 +101,7 @@ features: enabled: false url: "" +thread: + enabled: false + size: "5" + From a464b471ece5dce3b6291bbfdc8963ed3424ba5e Mon Sep 17 00:00:00 2001 From: Duc Nguyen Date: Tue, 13 May 2025 16:42:27 -0700 Subject: [PATCH 11/24] update --- charts/data-processing-service/templates/deployment.yaml | 2 +- charts/data-processing-service/values.yaml | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/charts/data-processing-service/templates/deployment.yaml b/charts/data-processing-service/templates/deployment.yaml index 6d6574935..404fee297 100644 --- a/charts/data-processing-service/templates/deployment.yaml +++ b/charts/data-processing-service/templates/deployment.yaml @@ -101,7 +101,7 @@ spec: - name: MODERNIZED_MATCHING_URL value: {{ .Values.features.modernizedMatching.url | default "" }} - name: THREAD_FLAG - value: {{ .Values.thread.enabled | default false | quote }} + value: {{ .Values.thread.enabled }} - name: THREAD_POOL_SIZE value: "{{ .Values.thread.size }}"" resources: diff --git a/charts/data-processing-service/values.yaml b/charts/data-processing-service/values.yaml index d6e2a55a1..aed9fe78a 100644 --- a/charts/data-processing-service/values.yaml +++ b/charts/data-processing-service/values.yaml @@ -97,3 +97,7 @@ features: modernizedMatching: enabled: false url: "" + +thread: + enabled: false + size: "5" From 5907fe992fbad12b388de47a7e9030ed8154e90f Mon Sep 17 00:00:00 2001 From: Duc Nguyen Date: Tue, 13 May 2025 16:43:31 -0700 Subject: [PATCH 12/24] update --- charts/data-processing-service/templates/deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/data-processing-service/templates/deployment.yaml b/charts/data-processing-service/templates/deployment.yaml index 404fee297..1f616c4c1 100644 --- a/charts/data-processing-service/templates/deployment.yaml +++ b/charts/data-processing-service/templates/deployment.yaml @@ -103,7 +103,7 @@ spec: - name: THREAD_FLAG value: {{ .Values.thread.enabled }} - name: THREAD_POOL_SIZE - value: "{{ .Values.thread.size }}"" + value: "{{ .Values.thread.size }}" resources: {{- toYaml .Values.resources | nindent 12 }} {{- with .Values.nodeSelector }} From 75746dc565dcc2312ddbec883b5d260e5844d2c6 Mon Sep 17 00:00:00 2001 From: Duc Nguyen Date: Tue, 13 May 2025 16:45:15 -0700 Subject: [PATCH 13/24] update --- charts/data-processing-service/templates/deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/data-processing-service/templates/deployment.yaml b/charts/data-processing-service/templates/deployment.yaml index 1f616c4c1..4ecfe86fd 100644 --- a/charts/data-processing-service/templates/deployment.yaml +++ b/charts/data-processing-service/templates/deployment.yaml @@ -101,7 +101,7 @@ spec: - name: MODERNIZED_MATCHING_URL value: {{ .Values.features.modernizedMatching.url | default "" }} - name: THREAD_FLAG - value: {{ .Values.thread.enabled }} + value: {{ .Values.thread.enabled | quote}} - name: THREAD_POOL_SIZE value: "{{ .Values.thread.size }}" resources: From fe130c2a9fff3fba18b64855f7bb7d37a7b40ccf Mon Sep 17 00:00:00 2001 From: Duc Nguyen Date: Tue, 13 May 2025 17:12:42 -0700 Subject: [PATCH 14/24] update --- charts/data-processing-service/values-feature.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/data-processing-service/values-feature.yaml b/charts/data-processing-service/values-feature.yaml index b70cbceb9..9e12ad6fc 100644 --- a/charts/data-processing-service/values-feature.yaml +++ b/charts/data-processing-service/values-feature.yaml @@ -5,7 +5,7 @@ env: "feature" image: repository: "" pullPolicy: IfNotPresent - tag: 1.0.1-SNAPSHOT-PERF-3-5-13-2025-1.fda2445 + tag: 1.0.1-SNAPSHOT-THREAD-5132025.1.fda2445 imagePullSecrets: [] nameOverride: "" From bb863db87d72a9778a06c0d45b4788d468c64623 Mon Sep 17 00:00:00 2001 From: Duc Nguyen Date: Tue, 13 May 2025 17:41:37 -0700 Subject: [PATCH 15/24] update image --- charts/data-processing-service/values-feature.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/data-processing-service/values-feature.yaml b/charts/data-processing-service/values-feature.yaml index 9e12ad6fc..063a9428f 100644 --- a/charts/data-processing-service/values-feature.yaml +++ b/charts/data-processing-service/values-feature.yaml @@ -5,7 +5,7 @@ env: "feature" image: repository: "" pullPolicy: IfNotPresent - tag: 1.0.1-SNAPSHOT-THREAD-5132025.1.fda2445 + tag: 1.0.1-SNAPSHOT-5132025-2.78f470b imagePullSecrets: [] nameOverride: "" From ec61693f837f55182b84472aa770a4a59fe98279 Mon Sep 17 00:00:00 2001 From: Duc Nguyen Date: Fri, 16 May 2025 22:07:30 -0700 Subject: [PATCH 16/24] update --- charts/data-processing-service/values-feature.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/data-processing-service/values-feature.yaml b/charts/data-processing-service/values-feature.yaml index 063a9428f..a96f47ef5 100644 --- a/charts/data-processing-service/values-feature.yaml +++ b/charts/data-processing-service/values-feature.yaml @@ -5,7 +5,7 @@ env: "feature" image: repository: "" pullPolicy: IfNotPresent - tag: 1.0.1-SNAPSHOT-5132025-2.78f470b + tag: 1.0.1-SNAPSHOT-PERF-5162025-1.2cc17da imagePullSecrets: [] nameOverride: "" From 6c1a4a36159a31e6696cfc7966d7db2b500ac0bc Mon Sep 17 00:00:00 2001 From: Duc Nguyen Date: Fri, 16 May 2025 23:42:08 -0700 Subject: [PATCH 17/24] support for custom hikari --- .../data-processing-service/templates/deployment.yaml | 10 ++++++++++ charts/data-processing-service/values-dts1.yaml | 6 ++++++ charts/data-processing-service/values-feature.yaml | 6 ++++++ charts/data-processing-service/values-int1.yaml | 7 +++++++ charts/data-processing-service/values.yaml | 9 +++++++++ 5 files changed, 38 insertions(+) diff --git a/charts/data-processing-service/templates/deployment.yaml b/charts/data-processing-service/templates/deployment.yaml index 4ecfe86fd..c0359530a 100644 --- a/charts/data-processing-service/templates/deployment.yaml +++ b/charts/data-processing-service/templates/deployment.yaml @@ -104,6 +104,16 @@ spec: value: {{ .Values.thread.enabled | quote}} - name: THREAD_POOL_SIZE value: "{{ .Values.thread.size }}" + - name: HIKARI_POOL_SIZE + value: "{{ .Values.hikari.poolSize }}" + - name: HIKARI_MINIMUM_IDLE + value: "{{ .Values.hikari.minIdle }}" + - name: HIKARI_CON_TIMEOUT + value: "{{ .Values.hikari.conTimeout }}" + - name: HIKARI_MAX_LIFETIME + value: "{{ .Values.hikari.maxLifeTime }}" + - name: LEAK_DETECTION_THRESHOLD + value: "{{ .Values.hikari.leakDetectThresHold }}" resources: {{- toYaml .Values.resources | nindent 12 }} {{- with .Values.nodeSelector }} diff --git a/charts/data-processing-service/values-dts1.yaml b/charts/data-processing-service/values-dts1.yaml index 002c39ff4..297ac8901 100644 --- a/charts/data-processing-service/values-dts1.yaml +++ b/charts/data-processing-service/values-dts1.yaml @@ -97,3 +97,9 @@ features: enabled: false url: "" +hikari: + poolSize: "20" + minIdle: "5" + conTimeout: "30000" + maxLifeTime: "5000" + leakDetectThresHold: "10000" diff --git a/charts/data-processing-service/values-feature.yaml b/charts/data-processing-service/values-feature.yaml index a96f47ef5..cb59f1e96 100644 --- a/charts/data-processing-service/values-feature.yaml +++ b/charts/data-processing-service/values-feature.yaml @@ -105,3 +105,9 @@ thread: enabled: false size: "5" +hikari: + poolSize: "20" + minIdle: "5" + conTimeout: "30000" + maxLifeTime: "5000" + leakDetectThresHold: "10000" diff --git a/charts/data-processing-service/values-int1.yaml b/charts/data-processing-service/values-int1.yaml index a7a59a628..d0fe62010 100644 --- a/charts/data-processing-service/values-int1.yaml +++ b/charts/data-processing-service/values-int1.yaml @@ -97,3 +97,10 @@ features: modernizedMatching: enabled: true url: "http://deduplication-api.default.svc.cluster.local:8083/nbs/api/deduplication/" + +hikari: + poolSize: "20" + minIdle: "5" + conTimeout: "30000" + maxLifeTime: "5000" + leakDetectThresHold: "10000" diff --git a/charts/data-processing-service/values.yaml b/charts/data-processing-service/values.yaml index aed9fe78a..ac46fdc7a 100644 --- a/charts/data-processing-service/values.yaml +++ b/charts/data-processing-service/values.yaml @@ -101,3 +101,12 @@ features: thread: enabled: false size: "5" + + +hikari: + poolSize: "20" + minIdle: "5" + conTimeout: "30000" + maxLifeTime: "5000" + leakDetectThresHold: "10000" + From ac6d4b6c0060053b1361848cbac1dd1366c47ab7 Mon Sep 17 00:00:00 2001 From: Duc Nguyen Date: Wed, 21 May 2025 21:00:39 -0700 Subject: [PATCH 18/24] update --- charts/data-processing-service/values-feature.yaml | 2 +- charts/dataingestion-service/values-feature.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/data-processing-service/values-feature.yaml b/charts/data-processing-service/values-feature.yaml index cb59f1e96..8afc95518 100644 --- a/charts/data-processing-service/values-feature.yaml +++ b/charts/data-processing-service/values-feature.yaml @@ -5,7 +5,7 @@ env: "feature" image: repository: "" pullPolicy: IfNotPresent - tag: 1.0.1-SNAPSHOT-PERF-5162025-1.2cc17da + tag: 1.0.1-SNAPSHOT-05212025-DI-1.f5a77f8 imagePullSecrets: [] nameOverride: "" diff --git a/charts/dataingestion-service/values-feature.yaml b/charts/dataingestion-service/values-feature.yaml index f3f1f0bba..5ce7d9978 100644 --- a/charts/dataingestion-service/values-feature.yaml +++ b/charts/dataingestion-service/values-feature.yaml @@ -5,7 +5,7 @@ env: "feature" image: repository: "" pullPolicy: IfNotPresent - tag: 1.0.1-SNAPSHOT.3998b71 + tag: 1.0.1-SNAPSHOT-05212025-DI-1.f5a77f8 imagePullSecrets: [] nameOverride: "" From 4d227a1f4e16ac33d3c51c1435bb19fa3dd3d1cb Mon Sep 17 00:00:00 2001 From: Duc Nguyen Date: Wed, 21 May 2025 21:01:34 -0700 Subject: [PATCH 19/24] update iamge --- charts/data-processing-service/values-feature.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/data-processing-service/values-feature.yaml b/charts/data-processing-service/values-feature.yaml index 8afc95518..50ba8db07 100644 --- a/charts/data-processing-service/values-feature.yaml +++ b/charts/data-processing-service/values-feature.yaml @@ -5,7 +5,7 @@ env: "feature" image: repository: "" pullPolicy: IfNotPresent - tag: 1.0.1-SNAPSHOT-05212025-DI-1.f5a77f8 + tag: 1.0.1-SNAPSHOT-05212025-DP-1.f5a77f8 imagePullSecrets: [] nameOverride: "" From 31ff7ec3ed61fe2a4e38ab2da1175971694dc936 Mon Sep 17 00:00:00 2001 From: Duc Nguyen Date: Wed, 21 May 2025 21:31:43 -0700 Subject: [PATCH 20/24] update param --- charts/data-processing-service/templates/deployment.yaml | 4 ++++ charts/data-processing-service/values-dts1.yaml | 7 +++++++ charts/data-processing-service/values-feature.yaml | 6 +++++- charts/data-processing-service/values-test2.yaml | 7 +++++++ charts/data-processing-service/values.yaml | 2 ++ 5 files changed, 25 insertions(+), 1 deletion(-) diff --git a/charts/data-processing-service/templates/deployment.yaml b/charts/data-processing-service/templates/deployment.yaml index c0359530a..9311ba54c 100644 --- a/charts/data-processing-service/templates/deployment.yaml +++ b/charts/data-processing-service/templates/deployment.yaml @@ -104,6 +104,10 @@ spec: value: {{ .Values.thread.enabled | quote}} - name: THREAD_POOL_SIZE value: "{{ .Values.thread.size }}" + - name: THREAD_BATCH_SIZE + value: "{{ .Values.thread.batchSize }}" + - name: PROCESSOR_DELAY + value: "{{ .Values.thread.processorDelay }}" - name: HIKARI_POOL_SIZE value: "{{ .Values.hikari.poolSize }}" - name: HIKARI_MINIMUM_IDLE diff --git a/charts/data-processing-service/values-dts1.yaml b/charts/data-processing-service/values-dts1.yaml index 297ac8901..b9bb6a4e7 100644 --- a/charts/data-processing-service/values-dts1.yaml +++ b/charts/data-processing-service/values-dts1.yaml @@ -92,6 +92,13 @@ probes: kafkaThread: "1" +thread: + enabled: false + size: "5" + batchSize: "50" + processorDelay: "30000" + + features: modernizedMatching: enabled: false diff --git a/charts/data-processing-service/values-feature.yaml b/charts/data-processing-service/values-feature.yaml index 50ba8db07..a67438b3f 100644 --- a/charts/data-processing-service/values-feature.yaml +++ b/charts/data-processing-service/values-feature.yaml @@ -101,9 +101,13 @@ features: enabled: false url: "" + thread: - enabled: false + enabled: true size: "5" + batchSize: "50" + processorDelay: "30000" + hikari: poolSize: "20" diff --git a/charts/data-processing-service/values-test2.yaml b/charts/data-processing-service/values-test2.yaml index 5d8f85044..439d4cdd2 100644 --- a/charts/data-processing-service/values-test2.yaml +++ b/charts/data-processing-service/values-test2.yaml @@ -93,6 +93,13 @@ probes: kafkaThread: "1" +thread: + enabled: false + size: "5" + batchSize: "50" + processorDelay: "30000" + + features: modernizedMatching: enabled: false diff --git a/charts/data-processing-service/values.yaml b/charts/data-processing-service/values.yaml index ac46fdc7a..e45f52d87 100644 --- a/charts/data-processing-service/values.yaml +++ b/charts/data-processing-service/values.yaml @@ -101,6 +101,8 @@ features: thread: enabled: false size: "5" + batchSize: "50" + processorDelay: "30000" hikari: From 9cd3b64ae2a1d5746463baa9ac29f10c5208110f Mon Sep 17 00:00:00 2001 From: Duc Nguyen Date: Tue, 27 May 2025 15:22:46 -0700 Subject: [PATCH 21/24] use specific node group for feature dp --- charts/data-processing-service/values-feature.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/charts/data-processing-service/values-feature.yaml b/charts/data-processing-service/values-feature.yaml index a67438b3f..a10f40aa9 100644 --- a/charts/data-processing-service/values-feature.yaml +++ b/charts/data-processing-service/values-feature.yaml @@ -56,7 +56,8 @@ autoscaling: maxReplicas: 100 targetCPUUtilizationPercentage: 80 -nodeSelector: {} +nodeSelector: + nodegroup: high-cpu tolerations: [] From d701a2d29a9ae3805ad68bd2f8f29e8d6cdbf839 Mon Sep 17 00:00:00 2001 From: Duc Nguyen Date: Tue, 27 May 2025 17:16:15 -0700 Subject: [PATCH 22/24] update --- charts/data-processing-service/values-feature.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/data-processing-service/values-feature.yaml b/charts/data-processing-service/values-feature.yaml index a10f40aa9..36c99a76e 100644 --- a/charts/data-processing-service/values-feature.yaml +++ b/charts/data-processing-service/values-feature.yaml @@ -5,7 +5,7 @@ env: "feature" image: repository: "" pullPolicy: IfNotPresent - tag: 1.0.1-SNAPSHOT-05212025-DP-1.f5a77f8 + tag: 1.0.1-SNAPSHOT-DP-05272025-1.f5a77f8 imagePullSecrets: [] nameOverride: "" From f96ad9227b67e213d235e4ae5d75012f3356e02f Mon Sep 17 00:00:00 2001 From: Duc Nguyen Date: Tue, 27 May 2025 17:28:55 -0700 Subject: [PATCH 23/24] update --- charts/data-processing-service/values-feature.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/data-processing-service/values-feature.yaml b/charts/data-processing-service/values-feature.yaml index 36c99a76e..534202889 100644 --- a/charts/data-processing-service/values-feature.yaml +++ b/charts/data-processing-service/values-feature.yaml @@ -5,7 +5,7 @@ env: "feature" image: repository: "" pullPolicy: IfNotPresent - tag: 1.0.1-SNAPSHOT-DP-05272025-1.f5a77f8 + tag: 1.0.1-SNAPSHOT-DP-05272025-3.fb49c12 imagePullSecrets: [] nameOverride: "" From cfc03db55249dc2bd617e6c2445c71d3927d0908 Mon Sep 17 00:00:00 2001 From: Duc Nguyen Date: Wed, 16 Jul 2025 13:29:17 -0700 Subject: [PATCH 24/24] Update DP template --- charts/data-processing-service/values-feature.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/data-processing-service/values-feature.yaml b/charts/data-processing-service/values-feature.yaml index 534202889..e053428ef 100644 --- a/charts/data-processing-service/values-feature.yaml +++ b/charts/data-processing-service/values-feature.yaml @@ -5,7 +5,7 @@ env: "feature" image: repository: "" pullPolicy: IfNotPresent - tag: 1.0.1-SNAPSHOT-DP-05272025-3.fb49c12 + tag: 1.0.1-SNAPSHOT.28f097e imagePullSecrets: [] nameOverride: "" @@ -107,7 +107,7 @@ thread: enabled: true size: "5" batchSize: "50" - processorDelay: "30000" + processorDelay: "10000" hikari: