From a71078fe3c2cfe29b520aef6a7a4834a06e058af Mon Sep 17 00:00:00 2001
From: mkottakota1 <149763406+mkottakota1@users.noreply.github.com>
Date: Thu, 20 Nov 2025 10:13:25 +0000
Subject: [PATCH 01/16] Provisioning Vertica in Kuberntes and running the
 Pytests

---
 .github/workflows/ci.yaml | 501 ++++++++++++++++++++++++++++++++------
 1 file changed, 429 insertions(+), 72 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 85877170..cb9fac86 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -1,11 +1,11 @@
-name: CI
+name: Vertica CI
 
 on: [push, pull_request]
 
 jobs:
   build:
-
     runs-on: ubuntu-latest
+
     strategy:
       matrix:
         python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13', 'pypy3.10']
@@ -18,83 +18,440 @@ jobs:
       CLIENT_SECRET: P9f8350QQIUhFfK1GF5sMhq4Dm3P6Sbs
 
     steps:
-      - name: Check out repository
+      # ---------------------------
+      # Checkout and setup
+      # ---------------------------
+      - name: Checkout repository
         uses: actions/checkout@v4
+
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
-      - name: Set up a Keycloak docker container
-        timeout-minutes: 5
-        run: |
-          docker network create -d bridge my-network
-          docker run -d -p 8080:8080 \
-            --name keycloak --network my-network \
-            -e KEYCLOAK_ADMIN=admin -e KEYCLOAK_ADMIN_PASSWORD=admin \
-            quay.io/keycloak/keycloak:23.0.4 start-dev
-          docker container ls
 
-      - name: Set up a Vertica server docker container
-        timeout-minutes: 15
-        run: |
-          docker run -d -p 5433:5433 -p 5444:5444 \
-            --name vertica_docker --network my-network \
-            opentext/vertica-ce:24.4.0-0
-          echo "Vertica startup ..."
-          until docker exec vertica_docker test -f /data/vertica/VMart/agent_start.out; do \
-            echo "..."; \
-            sleep 3; \
-          done;
-          echo "Vertica is up"
-          docker exec -u dbadmin vertica_docker /opt/vertica/bin/vsql -c "\l"
-          docker exec -u dbadmin vertica_docker /opt/vertica/bin/vsql -c "select version()"
+      # ---------------------------
+      # Kubernetes (KinD) + Helm setup
+      # ---------------------------
+      - name: Set up Kubernetes (KinD)
+        uses: helm/kind-action@v1.8.0
+        with:
+          cluster_name: vertica-ci
+          node_image: kindest/node:v1.29.0
 
-      - name: Configure Keycloak
-        run: |
-          echo "Wait for keycloak ready ..."
-          bash -c 'while true; do curl -s localhost:8080 &>/dev/null; ret=$?; [[ $ret -eq 0 ]] && break; echo "..."; sleep 3; done'
+      - name: Set up Helm
+        uses: azure/setup-helm@v3
+        with:
+          version: "3.11.3"
 
-          docker exec -i keycloak /bin/bash <<EOF
-            /opt/keycloak/bin/kcadm.sh config credentials --server http://localhost:8080 --realm master --user admin --password admin
-            /opt/keycloak/bin/kcadm.sh create realms -s realm=${REALM} -s enabled=true
-            /opt/keycloak/bin/kcadm.sh update realms/${REALM} -s accessTokenLifespan=3600
-            /opt/keycloak/bin/kcadm.sh get realms/${REALM}
-            /opt/keycloak/bin/kcadm.sh create users -r ${REALM} -s username=${USER} -s enabled=true
-            /opt/keycloak/bin/kcadm.sh set-password -r ${REALM} --username ${USER} --new-password ${PASSWORD}
-            /opt/keycloak/bin/kcadm.sh get users -r ${REALM}
-            /opt/keycloak/bin/kcadm.sh create clients -r ${REALM} -s clientId=${CLIENT_ID} -s enabled=true \
-              -s 'redirectUris=["/*"]' -s 'webOrigins=["/*"]' -s secret=${CLIENT_SECRET} -s directAccessGrantsEnabled=true -o
+      - name: Add Helm repositories
+        run: |
+          helm repo add vertica-charts https://vertica.github.io/charts || true
+          helm repo add bitnami https://charts.bitnami.com/bitnami || true
+          helm repo update
+      # ---------------------------
+      # MinIO Setup
+      # ---------------------------
+      - name: Install MinIO
+        run: |
+          kubectl create ns minio || true
+          cat <<'EOF' > minio.yaml
+          apiVersion: apps/v1
+          kind: Deployment
+          metadata:
+            name: minio
+            namespace: minio
+          spec:
+            replicas: 1
+            selector:
+              matchLabels:
+                app: minio
+            template:
+              metadata:
+                labels:
+                  app: minio
+              spec:
+                containers:
+                  - name: minio
+                    image: minio/minio:RELEASE.2025-09-07T16-13-09Z-cpuv1
+                    args: ["server", "/data"]
+                    env:
+                      - name: MINIO_ROOT_USER
+                        value: "minioadmin"
+                      - name: MINIO_ROOT_PASSWORD
+                        value: "minioadmin"
+                    ports:
+                      - containerPort: 9000
+                    volumeMounts:
+                      - name: data
+                        mountPath: /data
+                volumes:
+                  - name: data
+                    emptyDir: {}
+          ---
+          apiVersion: v1
+          kind: Service
+          metadata:
+            name: minio
+            namespace: minio
+          spec:
+            selector:
+              app: minio
+            ports:
+              - port: 9000
+                targetPort: 9000
           EOF
-
-          # Retrieving an Access Token
-          curl --location --request POST http://`hostname`:8080/realms/${REALM}/protocol/openid-connect/token \
-              --header 'Content-Type: application/x-www-form-urlencoded' \
-              --data-urlencode "username=${USER}" \
-              --data-urlencode "password=${PASSWORD}" \
-              --data-urlencode "client_id=${CLIENT_ID}" \
-              --data-urlencode "client_secret=${CLIENT_SECRET}" \
-              --data-urlencode 'grant_type=password' -o oauth.json
-          cat oauth.json | python3 -c 'import json,sys;obj=json.load(sys.stdin);print(obj["access_token"])' > access_token.txt
-
-          docker exec -u dbadmin vertica_docker /opt/vertica/bin/vsql -c "CREATE AUTHENTICATION v_oauth METHOD 'oauth' HOST '0.0.0.0/0';"
-          docker exec -u dbadmin vertica_docker /opt/vertica/bin/vsql -c "ALTER AUTHENTICATION v_oauth SET client_id = '${CLIENT_ID}';"
-          docker exec -u dbadmin vertica_docker /opt/vertica/bin/vsql -c "ALTER AUTHENTICATION v_oauth SET client_secret = '${CLIENT_SECRET}';"
-          docker exec -u dbadmin vertica_docker /opt/vertica/bin/vsql -c "ALTER AUTHENTICATION v_oauth SET discovery_url = 'http://`hostname`:8080/realms/${REALM}/.well-known/openid-configuration';"
-          docker exec -u dbadmin vertica_docker /opt/vertica/bin/vsql -c "ALTER AUTHENTICATION v_oauth SET introspect_url = 'http://`hostname`:8080/realms/${REALM}/protocol/openid-connect/token/introspect';"
-          docker exec -u dbadmin vertica_docker /opt/vertica/bin/vsql -c "SELECT * FROM client_auth WHERE auth_name='v_oauth';"
-          docker exec -u dbadmin vertica_docker /opt/vertica/bin/vsql -c "CREATE USER ${USER};"
-          docker exec -u dbadmin vertica_docker /opt/vertica/bin/vsql -c "GRANT AUTHENTICATION v_oauth TO ${USER};"
-          docker exec -u dbadmin vertica_docker /opt/vertica/bin/vsql -c "GRANT ALL ON SCHEMA PUBLIC TO ${USER};"
-          # A dbadmin-specific authentication record (connect remotely) is needed after setting up an OAuth user
-          docker exec -u dbadmin vertica_docker /opt/vertica/bin/vsql -c "CREATE AUTHENTICATION v_dbadmin_hash METHOD 'hash' HOST '0.0.0.0/0';"
-          docker exec -u dbadmin vertica_docker /opt/vertica/bin/vsql -c "ALTER AUTHENTICATION v_dbadmin_hash PRIORITY 10000;"
-          docker exec -u dbadmin vertica_docker /opt/vertica/bin/vsql -c "GRANT AUTHENTICATION v_dbadmin_hash TO dbadmin;"
-
-      - name: Install dependencies
-        run: pip install tox
-      - name: Run tests
+          kubectl apply -f minio.yaml
+          kubectl -n minio rollout status deployment/minio --timeout=2m || true
+          kubectl get pods -n minio -o wide || true
+          kubectl get svc -n minio || true
+      - name: Ensure MinIO bucket exists
+        run: |
+          kubectl run mc-client --rm -i --restart=Never \
+            --image=minio/mc:latest \
+            -n minio \
+            --command -- bash -c "
+              mc alias set localminio http://minio.minio.svc.cluster.local:9000 minioadmin minioadmin && \
+              mc mb --ignore-existing localminio/vertica-fleeting && \
+              mc ls localminio
+            "
+      - name: Create MinIO Secret
+        run: |
+          kubectl create ns my-verticadb-operator || true
+          kubectl delete secret communal-creds -n my-verticadb-operator --ignore-not-found
+          kubectl create secret generic communal-creds \
+            -n my-verticadb-operator \
+            --from-literal=accesskey="minioadmin" \
+            --from-literal=secretkey="minioadmin"
+          kubectl get secret communal-creds -n my-verticadb-operator -o yaml || true
+      # ---------------------------
+      # Vertica Operator + DB Deployment
+      # ---------------------------
+      - name: Install Vertica Operator
+        run: |
+          cat <<'EOF' > operator-values.yaml
+          installCRDs: true
+          controller:
+            extraEnv:
+              - name: AWS_REGION
+                value: "us-east-1"
+              - name: AWS_DEFAULT_REGION
+                value: "us-east-1"
+          EOF
+          helm upgrade --install vdb-op vertica-charts/verticadb-operator \
+            -n my-verticadb-operator -f operator-values.yaml --wait --timeout 10m
+          kubectl -n my-verticadb-operator get pods -o wide || true
+      - name: Deploy VerticaDB and per-node Services
+        run: |
+          cat <<'EOF' | kubectl apply -f -
+          apiVersion: vertica.com/v1
+          kind: VerticaDB
+          metadata:
+            name: verticadb-sample
+            namespace: my-verticadb-operator
+            annotations:
+              vertica.com/k-safety: "0"
+          spec:
+            image: opentext/vertica-k8s:latest
+            dbName: vdb
+            initPolicy: Create
+            communal:
+              path: s3://vertica-fleeting/mkottakota/
+              credentialSecret: communal-creds
+              endpoint: http://minio.minio.svc.cluster.local:9000
+              region: us-east-1
+            local:
+              dataPath: /data
+              depotPath: /depot
+            subclusters:
+              - name: defaultsubcluster
+                size: 1
+          ---
+          # -------------------------------------
+          # Per-node Services (needed for LB tests)
+          # -------------------------------------
+          apiVersion: v1
+          kind: Service
+          metadata:
+            name: vertica-node-0
+            namespace: my-verticadb-operator
+          spec:
+            selector:
+              statefulset.kubernetes.io/pod-name: verticadb-sample-defaultsubcluster-0
+            ports:
+              - port: 5433
+                targetPort: 5433
+          ---
+          apiVersion: v1
+          kind: Service
+          metadata:
+            name: vertica-node-1
+            namespace: my-verticadb-operator
+          spec:
+            selector:
+              statefulset.kubernetes.io/pod-name: verticadb-sample-defaultsubcluster-1
+            ports:
+              - port: 5433
+                targetPort: 5433
+          ---
+          apiVersion: v1
+          kind: Service
+          metadata:
+            name: vertica-node-2
+            namespace: my-verticadb-operator
+          spec:
+            selector:
+              statefulset.kubernetes.io/pod-name: verticadb-sample-defaultsubcluster-2
+            ports:
+              - port: 5433
+                targetPort: 5433
+          EOF
+      - name: Wait for Vertica readiness
+        run: |
+          NS=my-verticadb-operator
+          SS=verticadb-sample-defaultsubcluster
+          POD=${SS}-0
+          for i in {1..30}; do
+            kubectl get pod ${POD} -n ${NS} && break || sleep 10
+          done
+          kubectl wait --for=condition=Ready pod/${POD} -n ${NS} --timeout=5m || true
+          echo "🚀 Creating test runner pod..."
+          kubectl -n ${NS} run test-runner --image=python:3.13-slim --restart=Never --command -- sleep infinity
+          kubectl -n ${NS} wait --for=condition=Ready pod/test-runner --timeout=180s
+          echo "🩹 Adding Vertica pod entries to /etc/hosts in test pod..."
+          for p in $(kubectl -n $NS get pods -l app.kubernetes.io/instance=verticadb-sample -o jsonpath='{.items[*].metadata.name}'); do
+            IP=$(kubectl -n $NS get pod $p -o jsonpath='{.status.podIP}')
+            echo "$IP $p.$NS.svc.cluster.local $p" | kubectl -n $NS exec -i test-runner -- tee -a /etc/hosts >/dev/null
+            echo "✔ Added $p -> $IP"
+          done
+          echo "📂 Copying repository into pod..."
+          kubectl -n ${NS} cp . test-runner:/workspace
+      # ---------------------------
+      # Keycloak + OAuth setup
+      # ---------------------------
+      - name: Deploy Keycloak
+        run: |
+          kubectl create ns keycloak || true
+          cat <<'EOF' | kubectl apply -f -
+          apiVersion: apps/v1
+          kind: Deployment
+          metadata:
+            name: keycloak
+            namespace: keycloak
+          spec:
+            replicas: 1
+            selector:
+              matchLabels:
+                app: keycloak
+            template:
+              metadata:
+                labels:
+                  app: keycloak
+              spec:
+                containers:
+                  - name: keycloak
+                    image: quay.io/keycloak/keycloak:23.0.4
+                    args: ["start-dev"]
+                    env:
+                      - name: KEYCLOAK_ADMIN
+                        value: admin
+                      - name: KEYCLOAK_ADMIN_PASSWORD
+                        value: admin
+                    ports:
+                      - containerPort: 8080
+                    readinessProbe:
+                      httpGet:
+                        path: /
+                        port: 8080
+                      initialDelaySeconds: 20
+                      periodSeconds: 5
+                      failureThreshold: 6
+          ---
+          apiVersion: v1
+          kind: Service
+          metadata:
+            name: keycloak
+            namespace: keycloak
+          spec:
+            selector:
+              app: keycloak
+            ports:
+              - port: 8080
+                targetPort: 8080
+          EOF
+      - name: Wait for Keycloak readiness
+        run: |
+          kubectl -n keycloak rollout status deploy/keycloak --timeout=2m
+          kubectl -n keycloak get pods -o wide
+      - name: Configure Keycloak realm, client, and user
+        run: |
+          kubectl -n keycloak exec deploy/keycloak -- \
+            /opt/keycloak/bin/kcadm.sh config credentials \
+              --server http://localhost:8080 --realm master \
+              --user admin --password admin
+          kubectl -n keycloak exec deploy/keycloak -- \
+            /opt/keycloak/bin/kcadm.sh create realms -s realm=test -s enabled=true
+          kubectl -n keycloak exec deploy/keycloak -- \
+            /opt/keycloak/bin/kcadm.sh create clients -r test \
+              -s clientId=vertica -s enabled=true \
+              -s secret=P9f8350QQIUhFfK1GF5sMhq4Dm3P6Sbs \
+              -s 'redirectUris=["*"]' \
+              -s directAccessGrantsEnabled=true
+          kubectl -n keycloak exec deploy/keycloak -- \
+            /opt/keycloak/bin/kcadm.sh create users -r test \
+              -s username=oauth_user -s enabled=true
+          kubectl -n keycloak exec deploy/keycloak -- \
+            /opt/keycloak/bin/kcadm.sh set-password -r test \
+              --username oauth_user --new-password password
+      - name: Configure Vertica Authentication
+        run: |
+          NS=my-verticadb-operator
+          POD=verticadb-sample-defaultsubcluster-0
+          kubectl -n ${NS} exec ${POD} -c server -- bash -c "
+            /opt/vertica/bin/vsql -U dbadmin -c \"
+              CREATE AUTHENTICATION v_oauth METHOD 'oauth' HOST '0.0.0.0/0';
+              ALTER AUTHENTICATION v_oauth SET client_id = 'vertica';
+              ALTER AUTHENTICATION v_oauth SET client_secret = 'P9f8350QQIUhFfK1GF5sMhq4Dm3P6Sbs';
+              ALTER AUTHENTICATION v_oauth SET discovery_url = 'http://keycloak.keycloak.svc.cluster.local:8080/realms/test/.well-known/openid-configuration';
+              ALTER AUTHENTICATION v_oauth SET introspect_url = 'http://keycloak.keycloak.svc.cluster.local:8080/realms/test/protocol/openid-connect/token/introspect';
+              CREATE USER oauth_user;
+              GRANT AUTHENTICATION v_oauth TO oauth_user;
+              GRANT ALL ON SCHEMA PUBLIC TO oauth_user;
+              CREATE AUTHENTICATION v_dbadmin_hash METHOD 'hash' HOST '0.0.0.0/0';
+              ALTER AUTHENTICATION v_dbadmin_hash PRIORITY 10000;
+              GRANT AUTHENTICATION v_dbadmin_hash TO dbadmin;
+            \"
+          "
+          
+      # ---------------------------
+      # Testing section
+      # ---------------------------
+      - name: Check Vertica load balance policy
+        run: |
+          kubectl -n my-verticadb-operator exec verticadb-sample-defaultsubcluster-0 -c server -- \
+            /opt/vertica/bin/vsql -U dbadmin -c "SELECT get_load_balance_policy();"
+      - name: Run Python tests in-cluster
+        run: |
+          set -euo pipefail
+          NS=my-verticadb-operator
+          SVC=verticadb-sample-defaultsubcluster
+          LOCATOR="${SVC}.${NS}.svc.cluster.local:5433"
+          POD=py-test-runner
+          IMAGE=python:${{ matrix.python-version }}-slim
+          echo "Ensuring namespace ${NS} exists..."
+          kubectl get ns ${NS} >/dev/null 2>&1 || kubectl create ns ${NS}
+          echo "Waiting for Vertica service endpoints..."
+          WAIT_TIMEOUT=300
+          INTERVAL=5
+          deadline=$((SECONDS + WAIT_TIMEOUT))
+          while [ $SECONDS -lt $deadline ]; do
+            addrs=$(kubectl -n ${NS} get endpoints ${SVC} -o jsonpath='{.subsets[*].addresses[*].ip}' 2>/dev/null || true)
+            [ -n "$addrs" ] && break || sleep ${INTERVAL}
+          done
+          if [ -z "$addrs" ]; then
+            echo "Vertica service endpoints not found"
+            kubectl -n ${NS} get pods -o wide || true
+            kubectl -n ${NS} get endpoints ${SVC} -o yaml || true
+            exit 1
+          fi
+          echo "Creating Python test pod..."
+          kubectl -n ${NS} delete pod ${POD} --ignore-not-found || true
+          kubectl -n ${NS} run ${POD} --image=${IMAGE} --restart=Never --command -- sleep infinity
+          kubectl -n ${NS} wait --for=condition=Ready pod/${POD} --timeout=180s
+          echo "Copying repository into pod..."
+          kubectl -n ${NS} exec -i pod/${POD} -- mkdir -p /workspace
+          tar cf - . | kubectl -n ${NS} exec -i pod/${POD} -- tar xf - -C /workspace
+          echo "Installing dependencies..."
+          kubectl -n ${NS} exec pod/${POD} -- bash -lc 'apt-get update -qq && apt-get install -y -qq build-essential libssl-dev libpq-dev netcat-traditional curl || true'
+          kubectl -n ${NS} exec pod/${POD} -- bash -lc 'python -m pip install --upgrade pip >/dev/null 2>&1 || true; pip install tox pytest >/dev/null 2>&1 || true'
+          echo "Fetching token from Keycloak..."
+          CT_POD="curl-token-$$"
+          kubectl -n keycloak delete pod ${CT_POD} --ignore-not-found || true
+          kubectl -n keycloak run ${CT_POD} --restart=Never --image=curlimages/curl:latest --command -- sleep 120
+          kubectl -n keycloak wait --for=condition=Ready pod/${CT_POD} --timeout=120s || true
+          kubectl -n keycloak exec pod/${CT_POD} -- sh -c "
+            curl -s -X POST 'http://keycloak.keycloak.svc.cluster.local:8080/realms/${REALM}/protocol/openid-connect/token' \
+              -d 'client_id=${CLIENT_ID}' \
+              -d 'username=${USER}' \
+              -d 'password=${PASSWORD}' \
+              -d 'grant_type=password' \
+              -d 'client_secret=${CLIENT_SECRET}' > /tmp/token.json
+          "
+          kubectl -n keycloak cp ${CT_POD}:/tmp/token.json token.json || {
+            echo "Failed to copy token.json from curl pod"
+            kubectl -n keycloak logs ${CT_POD} || true
+            exit 1
+          }
+          kubectl -n keycloak delete pod ${CT_POD} --ignore-not-found || true
+          TOKEN=$(python3 -c 'import json; print(__import__("json").load(open("token.json")).get("access_token",""))')
+          if [ -z "$TOKEN" ]; then
+            echo "No access_token found in token.json"
+            cat token.json
+            exit 1
+          fi
+          echo "Access token retrieved (length: ${#TOKEN})"
+          printf '%s' "$TOKEN" | kubectl -n ${NS} exec -i pod/${POD} -- tee /workspace/access_token.txt >/dev/null
+          echo "🏃 Running Python tests inside pod..."
+          kubectl -n ${NS} exec -i pod/${POD} -- bash -lc "
+            set -euo pipefail
+            cd /workspace
+            export VP_TEST_OAUTH_ACCESS_TOKEN='${TOKEN}'
+            export VP_TEST_HOST='verticadb-sample-defaultsubcluster.my-verticadb-operator.svc.cluster.local'
+            export VP_TEST_PORT=5433
+            export VP_TEST_DATABASE='vdb'
+            export VP_TEST_OAUTH_USER='oauth_user'
+            export VP_TEST_USER='dbadmin'
+            export VP_TEST_PASSWORD=''
+            echo '🔍 Checking connectivity to Vertica...'
+            if command -v nc >/dev/null 2>&1; then
+              nc -zv \${VP_TEST_HOST} \${VP_TEST_PORT} || { echo '❌ Cannot reach Vertica host'; exit 1; }
+            else
+              timeout 5 bash -c 'cat < /dev/null > /dev/tcp/'\"\${VP_TEST_HOST}\"'/'\"\${VP_TEST_PORT}\"'' || { echo '❌ Cannot reach Vertica host'; exit 1; }
+            fi
+            echo 'Vertica reachable; performing token introspection...'
+            INTROSPECT_OUTPUT=\$(curl -s -X POST http://keycloak.keycloak.svc.cluster.local:8080/realms/test/protocol/openid-connect/token/introspect \
+              -d 'client_id=vertica' \
+              -d 'client_secret=P9f8350QQIUhFfK1GF5sMhq4Dm3P6Sbs' \
+              -d 'token='\${VP_TEST_OAUTH_ACCESS_TOKEN})
+            if echo \"\$INTROSPECT_OUTPUT\" | grep -q '\"active\":true'; then
+              echo 'Token introspection successful (active=true)'
+            else
+              echo 'Token introspection failed:'
+              echo \"\$INTROSPECT_OUTPUT\"; exit 1
+            fi
+            echo 'Running pytest suite via tox...'
+            tox -e py
+          "
+          echo "🧹 Cleaning up test pod..."
+          kubectl -n ${NS} delete pod ${POD} --ignore-not-found || true
+      # ---------------------------
+      # Final Teardown (K8s + KinD)
+      # ---------------------------
+      - name: Cleanup Kubernetes resources
+        if: always()
+        run: |
+          echo "🧹 Starting cleanup..."
+          echo "Deleting Python test runner pods..."
+          kubectl -n my-verticadb-operator delete pod test-runner --ignore-not-found || true
+          kubectl -n my-verticadb-operator delete pod py-test-runner --ignore-not-found || true
+          echo "Deleting Keycloak pods & services..."
+          kubectl delete deployment keycloak -n keycloak --ignore-not-found || true
+          kubectl delete service keycloak -n keycloak --ignore-not-found || true
+          kubectl delete ns keycloak --ignore-not-found || true
+          echo "Deleting VerticaDB and Operator..."
+          kubectl delete verticadb verticadb-sample -n my-verticadb-operator --ignore-not-found || true
+          helm uninstall vdb-op -n my-verticadb-operator || true
+          kubectl delete ns my-verticadb-operator --ignore-not-found || true
+          echo "Deleting MinIO..."
+          kubectl delete -f minio.yaml --ignore-not-found || true
+          kubectl delete ns minio --ignore-not-found || true
+          echo "Deleting leftover services..."
+          kubectl delete svc vertica-node-0 -n my-verticadb-operator --ignore-not-found || true
+          kubectl delete svc vertica-node-1 -n my-verticadb-operator --ignore-not-found || true
+          kubectl delete svc vertica-node-2 -n my-verticadb-operator --ignore-not-found || true
+          echo "Kubernetes resources cleanup done."
+      - name: Delete KinD cluster
+        if: always()
         run: |
-          export VP_TEST_USER=dbadmin
-          export VP_TEST_OAUTH_ACCESS_TOKEN=`cat access_token.txt`
-          export VP_TEST_OAUTH_USER=${USER}
-          tox -e py
+          echo "Deleting KinD cluster..."
+          kind delete cluster --name vertica-ci || true
+          echo "KinD cluster removed successfully"
\ No newline at end of file

From 32e6bf53dde60243e172abbc0d183812e50eb5a2 Mon Sep 17 00:00:00 2001
From: mkottakota1 <149763406+mkottakota1@users.noreply.github.com>
Date: Thu, 20 Nov 2025 10:31:56 +0000
Subject: [PATCH 02/16] Removing unwanted blanks

---
 .github/workflows/ci.yaml | 29 ++++++++++++++++++++++++++---
 1 file changed, 26 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index cb9fac86..0d497098 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -8,7 +8,7 @@ jobs:
 
     strategy:
       matrix:
-        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13', 'pypy3.10']
+        python-version: ['3.12']
 
     env:
       REALM: test
@@ -48,6 +48,7 @@ jobs:
           helm repo add vertica-charts https://vertica.github.io/charts || true
           helm repo add bitnami https://charts.bitnami.com/bitnami || true
           helm repo update
+
       # ---------------------------
       # MinIO Setup
       # ---------------------------
@@ -104,6 +105,7 @@ jobs:
           kubectl -n minio rollout status deployment/minio --timeout=2m || true
           kubectl get pods -n minio -o wide || true
           kubectl get svc -n minio || true
+
       - name: Ensure MinIO bucket exists
         run: |
           kubectl run mc-client --rm -i --restart=Never \
@@ -114,6 +116,7 @@ jobs:
               mc mb --ignore-existing localminio/vertica-fleeting && \
               mc ls localminio
             "
+
       - name: Create MinIO Secret
         run: |
           kubectl create ns my-verticadb-operator || true
@@ -123,6 +126,7 @@ jobs:
             --from-literal=accesskey="minioadmin" \
             --from-literal=secretkey="minioadmin"
           kubectl get secret communal-creds -n my-verticadb-operator -o yaml || true
+
       # ---------------------------
       # Vertica Operator + DB Deployment
       # ---------------------------
@@ -205,6 +209,7 @@ jobs:
               - port: 5433
                 targetPort: 5433
           EOF
+
       - name: Wait for Vertica readiness
         run: |
           NS=my-verticadb-operator
@@ -214,17 +219,22 @@ jobs:
             kubectl get pod ${POD} -n ${NS} && break || sleep 10
           done
           kubectl wait --for=condition=Ready pod/${POD} -n ${NS} --timeout=5m || true
+
           echo "🚀 Creating test runner pod..."
           kubectl -n ${NS} run test-runner --image=python:3.13-slim --restart=Never --command -- sleep infinity
           kubectl -n ${NS} wait --for=condition=Ready pod/test-runner --timeout=180s
+
           echo "🩹 Adding Vertica pod entries to /etc/hosts in test pod..."
           for p in $(kubectl -n $NS get pods -l app.kubernetes.io/instance=verticadb-sample -o jsonpath='{.items[*].metadata.name}'); do
             IP=$(kubectl -n $NS get pod $p -o jsonpath='{.status.podIP}')
             echo "$IP $p.$NS.svc.cluster.local $p" | kubectl -n $NS exec -i test-runner -- tee -a /etc/hosts >/dev/null
             echo "✔ Added $p -> $IP"
           done
+
           echo "📂 Copying repository into pod..."
           kubectl -n ${NS} cp . test-runner:/workspace
+
+
       # ---------------------------
       # Keycloak + OAuth setup
       # ---------------------------
@@ -278,10 +288,12 @@ jobs:
               - port: 8080
                 targetPort: 8080
           EOF
+
       - name: Wait for Keycloak readiness
         run: |
           kubectl -n keycloak rollout status deploy/keycloak --timeout=2m
           kubectl -n keycloak get pods -o wide
+
       - name: Configure Keycloak realm, client, and user
         run: |
           kubectl -n keycloak exec deploy/keycloak -- \
@@ -302,6 +314,7 @@ jobs:
           kubectl -n keycloak exec deploy/keycloak -- \
             /opt/keycloak/bin/kcadm.sh set-password -r test \
               --username oauth_user --new-password password
+
       - name: Configure Vertica Authentication
         run: |
           NS=my-verticadb-operator
@@ -329,6 +342,7 @@ jobs:
         run: |
           kubectl -n my-verticadb-operator exec verticadb-sample-defaultsubcluster-0 -c server -- \
             /opt/vertica/bin/vsql -U dbadmin -c "SELECT get_load_balance_policy();"
+
       - name: Run Python tests in-cluster
         run: |
           set -euo pipefail
@@ -421,34 +435,43 @@ jobs:
             echo 'Running pytest suite via tox...'
             tox -e py
           "
-          echo "🧹 Cleaning up test pod..."
+          echo "Cleaning up test pod..."
           kubectl -n ${NS} delete pod ${POD} --ignore-not-found || true
+
       # ---------------------------
       # Final Teardown (K8s + KinD)
       # ---------------------------
       - name: Cleanup Kubernetes resources
         if: always()
         run: |
-          echo "🧹 Starting cleanup..."
+          echo "Starting cleanup..."
+
           echo "Deleting Python test runner pods..."
           kubectl -n my-verticadb-operator delete pod test-runner --ignore-not-found || true
           kubectl -n my-verticadb-operator delete pod py-test-runner --ignore-not-found || true
+
           echo "Deleting Keycloak pods & services..."
           kubectl delete deployment keycloak -n keycloak --ignore-not-found || true
           kubectl delete service keycloak -n keycloak --ignore-not-found || true
           kubectl delete ns keycloak --ignore-not-found || true
+
           echo "Deleting VerticaDB and Operator..."
           kubectl delete verticadb verticadb-sample -n my-verticadb-operator --ignore-not-found || true
           helm uninstall vdb-op -n my-verticadb-operator || true
           kubectl delete ns my-verticadb-operator --ignore-not-found || true
+
           echo "Deleting MinIO..."
           kubectl delete -f minio.yaml --ignore-not-found || true
           kubectl delete ns minio --ignore-not-found || true
+
           echo "Deleting leftover services..."
           kubectl delete svc vertica-node-0 -n my-verticadb-operator --ignore-not-found || true
           kubectl delete svc vertica-node-1 -n my-verticadb-operator --ignore-not-found || true
           kubectl delete svc vertica-node-2 -n my-verticadb-operator --ignore-not-found || true
+
           echo "Kubernetes resources cleanup done."
+
+
       - name: Delete KinD cluster
         if: always()
         run: |

From 5b7c0aba02866a38e3a0439eafdbdc5cfb339332 Mon Sep 17 00:00:00 2001
From: mkottakota1 <149763406+mkottakota1@users.noreply.github.com>
Date: Thu, 20 Nov 2025 11:15:51 +0000
Subject: [PATCH 03/16] Running in MultiVersions

---
 .github/workflows/ci.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 0d497098..15067180 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -8,7 +8,7 @@ jobs:
 
     strategy:
       matrix:
-        python-version: ['3.12']
+        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13', 'pypy3.10']
 
     env:
       REALM: test

From 0c24318833a8604333be2f3bfc5839a00956c338 Mon Sep 17 00:00:00 2001
From: mkottakota1 <149763406+mkottakota1@users.noreply.github.com>
Date: Thu, 20 Nov 2025 11:27:23 +0000
Subject: [PATCH 04/16] Resolving failure for PYPY 3.10

---
 .github/workflows/ci.yaml | 35 ++++++++++-------------------------
 1 file changed, 10 insertions(+), 25 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 15067180..5f9abb28 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -48,7 +48,6 @@ jobs:
           helm repo add vertica-charts https://vertica.github.io/charts || true
           helm repo add bitnami https://charts.bitnami.com/bitnami || true
           helm repo update
-
       # ---------------------------
       # MinIO Setup
       # ---------------------------
@@ -105,7 +104,6 @@ jobs:
           kubectl -n minio rollout status deployment/minio --timeout=2m || true
           kubectl get pods -n minio -o wide || true
           kubectl get svc -n minio || true
-
       - name: Ensure MinIO bucket exists
         run: |
           kubectl run mc-client --rm -i --restart=Never \
@@ -116,7 +114,6 @@ jobs:
               mc mb --ignore-existing localminio/vertica-fleeting && \
               mc ls localminio
             "
-
       - name: Create MinIO Secret
         run: |
           kubectl create ns my-verticadb-operator || true
@@ -126,7 +123,6 @@ jobs:
             --from-literal=accesskey="minioadmin" \
             --from-literal=secretkey="minioadmin"
           kubectl get secret communal-creds -n my-verticadb-operator -o yaml || true
-
       # ---------------------------
       # Vertica Operator + DB Deployment
       # ---------------------------
@@ -209,7 +205,6 @@ jobs:
               - port: 5433
                 targetPort: 5433
           EOF
-
       - name: Wait for Vertica readiness
         run: |
           NS=my-verticadb-operator
@@ -219,22 +214,25 @@ jobs:
             kubectl get pod ${POD} -n ${NS} && break || sleep 10
           done
           kubectl wait --for=condition=Ready pod/${POD} -n ${NS} --timeout=5m || true
-
           echo "🚀 Creating test runner pod..."
-          kubectl -n ${NS} run test-runner --image=python:3.13-slim --restart=Never --command -- sleep infinity
-          kubectl -n ${NS} wait --for=condition=Ready pod/test-runner --timeout=180s
+          # Decide startup command based on image type
+          if [[ "${IMAGE}" == *"pypy"* ]]; then
+            START_CMD="python -c 'import time; time.sleep(10**6)'"
+          else
+            START_CMD="sleep infinity"
+          fi
+
+          kubectl -n ${NS} run ${POD} --image=${IMAGE} --restart=Never --command -- ${START_CMD}
 
+          kubectl -n ${NS} wait --for=condition=Ready pod/test-runner --timeout=180s
           echo "🩹 Adding Vertica pod entries to /etc/hosts in test pod..."
           for p in $(kubectl -n $NS get pods -l app.kubernetes.io/instance=verticadb-sample -o jsonpath='{.items[*].metadata.name}'); do
             IP=$(kubectl -n $NS get pod $p -o jsonpath='{.status.podIP}')
             echo "$IP $p.$NS.svc.cluster.local $p" | kubectl -n $NS exec -i test-runner -- tee -a /etc/hosts >/dev/null
             echo "✔ Added $p -> $IP"
           done
-
           echo "📂 Copying repository into pod..."
           kubectl -n ${NS} cp . test-runner:/workspace
-
-
       # ---------------------------
       # Keycloak + OAuth setup
       # ---------------------------
@@ -288,12 +286,10 @@ jobs:
               - port: 8080
                 targetPort: 8080
           EOF
-
       - name: Wait for Keycloak readiness
         run: |
           kubectl -n keycloak rollout status deploy/keycloak --timeout=2m
           kubectl -n keycloak get pods -o wide
-
       - name: Configure Keycloak realm, client, and user
         run: |
           kubectl -n keycloak exec deploy/keycloak -- \
@@ -314,7 +310,6 @@ jobs:
           kubectl -n keycloak exec deploy/keycloak -- \
             /opt/keycloak/bin/kcadm.sh set-password -r test \
               --username oauth_user --new-password password
-
       - name: Configure Vertica Authentication
         run: |
           NS=my-verticadb-operator
@@ -342,7 +337,6 @@ jobs:
         run: |
           kubectl -n my-verticadb-operator exec verticadb-sample-defaultsubcluster-0 -c server -- \
             /opt/vertica/bin/vsql -U dbadmin -c "SELECT get_load_balance_policy();"
-
       - name: Run Python tests in-cluster
         run: |
           set -euo pipefail
@@ -350,7 +344,7 @@ jobs:
           SVC=verticadb-sample-defaultsubcluster
           LOCATOR="${SVC}.${NS}.svc.cluster.local:5433"
           POD=py-test-runner
-          IMAGE=python:${{ matrix.python-version }}-slim
+          IMAGE=python:${{ matrix.python-version }}
           echo "Ensuring namespace ${NS} exists..."
           kubectl get ns ${NS} >/dev/null 2>&1 || kubectl create ns ${NS}
           echo "Waiting for Vertica service endpoints..."
@@ -437,7 +431,6 @@ jobs:
           "
           echo "Cleaning up test pod..."
           kubectl -n ${NS} delete pod ${POD} --ignore-not-found || true
-
       # ---------------------------
       # Final Teardown (K8s + KinD)
       # ---------------------------
@@ -445,33 +438,25 @@ jobs:
         if: always()
         run: |
           echo "Starting cleanup..."
-
           echo "Deleting Python test runner pods..."
           kubectl -n my-verticadb-operator delete pod test-runner --ignore-not-found || true
           kubectl -n my-verticadb-operator delete pod py-test-runner --ignore-not-found || true
-
           echo "Deleting Keycloak pods & services..."
           kubectl delete deployment keycloak -n keycloak --ignore-not-found || true
           kubectl delete service keycloak -n keycloak --ignore-not-found || true
           kubectl delete ns keycloak --ignore-not-found || true
-
           echo "Deleting VerticaDB and Operator..."
           kubectl delete verticadb verticadb-sample -n my-verticadb-operator --ignore-not-found || true
           helm uninstall vdb-op -n my-verticadb-operator || true
           kubectl delete ns my-verticadb-operator --ignore-not-found || true
-
           echo "Deleting MinIO..."
           kubectl delete -f minio.yaml --ignore-not-found || true
           kubectl delete ns minio --ignore-not-found || true
-
           echo "Deleting leftover services..."
           kubectl delete svc vertica-node-0 -n my-verticadb-operator --ignore-not-found || true
           kubectl delete svc vertica-node-1 -n my-verticadb-operator --ignore-not-found || true
           kubectl delete svc vertica-node-2 -n my-verticadb-operator --ignore-not-found || true
-
           echo "Kubernetes resources cleanup done."
-
-
       - name: Delete KinD cluster
         if: always()
         run: |

From 89ab9629c7827c04339029a828b29204c8e82e81 Mon Sep 17 00:00:00 2001
From: mkottakota1 <149763406+mkottakota1@users.noreply.github.com>
Date: Thu, 20 Nov 2025 16:31:27 +0000
Subject: [PATCH 05/16] Running without PYPY

---
 .github/workflows/ci.yaml | 37 ++++++++++++++++++++++++++-----------
 1 file changed, 26 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 5f9abb28..2b7047b6 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -8,7 +8,7 @@ jobs:
 
     strategy:
       matrix:
-        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13', 'pypy3.10']
+        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13']
 
     env:
       REALM: test
@@ -48,6 +48,7 @@ jobs:
           helm repo add vertica-charts https://vertica.github.io/charts || true
           helm repo add bitnami https://charts.bitnami.com/bitnami || true
           helm repo update
+
       # ---------------------------
       # MinIO Setup
       # ---------------------------
@@ -104,6 +105,7 @@ jobs:
           kubectl -n minio rollout status deployment/minio --timeout=2m || true
           kubectl get pods -n minio -o wide || true
           kubectl get svc -n minio || true
+
       - name: Ensure MinIO bucket exists
         run: |
           kubectl run mc-client --rm -i --restart=Never \
@@ -114,6 +116,7 @@ jobs:
               mc mb --ignore-existing localminio/vertica-fleeting && \
               mc ls localminio
             "
+
       - name: Create MinIO Secret
         run: |
           kubectl create ns my-verticadb-operator || true
@@ -123,6 +126,7 @@ jobs:
             --from-literal=accesskey="minioadmin" \
             --from-literal=secretkey="minioadmin"
           kubectl get secret communal-creds -n my-verticadb-operator -o yaml || true
+
       # ---------------------------
       # Vertica Operator + DB Deployment
       # ---------------------------
@@ -205,6 +209,7 @@ jobs:
               - port: 5433
                 targetPort: 5433
           EOF
+
       - name: Wait for Vertica readiness
         run: |
           NS=my-verticadb-operator
@@ -214,25 +219,22 @@ jobs:
             kubectl get pod ${POD} -n ${NS} && break || sleep 10
           done
           kubectl wait --for=condition=Ready pod/${POD} -n ${NS} --timeout=5m || true
-          echo "🚀 Creating test runner pod..."
-          # Decide startup command based on image type
-          if [[ "${IMAGE}" == *"pypy"* ]]; then
-            START_CMD="python -c 'import time; time.sleep(10**6)'"
-          else
-            START_CMD="sleep infinity"
-          fi
-
-          kubectl -n ${NS} run ${POD} --image=${IMAGE} --restart=Never --command -- ${START_CMD}
 
+          echo "🚀 Creating test runner pod..."
+          kubectl -n ${NS} run test-runner --image=python:3.13-slim --restart=Never --command -- sleep infinity
           kubectl -n ${NS} wait --for=condition=Ready pod/test-runner --timeout=180s
+
           echo "🩹 Adding Vertica pod entries to /etc/hosts in test pod..."
           for p in $(kubectl -n $NS get pods -l app.kubernetes.io/instance=verticadb-sample -o jsonpath='{.items[*].metadata.name}'); do
             IP=$(kubectl -n $NS get pod $p -o jsonpath='{.status.podIP}')
             echo "$IP $p.$NS.svc.cluster.local $p" | kubectl -n $NS exec -i test-runner -- tee -a /etc/hosts >/dev/null
             echo "✔ Added $p -> $IP"
           done
+
           echo "📂 Copying repository into pod..."
           kubectl -n ${NS} cp . test-runner:/workspace
+
+
       # ---------------------------
       # Keycloak + OAuth setup
       # ---------------------------
@@ -286,10 +288,12 @@ jobs:
               - port: 8080
                 targetPort: 8080
           EOF
+
       - name: Wait for Keycloak readiness
         run: |
           kubectl -n keycloak rollout status deploy/keycloak --timeout=2m
           kubectl -n keycloak get pods -o wide
+
       - name: Configure Keycloak realm, client, and user
         run: |
           kubectl -n keycloak exec deploy/keycloak -- \
@@ -310,6 +314,7 @@ jobs:
           kubectl -n keycloak exec deploy/keycloak -- \
             /opt/keycloak/bin/kcadm.sh set-password -r test \
               --username oauth_user --new-password password
+
       - name: Configure Vertica Authentication
         run: |
           NS=my-verticadb-operator
@@ -337,6 +342,7 @@ jobs:
         run: |
           kubectl -n my-verticadb-operator exec verticadb-sample-defaultsubcluster-0 -c server -- \
             /opt/vertica/bin/vsql -U dbadmin -c "SELECT get_load_balance_policy();"
+
       - name: Run Python tests in-cluster
         run: |
           set -euo pipefail
@@ -344,7 +350,7 @@ jobs:
           SVC=verticadb-sample-defaultsubcluster
           LOCATOR="${SVC}.${NS}.svc.cluster.local:5433"
           POD=py-test-runner
-          IMAGE=python:${{ matrix.python-version }}
+          IMAGE=python:${{ matrix.python-version }}-slim
           echo "Ensuring namespace ${NS} exists..."
           kubectl get ns ${NS} >/dev/null 2>&1 || kubectl create ns ${NS}
           echo "Waiting for Vertica service endpoints..."
@@ -431,6 +437,7 @@ jobs:
           "
           echo "Cleaning up test pod..."
           kubectl -n ${NS} delete pod ${POD} --ignore-not-found || true
+
       # ---------------------------
       # Final Teardown (K8s + KinD)
       # ---------------------------
@@ -438,25 +445,33 @@ jobs:
         if: always()
         run: |
           echo "Starting cleanup..."
+
           echo "Deleting Python test runner pods..."
           kubectl -n my-verticadb-operator delete pod test-runner --ignore-not-found || true
           kubectl -n my-verticadb-operator delete pod py-test-runner --ignore-not-found || true
+
           echo "Deleting Keycloak pods & services..."
           kubectl delete deployment keycloak -n keycloak --ignore-not-found || true
           kubectl delete service keycloak -n keycloak --ignore-not-found || true
           kubectl delete ns keycloak --ignore-not-found || true
+
           echo "Deleting VerticaDB and Operator..."
           kubectl delete verticadb verticadb-sample -n my-verticadb-operator --ignore-not-found || true
           helm uninstall vdb-op -n my-verticadb-operator || true
           kubectl delete ns my-verticadb-operator --ignore-not-found || true
+
           echo "Deleting MinIO..."
           kubectl delete -f minio.yaml --ignore-not-found || true
           kubectl delete ns minio --ignore-not-found || true
+
           echo "Deleting leftover services..."
           kubectl delete svc vertica-node-0 -n my-verticadb-operator --ignore-not-found || true
           kubectl delete svc vertica-node-1 -n my-verticadb-operator --ignore-not-found || true
           kubectl delete svc vertica-node-2 -n my-verticadb-operator --ignore-not-found || true
+
           echo "Kubernetes resources cleanup done."
+
+
       - name: Delete KinD cluster
         if: always()
         run: |

From 810ed57f6141323f7a76aeb4879c01314b65ec19 Mon Sep 17 00:00:00 2001
From: mkottakota1 <149763406+mkottakota1@users.noreply.github.com>
Date: Fri, 21 Nov 2025 07:02:49 +0000
Subject: [PATCH 06/16] Add PYPY in Python Matrix

---
 .github/workflows/ci.yaml | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 2b7047b6..ab8eb3ae 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -8,7 +8,7 @@ jobs:
 
     strategy:
       matrix:
-        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13']
+        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13','pypy3.10']
 
     env:
       REALM: test
@@ -369,7 +369,12 @@ jobs:
           fi
           echo "Creating Python test pod..."
           kubectl -n ${NS} delete pod ${POD} --ignore-not-found || true
-          kubectl -n ${NS} run ${POD} --image=${IMAGE} --restart=Never --command -- sleep infinity
+          if [[ "${IMAGE}" == *"pypy"* ]]; then
+            START_CMD="python -c 'import time; time.sleep(10**6)'"
+          else
+          START_CMD="sleep infinity"
+          fi
+          kubectl -n ${NS} run ${POD} --image=${IMAGE} --restart=Never --command -- ${START_CMD}
           kubectl -n ${NS} wait --for=condition=Ready pod/${POD} --timeout=180s
           echo "Copying repository into pod..."
           kubectl -n ${NS} exec -i pod/${POD} -- mkdir -p /workspace

From bd8b24f48ca748fd380fe70ca3cb15102c2ec035 Mon Sep 17 00:00:00 2001
From: mkottakota1 <149763406+mkottakota1@users.noreply.github.com>
Date: Fri, 21 Nov 2025 07:39:24 +0000
Subject: [PATCH 07/16] Adding debug logs for pyPY

---
 .github/workflows/ci.yaml | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index ab8eb3ae..32316443 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -370,12 +370,17 @@ jobs:
           echo "Creating Python test pod..."
           kubectl -n ${NS} delete pod ${POD} --ignore-not-found || true
           if [[ "${IMAGE}" == *"pypy"* ]]; then
-            START_CMD="python -c 'import time; time.sleep(10**6)'"
+            kubectl -n ${NS} run ${POD} --image=${IMAGE} --restart=Never --command -- python -c "import time; time.sleep(10**6)"
           else
-          START_CMD="sleep infinity"
+           kubectl -n ${NS} run ${POD} --image=${IMAGE} --restart=Never --command -- sleep infinity
+          fi
+          # Try waiting for Ready, but capture failure
+          if ! kubectl -n ${NS} wait --for=condition=Ready pod/${POD} --timeout=180s; then
+            echo "Pod did not become Ready. Collecting debug info..."
+            kubectl -n ${NS} describe pod ${POD} || true
+            kubectl -n ${NS} logs ${POD} || true
+            exit 1
           fi
-          kubectl -n ${NS} run ${POD} --image=${IMAGE} --restart=Never --command -- ${START_CMD}
-          kubectl -n ${NS} wait --for=condition=Ready pod/${POD} --timeout=180s
           echo "Copying repository into pod..."
           kubectl -n ${NS} exec -i pod/${POD} -- mkdir -p /workspace
           tar cf - . | kubectl -n ${NS} exec -i pod/${POD} -- tar xf - -C /workspace

From af486047530aead912a4f044b6ad90dc779e2dbf Mon Sep 17 00:00:00 2001
From: mkottakota1 <149763406+mkottakota1@users.noreply.github.com>
Date: Fri, 21 Nov 2025 08:46:25 +0000
Subject: [PATCH 08/16] Pull correct PYPY 3.10 image

---
 .github/workflows/ci.yaml | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 32316443..5491cf89 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -350,7 +350,13 @@ jobs:
           SVC=verticadb-sample-defaultsubcluster
           LOCATOR="${SVC}.${NS}.svc.cluster.local:5433"
           POD=py-test-runner
-          IMAGE=python:${{ matrix.python-version }}-slim
+          # Decide image based on matrix value
+          if [[ "${{ matrix.python-version }}" == pypy* ]]; then
+            VERSION="${{ matrix.python-version }}"   # e.g. "pypy3.10"
+            IMAGE="pypy:${VERSION#pypy}"             # strip "pypy" prefix in bash → "pypy:3.10"
+          else
+            IMAGE="python:${{ matrix.python-version }}"
+          fi
           echo "Ensuring namespace ${NS} exists..."
           kubectl get ns ${NS} >/dev/null 2>&1 || kubectl create ns ${NS}
           echo "Waiting for Vertica service endpoints..."

From 9f6cc7c3b6a179ebbabee42971e0dc590bd3bf1e Mon Sep 17 00:00:00 2001
From: mkottakota1 <149763406+mkottakota1@users.noreply.github.com>
Date: Fri, 21 Nov 2025 09:21:06 +0000
Subject: [PATCH 09/16] Installing PYPY dependencies

---
 .github/workflows/ci.yaml | 20 ++++++++++++++++++--
 1 file changed, 18 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 5491cf89..cc09eef5 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -391,8 +391,22 @@ jobs:
           kubectl -n ${NS} exec -i pod/${POD} -- mkdir -p /workspace
           tar cf - . | kubectl -n ${NS} exec -i pod/${POD} -- tar xf - -C /workspace
           echo "Installing dependencies..."
-          kubectl -n ${NS} exec pod/${POD} -- bash -lc 'apt-get update -qq && apt-get install -y -qq build-essential libssl-dev libpq-dev netcat-traditional curl || true'
-          kubectl -n ${NS} exec pod/${POD} -- bash -lc 'python -m pip install --upgrade pip >/dev/null 2>&1 || true; pip install tox pytest >/dev/null 2>&1 || true'
+          kubectl -n ${NS} exec pod/${POD} -- bash -lc '
+            set -e
+            apt-get update -qq
+            apt-get install -y -qq build-essential libssl-dev libpq-dev netcat-traditional curl
+            if command -v python >/dev/null 2>&1; then
+              python -m ensurepip || true
+              python -m pip install --upgrade pip
+              python -m pip install tox pytest
+            elif command -v pypy3 >/dev/null 2>&1; then
+              pypy3 -m ensurepip || true
+              pypy3 -m pip install --upgrade pip
+              pypy3 -m pip install tox pytest
+            fi
+            export PATH=$PATH:/root/.local/bin
+            which tox && tox --version
+          '
           echo "Fetching token from Keycloak..."
           CT_POD="curl-token-$$"
           kubectl -n keycloak delete pod ${CT_POD} --ignore-not-found || true
@@ -431,6 +445,8 @@ jobs:
             export VP_TEST_OAUTH_USER='oauth_user'
             export VP_TEST_USER='dbadmin'
             export VP_TEST_PASSWORD=''
+            # Add PyPy bin directory to PATH
+            export PATH=\$PATH:/opt/pypy/bin
             echo '🔍 Checking connectivity to Vertica...'
             if command -v nc >/dev/null 2>&1; then
               nc -zv \${VP_TEST_HOST} \${VP_TEST_PORT} || { echo '❌ Cannot reach Vertica host'; exit 1; }

From cf861cb658739976e0740a250eb0aa8fcaeae920 Mon Sep 17 00:00:00 2001
From: mkottakota1 <149763406+mkottakota1@users.noreply.github.com>
Date: Fri, 21 Nov 2025 09:41:47 +0000
Subject: [PATCH 10/16] Updating bin path for PYPY run

---
 .github/workflows/ci.yaml | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index cc09eef5..35c25f0a 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -404,7 +404,12 @@ jobs:
               pypy3 -m pip install --upgrade pip
               pypy3 -m pip install tox pytest
             fi
-            export PATH=$PATH:/root/.local/bin
+            # Conditionally adjust PATH for PyPy
+            if [[ "${{ matrix.python-version }}" == pypy* ]]; then
+              export PATH=$PATH:/opt/pypy/bin
+            else
+              export PATH=$PATH:/root/.local/bin
+            fi
             which tox && tox --version
           '
           echo "Fetching token from Keycloak..."
@@ -445,8 +450,6 @@ jobs:
             export VP_TEST_OAUTH_USER='oauth_user'
             export VP_TEST_USER='dbadmin'
             export VP_TEST_PASSWORD=''
-            # Add PyPy bin directory to PATH
-            export PATH=\$PATH:/opt/pypy/bin
             echo '🔍 Checking connectivity to Vertica...'
             if command -v nc >/dev/null 2>&1; then
               nc -zv \${VP_TEST_HOST} \${VP_TEST_PORT} || { echo '❌ Cannot reach Vertica host'; exit 1; }

From 12300d95cab2e68ffa41765755e05991d91549ed Mon Sep 17 00:00:00 2001
From: mkottakota1 <149763406+mkottakota1@users.noreply.github.com>
Date: Mon, 24 Nov 2025 04:22:47 +0000
Subject: [PATCH 11/16] Exporting PYPY path at the tox

---
 .github/workflows/ci.yaml | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 35c25f0a..2fbd2136 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -378,7 +378,7 @@ jobs:
           if [[ "${IMAGE}" == *"pypy"* ]]; then
             kubectl -n ${NS} run ${POD} --image=${IMAGE} --restart=Never --command -- python -c "import time; time.sleep(10**6)"
           else
-           kubectl -n ${NS} run ${POD} --image=${IMAGE} --restart=Never --command -- sleep infinity
+            kubectl -n ${NS} run ${POD} --image=${IMAGE} --restart=Never --command -- sleep infinity
           fi
           # Try waiting for Ready, but capture failure
           if ! kubectl -n ${NS} wait --for=condition=Ready pod/${POD} --timeout=180s; then
@@ -467,6 +467,11 @@ jobs:
               echo 'Token introspection failed:'
               echo \"\$INTROSPECT_OUTPUT\"; exit 1
             fi
+
+            if command -v pypy3 >/dev/null 2>&1; then
+              export PATH=\$PATH:/opt/pypy/bin
+            fi
+
             echo 'Running pytest suite via tox...'
             tox -e py
           "

From ac3b9677db5058933f92b18af4e4927a39bdc7ed Mon Sep 17 00:00:00 2001
From: mkottakota1 <149763406+mkottakota1@users.noreply.github.com>
Date: Mon, 24 Nov 2025 11:24:14 +0000
Subject: [PATCH 12/16] Added Client details in Github secrets, Handling token
 via  Kubernetes Secret, Updating minio image to latest, correction in python
 matrix spaces

---
 .github/workflows/ci.yaml | 155 ++++++++++++++++++++------------------
 1 file changed, 80 insertions(+), 75 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 2fbd2136..45761f7e 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -8,14 +8,14 @@ jobs:
 
     strategy:
       matrix:
-        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13','pypy3.10']
+        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13', 'pypy3.10']
 
     env:
-      REALM: test
-      USER: oauth_user
-      PASSWORD: password
-      CLIENT_ID: vertica
-      CLIENT_SECRET: P9f8350QQIUhFfK1GF5sMhq4Dm3P6Sbs
+      REALM: ${{ secrets.REALM }}
+      USER: ${{ secrets.USER }}
+      PASSWORD: ${{ secrets.PASSWORD }}
+      CLIENT_ID: ${{ secrets.CLIENT_ID }}
+      CLIENT_SECRET: ${{ secrets.CLIENT_SECRET }}
 
     steps:
       # ---------------------------
@@ -45,7 +45,7 @@ jobs:
 
       - name: Add Helm repositories
         run: |
-          helm repo add vertica-charts https://vertica.github.io/charts || true
+          helm repo add vertica-charts https://vertica.github.io/charts
           helm repo add bitnami https://charts.bitnami.com/bitnami || true
           helm repo update
 
@@ -54,7 +54,7 @@ jobs:
       # ---------------------------
       - name: Install MinIO
         run: |
-          kubectl create ns minio || true
+          kubectl create ns minio
           cat <<'EOF' > minio.yaml
           apiVersion: apps/v1
           kind: Deployment
@@ -73,7 +73,7 @@ jobs:
               spec:
                 containers:
                   - name: minio
-                    image: minio/minio:RELEASE.2025-09-07T16-13-09Z-cpuv1
+                    image: minio/minio:latest
                     args: ["server", "/data"]
                     env:
                       - name: MINIO_ROOT_USER
@@ -102,7 +102,7 @@ jobs:
                 targetPort: 9000
           EOF
           kubectl apply -f minio.yaml
-          kubectl -n minio rollout status deployment/minio --timeout=2m || true
+          kubectl -n minio rollout status deployment/minio --timeout=2m
           kubectl get pods -n minio -o wide || true
           kubectl get svc -n minio || true
 
@@ -119,7 +119,7 @@ jobs:
 
       - name: Create MinIO Secret
         run: |
-          kubectl create ns my-verticadb-operator || true
+          kubectl create ns my-verticadb-operator
           kubectl delete secret communal-creds -n my-verticadb-operator --ignore-not-found
           kubectl create secret generic communal-creds \
             -n my-verticadb-operator \
@@ -218,7 +218,7 @@ jobs:
           for i in {1..30}; do
             kubectl get pod ${POD} -n ${NS} && break || sleep 10
           done
-          kubectl wait --for=condition=Ready pod/${POD} -n ${NS} --timeout=5m || true
+          kubectl wait --for=condition=Ready pod/${POD} -n ${NS} --timeout=5m
 
           echo "🚀 Creating test runner pod..."
           kubectl -n ${NS} run test-runner --image=python:3.13-slim --restart=Never --command -- sleep infinity
@@ -240,7 +240,7 @@ jobs:
       # ---------------------------
       - name: Deploy Keycloak
         run: |
-          kubectl create ns keycloak || true
+          kubectl create ns keycloak
           cat <<'EOF' | kubectl apply -f -
           apiVersion: apps/v1
           kind: Deployment
@@ -334,15 +334,10 @@ jobs:
               GRANT AUTHENTICATION v_dbadmin_hash TO dbadmin;
             \"
           "
-          
+
       # ---------------------------
       # Testing section
       # ---------------------------
-      - name: Check Vertica load balance policy
-        run: |
-          kubectl -n my-verticadb-operator exec verticadb-sample-defaultsubcluster-0 -c server -- \
-            /opt/vertica/bin/vsql -U dbadmin -c "SELECT get_load_balance_policy();"
-
       - name: Run Python tests in-cluster
         run: |
           set -euo pipefail
@@ -373,25 +368,75 @@ jobs:
             kubectl -n ${NS} get endpoints ${SVC} -o yaml || true
             exit 1
           fi
-          echo "Creating Python test pod..."
-          kubectl -n ${NS} delete pod ${POD} --ignore-not-found || true
-          if [[ "${IMAGE}" == *"pypy"* ]]; then
-            kubectl -n ${NS} run ${POD} --image=${IMAGE} --restart=Never --command -- python -c "import time; time.sleep(10**6)"
-          else
-            kubectl -n ${NS} run ${POD} --image=${IMAGE} --restart=Never --command -- sleep infinity
+
+          TOKEN=$(kubectl -n keycloak run curl-client --rm -i --restart=Never \
+            --image=curlimages/curl:latest \
+            --command -- \
+            curl -s -X POST "http://keycloak.keycloak.svc.cluster.local:8080/realms/${REALM}/protocol/openid-connect/token" \
+            -d "client_id=${CLIENT_ID}" \
+            -d "username=${USER}" \
+            -d "password=${PASSWORD}" \
+            -d "grant_type=password" \
+            -d "client_secret=${CLIENT_SECRET}" | jq -r .access_token)
+
+          if [ -z "$TOKEN" ]; then
+            echo "Failed to fetch access token"
+            exit 1
           fi
-          # Try waiting for Ready, but capture failure
+
+          echo "Creating Kubernetes Secret with token..."
+          kubectl -n ${NS} delete secret oauth-token --ignore-not-found
+          kubectl -n ${NS} create secret generic oauth-token \
+            --from-literal=access_token="$TOKEN"
+
+          echo "Creating Python test pod with secret mount..."
+          kubectl -n ${NS} delete pod ${POD} --ignore-not-found || true
+          cat <<EOF | kubectl apply -f -
+          apiVersion: v1
+          kind: Pod
+          metadata:
+            name: ${POD}
+            namespace: ${NS}
+          spec:
+            restartPolicy: Never
+            containers:
+              - name: tester
+                image: ${IMAGE}
+                command: ["sleep", "infinity"]
+                env:
+                  - name: VP_TEST_OAUTH_ACCESS_TOKEN
+                    valueFrom:
+                      secretKeyRef:
+                        name: oauth-token
+                        key: access_token
+                  - name: VP_TEST_HOST
+                    value: verticadb-sample-defaultsubcluster.my-verticadb-operator.svc.cluster.local
+                  - name: VP_TEST_PORT
+                    value: "5433"
+                  - name: VP_TEST_DATABASE
+                    value: vdb
+                  - name: VP_TEST_OAUTH_USER
+                    value: oauth_user
+                  - name: VP_TEST_USER
+                    value: dbadmin
+                  - name: VP_TEST_PASSWORD
+                    value: ""
+          EOF
+
+           echo "Waiting for test pod readiness..."
           if ! kubectl -n ${NS} wait --for=condition=Ready pod/${POD} --timeout=180s; then
             echo "Pod did not become Ready. Collecting debug info..."
             kubectl -n ${NS} describe pod ${POD} || true
             kubectl -n ${NS} logs ${POD} || true
             exit 1
           fi
+
           echo "Copying repository into pod..."
-          kubectl -n ${NS} exec -i pod/${POD} -- mkdir -p /workspace
-          tar cf - . | kubectl -n ${NS} exec -i pod/${POD} -- tar xf - -C /workspace
+          kubectl -n ${NS} exec -i ${POD} -- mkdir -p /workspace
+          tar cf - . | kubectl -n ${NS} exec -i ${POD} -- tar xf - -C /workspace
+
           echo "Installing dependencies..."
-          kubectl -n ${NS} exec pod/${POD} -- bash -lc '
+          kubectl -n ${NS} exec ${POD} -- bash -lc '
             set -e
             apt-get update -qq
             apt-get install -y -qq build-essential libssl-dev libpq-dev netcat-traditional curl
@@ -404,70 +449,30 @@ jobs:
               pypy3 -m pip install --upgrade pip
               pypy3 -m pip install tox pytest
             fi
-            # Conditionally adjust PATH for PyPy
-            if [[ "${{ matrix.python-version }}" == pypy* ]]; then
+            if command -v pypy3 >/dev/null 2>&1; then
               export PATH=$PATH:/opt/pypy/bin
             else
               export PATH=$PATH:/root/.local/bin
             fi
             which tox && tox --version
           '
-          echo "Fetching token from Keycloak..."
-          CT_POD="curl-token-$$"
-          kubectl -n keycloak delete pod ${CT_POD} --ignore-not-found || true
-          kubectl -n keycloak run ${CT_POD} --restart=Never --image=curlimages/curl:latest --command -- sleep 120
-          kubectl -n keycloak wait --for=condition=Ready pod/${CT_POD} --timeout=120s || true
-          kubectl -n keycloak exec pod/${CT_POD} -- sh -c "
-            curl -s -X POST 'http://keycloak.keycloak.svc.cluster.local:8080/realms/${REALM}/protocol/openid-connect/token' \
-              -d 'client_id=${CLIENT_ID}' \
-              -d 'username=${USER}' \
-              -d 'password=${PASSWORD}' \
-              -d 'grant_type=password' \
-              -d 'client_secret=${CLIENT_SECRET}' > /tmp/token.json
-          "
-          kubectl -n keycloak cp ${CT_POD}:/tmp/token.json token.json || {
-            echo "Failed to copy token.json from curl pod"
-            kubectl -n keycloak logs ${CT_POD} || true
-            exit 1
-          }
-          kubectl -n keycloak delete pod ${CT_POD} --ignore-not-found || true
-          TOKEN=$(python3 -c 'import json; print(__import__("json").load(open("token.json")).get("access_token",""))')
-          if [ -z "$TOKEN" ]; then
-            echo "No access_token found in token.json"
-            cat token.json
-            exit 1
-          fi
-          echo "Access token retrieved (length: ${#TOKEN})"
-          printf '%s' "$TOKEN" | kubectl -n ${NS} exec -i pod/${POD} -- tee /workspace/access_token.txt >/dev/null
+
           echo "🏃 Running Python tests inside pod..."
-          kubectl -n ${NS} exec -i pod/${POD} -- bash -lc "
+          kubectl -n ${NS} exec -i ${POD} -- bash -lc "
             set -euo pipefail
             cd /workspace
-            export VP_TEST_OAUTH_ACCESS_TOKEN='${TOKEN}'
-            export VP_TEST_HOST='verticadb-sample-defaultsubcluster.my-verticadb-operator.svc.cluster.local'
-            export VP_TEST_PORT=5433
-            export VP_TEST_DATABASE='vdb'
-            export VP_TEST_OAUTH_USER='oauth_user'
-            export VP_TEST_USER='dbadmin'
-            export VP_TEST_PASSWORD=''
             echo '🔍 Checking connectivity to Vertica...'
-            if command -v nc >/dev/null 2>&1; then
-              nc -zv \${VP_TEST_HOST} \${VP_TEST_PORT} || { echo '❌ Cannot reach Vertica host'; exit 1; }
-            else
-              timeout 5 bash -c 'cat < /dev/null > /dev/tcp/'\"\${VP_TEST_HOST}\"'/'\"\${VP_TEST_PORT}\"'' || { echo '❌ Cannot reach Vertica host'; exit 1; }
-            fi
+            nc -zv \${VP_TEST_HOST} \${VP_TEST_PORT} || { echo '❌ Cannot reach Vertica host'; exit 1; }
             echo 'Vertica reachable; performing token introspection...'
             INTROSPECT_OUTPUT=\$(curl -s -X POST http://keycloak.keycloak.svc.cluster.local:8080/realms/test/protocol/openid-connect/token/introspect \
               -d 'client_id=vertica' \
-              -d 'client_secret=P9f8350QQIUhFfK1GF5sMhq4Dm3P6Sbs' \
+              -d 'client_secret=${CLIENT_SECRET}' \
               -d 'token='\${VP_TEST_OAUTH_ACCESS_TOKEN})
             if echo \"\$INTROSPECT_OUTPUT\" | grep -q '\"active\":true'; then
               echo 'Token introspection successful (active=true)'
             else
-              echo 'Token introspection failed:'
-              echo \"\$INTROSPECT_OUTPUT\"; exit 1
+              echo 'Token introspection failed:'; echo \"\$INTROSPECT_OUTPUT\"; exit 1
             fi
-
             if command -v pypy3 >/dev/null 2>&1; then
               export PATH=\$PATH:/opt/pypy/bin
             fi

From e7149155d69a18dafbcdc06ce5d6aae0630ad316 Mon Sep 17 00:00:00 2001
From: mkottakota1 <149763406+mkottakota1@users.noreply.github.com>
Date: Mon, 24 Nov 2025 11:49:23 +0000
Subject: [PATCH 13/16] Fix token parsing issue

---
 .github/workflows/ci.yaml | 27 +++++++++++++++++++++------
 1 file changed, 21 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 45761f7e..5eba0cf4 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -368,22 +368,37 @@ jobs:
             kubectl -n ${NS} get endpoints ${SVC} -o yaml || true
             exit 1
           fi
-
-          TOKEN=$(kubectl -n keycloak run curl-client --rm -i --restart=Never \
+          RAW=$(
+          kubectl -n keycloak run curl-client \
             --image=curlimages/curl:latest \
-            --command -- \
-            curl -s -X POST "http://keycloak.keycloak.svc.cluster.local:8080/realms/${REALM}/protocol/openid-connect/token" \
+            --restart=Never \
+            --rm -i --quiet \
+            -- curl -s -X POST "http://keycloak.keycloak.svc.cluster.local:8080/realms/${REALM}/protocol/openid-connect/token" \
             -d "client_id=${CLIENT_ID}" \
             -d "username=${USER}" \
             -d "password=${PASSWORD}" \
             -d "grant_type=password" \
-            -d "client_secret=${CLIENT_SECRET}" | jq -r .access_token)
+            -d "client_secret=${CLIENT_SECRET}"
+          )
+
+          # Validate JSON before parsing
+          if ! echo "$RAW" | jq . >/dev/null 2>&1; then
+            echo "Token endpoint did not return valid JSON:"
+            echo "$RAW"
+            exit 1
+          fi
+
+          # Extract the access token
+          TOKEN=$(echo "$RAW" | jq -r .access_token)
 
-          if [ -z "$TOKEN" ]; then
+          if [ -z "$TOKEN" ] || [ "$TOKEN" = "null" ]; then
             echo "Failed to fetch access token"
+            echo "$RAW"
             exit 1
           fi
 
+          echo "Access token retrieved (length: ${#TOKEN})"
+
           echo "Creating Kubernetes Secret with token..."
           kubectl -n ${NS} delete secret oauth-token --ignore-not-found
           kubectl -n ${NS} create secret generic oauth-token \

From 7a8277758431843b88ee69d0c5ac25488dc2ac1d Mon Sep 17 00:00:00 2001
From: mkottakota1 <149763406+mkottakota1@users.noreply.github.com>
Date: Mon, 24 Nov 2025 13:56:22 +0000
Subject: [PATCH 14/16] Updated Pythin version matrix

---
 .github/workflows/ci.yaml | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 5eba0cf4..e45bfd8f 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -8,7 +8,14 @@ jobs:
 
     strategy:
       matrix:
-        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13', 'pypy3.10']
+        python-version:
+          - '3.8'
+          - '3.9'
+          - '3.10'
+          - '3.11'
+          - '3.12'
+          - '3.13'
+          - 'pypy3.10'
 
     env:
       REALM: ${{ secrets.REALM }}

From fd322253492d127094fc08be44a265555c433a5b Mon Sep 17 00:00:00 2001
From: mkottakota1 <149763406+mkottakota1@users.noreply.github.com>
Date: Mon, 24 Nov 2025 14:48:15 +0000
Subject: [PATCH 15/16] Add retry block for token geneeration

---
 .github/workflows/ci.yaml | 86 ++++++++++++++++++++++++---------------
 1 file changed, 53 insertions(+), 33 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index e45bfd8f..502721ea 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -352,57 +352,72 @@ jobs:
           SVC=verticadb-sample-defaultsubcluster
           LOCATOR="${SVC}.${NS}.svc.cluster.local:5433"
           POD=py-test-runner
+
           # Decide image based on matrix value
           if [[ "${{ matrix.python-version }}" == pypy* ]]; then
-            VERSION="${{ matrix.python-version }}"   # e.g. "pypy3.10"
-            IMAGE="pypy:${VERSION#pypy}"             # strip "pypy" prefix in bash → "pypy:3.10"
+            VERSION="${{ matrix.python-version }}"   # "pypy3.10"
+            IMAGE="pypy:${VERSION#pypy}"             # "pypy:3.10"
           else
             IMAGE="python:${{ matrix.python-version }}"
           fi
+
           echo "Ensuring namespace ${NS} exists..."
           kubectl get ns ${NS} >/dev/null 2>&1 || kubectl create ns ${NS}
+
           echo "Waiting for Vertica service endpoints..."
           WAIT_TIMEOUT=300
           INTERVAL=5
           deadline=$((SECONDS + WAIT_TIMEOUT))
           while [ $SECONDS -lt $deadline ]; do
-            addrs=$(kubectl -n ${NS} get endpoints ${SVC} -o jsonpath='{.subsets[*].addresses[*].ip}' 2>/dev/null || true)
+            addrs=$(kubectl -n ${NS} get endpoints ${SVC} \
+              -o jsonpath='{.subsets[*].addresses[*].ip}' 2>/dev/null || true)
             [ -n "$addrs" ] && break || sleep ${INTERVAL}
           done
+
           if [ -z "$addrs" ]; then
             echo "Vertica service endpoints not found"
             kubectl -n ${NS} get pods -o wide || true
             kubectl -n ${NS} get endpoints ${SVC} -o yaml || true
             exit 1
           fi
-          RAW=$(
-          kubectl -n keycloak run curl-client \
-            --image=curlimages/curl:latest \
-            --restart=Never \
-            --rm -i --quiet \
-            -- curl -s -X POST "http://keycloak.keycloak.svc.cluster.local:8080/realms/${REALM}/protocol/openid-connect/token" \
-            -d "client_id=${CLIENT_ID}" \
-            -d "username=${USER}" \
-            -d "password=${PASSWORD}" \
-            -d "grant_type=password" \
-            -d "client_secret=${CLIENT_SECRET}"
-          )
-
-          # Validate JSON before parsing
-          if ! echo "$RAW" | jq . >/dev/null 2>&1; then
-            echo "Token endpoint did not return valid JSON:"
-            echo "$RAW"
-            exit 1
-          fi
 
-          # Extract the access token
-          TOKEN=$(echo "$RAW" | jq -r .access_token)
+          echo "Retrieving access token from Keycloak with retries..."
+          TOKEN=""
+          for i in {1..10}; do
+            echo "Attempt $i..."
+
+            RAW=$(
+              kubectl -n keycloak run curl-client \
+                --image=curlimages/curl:latest \
+                --restart=Never \
+                --rm -i --quiet \
+                --command -- curl -s -X POST \
+                  "http://keycloak.keycloak.svc.cluster.local:8080/realms/${REALM}/protocol/openid-connect/token" \
+                  -d "client_id=${CLIENT_ID}" \
+                  -d "username=${USER}" \
+                  -d "password=${PASSWORD}" \
+                  -d "grant_type=password" \
+                  -d "client_secret=${CLIENT_SECRET}"
+            ) || true
+
+            # JSON validity check
+            if echo "$RAW" | jq -e . >/dev/null 2>&1; then
+              TOKEN=$(echo "$RAW" | jq -r .access_token)
+            fi
 
-          if [ -z "$TOKEN" ] || [ "$TOKEN" = "null" ]; then
-            echo "Failed to fetch access token"
-            echo "$RAW"
-            exit 1
-          fi
+            if [ -n "${TOKEN}" ] && [ "${TOKEN}" != "null" ]; then
+              echo "Access token retrieved successfully."
+              break
+            fi
+
+            echo "Token fetch failed: $RAW"
+            sleep 5
+
+            if [ "$i" -eq 10 ]; then
+              echo "Failed to fetch access token after multiple retries."
+              exit 1
+            fi
+          done
 
           echo "Access token retrieved (length: ${#TOKEN})"
 
@@ -445,7 +460,7 @@ jobs:
                     value: ""
           EOF
 
-           echo "Waiting for test pod readiness..."
+          echo "Waiting for test pod readiness..."
           if ! kubectl -n ${NS} wait --for=condition=Ready pod/${POD} --timeout=180s; then
             echo "Pod did not become Ready. Collecting debug info..."
             kubectl -n ${NS} describe pod ${POD} || true
@@ -479,22 +494,26 @@ jobs:
             which tox && tox --version
           '
 
-          echo "🏃 Running Python tests inside pod..."
+          echo "Running Python tests inside pod..."
           kubectl -n ${NS} exec -i ${POD} -- bash -lc "
             set -euo pipefail
             cd /workspace
-            echo '🔍 Checking connectivity to Vertica...'
-            nc -zv \${VP_TEST_HOST} \${VP_TEST_PORT} || { echo '❌ Cannot reach Vertica host'; exit 1; }
+
+            echo 'Checking connectivity to Vertica...'
+            nc -zv \${VP_TEST_HOST} \${VP_TEST_PORT} || { echo 'Cannot reach Vertica host'; exit 1; }
+
             echo 'Vertica reachable; performing token introspection...'
             INTROSPECT_OUTPUT=\$(curl -s -X POST http://keycloak.keycloak.svc.cluster.local:8080/realms/test/protocol/openid-connect/token/introspect \
               -d 'client_id=vertica' \
               -d 'client_secret=${CLIENT_SECRET}' \
               -d 'token='\${VP_TEST_OAUTH_ACCESS_TOKEN})
+
             if echo \"\$INTROSPECT_OUTPUT\" | grep -q '\"active\":true'; then
               echo 'Token introspection successful (active=true)'
             else
               echo 'Token introspection failed:'; echo \"\$INTROSPECT_OUTPUT\"; exit 1
             fi
+
             if command -v pypy3 >/dev/null 2>&1; then
               export PATH=\$PATH:/opt/pypy/bin
             fi
@@ -502,6 +521,7 @@ jobs:
             echo 'Running pytest suite via tox...'
             tox -e py
           "
+
           echo "Cleaning up test pod..."
           kubectl -n ${NS} delete pod ${POD} --ignore-not-found || true
 

From fad8ef26f31dcf2d76dd368cf991cb818930d977 Mon Sep 17 00:00:00 2001
From: mkottakota1 <149763406+mkottakota1@users.noreply.github.com>
Date: Tue, 25 Nov 2025 09:55:45 +0000
Subject: [PATCH 16/16] Removed jq usage and started using Github secrets

---
 .github/workflows/ci.yaml | 32 ++++++++++++++++++++++----------
 1 file changed, 22 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 502721ea..464cc13f 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -151,6 +151,7 @@ jobs:
           helm upgrade --install vdb-op vertica-charts/verticadb-operator \
             -n my-verticadb-operator -f operator-values.yaml --wait --timeout 10m
           kubectl -n my-verticadb-operator get pods -o wide || true
+
       - name: Deploy VerticaDB and per-node Services
         run: |
           cat <<'EOF' | kubectl apply -f -
@@ -179,6 +180,9 @@ jobs:
           ---
           # -------------------------------------
           # Per-node Services (needed for LB tests)
+          # Note: node-1 and node-2 pods don't exist yet because subcluster size=1.
+          # These Services are intentional placeholders used when we scale the cluster
+          # for load-balancing and failover tests. Kubernetes allows zero-endpoint Services.
           # -------------------------------------
           apiVersion: v1
           kind: Service
@@ -311,8 +315,8 @@ jobs:
             /opt/keycloak/bin/kcadm.sh create realms -s realm=test -s enabled=true
           kubectl -n keycloak exec deploy/keycloak -- \
             /opt/keycloak/bin/kcadm.sh create clients -r test \
-              -s clientId=vertica -s enabled=true \
-              -s secret=P9f8350QQIUhFfK1GF5sMhq4Dm3P6Sbs \
+              -s clientId="${CLIENT_ID}" -s enabled=true \
+              -s secret="${CLIENT_SECRET}" \
               -s 'redirectUris=["*"]' \
               -s directAccessGrantsEnabled=true
           kubectl -n keycloak exec deploy/keycloak -- \
@@ -329,8 +333,8 @@ jobs:
           kubectl -n ${NS} exec ${POD} -c server -- bash -c "
             /opt/vertica/bin/vsql -U dbadmin -c \"
               CREATE AUTHENTICATION v_oauth METHOD 'oauth' HOST '0.0.0.0/0';
-              ALTER AUTHENTICATION v_oauth SET client_id = 'vertica';
-              ALTER AUTHENTICATION v_oauth SET client_secret = 'P9f8350QQIUhFfK1GF5sMhq4Dm3P6Sbs';
+              ALTER AUTHENTICATION v_oauth SET client_id = '${CLIENT_ID}';
+              ALTER AUTHENTICATION v_oauth SET client_secret = '${CLIENT_SECRET}';
               ALTER AUTHENTICATION v_oauth SET discovery_url = 'http://keycloak.keycloak.svc.cluster.local:8080/realms/test/.well-known/openid-configuration';
               ALTER AUTHENTICATION v_oauth SET introspect_url = 'http://keycloak.keycloak.svc.cluster.local:8080/realms/test/protocol/openid-connect/token/introspect';
               CREATE USER oauth_user;
@@ -400,17 +404,25 @@ jobs:
                   -d "client_secret=${CLIENT_SECRET}"
             ) || true
 
-            # JSON validity check
-            if echo "$RAW" | jq -e . >/dev/null 2>&1; then
-              TOKEN=$(echo "$RAW" | jq -r .access_token)
+            # Keep only the last line (JSON), remove kubectl noise
+            RAW=$(printf "%s" "$RAW" | sed -n '$p')
+
+            # Validate RAW is JSON
+            if ! printf '%s' "$RAW" | python3 -c 'import sys,json; json.load(sys.stdin)' >/dev/null 2>&1; then
+              echo "Token endpoint did not return valid JSON:"
+              printf '%s\n' "$RAW"
+              exit 1
             fi
 
-            if [ -n "${TOKEN}" ] && [ "${TOKEN}" != "null" ]; then
+            # Extract token (without printing it)
+            TOKEN=$(printf '%s' "$RAW" | python3 -c 'import sys,json; print(json.load(sys.stdin).get("access_token", ""))')
+
+            if [ -n "$TOKEN" ] && [ "$TOKEN" != "null" ]; then
               echo "Access token retrieved successfully."
               break
             fi
 
-            echo "Token fetch failed: $RAW"
+            echo "Token fetch failed, Keycloak may not be ready yet."
             sleep 5
 
             if [ "$i" -eq 10 ]; then
@@ -504,7 +516,7 @@ jobs:
 
             echo 'Vertica reachable; performing token introspection...'
             INTROSPECT_OUTPUT=\$(curl -s -X POST http://keycloak.keycloak.svc.cluster.local:8080/realms/test/protocol/openid-connect/token/introspect \
-              -d 'client_id=vertica' \
+              -d 'client_id=${CLIENT_ID}' \
               -d 'client_secret=${CLIENT_SECRET}' \
               -d 'token='\${VP_TEST_OAUTH_ACCESS_TOKEN})