diff --git a/.github/workflows/lint_and_test.yaml b/.github/workflows/lint_and_test.yaml new file mode 100644 index 0000000..f44ef5f --- /dev/null +++ b/.github/workflows/lint_and_test.yaml @@ -0,0 +1,47 @@ +name: Lint and Test Charts + +on: [pull_request] + +jobs: + lint-test: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Set up kubectl + uses: azure/setup-kubectl@v3 + + - name: Set up Helm + uses: azure/setup-helm@v3 + with: + version: v3.10.0 + + - name: Install bitnami repo + run: helm repo add bitnami https://charts.bitnami.com/bitnami + + - uses: actions/setup-python@v4 + with: + python-version: '3.9' + check-latest: true + + - name: Set up chart-testing + uses: helm/chart-testing-action@v2.3.1 + + - name: Run chart-testing (lint) + run: ct lint --target-branch ${{ github.event.repository.default_branch }} + + - name: start minikube + uses: medyagh/setup-minikube@latest + id: minikube + with: + cache: true + minikube-version: 1.22.0 + + - name: Copy and edit values.yaml + run: python -m pip install ruamel.yaml kubernetes && cd byzer-lang && python ./cp_write_values_yaml.py + + - name: Run byzer-lang-testing (install) + run: ct install --debug --chart-dirs ./byzer-lang --charts byzer-lang --helm-extra-args '--timeout 480s' diff --git a/.gitignore b/.gitignore index cff2d78..9c8ded9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ .idea/** -/byzer-lang/values.yaml /byzer-notebook/values.yaml /package.sh -/charts \ No newline at end of file +/charts +byzer-lang/values.yaml \ No newline at end of file diff --git a/byzer-lang/byzer-lang-headless-service.sh b/byzer-lang/byzer-lang-headless-service.sh deleted file mode 100755 index 241e41a..0000000 --- a/byzer-lang/byzer-lang-headless-service.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash - -driver_pod=$(kubectl get pod| grep "byzer-lang-deployment" | cut -d" " -f1) -[[ -z ${driver_pod} ]] && echo "Unable to get Byzer-lang driver pod" && exit 1 - -cat < str: + config.load_kube_config() + v1 = client.CoreV1Api() + api_server = v1.api_client.configuration.host + parse_result = urlparse(api_server) + if parse_result.hostname == "127.0.0.1": + host = socket.gethostbyname(socket.gethostname()) + api_server = parse_result.scheme + "://" + host + ":" + str(parse_result.port) + print(api_server) + return api_server + + +def rewrite_conf(conf: dict) -> dict: + + conf['clusterUrl'] = get_cluster_url() + + spark_conf = conf['spark'] + spark_conf['driver.memory'] = "1024m" + spark_conf['driver.cores'] = "1" + spark_conf['executor.memory'] = "512m" + spark_conf['kubernetes.driver.limit.cores'] = "400m" + spark_conf['kubernetes.driver.request.cores'] = "400m" + spark_conf['kubernetes.executor.limit.cores'] = "200m" + spark_conf['kubernetes.executor.request.cores'] = "200m" + + byzer_conf = conf['byzer'] + byzer_conf['mainJar'] = "byzer-lang-3.3.0-2.12-2.3.5.jar" + + image_conf = conf['image'] + image_conf["repository"] = "byzer/byzer-lang-k8s-full" + image_conf["tag"] = "3.3.0-2.3.5-2023-03-09" + + conf['fs']['cloud']['storage']['enabled'] = False + conf['readinessProbe']['initialDelaySeconds'] = 120 + conf['minReadySeconds'] = 120 + conf['livenessProbe']['initialDelaySeconds'] = 120 + + return conf + + +yaml = ruamel.yaml.YAML() +yaml.preserve_quotes = True +with open('values.example.yaml') as fp: + data = yaml.load(fp) +rewrite_conf(data) + +with open('values.yaml', 'w') as outfile: + yaml.dump(data, outfile) diff --git a/byzer-lang/index.yaml b/byzer-lang/index.yaml deleted file mode 100644 index db0ffc2..0000000 --- a/byzer-lang/index.yaml +++ /dev/null @@ -1,31 +0,0 @@ -apiVersion: v1 -entries: - common: - - annotations: - category: Infrastructure - apiVersion: v2 - appVersion: 2.2.2 - created: "2023-03-06T23:19:49.315352+08:00" - description: A Library Helm Chart for grouping common logic between bitnami charts. - This chart is not deployable by itself. - digest: d589f61a6529a986b5294bb76843431b8b0ab5816d3876d0b0fd631b8cc55ebe - home: https://github.com/bitnami/charts/tree/main/bitnami/common - icon: https://bitnami.com/downloads/logos/bitnami-mark.png - keywords: - - common - - helper - - template - - function - - bitnami - maintainers: - - name: Bitnami - url: https://github.com/bitnami/charts - name: common - sources: - - https://github.com/bitnami/charts - - https://www.bitnami.com/ - type: library - urls: - - charts/common-2.2.2.tgz - version: 2.2.2 -generated: "2023-03-06T23:19:49.311477+08:00" diff --git a/byzer-lang/templates/service.yaml b/byzer-lang/templates/service.yaml index 54f11c8..b8c38cd 100644 --- a/byzer-lang/templates/service.yaml +++ b/byzer-lang/templates/service.yaml @@ -11,5 +11,22 @@ spec: targetPort: http protocol: TCP name: http + selector: + {{- include "byzer-lang.selectorLabels" . | nindent 4 }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "byzer-lang.fullname" . }}-headless-service + labels: + {{- include "byzer-lang.labels" . | nindent 4 }} +spec: + type: ClusterIP + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + clusterIP: None selector: {{- include "byzer-lang.selectorLabels" . | nindent 4 }} \ No newline at end of file diff --git a/byzer-lang/templates/tests/test-connection.yaml b/byzer-lang/templates/tests/test-connection.yaml index 0953d2a..10a91b4 100644 --- a/byzer-lang/templates/tests/test-connection.yaml +++ b/byzer-lang/templates/tests/test-connection.yaml @@ -1,7 +1,7 @@ apiVersion: v1 kind: Pod metadata: - name: {{ include "byzer-lang.fullname" . }}-test-connection" + name: {{ include "byzer-lang.fullname" . }}-test-connection labels: {{- include "byzer-lang.labels" . | nindent 4 }} annotations: @@ -11,5 +11,5 @@ spec: - name: wget image: busybox command: ['wget'] - args: ['{{ include "byzer-lang.fullname" . }}:{{ .Values.service.port }}'] + args: ['{{ include "byzer-lang.fullname" . }}-service:{{ .Values.service.port }}'] restartPolicy: Never