WIP: Add support for running with a remote Armada server

.github/workflows/build.yaml

@@ -13,16 +13,16 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - scala_version: "2.12.15"
-            spark_version: "3.3.4"
-            java_version: "11"
+          # - scala_version: "2.12.15"
+          # spark_version: "3.3.4"
+          # java_version: "11"
           - scala_version: "2.12.18"
             spark_version: "3.5.5"
             java_version: "17"
 
-          - scala_version: "2.13.8"
-            spark_version: "3.3.4"
-            java_version: "11"
+          # - scala_version: "2.13.8"
+          # spark_version: "3.3.4"
+          # java_version: "11"
           - scala_version: "2.13.8"
             spark_version: "3.5.5"
             java_version: "17"
@@ -35,4 +35,4 @@ jobs:
         with:
           spark_version: ${{ matrix.spark_version }}
           scala_version: ${{ matrix.scala_version }}
-          java_version: ${{ matrix.java_version }}
+          java_version: ${{ matrix.java_version }}

.github/workflows/e2e.yaml

@@ -13,16 +13,16 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - scala_version: "2.12.15"
-            spark_version: "3.3.4"
-            java_version: "11"
+          # - scala_version: "2.12.15"
+          # spark_version: "3.3.4"
+          # java_version: "11"
           - scala_version: "2.12.18"
             spark_version: "3.5.5"
             java_version: "17"
 
-          - scala_version: "2.13.8"
-            spark_version: "3.3.4"
-            java_version: "11"
+          # - scala_version: "2.13.8"
+          # spark_version: "3.3.4"
+          # java_version: "11"
           - scala_version: "2.13.8"
             spark_version: "3.5.5"
             java_version: "17"

README.md

@@ -79,6 +79,90 @@ We recommend [kind](https://kind.sigs.k8s.io/) for local testing. If you are usi
 kind load docker-image $IMAGE_NAME --name armada
 ```
 
+### Running a remote Armada server using Armada Operator
+The default Armada Operator setup allows only localhost access.  You can quickly set up a local Armada server
+configured to allow external access from other hosts, useful for client development and testing. For this
+configuration:
+
+- Copy the file `e2e/kind-config-external-access.yaml` in this repository to `hack/kind-config.yaml`
+in your `armada-operator` repository.
+
+- Edit the newly-copied `hack/kind-config.yaml` as noted in the beginning comments of that file.
+
+- Run the armada-operator setup commands (usually `make kind-all`) to create and start your Armada instance.
+
+- Copy the `$HOME/.kube/config` and `$HOME/.armadctl.yaml` (that Armada Operator will generate) from the Armada
+server host to your `$HOME` directory on the client (local) host. Then edit the local `.kube/config` and on
+the line that has `server: https://0.0.0.0:6443`, change the `0.0.0.0` address to the IP address or hostname
+of the remote Armada server system.
+
+- Generate a copy of the client TLS key, cert, and CA-cert files: (1) go into the `e2e` subdirectory, and
+run `./extract-kind-cert.sh` - it will generate `client.crt`, `client.key`, and `ca.crt`, from the output
+of `kubectl config view`. These files can be left in this directory.
+
+- Copy the `$HOME/.armadactl.yaml` from the Armada server host to your home directory on your client system.
+
+- You should then be able to run `kubectl get pods -A` and see a list of the running pods on the remote
+Armada server, as well as running `armadactl get queues`.
+
+- Verify the functionality of your setup by editing `scripts/config.sh` and changing the following line:
+```
+ARMADA_MASTER=armada://192.168.12.135:30002
+```
+to the IP address or hostname of your Armada server. You should not need to change the port number.
+
+Also, set the location of the three TLS certificate files by adding/setting:
+```
+CLIENT_CERT_FILE=e2e/client.crt
+CLIENT_KEY_FILE=e2e/client.key
+CLUSTER_CA_FILE=e2e/ca.crt
+```
+
+-  You should be able to now verify the armada-spark configuration by running the E2E tests:
+```
+$ ./scripts/dev-e2e.sh
+```
+This will save its output to `e2e-test.log` for further debugging.
+
+---
+
+## Development
+
+Before submitting a pull request, please ensure that your code adheres to the project's coding standards and passes all tests.
+
+### Testing
+
+To run the unit tests, use the following command:
+
+```bash
+mvn test
+```
+
+To run the E2E tests, run Armada using the [Operator Quickstart](https://github.com/armadaproject/armada-operator?tab=readme-ov-file#quickstart) guide, then execute:
+
+```bash
+scripts/test-e2e.sh
+```
+### Linting
+
+To check the code for linting issues, use the following command:
+
+```bash
+mvn spotless:check
+```
+
+To automatically apply linting fixes, use:
+
+```bash
+mvn spotless:apply
+```
+
+### E2E
+
+Make sure that the [SparkPi](#sparkpi-example) job successfully runs on your Armada cluster before submitting a pull request.
+
+---
+
 ## Running Example Workloads
 
 ### SparkPi

e2e/extract-kind-cert.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+CONTEXT="kind-armada"
+
+E2E_DIR=$(realpath "$0" | xargs dirname)
+
+cd "$E2E_DIR" || (echo "Error: could not cd to $E2E_DIR"; exit 1)
+
+# What These Files Are
+#  -  client.crt: Your user (client) certificate
+#  -  client.key: The private key associated with the certificate
+#  -  ca.crt: The CA certificate used by the Kubernetes API server (for verifying client and server certs)
+
+# Extract the client certificate
+kubectl config view --raw -o json | jq -r \
+  ".users[] | select(.name == \"${CONTEXT}\") | .user.[\"client-certificate-data\"]" | base64 -d > client.crt
+
+# Extract the client key
+kubectl config view --raw -o json | jq -r \
+  ".users[] | select(.name == \"${CONTEXT}\") | .user.[\"client-key-data\"]" | base64 -d > client.key
+
+# Extract the cluster CA certificate
+kubectl config view --raw -o json | jq -r \
+  ".clusters[] | select(.name == \"${CONTEXT}\") | .cluster.[\"certificate-authority-data\"]" | base64 -d > ca.crt
+

e2e/kind-config-external-access.yaml

@@ -0,0 +1,52 @@
+# A kind configuration for running an Armada server that can be accessed
+# outside the host system, for working/developing with remote clients,
+# such as Armada-Spark clients.
+#
+# This configuration will allow you to run kubectl and armadactl
+# against the Armada instance on this system. To use this:
+# - Copy your $HOME/.kube/config on this system to the same directory
+#   on your remote client host, then modify that copied file so the
+#   IP address in there (0.0.0.0) is the address of the external interface
+#   mentioned below.
+# - Copy your $HOME/.armadactl.yaml to your $HOME directory on the remote
+#   client host, in that copied file, change the value of the 'armadaUrl'
+#   field from 'localhost' to the hostname (or IP address) of this server,
+#   and below that line a new line (at same indent level), add the entry
+#     forceNoTls: true
+# You should then be able to run `kubectl cluster-info` or
+# `armadactl get queues` without errors on the remote client host.
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+- role: control-plane
+  kubeadmConfigPatches:
+  - |
+    kind: ClusterConfiguration
+    apiServer:
+      certSANs:
+      - localhost
+      - 127.0.0.1
+      # replace the following line with the IP address
+      # of the external interface on this system
+      - 192.168.12.135
+      - 0.0.0.0
+  extraPortMappings:
+  # Lookout UI
+  - containerPort: 30000
+    hostPort: 30000
+    protocol: TCP
+  # Armada Server REST API
+  - containerPort: 30001
+    hostPort: 30001
+    protocol: TCP
+  # Armada Server gRPC API
+  - containerPort: 30002
+    hostPort: 30002
+    protocol: TCP
+  # Kubernetes API
+  - containerPort: 6443
+    hostPort: 6443
+    protocol: TCP
+- role: worker
+  labels:
+    armada-spark: true

example/jupyter/notebooks/jupyter_armada_spark.ipynb

@@ -82,7 +82,7 @@
     "driver_host = os.environ.get('SPARK_DRIVER_HOST')\n",
     "driver_port = os.environ.get('SPARK_DRIVER_PORT', '7078')\n",
     "block_manager_port = os.environ.get('SPARK_BLOCK_MANAGER_PORT', '10061')\n",
-    "armada_master = os.environ.get('ARMADA_MASTER', 'local://armada://host.docker.internal:30002')\n",
+    "armada_master = os.environ.get('ARMADA_MASTER', 'armada://host.docker.internal:30002')\n",
     "armada_queue = os.environ.get('ARMADA_QUEUE', 'default')\n",
     "armada_namespace = os.environ.get('ARMADA_NAMESPACE', 'default')\n",
     "image_name = os.environ.get('IMAGE_NAME', 'spark:armada')\n",