Add V100, CPU coverage tests and release build pipelines

.buildkite/amd-mi210-debug-tests.yml

@@ -0,0 +1,83 @@
+steps:
+  - label: ":cmake: Build AMD MI210 (Coverage)"
+    key: "build-mi210"
+    command: |
+      source /opt/spack-environment/activate.sh
+      export WORKSPACE=/workspace
+      mkdir -p /workspace/build
+      cd /workspace/build
+      FC=gfortran cmake -DCMAKE_INSTALL_PREFIX=/workspace/opt/self \
+            -DCMAKE_BUILD_TYPE="coverage" \
+            -DSELF_ENABLE_GPU=ON \
+            -DSELF_GPU_BACKEND=HIP \
+            -DSELF_ENABLE_TESTING=ON \
+            -DCMAKE_HIP_ARCHITECTURES="gfx90a" \
+            -DGPU_TARGETS="gfx90a" \
+            -DSELF_ENABLE_EXAMPLES=ON \
+            -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \
+            /workspace/
+      make -j
+      lcov --capture --initial \
+           --directory /workspace/build/src/ \
+           --output-file /workspace/initial.info
+    env:
+      slurm_partition: "main"
+      slurm_time: "01:00:00"
+      slurm_nodes: 1
+      slurm_ntasks: 2
+      slurm_cpus_per_task: 8
+      slurm_container_image: "docker://higherordermethods/selfish:latest-x86-rocm643-gfx90a"
+    agents:
+      queue: "galapagos"
+
+  - label: ":test_tube: Test AMD MI210 (Coverage)"
+    key: "test-mi210"
+    depends_on: "build-mi210"
+    command: |
+      source /opt/spack-environment/activate.sh
+      ctest --test-dir /workspace/build --output-on-failure
+    env:
+      slurm_partition: "main"
+      slurm_gres: "gpu:mi210:2"
+      slurm_time: "01:00:00"
+      slurm_nodes: 1
+      slurm_ntasks: 2
+      slurm_cpus_per_task: 8
+      slurm_container_image: "docker://higherordermethods/selfish:latest-x86-rocm643-gfx90a"
+    agents:
+      queue: "galapagos"
+
+  - label: ":coverage: Coverage AMD MI210"
+    key: "coverage-mi210"
+    depends_on: "test-mi210"
+    command: |
+      source /opt/spack-environment/activate.sh
+      lcov --capture \
+           --directory /workspace/build/src/ \
+           --output-file /workspace/ctest-capture.info
+      lcov --add-tracefile /workspace/initial.info \
+           --add-tracefile /workspace/ctest-capture.info \
+           --output-file /workspace/coverage.info
+    env:
+      slurm_partition: "main"
+      slurm_time: "01:00:00"
+      slurm_nodes: 1
+      slurm_ntasks: 2
+      slurm_cpus_per_task: 8
+      slurm_container_image: "docker://higherordermethods/selfish:latest-x86-rocm643-gfx90a"
+    agents:
+      queue: "galapagos"
+
+  - label: ":codecov: Upload Coverage (MI210)"
+    key: "upload-coverage-mi210"
+    depends_on: "coverage-mi210"
+    command: |
+      curl -Os https://uploader.codecov.io/latest/linux/codecov
+      chmod +x codecov
+      ./codecov -t "$${CODECOV_TOKEN}" \
+                -f "$${BUILDKITE_BUILD_CHECKOUT_PATH}/coverage.info" \
+                -F buildkite-rocm-mi210
+    secrets:
+      - CODECOV_TOKEN
+    agents:
+      queue: "galapagos"

.buildkite/nvidia-v100-debug-tests.yml

@@ -0,0 +1,82 @@
+steps:
+  - label: ":cmake: Build NVIDIA V100 (Coverage)"
+    key: "build-v100"
+    command: |
+      source /opt/spack-environment/activate.sh
+      export WORKSPACE=/workspace
+      mkdir -p /workspace/build
+      cd /workspace/build
+      FC=gfortran cmake -DCMAKE_INSTALL_PREFIX=/workspace/opt/self \
+            -DCMAKE_BUILD_TYPE="coverage" \
+            -DSELF_ENABLE_GPU=ON \
+            -DSELF_GPU_BACKEND=CUDA \
+            -DSELF_ENABLE_TESTING=ON \
+            -DCMAKE_CUDA_ARCHITECTURES="70" \
+            -DSELF_ENABLE_EXAMPLES=ON \
+            -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \
+            /workspace/
+      make -j
+      lcov --capture --initial \
+           --directory /workspace/build/src/ \
+           --output-file /workspace/initial.info
+    env:
+      slurm_partition: "main"
+      slurm_time: "01:00:00"
+      slurm_nodes: 1
+      slurm_ntasks: 2
+      slurm_cpus_per_task: 8
+      slurm_container_image: "docker://higherordermethods/selfish:latest-x86-cuda124-sm70"
+    agents:
+      queue: "galapagos"
+
+  - label: ":test_tube: Test NVIDIA V100 (Coverage)"
+    key: "test-v100"
+    depends_on: "build-v100"
+    command: |
+      source /opt/spack-environment/activate.sh
+      ctest --test-dir /workspace/build --output-on-failure
+    env:
+      slurm_partition: "main"
+      slurm_gres: "gpu:v100:2"
+      slurm_time: "01:00:00"
+      slurm_nodes: 1
+      slurm_ntasks: 2
+      slurm_cpus_per_task: 8
+      slurm_container_image: "docker://higherordermethods/selfish:latest-x86-cuda124-sm70"
+    agents:
+      queue: "galapagos"
+
+  - label: ":coverage: Coverage NVIDIA V100"
+    key: "coverage-v100"
+    depends_on: "test-v100"
+    command: |
+      source /opt/spack-environment/activate.sh
+      lcov --capture \
+           --directory /workspace/build/src/ \
+           --output-file /workspace/ctest-capture.info
+      lcov --add-tracefile /workspace/initial.info \
+           --add-tracefile /workspace/ctest-capture.info \
+           --output-file /workspace/coverage.info
+    env:
+      slurm_partition: "main"
+      slurm_time: "01:00:00"
+      slurm_nodes: 1
+      slurm_ntasks: 2
+      slurm_cpus_per_task: 8
+      slurm_container_image: "docker://higherordermethods/selfish:latest-x86-cuda124-sm70"
+    agents:
+      queue: "galapagos"
+
+  - label: ":codecov: Upload Coverage (V100)"
+    key: "upload-coverage-v100"
+    depends_on: "coverage-v100"
+    command: |
+      curl -Os https://uploader.codecov.io/latest/linux/codecov
+      chmod +x codecov
+      ./codecov -t "$${CODECOV_TOKEN}" \
+                -f "$${BUILDKITE_BUILD_CHECKOUT_PATH}/coverage.info" \
+                -F buildkite-cuda-v100
+    secrets:
+      - CODECOV_TOKEN
+    agents:
+      queue: "galapagos"

.buildkite/pipeline.yml

@@ -1,54 +1,22 @@
 steps:
-  - label: ":partyparrot: Build & Test AMD MI210 (Coverage)"
-    key: "build-test-mi210"
-    command: |
-      source /opt/spack-environment/activate.sh
-      export WORKSPACE=/workspace
-      mkdir -p /workspace/build
-      rocminfo
-      cd /workspace/build
-      FC=gfortran cmake -DCMAKE_INSTALL_PREFIX=/workspace/opt/self \
-            -DCMAKE_BUILD_TYPE="coverage" \
-            -DSELF_ENABLE_GPU=ON \
-            -DSELF_GPU_BACKEND=HIP \
-            -DSELF_ENABLE_TESTING=ON \
-            -DCMAKE_HIP_ARCHITECTURES="gfx90a" \
-            -DGPU_TARGETS="gfx90a" \
-            -DSELF_ENABLE_EXAMPLES=ON \
-            -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \
-            /workspace/
-      make -j
-      lcov --capture --initial \
-           --directory /workspace/build/src/ \
-           --output-file /workspace/initial.info
-      ctest --test-dir /workspace/build --output-on-failure
-      lcov --capture \
-           --directory /workspace/build/src/ \
-           --output-file /workspace/ctest-capture.info
-      lcov --add-tracefile /workspace/initial.info \
-           --add-tracefile /workspace/ctest-capture.info \
-           --output-file /workspace/coverage.info
-    env:
-      slurm_partition: "main"
-      slurm_gres: "gpu:mi210:2"
-      slurm_time: "01:00:00"
-      slurm_nodes: 1
-      slurm_ntasks: 2
-      slurm_cpus_per_task: 8
-      slurm_container_image: "docker://higherordermethods/selfish:latest-x86-rocm643-gfx90a"
-    agents:
-      queue: "galapagos"
+  # PR builds: coverage tests + release build/test (no push)
+  - label: ":pipeline: AMD MI210 Coverage Tests"
+    if: build.branch != "main"
+    command: buildkite-agent pipeline upload .buildkite/amd-mi210-debug-tests.yml
 
-  - label: ":codecov: Upload Coverage"
-    key: "upload-coverage"
-    depends_on: "build-test-mi210"
-    command: |
-      curl -Os https://uploader.codecov.io/latest/linux/codecov
-      chmod +x codecov
-      ./codecov -t "$${CODECOV_TOKEN}" \
-                -f "$${BUILDKITE_BUILD_CHECKOUT_PATH}/coverage.info" \
-                -F buildkite-rocm-mi210
-    secrets:
-      - CODECOV_TOKEN
-    agents:
-      queue: "galapagos"
+  - label: ":pipeline: NVIDIA V100 Coverage Tests"
+    if: build.branch != "main"
+    command: buildkite-agent pipeline upload .buildkite/nvidia-v100-debug-tests.yml
+
+  - label: ":pipeline: x86 CPU Coverage Tests"
+    if: build.branch != "main"
+    command: buildkite-agent pipeline upload .buildkite/x86-cpu-debug-tests.yml
+
+  - label: ":pipeline: Release Builds"
+    if: build.branch != "main"
+    command: buildkite-agent pipeline upload .buildkite/release-builds.yml
+
+  # Main branch: release build, test, and publish only
+  - label: ":pipeline: Release & Publish"
+    if: build.branch == "main"
+    command: buildkite-agent pipeline upload .buildkite/release-and-publish.yml

.buildkite/release-and-publish.yml

@@ -0,0 +1,137 @@
+steps:
+  # ---- AMD MI210 (gfx90a) Release ----
+  - label: ":docker: Build Release Image (x86-rocm643-gfx90a)"
+    key: "docker-build-gfx90a"
+    command: |
+      SELFISH_IMAGE="higherordermethods/selfish:latest-x86-rocm643-gfx90a"
+      docker pull "$${SELFISH_IMAGE}"
+      SELFISH_SHA=$$(docker inspect --format='{{index .RepoDigests 0}}' "$${SELFISH_IMAGE}" | cut -d@ -f2)
+      docker build \
+        --build-arg SELFISH_IMAGE="$${SELFISH_IMAGE}" \
+        --build-arg SELFISH_SHA="$${SELFISH_SHA}" \
+        -t higherordermethods/self:$${BUILDKITE_BUILD_NUMBER}-x86-rocm643-gfx90a \
+        -t higherordermethods/self:latest-x86-rocm643-gfx90a \
+        -f docker/x86_gfx90a/Dockerfile .
+    env:
+      slurm_nodelist: "oram"
+    agents:
+      queue: "galapagos"
+
+  - label: ":test_tube: Test Release Image (x86-rocm643-gfx90a)"
+    key: "test-release-gfx90a"
+    depends_on: "docker-build-gfx90a"
+    command: |
+      source /opt/spack-environment/activate.sh
+      ctest --test-dir /opt/self/build --output-on-failure
+    env:
+      slurm_partition: "main"
+      slurm_gres: "gpu:mi210:2"
+      slurm_time: "01:00:00"
+      slurm_nodes: 1
+      slurm_ntasks: 2
+      slurm_cpus_per_task: 8
+      slurm_container_image: "docker://higherordermethods/self:$${BUILDKITE_BUILD_NUMBER}-x86-rocm643-gfx90a"
+    agents:
+      queue: "galapagos"
+
+  - label: ":rocket: Push Release Image (x86-rocm643-gfx90a)"
+    key: "push-release-gfx90a"
+    depends_on: "test-release-gfx90a"
+    command: |
+      docker push higherordermethods/self:$${BUILDKITE_BUILD_NUMBER}-x86-rocm643-gfx90a
+      docker push higherordermethods/self:latest-x86-rocm643-gfx90a
+    env:
+      slurm_nodelist: "oram"
+    agents:
+      queue: "galapagos"
+
+  # ---- NVIDIA V100 (sm70) Release ----
+  - label: ":docker: Build Release Image (x86-cuda124-sm70)"
+    key: "docker-build-sm70"
+    command: |
+      SELFISH_IMAGE="higherordermethods/selfish:latest-x86-cuda124-sm70"
+      docker pull "$${SELFISH_IMAGE}"
+      SELFISH_SHA=$$(docker inspect --format='{{index .RepoDigests 0}}' "$${SELFISH_IMAGE}" | cut -d@ -f2)
+      docker build \
+        --build-arg SELFISH_IMAGE="$${SELFISH_IMAGE}" \
+        --build-arg SELFISH_SHA="$${SELFISH_SHA}" \
+        -t higherordermethods/self:$${BUILDKITE_BUILD_NUMBER}-x86-cuda124-sm70 \
+        -t higherordermethods/self:latest-x86-cuda124-sm70 \
+        -f docker/x86_sm70/Dockerfile .
+    env:
+      slurm_nodelist: "oram"
+    agents:
+      queue: "galapagos"
+
+  - label: ":test_tube: Test Release Image (x86-cuda124-sm70)"
+    key: "test-release-sm70"
+    depends_on: "docker-build-sm70"
+    command: |
+      source /opt/spack-environment/activate.sh
+      ctest --test-dir /opt/self/build --output-on-failure
+    env:
+      slurm_partition: "main"
+      slurm_gres: "gpu:v100:2"
+      slurm_time: "01:00:00"
+      slurm_nodes: 1
+      slurm_ntasks: 2
+      slurm_cpus_per_task: 8
+      slurm_container_image: "docker://higherordermethods/self:$${BUILDKITE_BUILD_NUMBER}-x86-cuda124-sm70"
+    agents:
+      queue: "galapagos"
+
+  - label: ":rocket: Push Release Image (x86-cuda124-sm70)"
+    key: "push-release-sm70"
+    depends_on: "test-release-sm70"
+    command: |
+      docker push higherordermethods/self:$${BUILDKITE_BUILD_NUMBER}-x86-cuda124-sm70
+      docker push higherordermethods/self:latest-x86-cuda124-sm70
+    env:
+      slurm_nodelist: "oram"
+    agents:
+      queue: "galapagos"
+
+  # ---- x86 CPU-only Release ----
+  - label: ":docker: Build Release Image (x86)"
+    key: "docker-build-x86"
+    command: |
+      SELFISH_IMAGE="higherordermethods/selfish:latest-x86"
+      docker pull "$${SELFISH_IMAGE}"
+      SELFISH_SHA=$$(docker inspect --format='{{index .RepoDigests 0}}' "$${SELFISH_IMAGE}" | cut -d@ -f2)
+      docker build \
+        --build-arg SELFISH_IMAGE="$${SELFISH_IMAGE}" \
+        --build-arg SELFISH_SHA="$${SELFISH_SHA}" \
+        -t higherordermethods/self:$${BUILDKITE_BUILD_NUMBER}-x86 \
+        -t higherordermethods/self:latest-x86 \
+        -f docker/x86/Dockerfile .
+    env:
+      slurm_nodelist: "oram"
+    agents:
+      queue: "galapagos"
+
+  - label: ":test_tube: Test Release Image (x86)"
+    key: "test-release-x86"
+    depends_on: "docker-build-x86"
+    command: |
+      source /opt/spack-environment/activate.sh
+      ctest --test-dir /opt/self/build --output-on-failure
+    env:
+      slurm_partition: "main"
+      slurm_time: "01:00:00"
+      slurm_nodes: 1
+      slurm_ntasks: 2
+      slurm_cpus_per_task: 8
+      slurm_container_image: "docker://higherordermethods/self:$${BUILDKITE_BUILD_NUMBER}-x86"
+    agents:
+      queue: "galapagos"
+
+  - label: ":rocket: Push Release Image (x86)"
+    key: "push-release-x86"
+    depends_on: "test-release-x86"
+    command: |
+      docker push higherordermethods/self:$${BUILDKITE_BUILD_NUMBER}-x86
+      docker push higherordermethods/self:latest-x86
+    env:
+      slurm_nodelist: "oram"
+    agents:
+      queue: "galapagos"