From 777036c80d70d84d43518d82c5f434007f3949f3 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Thu, 12 Mar 2026 11:29:30 +0100 Subject: [PATCH] chore: Bump to Iceberg 1.10.1 --- .../create-spark-ingestion-job.yaml | 3 ++- demos/end-to-end-security/create-spark-report.yaml | 3 ++- .../create-spark-anomaly-detection-job.yaml | 3 ++- stacks/airflow/airflow.yaml | 2 +- stacks/jupyterhub-pyspark-hdfs/spark_connect.yaml | 4 ++-- 5 files changed, 9 insertions(+), 6 deletions(-) diff --git a/demos/data-lakehouse-iceberg-trino-spark/create-spark-ingestion-job.yaml b/demos/data-lakehouse-iceberg-trino-spark/create-spark-ingestion-job.yaml index 8751e9f6..9638a4b2 100644 --- a/demos/data-lakehouse-iceberg-trino-spark/create-spark-ingestion-job.yaml +++ b/demos/data-lakehouse-iceberg-trino-spark/create-spark-ingestion-job.yaml @@ -144,12 +144,13 @@ data: stackable.tech/vendor: Stackable spec: sparkImage: + # Iceberg 1.10.1 only supports Spark 4.0.x productVersion: 4.0.1 mode: cluster mainApplicationFile: local:///stackable/spark/jobs/spark-ingest-into-lakehouse.py deps: packages: - - org.apache.iceberg:iceberg-spark-runtime-4.0_2.13:1.10.0 + - org.apache.iceberg:iceberg-spark-runtime-4.0_2.13:1.10.1 - org.apache.spark:spark-sql-kafka-0-10_2.13:4.0.1 s3connection: reference: minio diff --git a/demos/end-to-end-security/create-spark-report.yaml b/demos/end-to-end-security/create-spark-report.yaml index ff48f9d9..c48f7603 100644 --- a/demos/end-to-end-security/create-spark-report.yaml +++ b/demos/end-to-end-security/create-spark-report.yaml @@ -55,12 +55,13 @@ data: name: spark-report spec: sparkImage: + # Iceberg 1.10.1 only supports Spark 4.0.x productVersion: 4.0.1 mode: cluster mainApplicationFile: local:///stackable/spark/jobs/spark-report.py deps: packages: - - org.apache.iceberg:iceberg-spark-runtime-4.0_2.13:1.10.0 + - org.apache.iceberg:iceberg-spark-runtime-4.0_2.13:1.10.1 sparkConf: spark.driver.extraClassPath: /stackable/config/hdfs spark.executor.extraClassPath: /stackable/config/hdfs diff --git a/demos/spark-k8s-anomaly-detection-taxi-data/create-spark-anomaly-detection-job.yaml b/demos/spark-k8s-anomaly-detection-taxi-data/create-spark-anomaly-detection-job.yaml index 12b1614d..e2c139e1 100644 --- a/demos/spark-k8s-anomaly-detection-taxi-data/create-spark-anomaly-detection-job.yaml +++ b/demos/spark-k8s-anomaly-detection-taxi-data/create-spark-anomaly-detection-job.yaml @@ -51,12 +51,13 @@ data: name: spark-ad spec: sparkImage: + # Iceberg 1.10.1 only supports Spark 4.0.x productVersion: 4.0.1 mode: cluster mainApplicationFile: local:///spark-scripts/spark-ad.py deps: packages: - - org.apache.iceberg:iceberg-spark-runtime-4.0_2.13:1.10.0 + - org.apache.iceberg:iceberg-spark-runtime-4.0_2.13:1.10.1 requirements: - scikit-learn==1.4.0 s3connection: diff --git a/stacks/airflow/airflow.yaml b/stacks/airflow/airflow.yaml index ba514959..ebf0e9e0 100644 --- a/stacks/airflow/airflow.yaml +++ b/stacks/airflow/airflow.yaml @@ -537,7 +537,7 @@ data: spec: version: "1.0" sparkImage: - productVersion: 4.0.1 + productVersion: 4.1.1 mode: cluster mainApplicationFile: local:///stackable/spark/examples/src/main/python/pi.py job: diff --git a/stacks/jupyterhub-pyspark-hdfs/spark_connect.yaml b/stacks/jupyterhub-pyspark-hdfs/spark_connect.yaml index f7804e58..82715e9b 100644 --- a/stacks/jupyterhub-pyspark-hdfs/spark_connect.yaml +++ b/stacks/jupyterhub-pyspark-hdfs/spark_connect.yaml @@ -30,8 +30,8 @@ spec: image: # Using an image that includes scikit-learn (among other things) # because this package needs to be available on the executors. - custom: oci.stackable.tech/stackable/spark-connect-client:4.0.1-stackable0.0.0-dev - productVersion: 4.0.1 + custom: oci.stackable.tech/stackable/spark-connect-client:4.1.1-stackable0.0.0-dev + productVersion: 4.1.1 pullPolicy: IfNotPresent args: server: