CASSANDRA-18831 Add JDK21 support

.build/docker/_set_java.sh

@@ -56,7 +56,8 @@ fi
 ################################
 
 if grep "^ID=" /etc/os-release | grep -q 'debian\|ubuntu' ; then
-    sudo update-java-alternatives --set java-1.${java_version}.0-openjdk-$(dpkg --print-architecture)
+    sudo update-alternatives --set java /usr/lib/jvm/java-${java_version}-openjdk-$(dpkg --print-architecture)/bin/java
+    sudo update-alternatives --set javac /usr/lib/jvm/java-${java_version}-openjdk-$(dpkg --print-architecture)/bin/javac
 else
     sudo alternatives --set java $(alternatives --display java | grep "family java-${java_version}-openjdk" | cut -d' ' -f1)
     sudo alternatives --set javac $(alternatives --display javac | grep "family java-${java_version}-openjdk" | cut -d' ' -f1)

.build/docker/almalinux-build.docker

@@ -42,6 +42,7 @@ RUN yum -y install \
    git \
    java-11-openjdk-devel \
    java-17-openjdk-devel \
+   java-21-openjdk-devel \
    make \
    rpm-build \
    sudo \

.build/docker/build-artifacts.sh

@@ -18,5 +18,5 @@
 #
 # Creates the tarball artifact
 
-$(dirname "$0")/_docker_run.sh bullseye-build.docker build-artifacts.sh $1
+$(dirname "$0")/_docker_run.sh debian-build.docker build-artifacts.sh $1
 exit $?

.build/docker/build-debian.sh

@@ -32,5 +32,5 @@ echo
 #
 # Creates the debian package
 
-$(dirname "$0")/_docker_run.sh bullseye-build.docker docker/_build-debian.sh $1
+$(dirname "$0")/_docker_run.sh debian-build.docker docker/_build-debian.sh $1
 exit $?

.build/docker/build-jars.sh

@@ -18,5 +18,5 @@
 #
 # Build the jars
 
-$(dirname "$0")/_docker_run.sh bullseye-build.docker build-jars.sh $1
+$(dirname "$0")/_docker_run.sh debian-build.docker build-jars.sh $1
 exit $?

.build/docker/check-code.sh

@@ -20,4 +20,4 @@
 
 export CASSANDRA_DOCKER_ANT_OPTS="-Ddependency-check.home.base=/tmp"
 
-$(dirname "$0")/_docker_run.sh bullseye-build.docker check-code.sh $1
+$(dirname "$0")/_docker_run.sh debian-build.docker check-code.sh $1

.build/docker/bullseye-build.docker → .build/docker/debian-build.docker

@@ -36,16 +36,31 @@ RUN echo 'Acquire::http::Timeout "60";' > /etc/apt/apt.conf.d/80proxy.conf
 RUN echo 'Acquire::ftp::Timeout "60";' >> /etc/apt/apt.conf.d/80proxy.conf
 
 # install deps
-RUN until apt-get update \
-        && apt-get -y install ant build-essential curl devscripts ed git sudo \
+RUN until apt-get update && apt-get -y install ant build-essential curl devscripts ed git sudo \
                 python3-pip rsync procps dh-python quilt bash-completion ; \
     do echo "apt failed… trying again in 10s… " ; sleep 10 ; done
 
-RUN until apt-get update \
-        && apt-get install -y --no-install-recommends openjdk-11-jdk openjdk-17-jdk ; \
+RUN until apt-get install -y --no-install-recommends openjdk-11-jdk openjdk-17-jdk; \
     do echo "apt failed… trying again in 10s… " ; sleep 10 ; done
 
-RUN update-java-alternatives --set java-1.11.0-openjdk-$(dpkg --print-architecture)
+# Download and install OpenJDK 21.0.2 for the current architecture
+RUN sh -c '\
+    JDK_OS=linux ;\
+    JDK_TAR="openjdk-21.0.2_${JDK_OS}-$(uname -m | sed "s/x86_64/x64/")_bin.tar.gz" ;\
+    JDK_VERSION_SHAS="08db1392a48d4eb5ea5315cf8f18b89dbaf36cda663ba882cf03c704c9257ec2 a2def047a73941e01a73739f92755f86b895811afb1f91243db214cff5bdac3f 8fd09e15dc406387a0aba70bf5d99692874e999bf9cd9208b452b5d76ac922d3 b3d588e16ec1e0ef9805d8a696591bd518a5cea62567da8f53b5ce32d11d22e4" ;\
+    curl -L --fail --silent --retry 2 --retry-delay 5 --max-time 3600 "https://download.java.net/java/GA/jdk21.0.2/f2283984656d49d69e91c558476027ac/13/GPL/${JDK_TAR}" -o $JDK_TAR ;\
+    JDK_SHA="$(sha256sum $JDK_TAR | cut -d" " -f2)" ;\
+    echo "$JDK_VERSION_SHAS" | sed "s/ /\n/g" | grep -q "$JDK_SHA" || { echo "SHA256 mismatch for $JDK_TAR $JDK_SHA"; exit 1; } ;\
+    mkdir -p /usr/lib/jvm/java-21-openjdk-$(dpkg --print-architecture) ;\
+    tar -C /usr/lib/jvm/java-21-openjdk-$(dpkg --print-architecture) --strip-components=1 -xzf $JDK_TAR ;\
+    rm $JDK_TAR'
+
+# Add OpenJDK 21 to update-alternatives
+RUN update-alternatives --install /usr/bin/java java /usr/lib/jvm/java-21-openjdk-$(dpkg --print-architecture)/bin/java 1
+RUN update-alternatives --install /usr/bin/javac javac /usr/lib/jvm/java-21-openjdk-$(dpkg --print-architecture)/bin/javac 1
+
+RUN update-alternatives --set java /usr/lib/jvm/java-11-openjdk-$(dpkg --print-architecture)/bin/java
+RUN update-alternatives --set javac /usr/lib/jvm/java-11-openjdk-$(dpkg --print-architecture)/bin/javac
 
 # python3 is needed for the gen-doc target
 RUN pip install --upgrade pip
@@ -58,16 +73,20 @@ RUN sh -c '\
   GO_VERSION="1.24.3" ;\
   GO_VERSION_SHAS="3333f6ea53afa971e9078895eaa4ac7204a8c6b5c68c10e6bc9a33e8e391bdd8 a463cb59382bd7ae7d8f4c68846e73c4d589f223c589ac76871b66811ded7836 13e6fe3fcf65689d77d40e633de1e31c6febbdbcb846eb05fc2434ed2213e92b 64a3fa22142f627e78fac3018ce3d4aeace68b743eff0afda8aae0411df5e4fb" ;\
   GO_OS=linux ;\
-  [ $(uname) = "Darwin" ] && GO_OS=darwin ;\
-  GO_PLATFORM=amd64 ;\
-  [ $(uname -m) = "aarch64" ] && GO_PLATFORM=arm64 ;\
-  GO_TAR="go${GO_VERSION}.${GO_OS}-${GO_PLATFORM}.tar.gz" ;\
-  curl -L --fail --silent --retry 2 --retry-delay 5 --max-time 30 https://go.dev/dl/$GO_TAR -o $GO_TAR ;\
+  GO_TAR="go${GO_VERSION}.${GO_OS}-$(dpkg --print-architecture).tar.gz" ;\
+  curl -L --fail --silent --retry 2 --retry-delay 5 --max-time 3600 https://go.dev/dl/$GO_TAR -o $GO_TAR ;\
   GO_SHA="$(sha256sum $GO_TAR | cut -d" " -f2)" ;\
   echo "$GO_VERSION_SHAS" | sed "s/ /\n/g" | grep -q "$GO_SHA" || { echo "SHA256 mismatch for $GO_TAR $GO_SHA"; exit 1; } ;\
   tar -C /usr/local -xzf $GO_TAR ;\
   rm $GO_TAR'
 
 ENV GOROOT="/usr/local/go"
 ENV GOPATH="$BUILD_HOME/go"
-ENV PATH="$PATH:/usr/local/go/bin"
+ENV PATH="$PATH:/usr/local/go/bin"
+
+# allow lower UIDs and GIDs
+RUN sed -i 's/UID_MIN			 1000/UID_MIN			 100/' /etc/login.defs
+RUN sed -i 's/UID_MIN			 1000/UID_MIN			 10/' /etc/login.defs
+
+# suppress warnings about mismatching ownership
+RUN git config --global --add safe.directory ${CASSANDRA_DIR}

.build/docker/run-tests.sh

@@ -128,7 +128,7 @@ docker --version
 pushd ${cassandra_dir}/.build >/dev/null
 
 # build test image
-dockerfile="ubuntu2004_test.docker"
+dockerfile="ubuntu-test.docker"
 image_tag="$(md5sum docker/${dockerfile} | cut -d' ' -f1)"
 image_name="apache/cassandra-${dockerfile/.docker/}:${image_tag}"
 docker_mounts="-v ${cassandra_dir}:/home/cassandra/cassandra -v "${build_dir}":/home/cassandra/cassandra/build -v ${m2_dir}:/home/cassandra/.m2/repository"

.build/docker/ubuntu2004_test.docker → .build/docker/ubuntu-test.docker

@@ -11,7 +11,7 @@
 # limitations under the License.
 
 FROM ubuntu:20.04
-MAINTAINER Apache Cassandra <dev@cassandra.apache.org>
+LABEL org.opencontainers.image.authors="Apache Cassandra <dev@cassandra.apache.org>"
 
 # CONTEXT is expected to be cassandra/.build
 
@@ -52,7 +52,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
         python3.11 python3.11-venv python3.11-dev \
         virtualenv net-tools libev4 libev-dev wget gcc libxml2 libxslt1-dev \
         vim lsof sudo libjemalloc2 dumb-init locales rsync \
-        openjdk-8-jdk openjdk-11-jdk openjdk-17-jdk ant ant-optional
+        openjdk-8-jdk openjdk-11-jdk openjdk-17-jdk openjdk-21-jdk ant ant-optional
 
 
 RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.8 2
@@ -71,8 +71,9 @@ RUN chmod 0644 /opt/requirements.txt
 RUN pip3 install virtualenv virtualenv-clone
 RUN pip3 install --upgrade wheel
 
-# make Java 8 the default executable (we use to run all tests against Java 8)
-RUN update-java-alternatives --set java-1.8.0-openjdk-$(dpkg --print-architecture)
+# make Java 11 the default executable
+RUN update-alternatives --set java /usr/lib/jvm/java-11-openjdk-$(dpkg --print-architecture)/bin/java
+RUN update-alternatives --set javac /usr/lib/jvm/java-11-openjdk-$(dpkg --print-architecture)/bin/javac
 
 # enable legacy TLSv1 and TLSv1.1 (CASSANDRA-16848)
 RUN find /etc -type f -name java.security -exec sed -i 's/TLSv1, TLSv1.1//' {} \;

.build/parent-maven-pom.xml

@@ -36,8 +36,8 @@
     </license>
   </licenses>
   <properties>
-    <bytebuddy.version>1.12.13</bytebuddy.version>
-    <byteman.version>4.0.20</byteman.version>
+    <bytebuddy.version>1.17.8</bytebuddy.version>
+    <byteman.version>4.0.26</byteman.version>
     <netty.version>4.1.130.Final</netty.version>
     <ohc.version>0.5.1</ohc.version>
     <async-profiler.version>4.2</async-profiler.version>
@@ -494,13 +494,13 @@
       <dependency>
         <groupId>org.mockito</groupId>
         <artifactId>mockito-core</artifactId>
-        <version>4.7.0</version>
+        <version>5.12.0</version>
         <scope>test</scope>
       </dependency>
       <dependency>
         <groupId>org.mockito</groupId>
         <artifactId>mockito-inline</artifactId>
-        <version>4.7.0</version>
+        <version>5.2.0</version>
         <scope>test</scope>
       </dependency>
       <dependency>
@@ -891,7 +891,8 @@
       <dependency>
         <groupId>net.openhft</groupId>
         <artifactId>chronicle-queue</artifactId>
-        <version>5.23.37</version>
+        <!-- chronicle ea builds before the corresponding Final are code identical; Final builds are not made available on Maven -->
+        <version>5.25ea16</version>
         <exclusions>
           <exclusion>
             <artifactId>tools</artifactId>
@@ -907,7 +908,12 @@
       <dependency>
         <groupId>net.openhft</groupId>
         <artifactId>chronicle-core</artifactId>
-        <version>2.23.36</version>
+        <!-- While we broadly don't want to use ea libraries in the project, the Chronicle ecosystem has moved
+             to an approach of only allowing paid usage for non-ea builds. The ea builds are identical to the .0 release
+             of the next major version, so "ea" in this context signifies "stable GA build" for this major release line.
+             We plan to move away from chronicle for this and other reasons; the details of the conversation on the mailing
+             list can be found here: https://issues.apache.org/jira/browse/CASSANDRA-18831?focusedCommentId=17902034&page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel#comment-17902034 -->
+        <version>2.25ea14</version>
         <exclusions>
           <exclusion>
             <artifactId>chronicle-analytics</artifactId>
@@ -922,7 +928,8 @@
       <dependency>
         <groupId>net.openhft</groupId>
         <artifactId>chronicle-bytes</artifactId>
-        <version>2.23.33</version>
+        <!-- See above for details about exception re: inclusion of ea lib -->
+        <version>2.25ea10</version>
         <exclusions>
           <exclusion>
             <artifactId>annotations</artifactId>
@@ -933,7 +940,8 @@
       <dependency>
         <groupId>net.openhft</groupId>
         <artifactId>chronicle-wire</artifactId>
-        <version>2.23.39</version>
+        <!-- See above for details about exception re: inclusion of ea lib -->
+        <version>2.25ea15</version>
         <exclusions>
           <exclusion>
             <artifactId>compiler</artifactId>
@@ -949,7 +957,8 @@
       <dependency>
         <groupId>net.openhft</groupId>
         <artifactId>chronicle-threads</artifactId>
-        <version>2.23.25</version>
+        <!-- See above for details about exception re: inclusion of ea lib -->
+        <version>2.25ea7</version>
         <exclusions>
           <exclusion>
               <!-- pulls in affinity-3.23ea1 which pulls in third-party-bom-3.22.4-SNAPSHOT -->

.build/run-python-dtests.sh

@@ -124,7 +124,7 @@ set -e # enable immediate exit if venv setup fails
 # fresh virtualenv and test logs results everytime
 [[ "/" == "${DIST_DIR}" ]] || rm -rf "${DIST_DIR}/venv" "${DIST_DIR}/test/{html,output,logs}"
 
-# re-use when possible the pre-installed virtualenv found in the cassandra-ubuntu2004_test docker image
+# re-use when possible the pre-installed virtualenv found in the cassandra2004_test docker image
 virtualenv-clone ${BUILD_HOME}/env${python_version} ${DIST_DIR}/venv || virtualenv --python=python${python_version} ${DIST_DIR}/venv
 source ${DIST_DIR}/venv/bin/activate
 pip3 install --exists-action w -r ${CASSANDRA_DTEST_DIR}/requirements.txt

.build/run-tests.sh

@@ -251,7 +251,11 @@ _run_testlist() {
     for ((i=0; i < _test_iterations; i++)); do
       [ "${_test_iterations}" -eq 1 ] || printf "–––– run ${i}\n"
       set +o errexit
-      ant "$_testlist_target" -Dtest.classlistprefix="${_target_prefix}" -Dtest.classlistfile=<(echo "${testlist}") -Dtest.timeout="${_test_timeout}" ${ANT_TEST_OPTS}
+      ant "$_testlist_target" \
+        -Dtest.classlistprefix="${_target_prefix}" \
+        -Dtest.classlistfile=<(echo "${testlist}") \
+        -Dtest.timeout="${_test_timeout}" \
+        ${ANT_TEST_OPTS}
       ant_status=$?
       set -o errexit
       if [[ $ant_status -ne 0 ]]; then

.jenkins/Jenkinsfile

@@ -382,7 +382,7 @@ def test(command, cell) {
       withEnv(cell.collect { k, v -> "${k}=${v}" }) {
         ws("workspace/${JOB_NAME}/${BUILD_NUMBER}/${cell.step}/${cell.arch}/jdk-${cell.jdk}/python-${cell.python}") {
           fetchSource(cell.step, cell.arch, cell.jdk)
-          fetchDockerImages(['ubuntu2004_test'])
+          fetchDockerImages(['ubuntu-test'])
           def cell_suffix = "_jdk${cell.jdk}_python_${cell.python}_${cell.cython}_${cell.arch}_${cell.split}_${splits}"
           def logfile = "stage-logs/${JOB_NAME}_${BUILD_NUMBER}_${cell.step}${cell_suffix}_attempt${attempt}.log.xz"
           def script_vars = "#!/bin/bash \n set -o pipefail ; " // pipe to tee needs pipefail

NEWS.txt

@@ -80,6 +80,9 @@ using the provided 'sstableupgrade' tool.
 
 New features
 ------------
+    - JDK21 and Generational ZGC are now officially supported and generational ZGC is now the default Garbage Collector
+    when using JDK21. See jvm21-server.options for more details and configuration parameters. We do not recommend using
+    non-generational ZGC with Apache Cassandra.
     [The following is a placeholder, to be revised asap]
     - CEP-37 Auto Repair is a fully automated scheduler that provides repair orchestration within Apache Cassandra. This
     significantly reduces operational overhead by eliminating the need for operators to deploy external tools to submit
@@ -204,6 +207,11 @@ Upgrading
 Deprecation
 -----------
     - `use_deterministic_table_id` is no longer supported and should be removed from cassandra.yaml. Table IDs may still be supplied explicitly on CREATE.
+    - Updating dependencies for JDK21 necessitated updating Chronicle Queue, which has changed the enums used for log
+    rolling (cassandra.yaml -> full_query_logging_options:roll_cycle). Older legacy options will still work for the
+    foreseeable future but you will see warnings in logs and future dependency upgrades may break your log rolling param.
+    The default log rolling param has been changed from HOURLY to FAST_HOURLY, primarily different on how frequently
+    indexes are built (256 in FAST_HOURLY vs. 16 in HOURLY).
     - IEndpointSnitch has been deprecated as ClusterMetadata is now the source of truth regarding topology information.
       The responsibilities of the snitch have been broken out to a handful of new classes:
       * o.a.c.locator.Locator provides datacenter and rack info for endpoints. This is not configurable as topology is

bin/cassandra.in.sh

@@ -114,56 +114,36 @@ if [ -z $JAVA ] ; then
 fi
 
 # Matches variable 'java.supported' in build.xml
-java_versions_supported=11,17
+java_versions_supported="11 17 21"
+java_version_string=$(IFS=" "; echo "${java_versions_supported}")
 
 # Determine the sort of JVM we'll be running on.
-java_ver_output=`"${JAVA:-java}" -version 2>&1`
-jvmver=`echo "$java_ver_output" | grep '[openjdk|java] version' | awk -F'"' 'NR==1 {print $2}' | cut -d\- -f1`
-JVM_VERSION=${jvmver%_*}
-short=$(echo "${jvmver}" | cut -c1-2)
-
-# Unsupported JDKs below the upper supported version are not allowed
-if [ "$short" != "$(echo "$java_versions_supported" | cut -d, -f1)" ] && [ "$JVM_VERSION" \< "$(echo "$java_versions_supported" | cut -d, -f2)" ] ; then
-    echo "Unsupported Java $JVM_VERSION. Supported are $java_versions_supported"
-    exit 1;
-fi
-# Allow execution of supported Java versions, and newer if CASSANDRA_JDK_UNSUPPORTED is set
-is_supported_version=$(echo "$java_versions_supported" | tr "," '\n' | grep -F -x "$short")
-if [ -z "$is_supported_version" ] ; then
-    if [ -z "$CASSANDRA_JDK_UNSUPPORTED" ] ; then
-        echo "Unsupported Java $JVM_VERSION. Supported are $java_versions_supported"
-        echo "If you would like to test with newer Java versions set CASSANDRA_JDK_UNSUPPORTED to any value (for example, CASSANDRA_JDK_UNSUPPORTED=true). Unset the parameter for default behavior"
-        exit 1;
-    else
-        echo "######################################################################"
-        echo "Warning! You are using JDK$short. This Cassandra version only supports $java_versions_supported."
-        echo "######################################################################"
-    fi
+JAVA_VERSION=$(java -version 2>&1 | grep '[openjdk|java] version' | cut -d '"' -f2 | cut -d '.' -f1)
+
+supported=0
+for version in ${java_versions_supported}; do
+  if [ "$version" -eq "$JAVA_VERSION" ]; then
+      supported=1
+  fi
+done
+
+if [ "$supported" -eq 0 ]; then
+  if [ -z "$CASSANDRA_JDK_UNSUPPORTED" ]; then
+    echo "Unsupported Java $JAVA_VERSION. Supported are $java_version_string"
+    echo "If you would like to test with newer Java versions set CASSANDRA_JDK_UNSUPPORTED to any value (for example, CASSANDRA_JDK_UNSUPPORTED=true). Unset the parameter for default behavior"
+    exit 1
+  else
+    echo "######################################################################"
+    echo "Warning! You are using JDK$JAVA_VERSION. This Cassandra version only supports $java_version_string"
+    echo "######################################################################"
+  fi
 fi
-JAVA_VERSION=$short
-
-jvm=`echo "$java_ver_output" | grep -A 1 '[openjdk|java] version' | awk 'NR==2 {print $1}'`
-case "$jvm" in
-    OpenJDK)
-        JVM_VENDOR=OpenJDK
-        # this will be "64-Bit" or "32-Bit"
-        JVM_ARCH=`echo "$java_ver_output" | awk 'NR==3 {print $2}'`
-        ;;
-    "Java(TM)")
-        JVM_VENDOR=Oracle
-        # this will be "64-Bit" or "32-Bit"
-        JVM_ARCH=`echo "$java_ver_output" | awk 'NR==3 {print $3}'`
-        ;;
-    *)
-        # Help fill in other JVM values
-        JVM_VENDOR=other
-        JVM_ARCH=unknown
-        ;;
-esac
 
 # Read user-defined JVM options from jvm-server.options file
 JVM_OPTS_FILE=$CASSANDRA_CONF/jvm${jvmoptions_variant:--clients}.options
-if [ $JAVA_VERSION -ge 17 ] ; then
+if [ $JAVA_VERSION -ge 21 ] ; then
+    JVM_DEP_OPTS_FILE=$CASSANDRA_CONF/jvm21${jvmoptions_variant:--clients}.options
+elif [ $JAVA_VERSION -ge 17 ] ; then
     JVM_DEP_OPTS_FILE=$CASSANDRA_CONF/jvm17${jvmoptions_variant:--clients}.options
 elif [ $JAVA_VERSION -ge 11 ] ; then
     JVM_DEP_OPTS_FILE=$CASSANDRA_CONF/jvm11${jvmoptions_variant:--clients}.options