From 65e1ff5bab6c7d65a3229aa97bcac8b2033e6a95 Mon Sep 17 00:00:00 2001
From: Haoyang Li <haoyangl@nvidia.com>
Date: Thu, 26 Feb 2026 15:15:33 +0800
Subject: [PATCH 01/33] from_protobuf

Signed-off-by: Haoyang Li <haoyangl@nvidia.com>
---
 integration_tests/pom.xml                     |    2 +-
 integration_tests/run_pyspark_from_build.sh   |   69 +-
 integration_tests/src/main/python/data_gen.py |  402 ++-
 .../src/main/python/protobuf_test.py          | 2200 +++++++++++++++++
 .../src/main/python/spark_init_internal.py    |   29 +-
 .../protobuf_test/gen_nested_proto_data.sh    |   34 +
 .../nested_proto/device_req.proto             |    9 +
 .../nested_proto/generated/main_log.desc      |  Bin 0 -> 5838 bytes
 .../protobuf_test/nested_proto/main_log.proto |  101 +
 .../nested_proto/module_a_res.proto           |   90 +
 .../nested_proto/module_b_res.proto           |   27 +
 .../nested_proto/predictor_schema.proto       |   80 +
 pom.xml                                       |    2 +-
 .../nvidia/spark/rapids/GpuOverrides.scala    |    6 +-
 .../com/nvidia/spark/rapids/TypeChecks.scala  |    6 -
 .../protobuf/ProtobufDescriptorUtils.scala    |   87 +
 .../spark/sql/rapids/GpuFromProtobuf.scala    |  384 +++
 .../sql/rapids/complexTypeExtractors.scala    |   22 +-
 .../rapids/shims/ProtobufExprShims.scala      | 1013 ++++++++
 .../rapids/shims/Spark340PlusNonDBShims.scala |    4 +-
 20 files changed, 4546 insertions(+), 21 deletions(-)
 create mode 100644 integration_tests/src/main/python/protobuf_test.py
 create mode 100755 integration_tests/src/test/resources/protobuf_test/gen_nested_proto_data.sh
 create mode 100644 integration_tests/src/test/resources/protobuf_test/nested_proto/device_req.proto
 create mode 100644 integration_tests/src/test/resources/protobuf_test/nested_proto/generated/main_log.desc
 create mode 100644 integration_tests/src/test/resources/protobuf_test/nested_proto/main_log.proto
 create mode 100644 integration_tests/src/test/resources/protobuf_test/nested_proto/module_a_res.proto
 create mode 100644 integration_tests/src/test/resources/protobuf_test/nested_proto/module_b_res.proto
 create mode 100644 integration_tests/src/test/resources/protobuf_test/nested_proto/predictor_schema.proto
 create mode 100644 sql-plugin/src/main/scala/com/nvidia/spark/rapids/protobuf/ProtobufDescriptorUtils.scala
 create mode 100644 sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuFromProtobuf.scala
 create mode 100644 sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala

diff --git a/integration_tests/pom.xml b/integration_tests/pom.xml
index e3d91be0ce3..d8ee2301e5b 100644
--- a/integration_tests/pom.xml
+++ b/integration_tests/pom.xml
@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <!--
-  Copyright (c) 2020-2025, NVIDIA CORPORATION.
+  Copyright (c) 2020-2026, NVIDIA CORPORATION.
 
   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
diff --git a/integration_tests/run_pyspark_from_build.sh b/integration_tests/run_pyspark_from_build.sh
index 6550a3cc59f..a8e10e368a6 100755
--- a/integration_tests/run_pyspark_from_build.sh
+++ b/integration_tests/run_pyspark_from_build.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright (c) 2020-2025, NVIDIA CORPORATION.
+# Copyright (c) 2020-2026, NVIDIA CORPORATION.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -46,6 +46,9 @@
 #   To run all tests, including Avro tests:
 #     INCLUDE_SPARK_AVRO_JAR=true ./run_pyspark_from_build.sh
 #
+#   To run tests WITHOUT Protobuf tests (protobuf is included by default):
+#     INCLUDE_SPARK_PROTOBUF_JAR=false ./run_pyspark_from_build.sh
+#
 #   To run a specific test:
 #     TEST=my_test ./run_pyspark_from_build.sh
 #
@@ -141,9 +144,68 @@ else
         AVRO_JARS=""
     fi
 
-    # ALL_JARS includes dist.jar integration-test.jar avro.jar parquet.jar if they exist
+    # Protobuf support: Include spark-protobuf jar by default for protobuf_test.py
+    # Set INCLUDE_SPARK_PROTOBUF_JAR=false to disable
+    PROTOBUF_JARS=""
+    if [[ $( echo ${INCLUDE_SPARK_PROTOBUF_JAR} | tr '[:upper:]' '[:lower:]' ) != "false" ]];
+    then
+        export INCLUDE_SPARK_PROTOBUF_JAR=true
+        mkdir -p "${TARGET_DIR}/dependency"
+        
+        # Download spark-protobuf jar if not already in target/dependency
+        PROTOBUF_JAR_NAME="spark-protobuf_${SCALA_VERSION}-${VERSION_STRING}.jar"
+        PROTOBUF_JAR_PATH="${TARGET_DIR}/dependency/${PROTOBUF_JAR_NAME}"
+        
+        if [[ ! -f "$PROTOBUF_JAR_PATH" ]]; then
+            echo "Downloading spark-protobuf jar..."
+            PROTOBUF_MAVEN_URL="https://repo1.maven.org/maven2/org/apache/spark/spark-protobuf_${SCALA_VERSION}/${VERSION_STRING}/${PROTOBUF_JAR_NAME}"
+            if curl -sL -o "$PROTOBUF_JAR_PATH" "$PROTOBUF_MAVEN_URL"; then
+                echo "Downloaded spark-protobuf jar to $PROTOBUF_JAR_PATH"
+            else
+                echo "WARNING: Failed to download spark-protobuf jar from $PROTOBUF_MAVEN_URL"
+                rm -f "$PROTOBUF_JAR_PATH"
+            fi
+        fi
+        
+        # Also download protobuf-java jar (required dependency)
+        # Spark 3.5.x uses protobuf-java 3.25.1
+        PROTOBUF_JAVA_VERSION="3.25.1"
+        PROTOBUF_JAVA_JAR_NAME="protobuf-java-${PROTOBUF_JAVA_VERSION}.jar"
+        PROTOBUF_JAVA_JAR_PATH="${TARGET_DIR}/dependency/${PROTOBUF_JAVA_JAR_NAME}"
+        
+        if [[ ! -f "$PROTOBUF_JAVA_JAR_PATH" ]]; then
+            echo "Downloading protobuf-java jar..."
+            PROTOBUF_JAVA_MAVEN_URL="https://repo1.maven.org/maven2/com/google/protobuf/protobuf-java/${PROTOBUF_JAVA_VERSION}/${PROTOBUF_JAVA_JAR_NAME}"
+            if curl -sL -o "$PROTOBUF_JAVA_JAR_PATH" "$PROTOBUF_JAVA_MAVEN_URL"; then
+                echo "Downloaded protobuf-java jar to $PROTOBUF_JAVA_JAR_PATH"
+            else
+                echo "WARNING: Failed to download protobuf-java jar from $PROTOBUF_JAVA_MAVEN_URL"
+                rm -f "$PROTOBUF_JAVA_JAR_PATH"
+            fi
+        fi
+        
+        if [[ -f "$PROTOBUF_JAR_PATH" ]]; then
+            PROTOBUF_JARS="$PROTOBUF_JAR_PATH"
+            echo "Including spark-protobuf jar: $PROTOBUF_JAR_PATH"
+        fi
+        if [[ -f "$PROTOBUF_JAVA_JAR_PATH" ]]; then
+            PROTOBUF_JARS="$PROTOBUF_JARS $PROTOBUF_JAVA_JAR_PATH"
+            echo "Including protobuf-java jar: $PROTOBUF_JAVA_JAR_PATH"
+        fi
+        # Also add protobuf jars to driver classpath for Class.forName() to work
+        # This is needed because --jars only adds to executor classpath
+        if [[ -n "$PROTOBUF_JARS" ]]; then
+            PROTOBUF_DRIVER_CP=$(echo $PROTOBUF_JARS | tr ' ' ':')
+            export PYSP_TEST_spark_driver_extraClassPath="${PYSP_TEST_spark_driver_extraClassPath:+${PYSP_TEST_spark_driver_extraClassPath}:}${PROTOBUF_DRIVER_CP}"
+            echo "Added protobuf jars to driver classpath"
+        fi
+    else
+        export INCLUDE_SPARK_PROTOBUF_JAR=false
+    fi
+
+    # ALL_JARS includes dist.jar integration-test.jar avro.jar parquet.jar protobuf.jar if they exist
     # Remove non-existing paths and canonicalize the paths including get rid of links and `..`
-    ALL_JARS=$(readlink -e $PLUGIN_JAR $TEST_JARS $AVRO_JARS $PARQUET_HADOOP_TESTS || true)
+    ALL_JARS=$(readlink -e $PLUGIN_JAR $TEST_JARS $AVRO_JARS $PARQUET_HADOOP_TESTS $PROTOBUF_JARS || true)
     # `:` separated jars
     ALL_JARS="${ALL_JARS//$'\n'/:}"
 
@@ -404,6 +466,7 @@ else
       export PYSP_TEST_spark_gluten_loadLibFromJar=true
     fi
 
+
     SPARK_SHELL_SMOKE_TEST="${SPARK_SHELL_SMOKE_TEST:-0}"
     EXPLAIN_ONLY_CPU_SMOKE_TEST="${EXPLAIN_ONLY_CPU_SMOKE_TEST:-0}"
     SPARK_CONNECT_SMOKE_TEST="${SPARK_CONNECT_SMOKE_TEST:-0}"
diff --git a/integration_tests/src/main/python/data_gen.py b/integration_tests/src/main/python/data_gen.py
index fa7decac82d..b2d285b1af3 100644
--- a/integration_tests/src/main/python/data_gen.py
+++ b/integration_tests/src/main/python/data_gen.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2020-2025, NVIDIA CORPORATION.
+# Copyright (c) 2020-2026, NVIDIA CORPORATION.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -857,6 +857,406 @@ def gen_bytes():
             return bytes([ rand.randint(0, 255) for _ in range(length) ])
         self._start(rand, gen_bytes)
 
+
+# -----------------------------------------------------------------------------
+# Protobuf (simple types) generators/utilities (for from_protobuf/to_protobuf tests)
+# -----------------------------------------------------------------------------
+
+_PROTOBUF_WIRE_VARINT = 0
+_PROTOBUF_WIRE_64BIT = 1
+_PROTOBUF_WIRE_LEN_DELIM = 2
+_PROTOBUF_WIRE_32BIT = 5
+
+def _encode_protobuf_uvarint(value):
+    """Encode a non-negative integer as protobuf varint."""
+    if value is None:
+        raise ValueError("value must not be None")
+    if value < 0:
+        raise ValueError("uvarint only supports non-negative integers")
+    out = bytearray()
+    v = int(value)
+    while True:
+        b = v & 0x7F
+        v >>= 7
+        if v:
+            out.append(b | 0x80)
+        else:
+            out.append(b)
+            break
+    return bytes(out)
+
+def _encode_protobuf_key(field_number, wire_type):
+    return _encode_protobuf_uvarint((int(field_number) << 3) | int(wire_type))
+
+def _encode_protobuf_zigzag32(value):
+    """Encode a signed 32-bit integer using zigzag encoding (sint32)."""
+    return (value << 1) ^ (value >> 31)
+
+def _encode_protobuf_zigzag64(value):
+    """Encode a signed 64-bit integer using zigzag encoding (sint64)."""
+    return (value << 1) ^ (value >> 63)
+
+def _encode_protobuf_field(field_number, spark_type, value, encoding='default'):
+    """
+    Encode a single protobuf field for a subset of scalar types.
+    
+    encoding: 'default', 'fixed', 'zigzag' - determines how integers are encoded
+    
+    Notes on signed ints:
+    - Protobuf `int32`/`int64` use *varint* encoding of the two's-complement integer.
+    - Negative `int32` values are encoded as a 10-byte varint (because they are sign-extended to 64 bits).
+    - `sint32`/`sint64` use zigzag encoding for efficient negative number storage.
+    - `fixed32`/`fixed64` use fixed-width little-endian encoding.
+    """
+    if value is None:
+        return b""
+
+    if isinstance(spark_type, BooleanType):
+        return _encode_protobuf_key(field_number, _PROTOBUF_WIRE_VARINT) + _encode_protobuf_uvarint(1 if value else 0)
+    elif isinstance(spark_type, IntegerType):
+        if encoding == 'fixed':
+            # fixed32 / sfixed32: 4-byte little-endian (mask handles both signed and unsigned)
+            return _encode_protobuf_key(field_number, _PROTOBUF_WIRE_32BIT) + struct.pack("<I", int(value) & 0xFFFFFFFF)
+        elif encoding == 'zigzag':
+            # sint32: zigzag + varint
+            zigzag = _encode_protobuf_zigzag32(int(value))
+            return _encode_protobuf_key(field_number, _PROTOBUF_WIRE_VARINT) + _encode_protobuf_uvarint(zigzag)
+        else:
+            # int32: varint (negative values are sign-extended)
+            u64 = int(value) & 0xFFFFFFFFFFFFFFFF
+            return _encode_protobuf_key(field_number, _PROTOBUF_WIRE_VARINT) + _encode_protobuf_uvarint(u64)
+    elif isinstance(spark_type, LongType):
+        if encoding == 'fixed':
+            # fixed64 / sfixed64: 8-byte little-endian (mask handles both signed and unsigned)
+            return _encode_protobuf_key(field_number, _PROTOBUF_WIRE_64BIT) + struct.pack("<Q", int(value) & 0xFFFFFFFFFFFFFFFF)
+        elif encoding == 'zigzag':
+            # sint64: zigzag + varint
+            zigzag = _encode_protobuf_zigzag64(int(value))
+            return _encode_protobuf_key(field_number, _PROTOBUF_WIRE_VARINT) + _encode_protobuf_uvarint(zigzag)
+        else:
+            # int64: varint
+            u64 = int(value) & 0xFFFFFFFFFFFFFFFF
+            return _encode_protobuf_key(field_number, _PROTOBUF_WIRE_VARINT) + _encode_protobuf_uvarint(u64)
+    elif isinstance(spark_type, FloatType):
+        return _encode_protobuf_key(field_number, _PROTOBUF_WIRE_32BIT) + struct.pack("<f", float(value))
+    elif isinstance(spark_type, DoubleType):
+        return _encode_protobuf_key(field_number, _PROTOBUF_WIRE_64BIT) + struct.pack("<d", float(value))
+    elif isinstance(spark_type, StringType):
+        b = value.encode("utf-8")
+        return (_encode_protobuf_key(field_number, _PROTOBUF_WIRE_LEN_DELIM) +
+                _encode_protobuf_uvarint(len(b)) + b)
+    elif isinstance(spark_type, BinaryType):
+        # bytes field
+        b = bytes(value) if not isinstance(value, bytes) else value
+        return (_encode_protobuf_key(field_number, _PROTOBUF_WIRE_LEN_DELIM) +
+                _encode_protobuf_uvarint(len(b)) + b)
+    else:
+        raise ValueError("Unsupported type for protobuf simple generator: {}".format(spark_type))
+
+def _encode_protobuf_nested_message(field_number, child_fields, child_values):
+    """
+    Encode a nested protobuf message.
+    
+    child_fields: list of (field_name, field_number, spark_type, encoding)
+    child_values: dict mapping field_name to value
+    """
+    # First encode all child fields
+    child_encoded = b""
+    for (name, num, spark_type, enc) in child_fields:
+        if name in child_values and child_values[name] is not None:
+            child_encoded += _encode_protobuf_field(num, spark_type, child_values[name], enc)
+    
+    # Then wrap in length-delimited format
+    return (_encode_protobuf_key(field_number, _PROTOBUF_WIRE_LEN_DELIM) +
+            _encode_protobuf_uvarint(len(child_encoded)) + child_encoded)
+
+def _encode_protobuf_repeated_field(field_number, spark_element_type, values, encoding='default'):
+    """
+    Encode a repeated (non-packed) protobuf field.
+    Each element is encoded as a separate field with the same field number.
+    """
+    if values is None:
+        return b""
+    result = b""
+    for v in values:
+        if v is not None:
+            result += _encode_protobuf_field(field_number, spark_element_type, v, encoding)
+    return result
+
+def _encode_protobuf_packed_repeated(field_number, spark_element_type, values, encoding='default'):
+    """
+    Encode a packed repeated protobuf field (for numeric types).
+    All elements are packed into a single length-delimited field.
+    """
+    if values is None or len(values) == 0:
+        return b""
+    
+    # Encode all values without keys
+    packed_data = b""
+    for v in values:
+        if v is None:
+            continue
+        if isinstance(spark_element_type, BooleanType):
+            packed_data += _encode_protobuf_uvarint(1 if v else 0)
+        elif isinstance(spark_element_type, IntegerType):
+            if encoding == 'fixed':
+                packed_data += struct.pack("<i", int(v))
+            elif encoding == 'zigzag':
+                packed_data += _encode_protobuf_uvarint(_encode_protobuf_zigzag32(int(v)))
+            else:
+                u64 = int(v) & 0xFFFFFFFFFFFFFFFF
+                packed_data += _encode_protobuf_uvarint(u64)
+        elif isinstance(spark_element_type, LongType):
+            if encoding == 'fixed':
+                packed_data += struct.pack("<q", int(v))
+            elif encoding == 'zigzag':
+                packed_data += _encode_protobuf_uvarint(_encode_protobuf_zigzag64(int(v)))
+            else:
+                u64 = int(v) & 0xFFFFFFFFFFFFFFFF
+                packed_data += _encode_protobuf_uvarint(u64)
+        elif isinstance(spark_element_type, FloatType):
+            packed_data += struct.pack("<f", float(v))
+        elif isinstance(spark_element_type, DoubleType):
+            packed_data += struct.pack("<d", float(v))
+    
+    return (_encode_protobuf_key(field_number, _PROTOBUF_WIRE_LEN_DELIM) +
+            _encode_protobuf_uvarint(len(packed_data)) + packed_data)
+
+
+# ---------------------------------------------------------------------------
+# Protobuf field descriptors
+#
+# Each descriptor pairs a DataGen (for value generation) with protobuf metadata
+# (field number, encoding) and exposes a uniform interface:
+#   spark_field  -- StructField for the Spark schema
+#   start_gen(rand) -- initialise enclosed DataGen instances
+#   gen_value()  -- produce a Spark-compatible value
+#   encode(value) -- serialise the value to protobuf wire-format bytes
+# ---------------------------------------------------------------------------
+
+class PbScalar:
+    """Scalar protobuf field (bool, int32, int64, float, double, string, bytes)."""
+    def __init__(self, name, field_number, gen, encoding='default'):
+        self.name = name
+        self.field_number = field_number
+        self.gen = gen
+        self.encoding = encoding
+
+    @property
+    def spark_field(self):
+        return StructField(self.name, self.gen.data_type, nullable=self.gen.nullable)
+
+    def start_gen(self, rand):
+        self.gen.start(rand)
+
+    def gen_value(self):
+        return self.gen.gen()
+
+    def encode(self, value):
+        return _encode_protobuf_field(self.field_number, self.gen.data_type,
+                                      value, self.encoding)
+
+
+class PbNested:
+    """Non-repeated nested message field (produces a Spark StructType)."""
+    def __init__(self, name, field_number, children):
+        """children: list of PbScalar / PbNested / PbRepeated / PbRepeatedMessage"""
+        self.name = name
+        self.field_number = field_number
+        self.children = children
+
+    @property
+    def spark_field(self):
+        child_fields = [c.spark_field for c in self.children]
+        return StructField(self.name, StructType(child_fields), nullable=True)
+
+    def start_gen(self, rand):
+        for c in self.children:
+            c.start_gen(rand)
+
+    def gen_value(self):
+        child_vals = [c.gen_value() for c in self.children]
+        return tuple(child_vals) if child_vals else None
+
+    def encode(self, value):
+        if value is None:
+            return b""
+        child_encoded = b"".join(
+            c.encode(v) for c, v in zip(self.children, value))
+        return (_encode_protobuf_key(self.field_number, _PROTOBUF_WIRE_LEN_DELIM) +
+                _encode_protobuf_uvarint(len(child_encoded)) + child_encoded)
+
+
+class PbRepeated:
+    """Repeated scalar field (produces a Spark ArrayType of scalars)."""
+    def __init__(self, name, field_number, element_gen, packed=False,
+                 encoding='default', min_len=0, max_len=5):
+        self.name = name
+        self.field_number = field_number
+        self.element_gen = element_gen
+        self.packed = packed
+        self.encoding = encoding
+        self.min_len = min_len
+        self.max_len = max_len
+        self._rand = None
+
+    @property
+    def spark_field(self):
+        return StructField(
+            self.name,
+            ArrayType(self.element_gen.data_type,
+                      containsNull=self.element_gen.nullable),
+            nullable=True)
+
+    def start_gen(self, rand):
+        self._rand = rand
+        self.element_gen.start(rand)
+
+    def gen_value(self):
+        length = self._rand.randint(self.min_len, self.max_len)
+        return [self.element_gen.gen() for _ in range(length)]
+
+    def encode(self, value):
+        if value is None:
+            return b""
+        if self.packed:
+            return _encode_protobuf_packed_repeated(
+                self.field_number, self.element_gen.data_type, value,
+                self.encoding)
+        return _encode_protobuf_repeated_field(
+            self.field_number, self.element_gen.data_type, value,
+            self.encoding)
+
+
+class PbRepeatedMessage:
+    """Repeated nested message field (produces a Spark ArrayType(StructType))."""
+    def __init__(self, name, field_number, children, min_len=0, max_len=5):
+        """children: list of PbScalar / PbNested / PbRepeated / PbRepeatedMessage"""
+        self.name = name
+        self.field_number = field_number
+        self.children = children
+        self.min_len = min_len
+        self.max_len = max_len
+        self._rand = None
+
+    @property
+    def spark_field(self):
+        child_fields = [c.spark_field for c in self.children]
+        return StructField(
+            self.name,
+            ArrayType(StructType(child_fields), containsNull=True),
+            nullable=True)
+
+    def start_gen(self, rand):
+        self._rand = rand
+        for c in self.children:
+            c.start_gen(rand)
+
+    def gen_value(self):
+        length = self._rand.randint(self.min_len, self.max_len)
+        return [tuple(c.gen_value() for c in self.children) for _ in range(length)]
+
+    def encode(self, value):
+        if value is None:
+            return b""
+        parts = []
+        for element in value:
+            if element is not None:
+                child_encoded = b"".join(
+                    c.encode(v) for c, v in zip(self.children, element))
+                parts.append(
+                    _encode_protobuf_key(self.field_number, _PROTOBUF_WIRE_LEN_DELIM) +
+                    _encode_protobuf_uvarint(len(child_encoded)) + child_encoded)
+        return b"".join(parts)
+
+
+# ---------------------------------------------------------------------------
+# Unified protobuf message generator
+# ---------------------------------------------------------------------------
+
+class ProtobufMessageGen(DataGen):
+    """
+    Generate rows containing Spark columns for each protobuf field plus
+    a binary column with the serialised protobuf message.
+
+    Supports any mix of scalar, nested, repeated-scalar, and
+    repeated-message fields at arbitrary nesting depth.
+
+    Usage::
+
+        gen = ProtobufMessageGen([
+            PbScalar("id", 1, IntegerGen()),
+            PbNested("detail", 2, [
+                PbScalar("x", 1, IntegerGen()),
+            ]),
+            PbRepeated("tags", 3, StringGen()),
+            PbRepeatedMessage("items", 4, [
+                PbScalar("name", 1, StringGen()),
+            ], min_len=0, max_len=3),
+        ])
+        df = gen_df(spark, gen, length=100)
+    """
+    def __init__(self, fields, binary_col_name="bin", nullable=False):
+        """
+        fields: list of PbScalar / PbNested / PbRepeated / PbRepeatedMessage
+        """
+        self._pb_fields = fields
+        self._binary_col_name = binary_col_name
+
+        struct_fields = [f.spark_field for f in fields]
+        struct_fields.append(StructField(binary_col_name, BinaryType(), nullable=True))
+        super().__init__(StructType(struct_fields), nullable=nullable)
+
+    def __repr__(self):
+        return "ProtobufMessageGen(fields={})".format(len(self._pb_fields))
+
+    def _cache_repr(self):
+        def _field_repr(f):
+            if isinstance(f, PbScalar):
+                return "S:{}#{}:{}".format(f.name, f.field_number, str(f.gen.data_type))
+            elif isinstance(f, PbNested):
+                kids = ",".join(_field_repr(c) for c in f.children)
+                return "N:{}#{}:[{}]".format(f.name, f.field_number, kids)
+            elif isinstance(f, PbRepeated):
+                return "R:{}#{}:{}".format(f.name, f.field_number,
+                                           str(f.element_gen.data_type))
+            elif isinstance(f, PbRepeatedMessage):
+                kids = ",".join(_field_repr(c) for c in f.children)
+                return "RM:{}#{}:[{}]".format(f.name, f.field_number, kids)
+            return "?:{}".format(f.name)
+        parts = ",".join(_field_repr(f) for f in self._pb_fields)
+        return super()._cache_repr() + "(" + parts + "," + self._binary_col_name + ")"
+
+    def __eq__(self, other):
+        return (isinstance(other, ProtobufMessageGen) and
+                self._cache_repr() == other._cache_repr())
+
+    def __hash__(self):
+        return hash(self._cache_repr())
+
+    def start(self, rand):
+        for f in self._pb_fields:
+            f.start_gen(rand)
+
+        def make_row():
+            values = [f.gen_value() for f in self._pb_fields]
+            msg = b"".join(f.encode(v)
+                           for f, v in zip(self._pb_fields, values))
+            return tuple(values + [msg])
+
+        self._start(rand, make_row)
+
+
+def encode_pb_message(fields, values):
+    """Encode specific values into protobuf binary using field descriptors.
+
+    ``fields`` is a list of PbScalar / PbNested / PbRepeated / PbRepeatedMessage.
+    ``values`` is a parallel list of concrete values (one per field).
+    Pass ``None`` for a value to omit that field from the encoded message.
+    """
+    return b"".join(f.encode(v) for f, v in zip(fields, values))
+
+
 # Note: Current(2023/06/06) maxmium IT data size is 7282688 bytes, so LRU cache with maxsize 128
 # will lead to 7282688 * 128 = 932 MB additional memory usage in edge case, which is acceptable.
 @lru_cache(maxsize=128, typed=True)
diff --git a/integration_tests/src/main/python/protobuf_test.py b/integration_tests/src/main/python/protobuf_test.py
new file mode 100644
index 00000000000..052b38d953d
--- /dev/null
+++ b/integration_tests/src/main/python/protobuf_test.py
@@ -0,0 +1,2200 @@
+# Copyright (c) 2026, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import inspect
+import os
+import struct
+
+import pytest
+
+from asserts import assert_gpu_and_cpu_are_equal_collect
+from data_gen import (
+    BooleanGen, IntegerGen, LongGen, FloatGen, DoubleGen, StringGen, BinaryGen,
+    ProtobufMessageGen, PbScalar, PbNested, PbRepeated, PbRepeatedMessage,
+    encode_pb_message, gen_df, idfn
+)
+from marks import ignore_order
+from spark_session import with_cpu_session, is_before_spark_340
+import pyspark.sql.functions as f
+
+pytestmark = [pytest.mark.premerge_ci_1]
+
+
+# Random data generation configurations for simple scalars
+_random_scalar_test_configs = [
+    # (test_id, data_gen_config)
+    ("all_types", [
+        PbScalar("b", 1, BooleanGen()),
+        PbScalar("i32", 2, IntegerGen()),
+        PbScalar("i64", 3, LongGen()),
+        PbScalar("f32", 4, FloatGen()),
+        PbScalar("f64", 5, DoubleGen()),
+        PbScalar("s", 6, StringGen()),
+    ]),
+    ("integers_edge_cases", [
+        PbScalar("b", 1, BooleanGen()),
+        PbScalar("i32", 2, IntegerGen(
+            min_val=-2147483648, max_val=2147483647,
+            special_cases=[-2147483648, -1, 0, 1, 2147483647])),
+        PbScalar("i64", 3, LongGen(
+            min_val=-9223372036854775808, max_val=9223372036854775807,
+            special_cases=[-9223372036854775808, -1, 0, 1, 9223372036854775807])),
+        PbScalar("f32", 4, FloatGen()),
+        PbScalar("f64", 5, DoubleGen()),
+        PbScalar("s", 6, StringGen()),
+    ]),
+    ("floats_edge_cases", [
+        PbScalar("b", 1, BooleanGen()),
+        PbScalar("i32", 2, IntegerGen()),
+        PbScalar("i64", 3, LongGen()),
+        PbScalar("f32", 4, FloatGen(no_nans=True, special_cases=[-0.0, 0.0, 1.0, -1.0])),
+        PbScalar("f64", 5, DoubleGen(no_nans=True, special_cases=[-0.0, 0.0, 1.0, -1.0])),
+        PbScalar("s", 6, StringGen()),
+    ]),
+    ("large_dataset", [
+        PbScalar("b", 1, BooleanGen()),
+        PbScalar("i32", 2, IntegerGen()),
+        PbScalar("i64", 3, LongGen()),
+        PbScalar("f32", 4, FloatGen()),
+        PbScalar("f64", 5, DoubleGen()),
+        PbScalar("s", 6, StringGen(pattern="[a-z]{0,50}")),
+    ]),
+]
+
+
+def _try_import_from_protobuf():
+    try:
+        from pyspark.sql.protobuf.functions import from_protobuf
+        return from_protobuf
+    except Exception:
+        return None
+
+
+def _spark_protobuf_jvm_available(spark) -> bool:
+    """
+    `spark-protobuf` is an optional external module. PySpark may have the Python wrappers
+    even when the JVM side isn't present on the classpath, which manifests as:
+      TypeError: 'JavaPackage' object is not callable
+    when calling into `sc._jvm.org.apache.spark.sql.protobuf.functions.from_protobuf`.
+    """
+    jvm = spark.sparkContext._jvm
+    candidates = [
+        # Scala object `functions` compiles to `functions$`
+        "org.apache.spark.sql.protobuf.functions$",
+        # Some environments may expose it differently
+        "org.apache.spark.sql.protobuf.functions",
+    ]
+    for cls in candidates:
+        try:
+            jvm.java.lang.Class.forName(cls)
+            return True
+        except Exception:
+            continue
+    return False
+
+
+# ---------------------------------------------------------------------------
+# Shared fixture and helpers to reduce per-test boilerplate
+# ---------------------------------------------------------------------------
+
+@pytest.fixture(scope="module")
+def from_protobuf_fn():
+    """Skip the entire module if from_protobuf or the JVM module is unavailable."""
+    fn = _try_import_from_protobuf()
+    if fn is None:
+        pytest.skip("from_protobuf not available")
+    if not with_cpu_session(_spark_protobuf_jvm_available):
+        pytest.skip("spark-protobuf JVM not available")
+    return fn
+
+
+def _setup_protobuf_desc(spark_tmp_path, desc_name, build_fn):
+    """Build descriptor bytes via JVM, write to HDFS, return (desc_path, desc_bytes)."""
+    desc_path = spark_tmp_path + "/" + desc_name
+    desc_bytes = with_cpu_session(build_fn)
+    with_cpu_session(
+        lambda spark: _write_bytes_to_hadoop_path(spark, desc_path, desc_bytes))
+    return desc_path, desc_bytes
+
+
+def _call_from_protobuf(from_protobuf_fn, col, message_name,
+                         desc_path, desc_bytes, options=None):
+    """Call from_protobuf using the right API variant."""
+    sig = inspect.signature(from_protobuf_fn)
+    if "binaryDescriptorSet" in sig.parameters:
+        kw = dict(binaryDescriptorSet=bytearray(desc_bytes))
+        if options is not None:
+            kw["options"] = options
+        return from_protobuf_fn(col, message_name, **kw)
+    if options is not None and "options" in sig.parameters:
+        return from_protobuf_fn(col, message_name, desc_path, options)
+    return from_protobuf_fn(col, message_name, desc_path)
+
+
+def _build_simple_descriptor_set_bytes(spark):
+    """
+    Build a FileDescriptorSet for:
+      package test;
+      syntax = "proto2";
+      message Simple {
+        optional bool   b   = 1;
+        optional int32  i32 = 2;
+        optional int64  i64 = 3;
+        optional float  f32 = 4;
+        optional double f64 = 5;
+        optional string s   = 6;
+      }
+    """
+    D, fd = _new_proto2_file(spark, "simple.proto")
+
+    msg = D.DescriptorProto.newBuilder().setName("Simple")
+    label_opt = D.FieldDescriptorProto.Label.LABEL_OPTIONAL
+
+    def add_field(name, number, ftype):
+        msg.addField(
+            D.FieldDescriptorProto.newBuilder()
+              .setName(name)
+              .setNumber(number)
+              .setLabel(label_opt)
+              .setType(ftype)
+              .build()
+        )
+
+    add_field("b", 1, D.FieldDescriptorProto.Type.TYPE_BOOL)
+    add_field("i32", 2, D.FieldDescriptorProto.Type.TYPE_INT32)
+    add_field("i64", 3, D.FieldDescriptorProto.Type.TYPE_INT64)
+    add_field("f32", 4, D.FieldDescriptorProto.Type.TYPE_FLOAT)
+    add_field("f64", 5, D.FieldDescriptorProto.Type.TYPE_DOUBLE)
+    add_field("s", 6, D.FieldDescriptorProto.Type.TYPE_STRING)
+
+    fd.addMessageType(msg.build())
+
+    fds = D.FileDescriptorSet.newBuilder().addFile(fd.build()).build()
+    # py4j converts Java byte[] to a Python bytes-like object
+    return bytes(fds.toByteArray())
+
+
+def _write_bytes_to_hadoop_path(spark, path_str, data_bytes):
+    sc = spark.sparkContext
+    config = sc._jsc.hadoopConfiguration()
+    jpath = sc._jvm.org.apache.hadoop.fs.Path(path_str)
+    fs = sc._jvm.org.apache.hadoop.fs.FileSystem.get(config)
+    out = fs.create(jpath, True)
+    try:
+        out.write(bytearray(data_bytes))
+    finally:
+        out.close()
+
+
+def _new_proto2_file(spark, name):
+    """Create a proto2 FileDescriptorProto builder with common defaults."""
+    D = spark.sparkContext._jvm.com.google.protobuf.DescriptorProtos
+    fd = D.FileDescriptorProto.newBuilder() \
+        .setName(name) \
+        .setPackage("test")
+    try:
+        fd = fd.setSyntax("proto2")
+    except Exception:
+        pass
+    return D, fd
+
+
+@pytest.mark.skipif(is_before_spark_340(), reason="from_protobuf is Spark 3.4.0+")
+@ignore_order(local=True)
+def test_from_protobuf_simple_parquet_binary_round_trip(spark_tmp_path, from_protobuf_fn):
+    data_path = spark_tmp_path + "/PROTOBUF_SIMPLE_PARQUET/"
+    message_name = "test.Simple"
+    desc_path, desc_bytes = _setup_protobuf_desc(
+        spark_tmp_path, "simple.desc", _build_simple_descriptor_set_bytes)
+
+    # Build a DF with scalar columns + binary protobuf column and write to parquet
+    row_gen = ProtobufMessageGen([
+        PbScalar("b", 1, BooleanGen(nullable=True)),
+        PbScalar("i32", 2, IntegerGen(nullable=True, min_val=0, max_val=1 << 20)),
+        PbScalar("i64", 3, LongGen(nullable=True, min_val=0, max_val=1 << 40, special_cases=[])),
+        PbScalar("f32", 4, FloatGen(nullable=True, no_nans=True)),
+        PbScalar("f64", 5, DoubleGen(nullable=True, no_nans=True)),
+        PbScalar("s", 6, StringGen(nullable=True)),
+    ], binary_col_name="bin")
+
+    def write_parquet(spark):
+        df = gen_df(spark, row_gen)
+        df.write.mode("overwrite").parquet(data_path)
+
+    with_cpu_session(write_parquet)
+
+    # Sanity check correctness on CPU (decoded struct matches the original scalar columns)
+    def cpu_correctness_check(spark):
+        df = spark.read.parquet(data_path)
+        expected = f.struct(
+            f.col("b").alias("b"),
+            f.col("i32").alias("i32"),
+            f.col("i64").alias("i64"),
+            f.col("f32").alias("f32"),
+            f.col("f64").alias("f64"),
+            f.col("s").alias("s"),
+        ).alias("expected")
+        decoded = _call_from_protobuf(
+            from_protobuf_fn, f.col("bin"), message_name,
+            desc_path, desc_bytes).alias("decoded")
+        rows = df.select(expected, decoded).collect()
+        for r in rows:
+            assert r["expected"] == r["decoded"]
+
+    with_cpu_session(cpu_correctness_check)
+
+    # Main assertion: CPU and GPU results match for from_protobuf on a binary column read from parquet
+    def run_on_spark(spark):
+        df = spark.read.parquet(data_path)
+        decoded = _call_from_protobuf(
+            from_protobuf_fn, f.col("bin"), message_name, desc_path, desc_bytes)
+        return df.select(decoded.alias("decoded"))
+
+    assert_gpu_and_cpu_are_equal_collect(run_on_spark)
+
+
+def _load_nested_proto_desc_resource():
+    """Load the pre-compiled nested proto descriptor file.
+
+    The .desc file is checked into the repository under
+    integration_tests/src/test/resources/protobuf_test/nested_proto/generated/.
+    Returns the raw bytes or None if the file does not exist.
+    """
+    current_dir = os.path.dirname(os.path.abspath(__file__))
+    base_dir = os.path.dirname(os.path.dirname(os.path.dirname(current_dir)))
+    desc_file = os.path.join(base_dir, "src", "test", "resources",
+                             "protobuf_test", "nested_proto", "generated",
+                             "main_log.desc")
+    if not os.path.exists(desc_file):
+        return None
+    with open(desc_file, 'rb') as fp:
+        return fp.read()
+
+
+def _build_main_log_record_fields():
+    """Build PbField tree matching MainLogRecord schema from nested proto files."""
+    u32 = lambda: IntegerGen(min_val=0, max_val=100000)
+    u64 = lambda: LongGen(min_val=0, max_val=(1 << 50))
+
+    type_a_query_schema = [
+        PbScalar("keyword", 1, StringGen()),
+        PbScalar("session_id", 2, StringGen()),
+    ]
+    type_a_pair_schema = [
+        PbScalar("record_id", 1, StringGen()),
+        PbScalar("item_id", 2, StringGen()),
+    ]
+    schema_type_a = [
+        PbNested("query_schema", 1, type_a_query_schema),
+        PbRepeatedMessage("pair_schema", 2, type_a_pair_schema, min_len=0, max_len=3),
+    ]
+
+    type_b_query_schema = [
+        PbScalar("profile_tag_id", 1, StringGen()),
+        PbScalar("entity_id", 2, StringGen()),
+    ]
+    type_b_style_elem = [
+        PbScalar("template_id", 1, StringGen()),
+        PbScalar("material_id", 2, StringGen()),
+    ]
+    type_b_style_schema = [
+        PbRepeatedMessage("values", 1, type_b_style_elem, min_len=0, max_len=3),
+    ]
+    schema_type_b = [
+        PbNested("query_schema", 1, type_b_query_schema),
+        PbRepeatedMessage("style_schema", 2, type_b_style_schema, min_len=0, max_len=3),
+    ]
+
+    type_c_query_schema = [
+        PbScalar("keyword", 1, StringGen()),
+        PbScalar("category", 2, StringGen()),
+    ]
+    type_c_pair_schema = [
+        PbScalar("item_id", 1, StringGen()),
+        PbScalar("target_url", 2, StringGen()),
+    ]
+    type_c_style_schema = [
+        PbRepeatedMessage("values", 1, [], min_len=0, max_len=3),
+    ]
+    schema_type_c = [
+        PbNested("query_schema", 1, type_c_query_schema),
+        PbRepeatedMessage("pair_schema", 2, type_c_pair_schema, min_len=0, max_len=3),
+        PbRepeatedMessage("style_schema", 3, type_c_style_schema, min_len=0, max_len=3),
+    ]
+    predictor_schema = [
+        PbNested("type_a_schema", 1, schema_type_a),
+        PbNested("type_b_schema", 2, schema_type_b),
+        PbNested("type_c_schema", 3, schema_type_c),
+    ]
+
+    device_req_field = [
+        PbScalar("os_type", 1, IntegerGen()),
+        PbScalar("device_id", 2, BinaryGen(min_length=0, max_length=16)),
+    ]
+    partner_info = [
+        PbScalar("token", 1, StringGen()),
+        PbScalar("partner_id", 2, u64()),
+    ]
+    coordinate = [
+        PbScalar("x", 1, DoubleGen()),
+        PbScalar("y", 2, DoubleGen()),
+    ]
+    location_point = [
+        PbScalar("frequency", 1, u32()),
+        PbNested("coord", 2, coordinate),
+        PbScalar("timestamp", 3, u64()),
+    ]
+    change_log = [
+        PbScalar("value_before", 1, u32()),
+        PbScalar("parameters", 2, StringGen()),
+    ]
+    kv_pair = [
+        PbScalar("key", 1, BinaryGen(min_length=0, max_length=16)),
+        PbScalar("value", 2, BinaryGen(min_length=0, max_length=16)),
+    ]
+    style_config = [
+        PbScalar("style_id", 1, u32()),
+        PbRepeatedMessage("kv_pairs", 2, kv_pair, min_len=0, max_len=3),
+    ]
+    module_a_res = [
+        PbScalar("route_tag", 1, StringGen()),
+        PbScalar("status_tag", 2, IntegerGen()),
+        PbScalar("region_id", 3, u32()),
+        PbRepeated("experiment_ids", 4, StringGen(), packed=False, min_len=0, max_len=3),
+        PbScalar("quality_score", 5, DoubleGen()),
+        PbRepeatedMessage("location_points", 6, location_point, min_len=0, max_len=3),
+        PbRepeated("interest_ids", 7, u64(), packed=False, min_len=0, max_len=3),
+    ]
+    module_a_src_res = [
+        PbScalar("match_type", 1, u32()),
+    ]
+    module_a_detail = [
+        PbScalar("type_code", 1, u32()),
+        PbScalar("item_id", 2, u64()),
+        PbScalar("strategy_type", 3, IntegerGen()),
+        PbScalar("min_value", 4, LongGen()),
+        PbScalar("target_url", 5, BinaryGen(min_length=0, max_length=24)),
+        PbScalar("title", 6, StringGen()),
+        PbScalar("is_valid", 7, BooleanGen()),
+        PbScalar("score_ratio", 8, FloatGen()),
+        PbRepeated("template_ids", 9, u32(), packed=False, min_len=0, max_len=3),
+        PbRepeated("material_ids", 10, u64(), packed=False, min_len=0, max_len=3),
+        PbRepeatedMessage("styles", 11, style_config, min_len=0, max_len=3),
+        PbRepeatedMessage("change_logs", 12, change_log, min_len=0, max_len=3),
+        PbNested("partner_info", 13, partner_info),
+        PbNested("predictor_schema", 14, predictor_schema),
+    ]
+
+    block_element = [
+        PbScalar("element_id", 1, u64()),
+        PbRepeated("ref_ids", 2, u64(), packed=False, min_len=0, max_len=3),
+    ]
+    block_info = [
+        PbScalar("block_id", 1, u64()),
+        PbRepeatedMessage("elements", 2, block_element, min_len=0, max_len=3),
+    ]
+    module_b_detail = [
+        PbRepeated("tags", 1, u32(), packed=False, min_len=0, max_len=3),
+        PbScalar("item_id", 2, u64()),
+        PbScalar("name", 3, StringGen()),
+        PbRepeatedMessage("blocks", 4, block_info, min_len=0, max_len=3),
+    ]
+
+    request_info = [
+        PbScalar("page_num", 1, u32()),
+        PbScalar("channel_code", 2, StringGen()),
+        PbRepeated("experiment_ids", 3, u32(), packed=False, min_len=0, max_len=3),
+        PbScalar("is_filtered", 4, BooleanGen()),
+    ]
+    extended_req_info = [
+        PbNested("device_req_field", 1, device_req_field),
+    ]
+    server_added_field = [
+        PbScalar("region_code", 1, u32()),
+        PbScalar("flow_type", 2, StringGen()),
+        PbScalar("filter_result", 3, IntegerGen()),
+        PbRepeated("hit_rule_list", 4, IntegerGen(), packed=False, min_len=0, max_len=3),
+        PbScalar("request_time", 5, u64()),
+        PbScalar("skip_flag", 6, BooleanGen()),
+    ]
+    basic_info = [
+        PbNested("request_info", 1, request_info),
+        PbNested("extended_req_info", 2, extended_req_info),
+        PbNested("server_added_field", 3, server_added_field),
+    ]
+
+    channel_info = [
+        PbScalar("channel_id", 1, IntegerGen()),
+        PbNested("module_a_res", 2, module_a_res),
+    ]
+    src_channel_info = [
+        PbScalar("channel_id", 1, IntegerGen()),
+        PbNested("module_a_src_res", 2, module_a_src_res),
+    ]
+    item_detail_field = [
+        PbScalar("rank", 1, u32()),
+        PbScalar("record_id", 2, u64()),
+        PbScalar("keyword", 3, StringGen()),
+        PbNested("module_a_detail", 4, module_a_detail),
+        PbNested("module_b_detail", 5, module_b_detail),
+    ]
+    data_source_field = [
+        PbScalar("source_id", 1, u32()),
+        PbRepeatedMessage("src_channel_list", 2, src_channel_info, min_len=0, max_len=3),
+        PbScalar("billing_name", 3, StringGen()),
+        PbRepeatedMessage("item_list", 4, item_detail_field, min_len=0, max_len=3),
+        PbScalar("is_free", 5, BooleanGen()),
+    ]
+    log_content = [
+        PbNested("basic_info", 1, basic_info),
+        PbRepeatedMessage("channel_list", 2, channel_info, min_len=0, max_len=3),
+        PbRepeatedMessage("source_list", 3, data_source_field, min_len=0, max_len=3),
+    ]
+
+    return [
+        PbScalar(
+            "source", 1,
+            IntegerGen(min_val=0, max_val=1, nullable=False, special_cases=[4])),
+        PbScalar("timestamp", 2, LongGen(min_val=0, max_val=(1 << 50), nullable=False)),
+        PbScalar("user_id", 3, StringGen()),
+        PbScalar("account_id", 4, LongGen()),
+        PbScalar("client_ip", 5, IntegerGen(min_val=0, max_val=0x7FFFFFFF), encoding='fixed'),
+        PbNested("log_content", 6, log_content),
+    ]
+
+
+@pytest.mark.skipif(is_before_spark_340(), reason="from_protobuf is Spark 3.4.0+")
+@pytest.mark.parametrize("options", [None, {"enums.as.ints": "true"}])
+@ignore_order(local=True)
+def test_from_protobuf_customer_heavy_nested_proto(spark_tmp_path, from_protobuf_fn, options):
+    """Integration test with real nested proto: multi-level nesting, cross-file imports, enums."""
+    desc_bytes = _load_nested_proto_desc_resource()
+    if desc_bytes is None:
+        pytest.skip("nested_proto descriptor not found; run gen_nested_proto_data.sh first")
+
+    desc_path = spark_tmp_path + "/main_log.desc"
+    with_cpu_session(lambda spark: _write_bytes_to_hadoop_path(spark, desc_path, desc_bytes))
+
+    message_name = "com.test.proto.sample.MainLogRecord"
+    data_gen = ProtobufMessageGen(_build_main_log_record_fields())
+
+    def run_on_spark(spark):
+        generated = gen_df(spark, data_gen).select("bin")
+        decoded = _call_from_protobuf(
+            from_protobuf_fn, f.col("bin"), message_name, desc_path, desc_bytes,
+            options=options)
+
+        return generated.select(decoded.alias("decoded"))
+
+    assert_gpu_and_cpu_are_equal_collect(run_on_spark)
+
+
+def _build_nested_descriptor_set_bytes(spark):
+    """
+    Build a FileDescriptorSet for a message with both simple fields and nested message:
+      package test;
+      syntax = "proto2";
+      message Nested {
+        optional int32 x = 1;
+      }
+      message WithNested {
+        optional int32  simple_int  = 1;
+        optional string simple_str  = 2;
+        optional Nested nested_msg  = 3;   // nested message
+        optional int64  simple_long = 4;
+      }
+    """
+    D, fd = _new_proto2_file(spark, "nested.proto")
+
+    label_opt = D.FieldDescriptorProto.Label.LABEL_OPTIONAL
+
+    # Define Nested message
+    nested_msg = D.DescriptorProto.newBuilder().setName("Nested")
+    nested_msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("x")
+            .setNumber(1)
+            .setLabel(label_opt)
+            .setType(D.FieldDescriptorProto.Type.TYPE_INT32)
+            .build()
+    )
+    fd.addMessageType(nested_msg.build())
+
+    # Define WithNested message
+    with_nested_msg = D.DescriptorProto.newBuilder().setName("WithNested")
+    # simple_int
+    with_nested_msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("simple_int")
+            .setNumber(1)
+            .setLabel(label_opt)
+            .setType(D.FieldDescriptorProto.Type.TYPE_INT32)
+            .build()
+    )
+    # simple_str
+    with_nested_msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("simple_str")
+            .setNumber(2)
+            .setLabel(label_opt)
+            .setType(D.FieldDescriptorProto.Type.TYPE_STRING)
+            .build()
+    )
+    # nested_msg (nested message type)
+    with_nested_msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("nested_msg")
+            .setNumber(3)
+            .setLabel(label_opt)
+            .setType(D.FieldDescriptorProto.Type.TYPE_MESSAGE)
+            .setTypeName(".test.Nested")
+            .build()
+    )
+    # simple_long
+    with_nested_msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("simple_long")
+            .setNumber(4)
+            .setLabel(label_opt)
+            .setType(D.FieldDescriptorProto.Type.TYPE_INT64)
+            .build()
+    )
+    fd.addMessageType(with_nested_msg.build())
+
+    fds = D.FileDescriptorSet.newBuilder().addFile(fd.build()).build()
+    return bytes(fds.toByteArray())
+
+
+@pytest.mark.skipif(is_before_spark_340(), reason="from_protobuf is Spark 3.4.0+")
+@ignore_order(local=True)
+def test_from_protobuf_schema_projection_simple_fields_only(spark_tmp_path, from_protobuf_fn):
+    """
+    Test schema projection: when only simple fields are selected from a protobuf message
+    that also contains unsupported types (nested message), GPU should be able to decode
+    just the simple fields without falling back to CPU.
+    """
+    desc_path, desc_bytes = _setup_protobuf_desc(
+        spark_tmp_path, "nested.desc", _build_nested_descriptor_set_bytes)
+    message_name = "test.WithNested"
+
+    data_gen = ProtobufMessageGen([
+        PbScalar("simple_int", 1, IntegerGen()),
+        PbScalar("simple_str", 2, StringGen()),
+        PbScalar("simple_long", 4, LongGen()),
+    ])
+
+    def run_on_spark(spark):
+        df = gen_df(spark, data_gen)
+        decoded = _call_from_protobuf(
+            from_protobuf_fn, f.col("bin"), message_name, desc_path, desc_bytes)
+        # Only select simple fields, not the nested_msg field
+        return df.select(
+            decoded.getField("simple_int").alias("simple_int"),
+            decoded.getField("simple_str").alias("simple_str"),
+            decoded.getField("simple_long").alias("simple_long")
+        )
+
+    assert_gpu_and_cpu_are_equal_collect(run_on_spark)
+
+
+def _build_enum_descriptor_set_bytes(spark):
+    """
+    Build a FileDescriptorSet for a message with enum field:
+      package test;
+      syntax = "proto2";
+      message WithEnum {
+        enum Color {
+          RED = 0;
+          GREEN = 1;
+          BLUE = 2;
+        }
+        optional Color color = 1;
+        optional int32 count = 2;
+        optional string name = 3;
+      }
+    """
+    D, fd = _new_proto2_file(spark, "enum.proto")
+
+    label_opt = D.FieldDescriptorProto.Label.LABEL_OPTIONAL
+
+    # Define WithEnum message with nested enum
+    msg = D.DescriptorProto.newBuilder().setName("WithEnum")
+
+    # Add enum type definition
+    enum_type = D.EnumDescriptorProto.newBuilder().setName("Color")
+    enum_type.addValue(D.EnumValueDescriptorProto.newBuilder().setName("RED").setNumber(0).build())
+    enum_type.addValue(D.EnumValueDescriptorProto.newBuilder().setName("GREEN").setNumber(1).build())
+    enum_type.addValue(D.EnumValueDescriptorProto.newBuilder().setName("BLUE").setNumber(2).build())
+    msg.addEnumType(enum_type.build())
+
+    # Add color field (enum type)
+    msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("color")
+            .setNumber(1)
+            .setLabel(label_opt)
+            .setType(D.FieldDescriptorProto.Type.TYPE_ENUM)
+            .setTypeName(".test.WithEnum.Color")
+            .build()
+    )
+    # Add count field (int32)
+    msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("count")
+            .setNumber(2)
+            .setLabel(label_opt)
+            .setType(D.FieldDescriptorProto.Type.TYPE_INT32)
+            .build()
+    )
+    # Add name field (string)
+    msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("name")
+            .setNumber(3)
+            .setLabel(label_opt)
+            .setType(D.FieldDescriptorProto.Type.TYPE_STRING)
+            .build()
+    )
+
+    fd.addMessageType(msg.build())
+    fds = D.FileDescriptorSet.newBuilder().addFile(fd.build()).build()
+    return bytes(fds.toByteArray())
+
+
+@pytest.mark.skipif(is_before_spark_340(), reason="from_protobuf is Spark 3.4.0+")
+@pytest.mark.parametrize("enum_case", [
+    "as_int",
+    "unknown_as_int",
+    "as_string",
+    "unknown_as_string",
+], ids=lambda x: x)
+@ignore_order(local=True)
+def test_from_protobuf_enum_cases(spark_tmp_path, from_protobuf_fn, enum_case):
+    """Parametrized enum decoding tests for int/string modes and unknown values."""
+    desc_path, desc_bytes = _setup_protobuf_desc(
+        spark_tmp_path, "enum.desc", _build_enum_descriptor_set_bytes)
+    message_name = "test.WithEnum"
+
+    _ef_full = [
+        PbScalar("color", 1, IntegerGen()),
+        PbScalar("count", 2, IntegerGen()),
+        PbScalar("name", 3, StringGen()),
+    ]
+    _ef_partial = [PbScalar("color", 1, IntegerGen()), PbScalar("count", 2, IntegerGen())]
+
+    if enum_case == "as_int":
+        rows = [
+            (encode_pb_message(_ef_full, [1, 42, "test"]),),
+            (encode_pb_message(_ef_full, [0, 100, None]),),
+            (encode_pb_message(_ef_full, [None, 200, "hello"]),),
+            (None,),
+        ]
+        options = {"enums.as.ints": "true"}
+        select_mode = "fields3"
+    elif enum_case == "unknown_as_int":
+        rows = [(encode_pb_message(_ef_partial, [999, 42]),)]
+        options = {"enums.as.ints": "true", "mode": "PERMISSIVE"}
+        select_mode = "fields2"
+    elif enum_case == "as_string":
+        rows = [
+            (encode_pb_message(_ef_full, [1, 42, "test"]),),
+            (encode_pb_message(_ef_full, [0, 100, None]),),
+            (encode_pb_message(_ef_full, [2, 200, "hello"]),),
+            (encode_pb_message(_ef_full, [None, 300, "world"]),),
+            (None,),
+        ]
+        options = None
+        select_mode = "fields3"
+    else:
+        rows = [
+            (encode_pb_message(_ef_partial, [1, 10]),),
+            (encode_pb_message(_ef_partial, [999, 20]),),
+            (encode_pb_message(_ef_partial, [2, 30]),),
+        ]
+        options = {"mode": "PERMISSIVE"}
+        select_mode = "decoded"
+
+    def run_on_spark(spark):
+        df = spark.createDataFrame(rows, schema="bin binary")
+        decoded = _call_from_protobuf(
+            from_protobuf_fn, f.col("bin"), message_name, desc_path, desc_bytes,
+            options=options)
+        if select_mode == "fields3":
+            return df.select(
+                decoded.getField("color").alias("color"),
+                decoded.getField("count").alias("count"),
+                decoded.getField("name").alias("name"))
+        if select_mode == "fields2":
+            return df.select(
+                decoded.getField("color").alias("color"),
+                decoded.getField("count").alias("count"))
+        return df.select(decoded.alias("decoded"))
+
+    assert_gpu_and_cpu_are_equal_collect(run_on_spark)
+
+
+def _build_required_field_descriptor_set_bytes(spark):
+    """
+    Build a FileDescriptorSet for a message with required fields (proto2):
+      package test;
+      syntax = "proto2";
+      message WithRequired {
+        required int64 id = 1;
+        optional string name = 2;
+        optional int32 count = 3;
+      }
+    """
+    D, fd = _new_proto2_file(spark, "required.proto")
+
+    label_required = D.FieldDescriptorProto.Label.LABEL_REQUIRED
+    label_optional = D.FieldDescriptorProto.Label.LABEL_OPTIONAL
+
+    msg = D.DescriptorProto.newBuilder().setName("WithRequired")
+    
+    # id field (required)
+    msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("id")
+            .setNumber(1)
+            .setLabel(label_required)
+            .setType(D.FieldDescriptorProto.Type.TYPE_INT64)
+            .build()
+    )
+    # name field (optional)
+    msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("name")
+            .setNumber(2)
+            .setLabel(label_optional)
+            .setType(D.FieldDescriptorProto.Type.TYPE_STRING)
+            .build()
+    )
+    # count field (optional)
+    msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("count")
+            .setNumber(3)
+            .setLabel(label_optional)
+            .setType(D.FieldDescriptorProto.Type.TYPE_INT32)
+            .build()
+    )
+
+    fd.addMessageType(msg.build())
+    fds = D.FileDescriptorSet.newBuilder().addFile(fd.build()).build()
+    return bytes(fds.toByteArray())
+
+
+@pytest.mark.skipif(is_before_spark_340(), reason="from_protobuf is Spark 3.4.0+")
+@ignore_order(local=True)
+def test_from_protobuf_required_field_present(spark_tmp_path, from_protobuf_fn):
+    """
+    Test that required fields decode correctly when present.
+    GPU should produce same results as CPU.
+    """
+    desc_path, desc_bytes = _setup_protobuf_desc(
+        spark_tmp_path, "required.desc", _build_required_field_descriptor_set_bytes)
+    message_name = "test.WithRequired"
+
+    # Create test data with required field present
+    # Row 0: id=100, name="test", count=42
+    # Row 1: id=200, name missing, count missing
+    _rf = [
+        PbScalar("id", 1, LongGen()),
+        PbScalar("name", 2, StringGen()),
+        PbScalar("count", 3, IntegerGen()),
+    ]
+    test_data_row0 = encode_pb_message(_rf, [100, "test", 42])
+    test_data_row1 = encode_pb_message(_rf, [200, None, None])
+
+    def run_on_spark(spark):
+        df = spark.createDataFrame(
+            [(test_data_row0,), (test_data_row1,)],
+            schema="bin binary",
+        )
+        decoded = _call_from_protobuf(
+            from_protobuf_fn, f.col("bin"), message_name, desc_path, desc_bytes)
+        return df.select(
+            decoded.getField("id").alias("id"),
+            decoded.getField("name").alias("name"),
+            decoded.getField("count").alias("count")
+        )
+
+    assert_gpu_and_cpu_are_equal_collect(run_on_spark)
+
+
+def _build_default_value_descriptor_set_bytes(spark):
+    """
+    Build a FileDescriptorSet for a message with default values (proto2):
+      package test;
+      syntax = "proto2";
+      message WithDefaults {
+        optional int32 count = 1 [default = 42];
+        optional string name = 2 [default = "unknown"];
+        optional bool flag = 3 [default = true];
+      }
+    Note: Setting explicit defaults requires using FieldOptions which may not be
+    available via the simple DescriptorProtos API. For testing, we rely on proto2
+    implicit behavior where hasDefaultValue() returns true for optional fields.
+    """
+    D, fd = _new_proto2_file(spark, "defaults.proto")
+
+    label_optional = D.FieldDescriptorProto.Label.LABEL_OPTIONAL
+
+    msg = D.DescriptorProto.newBuilder().setName("WithDefaults")
+    
+    # count field with default
+    count_field = D.FieldDescriptorProto.newBuilder() \
+        .setName("count") \
+        .setNumber(1) \
+        .setLabel(label_optional) \
+        .setType(D.FieldDescriptorProto.Type.TYPE_INT32) \
+        .setDefaultValue("42")
+    msg.addField(count_field.build())
+    
+    # name field with default
+    name_field = D.FieldDescriptorProto.newBuilder() \
+        .setName("name") \
+        .setNumber(2) \
+        .setLabel(label_optional) \
+        .setType(D.FieldDescriptorProto.Type.TYPE_STRING) \
+        .setDefaultValue("unknown")
+    msg.addField(name_field.build())
+    
+    # flag field with default
+    flag_field = D.FieldDescriptorProto.newBuilder() \
+        .setName("flag") \
+        .setNumber(3) \
+        .setLabel(label_optional) \
+        .setType(D.FieldDescriptorProto.Type.TYPE_BOOL) \
+        .setDefaultValue("true")
+    msg.addField(flag_field.build())
+
+    fd.addMessageType(msg.build())
+    fds = D.FileDescriptorSet.newBuilder().addFile(fd.build()).build()
+    return bytes(fds.toByteArray())
+
+
+@pytest.mark.skipif(is_before_spark_340(), reason="from_protobuf is Spark 3.4.0+")
+@pytest.mark.parametrize("default_case", [
+    "field_present",
+    "missing_fields",
+    "string_default_only",
+], ids=lambda x: x)
+@ignore_order(local=True)
+def test_from_protobuf_default_values_cases(spark_tmp_path, from_protobuf_fn, default_case):
+    """Parametrized tests for proto2 default-value behavior."""
+    desc_path, desc_bytes = _setup_protobuf_desc(
+        spark_tmp_path, "defaults.desc", _build_default_value_descriptor_set_bytes)
+    message_name = "test.WithDefaults"
+
+    _df = [
+        PbScalar("count", 1, IntegerGen()),
+        PbScalar("name", 2, StringGen()),
+        PbScalar("flag", 3, BooleanGen()),
+    ]
+
+    if default_case == "field_present":
+        rows = [(encode_pb_message(_df, [100, "test", False]),)]
+        select_mode = "all"
+    elif default_case == "missing_fields":
+        rows = [
+            (encode_pb_message(_df, [None, None, None]),),
+            (encode_pb_message(_df, [100, None, None]),),
+        ]
+        select_mode = "all"
+    else:
+        rows = [(encode_pb_message(_df, [42, None, True]),)]
+        select_mode = "name_only"
+
+    def run_on_spark(spark):
+        df = spark.createDataFrame(rows, schema="bin binary")
+        decoded = _call_from_protobuf(
+            from_protobuf_fn, f.col("bin"), message_name, desc_path, desc_bytes)
+        if select_mode == "all":
+            return df.select(
+                decoded.getField("count").alias("count"),
+                decoded.getField("name").alias("name"),
+                decoded.getField("flag").alias("flag"))
+        return df.select(decoded.getField("name").alias("name"))
+
+    assert_gpu_and_cpu_are_equal_collect(run_on_spark)
+
+
+def _build_all_scalars_descriptor_set_bytes(spark):
+    """
+    Build a FileDescriptorSet for message with all scalar types:
+      message AllScalars {
+        optional bool b = 1;
+        optional int32 i32 = 2;
+        optional int64 i64 = 3;
+        optional float f32 = 4;
+        optional double f64 = 5;
+        optional string s = 6;
+        optional sint32 si32 = 7;  // zigzag encoding
+        optional sint64 si64 = 8;  // zigzag encoding
+        optional fixed32 fx32 = 9;
+        optional fixed64 fx64 = 10;
+      }
+    """
+    D, fd = _new_proto2_file(spark, "all_scalars.proto")
+
+    msg = D.DescriptorProto.newBuilder().setName("AllScalars")
+    label_opt = D.FieldDescriptorProto.Label.LABEL_OPTIONAL
+
+    def add_field(name, number, ftype):
+        msg.addField(
+            D.FieldDescriptorProto.newBuilder()
+              .setName(name)
+              .setNumber(number)
+              .setLabel(label_opt)
+              .setType(ftype)
+              .build()
+        )
+
+    T = D.FieldDescriptorProto.Type
+    add_field("b", 1, T.TYPE_BOOL)
+    add_field("i32", 2, T.TYPE_INT32)
+    add_field("i64", 3, T.TYPE_INT64)
+    add_field("f32", 4, T.TYPE_FLOAT)
+    add_field("f64", 5, T.TYPE_DOUBLE)
+    add_field("s", 6, T.TYPE_STRING)
+    add_field("si32", 7, T.TYPE_SINT32)
+    add_field("si64", 8, T.TYPE_SINT64)
+    add_field("fx32", 9, T.TYPE_FIXED32)
+    add_field("fx64", 10, T.TYPE_FIXED64)
+
+    fd.addMessageType(msg.build())
+    fds = D.FileDescriptorSet.newBuilder().addFile(fd.build()).build()
+    return bytes(fds.toByteArray())
+
+
+def _build_scalar_bytes_descriptor_set_bytes(spark):
+    """
+    Build a FileDescriptorSet for:
+      message ScalarBytes {
+        optional bytes payload = 1;
+      }
+    """
+    D, fd = _new_proto2_file(spark, "scalar_bytes.proto")
+    label_opt = D.FieldDescriptorProto.Label.LABEL_OPTIONAL
+
+    msg = D.DescriptorProto.newBuilder().setName("ScalarBytes")
+    msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("payload")
+            .setNumber(1)
+            .setLabel(label_opt)
+            .setType(D.FieldDescriptorProto.Type.TYPE_BYTES)
+            .build()
+    )
+    fd.addMessageType(msg.build())
+
+    fds = D.FileDescriptorSet.newBuilder().addFile(fd.build()).build()
+    return bytes(fds.toByteArray())
+
+
+def _scalar_test_id(config):
+    """Generate stable test ID using only the first element (test name)."""
+    return config[0] if isinstance(config, tuple) else str(config)
+
+
+@pytest.mark.skipif(is_before_spark_340(), reason="from_protobuf is Spark 3.4.0+")
+@pytest.mark.parametrize("test_config", _random_scalar_test_configs, ids=_scalar_test_id)
+@ignore_order(local=True)
+def test_from_protobuf_random_scalars(spark_tmp_path, from_protobuf_fn, test_config):
+    """
+    Parametrized test for from_protobuf with randomly generated scalar data.
+    Covers: all types, integer edge cases, float edge cases, large datasets.
+    """
+    test_id, field_configs = test_config
+
+    desc_path, desc_bytes = _setup_protobuf_desc(
+        spark_tmp_path, "simple.desc", _build_simple_descriptor_set_bytes)
+    message_name = "test.Simple"
+
+    data_gen = ProtobufMessageGen(field_configs)
+
+    def run_on_spark(spark):
+        df = gen_df(spark, data_gen)
+
+        decoded = _call_from_protobuf(
+            from_protobuf_fn, f.col("bin"), message_name, desc_path, desc_bytes)
+
+        # Select all decoded fields
+        return df.select(
+            decoded.getField("b").alias("b"),
+            decoded.getField("i32").alias("i32"),
+            decoded.getField("i64").alias("i64"),
+            decoded.getField("f32").alias("f32"),
+            decoded.getField("f64").alias("f64"),
+            decoded.getField("s").alias("s"),
+        )
+
+    assert_gpu_and_cpu_are_equal_collect(run_on_spark)
+
+
+@pytest.mark.skipif(is_before_spark_340(), reason="from_protobuf is Spark 3.4.0+")
+@ignore_order(local=True)
+def test_from_protobuf_all_scalar_types(spark_tmp_path, from_protobuf_fn):
+    """Decode all scalar wire encodings together in one message."""
+    desc_path, desc_bytes = _setup_protobuf_desc(
+        spark_tmp_path, "all_scalars.desc", _build_all_scalars_descriptor_set_bytes)
+    message_name = "test.AllScalars"
+
+    data_gen = ProtobufMessageGen([
+        PbScalar("b", 1, BooleanGen()),
+        PbScalar("i32", 2, IntegerGen()),
+        PbScalar("i64", 3, LongGen()),
+        PbScalar("f32", 4, FloatGen()),
+        PbScalar("f64", 5, DoubleGen()),
+        PbScalar("s", 6, StringGen()),
+        PbScalar("si32", 7, IntegerGen(
+            special_cases=[-1, 0, 1, -2147483648, 2147483647]), encoding='zigzag'),
+        PbScalar("si64", 8, LongGen(
+            special_cases=[-1, 0, 1, -9223372036854775808, 9223372036854775807]),
+            encoding='zigzag'),
+        PbScalar("fx32", 9, IntegerGen(
+            special_cases=[0, 1, -1, 2147483647, -2147483648]), encoding='fixed'),
+        PbScalar("fx64", 10, LongGen(
+            special_cases=[0, 1, -1]), encoding='fixed'),
+    ])
+
+    def run_on_spark(spark):
+        df = gen_df(spark, data_gen)
+        decoded = _call_from_protobuf(
+            from_protobuf_fn, f.col("bin"), message_name, desc_path, desc_bytes)
+        return df.select(
+            decoded.getField("b").alias("b"),
+            decoded.getField("i32").alias("i32"),
+            decoded.getField("i64").alias("i64"),
+            decoded.getField("f32").alias("f32"),
+            decoded.getField("f64").alias("f64"),
+            decoded.getField("s").alias("s"),
+            decoded.getField("si32").alias("si32"),
+            decoded.getField("si64").alias("si64"),
+            decoded.getField("fx32").alias("fx32"),
+            decoded.getField("fx64").alias("fx64"),
+        )
+
+    assert_gpu_and_cpu_are_equal_collect(run_on_spark)
+
+
+@pytest.mark.skipif(is_before_spark_340(), reason="from_protobuf is Spark 3.4.0+")
+@ignore_order(local=True)
+def test_from_protobuf_scalar_bytes(spark_tmp_path, from_protobuf_fn):
+    """Decode optional bytes scalar field, including empty bytes."""
+    desc_path, desc_bytes = _setup_protobuf_desc(
+        spark_tmp_path, "scalar_bytes.desc", _build_scalar_bytes_descriptor_set_bytes)
+    message_name = "test.ScalarBytes"
+
+    data_gen = ProtobufMessageGen([
+        PbScalar("payload", 1, BinaryGen(min_length=0, max_length=16)),
+    ])
+
+    def run_on_spark(spark):
+        df = gen_df(spark, data_gen)
+        decoded = _call_from_protobuf(
+            from_protobuf_fn, f.col("bin"), message_name, desc_path, desc_bytes)
+        return df.select(decoded.getField("payload").alias("payload"))
+
+    assert_gpu_and_cpu_are_equal_collect(run_on_spark)
+
+
+@pytest.mark.skipif(is_before_spark_340(), reason="from_protobuf is Spark 3.4.0+")
+@ignore_order(local=True)
+def test_from_protobuf_duplicate_fields(spark_tmp_path, from_protobuf_fn):
+    """Duplicate non-repeated field should follow protobuf last-one-wins semantics."""
+    desc_path, desc_bytes = _setup_protobuf_desc(
+        spark_tmp_path, "simple.desc", _build_simple_descriptor_set_bytes)
+    message_name = "test.Simple"
+
+    # i32 (field 2) appears twice in row0 and three times in row1.
+    test_data_row0 = bytes([0x10, 0x01, 0x10, 0x2A])
+    test_data_row1 = bytes([0x10, 0x03, 0x10, 0x04, 0x10, 0x05])
+
+    def run_on_spark(spark):
+        df = spark.createDataFrame(
+            [(test_data_row0,), (test_data_row1,), (None,)],
+            schema="bin binary")
+        decoded = _call_from_protobuf(
+            from_protobuf_fn, f.col("bin"), message_name, desc_path, desc_bytes)
+        return df.select(decoded.getField("i32").alias("i32"))
+
+    assert_gpu_and_cpu_are_equal_collect(run_on_spark)
+
+
+def _build_repeated_int_descriptor_set_bytes(spark):
+    """
+    Build a FileDescriptorSet for a message with repeated int32 field:
+      message WithRepeatedInt {
+        optional int32 id = 1;
+        repeated int32 values = 2;
+      }
+    """
+    D, fd = _new_proto2_file(spark, "repeated_int.proto")
+
+    label_opt = D.FieldDescriptorProto.Label.LABEL_OPTIONAL
+    label_rep = D.FieldDescriptorProto.Label.LABEL_REPEATED
+
+    msg = D.DescriptorProto.newBuilder().setName("WithRepeatedInt")
+    msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("id")
+            .setNumber(1)
+            .setLabel(label_opt)
+            .setType(D.FieldDescriptorProto.Type.TYPE_INT32)
+            .build()
+    )
+    msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("values")
+            .setNumber(2)
+            .setLabel(label_rep)
+            .setType(D.FieldDescriptorProto.Type.TYPE_INT32)
+            .build()
+    )
+    fd.addMessageType(msg.build())
+
+    fds = D.FileDescriptorSet.newBuilder().addFile(fd.build()).build()
+    return bytes(fds.toByteArray())
+
+
+@pytest.mark.skipif(is_before_spark_340(), reason="from_protobuf is Spark 3.4.0+")
+@ignore_order(local=True)
+def test_from_protobuf_repeated_int32(spark_tmp_path, from_protobuf_fn):
+    """
+    Test decoding repeated int32 field (ArrayType of integers).
+    """
+    desc_path, desc_bytes = _setup_protobuf_desc(
+        spark_tmp_path, "repeated_int.desc", _build_repeated_int_descriptor_set_bytes)
+    message_name = "test.WithRepeatedInt"
+
+    data_gen = ProtobufMessageGen([
+        PbScalar("id", 1, IntegerGen()),
+        PbRepeated("values", 2, IntegerGen(), min_len=0, max_len=10),
+    ])
+
+    def run_on_spark(spark):
+        df = gen_df(spark, data_gen)
+        decoded = _call_from_protobuf(
+            from_protobuf_fn, f.col("bin"), message_name, desc_path, desc_bytes)
+        return df.select(
+            decoded.getField("id").alias("id"),
+            decoded.getField("values").alias("values"),
+        )
+
+    assert_gpu_and_cpu_are_equal_collect(run_on_spark)
+
+
+def _build_repeated_string_descriptor_set_bytes(spark):
+    """
+    Build a FileDescriptorSet for a message with repeated string field:
+      message WithRepeatedString {
+        optional string name = 1;
+        repeated string tags = 2;
+      }
+    """
+    D, fd = _new_proto2_file(spark, "repeated_string.proto")
+
+    label_opt = D.FieldDescriptorProto.Label.LABEL_OPTIONAL
+    label_rep = D.FieldDescriptorProto.Label.LABEL_REPEATED
+
+    msg = D.DescriptorProto.newBuilder().setName("WithRepeatedString")
+    msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("name")
+            .setNumber(1)
+            .setLabel(label_opt)
+            .setType(D.FieldDescriptorProto.Type.TYPE_STRING)
+            .build()
+    )
+    msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("tags")
+            .setNumber(2)
+            .setLabel(label_rep)
+            .setType(D.FieldDescriptorProto.Type.TYPE_STRING)
+            .build()
+    )
+    fd.addMessageType(msg.build())
+
+    fds = D.FileDescriptorSet.newBuilder().addFile(fd.build()).build()
+    return bytes(fds.toByteArray())
+
+
+@pytest.mark.skipif(is_before_spark_340(), reason="from_protobuf is Spark 3.4.0+")
+@ignore_order(local=True)
+def test_from_protobuf_repeated_string(spark_tmp_path, from_protobuf_fn):
+    """
+    Test decoding repeated string field (ArrayType of strings).
+    """
+    desc_path, desc_bytes = _setup_protobuf_desc(
+        spark_tmp_path, "repeated_string.desc", _build_repeated_string_descriptor_set_bytes)
+    message_name = "test.WithRepeatedString"
+
+    data_gen = ProtobufMessageGen([
+        PbScalar("name", 1, StringGen()),
+        PbRepeated("tags", 2, StringGen(), min_len=0, max_len=5),
+    ])
+
+    def run_on_spark(spark):
+        df = gen_df(spark, data_gen)
+        decoded = _call_from_protobuf(
+            from_protobuf_fn, f.col("bin"), message_name, desc_path, desc_bytes)
+        return df.select(
+            decoded.getField("name").alias("name"),
+            decoded.getField("tags").alias("tags"),
+        )
+
+    assert_gpu_and_cpu_are_equal_collect(run_on_spark)
+
+
+@pytest.mark.skipif(is_before_spark_340(), reason="from_protobuf is Spark 3.4.0+")
+@ignore_order(local=True)
+def test_from_protobuf_nested_message(spark_tmp_path, from_protobuf_fn):
+    """
+    Test decoding nested message field (StructType).
+    """
+    desc_path, desc_bytes = _setup_protobuf_desc(
+        spark_tmp_path, "nested.desc", _build_nested_descriptor_set_bytes)
+    message_name = "test.WithNested"
+
+    data_gen = ProtobufMessageGen([
+        PbScalar("simple_int", 1, IntegerGen()),
+        PbScalar("simple_str", 2, StringGen(nullable=True)),
+        PbNested("nested_msg", 3, [PbScalar("x", 1, IntegerGen())]),
+        PbScalar("simple_long", 4, LongGen()),
+    ])
+
+    def run_on_spark(spark):
+        df = gen_df(spark, data_gen)
+        decoded = _call_from_protobuf(
+            from_protobuf_fn, f.col("bin"), message_name, desc_path, desc_bytes)
+        # Select all fields including nested
+        return df.select(
+            decoded.getField("simple_int").alias("simple_int"),
+            decoded.getField("simple_str").alias("simple_str"),
+            decoded.getField("nested_msg").alias("nested_msg"),
+            decoded.getField("simple_long").alias("simple_long"),
+        )
+
+    assert_gpu_and_cpu_are_equal_collect(run_on_spark)
+
+
+@pytest.mark.skipif(is_before_spark_340(), reason="from_protobuf is Spark 3.4.0+")
+@ignore_order(local=True)
+def test_from_protobuf_nested_message_field_access(spark_tmp_path, from_protobuf_fn):
+    """
+    Test accessing fields within nested message.
+    """
+    desc_path, desc_bytes = _setup_protobuf_desc(
+        spark_tmp_path, "nested.desc", _build_nested_descriptor_set_bytes)
+    message_name = "test.WithNested"
+
+    data_gen = ProtobufMessageGen([
+        PbScalar("simple_int", 1, IntegerGen()),
+        PbScalar("simple_str", 2, StringGen(nullable=True)),
+        PbNested("nested_msg", 3, [PbScalar("x", 1, IntegerGen())]),
+        PbScalar("simple_long", 4, LongGen()),
+    ])
+
+    def run_on_spark(spark):
+        df = gen_df(spark, data_gen)
+        decoded = _call_from_protobuf(
+            from_protobuf_fn, f.col("bin"), message_name, desc_path, desc_bytes)
+        # Access nested field directly
+        return df.select(
+            decoded.getField("simple_int").alias("simple_int"),
+            decoded.getField("nested_msg").getField("x").alias("nested_x"),
+        )
+
+    assert_gpu_and_cpu_are_equal_collect(run_on_spark)
+
+
+def _build_deep_nested_descriptor_set_bytes(spark):
+    """
+    Build a FileDescriptorSet for deeply nested message:
+      message Inner {
+        optional int32 value = 1;
+      }
+      message Middle {
+        optional string name = 1;
+        optional Inner inner = 2;
+      }
+      message Outer {
+        optional int32 id = 1;
+        optional Middle middle = 2;
+      }
+    """
+    D, fd = _new_proto2_file(spark, "deep_nested.proto")
+
+    label_opt = D.FieldDescriptorProto.Label.LABEL_OPTIONAL
+
+    # Inner message
+    inner_msg = D.DescriptorProto.newBuilder().setName("Inner")
+    inner_msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("value")
+            .setNumber(1)
+            .setLabel(label_opt)
+            .setType(D.FieldDescriptorProto.Type.TYPE_INT32)
+            .build()
+    )
+    fd.addMessageType(inner_msg.build())
+
+    # Middle message
+    middle_msg = D.DescriptorProto.newBuilder().setName("Middle")
+    middle_msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("name")
+            .setNumber(1)
+            .setLabel(label_opt)
+            .setType(D.FieldDescriptorProto.Type.TYPE_STRING)
+            .build()
+    )
+    middle_msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("inner")
+            .setNumber(2)
+            .setLabel(label_opt)
+            .setType(D.FieldDescriptorProto.Type.TYPE_MESSAGE)
+            .setTypeName(".test.Inner")
+            .build()
+    )
+    fd.addMessageType(middle_msg.build())
+
+    # Outer message
+    outer_msg = D.DescriptorProto.newBuilder().setName("Outer")
+    outer_msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("id")
+            .setNumber(1)
+            .setLabel(label_opt)
+            .setType(D.FieldDescriptorProto.Type.TYPE_INT32)
+            .build()
+    )
+    outer_msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("middle")
+            .setNumber(2)
+            .setLabel(label_opt)
+            .setType(D.FieldDescriptorProto.Type.TYPE_MESSAGE)
+            .setTypeName(".test.Middle")
+            .build()
+    )
+    fd.addMessageType(outer_msg.build())
+
+    fds = D.FileDescriptorSet.newBuilder().addFile(fd.build()).build()
+    return bytes(fds.toByteArray())
+
+
+@pytest.mark.skipif(is_before_spark_340(), reason="from_protobuf is Spark 3.4.0+")
+@ignore_order(local=True)
+def test_from_protobuf_deep_nested(spark_tmp_path, from_protobuf_fn):
+    """
+    Test decoding deeply nested messages (3 levels).
+    """
+    desc_path, desc_bytes = _setup_protobuf_desc(
+        spark_tmp_path, "deep_nested.desc", _build_deep_nested_descriptor_set_bytes)
+    message_name = "test.Outer"
+
+    data_gen = ProtobufMessageGen([
+        PbScalar("id", 1, IntegerGen()),
+        PbNested("middle", 2, [
+            PbScalar("name", 1, StringGen()),
+            PbNested("inner", 2, [PbScalar("value", 1, IntegerGen())]),
+        ]),
+    ])
+
+    def run_on_spark(spark):
+        df = gen_df(spark, data_gen)
+        decoded = _call_from_protobuf(
+            from_protobuf_fn, f.col("bin"), message_name, desc_path, desc_bytes)
+        return df.select(
+            decoded.getField("id").alias("id"),
+            decoded.getField("middle").alias("middle"),
+        )
+
+    assert_gpu_and_cpu_are_equal_collect(run_on_spark)
+
+
+def _build_repeated_message_descriptor_set_bytes(spark):
+    """
+    Build a FileDescriptorSet for repeated message field (array of structs):
+      message Item {
+        optional int32 id = 1;
+        optional string name = 2;
+      }
+      message Container {
+        optional string title = 1;
+        repeated Item items = 2;
+      }
+    """
+    D, fd = _new_proto2_file(spark, "repeated_message.proto")
+
+    label_opt = D.FieldDescriptorProto.Label.LABEL_OPTIONAL
+    label_rep = D.FieldDescriptorProto.Label.LABEL_REPEATED
+
+    # Item message
+    item_msg = D.DescriptorProto.newBuilder().setName("Item")
+    item_msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("id")
+            .setNumber(1)
+            .setLabel(label_opt)
+            .setType(D.FieldDescriptorProto.Type.TYPE_INT32)
+            .build()
+    )
+    item_msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("name")
+            .setNumber(2)
+            .setLabel(label_opt)
+            .setType(D.FieldDescriptorProto.Type.TYPE_STRING)
+            .build()
+    )
+    fd.addMessageType(item_msg.build())
+
+    # Container message
+    container_msg = D.DescriptorProto.newBuilder().setName("Container")
+    container_msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("title")
+            .setNumber(1)
+            .setLabel(label_opt)
+            .setType(D.FieldDescriptorProto.Type.TYPE_STRING)
+            .build()
+    )
+    container_msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("items")
+            .setNumber(2)
+            .setLabel(label_rep)
+            .setType(D.FieldDescriptorProto.Type.TYPE_MESSAGE)
+            .setTypeName(".test.Item")
+            .build()
+    )
+    fd.addMessageType(container_msg.build())
+
+    fds = D.FileDescriptorSet.newBuilder().addFile(fd.build()).build()
+    return bytes(fds.toByteArray())
+
+
+@pytest.mark.skipif(is_before_spark_340(), reason="from_protobuf is Spark 3.4.0+")
+@ignore_order(local=True)
+def test_from_protobuf_repeated_message(spark_tmp_path, from_protobuf_fn):
+    """
+    Test decoding repeated message field (ArrayType of StructType).
+    """
+    desc_path, desc_bytes = _setup_protobuf_desc(
+        spark_tmp_path, "repeated_message.desc", _build_repeated_message_descriptor_set_bytes)
+    message_name = "test.Container"
+
+    data_gen = ProtobufMessageGen([
+        PbScalar("title", 1, StringGen()),
+        PbRepeatedMessage("items", 2, [
+            PbScalar("id", 1, IntegerGen()),
+            PbScalar("name", 2, StringGen()),
+        ], min_len=0, max_len=5),
+    ])
+
+    def run_on_spark(spark):
+        df = gen_df(spark, data_gen)
+        decoded = _call_from_protobuf(
+            from_protobuf_fn, f.col("bin"), message_name, desc_path, desc_bytes)
+        return df.select(
+            decoded.getField("title").alias("title"),
+            decoded.getField("items").alias("items"),
+        )
+
+    assert_gpu_and_cpu_are_equal_collect(run_on_spark)
+
+
+def _build_nested_with_repeated_descriptor_set_bytes(spark):
+    """
+    Build a FileDescriptorSet for nested message with repeated fields:
+      message NestedWithRepeated {
+        optional string name = 1;
+        repeated int32 values = 2;
+        optional int32 count = 3;
+      }
+      message OuterWithNestedRepeated {
+        optional int32 id = 1;
+        optional NestedWithRepeated nested = 2;
+      }
+    """
+    D, fd = _new_proto2_file(spark, "nested_with_repeated.proto")
+
+    label_opt = D.FieldDescriptorProto.Label.LABEL_OPTIONAL
+    label_rep = D.FieldDescriptorProto.Label.LABEL_REPEATED
+
+    # NestedWithRepeated message
+    nested_msg = D.DescriptorProto.newBuilder().setName("NestedWithRepeated")
+    nested_msg.addField(D.FieldDescriptorProto.newBuilder()
+        .setName("name").setNumber(1).setLabel(label_opt)
+        .setType(D.FieldDescriptorProto.Type.TYPE_STRING))
+    nested_msg.addField(D.FieldDescriptorProto.newBuilder()
+        .setName("values").setNumber(2).setLabel(label_rep)
+        .setType(D.FieldDescriptorProto.Type.TYPE_INT32))
+    nested_msg.addField(D.FieldDescriptorProto.newBuilder()
+        .setName("count").setNumber(3).setLabel(label_opt)
+        .setType(D.FieldDescriptorProto.Type.TYPE_INT32))
+    fd.addMessageType(nested_msg)
+
+    # OuterWithNestedRepeated message
+    outer_msg = D.DescriptorProto.newBuilder().setName("OuterWithNestedRepeated")
+    outer_msg.addField(D.FieldDescriptorProto.newBuilder()
+        .setName("id").setNumber(1).setLabel(label_opt)
+        .setType(D.FieldDescriptorProto.Type.TYPE_INT32))
+    outer_msg.addField(D.FieldDescriptorProto.newBuilder()
+        .setName("nested").setNumber(2).setLabel(label_opt)
+        .setType(D.FieldDescriptorProto.Type.TYPE_MESSAGE)
+        .setTypeName(".test.NestedWithRepeated"))
+    fd.addMessageType(outer_msg)
+
+    fds = D.FileDescriptorSet.newBuilder().addFile(fd).build()
+    return bytes(fds.toByteArray())
+
+
+@pytest.mark.skipif(is_before_spark_340(), reason="from_protobuf is Spark 3.4.0+")
+@ignore_order(local=True)
+def test_from_protobuf_nested_with_repeated(spark_tmp_path, from_protobuf_fn):
+    """
+    Test decoding nested message that contains repeated fields.
+    Schema: OuterWithNestedRepeated { id, nested: NestedWithRepeated { name, values[], count } }
+    This tests StructType containing StructType containing ArrayType.
+    """
+    desc_path, desc_bytes = _setup_protobuf_desc(
+        spark_tmp_path, "nested_with_repeated.desc",
+        _build_nested_with_repeated_descriptor_set_bytes)
+    message_name = "test.OuterWithNestedRepeated"
+
+    data_gen = ProtobufMessageGen([
+        PbScalar("id", 1, IntegerGen()),
+        PbNested("nested", 2, [
+            PbScalar("name", 1, StringGen()),
+            PbRepeated("values", 2, IntegerGen(), min_len=0, max_len=5),
+            PbScalar("count", 3, IntegerGen()),
+        ]),
+    ])
+
+    def run_on_spark(spark):
+        df = gen_df(spark, data_gen)
+        decoded = _call_from_protobuf(
+            from_protobuf_fn, f.col("bin"), message_name, desc_path, desc_bytes)
+        return df.select(
+            decoded.getField("id").alias("id"),
+            decoded.getField("nested").alias("nested"),
+        )
+
+    assert_gpu_and_cpu_are_equal_collect(run_on_spark)
+
+
+def _build_repeated_with_nested_descriptor_set_bytes(spark):
+    """
+    Build a FileDescriptorSet for repeated message with nested message:
+      message Inner {
+        optional int32 value = 1;
+      }
+      message ItemWithNested {
+        optional int32 id = 1;
+        optional Inner inner = 2;
+        optional string name = 3;
+      }
+      message ContainerWithNestedItems {
+        optional string title = 1;
+        repeated ItemWithNested items = 2;
+      }
+    """
+    D, fd = _new_proto2_file(spark, "repeated_with_nested.proto")
+
+    label_opt = D.FieldDescriptorProto.Label.LABEL_OPTIONAL
+    label_rep = D.FieldDescriptorProto.Label.LABEL_REPEATED
+
+    # Inner message
+    inner_msg = D.DescriptorProto.newBuilder().setName("Inner")
+    inner_msg.addField(D.FieldDescriptorProto.newBuilder()
+        .setName("value").setNumber(1).setLabel(label_opt)
+        .setType(D.FieldDescriptorProto.Type.TYPE_INT32))
+    fd.addMessageType(inner_msg)
+
+    # ItemWithNested message
+    item_msg = D.DescriptorProto.newBuilder().setName("ItemWithNested")
+    item_msg.addField(D.FieldDescriptorProto.newBuilder()
+        .setName("id").setNumber(1).setLabel(label_opt)
+        .setType(D.FieldDescriptorProto.Type.TYPE_INT32))
+    item_msg.addField(D.FieldDescriptorProto.newBuilder()
+        .setName("inner").setNumber(2).setLabel(label_opt)
+        .setType(D.FieldDescriptorProto.Type.TYPE_MESSAGE)
+        .setTypeName(".test.Inner"))
+    item_msg.addField(D.FieldDescriptorProto.newBuilder()
+        .setName("name").setNumber(3).setLabel(label_opt)
+        .setType(D.FieldDescriptorProto.Type.TYPE_STRING))
+    fd.addMessageType(item_msg)
+
+    # ContainerWithNestedItems message
+    container_msg = D.DescriptorProto.newBuilder().setName("ContainerWithNestedItems")
+    container_msg.addField(D.FieldDescriptorProto.newBuilder()
+        .setName("title").setNumber(1).setLabel(label_opt)
+        .setType(D.FieldDescriptorProto.Type.TYPE_STRING))
+    container_msg.addField(D.FieldDescriptorProto.newBuilder()
+        .setName("items").setNumber(2).setLabel(label_rep)
+        .setType(D.FieldDescriptorProto.Type.TYPE_MESSAGE)
+        .setTypeName(".test.ItemWithNested"))
+    fd.addMessageType(container_msg)
+
+    fds = D.FileDescriptorSet.newBuilder().addFile(fd).build()
+    return bytes(fds.toByteArray())
+
+
+@pytest.mark.skipif(is_before_spark_340(), reason="from_protobuf is Spark 3.4.0+")
+@ignore_order(local=True)
+def test_from_protobuf_repeated_with_nested(spark_tmp_path, from_protobuf_fn):
+    """
+    Test decoding repeated message that contains nested message.
+    Schema: ContainerWithNestedItems { title, items[]: ItemWithNested { id, inner: Inner { value }, name } }
+    This tests ArrayType(StructType(StructType)) - nested struct inside repeated message.
+    """
+    desc_path, desc_bytes = _setup_protobuf_desc(
+        spark_tmp_path, "repeated_with_nested.desc",
+        _build_repeated_with_nested_descriptor_set_bytes)
+    message_name = "test.ContainerWithNestedItems"
+
+    data_gen = ProtobufMessageGen([
+        PbScalar("title", 1, StringGen()),
+        PbRepeatedMessage("items", 2, [
+            PbScalar("id", 1, IntegerGen()),
+            PbNested("inner", 2, [PbScalar("value", 1, IntegerGen())]),
+            PbScalar("name", 3, StringGen()),
+        ], min_len=0, max_len=3),
+    ])
+
+    def run_on_spark(spark):
+        df = gen_df(spark, data_gen)
+        decoded = _call_from_protobuf(
+            from_protobuf_fn, f.col("bin"), message_name, desc_path, desc_bytes)
+        return df.select(
+            decoded.getField("title").alias("title"),
+            decoded.getField("items").alias("items"),
+        )
+
+    assert_gpu_and_cpu_are_equal_collect(run_on_spark)
+
+
+def _build_packed_repeated_descriptor_set_bytes(spark):
+    """
+    Build a FileDescriptorSet for message with packed repeated fields (proto2):
+      message WithPackedRepeated {
+        optional int32 id = 1;
+        repeated int32 int_values = 2 [packed=true];
+        repeated double double_values = 3 [packed=true];
+        repeated bool bool_values = 4 [packed=true];
+      }
+    Uses proto2 with FieldOptions.packed=true (not proto3).
+    """
+    D, fd = _new_proto2_file(spark, "packed_repeated.proto")
+
+    label_rep = D.FieldDescriptorProto.Label.LABEL_REPEATED
+    label_opt = D.FieldDescriptorProto.Label.LABEL_OPTIONAL
+
+    # Build FieldOptions with packed=true for repeated numeric fields
+    packed_opts = D.FieldOptions.newBuilder().setPacked(True).build()
+
+    msg = D.DescriptorProto.newBuilder().setName("WithPackedRepeated")
+    msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("id")
+            .setNumber(1)
+            .setLabel(label_opt)
+            .setType(D.FieldDescriptorProto.Type.TYPE_INT32)
+            .build()
+    )
+    msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("int_values")
+            .setNumber(2)
+            .setLabel(label_rep)
+            .setType(D.FieldDescriptorProto.Type.TYPE_INT32)
+            .setOptions(packed_opts)
+            .build()
+    )
+    msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("double_values")
+            .setNumber(3)
+            .setLabel(label_rep)
+            .setType(D.FieldDescriptorProto.Type.TYPE_DOUBLE)
+            .setOptions(packed_opts)
+            .build()
+    )
+    msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("bool_values")
+            .setNumber(4)
+            .setLabel(label_rep)
+            .setType(D.FieldDescriptorProto.Type.TYPE_BOOL)
+            .setOptions(packed_opts)
+            .build()
+    )
+    fd.addMessageType(msg.build())
+
+    fds = D.FileDescriptorSet.newBuilder().addFile(fd.build()).build()
+    return bytes(fds.toByteArray())
+
+
+@pytest.mark.skipif(is_before_spark_340(), reason="from_protobuf is Spark 3.4.0+")
+@ignore_order(local=True)
+def test_from_protobuf_packed_repeated(spark_tmp_path, from_protobuf_fn):
+    """
+    Test packed repeated fields (int, double, bool) using DataGen.
+    """
+    desc_path, desc_bytes = _setup_protobuf_desc(
+        spark_tmp_path, "packed.desc", _build_packed_repeated_descriptor_set_bytes)
+    message_name = "test.WithPackedRepeated"
+
+    data_gen = ProtobufMessageGen([
+        PbScalar("id", 1, IntegerGen()),
+        PbRepeated("int_values", 2, IntegerGen(), packed=True, min_len=0, max_len=10),
+        PbRepeated("double_values", 3, DoubleGen(), packed=True, min_len=0, max_len=5),
+        PbRepeated("bool_values", 4, BooleanGen(), packed=True, min_len=0, max_len=5),
+    ])
+
+    def run_on_spark(spark):
+        df = gen_df(spark, data_gen)
+        decoded = _call_from_protobuf(
+            from_protobuf_fn, f.col("bin"), message_name, desc_path, desc_bytes)
+        return df.select(
+            decoded.getField("id").alias("id"),
+            decoded.getField("int_values").alias("int_values"),
+            decoded.getField("double_values").alias("double_values"),
+            decoded.getField("bool_values").alias("bool_values"),
+        )
+
+    assert_gpu_and_cpu_are_equal_collect(run_on_spark)
+
+
+def _build_repeated_all_types_descriptor_set_bytes(spark):
+    """
+    Build a FileDescriptorSet for message with various repeated field types:
+      message WithRepeatedAllTypes {
+        optional int32 id = 1;
+        repeated int64 long_values = 2;
+        repeated float float_values = 3;
+        repeated double double_values = 4;
+        repeated bool bool_values = 5;
+        repeated bytes bytes_values = 6;
+      }
+    """
+    D, fd = _new_proto2_file(spark, "repeated_all.proto")
+
+    label_rep = D.FieldDescriptorProto.Label.LABEL_REPEATED
+    label_opt = D.FieldDescriptorProto.Label.LABEL_OPTIONAL
+
+    msg = D.DescriptorProto.newBuilder().setName("WithRepeatedAllTypes")
+    msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("id")
+            .setNumber(1)
+            .setLabel(label_opt)
+            .setType(D.FieldDescriptorProto.Type.TYPE_INT32)
+            .build()
+    )
+    msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("long_values")
+            .setNumber(2)
+            .setLabel(label_rep)
+            .setType(D.FieldDescriptorProto.Type.TYPE_INT64)
+            .build()
+    )
+    msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("float_values")
+            .setNumber(3)
+            .setLabel(label_rep)
+            .setType(D.FieldDescriptorProto.Type.TYPE_FLOAT)
+            .build()
+    )
+    msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("double_values")
+            .setNumber(4)
+            .setLabel(label_rep)
+            .setType(D.FieldDescriptorProto.Type.TYPE_DOUBLE)
+            .build()
+    )
+    msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("bool_values")
+            .setNumber(5)
+            .setLabel(label_rep)
+            .setType(D.FieldDescriptorProto.Type.TYPE_BOOL)
+            .build()
+    )
+    msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("bytes_values")
+            .setNumber(6)
+            .setLabel(label_rep)
+            .setType(D.FieldDescriptorProto.Type.TYPE_BYTES)
+            .build()
+    )
+    fd.addMessageType(msg.build())
+
+    fds = D.FileDescriptorSet.newBuilder().addFile(fd.build()).build()
+    return bytes(fds.toByteArray())
+
+
+@pytest.mark.skipif(is_before_spark_340(), reason="from_protobuf is Spark 3.4.0+")
+@ignore_order(local=True)
+def test_from_protobuf_repeated_all_types(spark_tmp_path, from_protobuf_fn):
+    """Test repeated fields of various types (int64, float, double, bool, bytes) using DataGen."""
+    desc_path, desc_bytes = _setup_protobuf_desc(
+        spark_tmp_path, "repeated_all.desc", _build_repeated_all_types_descriptor_set_bytes)
+    message_name = "test.WithRepeatedAllTypes"
+
+    data_gen = ProtobufMessageGen([
+        PbScalar("id", 1, IntegerGen()),
+        PbRepeated("long_values", 2, LongGen()),
+        PbRepeated("float_values", 3, FloatGen()),
+        PbRepeated("double_values", 4, DoubleGen()),
+        PbRepeated("bool_values", 5, BooleanGen()),
+        PbRepeated("bytes_values", 6, BinaryGen()),
+    ])
+
+    def run_on_spark(spark):
+        df = gen_df(spark, data_gen)
+        decoded = _call_from_protobuf(
+            from_protobuf_fn, f.col("bin"), message_name, desc_path, desc_bytes)
+        return df.select(
+            decoded.getField("id").alias("id"),
+            decoded.getField("long_values").alias("long_values"),
+            decoded.getField("float_values").alias("float_values"),
+            decoded.getField("double_values").alias("double_values"),
+            decoded.getField("bool_values").alias("bool_values"),
+            decoded.getField("bytes_values").alias("bytes_values"),
+        )
+
+    assert_gpu_and_cpu_are_equal_collect(run_on_spark)
+
+
+@pytest.mark.skipif(is_before_spark_340(), reason="from_protobuf is Spark 3.4.0+")
+@ignore_order(local=True)
+def test_from_protobuf_large_repeated_array(spark_tmp_path, from_protobuf_fn):
+    """
+    Test decoding large repeated field (500-1000 elements).
+    Stress test for array handling.
+    """
+    desc_path, desc_bytes = _setup_protobuf_desc(
+        spark_tmp_path, "repeated_int.desc", _build_repeated_int_descriptor_set_bytes)
+    message_name = "test.WithRepeatedInt"
+
+    data_gen = ProtobufMessageGen([
+        PbScalar("id", 1, IntegerGen()),
+        PbRepeated("values", 2, IntegerGen(), min_len=500, max_len=1000),
+    ])
+
+    def run_on_spark(spark):
+        df = gen_df(spark, data_gen)
+        decoded = _call_from_protobuf(
+            from_protobuf_fn, f.col("bin"), message_name, desc_path, desc_bytes)
+        return df.select(
+            decoded.getField("id").alias("id"),
+            f.size(decoded.getField("values")).alias("array_size"),
+        )
+
+    assert_gpu_and_cpu_are_equal_collect(run_on_spark)
+
+
+def _build_signed_int_descriptor_set_bytes(spark):
+    """
+    Build a FileDescriptorSet for message with signed integer types:
+      message WithSignedInts {
+        optional sint32 si32 = 1;  // zigzag encoding
+        optional sint64 si64 = 2;  // zigzag encoding
+        optional sfixed32 sf32 = 3;  // fixed 4-byte
+        optional sfixed64 sf64 = 4;  // fixed 8-byte
+      }
+    """
+    D, fd = _new_proto2_file(spark, "signed_int.proto")
+
+    label_opt = D.FieldDescriptorProto.Label.LABEL_OPTIONAL
+
+    msg = D.DescriptorProto.newBuilder().setName("WithSignedInts")
+    msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("si32")
+            .setNumber(1)
+            .setLabel(label_opt)
+            .setType(D.FieldDescriptorProto.Type.TYPE_SINT32)
+            .build()
+    )
+    msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("si64")
+            .setNumber(2)
+            .setLabel(label_opt)
+            .setType(D.FieldDescriptorProto.Type.TYPE_SINT64)
+            .build()
+    )
+    msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("sf32")
+            .setNumber(3)
+            .setLabel(label_opt)
+            .setType(D.FieldDescriptorProto.Type.TYPE_SFIXED32)
+            .build()
+    )
+    msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("sf64")
+            .setNumber(4)
+            .setLabel(label_opt)
+            .setType(D.FieldDescriptorProto.Type.TYPE_SFIXED64)
+            .build()
+    )
+    fd.addMessageType(msg.build())
+
+    fds = D.FileDescriptorSet.newBuilder().addFile(fd.build()).build()
+    return bytes(fds.toByteArray())
+
+
+@pytest.mark.skipif(is_before_spark_340(), reason="from_protobuf is Spark 3.4.0+")
+@ignore_order(local=True)
+def test_from_protobuf_signed_integers(spark_tmp_path, from_protobuf_fn):
+    """
+    Test decoding signed integer types with zigzag encoding.
+    Zigzag: -1 -> 1, 1 -> 2, -2 -> 3, 2 -> 4, etc.
+    """
+    desc_path, desc_bytes = _setup_protobuf_desc(
+        spark_tmp_path, "signed.desc", _build_signed_int_descriptor_set_bytes)
+    message_name = "test.WithSignedInts"
+
+    data_gen = ProtobufMessageGen([
+        PbScalar("si32", 1, IntegerGen(
+            special_cases=[-1, 0, 1, -2147483648, 2147483647]), encoding='zigzag'),
+        PbScalar("si64", 2, LongGen(
+            special_cases=[-1, 0, 1, -9223372036854775808, 9223372036854775807]),
+            encoding='zigzag'),
+        PbScalar("sf32", 3, IntegerGen(
+            special_cases=[0, 1, -1, 2147483647, -2147483648]), encoding='fixed'),
+        PbScalar("sf64", 4, LongGen(
+            special_cases=[0, 1, -1]), encoding='fixed'),
+    ])
+
+    def run_on_spark(spark):
+        df = gen_df(spark, data_gen)
+        decoded = _call_from_protobuf(
+            from_protobuf_fn, f.col("bin"), message_name, desc_path, desc_bytes)
+        
+        return df.select(
+            decoded.getField("si32").alias("si32"),
+            decoded.getField("si64").alias("si64"),
+            decoded.getField("sf32").alias("sf32"),
+            decoded.getField("sf64").alias("sf64"),
+        )
+
+    assert_gpu_and_cpu_are_equal_collect(run_on_spark)
+
+
+def _build_fixed_int_descriptor_set_bytes(spark):
+    """
+    Build a FileDescriptorSet for message with fixed-width integer types:
+      message WithFixedInts {
+        optional fixed32 fx32 = 1;
+        optional fixed64 fx64 = 2;
+      }
+    """
+    D, fd = _new_proto2_file(spark, "fixed_int.proto")
+
+    label_opt = D.FieldDescriptorProto.Label.LABEL_OPTIONAL
+
+    msg = D.DescriptorProto.newBuilder().setName("WithFixedInts")
+    msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("fx32")
+            .setNumber(1)
+            .setLabel(label_opt)
+            .setType(D.FieldDescriptorProto.Type.TYPE_FIXED32)
+            .build()
+    )
+    msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("fx64")
+            .setNumber(2)
+            .setLabel(label_opt)
+            .setType(D.FieldDescriptorProto.Type.TYPE_FIXED64)
+            .build()
+    )
+    fd.addMessageType(msg.build())
+
+    fds = D.FileDescriptorSet.newBuilder().addFile(fd.build()).build()
+    return bytes(fds.toByteArray())
+
+
+@pytest.mark.skipif(is_before_spark_340(), reason="from_protobuf is Spark 3.4.0+")
+@ignore_order(local=True)
+def test_from_protobuf_fixed_integers(spark_tmp_path, from_protobuf_fn):
+    """
+    Test decoding fixed-width unsigned integer types.
+    """
+    desc_path, desc_bytes = _setup_protobuf_desc(
+        spark_tmp_path, "fixed.desc", _build_fixed_int_descriptor_set_bytes)
+    message_name = "test.WithFixedInts"
+
+    data_gen = ProtobufMessageGen([
+        PbScalar("fx32", 1, IntegerGen(
+            special_cases=[0, 1, -1, 2147483647, -2147483648]), encoding='fixed'),
+        PbScalar("fx64", 2, LongGen(
+            special_cases=[0, 1, -1]), encoding='fixed'),
+    ])
+
+    def run_on_spark(spark):
+        df = gen_df(spark, data_gen)
+        decoded = _call_from_protobuf(
+            from_protobuf_fn, f.col("bin"), message_name, desc_path, desc_bytes)
+        return df.select(
+            decoded.getField("fx32").alias("fx32"),
+            decoded.getField("fx64").alias("fx64"),
+        )
+
+    assert_gpu_and_cpu_are_equal_collect(run_on_spark)
+
+
+def _build_schema_projection_descriptor_set_bytes(spark):
+    """
+    Build a FileDescriptorSet for nested schema projection testing:
+      message Detail {
+        optional int32  a = 1;
+        optional int32  b = 2;
+        optional string c = 3;
+      }
+      message SchemaProj {
+        optional int32  id     = 1;
+        optional string name   = 2;
+        optional Detail detail = 3;
+        repeated Detail items  = 4;
+      }
+    The Detail message has 3 fields so we can test pruning subsets.
+    """
+    D, fd = _new_proto2_file(spark, "schema_proj.proto")
+
+    label_opt = D.FieldDescriptorProto.Label.LABEL_OPTIONAL
+    label_rep = D.FieldDescriptorProto.Label.LABEL_REPEATED
+
+    # Detail message: { a: int32, b: int32, c: string }
+    detail_msg = D.DescriptorProto.newBuilder().setName("Detail")
+    detail_msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("a").setNumber(1).setLabel(label_opt)
+            .setType(D.FieldDescriptorProto.Type.TYPE_INT32).build())
+    detail_msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("b").setNumber(2).setLabel(label_opt)
+            .setType(D.FieldDescriptorProto.Type.TYPE_INT32).build())
+    detail_msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("c").setNumber(3).setLabel(label_opt)
+            .setType(D.FieldDescriptorProto.Type.TYPE_STRING).build())
+    fd.addMessageType(detail_msg.build())
+
+    # SchemaProj message: { id, name, detail: Detail, items: repeated Detail }
+    main_msg = D.DescriptorProto.newBuilder().setName("SchemaProj")
+    main_msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("id").setNumber(1).setLabel(label_opt)
+            .setType(D.FieldDescriptorProto.Type.TYPE_INT32).build())
+    main_msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("name").setNumber(2).setLabel(label_opt)
+            .setType(D.FieldDescriptorProto.Type.TYPE_STRING).build())
+    main_msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("detail").setNumber(3).setLabel(label_opt)
+            .setType(D.FieldDescriptorProto.Type.TYPE_MESSAGE)
+            .setTypeName(".test.Detail").build())
+    main_msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("items").setNumber(4).setLabel(label_rep)
+            .setType(D.FieldDescriptorProto.Type.TYPE_MESSAGE)
+            .setTypeName(".test.Detail").build())
+    fd.addMessageType(main_msg.build())
+
+    fds = D.FileDescriptorSet.newBuilder().addFile(fd.build()).build()
+    return bytes(fds.toByteArray())
+
+
+# Field descriptors for SchemaProj: {id, name, detail: {a, b, c}, items[]: {a, b, c}}
+_detail_children = [
+    PbScalar("a", 1, IntegerGen()),
+    PbScalar("b", 2, IntegerGen()),
+    PbScalar("c", 3, StringGen()),
+]
+_schema_proj_fields = [
+    PbScalar("id", 1, IntegerGen()),
+    PbScalar("name", 2, StringGen()),
+    PbNested("detail", 3, _detail_children),
+    PbRepeatedMessage("items", 4, _detail_children),
+]
+
+_schema_proj_test_data = [
+    encode_pb_message(_schema_proj_fields,
+                      [1, "alice", (10, 20, "d1"), [(100, 200, "i1"), (101, 201, "i2")]]),
+    encode_pb_message(_schema_proj_fields,
+                      [2, "bob", (30, 40, "d2"), [(300, 400, "i3")]]),
+    encode_pb_message(_schema_proj_fields,
+                      [3, "carol", (50, 60, "d3"), []]),
+]
+
+
+_schema_proj_cases = [
+    ("nested_single_field", [("id", ("id",)), ("detail_a", ("detail", "a"))]),
+    ("nested_two_fields", [("detail_a", ("detail", "a")), ("detail_c", ("detail", "c"))]),
+    ("whole_struct_no_pruning", [("id", ("id",)), ("detail", ("detail",))]),
+    ("whole_and_subfield", [("detail", ("detail",)), ("detail_a", ("detail", "a"))]),
+    ("scalar_plus_nested", [("id", ("id",)), ("name", ("name",)), ("detail_a", ("detail", "a"))]),
+    ("repeated_msg_single_subfield", [("id", ("id",)), ("items_a", ("items", "a"))]),
+    ("repeated_msg_two_subfields", [("items_a", ("items", "a")), ("items_c", ("items", "c"))]),
+    ("repeated_whole_no_pruning", [("id", ("id",)), ("items", ("items",))]),
+    ("mix_struct_and_repeated", [("id", ("id",)), ("detail_a", ("detail", "a")), ("items_c", ("items", "c"))]),
+]
+
+
+def _get_field_by_path(expr, path):
+    current = expr
+    for name in path:
+        current = current.getField(name)
+    return current
+
+
+@pytest.mark.skipif(is_before_spark_340(), reason="from_protobuf is Spark 3.4.0+")
+@pytest.mark.parametrize("case_id,select_specs", _schema_proj_cases, ids=lambda c: c[0] if isinstance(c, tuple) else str(c))
+@ignore_order(local=True)
+def test_from_protobuf_schema_projection_cases(
+        spark_tmp_path, from_protobuf_fn, case_id, select_specs):
+    """Parametrized nested-schema projection tests."""
+    desc_path, desc_bytes = _setup_protobuf_desc(
+        spark_tmp_path, "schema_proj.desc", _build_schema_projection_descriptor_set_bytes)
+    message_name = "test.SchemaProj"
+
+    def run_on_spark(spark):
+        df = spark.createDataFrame(
+            [(d,) for d in _schema_proj_test_data], schema="bin binary")
+        decoded = _call_from_protobuf(
+            from_protobuf_fn, f.col("bin"), message_name, desc_path, desc_bytes)
+        selected = [_get_field_by_path(decoded, path).alias(alias)
+                    for alias, path in select_specs]
+        return df.select(*selected)
+
+    assert_gpu_and_cpu_are_equal_collect(run_on_spark)
diff --git a/integration_tests/src/main/python/spark_init_internal.py b/integration_tests/src/main/python/spark_init_internal.py
index 90861746b64..765bcac036d 100644
--- a/integration_tests/src/main/python/spark_init_internal.py
+++ b/integration_tests/src/main/python/spark_init_internal.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2020-2025, NVIDIA CORPORATION.
+# Copyright (c) 2020-2026, NVIDIA CORPORATION.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -61,11 +61,38 @@ def findspark_init():
     if spark_jars is not None:
         logging.info(f"Adding to findspark jars: {spark_jars}")
         findspark.add_jars(spark_jars)
+        # Also add to driver classpath so classes are available to Class.forName()
+        # This is needed for optional modules like spark-protobuf
+        _add_driver_classpath(spark_jars)
 
     if spark_jars_packages is not None:
         logging.info(f"Adding to findspark packages: {spark_jars_packages}")
         findspark.add_packages(spark_jars_packages)
 
+
+def _add_driver_classpath(jars):
+    """
+    Add jars to the driver classpath via PYSPARK_SUBMIT_ARGS.
+    findspark.add_jars() only adds --jars, which doesn't make classes available
+    to Class.forName() on the driver. This function adds --driver-class-path.
+    """
+    if not jars:
+        return
+    current_args = os.environ.get('PYSPARK_SUBMIT_ARGS', '')
+    # Remove trailing 'pyspark-shell' if present
+    if current_args.endswith('pyspark-shell'):
+        current_args = current_args[:-len('pyspark-shell')].strip()
+    # Skip if driver-class-path is already present
+    if '--driver-class-path' in current_args:
+        logging.info("driver-class-path already in PYSPARK_SUBMIT_ARGS, skipping")
+        return
+    # Add driver-class-path for each jar (use os.pathsep for platform independence)
+    jar_list = jars.replace(',', ' ').split()
+    driver_cp = os.pathsep.join(jar_list)
+    new_args = f"{current_args} --driver-class-path {driver_cp} pyspark-shell".strip()
+    os.environ['PYSPARK_SUBMIT_ARGS'] = new_args
+    logging.info(f"Updated PYSPARK_SUBMIT_ARGS with driver-class-path")
+
 def running_with_xdist(session, is_worker):
     try:
         import xdist
diff --git a/integration_tests/src/test/resources/protobuf_test/gen_nested_proto_data.sh b/integration_tests/src/test/resources/protobuf_test/gen_nested_proto_data.sh
new file mode 100755
index 00000000000..2283ac8a751
--- /dev/null
+++ b/integration_tests/src/test/resources/protobuf_test/gen_nested_proto_data.sh
@@ -0,0 +1,34 @@
+#!/bin/bash
+# Convenience script: compile nested_proto .proto files into a descriptor set.
+#
+# Usage:
+#   ./gen_nested_proto_data.sh
+#
+# The generated .desc file is checked into the repository and used by
+# integration tests in protobuf_test.py.  Re-run this script whenever
+# the .proto definitions under nested_proto/ change.
+
+set -e
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+PROTO_DIR="${SCRIPT_DIR}/nested_proto"
+OUTPUT_DIR="${SCRIPT_DIR}/nested_proto/generated"
+
+echo "=== Protobuf Descriptor Compiler ==="
+echo "Proto dir: ${PROTO_DIR}"
+echo ""
+
+# Create output directory
+mkdir -p "${OUTPUT_DIR}"
+
+# Compile proto files into a descriptor set (includes all imports)
+DESC_FILE="${OUTPUT_DIR}/main_log.desc"
+echo "Compiling proto files..."
+protoc \
+    --descriptor_set_out="${DESC_FILE}" \
+    --include_imports \
+    -I"${PROTO_DIR}" \
+    "${PROTO_DIR}/main_log.proto"
+
+echo "Generated: ${DESC_FILE}"
+echo "=== Done ==="
diff --git a/integration_tests/src/test/resources/protobuf_test/nested_proto/device_req.proto b/integration_tests/src/test/resources/protobuf_test/nested_proto/device_req.proto
new file mode 100644
index 00000000000..68024163e99
--- /dev/null
+++ b/integration_tests/src/test/resources/protobuf_test/nested_proto/device_req.proto
@@ -0,0 +1,9 @@
+package com.test.proto.sample;
+
+option java_outer_classname = "DeviceReqBean";
+
+// Device request field
+message DeviceReqField { 
+    optional int32 os_type = 1;           // int32
+    optional bytes device_id = 2;         // bytes
+}
diff --git a/integration_tests/src/test/resources/protobuf_test/nested_proto/generated/main_log.desc b/integration_tests/src/test/resources/protobuf_test/nested_proto/generated/main_log.desc
new file mode 100644
index 0000000000000000000000000000000000000000..6e8155238b3ffd41acba32f1090f10eb095ee416
GIT binary patch
literal 5838
zcmbVQU2h{<5nU%vVpsc%7A$69VYc^!VGz+OJOCph6nm1Lu$gS;W@h)X)v<3pt+d^q
z?sj(a1`F>z@B@&LkoXh)5F~g;yzm2%kT_NM_D8(incaEoTfVpYR-LMIs!jiL=*?xK
zr{N@v6F;3i*YjX^nZ#M_erpoXcQc)4X1$vR^JS#He>bRqnx8n61MVSpv+Jey1M6Yf
zxZAkr?0LIi>SEWNdWsYK%Hf0iYktmiKO0}s&&c>0t@)Ycem1|NpRw^XHh#R1Xdt(I
zveL=5^*yAWOMU%WoJ=Lkw#t|8TSTc&(=cB6;dIDXZ>zSo98J9ssV~k>gD|nazDw;y
zPvD9VaL(GQWei7C_uI4+W_n)w+U6x3{kcJoaUlNOUsLA=Uoa2WB>#ss$zKmhIda(E
z=tFXrxL@vS<L)MKeE&9AWjKoXsBz>yGpQR@Q}-_QP+AvZr2Q<Il`(g$^=U9e+(?RE
zWMOt)M&2?8#IC5v0gTSF>qs9&dhY&!9HeC#1(_~AQf`7jY?$N6Bn+a`BjqOed}6OE
zK5ZU;N!wRJw9+YH;S{`gOA5-%D0|5J%MFM|uK`gtaW^3X+$l$Y6QkSUHWxIJDMt=l
zlpdMT$5oW-3;YhXCdlM0POfcv7i;gxRQI?nhiU<v+O`GEjgke)OlSToiSkA<mS;)i
z4GhNP0^=MZV|fjPN>>F!|G9zC_%#r!ChaCdU%QDaS*tb?U+M<bn^KFnL0wJJ4XEi)
z9dhUKbcMnR{6wd>vizMIQ2>cKr$Le}baJ$~h}}17D~m7n!jRX4H<VercL`h|+wzuc
zK4^TP+SZ7syGJyR!F=HY73FrQ@lt#=RO2PDOo3r_?ft$%-QyUA2Vy;q!$szPkJ=ZB
z25l`S*K(?>N`w0|+M2LuDJ*C2?V8Rl->%GlcAJHB2p)uP$coWarNKL)_V{_QnCatq
z=DtlGp$Gq5U&M*FmvF>35|jIVNkK5zpq|tgIVZ(hyq{3(GzllUyCpxbxf7dl=lykq
z`i}*^eWlaGP)EX)NxTB@flUnpkPkM1NJWjZAX}yKE345K)JQD+vq;a(Y840DW$9r9
zDDyq)>6c4T<{ZQe1k+&?mDg6?(t0#a-FK+_ViiP8)hSd=54Yf_p`b=$JY$>tnEH`L
z&tEcp>2O=h_}v;{ZeXo?`C)-R<;=r}Mgg0P?Z8)4Ik^qj@_tIL&D2>kDUt=!0q0Mi
z`x*2juNtwipm-n9_D7#EdAS4Hguu&=j^ao6o3tg!3W^<N7T!5jn0a^{FD}BF`yFbT
zUbRx*RXegqI)6^B%PXJ7kqd$4<gJ?6%@r_GT=7*>?;p(~mWMhE!l(eAqwpv3G!Lc4
zhB4Mv+GW(6DyLn*m1YSO__av_!@i|JF~-Pw;JTQH3*W@s#D-1Pg5eVu!!_b&v^!`K
zp>hy6=LxzUX#mWlj9GAE!jv5%F+2F7rFOy;{z8#vDI6ac2;*T3ziucJlVS^_oh!MN
zVH?@&Vn(xq&qatWxt7ZoG6j*_c*_IY7AVr8BiXN=OG`MFJ>l+?Gm*CUqj;7MJL2GO
z4SWSbC^Cn@6{&kboq`NOn!_%_tRId~OL|q9#Y(Qln6f^p4}0ROZa!NZ(^bDXZ!~n&
z8Bxh8^ta7IA<kc^5FYy4oE;S{hT5qwOZ1<9iAwWB&Vikd#llMB9a3k6R=?z#4y41S
zrp+TO2+t2-V{1W6VK`3og#gYSg(An2n}0@a@gwqt(mQ9TrHdU|!@;kpWxeG+`A!X}
z)oZEZkoSj1-kPIoYr50~8=JAQtK_-9Be8h_JrJ>JD|rT4Y|9-D=Q?|JwI<i%D3#q$
zpEqbQ)mN+|P{&*K9qH~pl&wm?Fr0#n#?T6IfMk11ZO3WO2-YirVB9*YWei8tQGe7e
z{ET$4p#Nfen+ISJw9Bnmxbr2`NMWng-<>bS<o$Mox{vV=beEFPC-)cV@OYI>`0g9`
zdbOf|CYv}cG`Kw9RQ(N{uBtvmw@P)w1;lLGR@=Np{h&UB$s}HhK0+RuDBIS8pV11F
zC=~TviY)G^mN6VH-4Qv+4%iz)w#bAn>*o{%35DT}ig3z4d;e?@l|P}J={!ioiRmu$
z1z1}-yiqErEoY=vKvGOu3my4Um}XX|>X?#=4qL^E@FNz-IObwLG<h{%%)na5PY;7E
zFo;MAg82m!cI5r9fkw_R$`mRCAB#TIhY7ImcBpB5xn<^d_M8N%=o?@hy_o7Lw@67v
z_20M$c0YWIzkcq(8r)~(q9U$PBf*sYUhrrp#p3MMP3mWIV4wG!$OpznY4QGu?-RTt
zkAqco8O*f5Sj{DRj2FB<S<P7ibLeCsFx;BMA~V4k2RcTkyFSq2QyEf?4|E=|rc%`{
zMIpvbW5LYRUrUJlag}sz^ah@~Z{~t9Ozb^FCJ0v6-rqrE>j~nIcr%?=XC5*pi4YBU
z5yhVw0y7zCu|X&dlaC3CWo;E@0uLr1V}yRe=6ck99%g>RUr5pdJo9cT2Sx>(ls8hH
z7*KrvRFKZb^l5ERVdBnDFT<sO5n(9jL8zr#Y&gW)yCSF3I!q!ta#rLpqgx9|<Pk=j
z$`_lEZ+)efN^bH><}80Mc)x2<55w@+c(<ZK@$NL4aN_Lye^jHB_WhNaTXpRfjLrKE
zzG2pSUeYC9pKCO}LktZd0jMp0cb+3b#SGAOwS_lTf>b>mx9c3g^Dv6S#muh=uVZZ|
z&|-H?ZTZ`+D1rKnA7KuyXuoRlL3t1VcZWtwv?-AmX7Kb7EAMlpVC|TTDIf_Jm$t3t
znbT7KGsxxKto$=r`G>}cQt=n$6Y3YJP340@n%LQU1!&bMa?p&;EnS}b)+G=AuX0%(
zit;+Jw)^*q4DCyE?$G9^2cy9oE*|zzPX`T`X#4TUqod=4LG%8Pi;j`aU3`T<+CMot
OIQI8(q_P|h8vh3x_$jCW

literal 0
HcmV?d00001

diff --git a/integration_tests/src/test/resources/protobuf_test/nested_proto/main_log.proto b/integration_tests/src/test/resources/protobuf_test/nested_proto/main_log.proto
new file mode 100644
index 00000000000..8f683592109
--- /dev/null
+++ b/integration_tests/src/test/resources/protobuf_test/nested_proto/main_log.proto
@@ -0,0 +1,101 @@
+package com.test.proto.sample;
+
+import "module_a_res.proto";
+import "module_b_res.proto";
+import "device_req.proto";
+
+// ========== Enum type tests ==========
+enum SourceType {
+    WEB    = 0;   // web
+    APP    = 1;   // application
+    MOBILE = 4;   // mobile
+}
+
+enum ChannelType {
+    CHANNEL_A = 0;
+    CHANNEL_B = 1;
+}
+
+// ========== Main log record ==========
+message MainLogRecord {
+    // required fields
+    required SourceType source = 1;           // required enum
+    required uint64 timestamp = 2;            // required uint64
+
+    // optional scalar types - one of each
+    optional string user_id = 3;              // string
+    optional int64 account_id = 4;            // int64
+    optional fixed32 client_ip = 5;           // fixed32
+
+    // nested message
+    optional LogContent log_content = 6;
+}
+
+// ========== Log content (multi-level nesting) ==========
+message LogContent {
+    optional BasicInfo basic_info = 1;
+    repeated ChannelInfo channel_list = 2;
+    repeated DataSourceField source_list = 3;
+}
+
+// ========== Basic info (three-level nesting) ==========
+message BasicInfo {
+    optional RequestInfo request_info = 1;
+    optional ExtendedReqInfo extended_req_info = 2;
+    optional ServerAddedField server_added_field = 3;
+}
+
+// ========== Request info ==========
+message RequestInfo {
+    optional uint32 page_num = 1;                   // uint32
+    optional string channel_code = 2;               // string
+    repeated uint32 experiment_ids = 3;             // repeated uint32
+    optional bool is_filtered = 4;                  // bool
+}
+
+// ========== Extended request info (cross-file import) ==========
+message ExtendedReqInfo {
+    optional DeviceReqField device_req_field = 1;   // reference to external proto
+}
+
+// ========== Server-added fields ==========
+message ServerAddedField {
+    optional uint32 region_code = 1;                // uint32
+    optional string flow_type = 2;                  // string
+    optional int32 filter_result = 3;               // int32
+    repeated int32 hit_rule_list = 4;               // repeated int32
+    optional uint64 request_time = 5;               // uint64
+    optional bool skip_flag = 6;                    // bool
+}
+
+// ========== Channel info ==========
+message ChannelInfo {
+    optional int32 channel_id = 1;
+    optional ModuleAResField module_a_res = 2;      // reference to external proto
+}
+
+// ========== Source channel info ==========
+message SrcChannelInfo {
+    optional int32 channel_id = 1;
+    optional ModuleASrcResField module_a_src_res = 2;  // reference to external proto
+}
+
+// ========== Data source field ==========
+message DataSourceField {
+    optional uint32 source_id = 1;
+    repeated SrcChannelInfo src_channel_list = 2;
+    optional string billing_name = 3;
+    repeated ItemDetailField item_list = 4;
+    optional bool is_free = 5;
+}
+
+// ========== Item detail field ==========
+message ItemDetailField {
+    optional uint32 rank = 1;                       // uint32
+    optional uint64 record_id = 2;                  // uint64
+    optional string keyword = 3;                    // string
+
+    // cross-file message references
+    optional ModuleADetailField module_a_detail = 4;
+    optional ModuleBDetailField module_b_detail = 5;
+}
diff --git a/integration_tests/src/test/resources/protobuf_test/nested_proto/module_a_res.proto b/integration_tests/src/test/resources/protobuf_test/nested_proto/module_a_res.proto
new file mode 100644
index 00000000000..c9dbf2b9e35
--- /dev/null
+++ b/integration_tests/src/test/resources/protobuf_test/nested_proto/module_a_res.proto
@@ -0,0 +1,90 @@
+package com.test.proto.sample;
+
+option java_outer_classname = "ModuleARes";
+
+import "predictor_schema.proto";
+
+// ========== Test default values ==========
+message PartnerInfo {
+    optional string token = 1 [default = ""];      // default empty string
+    optional uint64 partner_id = 2 [default = 0];  // default 0
+}
+
+// ========== Test coordinate structure (simple nesting) ==========
+message Coordinate {
+    optional double x = 1;    // x coordinate - double
+    optional double y = 2;    // y coordinate - double
+}
+
+// ========== Test multi-level nesting ==========
+message LocationPoint {
+    optional uint32 frequency = 1;              // frequency - uint32
+    optional Coordinate coord = 2;              // coordinate - nested message
+    optional uint64 timestamp = 3;              // timestamp - uint64
+}
+
+// ========== Test change log ==========
+message ChangeLog {
+    optional uint32 value_before = 1;           // value before change
+    optional string parameters = 2;             // parameters
+}
+
+// ========== Test price change log ==========
+message PriceLog {
+    optional uint32 price_before = 1;
+}
+
+// ========== Module A response-level fields ==========
+message ModuleAResField {
+    optional string route_tag = 1;                  // route tag - string
+    optional int32 status_tag = 2;                  // status tag - int32
+    optional uint32 region_id = 3;                  // region id - uint32
+    repeated string experiment_ids = 4;             // experiment id list - repeated string
+    optional double quality_score = 5;              // quality score - double
+    repeated LocationPoint location_points = 6;    // location points - repeated nested message
+    repeated uint64 interest_ids = 7;               // interest id list - repeated uint64
+}
+
+// ========== Module A source response field ==========
+message ModuleASrcResField {
+    optional uint32 match_type = 1;    // match type
+}
+
+// ========== Key-value pair ==========
+message KVPair {
+    optional bytes key = 1;      // key - bytes
+    optional bytes value = 2;    // value - bytes
+}
+
+// ========== Style configuration ==========
+message StyleConfig {
+    optional uint32 style_id = 1;        // style id
+    repeated KVPair kv_pairs = 2;        // kv pair list - repeated nested message
+}
+
+// ========== Module A detail field (core complex structure) ==========
+message ModuleADetailField {
+    // scalar types - one or two of each
+    optional uint32 type_code = 1;          // uint32
+    optional uint64 item_id = 2;            // uint64
+    optional int32 strategy_type = 3;       // int32
+    optional int64 min_value = 4;           // int64
+    optional bytes target_url = 5;          // bytes
+    optional string title = 6;              // string
+    optional bool is_valid = 7;             // bool
+    optional float score_ratio = 8;         // float
+
+    // repeated scalar types
+    repeated uint32 template_ids = 9;       // repeated uint32
+    repeated uint64 material_ids = 10;      // repeated uint64
+
+    // repeated nested messages
+    repeated StyleConfig styles = 11;       // repeated message
+    repeated ChangeLog change_logs = 12;    // repeated message
+
+    // nested message
+    optional PartnerInfo partner_info = 13; // nested message
+
+    // cross-file import
+    optional PredictorSchema predictor_schema = 14;  // reference to external proto
+}
diff --git a/integration_tests/src/test/resources/protobuf_test/nested_proto/module_b_res.proto b/integration_tests/src/test/resources/protobuf_test/nested_proto/module_b_res.proto
new file mode 100644
index 00000000000..be38a10c083
--- /dev/null
+++ b/integration_tests/src/test/resources/protobuf_test/nested_proto/module_b_res.proto
@@ -0,0 +1,27 @@
+package com.test.proto.sample;
+
+// Module B response field
+message ModuleBResField { 
+    optional uint32 type_code = 1;        // uint32
+    optional string extra_info = 2;       // string
+}
+
+// Block element - tests repeated scalar types
+message BlockElement {
+    optional uint64 element_id = 1;       // uint64
+    repeated uint64 ref_ids = 2;          // repeated uint64
+}
+
+// Block info - tests repeated nested messages
+message BlockInfo {
+    optional uint64 block_id = 1;         // uint64
+    repeated BlockElement elements = 2;   // repeated message
+}
+
+// Module B detail field
+message ModuleBDetailField {
+    repeated uint32 tags = 1;             // repeated uint32
+    optional uint64 item_id = 2;          // uint64
+    optional string name = 3;             // string
+    repeated BlockInfo blocks = 4;        // repeated message
+}
diff --git a/integration_tests/src/test/resources/protobuf_test/nested_proto/predictor_schema.proto b/integration_tests/src/test/resources/protobuf_test/nested_proto/predictor_schema.proto
new file mode 100644
index 00000000000..9c0db8a6bcd
--- /dev/null
+++ b/integration_tests/src/test/resources/protobuf_test/nested_proto/predictor_schema.proto
@@ -0,0 +1,80 @@
+package com.test.proto.sample;
+
+// Predictor schema - tests multi-level schema nesting
+
+// ========== Main schema structure ==========
+message PredictorSchema {
+    optional SchemaTypeA type_a_schema = 1;      // nested
+    optional SchemaTypeB type_b_schema = 2;      // nested
+    optional SchemaTypeC type_c_schema = 3;      // nested (with repeated)
+}
+
+// ========== TypeA Query Schema ==========
+message TypeAQuerySchema {
+    optional string keyword = 1;            // keyword
+    optional string session_id = 2;         // session id
+}
+
+// ========== TypeA Pair Schema ==========
+message TypeAPairSchema {
+    optional string record_id = 1;          // record id
+    optional string item_id = 2;            // item id
+}
+
+// ========== TypeA Schema ==========
+message SchemaTypeA {
+    optional TypeAQuerySchema query_schema = 1;   // query-level schema
+    repeated TypeAPairSchema pair_schema = 2;     // pair list - repeated nested
+}
+
+// ========== TypeB Query Schema ==========
+message TypeBQuerySchema {
+    optional string profile_tag_id = 1;     // profile tag id
+    optional string entity_id = 2;          // entity id
+}
+
+// ========== TypeB Style Element ==========
+message TypeBStyleElem {
+    optional string template_id = 1;        // template id
+    optional string material_id = 2;        // material id
+}
+
+// ========== TypeB Style Schema ==========
+message TypeBStyleSchema {
+    repeated TypeBStyleElem values = 1;     // element list - repeated nested
+}
+
+// ========== TypeB Schema ==========
+message SchemaTypeB {
+    optional TypeBQuerySchema query_schema = 1;     // query-level schema
+    repeated TypeBStyleSchema style_schema = 2;     // style list - repeated nested
+}
+
+// ========== TypeC Query Schema ==========
+message TypeCQuerySchema {
+    optional string keyword = 1;            // keyword
+    optional string category = 2;           // category
+}
+
+// ========== TypeC Pair Schema ==========
+message TypeCPairSchema {
+    optional string item_id = 1;            // item id
+    optional string target_url = 2;         // target url
+}
+
+// ========== TypeC Style Element (empty structure test) ==========
+message TypeCStyleElem {
+    // empty message - tests empty structure handling
+}
+
+// ========== TypeC Style Schema ==========
+message TypeCStyleSchema {
+    repeated TypeCStyleElem values = 1;     // empty element list
+}
+
+// ========== TypeC Schema (full three-level structure) ==========
+message SchemaTypeC {
+    optional TypeCQuerySchema query_schema = 1;     // query schema
+    repeated TypeCPairSchema pair_schema = 2;       // pair list
+    repeated TypeCStyleSchema style_schema = 3;     // style list
+}
diff --git a/pom.xml b/pom.xml
index 6eeff9d35be..6e69fa554fe 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <!--
-  Copyright (c) 2020-2025, NVIDIA CORPORATION.
+  Copyright (c) 2020-2026, NVIDIA CORPORATION.
 
   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala
index 437c49608fe..535c905884a 100644
--- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala
+++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala
@@ -4094,12 +4094,14 @@ object GpuOverrides extends Logging {
       "Extracts the `ordinal`-th fields of all array elements for the data with the type of" +
         " array of struct",
       ExprChecks.unaryProject(
-        TypeSig.ARRAY.nested(TypeSig.commonCudfTypesWithNested),
+        TypeSig.ARRAY.nested((TypeSig.commonCudfTypes + TypeSig.DECIMAL_128 + TypeSig.NULL +
+            TypeSig.ARRAY + TypeSig.STRUCT + TypeSig.MAP + TypeSig.BINARY).nested()),
         TypeSig.ARRAY.nested(TypeSig.all),
         // we should allow all supported types for the children types signature of the nested
         // struct, even only a struct child is allowed for the array here. Since TypeSig supports
         // only one level signature for nested type.
-        TypeSig.ARRAY.nested(TypeSig.commonCudfTypesWithNested),
+        TypeSig.ARRAY.nested((TypeSig.commonCudfTypes + TypeSig.DECIMAL_128 + TypeSig.NULL +
+            TypeSig.ARRAY + TypeSig.STRUCT + TypeSig.MAP + TypeSig.BINARY).nested()),
         TypeSig.ARRAY.nested(TypeSig.all)),
       (e, conf, p, r) => new GpuGetArrayStructFieldsMeta(e, conf, p, r)
     ),
diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/TypeChecks.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/TypeChecks.scala
index dc9f226cbee..1dede77b5b0 100644
--- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/TypeChecks.scala
+++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/TypeChecks.scala
@@ -682,12 +682,6 @@ object TypeSig {
       TIMESTAMP + STRING + DECIMAL_128 + NULL + BINARY + CALENDAR + ARRAY + STRUCT +
       UDT).nested()
 
-  /**
-   * commonCudfTypes plus decimal, null and nested types.
-   */
-  val commonCudfTypesWithNested: TypeSig = (commonCudfTypes + DECIMAL_128 + NULL +
-      ARRAY + STRUCT + MAP).nested()
-
   /**
    * Different types of Pandas UDF support different sets of output type. Please refer to
    *   https://github.com/apache/spark/blob/master/python/pyspark/sql/udf.py#L98
diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/protobuf/ProtobufDescriptorUtils.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/protobuf/ProtobufDescriptorUtils.scala
new file mode 100644
index 00000000000..cabc8d2905d
--- /dev/null
+++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/protobuf/ProtobufDescriptorUtils.scala
@@ -0,0 +1,87 @@
+/*
+ * Copyright (c) 2026, NVIDIA CORPORATION.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.nvidia.spark.rapids.protobuf
+
+import scala.collection.JavaConverters._
+import scala.collection.mutable
+
+import com.google.protobuf.DescriptorProtos
+import com.google.protobuf.Descriptors
+
+/**
+ * Minimal descriptor utilities for locating a message descriptor in a FileDescriptorSet.
+ *
+ * This is intentionally lightweight for the "simple types" from_protobuf patch: it supports
+ * descriptor sets produced by `protoc --include_imports --descriptor_set_out=...`.
+ *
+ * NOTE: This utility is currently not used in the initial implementation, which relies on
+ * Spark's ProtobufUtils via reflection (buildMessageDescriptorWithSparkProtobuf). This class
+ * is preserved for potential future use cases where direct descriptor parsing is needed
+ * without depending on Spark's shaded protobuf classes.
+ */
+object ProtobufDescriptorUtils {
+
+  def buildMessageDescriptor(
+      fileDescriptorSetBytes: Array[Byte],
+      messageName: String): Descriptors.Descriptor = {
+    val fds = DescriptorProtos.FileDescriptorSet.parseFrom(fileDescriptorSetBytes)
+    val protos = fds.getFileList.asScala.toSeq
+    val byName = protos.map(p => p.getName -> p).toMap
+    val cache = mutable.HashMap.empty[String, Descriptors.FileDescriptor]
+
+    def buildFileDescriptor(name: String): Descriptors.FileDescriptor = {
+      cache.getOrElseUpdate(name, {
+        val p = byName.getOrElse(name,
+          throw new IllegalArgumentException(s"Missing FileDescriptorProto for '$name'"))
+        val deps = p.getDependencyList.asScala.map(buildFileDescriptor _).toArray
+        Descriptors.FileDescriptor.buildFrom(p, deps)
+      })
+    }
+
+    val fileDescriptors = protos.map(p => buildFileDescriptor(p.getName))
+    val candidates = fileDescriptors.iterator.flatMap(fd => findMessageDescriptors(fd, messageName))
+      .toSeq
+
+    candidates match {
+      case Seq(d) => d
+      case Seq() =>
+        throw new IllegalArgumentException(
+          s"Message '$messageName' not found in FileDescriptorSet")
+      case many =>
+        val names = many.map(_.getFullName).distinct.sorted
+        throw new IllegalArgumentException(
+          s"Message '$messageName' is ambiguous; matches: ${names.mkString(", ")}")
+    }
+  }
+
+  private def findMessageDescriptors(
+      fd: Descriptors.FileDescriptor,
+      messageName: String): Iterator[Descriptors.Descriptor] = {
+    def matches(d: Descriptors.Descriptor): Boolean = {
+      d.getName == messageName ||
+        d.getFullName == messageName ||
+        d.getFullName.endsWith("." + messageName)
+    }
+
+    def walk(d: Descriptors.Descriptor): Iterator[Descriptors.Descriptor] = {
+      val nested = d.getNestedTypes.asScala.iterator.flatMap(walk _)
+      if (matches(d)) Iterator.single(d) ++ nested else nested
+    }
+
+    fd.getMessageTypes.asScala.iterator.flatMap(walk _)
+  }
+}
\ No newline at end of file
diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuFromProtobuf.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuFromProtobuf.scala
new file mode 100644
index 00000000000..290670fff91
--- /dev/null
+++ b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuFromProtobuf.scala
@@ -0,0 +1,384 @@
+/*
+ * Copyright (c) 2025-2026, NVIDIA CORPORATION.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.rapids
+
+import ai.rapids.cudf
+import ai.rapids.cudf.{BinaryOp, CudfException, DType}
+import com.nvidia.spark.rapids.{GpuColumnVector, GpuUnaryExpression}
+import com.nvidia.spark.rapids.Arm.withResource
+import com.nvidia.spark.rapids.jni.Protobuf
+import com.nvidia.spark.rapids.shims.NullIntolerantShim
+
+import org.apache.spark.sql.catalyst.expressions.{ExpectsInputTypes, Expression}
+import org.apache.spark.sql.types._
+
+/**
+ * GPU implementation for Spark's `from_protobuf` decode path.
+ *
+ * This is designed to replace `org.apache.spark.sql.protobuf.ProtobufDataToCatalyst` when
+ * supported.
+ *
+ * The implementation uses a flattened schema representation where nested fields have parent
+ * indices pointing to their containing message field. For pure scalar schemas, all fields
+ * are top-level (parentIndices == -1, depthLevels == 0, isRepeated == false).
+ *
+ * Schema projection is supported: only fields in `decodedTopLevelIndices` are decoded by
+ * the GPU. Fields not in this array will be filled with null columns in post-processing
+ * to ensure the output matches `fullSchema`.
+ *
+ * @param fullSchema The complete output schema (must match the original expression's dataType)
+ * @param decodedTopLevelIndices Indices in fullSchema for top-level fields decoded by GPU.
+ *                               Must be sorted in ascending order.
+ * @param fieldNumbers Protobuf field numbers for all fields in flattened schema
+ * @param parentIndices Parent indices for all fields (-1 for top-level)
+ * @param depthLevels Nesting depth for all fields (0 for top-level)
+ * @param wireTypes Wire types for all fields
+ * @param outputTypeIds cuDF type IDs for all fields
+ * @param encodings Encodings for all fields
+ * @param isRepeated Whether each field is repeated
+ * @param isRequired Whether each field is required
+ * @param hasDefaultValue Whether each field has a default value
+ * @param defaultInts Default int/long values
+ * @param defaultFloats Default float/double values
+ * @param defaultBools Default bool values
+ * @param defaultStrings Default string/bytes values
+ * @param enumValidValues Valid enum values for each field
+ * @param enumNames Enum value names for enum-as-string fields. Parallel to enumValidValues.
+ * @param nestedPrunedFields For nested schema projection: maps top-level field name to the
+ *                           ordered list of decoded child field names. Only present for fields
+ *                           where children were pruned. If empty map, no nested pruning.
+ * @param failOnErrors If true, throw exception on malformed data
+ */
+case class GpuFromProtobuf(
+    fullSchema: StructType,
+    decodedTopLevelIndices: Array[Int],
+    fieldNumbers: Array[Int],
+    parentIndices: Array[Int],
+    depthLevels: Array[Int],
+    wireTypes: Array[Int],
+    outputTypeIds: Array[Int],
+    encodings: Array[Int],
+    isRepeated: Array[Boolean],
+    isRequired: Array[Boolean],
+    hasDefaultValue: Array[Boolean],
+    defaultInts: Array[Long],
+    defaultFloats: Array[Double],
+    defaultBools: Array[Boolean],
+    defaultStrings: Array[Array[Byte]],
+    enumValidValues: Array[Array[Int]],
+    enumNames: Array[Array[Array[Byte]]],
+    nestedPrunedFields: Map[String, Seq[String]],
+    failOnErrors: Boolean,
+    child: Expression)
+  extends GpuUnaryExpression with ExpectsInputTypes with NullIntolerantShim {
+
+  override def inputTypes: Seq[AbstractDataType] = Seq(BinaryType)
+
+  /**
+   * Build a partially pruned schema for the output:
+   * - Top-level: matches fullSchema (so GetStructField ordinals stay valid)
+   * - Nested ArrayType(StructType) fields: pruned to only contain decoded children
+   *   (GetArrayStructFields ordinals are remapped in GpuGetArrayStructFieldsMeta)
+   * - Nested StructType (non-repeated) fields: kept at full schema
+   *   (expansion inserts null columns cheaply; GetStructField ordinals stay valid)
+   */
+  private val outputSchema: StructType = {
+    val fields = fullSchema.fields.map { field =>
+      nestedPrunedFields.get(field.name) match {
+        case Some(childNames) =>
+          field.dataType match {
+            case ArrayType(st: StructType, containsNull) =>
+              // Pruned repeated message: only keep decoded children (Option A)
+              val prunedSt = StructType(
+                st.fields.filter(f => childNames.contains(f.name)))
+              field.copy(dataType = ArrayType(prunedSt, containsNull))
+            case _ =>
+              // Non-repeated struct or other: keep full type (expand with null cols)
+              field
+          }
+        case None => field
+      }
+    }
+    StructType(fields).asNullable
+  }
+
+  override def dataType: DataType = outputSchema
+
+  override def nullable: Boolean = true
+
+  // Identify which pruned fields are non-repeated structs (need expansion)
+  // vs repeated messages (pruned in outputSchema, no expansion needed)
+  private val nonRepeatedPrunedFields: Map[String, Seq[String]] =
+    nestedPrunedFields.filter { case (fieldName, _) =>
+      val idx = fullSchema.fieldIndex(fieldName)
+      fullSchema.fields(idx).dataType.isInstanceOf[StructType]
+    }
+
+  // Expansion is needed when not all top-level fields are decoded,
+  // or when non-repeated structs have pruned children
+  private val needsExpansion: Boolean =
+    decodedTopLevelIndices.length != fullSchema.fields.length ||
+    nonRepeatedPrunedFields.nonEmpty
+
+  override protected def doColumnar(input: GpuColumnVector): cudf.ColumnVector = {
+    val numRows = input.getRowCount.toInt
+    val jniResult = try {
+      Protobuf.decodeToStruct(
+        input.getBase,
+        fieldNumbers,
+        parentIndices,
+        depthLevels,
+        wireTypes,
+        outputTypeIds,
+        encodings,
+        isRepeated,
+        isRequired,
+        hasDefaultValue,
+        defaultInts,
+        defaultFloats,
+        defaultBools,
+        defaultStrings,
+        enumValidValues,
+        enumNames,
+        failOnErrors)
+    } catch {
+      case e: CudfException if failOnErrors =>
+        throw new org.apache.spark.SparkException("Malformed protobuf message", e)
+    }
+
+    // Expand if needed:
+    // - Top-level: insert null columns for non-decoded top-level fields
+    // - Non-repeated structs: expand pruned children with null columns
+    // - Repeated messages (ArrayType(StructType)): already pruned in outputSchema,
+    //   handled by GetArrayStructFields ordinal remapping (no expansion needed)
+    val expanded = if (needsExpansion) {
+      withResource(jniResult) { decoded =>
+        expandSchema(decoded, numRows)
+      }
+    } else {
+      jniResult
+    }
+
+    // Apply input nulls to output
+    if (input.getBase.hasNulls) {
+      withResource(expanded) { _ =>
+        expanded.mergeAndSetValidity(BinaryOp.BITWISE_AND, input.getBase)
+      }
+    } else {
+      expanded
+    }
+  }
+
+  /**
+   * Expand decoded struct to match outputSchema:
+   * - Top-level: insert null columns for non-decoded fields
+   * - Non-repeated structs with pruned children: expand by inserting null child columns
+   * - Repeated messages (ArrayType): no expansion (already pruned in outputSchema)
+   */
+  private def expandSchema(decoded: cudf.ColumnVector, numRows: Int): cudf.ColumnVector = {
+    val children = new Array[cudf.ColumnVector](fullSchema.fields.length)
+    var decodedIdx = 0
+
+    try {
+      for (i <- fullSchema.fields.indices) {
+        if (decodedIdx < decodedTopLevelIndices.length &&
+            decodedTopLevelIndices(decodedIdx) == i) {
+          val fieldName = fullSchema.fields(i).name
+
+          // Check if this is a non-repeated struct with pruned children
+          nonRepeatedPrunedFields.get(fieldName) match {
+            case Some(decodedNames) =>
+              // Expand non-repeated struct: insert null columns for pruned children
+              val targetSt = fullSchema.fields(i).dataType.asInstanceOf[StructType]
+              withResource(decoded.getChildColumnView(decodedIdx)) { childView =>
+                children(i) = expandStructChildren(childView, targetSt, decodedNames, numRows)
+              }
+            case None =>
+              // No expansion needed (either full schema or ArrayType pruned via Option A)
+              withResource(decoded.getChildColumnView(decodedIdx)) { childView =>
+                children(i) = childView.copyToColumnVector()
+              }
+          }
+          decodedIdx += 1
+        } else {
+          // This field was not decoded — create null column
+          children(i) = GpuColumnVector.columnVectorFromNull(
+            numRows, outputSchema.fields(i).dataType)
+        }
+      }
+
+      cudf.ColumnVector.makeStruct(numRows, children: _*)
+    } finally {
+      children.foreach(col => if (col != null) col.close())
+    }
+  }
+
+  /**
+   * Expand a non-repeated STRUCT column by inserting null columns for pruned children.
+   * This is cheap because non-repeated structs have only num_rows elements.
+   */
+  private def expandStructChildren(
+      decoded: cudf.ColumnView,
+      targetSchema: StructType,
+      decodedChildNames: Seq[String],
+      numRows: Int): cudf.ColumnVector = {
+    val children = new Array[cudf.ColumnVector](targetSchema.fields.length)
+    var decodedChildIdx = 0
+
+    try {
+      for (i <- targetSchema.fields.indices) {
+        if (decodedChildIdx < decodedChildNames.length &&
+            decodedChildNames(decodedChildIdx) == targetSchema.fields(i).name) {
+          withResource(decoded.getChildColumnView(decodedChildIdx)) { childView =>
+            children(i) = childView.copyToColumnVector()
+          }
+          decodedChildIdx += 1
+        } else {
+          children(i) = GpuColumnVector.columnVectorFromNull(
+            numRows, targetSchema.fields(i).dataType)
+        }
+      }
+
+      cudf.ColumnVector.makeStruct(numRows, children: _*)
+    } finally {
+      children.foreach(col => if (col != null) col.close())
+    }
+  }
+}
+
+object GpuFromProtobuf {
+  // Encodings from com.nvidia.spark.rapids.jni.Protobuf
+  val ENC_DEFAULT = 0
+  val ENC_FIXED   = 1
+  val ENC_ZIGZAG  = 2
+  val ENC_ENUM_STRING = 3
+
+  // Thread-local registry for pruned field ordinal mappings.
+  // When GpuFromProtobuf is created with nested pruning, it registers
+  // the field name -> pruned ordinal mapping. GpuGetArrayStructFieldsMeta
+  // reads this during convertToGpu to remap ordinals.
+  private val prunedFieldOrdinals = new ThreadLocal[Map[String, Int]]() {
+    override def initialValue(): Map[String, Int] = Map.empty
+  }
+
+  def registerPrunedFields(mappings: Map[String, Int]): Unit = {
+    prunedFieldOrdinals.set(prunedFieldOrdinals.get() ++ mappings)
+  }
+
+  def getPrunedOrdinal(fieldName: String): Int = {
+    prunedFieldOrdinals.get().getOrElse(fieldName, -1)
+  }
+
+  def clearPrunedFields(): Unit = {
+    prunedFieldOrdinals.set(Map.empty)
+  }
+
+  /**
+   * Maps a Spark DataType to the corresponding cuDF native type ID.
+   * Note: The encoding (varint/zigzag/fixed) is determined by the protobuf field type,
+   * not the Spark data type, so it must be set separately based on the protobuf schema.
+   *
+   * @return Some(typeId) for supported types, None for unsupported types
+   */
+  def sparkTypeToCudfIdOpt(dt: DataType): Option[Int] = dt match {
+    case BooleanType => Some(DType.BOOL8.getTypeId.getNativeId)
+    case IntegerType => Some(DType.INT32.getTypeId.getNativeId)
+    case LongType => Some(DType.INT64.getTypeId.getNativeId)
+    case FloatType => Some(DType.FLOAT32.getTypeId.getNativeId)
+    case DoubleType => Some(DType.FLOAT64.getTypeId.getNativeId)
+    case StringType => Some(DType.STRING.getTypeId.getNativeId)
+    case BinaryType => Some(DType.LIST.getTypeId.getNativeId)
+    case _ => None
+  }
+
+  /**
+   * Check if a Spark DataType is supported by the GPU protobuf decoder.
+   */
+  def isTypeSupported(dt: DataType): Boolean = sparkTypeToCudfIdOpt(dt).isDefined
+
+  /**
+   * Create an all-null column of the specified Spark DataType.
+   * This is used for fields with unsupported types (nested structs, arrays, etc.)
+   * that are not decoded but need to be present in the output struct.
+   */
+  def createNullColumn(dt: DataType, numRows: Int): cudf.ColumnVector = {
+    // Helper to create null arrays for boxed types
+    def nullBools = Array.fill[java.lang.Boolean](numRows)(null)
+    def nullInts = Array.fill[java.lang.Integer](numRows)(null)
+    def nullLongs = Array.fill[java.lang.Long](numRows)(null)
+    def nullFloats = Array.fill[java.lang.Float](numRows)(null)
+    def nullDoubles = Array.fill[java.lang.Double](numRows)(null)
+
+    dt match {
+      case BooleanType => cudf.ColumnVector.fromBoxedBooleans(nullBools: _*)
+      case IntegerType => cudf.ColumnVector.fromBoxedInts(nullInts: _*)
+      case LongType => cudf.ColumnVector.fromBoxedLongs(nullLongs: _*)
+      case FloatType => cudf.ColumnVector.fromBoxedFloats(nullFloats: _*)
+      case DoubleType => cudf.ColumnVector.fromBoxedDoubles(nullDoubles: _*)
+      case StringType => cudf.ColumnVector.fromStrings(Array.fill[String](numRows)(null): _*)
+      case BinaryType =>
+        // Binary is LIST<INT8> - create all-null list column using Scalar API
+        val elementType = new cudf.HostColumnVector.BasicType(true, DType.INT8)
+        withResource(cudf.Scalar.listFromNull(elementType)) { nullScalar =>
+          cudf.ColumnVector.fromScalar(nullScalar, numRows)
+        }
+      case st: StructType =>
+        // Recursively create null columns for struct fields
+        val children = st.fields.map(f => createNullColumn(f.dataType, numRows))
+        try {
+          withResource(cudf.ColumnVector.makeStruct(numRows, children: _*)) { structCol =>
+            // Set all rows to null - mergeAndSetValidity returns a NEW column
+            withResource(cudf.ColumnVector.fromBoxedBooleans(nullBools: _*)) { nullMask =>
+              structCol.mergeAndSetValidity(BinaryOp.BITWISE_AND, nullMask)
+            }
+          }
+        } finally {
+          children.foreach(_.close())
+        }
+      case ArrayType(elementType, _) =>
+        // Create empty arrays with all nulls using Scalar API
+        val cudfElementDType = sparkTypeToCudfIdOpt(elementType)
+          .map(id => DType.fromNative(id, 0))
+          .getOrElse(DType.INT8)  // fallback for nested complex types
+        val elemType = new cudf.HostColumnVector.BasicType(true, cudfElementDType)
+        withResource(cudf.Scalar.listFromNull(elemType)) { nullScalar =>
+          cudf.ColumnVector.fromScalar(nullScalar, numRows)
+        }
+      case MapType(keyType, valueType, _) =>
+        // Maps are represented as LIST<STRUCT<key, value>> in cuDF
+        // For all-null maps, we create a list column with STRUCT<key, value> element type
+        val cudfKeyDType = sparkTypeToCudfIdOpt(keyType)
+          .map(id => DType.fromNative(id, 0))
+          .getOrElse(DType.INT8)
+        val cudfValueDType = sparkTypeToCudfIdOpt(valueType)
+          .map(id => DType.fromNative(id, 0))
+          .getOrElse(DType.INT8)
+        // Create the struct type for map entries (key, value)
+        val keyFieldType = new cudf.HostColumnVector.BasicType(true, cudfKeyDType)
+        val valueFieldType = new cudf.HostColumnVector.BasicType(true, cudfValueDType)
+        val structType = new cudf.HostColumnVector.StructType(true, keyFieldType, valueFieldType)
+        // Create an all-null map column (list of structs)
+        withResource(cudf.Scalar.listFromNull(structType)) { nullScalar =>
+          cudf.ColumnVector.fromScalar(nullScalar, numRows)
+        }
+      case _ =>
+        // Fallback for any other types - create INT8 nulls as placeholder
+        // This should not happen in practice since unsupported types should be caught earlier
+        cudf.ColumnVector.fromBoxedBytes(Array.fill[java.lang.Byte](numRows)(null): _*)
+    }
+  }
+}
diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/complexTypeExtractors.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/complexTypeExtractors.scala
index bfd96dd7b87..972622bb825 100644
--- a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/complexTypeExtractors.scala
+++ b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/complexTypeExtractors.scala
@@ -409,8 +409,12 @@ class GpuGetArrayStructFieldsMeta(
      rule: DataFromReplacementRule)
   extends UnaryExprMeta[GetArrayStructFields](expr, conf, parent, rule) {
 
-  def convertToGpu(child: Expression): GpuExpression =
-    GpuGetArrayStructFields(child, expr.field, expr.ordinal, expr.numFields, expr.containsNull)
+  def convertToGpu(child: Expression): GpuExpression = {
+    // Check the global protobuf pruned field registry to remap ordinal
+    val runtimeOrd = GpuFromProtobuf.getPrunedOrdinal(expr.field.name)
+    GpuGetArrayStructFields(child, expr.field, expr.ordinal, expr.numFields,
+      expr.containsNull, runtimeOrd)
+  }
 }
 
 /**
@@ -424,7 +428,8 @@ case class GpuGetArrayStructFields(
     field: StructField,
     ordinal: Int,
     numFields: Int,
-    containsNull: Boolean) extends GpuUnaryExpression
+    containsNull: Boolean,
+    runtimeOrdinal: Int = -1) extends GpuUnaryExpression
     with ShimGetArrayStructFields
     with NullIntolerantShim {
 
@@ -435,7 +440,16 @@ case class GpuGetArrayStructFields(
   override protected def doColumnar(input: GpuColumnVector): ColumnVector = {
     val base = input.getBase
     val fieldView = withResource(base.getChildColumnView(0)) { structView =>
-      structView.getChildColumnView(ordinal)
+      val actualChildren = structView.getNumChildren
+      // Handle nested schema projection (Option A): the actual struct may have
+      // fewer children than numFields if GpuFromProtobuf pruned the schema.
+      // Use runtimeOrdinal (if set) when the struct was pruned.
+      val effectiveOrdinal = if (runtimeOrdinal >= 0 && actualChildren < numFields) {
+        runtimeOrdinal
+      } else {
+        ordinal
+      }
+      structView.getChildColumnView(effectiveOrdinal)
     }
     val listView = withResource(fieldView) { _ =>
       GpuListUtils.replaceListDataColumnAsView(base, fieldView)
diff --git a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala
new file mode 100644
index 00000000000..10061af5cc6
--- /dev/null
+++ b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala
@@ -0,0 +1,1013 @@
+/*
+ * Copyright (c) 2026, NVIDIA CORPORATION.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*** spark-rapids-shim-json-lines
+{"spark": "340"}
+{"spark": "341"}
+{"spark": "342"}
+{"spark": "343"}
+{"spark": "344"}
+{"spark": "350"}
+{"spark": "351"}
+{"spark": "352"}
+{"spark": "353"}
+{"spark": "354"}
+{"spark": "355"}
+{"spark": "356"}
+{"spark": "357"}
+{"spark": "400"}
+{"spark": "401"}
+spark-rapids-shim-json-lines ***/
+
+package com.nvidia.spark.rapids.shims
+
+import scala.collection.mutable
+import scala.util.Try
+
+import ai.rapids.cudf.DType
+import com.nvidia.spark.rapids._
+import com.nvidia.spark.rapids.jni.Protobuf.{WT_32BIT, WT_64BIT, WT_LEN, WT_VARINT}
+
+import org.apache.spark.sql.catalyst.expressions.{
+  AttributeReference, Expression, GetArrayStructFields, GetStructField, UnaryExpression
+}
+import org.apache.spark.sql.execution.ProjectExec
+import org.apache.spark.sql.rapids.GpuFromProtobuf
+import org.apache.spark.sql.types._
+
+/**
+ * Information about a protobuf field for schema projection support.
+ */
+private[shims] case class ProtobufFieldInfo(
+    fieldNumber: Int,
+    protoTypeName: String,
+    sparkType: DataType,
+    encoding: Int,
+    isSupported: Boolean,
+    unsupportedReason: Option[String],
+    isRequired: Boolean,
+    hasDefaultValue: Boolean,
+    defaultValue: Option[Any],  // Stored as protobuf-java type, will be converted for JNI
+    // Valid enum values for ENUM fields (used for validation)
+    enumValues: Option[Set[Int]] = None,
+    // Enum value-name mapping for ENUM -> STRING decoding (enumsAsInts=false)
+    enumNames: Option[Map[Int, String]] = None,
+    isRepeated: Boolean = false  // Whether this is a repeated field
+)
+
+/**
+ * Flattened field descriptor for nested protobuf schemas.
+ * Used to represent a hierarchical schema as a linear array for GPU processing.
+ */
+private[shims] case class FlattenedFieldDescriptor(
+    fieldNumber: Int,
+    parentIdx: Int,          // Index of parent field in flattened array (-1 for top-level)
+    depth: Int,              // Nesting depth (0 for top-level)
+    wireType: Int,           // Protobuf wire type
+    outputTypeId: Int,       // cudf type id for the output (element type for repeated)
+    encoding: Int,           // Encoding (default/fixed/zigzag)
+    isRepeated: Boolean,     // Whether this is a repeated field
+    isRequired: Boolean,     // Whether this field is required (proto2)
+    hasDefaultValue: Boolean,
+    defaultInt: Long,
+    defaultFloat: Double,
+    defaultBool: Boolean,
+    defaultString: Array[Byte],
+    enumValidValues: Array[Int],
+    enumNames: Array[Array[Byte]]
+)
+
+/**
+ * Spark 3.4+ optional integration for spark-protobuf expressions.
+ *
+ * spark-protobuf is an external module, so these rules must be registered by reflection.
+ */
+object ProtobufExprShims {
+  private[this] val protobufDataToCatalystClassName =
+    "org.apache.spark.sql.protobuf.ProtobufDataToCatalyst"
+
+  private[this] val sparkProtobufUtilsObjectClassName =
+    "org.apache.spark.sql.protobuf.utils.ProtobufUtils$"
+
+  def exprs: Map[Class[_ <: Expression], ExprRule[_ <: Expression]] = {
+    try {
+      val clazz = ShimReflectionUtils.loadClass(protobufDataToCatalystClassName)
+        .asInstanceOf[Class[_ <: UnaryExpression]]
+      Map(clazz.asInstanceOf[Class[_ <: Expression]] -> fromProtobufRule)
+    } catch {
+      case _: ClassNotFoundException => Map.empty
+    }
+  }
+
+  private def fromProtobufRule: ExprRule[_ <: Expression] = {
+    GpuOverrides.expr[UnaryExpression](
+      "Decode a BinaryType column (protobuf) into a Spark SQL struct",
+      ExprChecks.unaryProject(
+        // Use TypeSig.all here because schema projection determines which fields
+        // actually need GPU support. Detailed type checking is done in tagExprForGpu.
+        TypeSig.all,
+        TypeSig.all,
+        TypeSig.BINARY,
+        TypeSig.BINARY),
+      (e, conf, p, r) => new UnaryExprMeta[UnaryExpression](e, conf, p, r) {
+
+        // Full schema from the expression (must match original dataType for compatibility)
+        private var fullSchema: StructType = _
+        private var failOnErrors: Boolean = _
+
+        // Flattened schema variables for GPU decoding
+        private var flatFieldNumbers: Array[Int] = _
+        private var flatParentIndices: Array[Int] = _
+        private var flatDepthLevels: Array[Int] = _
+        private var flatWireTypes: Array[Int] = _
+        private var flatOutputTypeIds: Array[Int] = _
+        private var flatEncodings: Array[Int] = _
+        private var flatIsRepeated: Array[Boolean] = _
+        private var flatIsRequired: Array[Boolean] = _
+        private var flatHasDefaultValue: Array[Boolean] = _
+        private var flatDefaultInts: Array[Long] = _
+        private var flatDefaultFloats: Array[Double] = _
+        private var flatDefaultBools: Array[Boolean] = _
+        private var flatDefaultStrings: Array[Array[Byte]] = _
+        private var flatEnumValidValues: Array[Array[Int]] = _
+        private var flatEnumNames: Array[Array[Array[Byte]]] = _
+        // Indices in fullSchema for top-level fields that were decoded (for schema projection)
+        private var decodedTopLevelIndices: Array[Int] = _
+
+        override def tagExprForGpu(): Unit = {
+          fullSchema = e.dataType match {
+            case st: StructType => st
+            case other =>
+              willNotWorkOnGpu(
+                s"Only StructType output is supported for from_protobuf, got $other")
+              return
+          }
+
+          val options = getOptionsMap(e)
+          val supportedOptions = Set("enums.as.ints", "mode")
+          val unsupportedOptions = options.keys.filterNot(supportedOptions.contains)
+          if (unsupportedOptions.nonEmpty) {
+            val keys = unsupportedOptions.mkString(",")
+            willNotWorkOnGpu(
+              s"from_protobuf options are not supported yet on GPU: $keys")
+            return
+          }
+
+          val enumsAsInts = options.getOrElse("enums.as.ints", "false").toBoolean
+          failOnErrors = options.getOrElse("mode", "PERMISSIVE").equalsIgnoreCase("FAILFAST")
+          val messageName = getMessageName(e)
+
+          // Try to get descriptor: either file path (Spark 3.4.x) or binary bytes (Spark 3.5+)
+          val descFilePathOrBytes: Option[Either[String, Array[Byte]]] =
+            getDescFilePath(e).map(Left(_)).orElse {
+              getDescriptorBytes(e).map(Right(_))
+            }
+
+          if (descFilePathOrBytes.isEmpty) {
+            willNotWorkOnGpu(
+              "from_protobuf requires a descriptor set " +
+                "(descFilePath or binaryFileDescriptorSet)")
+            return
+          }
+
+          val msgDesc = try {
+            // Spark 3.4.x: buildDescriptor(messageName, descFilePath: Option[String])
+            // Spark 3.5+:  buildDescriptor(messageName, binaryFileDescriptorSet)
+            buildMessageDescriptorWithSparkProtobuf(messageName, descFilePathOrBytes.get)
+          } catch {
+            case t: Throwable =>
+              willNotWorkOnGpu(
+                s"Failed to resolve protobuf descriptor for message '$messageName': " +
+                  s"${t.getMessage}")
+              return
+          }
+
+          // Reject proto3 descriptors — GPU decoder only supports proto2 semantics.
+          // proto3 has different null/default-value behavior that the GPU path doesn't handle.
+          val protoSyntax = Try {
+            val fileDesc = invoke0[AnyRef](msgDesc, "getFile")
+            val syntaxObj = invoke0[AnyRef](fileDesc, "getSyntax")
+            typeName(syntaxObj)
+          }.getOrElse("")
+          if (protoSyntax == "PROTO3") {
+            willNotWorkOnGpu("proto3 descriptors are not supported; only proto2 is supported")
+            return
+          }
+
+          // Step 1: Analyze all fields and build field info map
+          val allFieldsInfo = analyzeAllFields(fullSchema, msgDesc, enumsAsInts, messageName)
+          if (allFieldsInfo.isEmpty) {
+            // Error was already reported in analyzeAllFields
+            return
+          }
+          val fieldsInfoMap = allFieldsInfo.get
+
+          // Step 2: Determine which fields are actually required by downstream operations
+          val requiredFieldNames = analyzeRequiredFields(fieldsInfoMap.keySet)
+
+          // Step 3: Check if all required fields are supported
+          val unsupportedRequired = requiredFieldNames.filter { name =>
+            fieldsInfoMap.get(name).exists(!_.isSupported)
+          }
+
+          if (unsupportedRequired.nonEmpty) {
+            val reasons = unsupportedRequired.map { name =>
+              val info = fieldsInfoMap(name)
+              s"${name}: ${info.unsupportedReason.getOrElse("unknown reason")}"
+            }
+            willNotWorkOnGpu(
+              s"Required fields not supported for from_protobuf: ${reasons.mkString(", ")}")
+            return
+          }
+
+          // Step 4: Identify which fields in fullSchema need to be decoded
+          // These are fields that are required AND supported
+          val indicesToDecode = fullSchema.fields.zipWithIndex.collect {
+            case (sf, idx) if requiredFieldNames.contains(sf.name) => idx
+          }
+
+          // Verify all fields to be decoded are actually supported
+          // (This catches edge cases where field analysis might have issues)
+          val unsupportedInDecode = indicesToDecode.filter { idx =>
+            val sf = fullSchema.fields(idx)
+            fieldsInfoMap.get(sf.name).exists(!_.isSupported)
+          }
+          if (unsupportedInDecode.nonEmpty) {
+            val reasons = unsupportedInDecode.map { idx =>
+              val sf = fullSchema.fields(idx)
+              val info = fieldsInfoMap(sf.name)
+              s"${sf.name}: ${info.unsupportedReason.getOrElse("unknown reason")}"
+            }
+            willNotWorkOnGpu(
+              s"Fields not supported for from_protobuf: ${reasons.mkString(", ")}")
+            return
+          }
+
+          // Step 5: Build flattened schema for GPU decoding.
+          // The flattened schema represents nested fields with parent indices.
+          // For pure scalar schemas, all fields are top-level (parentIdx == -1, depth == 0).
+          {
+            val flatFields = mutable.ArrayBuffer[FlattenedFieldDescriptor]()
+
+            // Helper to add a field and its children recursively
+            def addFieldWithChildren(
+                sf: StructField,
+                info: ProtobufFieldInfo,
+                parentIdx: Int,
+                depth: Int,
+                nestedMsgDesc: AnyRef): Unit = {
+
+              val currentIdx = flatFields.size
+
+              val outputType = sf.dataType match {
+                case ArrayType(elemType, _) =>
+                  elemType match {
+                    case _: StructType =>
+                      // Repeated message field: ArrayType(StructType) - element type is STRUCT
+                      DType.STRUCT.getTypeId.getNativeId
+                    case other =>
+                      GpuFromProtobuf.sparkTypeToCudfIdOpt(other)
+                        .getOrElse(DType.INT8.getTypeId.getNativeId)
+                  }
+                case _: StructType =>
+                  DType.STRUCT.getTypeId.getNativeId
+                case other =>
+                  GpuFromProtobuf.sparkTypeToCudfIdOpt(other)
+                    .getOrElse(DType.INT8.getTypeId.getNativeId)
+              }
+
+              val wireType = getWireType(info.protoTypeName, info.encoding)
+
+              val hasDefault = info.hasDefaultValue && info.defaultValue.isDefined
+              val (defInt, defFloat, defBool, defString) = if (hasDefault) {
+                val defVal = info.defaultValue.get
+                sf.dataType match {
+                  case BooleanType =>
+                    val b = defVal.asInstanceOf[java.lang.Boolean].booleanValue()
+                    (0L, 0.0, b, null: Array[Byte])
+                  case IntegerType | LongType =>
+                    val intVal = defVal match {
+                      case i: java.lang.Integer => i.longValue()
+                      case l: java.lang.Long => l.longValue()
+                      case _ => 0L
+                    }
+                    (intVal, 0.0, false, null: Array[Byte])
+                  case FloatType =>
+                    val f = defVal.asInstanceOf[java.lang.Float].doubleValue()
+                    (0L, f, false, null: Array[Byte])
+                  case DoubleType =>
+                    val d = defVal.asInstanceOf[java.lang.Double].doubleValue()
+                    (0L, d, false, null: Array[Byte])
+                  case StringType =>
+                    val str = defVal.asInstanceOf[String]
+                    val bytes = if (str != null) str.getBytes("UTF-8") else null
+                    (0L, 0.0, false, bytes)
+                  case _ => (0L, 0.0, false, null: Array[Byte])
+                }
+              } else {
+                (0L, 0.0, false, null: Array[Byte])
+              }
+
+              val enumValsArr = info.enumValues.map(_.toArray.sorted).orNull
+              val enumNamesArr = info.enumNames.map { nameMap =>
+                val sorted = nameMap.toSeq.sortBy(_._1)
+                sorted.map { case (_, enumName) => enumName.getBytes("UTF-8") }.toArray
+              }.orNull
+
+              flatFields += FlattenedFieldDescriptor(
+                fieldNumber = info.fieldNumber,
+                parentIdx = parentIdx,
+                depth = depth,
+                wireType = wireType,
+                outputTypeId = outputType,
+                encoding = info.encoding,
+                isRepeated = info.isRepeated,
+                isRequired = info.isRequired,
+                hasDefaultValue = info.hasDefaultValue,
+                defaultInt = defInt,
+                defaultFloat = defFloat,
+                defaultBool = defBool,
+                defaultString = defString,
+                enumValidValues = enumValsArr,
+                enumNames = enumNamesArr
+              )
+
+              // For nested struct types (including repeated message = ArrayType(StructType)), 
+              // add child fields
+              sf.dataType match {
+                case st: StructType if nestedMsgDesc != null =>
+                  // Non-repeated nested message
+                  addChildFieldsFromStruct(st, nestedMsgDesc, sf.name, currentIdx, depth)
+                  
+                case ArrayType(st: StructType, _) if nestedMsgDesc != null =>
+                  // Repeated message field (pruned via Option A ordinal remapping)
+                  addChildFieldsFromStruct(st, nestedMsgDesc, sf.name, currentIdx, depth)
+                  
+                case _ => // Not a struct, no children to add
+              }
+            }
+            
+            // Helper to add child fields from a struct type.
+            // Applies nested schema pruning for ALL struct types (both repeated and non-repeated).
+            // For repeated message fields (ArrayType(StructType)), the pruned output is handled
+            // by ordinal remapping in GpuGetArrayStructFieldsMeta (Option A), not by expansion.
+            def addChildFieldsFromStruct(
+                st: StructType,
+                parentMsgDesc: AnyRef,
+                fieldName: String,
+                parentIdx: Int,
+                parentDepth: Int): Unit = {
+              val fd = invoke1[AnyRef](
+                parentMsgDesc, "findFieldByName", classOf[String], fieldName)
+              if (fd != null) {
+                Try {
+                  val childMsgDesc = invoke0[AnyRef](fd, "getMessageType")
+                  // Filter children based on nested schema projection requirements.
+                  val requiredChildren = nestedFieldRequirements.get(fieldName)
+                  val filteredFields = requiredChildren match {
+                    case Some(Some(childNames)) =>
+                      // Only add required children (nested schema projection)
+                      st.fields.filter(f => childNames.contains(f.name))
+                    case _ =>
+                      // None (field not in map) or Some(None) (whole field needed)
+                      st.fields
+                  }
+                  filteredFields.foreach { childSf =>
+                    val childFd = invoke1[AnyRef](
+                      childMsgDesc, "findFieldByName", classOf[String], childSf.name)
+                    if (childFd != null) {
+                      val childProtoType = invoke0[AnyRef](childFd, "getType")
+                      val childProtoTypeName = typeName(childProtoType)
+                      val childFieldNumber = invoke0[java.lang.Integer](
+                        childFd, "getNumber").intValue()
+                      val childIsRepeated = Try {
+                        invoke0[java.lang.Boolean](childFd, "isRepeated").booleanValue()
+                      }.getOrElse(false)
+                      val childIsRequired = Try {
+                        invoke0[java.lang.Boolean](childFd, "isRequired").booleanValue()
+                      }.getOrElse(false)
+                      val childHasDefault = Try {
+                        invoke0[java.lang.Boolean](childFd, "hasDefaultValue").booleanValue()
+                      }.getOrElse(false)
+                      val (_, _, childEncoding) = checkFieldSupport(
+                        childSf.dataType, childProtoTypeName, childIsRepeated, enumsAsInts)
+
+                      val childInfo = ProtobufFieldInfo(
+                        fieldNumber = childFieldNumber,
+                        protoTypeName = childProtoTypeName,
+                        sparkType = childSf.dataType,
+                        encoding = childEncoding,
+                        isSupported = true,
+                        unsupportedReason = None,
+                        isRequired = childIsRequired,
+                        hasDefaultValue = childHasDefault,
+                        defaultValue = None,
+                        enumValues = None,
+                        enumNames = None,
+                        isRepeated = childIsRepeated
+                      )
+
+                      addFieldWithChildren(
+                        childSf, childInfo, parentIdx, parentDepth + 1, childMsgDesc)
+                    }
+                  }
+                }
+              }
+            }
+
+            // Only add top-level fields that are actually required (schema projection).
+            // This significantly reduces GPU memory and computation for schemas with many
+            // fields when only a few are needed. The Scala layer will post-process the
+            // output to insert null columns for non-decoded fields.
+            decodedTopLevelIndices = indicesToDecode
+            indicesToDecode.foreach { schemaIdx =>
+              val sf = fullSchema.fields(schemaIdx)
+              val info = fieldsInfoMap(sf.name)
+              addFieldWithChildren(sf, info, -1, 0, msgDesc)
+            }
+
+            // Populate flattened schema variables
+            val flat = flatFields.toArray
+            flatFieldNumbers = flat.map(_.fieldNumber)
+            flatParentIndices = flat.map(_.parentIdx)
+            flatDepthLevels = flat.map(_.depth)
+            flatWireTypes = flat.map(_.wireType)
+            flatOutputTypeIds = flat.map(_.outputTypeId)
+            flatEncodings = flat.map(_.encoding)
+            flatIsRepeated = flat.map(_.isRepeated)
+            flatIsRequired = flat.map(_.isRequired)
+            flatHasDefaultValue = flat.map(_.hasDefaultValue)
+            flatDefaultInts = flat.map(_.defaultInt)
+            flatDefaultFloats = flat.map(_.defaultFloat)
+            flatDefaultBools = flat.map(_.defaultBool)
+            flatDefaultStrings = flat.map(_.defaultString)
+            flatEnumValidValues = flat.map(_.enumValidValues)
+            flatEnumNames = flat.map(_.enumNames)
+          }
+        }
+
+        /**
+         * Analyze all fields in the schema and build a map of field name to ProtobufFieldInfo.
+         * Returns None if there's an error that should abort processing.
+         */
+        private def analyzeAllFields(
+            schema: StructType,
+            msgDesc: AnyRef,
+            enumsAsInts: Boolean,
+            messageName: String): Option[Map[String, ProtobufFieldInfo]] = {
+          val result = mutable.Map[String, ProtobufFieldInfo]()
+
+          for (sf <- schema.fields) {
+            val fd = invoke1[AnyRef](msgDesc, "findFieldByName", classOf[String], sf.name)
+            if (fd == null) {
+              willNotWorkOnGpu(
+                s"Protobuf field '${sf.name}' not found in message '$messageName'")
+              return None
+            }
+
+            val isRepeated = Try {
+              invoke0[java.lang.Boolean](fd, "isRepeated").booleanValue()
+            }.getOrElse(false)
+
+            // Check if field is required (proto2 required fields)
+            val isFieldRequired = Try {
+              invoke0[java.lang.Boolean](fd, "isRequired").booleanValue()
+            }.getOrElse(false)
+
+            // Check if field has a default value (proto2 [default = xxx])
+            val hasDefault = Try {
+              invoke0[java.lang.Boolean](fd, "hasDefaultValue").booleanValue()
+            }.getOrElse(false)
+
+            // Get the default value if it exists
+            val defaultVal = if (hasDefault) {
+              Try(Some(invoke0[AnyRef](fd, "getDefaultValue"))).getOrElse(None)
+            } else {
+              None
+            }
+
+            val protoType = invoke0[AnyRef](fd, "getType")
+            val protoTypeName = typeName(protoType)
+            val fieldNumber = invoke0[java.lang.Integer](fd, "getNumber").intValue()
+
+            // Check field support and determine encoding
+            val (isSupported, unsupportedReason, encoding) =
+              checkFieldSupport(sf.dataType, protoTypeName, isRepeated, enumsAsInts)
+
+            // Extract enum values and (for enumsAsInts=false) value-name mapping.
+            val (enumVals, enumNameMap): (Option[Set[Int]], Option[Map[Int, String]]) =
+              if (protoTypeName == "ENUM") {
+                Try {
+                  val enumType = invoke0[AnyRef](fd, "getEnumType")
+                  val values = invoke0[java.util.List[_]](enumType, "getValues")
+                  import scala.collection.JavaConverters._
+                  val pairs = values.asScala.map { v =>
+                    val ev = v.asInstanceOf[AnyRef]
+                    val num = invoke0[java.lang.Integer](ev, "getNumber").intValue()
+                    val name = invoke0[String](ev, "getName")
+                    (num, name)
+                  }
+                  if (enumsAsInts) {
+                    (Some(pairs.map(_._1).toSet), None)
+                  } else {
+                    (Some(pairs.map(_._1).toSet), Some(pairs.toMap))
+                  }
+                }.getOrElse((None, None))
+              } else {
+                (None, None)
+              }
+
+            result(sf.name) = ProtobufFieldInfo(
+              fieldNumber = fieldNumber,
+              protoTypeName = protoTypeName,
+              sparkType = sf.dataType,
+              encoding = encoding,
+              isSupported = isSupported,
+              unsupportedReason = unsupportedReason,
+              isRequired = isFieldRequired,
+              hasDefaultValue = hasDefault,
+              defaultValue = defaultVal,
+              enumValues = enumVals,
+              enumNames = enumNameMap,
+              isRepeated = isRepeated
+            )
+          }
+
+          Some(result.toMap)
+        }
+
+        /**
+         * Check if a field type is supported and return encoding information.
+         * @return (isSupported, unsupportedReason, encoding)
+         */
+        private def checkFieldSupport(
+            sparkType: DataType,
+            protoTypeName: String,
+            isRepeated: Boolean,
+            enumsAsInts: Boolean): (Boolean, Option[String], Int) = {
+
+          // Handle repeated fields (arrays)
+          if (isRepeated) {
+            sparkType match {
+              case ArrayType(elementType, _) =>
+                // Check if element type is supported
+                elementType match {
+                  case BooleanType | IntegerType | LongType | FloatType | DoubleType |
+                       StringType | BinaryType =>
+                    // Supported repeated scalar - determine encoding from proto type
+                    return checkScalarEncoding(elementType, protoTypeName, enumsAsInts)
+                  case _: StructType =>
+                    // Repeated nested message (array of structs) - supported on GPU
+                    return (true, None, GpuFromProtobuf.ENC_DEFAULT)
+                  case _ =>
+                    return (false, Some(s"unsupported repeated element type: $elementType"),
+                      GpuFromProtobuf.ENC_DEFAULT)
+                }
+              case _ =>
+                return (false, Some(s"repeated field should map to ArrayType, got: $sparkType"),
+                  GpuFromProtobuf.ENC_DEFAULT)
+            }
+          }
+
+          // Handle nested messages (non-repeated)
+          if (protoTypeName == "MESSAGE") {
+            sparkType match {
+              case _: StructType =>
+                return (true, None, GpuFromProtobuf.ENC_DEFAULT)
+              case _ =>
+                return (false, Some(s"nested message should map to StructType, got: $sparkType"),
+                  GpuFromProtobuf.ENC_DEFAULT)
+            }
+          }
+
+          // Check Spark type is one of the supported simple types
+          sparkType match {
+            case BooleanType | IntegerType | LongType | FloatType | DoubleType |
+                 StringType | BinaryType =>
+              // Supported Spark type, continue to check encoding
+            case other =>
+              return (false, Some(s"unsupported Spark type: $other"), GpuFromProtobuf.ENC_DEFAULT)
+          }
+
+          checkScalarEncoding(sparkType, protoTypeName, enumsAsInts)
+        }
+
+        /**
+         * Determine encoding for scalar types.
+         */
+        private def checkScalarEncoding(
+            sparkType: DataType,
+            protoTypeName: String,
+            enumsAsInts: Boolean): (Boolean, Option[String], Int) = {
+
+          // Determine encoding based on Spark type and proto type combination
+          val encoding = (sparkType, protoTypeName) match {
+            case (BooleanType, "BOOL") => Some(GpuFromProtobuf.ENC_DEFAULT)
+            case (IntegerType, "INT32" | "UINT32") => Some(GpuFromProtobuf.ENC_DEFAULT)
+            case (IntegerType, "SINT32") => Some(GpuFromProtobuf.ENC_ZIGZAG)
+            case (IntegerType, "FIXED32" | "SFIXED32") => Some(GpuFromProtobuf.ENC_FIXED)
+            case (LongType, "INT64" | "UINT64") => Some(GpuFromProtobuf.ENC_DEFAULT)
+            case (LongType, "SINT64") => Some(GpuFromProtobuf.ENC_ZIGZAG)
+            case (LongType, "FIXED64" | "SFIXED64") => Some(GpuFromProtobuf.ENC_FIXED)
+            // Spark may upcast smaller integers to LongType
+            case (LongType, "INT32" | "UINT32" | "SINT32" | "FIXED32" | "SFIXED32") =>
+              val enc = protoTypeName match {
+                case "SINT32" => GpuFromProtobuf.ENC_ZIGZAG
+                case "FIXED32" | "SFIXED32" => GpuFromProtobuf.ENC_FIXED
+                case _ => GpuFromProtobuf.ENC_DEFAULT
+              }
+              Some(enc)
+            case (FloatType, "FLOAT") => Some(GpuFromProtobuf.ENC_DEFAULT)
+            case (DoubleType, "DOUBLE") => Some(GpuFromProtobuf.ENC_DEFAULT)
+            case (StringType, "STRING") => Some(GpuFromProtobuf.ENC_DEFAULT)
+            case (BinaryType, "BYTES") => Some(GpuFromProtobuf.ENC_DEFAULT)
+            case (IntegerType, "ENUM") if enumsAsInts => Some(GpuFromProtobuf.ENC_DEFAULT)
+            case (StringType, "ENUM") if !enumsAsInts => Some(GpuFromProtobuf.ENC_ENUM_STRING)
+            case _ => None
+          }
+
+          encoding match {
+            case Some(enc) => (true, None, enc)
+            case None =>
+              (false,
+                Some(s"type mismatch: Spark $sparkType vs Protobuf $protoTypeName"),
+                GpuFromProtobuf.ENC_DEFAULT)
+          }
+        }
+
+        /**
+         * Get wire type constant for a given protobuf type name and encoding.
+         */
+        private def getWireType(protoTypeName: String, encoding: Int): Int = {
+          protoTypeName match {
+            case "BOOL" | "INT32" | "UINT32" | "SINT32" | "INT64" | "UINT64" | "SINT64" | "ENUM" =>
+              if (encoding == GpuFromProtobuf.ENC_FIXED) {
+                if (protoTypeName.contains("64")) WT_64BIT else WT_32BIT
+              } else {
+                WT_VARINT
+              }
+            case "FIXED32" | "SFIXED32" | "FLOAT" => WT_32BIT
+            case "FIXED64" | "SFIXED64" | "DOUBLE" => WT_64BIT
+            case "STRING" | "BYTES" | "MESSAGE" => WT_LEN
+            case _ => WT_VARINT  // default
+          }
+        }
+
+        /**
+         * Analyze which fields are actually required by downstream operations.
+         * Currently supports analyzing parent Project expressions.
+         *
+         * @param allFieldNames All field names in the full schema
+         * @return Set of field names that are actually required
+         */
+        private def analyzeRequiredFields(allFieldNames: Set[String]): Set[String] = {
+          val parentPlanOpt = findParentPlanMeta()
+
+          parentPlanOpt match {
+            case Some(planMeta) =>
+              analyzeDownstreamProject(planMeta) match {
+                case Some(fields) if fields.nonEmpty =>
+                  fields
+                case _ =>
+                  planMeta.parent match {
+                    case Some(grandParentMeta: SparkPlanMeta[_]) =>
+                      analyzeDownstreamProject(grandParentMeta) match {
+                        case Some(fields) if fields.nonEmpty => fields
+                        case _ => allFieldNames
+                      }
+                    case _ => allFieldNames
+                  }
+              }
+            case None =>
+              allFieldNames
+          }
+        }
+
+        /**
+         * Find the parent SparkPlanMeta by traversing up the parent chain.
+         */
+        private def findParentPlanMeta(): Option[SparkPlanMeta[_]] = {
+          def traverse(meta: Option[RapidsMeta[_, _, _]]): Option[SparkPlanMeta[_]] = {
+            meta match {
+              case Some(p: SparkPlanMeta[_]) => Some(p)
+              case Some(p: RapidsMeta[_, _, _]) => traverse(p.parent)
+              case _ => None
+            }
+          }
+          traverse(parent)
+        }
+
+        /**
+         * Nested field requirements: for each top-level field, what children are needed?
+         * - None means the whole field is needed (all children)
+         * - Some(Set("a","b")) means only children a and b are needed
+         */
+        // Populated during analyzeDownstreamProject, used by addChildFieldsFromStruct
+        private var nestedFieldRequirements: Map[String, Option[Set[String]]] = Map.empty
+
+        /**
+         * Analyze a Project plan to find which struct fields are actually used.
+         * This looks for GetStructField expressions that reference our protobuf output.
+         * Also detects nested field access (e.g., decoded.ad_info.winfoid) for nested
+         * schema projection.
+         */
+        private def analyzeDownstreamProject(planMeta: SparkPlanMeta[_]): Option[Set[String]] = {
+          planMeta.wrapped match {
+            case p: ProjectExec =>
+              // Collect field references with nested child tracking
+              // Key = top-level field name
+              // Value = None (need whole field) or Some(Set(...)) (only need specific children)
+              val fieldReqs = mutable.Map[String, Option[Set[String]]]()
+              var hasDirectStructRef = false
+
+              p.projectList.foreach { expr =>
+                collectStructFieldReferences(expr, fieldReqs, hasDirectStructRefHolder = () => {
+                  hasDirectStructRef = true
+                })
+              }
+
+              if (hasDirectStructRef) {
+                None
+              } else if (fieldReqs.nonEmpty) {
+                nestedFieldRequirements = fieldReqs.toMap
+                Some(fieldReqs.keySet.toSet)
+              } else {
+                None
+              }
+            case _ =>
+              None
+          }
+        }
+
+        /**
+         * Get the field name from a GetStructField expression using its ordinal and schema.
+         */
+        private def getFieldName(ordinal: Int, nameOpt: Option[String],
+            schema: StructType): String = {
+          nameOpt.getOrElse {
+            if (ordinal < schema.fields.length) schema.fields(ordinal).name
+            else s"_$ordinal"
+          }
+        }
+
+        /**
+         * Recursively collect field names and nested child requirements from
+         * GetStructField expressions. Detects patterns like:
+         *   - decoded.field_name         -> field_name: None (whole field)
+         *   - decoded.ad_info.winfoid    -> ad_info: Some({winfoid})
+         *   - decoded.ad_info            -> ad_info: None (whole field)
+         *
+         * When both whole-field and sub-field access exist, whole-field wins (None).
+         */
+        /**
+         * Helper: record a nested child field requirement for a parent field.
+         * Merges with existing requirements (whole-field wins over sub-field).
+         */
+        private def addNestedFieldReq(
+            fieldReqs: mutable.Map[String, Option[Set[String]]],
+            parentName: String,
+            childName: String): Unit = {
+          fieldReqs.get(parentName) match {
+            case Some(None) => // Already need whole field, keep it
+            case Some(Some(existing)) =>
+              fieldReqs(parentName) = Some(existing + childName)
+            case None =>
+              fieldReqs(parentName) = Some(Set(childName))
+          }
+        }
+
+        private def collectStructFieldReferences(
+            expr: Expression,
+            fieldReqs: mutable.Map[String, Option[Set[String]]],
+            hasDirectStructRefHolder: () => Unit): Unit = {
+          expr match {
+            // Pattern: decoded.parent_struct.child_field (non-array struct)
+            case GetStructField(child, ordinal, nameOpt) =>
+              child match {
+                case GetStructField(innerChild, innerOrdinal, innerNameOpt)
+                    if isProtobufStructReference(innerChild) =>
+                  val parentName = getFieldName(innerOrdinal, innerNameOpt, fullSchema)
+                  val parentType = fullSchema.fields(innerOrdinal).dataType
+                  val childSchema = parentType match {
+                    case st: StructType => st
+                    case ArrayType(st: StructType, _) => st
+                    case _ => null
+                  }
+                  if (childSchema != null) {
+                    val childName = getFieldName(ordinal, nameOpt, childSchema)
+                    addNestedFieldReq(fieldReqs, parentName, childName)
+                  } else {
+                    fieldReqs(parentName) = None
+                  }
+
+                case _ if isProtobufStructReference(child) =>
+                  // Direct top-level access: decoded.field_name (whole field)
+                  val fieldName = getFieldName(ordinal, nameOpt, fullSchema)
+                  fieldReqs(fieldName) = None
+
+                case _ =>
+                  collectStructFieldReferences(child, fieldReqs, hasDirectStructRefHolder)
+              }
+
+            // Pattern: decoded.ad_info.winfoid where ad_info is ArrayType(StructType)
+            // Spark generates: GetArrayStructFields(GetStructField(decoded, ad_info_ord), field)
+            case gasf: GetArrayStructFields =>
+              gasf.child match {
+                case GetStructField(innerChild, innerOrdinal, innerNameOpt)
+                    if isProtobufStructReference(innerChild) =>
+                  // Nested array-struct access: decoded.array_field.child_field
+                  val parentName = getFieldName(innerOrdinal, innerNameOpt, fullSchema)
+                  val childName = gasf.field.name
+                  addNestedFieldReq(fieldReqs, parentName, childName)
+
+                case _ =>
+                  // Not a direct protobuf reference, recurse into children
+                  gasf.children.foreach { child =>
+                    collectStructFieldReferences(child, fieldReqs, hasDirectStructRefHolder)
+                  }
+              }
+
+            case _ =>
+              if (isProtobufStructReference(expr)) {
+                hasDirectStructRefHolder()
+              }
+              expr.children.foreach { child =>
+                collectStructFieldReferences(child, fieldReqs, hasDirectStructRefHolder)
+              }
+          }
+        }
+
+        /**
+         * Check if an expression references the output of a protobuf decode expression.
+         * This can be either:
+         * 1. The ProtobufDataToCatalyst expression itself
+         * 2. An AttributeReference that references the output of ProtobufDataToCatalyst
+         *    (when accessing from a downstream ProjectExec)
+         */
+        private def isProtobufStructReference(expr: Expression): Boolean = {
+          // Check if expr is a ProtobufDataToCatalyst expression
+          if (expr.getClass.getName.contains("ProtobufDataToCatalyst")) {
+            return true
+          }
+          
+          // Check if expr is an AttributeReference with the same schema as our protobuf output
+          // This handles the case where GetStructField references a column from a parent Project
+          expr match {
+            case attr: AttributeReference =>
+              // Check if the data type matches our full schema (struct type from protobuf)
+              attr.dataType match {
+                case st: StructType => 
+                  // Compare field names and types only. We intentionally do not compare
+                  // nullable flags because schema transformations (like projections or
+                  // certain optimizations) may change nullability while the underlying
+                  // schema structure remains the same. For schema projection detection,
+                  // matching names and types is sufficient to identify protobuf output.
+                  st.fields.length == fullSchema.fields.length &&
+                    st.fields.zip(fullSchema.fields).forall { case (a, b) =>
+                      a.name == b.name && a.dataType == b.dataType
+                    }
+                case _ => false
+              }
+            case _ => false
+          }
+        }
+
+        override def convertToGpu(child: Expression): GpuExpression = {
+          // Build pruned fields map for ALL struct types (both repeated and non-repeated).
+          // For repeated message fields (ArrayType(StructType)), pruning is handled via
+          // ordinal remapping in GpuGetArrayStructFieldsMeta (Option A) instead of expansion.
+          val prunedFieldsMap: Map[String, Seq[String]] = nestedFieldRequirements.collect {
+            case (fieldName, Some(childNames)) =>
+              val fieldIdx = fullSchema.fieldIndex(fieldName)
+              val childSchema = fullSchema.fields(fieldIdx).dataType match {
+                case st: StructType => st
+                case ArrayType(st: StructType, _) => st
+                case _ => null
+              }
+              if (childSchema != null) {
+                // Preserve the original field order
+                val orderedNames = childSchema.fields
+                  .map(_.name)
+                  .filter(childNames.contains)
+                  .toSeq
+                fieldName -> orderedNames
+              } else {
+                fieldName -> childNames.toSeq
+              }
+          }
+
+          // Register pruned field ordinal mappings for GpuGetArrayStructFieldsMeta.
+          // For each pruned ArrayType(StructType) field, map child field names to their
+          // ordinal in the pruned struct so runtime column access uses correct indices.
+          val ordinalMappings = prunedFieldsMap.flatMap { case (parentName, childNames) =>
+            val fieldIdx = fullSchema.fieldIndex(parentName)
+            fullSchema.fields(fieldIdx).dataType match {
+              case ArrayType(_: StructType, _) =>
+                childNames.zipWithIndex.map { case (name, idx) => name -> idx }
+              case _ => Seq.empty
+            }
+          }
+          if (ordinalMappings.nonEmpty) {
+            GpuFromProtobuf.registerPrunedFields(ordinalMappings)
+          }
+
+          GpuFromProtobuf(
+            fullSchema, decodedTopLevelIndices, flatFieldNumbers, flatParentIndices,
+            flatDepthLevels, flatWireTypes, flatOutputTypeIds, flatEncodings,
+            flatIsRepeated, flatIsRequired, flatHasDefaultValue, flatDefaultInts,
+            flatDefaultFloats, flatDefaultBools, flatDefaultStrings, flatEnumValidValues,
+            flatEnumNames,
+            prunedFieldsMap, failOnErrors, child)
+        }
+      }
+    )
+  }
+
+  private def getMessageName(e: Expression): String =
+    invoke0[String](e, "messageName")
+
+  /**
+   * Newer Spark versions may carry an in-expression descriptor set payload.
+   * - Spark 3.5.x: binaryFileDescriptorSet: Option[Array[Byte]]
+   * - Spark 4.x: binaryDescriptorSet (may be Array[Byte] or Option[Array[Byte]])
+   * Spark 3.4.x does not have this, so callers should fall back to descFilePath().
+   */
+  private def getDescriptorBytes(e: Expression): Option[Array[Byte]] = {
+    // Spark 3.5.x: binaryFileDescriptorSet (note: "File" in the name)
+    val spark35Result = Try(invoke0[Option[Array[Byte]]](e, "binaryFileDescriptorSet"))
+      .toOption.flatten
+    spark35Result.orElse {
+      // Spark 4.x: binaryDescriptorSet - may be Array[Byte] or Option[Array[Byte]]
+      val direct = Try(invoke0[Array[Byte]](e, "binaryDescriptorSet")).toOption
+      direct.orElse {
+        Try(invoke0[Option[Array[Byte]]](e, "binaryDescriptorSet")).toOption.flatten
+      }
+    }
+  }
+
+  /**
+   * Get descriptor file path from expression.
+   * Only available in Spark 3.4.x. Spark 3.5+ uses binaryFileDescriptorSet instead.
+   */
+  private def getDescFilePath(e: Expression): Option[String] =
+    Try(invoke0[Option[String]](e, "descFilePath")).toOption.flatten
+
+  /**
+   * Build message descriptor using Spark's ProtobufUtils.
+   * Supports both Spark 3.4.x (descFilePath: Option[String]) and
+   * Spark 3.5+ (binaryFileDescriptorSet: Option[Array[Byte]]).
+   *
+   * @param messageName The protobuf message name
+   * @param descFilePathOrBytes Either a file path (String) or binary descriptor bytes (Array[Byte])
+   */
+  private def buildMessageDescriptorWithSparkProtobuf(
+      messageName: String,
+      descFilePathOrBytes: Either[String, Array[Byte]]): AnyRef = {
+    val cls = ShimReflectionUtils.loadClass(sparkProtobufUtilsObjectClassName)
+    val module = cls.getField("MODULE$").get(null)
+
+    descFilePathOrBytes match {
+      case Left(filePath) =>
+        // Spark 3.4.x: buildDescriptor(messageName: String, descFilePath: Option[String])
+        val m = cls.getMethod("buildDescriptor", classOf[String], classOf[scala.Option[_]])
+        m.invoke(module, messageName, Some(filePath)).asInstanceOf[AnyRef]
+      case Right(bytes) =>
+        // Spark 3.5+: buildDescriptor(messageName, binaryFileDescriptorSet)
+        val m = cls.getMethod("buildDescriptor", classOf[String], classOf[scala.Option[_]])
+        m.invoke(module, messageName, Some(bytes)).asInstanceOf[AnyRef]
+    }
+  }
+
+  private def typeName(t: AnyRef): String = {
+    if (t == null) {
+      "null"
+    } else {
+      // Prefer Enum.name() when available; fall back to toString.
+      Try(invoke0[String](t, "name")).getOrElse(t.toString)
+    }
+  }
+
+  private def getOptionsMap(e: Expression): Map[String, String] = {
+    val opt = Try(invoke0[scala.collection.Map[String, String]](e, "options")).toOption
+    opt.map(_.toMap).getOrElse(Map.empty)
+  }
+
+  private def invoke0[T](obj: AnyRef, method: String): T =
+    obj.getClass.getMethod(method).invoke(obj).asInstanceOf[T]
+
+  private def invoke1[T](obj: AnyRef, method: String, arg0Cls: Class[_], arg0: AnyRef): T =
+    obj.getClass.getMethod(method, arg0Cls).invoke(obj, arg0).asInstanceOf[T]
+}
diff --git a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/Spark340PlusNonDBShims.scala b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/Spark340PlusNonDBShims.scala
index 6e28a071a00..56bfa229051 100644
--- a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/Spark340PlusNonDBShims.scala
+++ b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/Spark340PlusNonDBShims.scala
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022-2025, NVIDIA CORPORATION.
+ * Copyright (c) 2022-2026, NVIDIA CORPORATION.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -162,7 +162,7 @@ trait Spark340PlusNonDBShims extends Spark331PlusNonDBShims {
       ),
       GpuElementAtMeta.elementAtRule(true)
     ).map(r => (r.getClassFor.asSubclass(classOf[Expression]), r)).toMap
-    super.getExprs ++ shimExprs
+    super.getExprs ++ shimExprs ++ ProtobufExprShims.exprs
   }
 
   override def getDataWriteCmds: Map[Class[_ <: DataWritingCommand],

From a225f9a3bbf63515fb233d0f6d2c153645d52f68 Mon Sep 17 00:00:00 2001
From: Haoyang Li <haoyangl@nvidia.com>
Date: Thu, 26 Feb 2026 16:47:34 +0800
Subject: [PATCH 02/33] address self review comments

Signed-off-by: Haoyang Li <haoyangl@nvidia.com>
---
 .../src/main/python/protobuf_test.py          | 126 ++++++++++++++++++
 .../protobuf/ProtobufDescriptorUtils.scala    |  87 ------------
 .../spark/sql/rapids/GpuFromProtobuf.scala    |  79 +----------
 .../rapids/shims/ProtobufExprShims.scala      |  76 ++++++++---
 4 files changed, 185 insertions(+), 183 deletions(-)
 delete mode 100644 sql-plugin/src/main/scala/com/nvidia/spark/rapids/protobuf/ProtobufDescriptorUtils.scala

diff --git a/integration_tests/src/main/python/protobuf_test.py b/integration_tests/src/main/python/protobuf_test.py
index 052b38d953d..2d4bc14a34f 100644
--- a/integration_tests/src/main/python/protobuf_test.py
+++ b/integration_tests/src/main/python/protobuf_test.py
@@ -672,6 +672,89 @@ def _build_enum_descriptor_set_bytes(spark):
     return bytes(fds.toByteArray())
 
 
+def _build_nested_enum_descriptor_set_bytes(spark):
+    """
+    Build a FileDescriptorSet for a nested enum message:
+      package test;
+      syntax = "proto2";
+      message WithNestedEnum {
+        optional int32 id = 1;
+        optional Detail detail = 2;
+        optional string name = 3;
+      }
+      message Detail {
+        enum Status {
+          UNKNOWN = 0;
+          OK = 1;
+          BAD = 2;
+        }
+        optional Status status = 1;
+        optional int32 count = 2;
+      }
+    """
+    D, fd = _new_proto2_file(spark, "nested_enum.proto")
+    label_opt = D.FieldDescriptorProto.Label.LABEL_OPTIONAL
+
+    detail_msg = D.DescriptorProto.newBuilder().setName("Detail")
+    detail_enum = D.EnumDescriptorProto.newBuilder().setName("Status")
+    detail_enum.addValue(
+        D.EnumValueDescriptorProto.newBuilder().setName("UNKNOWN").setNumber(0).build())
+    detail_enum.addValue(
+        D.EnumValueDescriptorProto.newBuilder().setName("OK").setNumber(1).build())
+    detail_enum.addValue(
+        D.EnumValueDescriptorProto.newBuilder().setName("BAD").setNumber(2).build())
+    detail_msg.addEnumType(detail_enum.build())
+    detail_msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("status")
+            .setNumber(1)
+            .setLabel(label_opt)
+            .setType(D.FieldDescriptorProto.Type.TYPE_ENUM)
+            .setTypeName(".test.Detail.Status")
+            .build()
+    )
+    detail_msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("count")
+            .setNumber(2)
+            .setLabel(label_opt)
+            .setType(D.FieldDescriptorProto.Type.TYPE_INT32)
+            .build()
+    )
+    fd.addMessageType(detail_msg.build())
+
+    top_msg = D.DescriptorProto.newBuilder().setName("WithNestedEnum")
+    top_msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("id")
+            .setNumber(1)
+            .setLabel(label_opt)
+            .setType(D.FieldDescriptorProto.Type.TYPE_INT32)
+            .build()
+    )
+    top_msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("detail")
+            .setNumber(2)
+            .setLabel(label_opt)
+            .setType(D.FieldDescriptorProto.Type.TYPE_MESSAGE)
+            .setTypeName(".test.Detail")
+            .build()
+    )
+    top_msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("name")
+            .setNumber(3)
+            .setLabel(label_opt)
+            .setType(D.FieldDescriptorProto.Type.TYPE_STRING)
+            .build()
+    )
+    fd.addMessageType(top_msg.build())
+
+    fds = D.FileDescriptorSet.newBuilder().addFile(fd.build()).build()
+    return bytes(fds.toByteArray())
+
+
 @pytest.mark.skipif(is_before_spark_340(), reason="from_protobuf is Spark 3.4.0+")
 @pytest.mark.parametrize("enum_case", [
     "as_int",
@@ -744,6 +827,49 @@ def run_on_spark(spark):
     assert_gpu_and_cpu_are_equal_collect(run_on_spark)
 
 
+@pytest.mark.skipif(is_before_spark_340(), reason="from_protobuf is Spark 3.4.0+")
+@ignore_order(local=True)
+def test_from_protobuf_nested_enum_permissive_invalid_row_null(spark_tmp_path, from_protobuf_fn):
+    """
+    Nested enum decode parity test:
+      - valid nested enum values decode to names in string mode
+      - missing nested enum stays null
+    This protects nested enum metadata propagation in GPU schema flattening.
+    """
+    desc_path, desc_bytes = _setup_protobuf_desc(
+        spark_tmp_path, "nested_enum.desc", _build_nested_enum_descriptor_set_bytes)
+    message_name = "test.WithNestedEnum"
+
+    fields = [
+        PbScalar("id", 1, IntegerGen()),
+        PbNested("detail", 2, [
+            PbScalar("status", 1, IntegerGen()),
+            PbScalar("count", 2, IntegerGen()),
+        ]),
+        PbScalar("name", 3, StringGen(nullable=True)),
+    ]
+    rows = [
+        (0, encode_pb_message(fields, [1, (1, 10), "ok"])),       # status=OK
+        (1, encode_pb_message(fields, [2, (2, 20), "bad"])),      # status=BAD
+        (2, encode_pb_message(fields, [3, (None, 30), "none"])),  # missing enum
+        (3, None),
+    ]
+
+    def run_on_spark(spark):
+        df = spark.createDataFrame(rows, schema="idx int, bin binary")
+        decoded = _call_from_protobuf(
+            from_protobuf_fn, f.col("bin"), message_name, desc_path, desc_bytes,
+            options={"mode": "PERMISSIVE"})
+        return df.select(
+            f.col("idx"),
+            decoded.isNull().alias("decoded_is_null"),
+            decoded.getField("id").alias("id"),
+            decoded.getField("detail").getField("status").alias("status"),
+            decoded.getField("name").alias("name")).orderBy("idx")
+
+    assert_gpu_and_cpu_are_equal_collect(run_on_spark)
+
+
 def _build_required_field_descriptor_set_bytes(spark):
     """
     Build a FileDescriptorSet for a message with required fields (proto2):
diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/protobuf/ProtobufDescriptorUtils.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/protobuf/ProtobufDescriptorUtils.scala
deleted file mode 100644
index cabc8d2905d..00000000000
--- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/protobuf/ProtobufDescriptorUtils.scala
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Copyright (c) 2026, NVIDIA CORPORATION.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.nvidia.spark.rapids.protobuf
-
-import scala.collection.JavaConverters._
-import scala.collection.mutable
-
-import com.google.protobuf.DescriptorProtos
-import com.google.protobuf.Descriptors
-
-/**
- * Minimal descriptor utilities for locating a message descriptor in a FileDescriptorSet.
- *
- * This is intentionally lightweight for the "simple types" from_protobuf patch: it supports
- * descriptor sets produced by `protoc --include_imports --descriptor_set_out=...`.
- *
- * NOTE: This utility is currently not used in the initial implementation, which relies on
- * Spark's ProtobufUtils via reflection (buildMessageDescriptorWithSparkProtobuf). This class
- * is preserved for potential future use cases where direct descriptor parsing is needed
- * without depending on Spark's shaded protobuf classes.
- */
-object ProtobufDescriptorUtils {
-
-  def buildMessageDescriptor(
-      fileDescriptorSetBytes: Array[Byte],
-      messageName: String): Descriptors.Descriptor = {
-    val fds = DescriptorProtos.FileDescriptorSet.parseFrom(fileDescriptorSetBytes)
-    val protos = fds.getFileList.asScala.toSeq
-    val byName = protos.map(p => p.getName -> p).toMap
-    val cache = mutable.HashMap.empty[String, Descriptors.FileDescriptor]
-
-    def buildFileDescriptor(name: String): Descriptors.FileDescriptor = {
-      cache.getOrElseUpdate(name, {
-        val p = byName.getOrElse(name,
-          throw new IllegalArgumentException(s"Missing FileDescriptorProto for '$name'"))
-        val deps = p.getDependencyList.asScala.map(buildFileDescriptor _).toArray
-        Descriptors.FileDescriptor.buildFrom(p, deps)
-      })
-    }
-
-    val fileDescriptors = protos.map(p => buildFileDescriptor(p.getName))
-    val candidates = fileDescriptors.iterator.flatMap(fd => findMessageDescriptors(fd, messageName))
-      .toSeq
-
-    candidates match {
-      case Seq(d) => d
-      case Seq() =>
-        throw new IllegalArgumentException(
-          s"Message '$messageName' not found in FileDescriptorSet")
-      case many =>
-        val names = many.map(_.getFullName).distinct.sorted
-        throw new IllegalArgumentException(
-          s"Message '$messageName' is ambiguous; matches: ${names.mkString(", ")}")
-    }
-  }
-
-  private def findMessageDescriptors(
-      fd: Descriptors.FileDescriptor,
-      messageName: String): Iterator[Descriptors.Descriptor] = {
-    def matches(d: Descriptors.Descriptor): Boolean = {
-      d.getName == messageName ||
-        d.getFullName == messageName ||
-        d.getFullName.endsWith("." + messageName)
-    }
-
-    def walk(d: Descriptors.Descriptor): Iterator[Descriptors.Descriptor] = {
-      val nested = d.getNestedTypes.asScala.iterator.flatMap(walk _)
-      if (matches(d)) Iterator.single(d) ++ nested else nested
-    }
-
-    fd.getMessageTypes.asScala.iterator.flatMap(walk _)
-  }
-}
\ No newline at end of file
diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuFromProtobuf.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuFromProtobuf.scala
index 290670fff91..941fbf160ec 100644
--- a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuFromProtobuf.scala
+++ b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuFromProtobuf.scala
@@ -156,8 +156,11 @@ case class GpuFromProtobuf(
         enumNames,
         failOnErrors)
     } catch {
-      case e: CudfException if failOnErrors =>
-        throw new org.apache.spark.SparkException("Malformed protobuf message", e)
+      case e: CudfException =>
+        if (failOnErrors) {
+          throw new org.apache.spark.SparkException("Malformed protobuf message", e)
+        }
+        throw e
     }
 
     // Expand if needed:
@@ -309,76 +312,4 @@ object GpuFromProtobuf {
    * Check if a Spark DataType is supported by the GPU protobuf decoder.
    */
   def isTypeSupported(dt: DataType): Boolean = sparkTypeToCudfIdOpt(dt).isDefined
-
-  /**
-   * Create an all-null column of the specified Spark DataType.
-   * This is used for fields with unsupported types (nested structs, arrays, etc.)
-   * that are not decoded but need to be present in the output struct.
-   */
-  def createNullColumn(dt: DataType, numRows: Int): cudf.ColumnVector = {
-    // Helper to create null arrays for boxed types
-    def nullBools = Array.fill[java.lang.Boolean](numRows)(null)
-    def nullInts = Array.fill[java.lang.Integer](numRows)(null)
-    def nullLongs = Array.fill[java.lang.Long](numRows)(null)
-    def nullFloats = Array.fill[java.lang.Float](numRows)(null)
-    def nullDoubles = Array.fill[java.lang.Double](numRows)(null)
-
-    dt match {
-      case BooleanType => cudf.ColumnVector.fromBoxedBooleans(nullBools: _*)
-      case IntegerType => cudf.ColumnVector.fromBoxedInts(nullInts: _*)
-      case LongType => cudf.ColumnVector.fromBoxedLongs(nullLongs: _*)
-      case FloatType => cudf.ColumnVector.fromBoxedFloats(nullFloats: _*)
-      case DoubleType => cudf.ColumnVector.fromBoxedDoubles(nullDoubles: _*)
-      case StringType => cudf.ColumnVector.fromStrings(Array.fill[String](numRows)(null): _*)
-      case BinaryType =>
-        // Binary is LIST<INT8> - create all-null list column using Scalar API
-        val elementType = new cudf.HostColumnVector.BasicType(true, DType.INT8)
-        withResource(cudf.Scalar.listFromNull(elementType)) { nullScalar =>
-          cudf.ColumnVector.fromScalar(nullScalar, numRows)
-        }
-      case st: StructType =>
-        // Recursively create null columns for struct fields
-        val children = st.fields.map(f => createNullColumn(f.dataType, numRows))
-        try {
-          withResource(cudf.ColumnVector.makeStruct(numRows, children: _*)) { structCol =>
-            // Set all rows to null - mergeAndSetValidity returns a NEW column
-            withResource(cudf.ColumnVector.fromBoxedBooleans(nullBools: _*)) { nullMask =>
-              structCol.mergeAndSetValidity(BinaryOp.BITWISE_AND, nullMask)
-            }
-          }
-        } finally {
-          children.foreach(_.close())
-        }
-      case ArrayType(elementType, _) =>
-        // Create empty arrays with all nulls using Scalar API
-        val cudfElementDType = sparkTypeToCudfIdOpt(elementType)
-          .map(id => DType.fromNative(id, 0))
-          .getOrElse(DType.INT8)  // fallback for nested complex types
-        val elemType = new cudf.HostColumnVector.BasicType(true, cudfElementDType)
-        withResource(cudf.Scalar.listFromNull(elemType)) { nullScalar =>
-          cudf.ColumnVector.fromScalar(nullScalar, numRows)
-        }
-      case MapType(keyType, valueType, _) =>
-        // Maps are represented as LIST<STRUCT<key, value>> in cuDF
-        // For all-null maps, we create a list column with STRUCT<key, value> element type
-        val cudfKeyDType = sparkTypeToCudfIdOpt(keyType)
-          .map(id => DType.fromNative(id, 0))
-          .getOrElse(DType.INT8)
-        val cudfValueDType = sparkTypeToCudfIdOpt(valueType)
-          .map(id => DType.fromNative(id, 0))
-          .getOrElse(DType.INT8)
-        // Create the struct type for map entries (key, value)
-        val keyFieldType = new cudf.HostColumnVector.BasicType(true, cudfKeyDType)
-        val valueFieldType = new cudf.HostColumnVector.BasicType(true, cudfValueDType)
-        val structType = new cudf.HostColumnVector.StructType(true, keyFieldType, valueFieldType)
-        // Create an all-null map column (list of structs)
-        withResource(cudf.Scalar.listFromNull(structType)) { nullScalar =>
-          cudf.ColumnVector.fromScalar(nullScalar, numRows)
-        }
-      case _ =>
-        // Fallback for any other types - create INT8 nulls as placeholder
-        // This should not happen in practice since unsupported types should be caught earlier
-        cudf.ColumnVector.fromBoxedBytes(Array.fill[java.lang.Byte](numRows)(null): _*)
-    }
-  }
 }
diff --git a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala
index 10061af5cc6..51463b0f3d6 100644
--- a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala
+++ b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala
@@ -34,6 +34,8 @@ spark-rapids-shim-json-lines ***/
 
 package com.nvidia.spark.rapids.shims
 
+import java.lang.ReflectiveOperationException
+
 import scala.collection.mutable
 import scala.util.Try
 
@@ -203,7 +205,9 @@ object ProtobufExprShims {
             typeName(syntaxObj)
           }.getOrElse("")
           if (protoSyntax == "PROTO3") {
-            willNotWorkOnGpu("proto3 descriptors are not supported; only proto2 is supported")
+            willNotWorkOnGpu(
+              "proto3 syntax is not supported by the GPU protobuf decoder; " +
+                "only proto2 is supported. The query will fall back to CPU.")
             return
           }
 
@@ -373,7 +377,7 @@ object ProtobufExprShims {
               val fd = invoke1[AnyRef](
                 parentMsgDesc, "findFieldByName", classOf[String], fieldName)
               if (fd != null) {
-                Try {
+                try {
                   val childMsgDesc = invoke0[AnyRef](fd, "getMessageType")
                   // Filter children based on nested schema projection requirements.
                   val requiredChildren = nestedFieldRequirements.get(fieldName)
@@ -405,6 +409,29 @@ object ProtobufExprShims {
                       val (_, _, childEncoding) = checkFieldSupport(
                         childSf.dataType, childProtoTypeName, childIsRepeated, enumsAsInts)
 
+                      val (childEnumVals, childEnumNameMap): (Option[Set[Int]],
+                        Option[Map[Int, String]]) =
+                        if (childProtoTypeName == "ENUM") {
+                          Try {
+                            val enumType = invoke0[AnyRef](childFd, "getEnumType")
+                            val values = invoke0[java.util.List[_]](enumType, "getValues")
+                            import scala.collection.JavaConverters._
+                            val pairs = values.asScala.map { v =>
+                              val ev = v.asInstanceOf[AnyRef]
+                              val num = invoke0[java.lang.Integer](ev, "getNumber").intValue()
+                              val name = invoke0[String](ev, "getName")
+                              (num, name)
+                            }
+                            if (enumsAsInts) {
+                              (Some(pairs.map(_._1).toSet), None)
+                            } else {
+                              (Some(pairs.map(_._1).toSet), Some(pairs.toMap))
+                            }
+                          }.getOrElse((None, None))
+                        } else {
+                          (None, None)
+                        }
+
                       val childInfo = ProtobufFieldInfo(
                         fieldNumber = childFieldNumber,
                         protoTypeName = childProtoTypeName,
@@ -415,8 +442,8 @@ object ProtobufExprShims {
                         isRequired = childIsRequired,
                         hasDefaultValue = childHasDefault,
                         defaultValue = None,
-                        enumValues = None,
-                        enumNames = None,
+                        enumValues = childEnumVals,
+                        enumNames = childEnumNameMap,
                         isRepeated = childIsRepeated
                       )
 
@@ -424,6 +451,9 @@ object ProtobufExprShims {
                         childSf, childInfo, parentIdx, parentDepth + 1, childMsgDesc)
                     }
                   }
+                } catch {
+                  case _: ReflectiveOperationException =>
+                  // Ignore reflection failures and let remaining fields continue.
                 }
               }
             }
@@ -662,7 +692,9 @@ object ProtobufExprShims {
             case "FIXED32" | "SFIXED32" | "FLOAT" => WT_32BIT
             case "FIXED64" | "SFIXED64" | "DOUBLE" => WT_64BIT
             case "STRING" | "BYTES" | "MESSAGE" => WT_LEN
-            case _ => WT_VARINT  // default
+            case other =>
+              throw new IllegalStateException(
+                s"Unknown protobuf type name '$other' - cannot determine wire type")
           }
         }
 
@@ -862,30 +894,30 @@ object ProtobufExprShims {
           if (expr.getClass.getName.contains("ProtobufDataToCatalyst")) {
             return true
           }
-          
-          // Check if expr is an AttributeReference with the same schema as our protobuf output
-          // This handles the case where GetStructField references a column from a parent Project
+
+          // Prefer exact attribute identity. Avoid matching by schema shape because unrelated
+          // struct columns can share the same field names/types.
+          val protobufOutputExprId: Option[org.apache.spark.sql.catalyst.expressions.ExprId] =
+            parent.flatMap { meta =>
+              meta.wrapped match {
+                case alias: org.apache.spark.sql.catalyst.expressions.Alias
+                    if alias.child eq e =>
+                  Some(alias.exprId)
+                case _ => None
+              }
+            }
+
           expr match {
             case attr: AttributeReference =>
-              // Check if the data type matches our full schema (struct type from protobuf)
-              attr.dataType match {
-                case st: StructType => 
-                  // Compare field names and types only. We intentionally do not compare
-                  // nullable flags because schema transformations (like projections or
-                  // certain optimizations) may change nullability while the underlying
-                  // schema structure remains the same. For schema projection detection,
-                  // matching names and types is sufficient to identify protobuf output.
-                  st.fields.length == fullSchema.fields.length &&
-                    st.fields.zip(fullSchema.fields).forall { case (a, b) =>
-                      a.name == b.name && a.dataType == b.dataType
-                    }
-                case _ => false
-              }
+              protobufOutputExprId.exists(_ == attr.exprId)
             case _ => false
           }
         }
 
         override def convertToGpu(child: Expression): GpuExpression = {
+          // Ensure thread-local ordinal mapping doesn't leak across query conversions.
+          GpuFromProtobuf.clearPrunedFields()
+
           // Build pruned fields map for ALL struct types (both repeated and non-repeated).
           // For repeated message fields (ArrayType(StructType)), pruning is handled via
           // ordinal remapping in GpuGetArrayStructFieldsMeta (Option A) instead of expansion.

From e517e43a43b0824237ab1573bee8348780e7f567 Mon Sep 17 00:00:00 2001
From: Haoyang Li <haoyangl@nvidia.com>
Date: Thu, 26 Feb 2026 21:46:19 +0800
Subject: [PATCH 03/33] fix schema projection

Signed-off-by: Haoyang Li <haoyangl@nvidia.com>
---
 .../nvidia/spark/rapids/GpuCanonicalize.scala |   3 +-
 .../nvidia/spark/rapids/GpuOverrides.scala    |   5 +-
 .../spark/sql/rapids/GpuFromProtobuf.scala    | 101 ++++--------------
 .../sql/rapids/complexTypeExtractors.scala    |  65 +++++++----
 .../rapids/shims/ProtobufExprShims.scala      |   4 +-
 5 files changed, 70 insertions(+), 108 deletions(-)

diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuCanonicalize.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuCanonicalize.scala
index 2dd39fa2375..caeaea66e07 100644
--- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuCanonicalize.scala
+++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuCanonicalize.scala
@@ -51,7 +51,8 @@ object GpuCanonicalize {
     case a: AttributeReference =>
       AttributeReference("none", TrampolineUtil.asNullable(a.dataType))(exprId = a.exprId)
     case GetStructField(child, ordinal, Some(_)) => GetStructField(child, ordinal, None)
-    case GpuGetStructField(child, ordinal, Some(_)) => GpuGetStructField(child, ordinal, None)
+    case GpuGetStructField(child, ordinal, Some(_), runtimeOrdinal, resolvedType) =>
+      GpuGetStructField(child, ordinal, None, runtimeOrdinal, resolvedType)
     case _ => e
   }
 
diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala
index 535c905884a..d234801b6ae 100644
--- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala
+++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala
@@ -2643,10 +2643,7 @@ object GpuOverrides extends Logging {
         TypeSig.STRUCT.nested(TypeSig.commonCudfTypes + TypeSig.ARRAY +
             TypeSig.STRUCT + TypeSig.MAP + TypeSig.NULL + TypeSig.DECIMAL_128 + TypeSig.BINARY),
         TypeSig.STRUCT.nested(TypeSig.all)),
-      (expr, conf, p, r) => new UnaryExprMeta[GetStructField](expr, conf, p, r) {
-        override def convertToGpu(arr: Expression): GpuExpression =
-          GpuGetStructField(arr, expr.ordinal, expr.name)
-      }),
+      (expr, conf, p, r) => new GpuGetStructFieldMeta(expr, conf, p, r)),
     expr[GetArrayItem](
       "Gets the field at `ordinal` in the Array",
       ExprChecks.binaryProject(
diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuFromProtobuf.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuFromProtobuf.scala
index 941fbf160ec..a94d1f9c8ff 100644
--- a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuFromProtobuf.scala
+++ b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuFromProtobuf.scala
@@ -89,12 +89,10 @@ case class GpuFromProtobuf(
   override def inputTypes: Seq[AbstractDataType] = Seq(BinaryType)
 
   /**
-   * Build a partially pruned schema for the output:
-   * - Top-level: matches fullSchema (so GetStructField ordinals stay valid)
-   * - Nested ArrayType(StructType) fields: pruned to only contain decoded children
-   *   (GetArrayStructFields ordinals are remapped in GpuGetArrayStructFieldsMeta)
-   * - Nested StructType (non-repeated) fields: kept at full schema
-   *   (expansion inserts null columns cheaply; GetStructField ordinals stay valid)
+   * Build a pruned schema for nested struct fields:
+   * - Top-level field order still matches fullSchema
+   * - Nested ArrayType(StructType) and StructType are both pruned to decoded children
+   * - Runtime ordinals are remapped by GpuGetStructField/GpuGetArrayStructFields
    */
   private val outputSchema: StructType = {
     val fields = fullSchema.fields.map { field =>
@@ -102,13 +100,14 @@ case class GpuFromProtobuf(
         case Some(childNames) =>
           field.dataType match {
             case ArrayType(st: StructType, containsNull) =>
-              // Pruned repeated message: only keep decoded children (Option A)
               val prunedSt = StructType(
                 st.fields.filter(f => childNames.contains(f.name)))
               field.copy(dataType = ArrayType(prunedSt, containsNull))
-            case _ =>
-              // Non-repeated struct or other: keep full type (expand with null cols)
-              field
+            case st: StructType =>
+              val prunedSt = StructType(
+                st.fields.filter(f => childNames.contains(f.name)))
+              field.copy(dataType = prunedSt)
+            case _ => field
           }
         case None => field
       }
@@ -120,19 +119,9 @@ case class GpuFromProtobuf(
 
   override def nullable: Boolean = true
 
-  // Identify which pruned fields are non-repeated structs (need expansion)
-  // vs repeated messages (pruned in outputSchema, no expansion needed)
-  private val nonRepeatedPrunedFields: Map[String, Seq[String]] =
-    nestedPrunedFields.filter { case (fieldName, _) =>
-      val idx = fullSchema.fieldIndex(fieldName)
-      fullSchema.fields(idx).dataType.isInstanceOf[StructType]
-    }
-
-  // Expansion is needed when not all top-level fields are decoded,
-  // or when non-repeated structs have pruned children
+  // Expansion is needed only when not all top-level fields are decoded.
   private val needsExpansion: Boolean =
-    decodedTopLevelIndices.length != fullSchema.fields.length ||
-    nonRepeatedPrunedFields.nonEmpty
+    decodedTopLevelIndices.length != fullSchema.fields.length
 
   override protected def doColumnar(input: GpuColumnVector): cudf.ColumnVector = {
     val numRows = input.getRowCount.toInt
@@ -156,18 +145,11 @@ case class GpuFromProtobuf(
         enumNames,
         failOnErrors)
     } catch {
-      case e: CudfException =>
-        if (failOnErrors) {
-          throw new org.apache.spark.SparkException("Malformed protobuf message", e)
-        }
-        throw e
+      case e: CudfException if failOnErrors =>
+        throw new org.apache.spark.SparkException("Malformed protobuf message", e)
     }
 
-    // Expand if needed:
-    // - Top-level: insert null columns for non-decoded top-level fields
-    // - Non-repeated structs: expand pruned children with null columns
-    // - Repeated messages (ArrayType(StructType)): already pruned in outputSchema,
-    //   handled by GetArrayStructFields ordinal remapping (no expansion needed)
+    // Expand only at top-level: insert null columns for non-decoded top-level fields.
     val expanded = if (needsExpansion) {
       withResource(jniResult) { decoded =>
         expandSchema(decoded, numRows)
@@ -187,10 +169,8 @@ case class GpuFromProtobuf(
   }
 
   /**
-   * Expand decoded struct to match outputSchema:
-   * - Top-level: insert null columns for non-decoded fields
-   * - Non-repeated structs with pruned children: expand by inserting null child columns
-   * - Repeated messages (ArrayType): no expansion (already pruned in outputSchema)
+   * Expand decoded top-level struct to match outputSchema by inserting null columns for
+   * non-decoded top-level fields.
    */
   private def expandSchema(decoded: cudf.ColumnVector, numRows: Int): cudf.ColumnVector = {
     val children = new Array[cudf.ColumnVector](fullSchema.fields.length)
@@ -200,21 +180,8 @@ case class GpuFromProtobuf(
       for (i <- fullSchema.fields.indices) {
         if (decodedIdx < decodedTopLevelIndices.length &&
             decodedTopLevelIndices(decodedIdx) == i) {
-          val fieldName = fullSchema.fields(i).name
-
-          // Check if this is a non-repeated struct with pruned children
-          nonRepeatedPrunedFields.get(fieldName) match {
-            case Some(decodedNames) =>
-              // Expand non-repeated struct: insert null columns for pruned children
-              val targetSt = fullSchema.fields(i).dataType.asInstanceOf[StructType]
-              withResource(decoded.getChildColumnView(decodedIdx)) { childView =>
-                children(i) = expandStructChildren(childView, targetSt, decodedNames, numRows)
-              }
-            case None =>
-              // No expansion needed (either full schema or ArrayType pruned via Option A)
-              withResource(decoded.getChildColumnView(decodedIdx)) { childView =>
-                children(i) = childView.copyToColumnVector()
-              }
+          withResource(decoded.getChildColumnView(decodedIdx)) { childView =>
+            children(i) = childView.copyToColumnVector()
           }
           decodedIdx += 1
         } else {
@@ -229,38 +196,6 @@ case class GpuFromProtobuf(
       children.foreach(col => if (col != null) col.close())
     }
   }
-
-  /**
-   * Expand a non-repeated STRUCT column by inserting null columns for pruned children.
-   * This is cheap because non-repeated structs have only num_rows elements.
-   */
-  private def expandStructChildren(
-      decoded: cudf.ColumnView,
-      targetSchema: StructType,
-      decodedChildNames: Seq[String],
-      numRows: Int): cudf.ColumnVector = {
-    val children = new Array[cudf.ColumnVector](targetSchema.fields.length)
-    var decodedChildIdx = 0
-
-    try {
-      for (i <- targetSchema.fields.indices) {
-        if (decodedChildIdx < decodedChildNames.length &&
-            decodedChildNames(decodedChildIdx) == targetSchema.fields(i).name) {
-          withResource(decoded.getChildColumnView(decodedChildIdx)) { childView =>
-            children(i) = childView.copyToColumnVector()
-          }
-          decodedChildIdx += 1
-        } else {
-          children(i) = GpuColumnVector.columnVectorFromNull(
-            numRows, targetSchema.fields(i).dataType)
-        }
-      }
-
-      cudf.ColumnVector.makeStruct(numRows, children: _*)
-    } finally {
-      children.foreach(col => if (col != null) col.close())
-    }
-  }
 }
 
 object GpuFromProtobuf {
diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/complexTypeExtractors.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/complexTypeExtractors.scala
index 972622bb825..3a1daa87d57 100644
--- a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/complexTypeExtractors.scala
+++ b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/complexTypeExtractors.scala
@@ -32,7 +32,12 @@ import org.apache.spark.sql.rapids.shims.RapidsErrorUtils
 import org.apache.spark.sql.types.{AbstractDataType, AnyDataType, ArrayType, BooleanType, DataType, IntegralType, LongType, MapType, StructField, StructType}
 import org.apache.spark.sql.vectorized.ColumnarBatch
 
-case class GpuGetStructField(child: Expression, ordinal: Int, name: Option[String] = None)
+case class GpuGetStructField(
+    child: Expression,
+    ordinal: Int,
+    name: Option[String] = None,
+    runtimeOrdinal: Int = -1,
+    resolvedType: Option[DataType] = None)
     extends ShimUnaryExpression
     with GpuExpression
     with ShimGetStructField
@@ -40,22 +45,38 @@ case class GpuGetStructField(child: Expression, ordinal: Int, name: Option[Strin
 
   lazy val childSchema: StructType = child.dataType.asInstanceOf[StructType]
 
-  override def dataType: DataType = childSchema(ordinal).dataType
-  override def nullable: Boolean = child.nullable || childSchema(ordinal).nullable
+  override def dataType: DataType =
+    resolvedType.getOrElse(childSchema(ordinal).dataType)
+
+  override def nullable: Boolean =
+    child.nullable || resolvedType.map(_ => true)
+      .getOrElse(childSchema(ordinal).nullable)
 
   override def toString: String = {
-    val fieldName = if (resolved) childSchema(ordinal).name else s"_$ordinal"
+    val fieldName = if (runtimeOrdinal >= 0) {
+      name.getOrElse(s"_$ordinal")
+    } else if (resolved) {
+      childSchema(ordinal).name
+    } else {
+      s"_$ordinal"
+    }
     s"$child.${name.getOrElse(fieldName)}"
   }
 
-  override def sql: String =
-    child.sql + s".${quoteIdentifier(name.getOrElse(childSchema(ordinal).name))}"
+  override def sql: String = {
+    val fieldName = if (runtimeOrdinal >= 0) name.getOrElse(s"_$ordinal")
+      else childSchema(ordinal).name
+    child.sql + s".${quoteIdentifier(name.getOrElse(fieldName))}"
+  }
+
+  private def effectiveOrdinal: Int =
+    if (runtimeOrdinal >= 0) runtimeOrdinal else ordinal
 
   override def columnarEvalAny(batch: ColumnarBatch): Any = {
     val dt = dataType
     withResourceIfAllowed(child.columnarEvalAny(batch)) {
       case cv: GpuColumnVector =>
-        withResource(cv.getBase.getChildColumnView(ordinal)) { view =>
+        withResource(cv.getBase.getChildColumnView(effectiveOrdinal)) { view =>
           GpuColumnVector.from(view.copyToColumnVector(), dt)
         }
       case s: GpuScalar =>
@@ -64,7 +85,7 @@ case class GpuGetStructField(child: Expression, ordinal: Int, name: Option[Strin
           GpuScalar(null, dt)
         } else {
           withResource(s.getBase.getChildrenFromStructScalar) { children =>
-            GpuScalar.wrap(children(ordinal).getScalarElement(0), dt)
+            GpuScalar.wrap(children(effectiveOrdinal).getScalarElement(0), dt)
           }
         }
       case other =>
@@ -402,6 +423,21 @@ case class GpuArrayPosition(left: Expression, right: Expression)
   }
 }
 
+class GpuGetStructFieldMeta(
+    expr: GetStructField,
+    conf: RapidsConf,
+    parent: Option[RapidsMeta[_, _, _]],
+    rule: DataFromReplacementRule)
+  extends UnaryExprMeta[GetStructField](expr, conf, parent, rule) {
+
+  def convertToGpu(child: Expression): GpuExpression = {
+    val fieldName = expr.name.getOrElse(expr.childSchema(expr.ordinal).name)
+    val runtimeOrd = GpuFromProtobuf.getPrunedOrdinal(fieldName)
+    val resolvedType = if (runtimeOrd >= 0) Some(expr.dataType) else None
+    GpuGetStructField(child, expr.ordinal, expr.name, runtimeOrd, resolvedType)
+  }
+}
+
 class GpuGetArrayStructFieldsMeta(
      expr: GetArrayStructFields,
      conf: RapidsConf,
@@ -410,7 +446,6 @@ class GpuGetArrayStructFieldsMeta(
   extends UnaryExprMeta[GetArrayStructFields](expr, conf, parent, rule) {
 
   def convertToGpu(child: Expression): GpuExpression = {
-    // Check the global protobuf pruned field registry to remap ordinal
     val runtimeOrd = GpuFromProtobuf.getPrunedOrdinal(expr.field.name)
     GpuGetArrayStructFields(child, expr.field, expr.ordinal, expr.numFields,
       expr.containsNull, runtimeOrd)
@@ -437,18 +472,12 @@ case class GpuGetArrayStructFields(
   override def toString: String = s"$child.${field.name}"
   override def sql: String = s"${child.sql}.${quoteIdentifier(field.name)}"
 
+  private def effectiveOrdinal: Int =
+    if (runtimeOrdinal >= 0) runtimeOrdinal else ordinal
+
   override protected def doColumnar(input: GpuColumnVector): ColumnVector = {
     val base = input.getBase
     val fieldView = withResource(base.getChildColumnView(0)) { structView =>
-      val actualChildren = structView.getNumChildren
-      // Handle nested schema projection (Option A): the actual struct may have
-      // fewer children than numFields if GpuFromProtobuf pruned the schema.
-      // Use runtimeOrdinal (if set) when the struct was pruned.
-      val effectiveOrdinal = if (runtimeOrdinal >= 0 && actualChildren < numFields) {
-        runtimeOrdinal
-      } else {
-        ordinal
-      }
       structView.getChildColumnView(effectiveOrdinal)
     }
     val listView = withResource(fieldView) { _ =>
diff --git a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala
index 51463b0f3d6..01db878a3cb 100644
--- a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala
+++ b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala
@@ -942,12 +942,12 @@ object ProtobufExprShims {
           }
 
           // Register pruned field ordinal mappings for GpuGetArrayStructFieldsMeta.
-          // For each pruned ArrayType(StructType) field, map child field names to their
+          // For each pruned struct field, map child field names to their
           // ordinal in the pruned struct so runtime column access uses correct indices.
           val ordinalMappings = prunedFieldsMap.flatMap { case (parentName, childNames) =>
             val fieldIdx = fullSchema.fieldIndex(parentName)
             fullSchema.fields(fieldIdx).dataType match {
-              case ArrayType(_: StructType, _) =>
+              case ArrayType(_: StructType, _) | _: StructType =>
                 childNames.zipWithIndex.map { case (name, idx) => name -> idx }
               case _ => Seq.empty
             }

From bc1bee7371e8a576580c88440e0d202e83a49f18 Mon Sep 17 00:00:00 2001
From: Haoyang Li <haoyangl@nvidia.com>
Date: Sat, 28 Feb 2026 16:28:27 +0800
Subject: [PATCH 04/33] address comments

Signed-off-by: Haoyang Li <haoyangl@nvidia.com>
---
 .../src/main/python/protobuf_test.py          | 146 ++++++++++++
 .../spark/sql/rapids/GpuFromProtobuf.scala    |  34 +--
 .../rapids/shims/ProtobufExprShims.scala      | 222 +++++++++---------
 3 files changed, 274 insertions(+), 128 deletions(-)

diff --git a/integration_tests/src/main/python/protobuf_test.py b/integration_tests/src/main/python/protobuf_test.py
index 2d4bc14a34f..179e59d5543 100644
--- a/integration_tests/src/main/python/protobuf_test.py
+++ b/integration_tests/src/main/python/protobuf_test.py
@@ -2304,6 +2304,152 @@ def _get_field_by_path(expr, path):
     return current
 
 
+def _build_deep_nested_5_level_descriptor_set_bytes(spark):
+    """
+    Build a FileDescriptorSet for 5-level deep nesting:
+      message Level5 { optional int32 val5 = 1; }
+      message Level4 { optional int32 val4 = 1; optional Level5 level5 = 2; }
+      message Level3 { optional int32 val3 = 1; optional Level4 level4 = 2; }
+      message Level2 { optional int32 val2 = 1; optional Level3 level3 = 2; }
+      message Level1 { optional int32 val1 = 1; optional Level2 level2 = 2; }
+    """
+    D, fd = _new_proto2_file(spark, "deep_nested_5_level.proto")
+
+    label_opt = D.FieldDescriptorProto.Label.LABEL_OPTIONAL
+
+    # Level5 message
+    level5_msg = D.DescriptorProto.newBuilder().setName("Level5")
+    level5_msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("val5")
+            .setNumber(1)
+            .setLabel(label_opt)
+            .setType(D.FieldDescriptorProto.Type.TYPE_INT32)
+            .build()
+    )
+    fd.addMessageType(level5_msg.build())
+
+    # Level4 message
+    level4_msg = D.DescriptorProto.newBuilder().setName("Level4")
+    level4_msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("val4")
+            .setNumber(1)
+            .setLabel(label_opt)
+            .setType(D.FieldDescriptorProto.Type.TYPE_INT32)
+            .build()
+    )
+    level4_msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("level5")
+            .setNumber(2)
+            .setLabel(label_opt)
+            .setType(D.FieldDescriptorProto.Type.TYPE_MESSAGE)
+            .setTypeName(".test.Level5")
+            .build()
+    )
+    fd.addMessageType(level4_msg.build())
+
+    # Level3 message
+    level3_msg = D.DescriptorProto.newBuilder().setName("Level3")
+    level3_msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("val3")
+            .setNumber(1)
+            .setLabel(label_opt)
+            .setType(D.FieldDescriptorProto.Type.TYPE_INT32)
+            .build()
+    )
+    level3_msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("level4")
+            .setNumber(2)
+            .setLabel(label_opt)
+            .setType(D.FieldDescriptorProto.Type.TYPE_MESSAGE)
+            .setTypeName(".test.Level4")
+            .build()
+    )
+    fd.addMessageType(level3_msg.build())
+
+    # Level2 message
+    level2_msg = D.DescriptorProto.newBuilder().setName("Level2")
+    level2_msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("val2")
+            .setNumber(1)
+            .setLabel(label_opt)
+            .setType(D.FieldDescriptorProto.Type.TYPE_INT32)
+            .build()
+    )
+    level2_msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("level3")
+            .setNumber(2)
+            .setLabel(label_opt)
+            .setType(D.FieldDescriptorProto.Type.TYPE_MESSAGE)
+            .setTypeName(".test.Level3")
+            .build()
+    )
+    fd.addMessageType(level2_msg.build())
+
+    # Level1 message
+    level1_msg = D.DescriptorProto.newBuilder().setName("Level1")
+    level1_msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("val1")
+            .setNumber(1)
+            .setLabel(label_opt)
+            .setType(D.FieldDescriptorProto.Type.TYPE_INT32)
+            .build()
+    )
+    level1_msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("level2")
+            .setNumber(2)
+            .setLabel(label_opt)
+            .setType(D.FieldDescriptorProto.Type.TYPE_MESSAGE)
+            .setTypeName(".test.Level2")
+            .build()
+    )
+    fd.addMessageType(level1_msg.build())
+
+    fds = D.FileDescriptorSet.newBuilder().addFile(fd.build()).build()
+    return bytes(fds.toByteArray())
+
+
+@pytest.mark.skipif(is_before_spark_340(), reason="from_protobuf is Spark 3.4.0+")
+@ignore_order(local=True)
+def test_from_protobuf_deep_nesting_5_levels(spark_tmp_path, from_protobuf_fn):
+    desc_path, desc_bytes = _setup_protobuf_desc(
+        spark_tmp_path, "deep_nested_5_level.desc",
+        _build_deep_nested_5_level_descriptor_set_bytes)
+    message_name = "test.Level1"
+    data_gen = ProtobufMessageGen([
+        PbScalar("val1", 1, IntegerGen()),
+        PbNested("level2", 2, [
+            PbScalar("val2", 1, IntegerGen()),
+            PbNested("level3", 2, [
+                PbScalar("val3", 1, IntegerGen()),
+                PbNested("level4", 2, [
+                    PbScalar("val4", 1, IntegerGen()),
+                    PbNested("level5", 2, [
+                        PbScalar("val5", 1, IntegerGen()),
+                    ]),
+                ]),
+            ]),
+        ]),
+    ])
+    def run_on_spark(spark):
+        df = gen_df(spark, data_gen)
+        decoded = _call_from_protobuf(
+            from_protobuf_fn, f.col("bin"), message_name, desc_path, desc_bytes)
+        return df.select(
+            decoded.getField("val1").alias("val1"),
+            decoded.getField("level2").alias("level2"),
+        )
+    assert_gpu_and_cpu_are_equal_collect(run_on_spark)
+
+
 @pytest.mark.skipif(is_before_spark_340(), reason="from_protobuf is Spark 3.4.0+")
 @pytest.mark.parametrize("case_id,select_specs", _schema_proj_cases, ids=lambda c: c[0] if isinstance(c, tuple) else str(c))
 @ignore_order(local=True)
diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuFromProtobuf.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuFromProtobuf.scala
index a94d1f9c8ff..106ecbccde2 100644
--- a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuFromProtobuf.scala
+++ b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuFromProtobuf.scala
@@ -20,7 +20,7 @@ import ai.rapids.cudf
 import ai.rapids.cudf.{BinaryOp, CudfException, DType}
 import com.nvidia.spark.rapids.{GpuColumnVector, GpuUnaryExpression}
 import com.nvidia.spark.rapids.Arm.withResource
-import com.nvidia.spark.rapids.jni.Protobuf
+import com.nvidia.spark.rapids.jni.{Protobuf, ProtobufSchemaDescriptor}
 import com.nvidia.spark.rapids.shims.NullIntolerantShim
 
 import org.apache.spark.sql.catalyst.expressions.{ExpectsInputTypes, Expression}
@@ -123,27 +123,15 @@ case class GpuFromProtobuf(
   private val needsExpansion: Boolean =
     decodedTopLevelIndices.length != fullSchema.fields.length
 
+  @transient private lazy val schema = new ProtobufSchemaDescriptor(
+    fieldNumbers, parentIndices, depthLevels, wireTypes, outputTypeIds, encodings,
+    isRepeated, isRequired, hasDefaultValue, defaultInts, defaultFloats, defaultBools,
+    defaultStrings, enumValidValues, enumNames)
+
   override protected def doColumnar(input: GpuColumnVector): cudf.ColumnVector = {
     val numRows = input.getRowCount.toInt
     val jniResult = try {
-      Protobuf.decodeToStruct(
-        input.getBase,
-        fieldNumbers,
-        parentIndices,
-        depthLevels,
-        wireTypes,
-        outputTypeIds,
-        encodings,
-        isRepeated,
-        isRequired,
-        hasDefaultValue,
-        defaultInts,
-        defaultFloats,
-        defaultBools,
-        defaultStrings,
-        enumValidValues,
-        enumNames,
-        failOnErrors)
+      Protobuf.decodeToStruct(input.getBase, schema, failOnErrors)
     } catch {
       case e: CudfException if failOnErrors =>
         throw new org.apache.spark.SparkException("Malformed protobuf message", e)
@@ -199,16 +187,16 @@ case class GpuFromProtobuf(
 }
 
 object GpuFromProtobuf {
-  // Encodings from com.nvidia.spark.rapids.jni.Protobuf
   val ENC_DEFAULT = 0
   val ENC_FIXED   = 1
   val ENC_ZIGZAG  = 2
   val ENC_ENUM_STRING = 3
 
   // Thread-local registry for pruned field ordinal mappings.
-  // When GpuFromProtobuf is created with nested pruning, it registers
-  // the field name -> pruned ordinal mapping. GpuGetArrayStructFieldsMeta
-  // reads this during convertToGpu to remap ordinals.
+  // Set by ProtobufExprShims.convertToGpu during plan conversion (single-threaded per query),
+  // consumed by GpuGetStructFieldMeta / GpuGetArrayStructFieldsMeta during the same conversion.
+  // A ThreadLocal is necessary here because the expression metas are general-purpose overrides
+  // (not protobuf-specific) and their parent meta chain doesn't contain the protobuf meta.
   private val prunedFieldOrdinals = new ThreadLocal[Map[String, Int]]() {
     override def initialValue(): Map[String, Int] = Map.empty
   }
diff --git a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala
index 01db878a3cb..8e181357a3d 100644
--- a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala
+++ b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala
@@ -126,7 +126,6 @@ object ProtobufExprShims {
         TypeSig.BINARY),
       (e, conf, p, r) => new UnaryExprMeta[UnaryExpression](e, conf, p, r) {
 
-        // Full schema from the expression (must match original dataType for compatibility)
         private var fullSchema: StructType = _
         private var failOnErrors: Boolean = _
 
@@ -199,11 +198,7 @@ object ProtobufExprShims {
 
           // Reject proto3 descriptors — GPU decoder only supports proto2 semantics.
           // proto3 has different null/default-value behavior that the GPU path doesn't handle.
-          val protoSyntax = Try {
-            val fileDesc = invoke0[AnyRef](msgDesc, "getFile")
-            val syntaxObj = invoke0[AnyRef](fileDesc, "getSyntax")
-            typeName(syntaxObj)
-          }.getOrElse("")
+          val protoSyntax = PbReflect.getFileSyntax(msgDesc)
           if (protoSyntax == "PROTO3") {
             willNotWorkOnGpu(
               "proto3 syntax is not supported by the GPU protobuf decoder; " +
@@ -357,7 +352,7 @@ object ProtobufExprShims {
                   addChildFieldsFromStruct(st, nestedMsgDesc, sf.name, currentIdx, depth)
                   
                 case ArrayType(st: StructType, _) if nestedMsgDesc != null =>
-                  // Repeated message field (pruned via Option A ordinal remapping)
+                  // Repeated message field (pruned via ordinal remapping)
                   addChildFieldsFromStruct(st, nestedMsgDesc, sf.name, currentIdx, depth)
                   
                 case _ => // Not a struct, no children to add
@@ -366,46 +361,33 @@ object ProtobufExprShims {
             
             // Helper to add child fields from a struct type.
             // Applies nested schema pruning for ALL struct types (both repeated and non-repeated).
-            // For repeated message fields (ArrayType(StructType)), the pruned output is handled
-            // by ordinal remapping in GpuGetArrayStructFieldsMeta (Option A), not by expansion.
+            // Pruned output is handled by ordinal remapping in GpuGetStructFieldMeta /
+            // GpuGetArrayStructFieldsMeta, not by null column expansion.
             def addChildFieldsFromStruct(
                 st: StructType,
                 parentMsgDesc: AnyRef,
                 fieldName: String,
                 parentIdx: Int,
                 parentDepth: Int): Unit = {
-              val fd = invoke1[AnyRef](
-                parentMsgDesc, "findFieldByName", classOf[String], fieldName)
+              val fd = PbReflect.findFieldByName(parentMsgDesc, fieldName)
               if (fd != null) {
                 try {
-                  val childMsgDesc = invoke0[AnyRef](fd, "getMessageType")
-                  // Filter children based on nested schema projection requirements.
+                  val childMsgDesc = PbReflect.getMessageType(fd)
                   val requiredChildren = nestedFieldRequirements.get(fieldName)
                   val filteredFields = requiredChildren match {
                     case Some(Some(childNames)) =>
-                      // Only add required children (nested schema projection)
                       st.fields.filter(f => childNames.contains(f.name))
                     case _ =>
-                      // None (field not in map) or Some(None) (whole field needed)
                       st.fields
                   }
                   filteredFields.foreach { childSf =>
-                    val childFd = invoke1[AnyRef](
-                      childMsgDesc, "findFieldByName", classOf[String], childSf.name)
+                    val childFd = PbReflect.findFieldByName(childMsgDesc, childSf.name)
                     if (childFd != null) {
-                      val childProtoType = invoke0[AnyRef](childFd, "getType")
-                      val childProtoTypeName = typeName(childProtoType)
-                      val childFieldNumber = invoke0[java.lang.Integer](
-                        childFd, "getNumber").intValue()
-                      val childIsRepeated = Try {
-                        invoke0[java.lang.Boolean](childFd, "isRepeated").booleanValue()
-                      }.getOrElse(false)
-                      val childIsRequired = Try {
-                        invoke0[java.lang.Boolean](childFd, "isRequired").booleanValue()
-                      }.getOrElse(false)
-                      val childHasDefault = Try {
-                        invoke0[java.lang.Boolean](childFd, "hasDefaultValue").booleanValue()
-                      }.getOrElse(false)
+                      val childProtoTypeName = typeName(PbReflect.getFieldType(childFd))
+                      val childFieldNumber = PbReflect.getFieldNumber(childFd)
+                      val childIsRepeated = PbReflect.isRepeated(childFd)
+                      val childIsRequired = PbReflect.isRequired(childFd)
+                      val childHasDefault = PbReflect.hasDefaultValue(childFd)
                       val (_, _, childEncoding) = checkFieldSupport(
                         childSf.dataType, childProtoTypeName, childIsRepeated, enumsAsInts)
 
@@ -413,15 +395,8 @@ object ProtobufExprShims {
                         Option[Map[Int, String]]) =
                         if (childProtoTypeName == "ENUM") {
                           Try {
-                            val enumType = invoke0[AnyRef](childFd, "getEnumType")
-                            val values = invoke0[java.util.List[_]](enumType, "getValues")
-                            import scala.collection.JavaConverters._
-                            val pairs = values.asScala.map { v =>
-                              val ev = v.asInstanceOf[AnyRef]
-                              val num = invoke0[java.lang.Integer](ev, "getNumber").intValue()
-                              val name = invoke0[String](ev, "getName")
-                              (num, name)
-                            }
+                            val pairs = PbReflect.getEnumValues(
+                              PbReflect.getEnumType(childFd))
                             if (enumsAsInts) {
                               (Some(pairs.map(_._1).toSet), None)
                             } else {
@@ -501,55 +476,28 @@ object ProtobufExprShims {
           val result = mutable.Map[String, ProtobufFieldInfo]()
 
           for (sf <- schema.fields) {
-            val fd = invoke1[AnyRef](msgDesc, "findFieldByName", classOf[String], sf.name)
+            val fd = PbReflect.findFieldByName(msgDesc, sf.name)
             if (fd == null) {
               willNotWorkOnGpu(
                 s"Protobuf field '${sf.name}' not found in message '$messageName'")
               return None
             }
 
-            val isRepeated = Try {
-              invoke0[java.lang.Boolean](fd, "isRepeated").booleanValue()
-            }.getOrElse(false)
-
-            // Check if field is required (proto2 required fields)
-            val isFieldRequired = Try {
-              invoke0[java.lang.Boolean](fd, "isRequired").booleanValue()
-            }.getOrElse(false)
-
-            // Check if field has a default value (proto2 [default = xxx])
-            val hasDefault = Try {
-              invoke0[java.lang.Boolean](fd, "hasDefaultValue").booleanValue()
-            }.getOrElse(false)
-
-            // Get the default value if it exists
-            val defaultVal = if (hasDefault) {
-              Try(Some(invoke0[AnyRef](fd, "getDefaultValue"))).getOrElse(None)
-            } else {
-              None
-            }
+            val isRepeated = PbReflect.isRepeated(fd)
+            val isFieldRequired = PbReflect.isRequired(fd)
+            val hasDefault = PbReflect.hasDefaultValue(fd)
+            val defaultVal = if (hasDefault) PbReflect.getDefaultValue(fd) else None
 
-            val protoType = invoke0[AnyRef](fd, "getType")
-            val protoTypeName = typeName(protoType)
-            val fieldNumber = invoke0[java.lang.Integer](fd, "getNumber").intValue()
+            val protoTypeName = typeName(PbReflect.getFieldType(fd))
+            val fieldNumber = PbReflect.getFieldNumber(fd)
 
-            // Check field support and determine encoding
             val (isSupported, unsupportedReason, encoding) =
               checkFieldSupport(sf.dataType, protoTypeName, isRepeated, enumsAsInts)
 
-            // Extract enum values and (for enumsAsInts=false) value-name mapping.
             val (enumVals, enumNameMap): (Option[Set[Int]], Option[Map[Int, String]]) =
               if (protoTypeName == "ENUM") {
                 Try {
-                  val enumType = invoke0[AnyRef](fd, "getEnumType")
-                  val values = invoke0[java.util.List[_]](enumType, "getValues")
-                  import scala.collection.JavaConverters._
-                  val pairs = values.asScala.map { v =>
-                    val ev = v.asInstanceOf[AnyRef]
-                    val num = invoke0[java.lang.Integer](ev, "getNumber").intValue()
-                    val name = invoke0[String](ev, "getName")
-                    (num, name)
-                  }
+                  val pairs = PbReflect.getEnumValues(PbReflect.getEnumType(fd))
                   if (enumsAsInts) {
                     (Some(pairs.map(_._1).toSet), None)
                   } else {
@@ -915,12 +863,11 @@ object ProtobufExprShims {
         }
 
         override def convertToGpu(child: Expression): GpuExpression = {
-          // Ensure thread-local ordinal mapping doesn't leak across query conversions.
           GpuFromProtobuf.clearPrunedFields()
 
           // Build pruned fields map for ALL struct types (both repeated and non-repeated).
-          // For repeated message fields (ArrayType(StructType)), pruning is handled via
-          // ordinal remapping in GpuGetArrayStructFieldsMeta (Option A) instead of expansion.
+          // Pruning is handled via ordinal remapping in GpuGetStructFieldMeta /
+          // GpuGetArrayStructFieldsMeta.
           val prunedFieldsMap: Map[String, Seq[String]] = nestedFieldRequirements.collect {
             case (fieldName, Some(childNames)) =>
               val fieldIdx = fullSchema.fieldIndex(fieldName)
@@ -930,7 +877,6 @@ object ProtobufExprShims {
                 case _ => null
               }
               if (childSchema != null) {
-                // Preserve the original field order
                 val orderedNames = childSchema.fields
                   .map(_.name)
                   .filter(childNames.contains)
@@ -941,9 +887,6 @@ object ProtobufExprShims {
               }
           }
 
-          // Register pruned field ordinal mappings for GpuGetArrayStructFieldsMeta.
-          // For each pruned struct field, map child field names to their
-          // ordinal in the pruned struct so runtime column access uses correct indices.
           val ordinalMappings = prunedFieldsMap.flatMap { case (parentName, childNames) =>
             val fieldIdx = fullSchema.fieldIndex(parentName)
             fullSchema.fields(fieldIdx).dataType match {
@@ -969,33 +912,21 @@ object ProtobufExprShims {
   }
 
   private def getMessageName(e: Expression): String =
-    invoke0[String](e, "messageName")
+    PbReflect.invoke0[String](e, "messageName")
 
-  /**
-   * Newer Spark versions may carry an in-expression descriptor set payload.
-   * - Spark 3.5.x: binaryFileDescriptorSet: Option[Array[Byte]]
-   * - Spark 4.x: binaryDescriptorSet (may be Array[Byte] or Option[Array[Byte]])
-   * Spark 3.4.x does not have this, so callers should fall back to descFilePath().
-   */
   private def getDescriptorBytes(e: Expression): Option[Array[Byte]] = {
-    // Spark 3.5.x: binaryFileDescriptorSet (note: "File" in the name)
-    val spark35Result = Try(invoke0[Option[Array[Byte]]](e, "binaryFileDescriptorSet"))
+    val spark35Result = Try(PbReflect.invoke0[Option[Array[Byte]]](e, "binaryFileDescriptorSet"))
       .toOption.flatten
     spark35Result.orElse {
-      // Spark 4.x: binaryDescriptorSet - may be Array[Byte] or Option[Array[Byte]]
-      val direct = Try(invoke0[Array[Byte]](e, "binaryDescriptorSet")).toOption
+      val direct = Try(PbReflect.invoke0[Array[Byte]](e, "binaryDescriptorSet")).toOption
       direct.orElse {
-        Try(invoke0[Option[Array[Byte]]](e, "binaryDescriptorSet")).toOption.flatten
+        Try(PbReflect.invoke0[Option[Array[Byte]]](e, "binaryDescriptorSet")).toOption.flatten
       }
     }
   }
 
-  /**
-   * Get descriptor file path from expression.
-   * Only available in Spark 3.4.x. Spark 3.5+ uses binaryFileDescriptorSet instead.
-   */
   private def getDescFilePath(e: Expression): Option[String] =
-    Try(invoke0[Option[String]](e, "descFilePath")).toOption.flatten
+    Try(PbReflect.invoke0[Option[String]](e, "descFilePath")).toOption.flatten
 
   /**
    * Build message descriptor using Spark's ProtobufUtils.
@@ -1027,19 +958,100 @@ object ProtobufExprShims {
     if (t == null) {
       "null"
     } else {
-      // Prefer Enum.name() when available; fall back to toString.
-      Try(invoke0[String](t, "name")).getOrElse(t.toString)
+      Try(PbReflect.invoke0[String](t, "name")).getOrElse(t.toString)
     }
   }
 
   private def getOptionsMap(e: Expression): Map[String, String] = {
-    val opt = Try(invoke0[scala.collection.Map[String, String]](e, "options")).toOption
+    val opt = Try(PbReflect.invoke0[scala.collection.Map[String, String]](e, "options")).toOption
     opt.map(_.toMap).getOrElse(Map.empty)
   }
 
-  private def invoke0[T](obj: AnyRef, method: String): T =
-    obj.getClass.getMethod(method).invoke(obj).asInstanceOf[T]
+  /**
+   * Cached reflection helper for protobuf-java descriptor APIs.
+   *
+   * All protobuf descriptor method calls go through this object so that:
+   *  1. java.lang.reflect.Method objects are cached (ConcurrentHashMap)
+   *  2. Missing methods produce a clear UnsupportedOperationException with the
+   *     class name and loaded protobuf-java version, instead of a raw
+   *     NoSuchMethodException.
+   */
+  private[shims] object PbReflect {
+    import java.lang.reflect.Method
+    private val cache = new java.util.concurrent.ConcurrentHashMap[String, Method]()
+
+    private def protobufJavaVersion: String = Try {
+      val rtCls = Class.forName("com.google.protobuf.RuntimeVersion")
+      val domain = rtCls.getField("DOMAIN").get(null)
+      val major = rtCls.getField("MAJOR").get(null)
+      val minor = rtCls.getField("MINOR").get(null)
+      val patch = rtCls.getField("PATCH").get(null)
+      s"$domain-$major.$minor.$patch"
+    }.getOrElse("unknown")
+
+    private def cached(cls: Class[_], name: String, paramTypes: Class[_]*): Method = {
+      val key = s"${cls.getName}#$name(${paramTypes.map(_.getName).mkString(",")})"
+      cache.computeIfAbsent(key, _ => {
+        try {
+          cls.getMethod(name, paramTypes: _*)
+        } catch {
+          case ex: NoSuchMethodException =>
+            throw new UnsupportedOperationException(
+              s"protobuf-java method not found: ${cls.getSimpleName}.$name " +
+                s"(protobuf-java version: $protobufJavaVersion). " +
+                s"This may indicate an incompatible protobuf-java library version.",
+              ex)
+        }
+      })
+    }
 
-  private def invoke1[T](obj: AnyRef, method: String, arg0Cls: Class[_], arg0: AnyRef): T =
-    obj.getClass.getMethod(method, arg0Cls).invoke(obj, arg0).asInstanceOf[T]
+    def invoke0[T](obj: AnyRef, method: String): T =
+      cached(obj.getClass, method).invoke(obj).asInstanceOf[T]
+
+    def invoke1[T](obj: AnyRef, method: String, arg0Cls: Class[_], arg0: AnyRef): T =
+      cached(obj.getClass, method, arg0Cls).invoke(obj, arg0).asInstanceOf[T]
+
+    // ---- Typed helpers for common Descriptor operations ----
+
+    def findFieldByName(msgDesc: AnyRef, name: String): AnyRef =
+      invoke1[AnyRef](msgDesc, "findFieldByName", classOf[String], name)
+
+    def getFieldNumber(fd: AnyRef): Int =
+      invoke0[java.lang.Integer](fd, "getNumber").intValue()
+
+    def getFieldType(fd: AnyRef): AnyRef = invoke0[AnyRef](fd, "getType")
+
+    def isRepeated(fd: AnyRef): Boolean =
+      Try(invoke0[java.lang.Boolean](fd, "isRepeated").booleanValue()).getOrElse(false)
+
+    def isRequired(fd: AnyRef): Boolean =
+      Try(invoke0[java.lang.Boolean](fd, "isRequired").booleanValue()).getOrElse(false)
+
+    def hasDefaultValue(fd: AnyRef): Boolean =
+      Try(invoke0[java.lang.Boolean](fd, "hasDefaultValue").booleanValue()).getOrElse(false)
+
+    def getDefaultValue(fd: AnyRef): Option[AnyRef] =
+      Try(Some(invoke0[AnyRef](fd, "getDefaultValue"))).getOrElse(None)
+
+    def getMessageType(fd: AnyRef): AnyRef = invoke0[AnyRef](fd, "getMessageType")
+
+    def getEnumType(fd: AnyRef): AnyRef = invoke0[AnyRef](fd, "getEnumType")
+
+    def getEnumValues(enumType: AnyRef): Seq[(Int, String)] = {
+      import scala.collection.JavaConverters._
+      val values = invoke0[java.util.List[_]](enumType, "getValues")
+      values.asScala.map { v =>
+        val ev = v.asInstanceOf[AnyRef]
+        val num = invoke0[java.lang.Integer](ev, "getNumber").intValue()
+        val name = invoke0[String](ev, "getName")
+        (num, name)
+      }.toSeq
+    }
+
+    def getFileSyntax(msgDesc: AnyRef): String = Try {
+      val fileDesc = invoke0[AnyRef](msgDesc, "getFile")
+      val syntaxObj = invoke0[AnyRef](fileDesc, "getSyntax")
+      typeName(syntaxObj)
+    }.getOrElse("")
+  }
 }

From 62679af22ec8648d57b9a810b42d73482d0c801a Mon Sep 17 00:00:00 2001
From: Haoyang Li <haoyangl@nvidia.com>
Date: Mon, 2 Mar 2026 13:00:01 +0800
Subject: [PATCH 05/33] bug fix and clean up

Signed-off-by: Haoyang Li <haoyangl@nvidia.com>
---
 .../src/main/python/protobuf_test.py          | 203 +++++++++++++++++
 .../nvidia/spark/rapids/GpuCanonicalize.scala |   4 +-
 .../spark/sql/rapids/GpuFromProtobuf.scala    | 102 +--------
 .../sql/rapids/complexTypeExtractors.scala    |  51 ++---
 .../rapids/shims/ProtobufExprShims.scala      | 212 ++++++++++++------
 5 files changed, 368 insertions(+), 204 deletions(-)

diff --git a/integration_tests/src/main/python/protobuf_test.py b/integration_tests/src/main/python/protobuf_test.py
index 179e59d5543..13be6332ce9 100644
--- a/integration_tests/src/main/python/protobuf_test.py
+++ b/integration_tests/src/main/python/protobuf_test.py
@@ -2470,3 +2470,206 @@ def run_on_spark(spark):
         return df.select(*selected)
 
     assert_gpu_and_cpu_are_equal_collect(run_on_spark)
+
+def _build_name_collision_descriptor_set_bytes(spark):
+    D, fd = _new_proto2_file(spark, "name_collision.proto")
+    label_opt = D.FieldDescriptorProto.Label.LABEL_OPTIONAL
+
+    # User message
+    user_msg = D.DescriptorProto.newBuilder().setName("User")
+    user_msg.addField(D.FieldDescriptorProto.newBuilder().setName("age").setNumber(1).setLabel(label_opt).setType(D.FieldDescriptorProto.Type.TYPE_INT32).build())
+    user_msg.addField(D.FieldDescriptorProto.newBuilder().setName("id").setNumber(2).setLabel(label_opt).setType(D.FieldDescriptorProto.Type.TYPE_INT32).build())
+    fd.addMessageType(user_msg.build())
+
+    # Ad message
+    ad_msg = D.DescriptorProto.newBuilder().setName("Ad")
+    ad_msg.addField(D.FieldDescriptorProto.newBuilder().setName("id").setNumber(1).setLabel(label_opt).setType(D.FieldDescriptorProto.Type.TYPE_INT32).build())
+    fd.addMessageType(ad_msg.build())
+
+    # Event message
+    event_msg = D.DescriptorProto.newBuilder().setName("Event")
+    event_msg.addField(D.FieldDescriptorProto.newBuilder().setName("user_info").setNumber(1).setLabel(label_opt).setType(D.FieldDescriptorProto.Type.TYPE_MESSAGE).setTypeName(".test.User").build())
+    event_msg.addField(D.FieldDescriptorProto.newBuilder().setName("ad_info").setNumber(2).setLabel(label_opt).setType(D.FieldDescriptorProto.Type.TYPE_MESSAGE).setTypeName(".test.Ad").build())
+    fd.addMessageType(event_msg.build())
+
+    fds = D.FileDescriptorSet.newBuilder().addFile(fd.build()).build()
+    return bytes(fds.toByteArray())
+
+@pytest.mark.skipif(is_before_spark_340(), reason="from_protobuf is Spark 3.4.0+")
+@ignore_order(local=True)
+def test_from_protobuf_bug1_name_collision(spark_tmp_path, from_protobuf_fn):
+    desc_path, desc_bytes = _setup_protobuf_desc(
+        spark_tmp_path, "name_collision.desc",
+        _build_name_collision_descriptor_set_bytes)
+    message_name = "test.Event"
+
+    data_gen = ProtobufMessageGen([
+        PbNested("user_info", 1, [
+            PbScalar("age", 1, IntegerGen()),
+            PbScalar("id", 2, IntegerGen()),
+        ]),
+        PbNested("ad_info", 2, [
+            PbScalar("id", 1, IntegerGen()),
+        ]),
+    ])
+
+    def run_on_spark(spark):
+        df = gen_df(spark, data_gen)
+        decoded = _call_from_protobuf(
+            from_protobuf_fn, f.col("bin"), message_name, desc_path, desc_bytes)
+        
+        return df.select(
+            decoded.getField("user_info").getField("age").alias("age"),
+            decoded.getField("user_info").getField("id").alias("user_id"),
+            decoded.getField("ad_info").getField("id").alias("ad_id")
+        )
+
+    assert_gpu_and_cpu_are_equal_collect(run_on_spark)
+
+
+def _build_filter_jump_descriptor_set_bytes(spark):
+    D, fd = _new_proto2_file(spark, "filter_jump.proto")
+    label_opt = D.FieldDescriptorProto.Label.LABEL_OPTIONAL
+
+    msg = D.DescriptorProto.newBuilder().setName("Event")
+    msg.addField(D.FieldDescriptorProto.newBuilder().setName("status").setNumber(1).setLabel(label_opt).setType(D.FieldDescriptorProto.Type.TYPE_INT32).build())
+    msg.addField(D.FieldDescriptorProto.newBuilder().setName("ad_info").setNumber(2).setLabel(label_opt).setType(D.FieldDescriptorProto.Type.TYPE_STRING).build())
+    fd.addMessageType(msg.build())
+
+    fds = D.FileDescriptorSet.newBuilder().addFile(fd.build()).build()
+    return bytes(fds.toByteArray())
+
+@pytest.mark.skipif(is_before_spark_340(), reason="from_protobuf is Spark 3.4.0+")
+@ignore_order(local=True)
+def test_from_protobuf_bug2_filter_jump(spark_tmp_path, from_protobuf_fn):
+    desc_path, desc_bytes = _setup_protobuf_desc(
+        spark_tmp_path, "filter_jump.desc",
+        _build_filter_jump_descriptor_set_bytes)
+    message_name = "test.Event"
+
+    data_gen = ProtobufMessageGen([
+        PbScalar("status", 1, IntegerGen(min_val=1, max_val=1)),
+        PbScalar("ad_info", 2, StringGen()),
+    ])
+
+    def run_on_spark(spark):
+        df = gen_df(spark, data_gen)
+        pb_expr1 = _call_from_protobuf(
+            from_protobuf_fn, f.col("bin"), message_name, desc_path, desc_bytes)
+        pb_expr2 = _call_from_protobuf(
+            from_protobuf_fn, f.col("bin"), message_name, desc_path, desc_bytes)
+        
+        return df.filter(pb_expr1.getField("status") == 1).select(pb_expr2.getField("ad_info").alias("ad_info"))
+
+    assert_gpu_and_cpu_are_equal_collect(run_on_spark)
+
+
+def _build_unrelated_struct_name_collision_descriptor_set_bytes(spark):
+    D, fd = _new_proto2_file(spark, "unrelated_struct.proto")
+    label_opt = D.FieldDescriptorProto.Label.LABEL_OPTIONAL
+
+    # Nested message
+    nested_msg = D.DescriptorProto.newBuilder().setName("Nested")
+    nested_msg.addField(D.FieldDescriptorProto.newBuilder().setName("dummy").setNumber(1).setLabel(label_opt).setType(D.FieldDescriptorProto.Type.TYPE_INT32).build())
+    nested_msg.addField(D.FieldDescriptorProto.newBuilder().setName("winfoid").setNumber(2).setLabel(label_opt).setType(D.FieldDescriptorProto.Type.TYPE_INT32).build())
+    fd.addMessageType(nested_msg.build())
+
+    msg = D.DescriptorProto.newBuilder().setName("Event")
+    msg.addField(D.FieldDescriptorProto.newBuilder().setName("ad_info").setNumber(1).setLabel(label_opt).setType(D.FieldDescriptorProto.Type.TYPE_MESSAGE).setTypeName(".test.Nested").build())
+    fd.addMessageType(msg.build())
+
+    fds = D.FileDescriptorSet.newBuilder().addFile(fd.build()).build()
+    return bytes(fds.toByteArray())
+
+@pytest.mark.skipif(is_before_spark_340(), reason="from_protobuf is Spark 3.4.0+")
+@ignore_order(local=True)
+def test_from_protobuf_bug3_unrelated_struct_name_collision(spark_tmp_path, from_protobuf_fn):
+    desc_path, desc_bytes = _setup_protobuf_desc(
+        spark_tmp_path, "unrelated_struct.desc",
+        _build_unrelated_struct_name_collision_descriptor_set_bytes)
+    message_name = "test.Event"
+
+    data_gen = ProtobufMessageGen([
+        PbNested("ad_info", 1, [
+            PbScalar("dummy", 1, IntegerGen()),
+            PbScalar("winfoid", 2, IntegerGen()),
+        ]),
+    ])
+
+    def run_on_spark(spark):
+        df = gen_df(spark, data_gen)
+        # Write to parquet to prevent Catalyst from optimizing away the GetStructField,
+        # and to ensure it runs on the GPU.
+        df_with_other = df.withColumn("other_struct",
+                                      f.struct(f.lit("hello").alias("dummy_str"), f.lit(42).alias("winfoid")))
+
+        path = spark_tmp_path + "/bug3_data.parquet"
+        df_with_other.write.mode("overwrite").parquet(path)
+        read_df = spark.read.parquet(path)
+        
+        decoded = _call_from_protobuf(
+            from_protobuf_fn, f.col("bin"), message_name, desc_path, desc_bytes)
+        
+        # We only select decoded.ad_info.winfoid, so dummy is pruned.
+        # winfoid gets ordinal 0 in the pruned schema.
+        # But for other_struct, winfoid is ordinal 1.
+        # GpuGetStructFieldMeta will see "winfoid", query the ThreadLocal, get 0, 
+        # and extract ordinal 0 ("hello") for other_winfoid, causing a mismatch!
+        return read_df.select(
+            decoded.getField("ad_info").getField("winfoid").alias("pb_winfoid"),
+            f.col("other_struct").getField("winfoid").alias("other_winfoid")
+        )
+
+    assert_gpu_and_cpu_are_equal_collect(run_on_spark)
+
+
+def _build_max_depth_descriptor_set_bytes(spark):
+    D, fd = _new_proto2_file(spark, "max_depth.proto")
+    label_opt = D.FieldDescriptorProto.Label.LABEL_OPTIONAL
+
+    # Generate 12 levels of nesting
+    for i in range(12, 0, -1):
+        msg = D.DescriptorProto.newBuilder().setName(f"Level{i}")
+        msg.addField(D.FieldDescriptorProto.newBuilder().setName(f"val{i}").setNumber(1).setLabel(label_opt).setType(D.FieldDescriptorProto.Type.TYPE_INT32).build())
+        if i < 12:
+            msg.addField(D.FieldDescriptorProto.newBuilder().setName(f"level{i+1}").setNumber(2).setLabel(label_opt).setType(D.FieldDescriptorProto.Type.TYPE_MESSAGE).setTypeName(f".test.Level{i+1}").build())
+        fd.addMessageType(msg.build())
+
+    fds = D.FileDescriptorSet.newBuilder().addFile(fd.build()).build()
+    return bytes(fds.toByteArray())
+
+@pytest.mark.skipif(is_before_spark_340(), reason="from_protobuf is Spark 3.4.0+")
+@ignore_order(local=True)
+def test_from_protobuf_bug4_max_depth(spark_tmp_path, from_protobuf_fn):
+    desc_path, desc_bytes = _setup_protobuf_desc(
+        spark_tmp_path, "max_depth.desc",
+        _build_max_depth_descriptor_set_bytes)
+    message_name = "test.Level1"
+
+    # Build the deeply nested data gen spec
+    def build_nested_gen(level):
+        if level == 12:
+            return [PbScalar(f"val{level}", 1, IntegerGen())]
+        return [
+            PbScalar(f"val{level}", 1, IntegerGen()),
+            PbNested(f"level{level+1}", 2, build_nested_gen(level+1))
+        ]
+
+    data_gen = ProtobufMessageGen(build_nested_gen(1))
+
+    def run_on_spark(spark):
+        df = gen_df(spark, data_gen)
+        decoded = _call_from_protobuf(
+            from_protobuf_fn, f.col("bin"), message_name, desc_path, desc_bytes)
+        # Deep access
+        field = decoded
+        for i in range(2, 13):
+            field = field.getField(f"level{i}")
+        return df.select(field.getField("val12").alias("val12"))
+
+    # Depth 12 exceeds GPU max nesting depth (10), so the query should
+    # gracefully fall back to CPU. Verify that it still produces correct
+    # results (CPU path) without crashing.
+    from spark_session import with_cpu_session
+    cpu_result = with_cpu_session(lambda spark: run_on_spark(spark).collect())
+    assert len(cpu_result) > 0
diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuCanonicalize.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuCanonicalize.scala
index caeaea66e07..7adee94cad0 100644
--- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuCanonicalize.scala
+++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuCanonicalize.scala
@@ -51,8 +51,8 @@ object GpuCanonicalize {
     case a: AttributeReference =>
       AttributeReference("none", TrampolineUtil.asNullable(a.dataType))(exprId = a.exprId)
     case GetStructField(child, ordinal, Some(_)) => GetStructField(child, ordinal, None)
-    case GpuGetStructField(child, ordinal, Some(_), runtimeOrdinal, resolvedType) =>
-      GpuGetStructField(child, ordinal, None, runtimeOrdinal, resolvedType)
+    case GpuGetStructField(child, ordinal, Some(_)) =>
+      GpuGetStructField(child, ordinal, None)
     case _ => e
   }
 
diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuFromProtobuf.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuFromProtobuf.scala
index 106ecbccde2..abeac6ba01e 100644
--- a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuFromProtobuf.scala
+++ b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuFromProtobuf.scala
@@ -65,7 +65,7 @@ import org.apache.spark.sql.types._
  */
 case class GpuFromProtobuf(
     fullSchema: StructType,
-    decodedTopLevelIndices: Array[Int],
+    decodedSchema: StructType,
     fieldNumbers: Array[Int],
     parentIndices: Array[Int],
     depthLevels: Array[Int],
@@ -81,55 +81,22 @@ case class GpuFromProtobuf(
     defaultStrings: Array[Array[Byte]],
     enumValidValues: Array[Array[Int]],
     enumNames: Array[Array[Array[Byte]]],
-    nestedPrunedFields: Map[String, Seq[String]],
     failOnErrors: Boolean,
     child: Expression)
   extends GpuUnaryExpression with ExpectsInputTypes with NullIntolerantShim {
 
   override def inputTypes: Seq[AbstractDataType] = Seq(BinaryType)
 
-  /**
-   * Build a pruned schema for nested struct fields:
-   * - Top-level field order still matches fullSchema
-   * - Nested ArrayType(StructType) and StructType are both pruned to decoded children
-   * - Runtime ordinals are remapped by GpuGetStructField/GpuGetArrayStructFields
-   */
-  private val outputSchema: StructType = {
-    val fields = fullSchema.fields.map { field =>
-      nestedPrunedFields.get(field.name) match {
-        case Some(childNames) =>
-          field.dataType match {
-            case ArrayType(st: StructType, containsNull) =>
-              val prunedSt = StructType(
-                st.fields.filter(f => childNames.contains(f.name)))
-              field.copy(dataType = ArrayType(prunedSt, containsNull))
-            case st: StructType =>
-              val prunedSt = StructType(
-                st.fields.filter(f => childNames.contains(f.name)))
-              field.copy(dataType = prunedSt)
-            case _ => field
-          }
-        case None => field
-      }
-    }
-    StructType(fields).asNullable
-  }
-
-  override def dataType: DataType = outputSchema
+  override def dataType: DataType = decodedSchema
 
   override def nullable: Boolean = true
 
-  // Expansion is needed only when not all top-level fields are decoded.
-  private val needsExpansion: Boolean =
-    decodedTopLevelIndices.length != fullSchema.fields.length
-
   @transient private lazy val schema = new ProtobufSchemaDescriptor(
     fieldNumbers, parentIndices, depthLevels, wireTypes, outputTypeIds, encodings,
     isRepeated, isRequired, hasDefaultValue, defaultInts, defaultFloats, defaultBools,
     defaultStrings, enumValidValues, enumNames)
 
   override protected def doColumnar(input: GpuColumnVector): cudf.ColumnVector = {
-    val numRows = input.getRowCount.toInt
     val jniResult = try {
       Protobuf.decodeToStruct(input.getBase, schema, failOnErrors)
     } catch {
@@ -137,51 +104,13 @@ case class GpuFromProtobuf(
         throw new org.apache.spark.SparkException("Malformed protobuf message", e)
     }
 
-    // Expand only at top-level: insert null columns for non-decoded top-level fields.
-    val expanded = if (needsExpansion) {
-      withResource(jniResult) { decoded =>
-        expandSchema(decoded, numRows)
-      }
-    } else {
-      jniResult
-    }
-
     // Apply input nulls to output
     if (input.getBase.hasNulls) {
-      withResource(expanded) { _ =>
-        expanded.mergeAndSetValidity(BinaryOp.BITWISE_AND, input.getBase)
+      withResource(jniResult) { _ =>
+        jniResult.mergeAndSetValidity(BinaryOp.BITWISE_AND, input.getBase)
       }
     } else {
-      expanded
-    }
-  }
-
-  /**
-   * Expand decoded top-level struct to match outputSchema by inserting null columns for
-   * non-decoded top-level fields.
-   */
-  private def expandSchema(decoded: cudf.ColumnVector, numRows: Int): cudf.ColumnVector = {
-    val children = new Array[cudf.ColumnVector](fullSchema.fields.length)
-    var decodedIdx = 0
-
-    try {
-      for (i <- fullSchema.fields.indices) {
-        if (decodedIdx < decodedTopLevelIndices.length &&
-            decodedTopLevelIndices(decodedIdx) == i) {
-          withResource(decoded.getChildColumnView(decodedIdx)) { childView =>
-            children(i) = childView.copyToColumnVector()
-          }
-          decodedIdx += 1
-        } else {
-          // This field was not decoded — create null column
-          children(i) = GpuColumnVector.columnVectorFromNull(
-            numRows, outputSchema.fields(i).dataType)
-        }
-      }
-
-      cudf.ColumnVector.makeStruct(numRows, children: _*)
-    } finally {
-      children.foreach(col => if (col != null) col.close())
+      jniResult
     }
   }
 }
@@ -192,27 +121,6 @@ object GpuFromProtobuf {
   val ENC_ZIGZAG  = 2
   val ENC_ENUM_STRING = 3
 
-  // Thread-local registry for pruned field ordinal mappings.
-  // Set by ProtobufExprShims.convertToGpu during plan conversion (single-threaded per query),
-  // consumed by GpuGetStructFieldMeta / GpuGetArrayStructFieldsMeta during the same conversion.
-  // A ThreadLocal is necessary here because the expression metas are general-purpose overrides
-  // (not protobuf-specific) and their parent meta chain doesn't contain the protobuf meta.
-  private val prunedFieldOrdinals = new ThreadLocal[Map[String, Int]]() {
-    override def initialValue(): Map[String, Int] = Map.empty
-  }
-
-  def registerPrunedFields(mappings: Map[String, Int]): Unit = {
-    prunedFieldOrdinals.set(prunedFieldOrdinals.get() ++ mappings)
-  }
-
-  def getPrunedOrdinal(fieldName: String): Int = {
-    prunedFieldOrdinals.get().getOrElse(fieldName, -1)
-  }
-
-  def clearPrunedFields(): Unit = {
-    prunedFieldOrdinals.set(Map.empty)
-  }
-
   /**
    * Maps a Spark DataType to the corresponding cuDF native type ID.
    * Note: The encoding (varint/zigzag/fixed) is determined by the protobuf field type,
diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/complexTypeExtractors.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/complexTypeExtractors.scala
index 3a1daa87d57..5ecfbbca1fb 100644
--- a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/complexTypeExtractors.scala
+++ b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/complexTypeExtractors.scala
@@ -35,9 +35,7 @@ import org.apache.spark.sql.vectorized.ColumnarBatch
 case class GpuGetStructField(
     child: Expression,
     ordinal: Int,
-    name: Option[String] = None,
-    runtimeOrdinal: Int = -1,
-    resolvedType: Option[DataType] = None)
+    name: Option[String] = None)
     extends ShimUnaryExpression
     with GpuExpression
     with ShimGetStructField
@@ -45,17 +43,13 @@ case class GpuGetStructField(
 
   lazy val childSchema: StructType = child.dataType.asInstanceOf[StructType]
 
-  override def dataType: DataType =
-    resolvedType.getOrElse(childSchema(ordinal).dataType)
+  override def dataType: DataType = childSchema(ordinal).dataType
 
   override def nullable: Boolean =
-    child.nullable || resolvedType.map(_ => true)
-      .getOrElse(childSchema(ordinal).nullable)
+    child.nullable || childSchema(ordinal).nullable
 
   override def toString: String = {
-    val fieldName = if (runtimeOrdinal >= 0) {
-      name.getOrElse(s"_$ordinal")
-    } else if (resolved) {
+    val fieldName = if (resolved) {
       childSchema(ordinal).name
     } else {
       s"_$ordinal"
@@ -64,28 +58,23 @@ case class GpuGetStructField(
   }
 
   override def sql: String = {
-    val fieldName = if (runtimeOrdinal >= 0) name.getOrElse(s"_$ordinal")
-      else childSchema(ordinal).name
+    val fieldName = childSchema(ordinal).name
     child.sql + s".${quoteIdentifier(name.getOrElse(fieldName))}"
   }
 
-  private def effectiveOrdinal: Int =
-    if (runtimeOrdinal >= 0) runtimeOrdinal else ordinal
-
   override def columnarEvalAny(batch: ColumnarBatch): Any = {
     val dt = dataType
     withResourceIfAllowed(child.columnarEvalAny(batch)) {
       case cv: GpuColumnVector =>
-        withResource(cv.getBase.getChildColumnView(effectiveOrdinal)) { view =>
+        withResource(cv.getBase.getChildColumnView(ordinal)) { view =>
           GpuColumnVector.from(view.copyToColumnVector(), dt)
         }
       case s: GpuScalar =>
-        // For a scalar in we want a scalar out.
         if (!s.isValid) {
           GpuScalar(null, dt)
         } else {
           withResource(s.getBase.getChildrenFromStructScalar) { children =>
-            GpuScalar.wrap(children(effectiveOrdinal).getScalarElement(0), dt)
+            GpuScalar.wrap(children(ordinal).getScalarElement(0), dt)
           }
         }
       case other =>
@@ -431,10 +420,11 @@ class GpuGetStructFieldMeta(
   extends UnaryExprMeta[GetStructField](expr, conf, parent, rule) {
 
   def convertToGpu(child: Expression): GpuExpression = {
-    val fieldName = expr.name.getOrElse(expr.childSchema(expr.ordinal).name)
-    val runtimeOrd = GpuFromProtobuf.getPrunedOrdinal(fieldName)
-    val resolvedType = if (runtimeOrd >= 0) Some(expr.dataType) else None
-    GpuGetStructField(child, expr.ordinal, expr.name, runtimeOrd, resolvedType)
+    val runtimeOrd = expr.getTagValue(
+      ProtobufExprShims.PRUNED_ORDINAL_TAG).getOrElse(-1)
+    val effectiveOrd =
+      if (runtimeOrd >= 0) runtimeOrd else expr.ordinal
+    GpuGetStructField(child, effectiveOrd, expr.name)
   }
 }
 
@@ -446,9 +436,12 @@ class GpuGetArrayStructFieldsMeta(
   extends UnaryExprMeta[GetArrayStructFields](expr, conf, parent, rule) {
 
   def convertToGpu(child: Expression): GpuExpression = {
-    val runtimeOrd = GpuFromProtobuf.getPrunedOrdinal(expr.field.name)
-    GpuGetArrayStructFields(child, expr.field, expr.ordinal, expr.numFields,
-      expr.containsNull, runtimeOrd)
+    val runtimeOrd = expr.getTagValue(
+      ProtobufExprShims.PRUNED_ORDINAL_TAG).getOrElse(-1)
+    val effectiveOrd =
+      if (runtimeOrd >= 0) runtimeOrd else expr.ordinal
+    GpuGetArrayStructFields(child, expr.field,
+      effectiveOrd, expr.numFields, expr.containsNull)
   }
 }
 
@@ -463,8 +456,7 @@ case class GpuGetArrayStructFields(
     field: StructField,
     ordinal: Int,
     numFields: Int,
-    containsNull: Boolean,
-    runtimeOrdinal: Int = -1) extends GpuUnaryExpression
+    containsNull: Boolean) extends GpuUnaryExpression
     with ShimGetArrayStructFields
     with NullIntolerantShim {
 
@@ -472,13 +464,10 @@ case class GpuGetArrayStructFields(
   override def toString: String = s"$child.${field.name}"
   override def sql: String = s"${child.sql}.${quoteIdentifier(field.name)}"
 
-  private def effectiveOrdinal: Int =
-    if (runtimeOrdinal >= 0) runtimeOrdinal else ordinal
-
   override protected def doColumnar(input: GpuColumnVector): ColumnVector = {
     val base = input.getBase
     val fieldView = withResource(base.getChildColumnView(0)) { structView =>
-      structView.getChildColumnView(effectiveOrdinal)
+      structView.getChildColumnView(ordinal)
     }
     val listView = withResource(fieldView) { _ =>
       GpuListUtils.replaceListDataColumnAsView(base, fieldView)
diff --git a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala
index 8e181357a3d..10701c0a616 100644
--- a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala
+++ b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala
@@ -104,6 +104,9 @@ object ProtobufExprShims {
   private[this] val sparkProtobufUtilsObjectClassName =
     "org.apache.spark.sql.protobuf.utils.ProtobufUtils$"
 
+  val PRUNED_ORDINAL_TAG =
+    new org.apache.spark.sql.catalyst.trees.TreeNodeTag[Int]("GPU_PRUNED_ORDINAL")
+
   def exprs: Map[Class[_ <: Expression], ExprRule[_ <: Expression]] = {
     try {
       val clazz = ShimReflectionUtils.loadClass(protobufDataToCatalystClassName)
@@ -271,6 +274,11 @@ object ProtobufExprShims {
 
               val currentIdx = flatFields.size
 
+              if (depth >= 10) {
+                willNotWorkOnGpu("Protobuf nesting depth exceeds maximum supported depth of 10")
+                return
+              }
+
               val outputType = sf.dataType match {
                 case ArrayType(elemType, _) =>
                   elemType match {
@@ -653,26 +661,43 @@ object ProtobufExprShims {
          * @param allFieldNames All field names in the full schema
          * @return Set of field names that are actually required
          */
+        private var targetExprsToRemap: Seq[Expression] = Seq.empty
+
         private def analyzeRequiredFields(allFieldNames: Set[String]): Set[String] = {
-          val parentPlanOpt = findParentPlanMeta()
+          val fieldReqs = mutable.Map[String, Option[Set[String]]]()
+          var hasDirectStructRef = false
+          val holder = () => { hasDirectStructRef = true }
+
+          var currentMeta: Option[SparkPlanMeta[_]] = findParentPlanMeta()
+          var foundProject = false
+          var safeToPrune = true
+          val collectedExprs = mutable.ArrayBuffer[Expression]()
+
+          while (currentMeta.isDefined && !foundProject && safeToPrune) {
+            currentMeta.get.wrapped match {
+              case p: ProjectExec =>
+                collectedExprs ++= p.projectList
+                p.projectList.foreach(collectStructFieldReferences(_, fieldReqs, holder))
+                foundProject = true
+              case f: org.apache.spark.sql.execution.FilterExec =>
+                collectedExprs += f.condition
+                collectStructFieldReferences(f.condition, fieldReqs, holder)
+                currentMeta = currentMeta.get.parent match {
+                  case Some(pm: SparkPlanMeta[_]) => Some(pm)
+                  case _ => None
+                }
+              case _ =>
+                safeToPrune = false
+            }
+          }
 
-          parentPlanOpt match {
-            case Some(planMeta) =>
-              analyzeDownstreamProject(planMeta) match {
-                case Some(fields) if fields.nonEmpty =>
-                  fields
-                case _ =>
-                  planMeta.parent match {
-                    case Some(grandParentMeta: SparkPlanMeta[_]) =>
-                      analyzeDownstreamProject(grandParentMeta) match {
-                        case Some(fields) if fields.nonEmpty => fields
-                        case _ => allFieldNames
-                      }
-                    case _ => allFieldNames
-                  }
-              }
-            case None =>
-              allFieldNames
+          if (!safeToPrune || !foundProject || hasDirectStructRef || fieldReqs.isEmpty) {
+            targetExprsToRemap = Seq.empty
+            allFieldNames
+          } else {
+            nestedFieldRequirements = fieldReqs.toMap
+            targetExprsToRemap = collectedExprs.toSeq
+            fieldReqs.keySet.toSet
           }
         }
 
@@ -698,39 +723,6 @@ object ProtobufExprShims {
         // Populated during analyzeDownstreamProject, used by addChildFieldsFromStruct
         private var nestedFieldRequirements: Map[String, Option[Set[String]]] = Map.empty
 
-        /**
-         * Analyze a Project plan to find which struct fields are actually used.
-         * This looks for GetStructField expressions that reference our protobuf output.
-         * Also detects nested field access (e.g., decoded.ad_info.winfoid) for nested
-         * schema projection.
-         */
-        private def analyzeDownstreamProject(planMeta: SparkPlanMeta[_]): Option[Set[String]] = {
-          planMeta.wrapped match {
-            case p: ProjectExec =>
-              // Collect field references with nested child tracking
-              // Key = top-level field name
-              // Value = None (need whole field) or Some(Set(...)) (only need specific children)
-              val fieldReqs = mutable.Map[String, Option[Set[String]]]()
-              var hasDirectStructRef = false
-
-              p.projectList.foreach { expr =>
-                collectStructFieldReferences(expr, fieldReqs, hasDirectStructRefHolder = () => {
-                  hasDirectStructRef = true
-                })
-              }
-
-              if (hasDirectStructRef) {
-                None
-              } else if (fieldReqs.nonEmpty) {
-                nestedFieldRequirements = fieldReqs.toMap
-                Some(fieldReqs.keySet.toSet)
-              } else {
-                None
-              }
-            case _ =>
-              None
-          }
-        }
 
         /**
          * Get the field name from a GetStructField expression using its ordinal and schema.
@@ -838,18 +830,30 @@ object ProtobufExprShims {
          *    (when accessing from a downstream ProjectExec)
          */
         private def isProtobufStructReference(expr: Expression): Boolean = {
-          // Check if expr is a ProtobufDataToCatalyst expression
-          if (expr.getClass.getName.contains("ProtobufDataToCatalyst")) {
+          if (expr eq e) {
             return true
           }
 
-          // Prefer exact attribute identity. Avoid matching by schema shape because unrelated
-          // struct columns can share the same field names/types.
-          val protobufOutputExprId: Option[org.apache.spark.sql.catalyst.expressions.ExprId] =
+          // Catalyst may create duplicate ProtobufDataToCatalyst
+          // instances for each GetStructField access. Match copies
+          // by class + identical input child so that
+          // analyzeRequiredFields detects all field accesses in one
+          // pass, keeping schema projection correct.
+          if (expr.getClass == e.getClass &&
+              expr.children.nonEmpty &&
+              e.children.nonEmpty &&
+              ((expr.children.head eq e.children.head) ||
+                expr.children.head.semanticEquals(
+                  e.children.head))) {
+            return true
+          }
+
+          val protobufOutputExprId
+            : Option[org.apache.spark.sql.catalyst.expressions.ExprId] =
             parent.flatMap { meta =>
               meta.wrapped match {
-                case alias: org.apache.spark.sql.catalyst.expressions.Alias
-                    if alias.child eq e =>
+                case alias: org.apache.spark.sql.catalyst.expressions
+                      .Alias if alias.child eq e =>
                   Some(alias.exprId)
                 case _ => None
               }
@@ -863,11 +867,6 @@ object ProtobufExprShims {
         }
 
         override def convertToGpu(child: Expression): GpuExpression = {
-          GpuFromProtobuf.clearPrunedFields()
-
-          // Build pruned fields map for ALL struct types (both repeated and non-repeated).
-          // Pruning is handled via ordinal remapping in GpuGetStructFieldMeta /
-          // GpuGetArrayStructFieldsMeta.
           val prunedFieldsMap: Map[String, Seq[String]] = nestedFieldRequirements.collect {
             case (fieldName, Some(childNames)) =>
               val fieldIdx = fullSchema.fieldIndex(fieldName)
@@ -887,25 +886,90 @@ object ProtobufExprShims {
               }
           }
 
-          val ordinalMappings = prunedFieldsMap.flatMap { case (parentName, childNames) =>
-            val fieldIdx = fullSchema.fieldIndex(parentName)
-            fullSchema.fields(fieldIdx).dataType match {
-              case ArrayType(_: StructType, _) | _: StructType =>
-                childNames.zipWithIndex.map { case (name, idx) => name -> idx }
-              case _ => Seq.empty
+          def registerExprs(expr: Expression): Unit = {
+            expr match {
+              case gsf @ GetStructField(childExpr, ordinal, nameOpt) =>
+                childExpr match {
+                  case GetStructField(innerChild, innerOrdinal, innerNameOpt)
+                      if isProtobufStructReference(innerChild) =>
+                    val parentName = getFieldName(innerOrdinal, innerNameOpt, fullSchema)
+                    val parentType = fullSchema.fields(innerOrdinal).dataType
+                    val childSchema = parentType match {
+                      case st: StructType => st
+                      case ArrayType(st: StructType, _) => st
+                      case _ => null
+                    }
+                    if (childSchema != null) {
+                      val childName = getFieldName(ordinal, nameOpt, childSchema)
+                      prunedFieldsMap.get(parentName).foreach { orderedChildren =>
+                        val runtimeOrd = orderedChildren.indexOf(childName)
+                        if (runtimeOrd >= 0) {
+                          gsf.setTagValue(ProtobufExprShims.PRUNED_ORDINAL_TAG, runtimeOrd)
+                        }
+                      }
+                    }
+                  case _ if isProtobufStructReference(childExpr) =>
+                    val runtimeOrd = decodedTopLevelIndices.indexOf(ordinal)
+                    if (runtimeOrd >= 0) {
+                      gsf.setTagValue(ProtobufExprShims.PRUNED_ORDINAL_TAG, runtimeOrd)
+                    }
+                  case _ =>
+                }
+
+              case gasf @ GetArrayStructFields(childExpr, field, _, _,
+                _) =>
+                childExpr match {
+                  case GetStructField(innerChild, innerOrdinal, innerNameOpt)
+                      if isProtobufStructReference(innerChild) =>
+                    val parentName = getFieldName(innerOrdinal, innerNameOpt, fullSchema)
+                    val childName = field.name
+                    prunedFieldsMap.get(parentName).foreach { orderedChildren =>
+                      val runtimeOrd = orderedChildren.indexOf(childName)
+                      if (runtimeOrd >= 0) {
+                        gasf.setTagValue(ProtobufExprShims.PRUNED_ORDINAL_TAG, runtimeOrd)
+                      }
+                    }
+                  case _ =>
+                }
+              case _ =>
             }
+            expr.children.foreach(registerExprs)
           }
-          if (ordinalMappings.nonEmpty) {
-            GpuFromProtobuf.registerPrunedFields(ordinalMappings)
+
+          targetExprsToRemap.foreach(registerExprs)
+
+          val decodedSchema = {
+            val decodedFields = decodedTopLevelIndices.map { idx =>
+              val field = fullSchema.fields(idx)
+              prunedFieldsMap.get(field.name) match {
+                case Some(childNames) =>
+                  field.dataType match {
+                    case ArrayType(st: StructType, cn) =>
+                      val pruned = StructType(
+                        st.fields.filter(f =>
+                          childNames.contains(f.name)))
+                      field.copy(dataType = ArrayType(pruned, cn))
+                    case st: StructType =>
+                      val pruned = StructType(
+                        st.fields.filter(f =>
+                          childNames.contains(f.name)))
+                      field.copy(dataType = pruned)
+                    case _ => field
+                  }
+                case None => field
+              }
+            }
+            StructType(decodedFields.map(f =>
+              f.copy(nullable = true)))
           }
 
           GpuFromProtobuf(
-            fullSchema, decodedTopLevelIndices, flatFieldNumbers, flatParentIndices,
+            fullSchema, decodedSchema,
+            flatFieldNumbers, flatParentIndices,
             flatDepthLevels, flatWireTypes, flatOutputTypeIds, flatEncodings,
             flatIsRepeated, flatIsRequired, flatHasDefaultValue, flatDefaultInts,
             flatDefaultFloats, flatDefaultBools, flatDefaultStrings, flatEnumValidValues,
-            flatEnumNames,
-            prunedFieldsMap, failOnErrors, child)
+            flatEnumNames, failOnErrors, child)
         }
       }
     )

From f7d95517900ab4f731dc3a94cab8c1b3699d072a Mon Sep 17 00:00:00 2001
From: Haoyang Li <haoyangl@nvidia.com>
Date: Tue, 3 Mar 2026 12:13:23 +0800
Subject: [PATCH 06/33] bug fix and clean up

Signed-off-by: Haoyang Li <haoyangl@nvidia.com>
---
 .../src/main/python/protobuf_test.py          | 369 ++++++++++++++++++
 .../rapids/shims/ProtobufExprShims.scala      | 261 ++++++++-----
 2 files changed, 523 insertions(+), 107 deletions(-)

diff --git a/integration_tests/src/main/python/protobuf_test.py b/integration_tests/src/main/python/protobuf_test.py
index 13be6332ce9..cfdbf031fa9 100644
--- a/integration_tests/src/main/python/protobuf_test.py
+++ b/integration_tests/src/main/python/protobuf_test.py
@@ -2673,3 +2673,372 @@ def run_on_spark(spark):
     from spark_session import with_cpu_session
     cpu_result = with_cpu_session(lambda spark: run_on_spark(spark).collect())
     assert len(cpu_result) > 0
+
+
+# ===========================================================================
+# Regression tests for known bugs found by code review
+# ===========================================================================
+
+def _encode_varint(value):
+    """Encode a non-negative integer as a protobuf varint (for hand-crafting test bytes)."""
+    out = bytearray()
+    v = int(value)
+    while True:
+        b = v & 0x7F
+        v >>= 7
+        if v:
+            out.append(b | 0x80)
+        else:
+            out.append(b)
+            break
+    return bytes(out)
+
+
+def _encode_tag(field_number, wire_type):
+    return _encode_varint((field_number << 3) | wire_type)
+
+
+# ---------------------------------------------------------------------------
+# Bug 1: BOOL8 truncation — non-canonical bool varint values >= 256
+#
+# Protobuf spec: bool is a varint; any non-zero value means true.
+# CPU decoder (protobuf-java): CodedInputStream.readBool() = readRawVarint64() != 0  →  true
+# GPU decoder: extract_varint_kernel writes static_cast<uint8_t>(v).
+#   For v = 256, static_cast<uint8_t>(256) == 0  →  false.  BUG.
+# ---------------------------------------------------------------------------
+
+@pytest.mark.skipif(is_before_spark_340(), reason="from_protobuf is Spark 3.4.0+")
+@ignore_order(local=True)
+def test_from_protobuf_bool_noncanonical_varint_scalar(spark_tmp_path, from_protobuf_fn):
+    """Scalar bool field encoded as varint 256 should decode as true, not false."""
+    desc_path, desc_bytes = _setup_protobuf_desc(
+        spark_tmp_path, "simple_bool_bug.desc", _build_simple_descriptor_set_bytes)
+    message_name = "test.Simple"
+
+    # Hand-craft protobuf wire bytes:
+    #   field 1 (bool b): tag = (1<<3)|0 = 0x08, value = varint(256) = 0x80 0x02
+    #   field 2 (int32 i32): tag = (2<<3)|0 = 0x10, value = varint(99) = 0x63
+    # Varint 256 is a perfectly valid encoding; CPU reads it as true.
+    # GPU truncates uint8_t(256) = 0 → false.
+    row_bool_256 = _encode_tag(1, 0) + _encode_varint(256) + \
+                   _encode_tag(2, 0) + _encode_varint(99)
+
+    # Control row: canonical bool true (varint 1) — should work on both
+    row_bool_1 = _encode_tag(1, 0) + _encode_varint(1) + \
+                 _encode_tag(2, 0) + _encode_varint(100)
+
+    # Another problematic value: 512 → uint8_t(0) → false
+    row_bool_512 = _encode_tag(1, 0) + _encode_varint(512) + \
+                   _encode_tag(2, 0) + _encode_varint(101)
+
+    def run_on_spark(spark):
+        df = spark.createDataFrame(
+            [(row_bool_256,), (row_bool_1,), (row_bool_512,)],
+            schema="bin binary",
+        )
+        decoded = _call_from_protobuf(
+            from_protobuf_fn, f.col("bin"), message_name, desc_path, desc_bytes)
+        return df.select(
+            decoded.getField("b").alias("b"),
+            decoded.getField("i32").alias("i32"),
+        )
+
+    assert_gpu_and_cpu_are_equal_collect(run_on_spark)
+
+
+def _build_repeated_bool_descriptor_set_bytes(spark):
+    """
+    message WithRepeatedBool {
+        optional int32 id = 1;
+        repeated bool flags = 2;
+    }
+    """
+    D, fd = _new_proto2_file(spark, "repeated_bool.proto")
+    label_opt = D.FieldDescriptorProto.Label.LABEL_OPTIONAL
+    label_rep = D.FieldDescriptorProto.Label.LABEL_REPEATED
+    msg = D.DescriptorProto.newBuilder().setName("WithRepeatedBool")
+    msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("id").setNumber(1).setLabel(label_opt)
+            .setType(D.FieldDescriptorProto.Type.TYPE_INT32).build())
+    msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("flags").setNumber(2).setLabel(label_rep)
+            .setType(D.FieldDescriptorProto.Type.TYPE_BOOL).build())
+    fd.addMessageType(msg.build())
+    fds = D.FileDescriptorSet.newBuilder().addFile(fd.build()).build()
+    return bytes(fds.toByteArray())
+
+
+@pytest.mark.skipif(is_before_spark_340(), reason="from_protobuf is Spark 3.4.0+")
+@ignore_order(local=True)
+def test_from_protobuf_bool_noncanonical_varint_repeated(spark_tmp_path, from_protobuf_fn):
+    """Repeated bool with non-canonical varint values (256, 512) should all decode as true."""
+    desc_path, desc_bytes = _setup_protobuf_desc(
+        spark_tmp_path, "repeated_bool_bug.desc", _build_repeated_bool_descriptor_set_bytes)
+    message_name = "test.WithRepeatedBool"
+
+    # Repeated bool field 2 (wire type 0 = varint), unpacked.
+    # Three elements: varint(256), varint(1), varint(512)
+    # CPU: [true, true, true]
+    # GPU (buggy): [false, true, false]  — because uint8(256)=0, uint8(512)=0
+    row = (_encode_tag(1, 0) + _encode_varint(42) +
+           _encode_tag(2, 0) + _encode_varint(256) +
+           _encode_tag(2, 0) + _encode_varint(1) +
+           _encode_tag(2, 0) + _encode_varint(512))
+
+    def run_on_spark(spark):
+        df = spark.createDataFrame([(row,)], schema="bin binary")
+        decoded = _call_from_protobuf(
+            from_protobuf_fn, f.col("bin"), message_name, desc_path, desc_bytes)
+        return df.select(
+            decoded.getField("id").alias("id"),
+            decoded.getField("flags").alias("flags"),
+        )
+
+    assert_gpu_and_cpu_are_equal_collect(run_on_spark)
+
+
+# ---------------------------------------------------------------------------
+# Bug 2: Nested message child field default values are lost
+#
+# In ProtobufExprShims.scala, addChildFieldsFromStruct always sets
+# defaultValue = None for nested children, even when hasDefaultValue = true.
+# This means proto2 defaults for fields inside nested messages are never
+# passed to the GPU decoder.
+#
+# CPU: missing child field → proto2 default value
+# GPU: missing child field → null
+# ---------------------------------------------------------------------------
+
+def _build_nested_with_defaults_descriptor_set_bytes(spark):
+    """
+    message Inner {
+        optional int32  count = 1 [default = 42];
+        optional string label = 2 [default = "hello"];
+        optional bool   flag  = 3 [default = true];
+    }
+    message OuterWithNestedDefaults {
+        optional int32 id    = 1;
+        optional Inner inner = 2;
+    }
+    """
+    D, fd = _new_proto2_file(spark, "nested_defaults.proto")
+    label_opt = D.FieldDescriptorProto.Label.LABEL_OPTIONAL
+
+    inner = D.DescriptorProto.newBuilder().setName("Inner")
+    inner.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("count").setNumber(1).setLabel(label_opt)
+            .setType(D.FieldDescriptorProto.Type.TYPE_INT32)
+            .setDefaultValue("42").build())
+    inner.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("label").setNumber(2).setLabel(label_opt)
+            .setType(D.FieldDescriptorProto.Type.TYPE_STRING)
+            .setDefaultValue("hello").build())
+    inner.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("flag").setNumber(3).setLabel(label_opt)
+            .setType(D.FieldDescriptorProto.Type.TYPE_BOOL)
+            .setDefaultValue("true").build())
+    fd.addMessageType(inner.build())
+
+    outer = D.DescriptorProto.newBuilder().setName("OuterWithNestedDefaults")
+    outer.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("id").setNumber(1).setLabel(label_opt)
+            .setType(D.FieldDescriptorProto.Type.TYPE_INT32).build())
+    outer.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("inner").setNumber(2).setLabel(label_opt)
+            .setType(D.FieldDescriptorProto.Type.TYPE_MESSAGE)
+            .setTypeName(".test.Inner").build())
+    fd.addMessageType(outer.build())
+
+    fds = D.FileDescriptorSet.newBuilder().addFile(fd.build()).build()
+    return bytes(fds.toByteArray())
+
+
+@pytest.mark.skipif(is_before_spark_340(), reason="from_protobuf is Spark 3.4.0+")
+@ignore_order(local=True)
+def test_from_protobuf_nested_child_default_values(spark_tmp_path, from_protobuf_fn):
+    """Proto2 default values for fields inside nested messages must be honored.
+
+    When `inner` is present but its child fields are absent, the CPU decoder
+    returns the proto2 defaults (count=42, label="hello", flag=true).
+    The GPU decoder currently returns null for all three because
+    defaultValue is never populated for nested children.
+    """
+    desc_path, desc_bytes = _setup_protobuf_desc(
+        spark_tmp_path, "nested_defaults.desc",
+        _build_nested_with_defaults_descriptor_set_bytes)
+    message_name = "test.OuterWithNestedDefaults"
+
+    # Row 1: outer.id = 10, inner is present but EMPTY (0-length nested message).
+    #   Wire: field 1 varint(10), field 2 length-delimited with length 0.
+    #   CPU should fill inner.count=42, inner.label="hello", inner.flag=true.
+    row_empty_inner = (_encode_tag(1, 0) + _encode_varint(10) +
+                       _encode_tag(2, 2) + _encode_varint(0))
+
+    # Row 2: outer.id = 20, inner has only count=7 (label and flag should get defaults).
+    inner_partial = _encode_tag(1, 0) + _encode_varint(7)
+    row_partial_inner = (_encode_tag(1, 0) + _encode_varint(20) +
+                         _encode_tag(2, 2) + _encode_varint(len(inner_partial)) +
+                         inner_partial)
+
+    # Row 3: outer.id = 30, inner is fully absent → inner itself is null.
+    row_no_inner = _encode_tag(1, 0) + _encode_varint(30)
+
+    def run_on_spark(spark):
+        df = spark.createDataFrame(
+            [(row_empty_inner,), (row_partial_inner,), (row_no_inner,)],
+            schema="bin binary",
+        )
+        decoded = _call_from_protobuf(
+            from_protobuf_fn, f.col("bin"), message_name, desc_path, desc_bytes)
+        return df.select(
+            decoded.getField("id").alias("id"),
+            decoded.getField("inner").getField("count").alias("inner_count"),
+            decoded.getField("inner").getField("label").alias("inner_label"),
+            decoded.getField("inner").getField("flag").alias("inner_flag"),
+        )
+
+    assert_gpu_and_cpu_are_equal_collect(run_on_spark)
+
+
+# ===========================================================================
+# Deep nested schema pruning tests
+#
+# These verify that the GPU path correctly prunes nested fields at depth > 2.
+# Previously, collectStructFieldReferences only recognized 2-level
+# GetStructField chains, so accessing decoded.level2.level3.val3 would
+# decode ALL of level3's children instead of only val3.
+# ===========================================================================
+
+def _deep_5_level_data_gen():
+    return ProtobufMessageGen([
+        PbScalar("val1", 1, IntegerGen()),
+        PbNested("level2", 2, [
+            PbScalar("val2", 1, IntegerGen()),
+            PbNested("level3", 2, [
+                PbScalar("val3", 1, IntegerGen()),
+                PbNested("level4", 2, [
+                    PbScalar("val4", 1, IntegerGen()),
+                    PbNested("level5", 2, [
+                        PbScalar("val5", 1, IntegerGen()),
+                    ]),
+                ]),
+            ]),
+        ]),
+    ])
+
+
+@pytest.mark.skipif(is_before_spark_340(), reason="from_protobuf is Spark 3.4.0+")
+@ignore_order(local=True)
+def test_deep_pruning_3_level_leaf(spark_tmp_path, from_protobuf_fn):
+    """Access decoded.level2.level3.val3 -- triggers 3-level deep pruning."""
+    desc_path, desc_bytes = _setup_protobuf_desc(
+        spark_tmp_path, "dp3.desc", _build_deep_nested_5_level_descriptor_set_bytes)
+    message_name = "test.Level1"
+    data_gen = _deep_5_level_data_gen()
+
+    def run_on_spark(spark):
+        df = gen_df(spark, data_gen)
+        decoded = _call_from_protobuf(
+            from_protobuf_fn, f.col("bin"), message_name, desc_path, desc_bytes)
+        return df.select(
+            decoded.getField("val1").alias("val1"),
+            decoded.getField("level2").getField("level3").getField("val3").alias("deep_val3"),
+        )
+
+    assert_gpu_and_cpu_are_equal_collect(run_on_spark)
+
+
+@pytest.mark.skipif(is_before_spark_340(), reason="from_protobuf is Spark 3.4.0+")
+@ignore_order(local=True)
+def test_deep_pruning_5_level_leaf(spark_tmp_path, from_protobuf_fn):
+    """Access decoded.level2.level3.level4.level5.val5 -- deepest leaf."""
+    desc_path, desc_bytes = _setup_protobuf_desc(
+        spark_tmp_path, "dp5.desc", _build_deep_nested_5_level_descriptor_set_bytes)
+    message_name = "test.Level1"
+    data_gen = _deep_5_level_data_gen()
+
+    def run_on_spark(spark):
+        df = gen_df(spark, data_gen)
+        decoded = _call_from_protobuf(
+            from_protobuf_fn, f.col("bin"), message_name, desc_path, desc_bytes)
+        return df.select(
+            _get_field_by_path(decoded, ["level2", "level3", "level4", "level5", "val5"])
+                .alias("val5"),
+        )
+
+    assert_gpu_and_cpu_are_equal_collect(run_on_spark)
+
+
+@pytest.mark.skipif(is_before_spark_340(), reason="from_protobuf is Spark 3.4.0+")
+@ignore_order(local=True)
+def test_deep_pruning_mixed_depths(spark_tmp_path, from_protobuf_fn):
+    """Access leaves at different depths in the same query.
+
+    Select val1 (depth 1), val2 (depth 2), val3 (depth 3), and val5 (depth 5)
+    to exercise pruning at every intermediate level simultaneously.
+    """
+    desc_path, desc_bytes = _setup_protobuf_desc(
+        spark_tmp_path, "dp_mix.desc", _build_deep_nested_5_level_descriptor_set_bytes)
+    message_name = "test.Level1"
+    data_gen = _deep_5_level_data_gen()
+
+    def run_on_spark(spark):
+        df = gen_df(spark, data_gen)
+        decoded = _call_from_protobuf(
+            from_protobuf_fn, f.col("bin"), message_name, desc_path, desc_bytes)
+        return df.select(
+            decoded.getField("val1").alias("val1"),
+            decoded.getField("level2").getField("val2").alias("val2"),
+            _get_field_by_path(decoded, ["level2", "level3", "val3"]).alias("val3"),
+            _get_field_by_path(decoded, ["level2", "level3", "level4", "level5", "val5"])
+                .alias("val5"),
+        )
+
+    assert_gpu_and_cpu_are_equal_collect(run_on_spark)
+
+
+@pytest.mark.skipif(is_before_spark_340(), reason="from_protobuf is Spark 3.4.0+")
+@ignore_order(local=True)
+def test_deep_pruning_sibling_at_depth_3(spark_tmp_path, from_protobuf_fn):
+    """At depth 3, access val3 but NOT level4 -- level4 subtree should be pruned."""
+    desc_path, desc_bytes = _setup_protobuf_desc(
+        spark_tmp_path, "dp_sib3.desc", _build_deep_nested_5_level_descriptor_set_bytes)
+    message_name = "test.Level1"
+    data_gen = _deep_5_level_data_gen()
+
+    def run_on_spark(spark):
+        df = gen_df(spark, data_gen)
+        decoded = _call_from_protobuf(
+            from_protobuf_fn, f.col("bin"), message_name, desc_path, desc_bytes)
+        return df.select(
+            _get_field_by_path(decoded, ["level2", "level3", "val3"]).alias("val3"),
+        )
+
+    assert_gpu_and_cpu_are_equal_collect(run_on_spark)
+
+
+@pytest.mark.skipif(is_before_spark_340(), reason="from_protobuf is Spark 3.4.0+")
+@ignore_order(local=True)
+def test_deep_pruning_whole_struct_at_depth_3(spark_tmp_path, from_protobuf_fn):
+    """Select the whole level3 struct -- no deep pruning inside level3."""
+    desc_path, desc_bytes = _setup_protobuf_desc(
+        spark_tmp_path, "dp_whole3.desc", _build_deep_nested_5_level_descriptor_set_bytes)
+    message_name = "test.Level1"
+    data_gen = _deep_5_level_data_gen()
+
+    def run_on_spark(spark):
+        df = gen_df(spark, data_gen)
+        decoded = _call_from_protobuf(
+            from_protobuf_fn, f.col("bin"), message_name, desc_path, desc_bytes)
+        return df.select(
+            decoded.getField("level2").getField("level3").alias("level3"),
+        )
+
+    assert_gpu_and_cpu_are_equal_collect(run_on_spark)
diff --git a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala
index 10701c0a616..1b211908306 100644
--- a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala
+++ b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala
@@ -264,13 +264,15 @@ object ProtobufExprShims {
           {
             val flatFields = mutable.ArrayBuffer[FlattenedFieldDescriptor]()
 
-            // Helper to add a field and its children recursively
+            // Helper to add a field and its children recursively.
+            // pathPrefix is the dot-path of ancestor fields (empty for top-level).
             def addFieldWithChildren(
                 sf: StructField,
                 info: ProtobufFieldInfo,
                 parentIdx: Int,
                 depth: Int,
-                nestedMsgDesc: AnyRef): Unit = {
+                nestedMsgDesc: AnyRef,
+                pathPrefix: String = ""): Unit = {
 
               val currentIdx = flatFields.size
 
@@ -356,32 +358,33 @@ object ProtobufExprShims {
               // add child fields
               sf.dataType match {
                 case st: StructType if nestedMsgDesc != null =>
-                  // Non-repeated nested message
-                  addChildFieldsFromStruct(st, nestedMsgDesc, sf.name, currentIdx, depth)
+                  addChildFieldsFromStruct(
+                    st, nestedMsgDesc, sf.name, currentIdx, depth, pathPrefix)
                   
                 case ArrayType(st: StructType, _) if nestedMsgDesc != null =>
-                  // Repeated message field (pruned via ordinal remapping)
-                  addChildFieldsFromStruct(st, nestedMsgDesc, sf.name, currentIdx, depth)
+                  addChildFieldsFromStruct(
+                    st, nestedMsgDesc, sf.name, currentIdx, depth, pathPrefix)
                   
                 case _ => // Not a struct, no children to add
               }
             }
             
             // Helper to add child fields from a struct type.
-            // Applies nested schema pruning for ALL struct types (both repeated and non-repeated).
-            // Pruned output is handled by ordinal remapping in GpuGetStructFieldMeta /
-            // GpuGetArrayStructFieldsMeta, not by null column expansion.
+            // Applies nested schema pruning at arbitrary depth using path-based
+            // lookup into nestedFieldRequirements.
             def addChildFieldsFromStruct(
                 st: StructType,
                 parentMsgDesc: AnyRef,
                 fieldName: String,
                 parentIdx: Int,
-                parentDepth: Int): Unit = {
+                parentDepth: Int,
+                pathPrefix: String): Unit = {
+              val path = if (pathPrefix.isEmpty) fieldName else s"$pathPrefix.$fieldName"
               val fd = PbReflect.findFieldByName(parentMsgDesc, fieldName)
               if (fd != null) {
                 try {
                   val childMsgDesc = PbReflect.getMessageType(fd)
-                  val requiredChildren = nestedFieldRequirements.get(fieldName)
+                  val requiredChildren = nestedFieldRequirements.get(path)
                   val filteredFields = requiredChildren match {
                     case Some(Some(childNames)) =>
                       st.fields.filter(f => childNames.contains(f.name))
@@ -415,6 +418,9 @@ object ProtobufExprShims {
                           (None, None)
                         }
 
+                      val childDefaultVal =
+                        if (childHasDefault) PbReflect.getDefaultValue(childFd) else None
+
                       val childInfo = ProtobufFieldInfo(
                         fieldNumber = childFieldNumber,
                         protoTypeName = childProtoTypeName,
@@ -424,14 +430,14 @@ object ProtobufExprShims {
                         unsupportedReason = None,
                         isRequired = childIsRequired,
                         hasDefaultValue = childHasDefault,
-                        defaultValue = None,
+                        defaultValue = childDefaultVal,
                         enumValues = childEnumVals,
                         enumNames = childEnumNameMap,
                         isRepeated = childIsRepeated
                       )
 
                       addFieldWithChildren(
-                        childSf, childInfo, parentIdx, parentDepth + 1, childMsgDesc)
+                        childSf, childInfo, parentIdx, parentDepth + 1, childMsgDesc, path)
                     }
                   }
                 } catch {
@@ -716,17 +722,16 @@ object ProtobufExprShims {
         }
 
         /**
-         * Nested field requirements: for each top-level field, what children are needed?
-         * - None means the whole field is needed (all children)
-         * - Some(Set("a","b")) means only children a and b are needed
+         * Nested field requirements: maps a field path to child requirements.
+         * Keys are dot-separated paths from the protobuf root:
+         *   - "level1" -> Some(Set("level2"))          (top-level struct pruning)
+         *   - "level1.level2" -> Some(Set("level3"))   (deep nested pruning)
+         *   - "field" -> None                           (whole field needed)
+         *
+         * Top-level names (keys without dots) also determine which fields are decoded.
          */
-        // Populated during analyzeDownstreamProject, used by addChildFieldsFromStruct
         private var nestedFieldRequirements: Map[String, Option[Set[String]]] = Map.empty
 
-
-        /**
-         * Get the field name from a GetStructField expression using its ordinal and schema.
-         */
         private def getFieldName(ordinal: Int, nameOpt: Option[String],
             schema: StructType): String = {
           nameOpt.getOrElse {
@@ -736,28 +741,84 @@ object ProtobufExprShims {
         }
 
         /**
-         * Recursively collect field names and nested child requirements from
-         * GetStructField expressions. Detects patterns like:
-         *   - decoded.field_name         -> field_name: None (whole field)
-         *   - decoded.ad_info.winfoid    -> ad_info: Some({winfoid})
-         *   - decoded.ad_info            -> ad_info: None (whole field)
-         *
-         * When both whole-field and sub-field access exist, whole-field wins (None).
+         * Navigate the Spark schema tree by following a dot-separated path of
+         * field names. Returns the StructType at the end of the path, unwrapping
+         * ArrayType(StructType) along the way, or null if the path is invalid.
          */
+        private def resolveSchemaAtPath(root: StructType, path: Seq[String]): StructType = {
+          var current: StructType = root
+          for (name <- path) {
+            val field = current.fields.find(_.name == name).orNull
+            if (field == null) return null
+            field.dataType match {
+              case st: StructType => current = st
+              case ArrayType(st: StructType, _) => current = st
+              case _ => return null
+            }
+          }
+          current
+        }
+
         /**
-         * Helper: record a nested child field requirement for a parent field.
-         * Merges with existing requirements (whole-field wins over sub-field).
+         * Walk a GetStructField chain upward until it reaches the protobuf
+         * reference expression, returning the sequence of field names forming
+         * the access path. Returns None if the chain does not terminate at a
+         * protobuf reference.
+         *
+         * Example: for `GetStructField(GetStructField(decoded, a_ord), b_ord)`
+         *   → Some(Seq("a", "b"))
          */
+        private def resolveFieldAccessChain(
+            expr: Expression): Option[Seq[String]] = {
+          expr match {
+            case GetStructField(child, ordinal, nameOpt) =>
+              if (isProtobufStructReference(child)) {
+                Some(Seq(getFieldName(ordinal, nameOpt, fullSchema)))
+              } else {
+                resolveFieldAccessChain(child).flatMap { parentPath =>
+                  val parentSchema = if (parentPath.isEmpty) fullSchema
+                                     else resolveSchemaAtPath(fullSchema, parentPath)
+                  if (parentSchema != null) {
+                    Some(parentPath :+ getFieldName(ordinal, nameOpt, parentSchema))
+                  } else {
+                    None
+                  }
+                }
+              }
+            case _ if isProtobufStructReference(expr) =>
+              Some(Seq.empty)
+            case _ =>
+              None
+          }
+        }
+
         private def addNestedFieldReq(
             fieldReqs: mutable.Map[String, Option[Set[String]]],
-            parentName: String,
+            parentKey: String,
             childName: String): Unit = {
-          fieldReqs.get(parentName) match {
+          fieldReqs.get(parentKey) match {
             case Some(None) => // Already need whole field, keep it
             case Some(Some(existing)) =>
-              fieldReqs(parentName) = Some(existing + childName)
+              fieldReqs(parentKey) = Some(existing + childName)
             case None =>
-              fieldReqs(parentName) = Some(Set(childName))
+              fieldReqs(parentKey) = Some(Set(childName))
+          }
+        }
+
+        /**
+         * Register pruning requirements at every level of a field access path.
+         * For path = ["a", "b"] with leafName = "c":
+         *   "a" -> needs child "b"
+         *   "a.b" -> needs child "c"
+         */
+        private def registerPathRequirements(
+            fieldReqs: mutable.Map[String, Option[Set[String]]],
+            path: Seq[String],
+            leafName: String): Unit = {
+          for (i <- path.indices) {
+            val pathKey = path.take(i + 1).mkString(".")
+            val childName = if (i < path.length - 1) path(i + 1) else leafName
+            addNestedFieldReq(fieldReqs, pathKey, childName)
           }
         }
 
@@ -766,47 +827,31 @@ object ProtobufExprShims {
             fieldReqs: mutable.Map[String, Option[Set[String]]],
             hasDirectStructRefHolder: () => Unit): Unit = {
           expr match {
-            // Pattern: decoded.parent_struct.child_field (non-array struct)
             case GetStructField(child, ordinal, nameOpt) =>
-              child match {
-                case GetStructField(innerChild, innerOrdinal, innerNameOpt)
-                    if isProtobufStructReference(innerChild) =>
-                  val parentName = getFieldName(innerOrdinal, innerNameOpt, fullSchema)
-                  val parentType = fullSchema.fields(innerOrdinal).dataType
-                  val childSchema = parentType match {
-                    case st: StructType => st
-                    case ArrayType(st: StructType, _) => st
-                    case _ => null
-                  }
-                  if (childSchema != null) {
-                    val childName = getFieldName(ordinal, nameOpt, childSchema)
-                    addNestedFieldReq(fieldReqs, parentName, childName)
+              resolveFieldAccessChain(child) match {
+                case Some(parentPath) =>
+                  val parentSchema = if (parentPath.isEmpty) fullSchema
+                                     else resolveSchemaAtPath(fullSchema, parentPath)
+                  if (parentSchema != null) {
+                    val fieldName = getFieldName(ordinal, nameOpt, parentSchema)
+                    if (parentPath.isEmpty) {
+                      // Direct top-level access: decoded.field_name (whole field)
+                      fieldReqs(fieldName) = None
+                    } else {
+                      registerPathRequirements(fieldReqs, parentPath, fieldName)
+                    }
                   } else {
-                    fieldReqs(parentName) = None
+                    collectStructFieldReferences(child, fieldReqs, hasDirectStructRefHolder)
                   }
-
-                case _ if isProtobufStructReference(child) =>
-                  // Direct top-level access: decoded.field_name (whole field)
-                  val fieldName = getFieldName(ordinal, nameOpt, fullSchema)
-                  fieldReqs(fieldName) = None
-
-                case _ =>
+                case None =>
                   collectStructFieldReferences(child, fieldReqs, hasDirectStructRefHolder)
               }
 
-            // Pattern: decoded.ad_info.winfoid where ad_info is ArrayType(StructType)
-            // Spark generates: GetArrayStructFields(GetStructField(decoded, ad_info_ord), field)
             case gasf: GetArrayStructFields =>
-              gasf.child match {
-                case GetStructField(innerChild, innerOrdinal, innerNameOpt)
-                    if isProtobufStructReference(innerChild) =>
-                  // Nested array-struct access: decoded.array_field.child_field
-                  val parentName = getFieldName(innerOrdinal, innerNameOpt, fullSchema)
-                  val childName = gasf.field.name
-                  addNestedFieldReq(fieldReqs, parentName, childName)
-
+              resolveFieldAccessChain(gasf.child) match {
+                case Some(parentPath) if parentPath.nonEmpty =>
+                  registerPathRequirements(fieldReqs, parentPath, gasf.field.name)
                 case _ =>
-                  // Not a direct protobuf reference, recurse into children
                   gasf.children.foreach { child =>
                     collectStructFieldReferences(child, fieldReqs, hasDirectStructRefHolder)
                   }
@@ -868,47 +913,37 @@ object ProtobufExprShims {
 
         override def convertToGpu(child: Expression): GpuExpression = {
           val prunedFieldsMap: Map[String, Seq[String]] = nestedFieldRequirements.collect {
-            case (fieldName, Some(childNames)) =>
-              val fieldIdx = fullSchema.fieldIndex(fieldName)
-              val childSchema = fullSchema.fields(fieldIdx).dataType match {
-                case st: StructType => st
-                case ArrayType(st: StructType, _) => st
-                case _ => null
-              }
+            case (pathKey, Some(childNames)) =>
+              val pathParts = pathKey.split("\\.").toSeq
+              val childSchema = resolveSchemaAtPath(fullSchema, pathParts)
               if (childSchema != null) {
                 val orderedNames = childSchema.fields
                   .map(_.name)
                   .filter(childNames.contains)
                   .toSeq
-                fieldName -> orderedNames
+                pathKey -> orderedNames
               } else {
-                fieldName -> childNames.toSeq
+                pathKey -> childNames.toSeq
               }
           }
 
           def registerExprs(expr: Expression): Unit = {
             expr match {
               case gsf @ GetStructField(childExpr, ordinal, nameOpt) =>
-                childExpr match {
-                  case GetStructField(innerChild, innerOrdinal, innerNameOpt)
-                      if isProtobufStructReference(innerChild) =>
-                    val parentName = getFieldName(innerOrdinal, innerNameOpt, fullSchema)
-                    val parentType = fullSchema.fields(innerOrdinal).dataType
-                    val childSchema = parentType match {
-                      case st: StructType => st
-                      case ArrayType(st: StructType, _) => st
-                      case _ => null
-                    }
-                    if (childSchema != null) {
-                      val childName = getFieldName(ordinal, nameOpt, childSchema)
-                      prunedFieldsMap.get(parentName).foreach { orderedChildren =>
+                resolveFieldAccessChain(childExpr) match {
+                  case Some(parentPath) if parentPath.nonEmpty =>
+                    val parentSchema = resolveSchemaAtPath(fullSchema, parentPath)
+                    if (parentSchema != null) {
+                      val pathKey = parentPath.mkString(".")
+                      val childName = getFieldName(ordinal, nameOpt, parentSchema)
+                      prunedFieldsMap.get(pathKey).foreach { orderedChildren =>
                         val runtimeOrd = orderedChildren.indexOf(childName)
                         if (runtimeOrd >= 0) {
                           gsf.setTagValue(ProtobufExprShims.PRUNED_ORDINAL_TAG, runtimeOrd)
                         }
                       }
                     }
-                  case _ if isProtobufStructReference(childExpr) =>
+                  case Some(parentPath) if parentPath.isEmpty =>
                     val runtimeOrd = decodedTopLevelIndices.indexOf(ordinal)
                     if (runtimeOrd >= 0) {
                       gsf.setTagValue(ProtobufExprShims.PRUNED_ORDINAL_TAG, runtimeOrd)
@@ -916,15 +951,12 @@ object ProtobufExprShims {
                   case _ =>
                 }
 
-              case gasf @ GetArrayStructFields(childExpr, field, _, _,
-                _) =>
-                childExpr match {
-                  case GetStructField(innerChild, innerOrdinal, innerNameOpt)
-                      if isProtobufStructReference(innerChild) =>
-                    val parentName = getFieldName(innerOrdinal, innerNameOpt, fullSchema)
-                    val childName = field.name
-                    prunedFieldsMap.get(parentName).foreach { orderedChildren =>
-                      val runtimeOrd = orderedChildren.indexOf(childName)
+              case gasf @ GetArrayStructFields(childExpr, field, _, _, _) =>
+                resolveFieldAccessChain(childExpr) match {
+                  case Some(parentPath) if parentPath.nonEmpty =>
+                    val pathKey = parentPath.mkString(".")
+                    prunedFieldsMap.get(pathKey).foreach { orderedChildren =>
+                      val runtimeOrd = orderedChildren.indexOf(field.name)
                       if (runtimeOrd >= 0) {
                         gasf.setTagValue(ProtobufExprShims.PRUNED_ORDINAL_TAG, runtimeOrd)
                       }
@@ -939,26 +971,41 @@ object ProtobufExprShims {
           targetExprsToRemap.foreach(registerExprs)
 
           val decodedSchema = {
-            val decodedFields = decodedTopLevelIndices.map { idx =>
-              val field = fullSchema.fields(idx)
-              prunedFieldsMap.get(field.name) match {
+            def applyPruning(field: StructField, prefix: String): StructField = {
+              val path = if (prefix.isEmpty) field.name else s"$prefix.${field.name}"
+              prunedFieldsMap.get(path) match {
                 case Some(childNames) =>
                   field.dataType match {
                     case ArrayType(st: StructType, cn) =>
                       val pruned = StructType(
-                        st.fields.filter(f =>
-                          childNames.contains(f.name)))
+                        st.fields.filter(f => childNames.contains(f.name))
+                          .map(f => applyPruning(f, path)))
                       field.copy(dataType = ArrayType(pruned, cn))
                     case st: StructType =>
                       val pruned = StructType(
-                        st.fields.filter(f =>
-                          childNames.contains(f.name)))
+                        st.fields.filter(f => childNames.contains(f.name))
+                          .map(f => applyPruning(f, path)))
                       field.copy(dataType = pruned)
                     case _ => field
                   }
-                case None => field
+                case None =>
+                  field.dataType match {
+                    case ArrayType(st: StructType, cn) =>
+                      val recursed = StructType(st.fields.map(f => applyPruning(f, path)))
+                      if (recursed != st) field.copy(dataType = ArrayType(recursed, cn))
+                      else field
+                    case st: StructType =>
+                      val recursed = StructType(st.fields.map(f => applyPruning(f, path)))
+                      if (recursed != st) field.copy(dataType = recursed)
+                      else field
+                    case _ => field
+                  }
               }
             }
+
+            val decodedFields = decodedTopLevelIndices.map { idx =>
+              applyPruning(fullSchema.fields(idx), "")
+            }
             StructType(decodedFields.map(f =>
               f.copy(nullable = true)))
           }

From 802488d8bbc14734fae12d47685d7bc5b65f9241 Mon Sep 17 00:00:00 2001
From: Haoyang Li <haoyangl@nvidia.com>
Date: Tue, 3 Mar 2026 15:36:32 +0800
Subject: [PATCH 07/33] address cc comments

Signed-off-by: Haoyang Li <haoyangl@nvidia.com>
---
 .../src/main/python/protobuf_test.py          | 86 ++++++++++++++++++-
 .../spark/sql/rapids/GpuFromProtobuf.scala    | 18 ++--
 .../rapids/shims/ProtobufExprShims.scala      | 19 ++--
 3 files changed, 106 insertions(+), 17 deletions(-)

diff --git a/integration_tests/src/main/python/protobuf_test.py b/integration_tests/src/main/python/protobuf_test.py
index cfdbf031fa9..b7562ed173f 100644
--- a/integration_tests/src/main/python/protobuf_test.py
+++ b/integration_tests/src/main/python/protobuf_test.py
@@ -18,7 +18,7 @@
 
 import pytest
 
-from asserts import assert_gpu_and_cpu_are_equal_collect
+from asserts import assert_gpu_and_cpu_are_equal_collect, assert_gpu_and_cpu_error
 from data_gen import (
     BooleanGen, IntegerGen, LongGen, FloatGen, DoubleGen, StringGen, BinaryGen,
     ProtobufMessageGen, PbScalar, PbNested, PbRepeated, PbRepeatedMessage,
@@ -3042,3 +3042,87 @@ def run_on_spark(spark):
         )
 
     assert_gpu_and_cpu_are_equal_collect(run_on_spark)
+
+
+# ===========================================================================
+# FAILFAST mode tests
+# ===========================================================================
+
+@pytest.mark.skipif(is_before_spark_340(), reason="from_protobuf is Spark 3.4.0+")
+def test_from_protobuf_failfast_malformed_data(spark_tmp_path, from_protobuf_fn):
+    """FAILFAST mode should throw on malformed protobuf data (both CPU and GPU)."""
+    desc_path, desc_bytes = _setup_protobuf_desc(
+        spark_tmp_path, "failfast.desc", _build_simple_descriptor_set_bytes)
+    message_name = "test.Simple"
+
+    # Craft a valid row and a malformed row (truncated varint with continuation bit)
+    valid_row = _encode_tag(1, 0) + _encode_varint(1) + \
+                _encode_tag(2, 0) + _encode_varint(42)
+    malformed_row = bytes([0x08, 0x80])  # field 1, varint, but only continuation byte -- no end
+
+    def run_on_spark(spark):
+        df = spark.createDataFrame(
+            [(valid_row,), (malformed_row,)],
+            schema="bin binary",
+        )
+        decoded = _call_from_protobuf(
+            from_protobuf_fn, f.col("bin"), message_name, desc_path, desc_bytes,
+            options={"mode": "FAILFAST"})
+        # Must call .collect() so the exception surfaces inside with_*_session
+        return df.select(decoded.getField("b").alias("b")).collect()
+
+    assert_gpu_and_cpu_error(run_on_spark, {}, "Malformed")
+
+
+@pytest.mark.skipif(is_before_spark_340(), reason="from_protobuf is Spark 3.4.0+")
+@ignore_order(local=True)
+def test_from_protobuf_permissive_malformed_returns_null(spark_tmp_path, from_protobuf_fn):
+    """PERMISSIVE mode should return null for malformed rows, not throw.
+
+    Note: Spark's from_protobuf defaults to FAILFAST (unlike JSON/CSV which
+    default to PERMISSIVE), so mode must be set explicitly.
+    """
+    desc_path, desc_bytes = _setup_protobuf_desc(
+        spark_tmp_path, "permissive.desc", _build_simple_descriptor_set_bytes)
+    message_name = "test.Simple"
+
+    valid_row = _encode_tag(2, 0) + _encode_varint(99)
+    malformed_row = bytes([0x08, 0x80])  # truncated varint
+
+    def run_on_spark(spark):
+        df = spark.createDataFrame(
+            [(valid_row,), (malformed_row,)],
+            schema="bin binary",
+        )
+        decoded = _call_from_protobuf(
+            from_protobuf_fn, f.col("bin"), message_name, desc_path, desc_bytes,
+            options={"mode": "PERMISSIVE"})
+        return df.select(
+            decoded.getField("i32").alias("i32"),
+        )
+
+    assert_gpu_and_cpu_are_equal_collect(run_on_spark)
+
+
+@pytest.mark.skipif(is_before_spark_340(), reason="from_protobuf is Spark 3.4.0+")
+@ignore_order(local=True)
+def test_from_protobuf_all_null_input(spark_tmp_path, from_protobuf_fn):
+    """All rows in the input binary column are null (not empty bytes, actual nulls).
+    GPU should produce all-null struct rows matching CPU behavior."""
+    desc_path, desc_bytes = _setup_protobuf_desc(
+        spark_tmp_path, "allnull.desc", _build_simple_descriptor_set_bytes)
+    message_name = "test.Simple"
+
+    def run_on_spark(spark):
+        df = spark.createDataFrame(
+            [(None,), (None,), (None,)],
+            schema="bin binary",
+        )
+        decoded = _call_from_protobuf(
+            from_protobuf_fn, f.col("bin"), message_name, desc_path, desc_bytes)
+        return df.select(
+            decoded.getField("i32").alias("i32"),
+            decoded.getField("s").alias("s"),
+        )
+
+    assert_gpu_and_cpu_are_equal_collect(run_on_spark)
diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuFromProtobuf.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuFromProtobuf.scala
index abeac6ba01e..d7cf9896c2e 100644
--- a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuFromProtobuf.scala
+++ b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuFromProtobuf.scala
@@ -36,13 +36,15 @@ import org.apache.spark.sql.types._
  * indices pointing to their containing message field. For pure scalar schemas, all fields
  * are top-level (parentIndices == -1, depthLevels == 0, isRepeated == false).
  *
- * Schema projection is supported: only fields in `decodedTopLevelIndices` are decoded by
- * the GPU. Fields not in this array will be filled with null columns in post-processing
- * to ensure the output matches `fullSchema`.
+ * Schema projection is supported: `decodedSchema` contains only the top-level fields and
+ * nested children that are actually referenced by downstream operators. Fields present in
+ * the original expression's output type but absent from `decodedSchema` will be filled with
+ * null columns by `expandSchema` in the shim layer.
  *
- * @param fullSchema The complete output schema (must match the original expression's dataType)
- * @param decodedTopLevelIndices Indices in fullSchema for top-level fields decoded by GPU.
- *                               Must be sorted in ascending order.
+ * @param decodedSchema The pruned schema containing only the fields decoded by the GPU.
+ *                      Top-level fields not in this schema get null columns; nested structs
+ *                      may have fewer children than the full schema (ordinal remapping handles
+ *                      the difference).
  * @param fieldNumbers Protobuf field numbers for all fields in flattened schema
  * @param parentIndices Parent indices for all fields (-1 for top-level)
  * @param depthLevels Nesting depth for all fields (0 for top-level)
@@ -58,13 +60,9 @@ import org.apache.spark.sql.types._
  * @param defaultStrings Default string/bytes values
  * @param enumValidValues Valid enum values for each field
  * @param enumNames Enum value names for enum-as-string fields. Parallel to enumValidValues.
- * @param nestedPrunedFields For nested schema projection: maps top-level field name to the
- *                           ordered list of decoded child field names. Only present for fields
- *                           where children were pruned. If empty map, no nested pruning.
  * @param failOnErrors If true, throw exception on malformed data
  */
 case class GpuFromProtobuf(
-    fullSchema: StructType,
     decodedSchema: StructType,
     fieldNumbers: Array[Int],
     parentIndices: Array[Int],
diff --git a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala
index 1b211908306..89835666bed 100644
--- a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala
+++ b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala
@@ -160,7 +160,13 @@ object ProtobufExprShims {
               return
           }
 
-          val options = getOptionsMap(e)
+          val options = getOptionsMap(e) match {
+            case Some(m) => m
+            case None =>
+              willNotWorkOnGpu(
+                "Cannot read from_protobuf options via reflection; falling back to CPU")
+              return
+          }
           val supportedOptions = Set("enums.as.ints", "mode")
           val unsupportedOptions = options.keys.filterNot(supportedOptions.contains)
           if (unsupportedOptions.nonEmpty) {
@@ -171,7 +177,7 @@ object ProtobufExprShims {
           }
 
           val enumsAsInts = options.getOrElse("enums.as.ints", "false").toBoolean
-          failOnErrors = options.getOrElse("mode", "PERMISSIVE").equalsIgnoreCase("FAILFAST")
+          failOnErrors = options.getOrElse("mode", "FAILFAST").equalsIgnoreCase("FAILFAST")
           val messageName = getMessageName(e)
 
           // Try to get descriptor: either file path (Spark 3.4.x) or binary bytes (Spark 3.5+)
@@ -1011,7 +1017,7 @@ object ProtobufExprShims {
           }
 
           GpuFromProtobuf(
-            fullSchema, decodedSchema,
+            decodedSchema,
             flatFieldNumbers, flatParentIndices,
             flatDepthLevels, flatWireTypes, flatOutputTypeIds, flatEncodings,
             flatIsRepeated, flatIsRequired, flatHasDefaultValue, flatDefaultInts,
@@ -1073,9 +1079,10 @@ object ProtobufExprShims {
     }
   }
 
-  private def getOptionsMap(e: Expression): Map[String, String] = {
-    val opt = Try(PbReflect.invoke0[scala.collection.Map[String, String]](e, "options")).toOption
-    opt.map(_.toMap).getOrElse(Map.empty)
+  private def getOptionsMap(e: Expression): Option[Map[String, String]] = {
+    Try(PbReflect.invoke0[scala.collection.Map[String, String]](e, "options"))
+      .map(_.toMap)
+      .toOption
   }
 
   /**

From 030fdf86c4fdd36469eec5f393f4d6bf8081a0a2 Mon Sep 17 00:00:00 2001
From: Haoyang Li <haoyangl@nvidia.com>
Date: Tue, 3 Mar 2026 16:37:26 +0800
Subject: [PATCH 08/33] codex review and address

Signed-off-by: Haoyang Li <haoyangl@nvidia.com>
---
 .../rapids/shims/ProtobufExprShims.scala      | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

diff --git a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala
index 89835666bed..04ea0fff756 100644
--- a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala
+++ b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala
@@ -405,8 +405,15 @@ object ProtobufExprShims {
                       val childIsRepeated = PbReflect.isRepeated(childFd)
                       val childIsRequired = PbReflect.isRequired(childFd)
                       val childHasDefault = PbReflect.hasDefaultValue(childFd)
-                      val (_, _, childEncoding) = checkFieldSupport(
-                        childSf.dataType, childProtoTypeName, childIsRepeated, enumsAsInts)
+                      val (childIsSupported, childUnsupportedReason, childEncoding) =
+                        checkFieldSupport(
+                          childSf.dataType, childProtoTypeName, childIsRepeated, enumsAsInts)
+
+                      if (!childIsSupported) {
+                        willNotWorkOnGpu(
+                          s"Nested field '${childSf.name}' at '$path': " +
+                            childUnsupportedReason.getOrElse("unsupported type"))
+                      }
 
                       val (childEnumVals, childEnumNameMap): (Option[Set[Int]],
                         Option[Map[Int, String]]) =
@@ -890,6 +897,14 @@ object ProtobufExprShims {
           // by class + identical input child so that
           // analyzeRequiredFields detects all field accesses in one
           // pass, keeping schema projection correct.
+          //
+          // Known limitation: if two different from_protobuf calls
+          // (different messageName / descriptor) share the same binary
+          // input column within one ProjectExec, this check will
+          // incorrectly treat them as the same expression, potentially
+          // mixing up field requirements and ordinal remapping. This is
+          // acceptable because decoding the same bytes as two different
+          // message types is not a meaningful query.
           if (expr.getClass == e.getClass &&
               expr.children.nonEmpty &&
               e.children.nonEmpty &&

From 73ce21b98c7018a213c7a18bc084ddf42a355f61 Mon Sep 17 00:00:00 2001
From: Haoyang Li <haoyangl@nvidia.com>
Date: Tue, 3 Mar 2026 17:21:05 +0800
Subject: [PATCH 09/33] verify and fix shim build error

Signed-off-by: Haoyang Li <haoyangl@nvidia.com>
---
 docs/supported_ops.md                                    | 4 ++--
 .../apache/spark/sql/rapids/complexTypeExtractors.scala  | 9 +++++++--
 .../nvidia/spark/rapids/shims/ProtobufExprShims.scala    | 2 +-
 3 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/docs/supported_ops.md b/docs/supported_ops.md
index b6b84bdfc3d..e6691244efa 100644
--- a/docs/supported_ops.md
+++ b/docs/supported_ops.md
@@ -7988,7 +7988,7 @@ are limited.
 <td> </td>
 <td> </td>
 <td> </td>
-<td><em>PS<br/>UTC is only supported TZ for child TIMESTAMP;<br/>unsupported child types BINARY, CALENDAR, UDT, DAYTIME, YEARMONTH</em></td>
+<td><em>PS<br/>UTC is only supported TZ for child TIMESTAMP;<br/>unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH</em></td>
 <td> </td>
 <td> </td>
 <td> </td>
@@ -8011,7 +8011,7 @@ are limited.
 <td> </td>
 <td> </td>
 <td> </td>
-<td><em>PS<br/>UTC is only supported TZ for child TIMESTAMP;<br/>unsupported child types BINARY, CALENDAR, UDT, DAYTIME, YEARMONTH</em></td>
+<td><em>PS<br/>UTC is only supported TZ for child TIMESTAMP;<br/>unsupported child types CALENDAR, UDT, DAYTIME, YEARMONTH</em></td>
 <td> </td>
 <td> </td>
 <td> </td>
diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/complexTypeExtractors.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/complexTypeExtractors.scala
index 5ecfbbca1fb..d14623322e0 100644
--- a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/complexTypeExtractors.scala
+++ b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/complexTypeExtractors.scala
@@ -412,6 +412,11 @@ case class GpuArrayPosition(left: Expression, right: Expression)
   }
 }
 
+object GpuStructFieldOrdinalTag {
+  val PRUNED_ORDINAL_TAG =
+    new org.apache.spark.sql.catalyst.trees.TreeNodeTag[Int]("GPU_PRUNED_ORDINAL")
+}
+
 class GpuGetStructFieldMeta(
     expr: GetStructField,
     conf: RapidsConf,
@@ -421,7 +426,7 @@ class GpuGetStructFieldMeta(
 
   def convertToGpu(child: Expression): GpuExpression = {
     val runtimeOrd = expr.getTagValue(
-      ProtobufExprShims.PRUNED_ORDINAL_TAG).getOrElse(-1)
+      GpuStructFieldOrdinalTag.PRUNED_ORDINAL_TAG).getOrElse(-1)
     val effectiveOrd =
       if (runtimeOrd >= 0) runtimeOrd else expr.ordinal
     GpuGetStructField(child, effectiveOrd, expr.name)
@@ -437,7 +442,7 @@ class GpuGetArrayStructFieldsMeta(
 
   def convertToGpu(child: Expression): GpuExpression = {
     val runtimeOrd = expr.getTagValue(
-      ProtobufExprShims.PRUNED_ORDINAL_TAG).getOrElse(-1)
+      GpuStructFieldOrdinalTag.PRUNED_ORDINAL_TAG).getOrElse(-1)
     val effectiveOrd =
       if (runtimeOrd >= 0) runtimeOrd else expr.ordinal
     GpuGetArrayStructFields(child, expr.field,
diff --git a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala
index 04ea0fff756..7ba172be20c 100644
--- a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala
+++ b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala
@@ -105,7 +105,7 @@ object ProtobufExprShims {
     "org.apache.spark.sql.protobuf.utils.ProtobufUtils$"
 
   val PRUNED_ORDINAL_TAG =
-    new org.apache.spark.sql.catalyst.trees.TreeNodeTag[Int]("GPU_PRUNED_ORDINAL")
+    org.apache.spark.sql.rapids.GpuStructFieldOrdinalTag.PRUNED_ORDINAL_TAG
 
   def exprs: Map[Class[_ <: Expression], ExprRule[_ <: Expression]] = {
     try {

From 685c8853e6c24c670b11493bcc32c60d976ee034 Mon Sep 17 00:00:00 2001
From: Haoyang Li <haoyangl@nvidia.com>
Date: Tue, 3 Mar 2026 17:22:07 +0800
Subject: [PATCH 10/33] Update copyright year in pom.xml

---
 integration_tests/pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/integration_tests/pom.xml b/integration_tests/pom.xml
index d8ee2301e5b..e3d91be0ce3 100644
--- a/integration_tests/pom.xml
+++ b/integration_tests/pom.xml
@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <!--
-  Copyright (c) 2020-2026, NVIDIA CORPORATION.
+  Copyright (c) 2020-2025, NVIDIA CORPORATION.
 
   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.

From 781e639ce80d443cef5afdfae1cc50fe1995de4c Mon Sep 17 00:00:00 2001
From: Haoyang Li <haoyangl@nvidia.com>
Date: Tue, 3 Mar 2026 17:54:48 +0800
Subject: [PATCH 11/33] Update copyright year in pom.xml

---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 6e69fa554fe..6eeff9d35be 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <!--
-  Copyright (c) 2020-2026, NVIDIA CORPORATION.
+  Copyright (c) 2020-2025, NVIDIA CORPORATION.
 
   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.

From 43a4c088930614a55372424e0eacb982db8d9d2e Mon Sep 17 00:00:00 2001
From: Haoyang Li <haoyangl@nvidia.com>
Date: Tue, 3 Mar 2026 18:22:11 +0800
Subject: [PATCH 12/33] clean up

Signed-off-by: Haoyang Li <haoyangl@nvidia.com>
---
 .../main/scala/com/nvidia/spark/rapids/GpuCanonicalize.scala   | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuCanonicalize.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuCanonicalize.scala
index 7adee94cad0..2dd39fa2375 100644
--- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuCanonicalize.scala
+++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuCanonicalize.scala
@@ -51,8 +51,7 @@ object GpuCanonicalize {
     case a: AttributeReference =>
       AttributeReference("none", TrampolineUtil.asNullable(a.dataType))(exprId = a.exprId)
     case GetStructField(child, ordinal, Some(_)) => GetStructField(child, ordinal, None)
-    case GpuGetStructField(child, ordinal, Some(_)) =>
-      GpuGetStructField(child, ordinal, None)
+    case GpuGetStructField(child, ordinal, Some(_)) => GpuGetStructField(child, ordinal, None)
     case _ => e
   }
 

From c1ef9fbf1ce046d1f69be152f02e57479daeaa57 Mon Sep 17 00:00:00 2001
From: Haoyang Li <haoyangl@nvidia.com>
Date: Thu, 5 Mar 2026 10:15:29 +0800
Subject: [PATCH 13/33] address comments

Signed-off-by: Haoyang Li <haoyangl@nvidia.com>
---
 integration_tests/run_pyspark_from_build.sh   | 20 +++++++--
 .../src/main/python/protobuf_test.py          | 41 ++++++++-----------
 .../src/main/python/spark_init_internal.py    | 22 ++++++----
 3 files changed, 48 insertions(+), 35 deletions(-)

diff --git a/integration_tests/run_pyspark_from_build.sh b/integration_tests/run_pyspark_from_build.sh
index 3ea01250d4b..16ed5bd4148 100755
--- a/integration_tests/run_pyspark_from_build.sh
+++ b/integration_tests/run_pyspark_from_build.sh
@@ -167,9 +167,23 @@ else
             fi
         fi
         
-        # Also download protobuf-java jar (required dependency)
-        # Spark 3.5.x uses protobuf-java 3.25.1
-        PROTOBUF_JAVA_VERSION="3.25.1"
+        # Also download protobuf-java jar (required dependency).
+        # Detect version from the jar bundled with Spark, fall back to version mapping.
+        PROTOBUF_JAVA_VERSION=""
+        BUNDLED_PB_JAR=$(ls "$SPARK_HOME"/jars/protobuf-java-[0-9]*.jar 2>/dev/null | head -1)
+        if [[ -n "$BUNDLED_PB_JAR" ]]; then
+            PROTOBUF_JAVA_VERSION=$(basename "$BUNDLED_PB_JAR" | sed 's/protobuf-java-\(.*\)\.jar/\1/')
+            echo "Detected protobuf-java version $PROTOBUF_JAVA_VERSION from SPARK_HOME"
+        fi
+        if [[ -z "$PROTOBUF_JAVA_VERSION" ]]; then
+            case "$VERSION_STRING" in
+                3.4.*) PROTOBUF_JAVA_VERSION="3.25.1" ;;
+                3.5.*) PROTOBUF_JAVA_VERSION="3.25.1" ;;
+                4.0.*) PROTOBUF_JAVA_VERSION="4.29.3" ;;
+                *)     PROTOBUF_JAVA_VERSION="3.25.1" ;;
+            esac
+            echo "Using protobuf-java version $PROTOBUF_JAVA_VERSION based on Spark $VERSION_STRING"
+        fi
         PROTOBUF_JAVA_JAR_NAME="protobuf-java-${PROTOBUF_JAVA_VERSION}.jar"
         PROTOBUF_JAVA_JAR_PATH="${TARGET_DIR}/dependency/${PROTOBUF_JAVA_JAR_NAME}"
         
diff --git a/integration_tests/src/main/python/protobuf_test.py b/integration_tests/src/main/python/protobuf_test.py
index b7562ed173f..2de47ea2d0f 100644
--- a/integration_tests/src/main/python/protobuf_test.py
+++ b/integration_tests/src/main/python/protobuf_test.py
@@ -2710,16 +2710,16 @@ def _encode_tag(field_number, wire_type):
 @pytest.mark.skipif(is_before_spark_340(), reason="from_protobuf is Spark 3.4.0+")
 @ignore_order(local=True)
 def test_from_protobuf_bool_noncanonical_varint_scalar(spark_tmp_path, from_protobuf_fn):
-    """Scalar bool field encoded as varint 256 should decode as true, not false."""
+    """Regression test: scalar bool encoded as non-canonical varint (e.g. 256) must decode as true.
+
+    Protobuf allows any non-zero varint for bool true. The GPU decoder previously
+    truncated to uint8_t, causing values >= 256 to wrap to 0 (false).
+    """
     desc_path, desc_bytes = _setup_protobuf_desc(
         spark_tmp_path, "simple_bool_bug.desc", _build_simple_descriptor_set_bytes)
     message_name = "test.Simple"
 
-    # Hand-craft protobuf wire bytes:
-    #   field 1 (bool b): tag = (1<<3)|0 = 0x08, value = varint(256) = 0x80 0x02
-    #   field 2 (int32 i32): tag = (2<<3)|0 = 0x10, value = varint(99) = 0x63
-    # Varint 256 is a perfectly valid encoding; CPU reads it as true.
-    # GPU truncates uint8_t(256) = 0 → false.
+    # varint(256) = 0x80 0x02, varint(512) = 0x80 0x04 — valid non-canonical bool true
     row_bool_256 = _encode_tag(1, 0) + _encode_varint(256) + \
                    _encode_tag(2, 0) + _encode_varint(99)
 
@@ -2727,7 +2727,7 @@ def test_from_protobuf_bool_noncanonical_varint_scalar(spark_tmp_path, from_prot
     row_bool_1 = _encode_tag(1, 0) + _encode_varint(1) + \
                  _encode_tag(2, 0) + _encode_varint(100)
 
-    # Another problematic value: 512 → uint8_t(0) → false
+    # Another non-canonical value: varint(512)
     row_bool_512 = _encode_tag(1, 0) + _encode_varint(512) + \
                    _encode_tag(2, 0) + _encode_varint(101)
 
@@ -2773,15 +2773,16 @@ def _build_repeated_bool_descriptor_set_bytes(spark):
 @pytest.mark.skipif(is_before_spark_340(), reason="from_protobuf is Spark 3.4.0+")
 @ignore_order(local=True)
 def test_from_protobuf_bool_noncanonical_varint_repeated(spark_tmp_path, from_protobuf_fn):
-    """Repeated bool with non-canonical varint values (256, 512) should all decode as true."""
+    """Regression test: repeated bool with non-canonical varint values must all decode as true.
+
+    Same uint8_t truncation issue as the scalar case, exercised with repeated fields.
+    """
     desc_path, desc_bytes = _setup_protobuf_desc(
         spark_tmp_path, "repeated_bool_bug.desc", _build_repeated_bool_descriptor_set_bytes)
     message_name = "test.WithRepeatedBool"
 
     # Repeated bool field 2 (wire type 0 = varint), unpacked.
-    # Three elements: varint(256), varint(1), varint(512)
-    # CPU: [true, true, true]
-    # GPU (buggy): [false, true, false]  — because uint8(256)=0, uint8(512)=0
+    # Three elements: varint(256), varint(1), varint(512) — all should decode as true.
     row = (_encode_tag(1, 0) + _encode_varint(42) +
            _encode_tag(2, 0) + _encode_varint(256) +
            _encode_tag(2, 0) + _encode_varint(1) +
@@ -2800,15 +2801,7 @@ def run_on_spark(spark):
 
 
 # ---------------------------------------------------------------------------
-# Bug 2: Nested message child field default values are lost
-#
-# In ProtobufExprShims.scala, addChildFieldsFromStruct always sets
-# defaultValue = None for nested children, even when hasDefaultValue = true.
-# This means proto2 defaults for fields inside nested messages are never
-# passed to the GPU decoder.
-#
-# CPU: missing child field → proto2 default value
-# GPU: missing child field → null
+# Regression guard: nested message child field default values
 # ---------------------------------------------------------------------------
 
 def _build_nested_with_defaults_descriptor_set_bytes(spark):
@@ -2863,12 +2856,10 @@ def _build_nested_with_defaults_descriptor_set_bytes(spark):
 @pytest.mark.skipif(is_before_spark_340(), reason="from_protobuf is Spark 3.4.0+")
 @ignore_order(local=True)
 def test_from_protobuf_nested_child_default_values(spark_tmp_path, from_protobuf_fn):
-    """Proto2 default values for fields inside nested messages must be honored.
+    """Regression test: proto2 default values for fields inside nested messages must be honored.
 
-    When `inner` is present but its child fields are absent, the CPU decoder
-    returns the proto2 defaults (count=42, label="hello", flag=true).
-    The GPU decoder currently returns null for all three because
-    defaultValue is never populated for nested children.
+    When `inner` is present but its child fields are absent, the decoder must
+    return the proto2 defaults (count=42, label="hello", flag=true), not null.
     """
     desc_path, desc_bytes = _setup_protobuf_desc(
         spark_tmp_path, "nested_defaults.desc",
diff --git a/integration_tests/src/main/python/spark_init_internal.py b/integration_tests/src/main/python/spark_init_internal.py
index a0053623f95..0463ac754c5 100644
--- a/integration_tests/src/main/python/spark_init_internal.py
+++ b/integration_tests/src/main/python/spark_init_internal.py
@@ -82,14 +82,22 @@ def _add_driver_classpath(jars):
     # Remove trailing 'pyspark-shell' if present
     if current_args.endswith('pyspark-shell'):
         current_args = current_args[:-len('pyspark-shell')].strip()
-    # Skip if driver-class-path is already present
-    if '--driver-class-path' in current_args:
-        logging.info("driver-class-path already in PYSPARK_SUBMIT_ARGS, skipping")
-        return
-    # Add driver-class-path for each jar (use os.pathsep for platform independence)
     jar_list = jars.replace(',', ' ').split()
-    driver_cp = os.pathsep.join(jar_list)
-    new_args = f"{current_args} --driver-class-path {driver_cp} pyspark-shell".strip()
+    new_cp = os.pathsep.join(jar_list)
+    if '--driver-class-path' in current_args:
+        match = re.search(r'--driver-class-path\s+(\S+)', current_args)
+        if match:
+            existing_cp = match.group(1)
+            merged_cp = existing_cp + os.pathsep + new_cp
+            current_args = re.sub(
+                r'--driver-class-path\s+\S+',
+                f'--driver-class-path {merged_cp}',
+                current_args)
+        else:
+            current_args += f' --driver-class-path {new_cp}'
+    else:
+        current_args += f' --driver-class-path {new_cp}'
+    new_args = f"{current_args} pyspark-shell".strip()
     os.environ['PYSPARK_SUBMIT_ARGS'] = new_args
     logging.info(f"Updated PYSPARK_SUBMIT_ARGS with driver-class-path")
 

From 8a2c007bbfc29e4ffadf3c4e5d86b50dd4cc1dbd Mon Sep 17 00:00:00 2001
From: Haoyang Li <haoyangl@nvidia.com>
Date: Thu, 5 Mar 2026 12:40:56 +0800
Subject: [PATCH 14/33] address comments

Signed-off-by: Haoyang Li <haoyangl@nvidia.com>
---
 .../spark/sql/rapids/GpuFromProtobuf.scala    |  6 +-
 .../rapids/shims/ProtobufExprShims.scala      | 76 +++++++++----------
 2 files changed, 43 insertions(+), 39 deletions(-)

diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuFromProtobuf.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuFromProtobuf.scala
index d7cf9896c2e..5b39aafb56d 100644
--- a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuFromProtobuf.scala
+++ b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuFromProtobuf.scala
@@ -23,6 +23,7 @@ import com.nvidia.spark.rapids.Arm.withResource
 import com.nvidia.spark.rapids.jni.{Protobuf, ProtobufSchemaDescriptor}
 import com.nvidia.spark.rapids.shims.NullIntolerantShim
 
+import org.apache.spark.internal.Logging
 import org.apache.spark.sql.catalyst.expressions.{ExpectsInputTypes, Expression}
 import org.apache.spark.sql.types._
 
@@ -81,7 +82,7 @@ case class GpuFromProtobuf(
     enumNames: Array[Array[Array[Byte]]],
     failOnErrors: Boolean,
     child: Expression)
-  extends GpuUnaryExpression with ExpectsInputTypes with NullIntolerantShim {
+  extends GpuUnaryExpression with ExpectsInputTypes with NullIntolerantShim with Logging {
 
   override def inputTypes: Seq[AbstractDataType] = Seq(BinaryType)
 
@@ -100,6 +101,9 @@ case class GpuFromProtobuf(
     } catch {
       case e: CudfException if failOnErrors =>
         throw new org.apache.spark.SparkException("Malformed protobuf message", e)
+      case e: CudfException =>
+        logWarning(s"Unexpected CudfException in PERMISSIVE mode: ${e.getMessage}", e)
+        throw e
     }
 
     // Apply input nulls to output
diff --git a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala
index 7ba172be20c..90dc86f15d9 100644
--- a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala
+++ b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala
@@ -208,7 +208,7 @@ object ProtobufExprShims {
           // Reject proto3 descriptors — GPU decoder only supports proto2 semantics.
           // proto3 has different null/default-value behavior that the GPU path doesn't handle.
           val protoSyntax = PbReflect.getFileSyntax(msgDesc)
-          if (protoSyntax == "PROTO3") {
+          if (protoSyntax == "PROTO3" || protoSyntax.isEmpty) {
             willNotWorkOnGpu(
               "proto3 syntax is not supported by the GPU protobuf decoder; " +
                 "only proto2 is supported. The query will fall back to CPU.")
@@ -413,44 +413,44 @@ object ProtobufExprShims {
                         willNotWorkOnGpu(
                           s"Nested field '${childSf.name}' at '$path': " +
                             childUnsupportedReason.getOrElse("unsupported type"))
+                      } else {
+                        val (childEnumVals, childEnumNameMap): (Option[Set[Int]],
+                          Option[Map[Int, String]]) =
+                          if (childProtoTypeName == "ENUM") {
+                            Try {
+                              val pairs = PbReflect.getEnumValues(
+                                PbReflect.getEnumType(childFd))
+                              if (enumsAsInts) {
+                                (Some(pairs.map(_._1).toSet), None)
+                              } else {
+                                (Some(pairs.map(_._1).toSet), Some(pairs.toMap))
+                              }
+                            }.getOrElse((None, None))
+                          } else {
+                            (None, None)
+                          }
+
+                        val childDefaultVal =
+                          if (childHasDefault) PbReflect.getDefaultValue(childFd) else None
+
+                        val childInfo = ProtobufFieldInfo(
+                          fieldNumber = childFieldNumber,
+                          protoTypeName = childProtoTypeName,
+                          sparkType = childSf.dataType,
+                          encoding = childEncoding,
+                          isSupported = true,
+                          unsupportedReason = None,
+                          isRequired = childIsRequired,
+                          hasDefaultValue = childHasDefault,
+                          defaultValue = childDefaultVal,
+                          enumValues = childEnumVals,
+                          enumNames = childEnumNameMap,
+                          isRepeated = childIsRepeated
+                        )
+
+                        addFieldWithChildren(
+                          childSf, childInfo, parentIdx, parentDepth + 1, childMsgDesc, path)
                       }
-
-                      val (childEnumVals, childEnumNameMap): (Option[Set[Int]],
-                        Option[Map[Int, String]]) =
-                        if (childProtoTypeName == "ENUM") {
-                          Try {
-                            val pairs = PbReflect.getEnumValues(
-                              PbReflect.getEnumType(childFd))
-                            if (enumsAsInts) {
-                              (Some(pairs.map(_._1).toSet), None)
-                            } else {
-                              (Some(pairs.map(_._1).toSet), Some(pairs.toMap))
-                            }
-                          }.getOrElse((None, None))
-                        } else {
-                          (None, None)
-                        }
-
-                      val childDefaultVal =
-                        if (childHasDefault) PbReflect.getDefaultValue(childFd) else None
-
-                      val childInfo = ProtobufFieldInfo(
-                        fieldNumber = childFieldNumber,
-                        protoTypeName = childProtoTypeName,
-                        sparkType = childSf.dataType,
-                        encoding = childEncoding,
-                        isSupported = true,
-                        unsupportedReason = None,
-                        isRequired = childIsRequired,
-                        hasDefaultValue = childHasDefault,
-                        defaultValue = childDefaultVal,
-                        enumValues = childEnumVals,
-                        enumNames = childEnumNameMap,
-                        isRepeated = childIsRepeated
-                      )
-
-                      addFieldWithChildren(
-                        childSf, childInfo, parentIdx, parentDepth + 1, childMsgDesc, path)
                     }
                   }
                 } catch {

From 035fd1d3da8030ddf812f7079417bbae881c6ccf Mon Sep 17 00:00:00 2001
From: Haoyang Li <haoyangl@nvidia.com>
Date: Thu, 5 Mar 2026 13:32:08 +0800
Subject: [PATCH 15/33] address comments

Signed-off-by: Haoyang Li <haoyangl@nvidia.com>
---
 .../com/nvidia/spark/rapids/shims/ProtobufExprShims.scala   | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala
index 90dc86f15d9..ea6aa28e3c8 100644
--- a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala
+++ b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala
@@ -330,6 +330,12 @@ object ProtobufExprShims {
                     val str = defVal.asInstanceOf[String]
                     val bytes = if (str != null) str.getBytes("UTF-8") else null
                     (0L, 0.0, false, bytes)
+                  case BinaryType =>
+                    val bytes = Try {
+                      val toByteArray = defVal.getClass.getMethod("toByteArray")
+                      toByteArray.invoke(defVal).asInstanceOf[Array[Byte]]
+                    }.getOrElse(null: Array[Byte])
+                    (0L, 0.0, false, bytes)
                   case _ => (0L, 0.0, false, null: Array[Byte])
                 }
               } else {

From c812754d964bfbf0d5f0205e17e83c374d9b13b4 Mon Sep 17 00:00:00 2001
From: Haoyang Li <haoyangl@nvidia.com>
Date: Thu, 5 Mar 2026 14:01:56 +0800
Subject: [PATCH 16/33] fix shim

Signed-off-by: Haoyang Li <haoyangl@nvidia.com>
---
 .../scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala
index ea6aa28e3c8..d6bb6ca8424 100644
--- a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala
+++ b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala
@@ -30,6 +30,8 @@
 {"spark": "357"}
 {"spark": "400"}
 {"spark": "401"}
+{"spark": "402"}
+{"spark": "411"}
 spark-rapids-shim-json-lines ***/
 
 package com.nvidia.spark.rapids.shims

From 4551f285d2c0a9efb4d831707efe511b9e05970f Mon Sep 17 00:00:00 2001
From: Haoyang Li <haoyangl@nvidia.com>
Date: Thu, 5 Mar 2026 14:49:44 +0800
Subject: [PATCH 17/33] address comments

Signed-off-by: Haoyang Li <haoyangl@nvidia.com>
---
 integration_tests/run_pyspark_from_build.sh                     | 2 +-
 integration_tests/src/main/python/spark_init_internal.py        | 2 +-
 .../scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/integration_tests/run_pyspark_from_build.sh b/integration_tests/run_pyspark_from_build.sh
index 16ed5bd4148..c4e8edf3568 100755
--- a/integration_tests/run_pyspark_from_build.sh
+++ b/integration_tests/run_pyspark_from_build.sh
@@ -209,7 +209,7 @@ else
         # Also add protobuf jars to driver classpath for Class.forName() to work
         # This is needed because --jars only adds to executor classpath
         if [[ -n "$PROTOBUF_JARS" ]]; then
-            PROTOBUF_DRIVER_CP=$(echo $PROTOBUF_JARS | tr ' ' ':')
+            PROTOBUF_DRIVER_CP=$(echo "$PROTOBUF_JARS" | tr ' ' ':')
             export PYSP_TEST_spark_driver_extraClassPath="${PYSP_TEST_spark_driver_extraClassPath:+${PYSP_TEST_spark_driver_extraClassPath}:}${PROTOBUF_DRIVER_CP}"
             echo "Added protobuf jars to driver classpath"
         fi
diff --git a/integration_tests/src/main/python/spark_init_internal.py b/integration_tests/src/main/python/spark_init_internal.py
index 0463ac754c5..e40abd15570 100644
--- a/integration_tests/src/main/python/spark_init_internal.py
+++ b/integration_tests/src/main/python/spark_init_internal.py
@@ -91,7 +91,7 @@ def _add_driver_classpath(jars):
             merged_cp = existing_cp + os.pathsep + new_cp
             current_args = re.sub(
                 r'--driver-class-path\s+\S+',
-                f'--driver-class-path {merged_cp}',
+                lambda m: f'--driver-class-path {merged_cp}',
                 current_args)
         else:
             current_args += f' --driver-class-path {new_cp}'
diff --git a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala
index d6bb6ca8424..7edfa05ad06 100644
--- a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala
+++ b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala
@@ -359,7 +359,7 @@ object ProtobufExprShims {
                 encoding = info.encoding,
                 isRepeated = info.isRepeated,
                 isRequired = info.isRequired,
-                hasDefaultValue = info.hasDefaultValue,
+                hasDefaultValue = hasDefault,
                 defaultInt = defInt,
                 defaultFloat = defFloat,
                 defaultBool = defBool,

From ca3b7c571f39d35c0017f0c9d9b5675c869e3464 Mon Sep 17 00:00:00 2001
From: Haoyang Li <haoyangl@nvidia.com>
Date: Thu, 5 Mar 2026 16:17:01 +0800
Subject: [PATCH 18/33] address comments

Signed-off-by: Haoyang Li <haoyangl@nvidia.com>
---
 .../src/main/python/protobuf_test.py          | 68 +++++++++++++++++++
 .../nested_proto/device_req.proto             |  2 +
 .../protobuf_test/nested_proto/main_log.proto |  2 +
 .../nested_proto/module_a_res.proto           |  2 +
 .../nested_proto/module_b_res.proto           |  2 +
 .../nested_proto/predictor_schema.proto       |  2 +
 .../spark/sql/rapids/GpuFromProtobuf.scala    | 12 ++--
 .../rapids/shims/ProtobufExprShims.scala      |  6 +-
 8 files changed, 88 insertions(+), 8 deletions(-)

diff --git a/integration_tests/src/main/python/protobuf_test.py b/integration_tests/src/main/python/protobuf_test.py
index 2de47ea2d0f..28940305a57 100644
--- a/integration_tests/src/main/python/protobuf_test.py
+++ b/integration_tests/src/main/python/protobuf_test.py
@@ -870,6 +870,74 @@ def run_on_spark(spark):
     assert_gpu_and_cpu_are_equal_collect(run_on_spark)
 
 
+def _build_repeated_enum_descriptor_set_bytes(spark):
+    """
+    message WithRepeatedEnum {
+        optional int32 id = 1;
+        repeated Priority priority = 2;
+        enum Priority { UNKNOWN = 0; FOO = 1; BAR = 2; }
+    }
+    """
+    D, fd = _new_proto2_file(spark, "repeated_enum.proto")
+    label_opt = D.FieldDescriptorProto.Label.LABEL_OPTIONAL
+    label_rep = D.FieldDescriptorProto.Label.LABEL_REPEATED
+
+    msg = D.DescriptorProto.newBuilder().setName("WithRepeatedEnum")
+
+    enum_type = D.EnumDescriptorProto.newBuilder().setName("Priority")
+    enum_type.addValue(D.EnumValueDescriptorProto.newBuilder().setName("UNKNOWN").setNumber(0).build())
+    enum_type.addValue(D.EnumValueDescriptorProto.newBuilder().setName("FOO").setNumber(1).build())
+    enum_type.addValue(D.EnumValueDescriptorProto.newBuilder().setName("BAR").setNumber(2).build())
+    msg.addEnumType(enum_type.build())
+
+    msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("id").setNumber(1).setLabel(label_opt)
+            .setType(D.FieldDescriptorProto.Type.TYPE_INT32).build())
+    msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("priority").setNumber(2).setLabel(label_rep)
+            .setType(D.FieldDescriptorProto.Type.TYPE_ENUM)
+            .setTypeName(".test.WithRepeatedEnum.Priority").build())
+
+    fd.addMessageType(msg.build())
+    fds = D.FileDescriptorSet.newBuilder().addFile(fd.build()).build()
+    return bytes(fds.toByteArray())
+
+
+@pytest.mark.skipif(is_before_spark_340(), reason="from_protobuf is Spark 3.4.0+")
+@ignore_order(local=True)
+def test_from_protobuf_repeated_enum_invalid_permissive(spark_tmp_path, from_protobuf_fn):
+    """Repeated enum with invalid values in PERMISSIVE mode.
+
+    Row with any invalid enum value should become null (entire struct row),
+    while rows with all-valid enum values decode normally.
+    """
+    desc_path, desc_bytes = _setup_protobuf_desc(
+        spark_tmp_path, "repeated_enum.desc", _build_repeated_enum_descriptor_set_bytes)
+    message_name = "test.WithRepeatedEnum"
+
+    row1 = (_encode_tag(1, 0) + _encode_varint(1) +
+            _encode_tag(2, 0) + _encode_varint(0) +
+            _encode_tag(2, 0) + _encode_varint(1) +
+            _encode_tag(2, 0) + _encode_varint(2))
+    row2 = (_encode_tag(1, 0) + _encode_varint(2) +
+            _encode_tag(2, 0) + _encode_varint(1) +
+            _encode_tag(2, 0) + _encode_varint(99))
+    row3 = (_encode_tag(1, 0) + _encode_varint(3) +
+            _encode_tag(2, 0) + _encode_varint(0))
+
+    def run_on_spark(spark):
+        df = spark.createDataFrame(
+            [(row1,), (row2,), (row3,)], schema="bin binary")
+        decoded = _call_from_protobuf(
+            from_protobuf_fn, f.col("bin"), message_name, desc_path, desc_bytes,
+            options={"mode": "PERMISSIVE"})
+        return df.select(decoded.alias("decoded"))
+
+    assert_gpu_and_cpu_are_equal_collect(run_on_spark)
+
+
 def _build_required_field_descriptor_set_bytes(spark):
     """
     Build a FileDescriptorSet for a message with required fields (proto2):
diff --git a/integration_tests/src/test/resources/protobuf_test/nested_proto/device_req.proto b/integration_tests/src/test/resources/protobuf_test/nested_proto/device_req.proto
index 68024163e99..c4d86951d96 100644
--- a/integration_tests/src/test/resources/protobuf_test/nested_proto/device_req.proto
+++ b/integration_tests/src/test/resources/protobuf_test/nested_proto/device_req.proto
@@ -1,3 +1,5 @@
+syntax = "proto2";
+
 package com.test.proto.sample;
 
 option java_outer_classname = "DeviceReqBean";
diff --git a/integration_tests/src/test/resources/protobuf_test/nested_proto/main_log.proto b/integration_tests/src/test/resources/protobuf_test/nested_proto/main_log.proto
index 8f683592109..f9a325a9f2e 100644
--- a/integration_tests/src/test/resources/protobuf_test/nested_proto/main_log.proto
+++ b/integration_tests/src/test/resources/protobuf_test/nested_proto/main_log.proto
@@ -1,3 +1,5 @@
+syntax = "proto2";
+
 package com.test.proto.sample;
 
 import "module_a_res.proto";
diff --git a/integration_tests/src/test/resources/protobuf_test/nested_proto/module_a_res.proto b/integration_tests/src/test/resources/protobuf_test/nested_proto/module_a_res.proto
index c9dbf2b9e35..9a2674e5dd5 100644
--- a/integration_tests/src/test/resources/protobuf_test/nested_proto/module_a_res.proto
+++ b/integration_tests/src/test/resources/protobuf_test/nested_proto/module_a_res.proto
@@ -1,3 +1,5 @@
+syntax = "proto2";
+
 package com.test.proto.sample;
 
 option java_outer_classname = "ModuleARes";
diff --git a/integration_tests/src/test/resources/protobuf_test/nested_proto/module_b_res.proto b/integration_tests/src/test/resources/protobuf_test/nested_proto/module_b_res.proto
index be38a10c083..0c742739835 100644
--- a/integration_tests/src/test/resources/protobuf_test/nested_proto/module_b_res.proto
+++ b/integration_tests/src/test/resources/protobuf_test/nested_proto/module_b_res.proto
@@ -1,3 +1,5 @@
+syntax = "proto2";
+
 package com.test.proto.sample;
 
 // Module B response field
diff --git a/integration_tests/src/test/resources/protobuf_test/nested_proto/predictor_schema.proto b/integration_tests/src/test/resources/protobuf_test/nested_proto/predictor_schema.proto
index 9c0db8a6bcd..bc7d86c67e2 100644
--- a/integration_tests/src/test/resources/protobuf_test/nested_proto/predictor_schema.proto
+++ b/integration_tests/src/test/resources/protobuf_test/nested_proto/predictor_schema.proto
@@ -1,3 +1,5 @@
+syntax = "proto2";
+
 package com.test.proto.sample;
 
 // Predictor schema - tests multi-level schema nesting
diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuFromProtobuf.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuFromProtobuf.scala
index 5b39aafb56d..1bd7c7364c7 100644
--- a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuFromProtobuf.scala
+++ b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuFromProtobuf.scala
@@ -38,14 +38,14 @@ import org.apache.spark.sql.types._
  * are top-level (parentIndices == -1, depthLevels == 0, isRepeated == false).
  *
  * Schema projection is supported: `decodedSchema` contains only the top-level fields and
- * nested children that are actually referenced by downstream operators. Fields present in
- * the original expression's output type but absent from `decodedSchema` will be filled with
- * null columns by `expandSchema` in the shim layer.
+ * nested children that are actually referenced by downstream operators. Downstream
+ * `GetStructField` and `GetArrayStructFields` nodes have their ordinals rewritten via
+ * `PRUNED_ORDINAL_TAG` to index into the pruned schema. Unreferenced fields are never
+ * accessed, so no null-column filling is needed.
  *
  * @param decodedSchema The pruned schema containing only the fields decoded by the GPU.
- *                      Top-level fields not in this schema get null columns; nested structs
- *                      may have fewer children than the full schema (ordinal remapping handles
- *                      the difference).
+ *                      Only fields referenced by downstream operators are included;
+ *                      ordinal remapping ensures correct field access into the pruned output.
  * @param fieldNumbers Protobuf field numbers for all fields in flattened schema
  * @param parentIndices Parent indices for all fields (-1 for top-level)
  * @param depthLevels Nesting depth for all fields (0 for top-level)
diff --git a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala
index 7edfa05ad06..38e35b0b6ab 100644
--- a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala
+++ b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala
@@ -676,8 +676,10 @@ object ProtobufExprShims {
             case "FIXED64" | "SFIXED64" | "DOUBLE" => WT_64BIT
             case "STRING" | "BYTES" | "MESSAGE" => WT_LEN
             case other =>
-              throw new IllegalStateException(
-                s"Unknown protobuf type name '$other' - cannot determine wire type")
+              willNotWorkOnGpu(
+                s"Unknown protobuf type name '$other' - cannot determine wire type; " +
+                  "falling back to CPU")
+              WT_VARINT
           }
         }
 

From 6860c917293a3f475936e8270ad5b701dfee029f Mon Sep 17 00:00:00 2001
From: Haoyang Li <haoyangl@nvidia.com>
Date: Thu, 5 Mar 2026 17:07:20 +0800
Subject: [PATCH 19/33] address comments

Signed-off-by: Haoyang Li <haoyangl@nvidia.com>
---
 integration_tests/run_pyspark_from_build.sh                 | 6 +++---
 .../com/nvidia/spark/rapids/shims/ProtobufExprShims.scala   | 4 ++--
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/integration_tests/run_pyspark_from_build.sh b/integration_tests/run_pyspark_from_build.sh
index c4e8edf3568..5a7db886da1 100755
--- a/integration_tests/run_pyspark_from_build.sh
+++ b/integration_tests/run_pyspark_from_build.sh
@@ -159,7 +159,7 @@ else
         if [[ ! -f "$PROTOBUF_JAR_PATH" ]]; then
             echo "Downloading spark-protobuf jar..."
             PROTOBUF_MAVEN_URL="https://repo1.maven.org/maven2/org/apache/spark/spark-protobuf_${SCALA_VERSION}/${VERSION_STRING}/${PROTOBUF_JAR_NAME}"
-            if curl -sL -o "$PROTOBUF_JAR_PATH" "$PROTOBUF_MAVEN_URL"; then
+            if curl -fsL -o "$PROTOBUF_JAR_PATH" "$PROTOBUF_MAVEN_URL"; then
                 echo "Downloaded spark-protobuf jar to $PROTOBUF_JAR_PATH"
             else
                 echo "WARNING: Failed to download spark-protobuf jar from $PROTOBUF_MAVEN_URL"
@@ -190,7 +190,7 @@ else
         if [[ ! -f "$PROTOBUF_JAVA_JAR_PATH" ]]; then
             echo "Downloading protobuf-java jar..."
             PROTOBUF_JAVA_MAVEN_URL="https://repo1.maven.org/maven2/com/google/protobuf/protobuf-java/${PROTOBUF_JAVA_VERSION}/${PROTOBUF_JAVA_JAR_NAME}"
-            if curl -sL -o "$PROTOBUF_JAVA_JAR_PATH" "$PROTOBUF_JAVA_MAVEN_URL"; then
+            if curl -fsL -o "$PROTOBUF_JAVA_JAR_PATH" "$PROTOBUF_JAVA_MAVEN_URL"; then
                 echo "Downloaded protobuf-java jar to $PROTOBUF_JAVA_JAR_PATH"
             else
                 echo "WARNING: Failed to download protobuf-java jar from $PROTOBUF_JAVA_MAVEN_URL"
@@ -203,7 +203,7 @@ else
             echo "Including spark-protobuf jar: $PROTOBUF_JAR_PATH"
         fi
         if [[ -f "$PROTOBUF_JAVA_JAR_PATH" ]]; then
-            PROTOBUF_JARS="$PROTOBUF_JARS $PROTOBUF_JAVA_JAR_PATH"
+            PROTOBUF_JARS="${PROTOBUF_JARS:+$PROTOBUF_JARS }$PROTOBUF_JAVA_JAR_PATH"
             echo "Including protobuf-java jar: $PROTOBUF_JAVA_JAR_PATH"
         fi
         # Also add protobuf jars to driver classpath for Class.forName() to work
diff --git a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala
index 38e35b0b6ab..0c9c7d9b79e 100644
--- a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala
+++ b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala
@@ -210,9 +210,9 @@ object ProtobufExprShims {
           // Reject proto3 descriptors — GPU decoder only supports proto2 semantics.
           // proto3 has different null/default-value behavior that the GPU path doesn't handle.
           val protoSyntax = PbReflect.getFileSyntax(msgDesc)
-          if (protoSyntax == "PROTO3" || protoSyntax.isEmpty) {
+          if (protoSyntax == "PROTO3" || protoSyntax == "EDITIONS" || protoSyntax.isEmpty) {
             willNotWorkOnGpu(
-              "proto3 syntax is not supported by the GPU protobuf decoder; " +
+              "proto3/editions syntax is not supported by the GPU protobuf decoder; " +
                 "only proto2 is supported. The query will fall back to CPU.")
             return
           }

From c1c919bba02894105ecbd008fba2a2e7b348e517 Mon Sep 17 00:00:00 2001
From: Haoyang Li <haoyangl@nvidia.com>
Date: Thu, 5 Mar 2026 17:52:23 +0800
Subject: [PATCH 20/33] address comments

Signed-off-by: Haoyang Li <haoyangl@nvidia.com>
---
 .../spark/rapids/shims/ProtobufExprShims.scala     | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala
index 0c9c7d9b79e..f19abc05540 100644
--- a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala
+++ b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala
@@ -178,7 +178,13 @@ object ProtobufExprShims {
             return
           }
 
-          val enumsAsInts = options.getOrElse("enums.as.ints", "false").toBoolean
+          val enumsAsInts = Try(options.getOrElse("enums.as.ints", "false").toBoolean) match {
+            case scala.util.Success(v) => v
+            case scala.util.Failure(_) =>
+              willNotWorkOnGpu("Invalid value for from_protobuf option 'enums.as.ints': " +
+                s"'${options.getOrElse("enums.as.ints", "")}' (expected true/false)")
+              return
+          }
           failOnErrors = options.getOrElse("mode", "FAILFAST").equalsIgnoreCase("FAILFAST")
           val messageName = getMessageName(e)
 
@@ -462,8 +468,10 @@ object ProtobufExprShims {
                     }
                   }
                 } catch {
-                  case _: ReflectiveOperationException =>
-                  // Ignore reflection failures and let remaining fields continue.
+                  case ex: ReflectiveOperationException =>
+                    willNotWorkOnGpu(
+                      s"Failed to reflect message type for nested field '$fieldName': " +
+                        ex.getMessage)
                 }
               }
             }

From d71f7a19ca59efd80c7a2d444524472617ed45e4 Mon Sep 17 00:00:00 2001
From: Haoyang Li <haoyangl@nvidia.com>
Date: Thu, 5 Mar 2026 21:38:22 +0800
Subject: [PATCH 21/33] address comments

Signed-off-by: Haoyang Li <haoyangl@nvidia.com>
---
 .../com/nvidia/spark/rapids/shims/ProtobufExprShims.scala  | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala
index f19abc05540..1c95cfae8e6 100644
--- a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala
+++ b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala
@@ -478,8 +478,8 @@ object ProtobufExprShims {
 
             // Only add top-level fields that are actually required (schema projection).
             // This significantly reduces GPU memory and computation for schemas with many
-            // fields when only a few are needed. The Scala layer will post-process the
-            // output to insert null columns for non-decoded fields.
+            // fields when only a few are needed. Downstream GetStructField ordinals are
+            // remapped via PRUNED_ORDINAL_TAG to index into the pruned output.
             decodedTopLevelIndices = indicesToDecode
             indicesToDecode.foreach { schemaIdx =>
               val sf = fullSchema.fields(schemaIdx)
@@ -693,7 +693,8 @@ object ProtobufExprShims {
 
         /**
          * Analyze which fields are actually required by downstream operations.
-         * Currently supports analyzing parent Project expressions.
+         * Traverses parent FilterExec and ProjectExec nodes to collect
+         * struct field references, then returns the set of required top-level field names.
          *
          * @param allFieldNames All field names in the full schema
          * @return Set of field names that are actually required

From 0c9385a1f3edb8e56fda20a3188452c20a1230d1 Mon Sep 17 00:00:00 2001
From: Haoyang Li <haoyangl@nvidia.com>
Date: Fri, 6 Mar 2026 11:31:05 +0800
Subject: [PATCH 22/33] address comments

Signed-off-by: Haoyang Li <haoyangl@nvidia.com>
---
 .../src/main/python/spark_init_internal.py    |  2 +-
 .../rapids/shims/ProtobufExprShims.scala      | 40 +++++++++++++++----
 2 files changed, 33 insertions(+), 9 deletions(-)

diff --git a/integration_tests/src/main/python/spark_init_internal.py b/integration_tests/src/main/python/spark_init_internal.py
index e40abd15570..95f988cbc2b 100644
--- a/integration_tests/src/main/python/spark_init_internal.py
+++ b/integration_tests/src/main/python/spark_init_internal.py
@@ -82,7 +82,7 @@ def _add_driver_classpath(jars):
     # Remove trailing 'pyspark-shell' if present
     if current_args.endswith('pyspark-shell'):
         current_args = current_args[:-len('pyspark-shell')].strip()
-    jar_list = jars.replace(',', ' ').split()
+    jar_list = [j.strip() for j in jars.split(',') if j.strip()]
     new_cp = os.pathsep.join(jar_list)
     if '--driver-class-path' in current_args:
         match = re.search(r'--driver-class-path\s+(\S+)', current_args)
diff --git a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala
index 1c95cfae8e6..b973641cf85 100644
--- a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala
+++ b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala
@@ -99,7 +99,7 @@ private[shims] case class FlattenedFieldDescriptor(
  *
  * spark-protobuf is an external module, so these rules must be registered by reflection.
  */
-object ProtobufExprShims {
+object ProtobufExprShims extends org.apache.spark.internal.Logging {
   private[this] val protobufDataToCatalystClassName =
     "org.apache.spark.sql.protobuf.ProtobufDataToCatalyst"
 
@@ -116,6 +116,9 @@ object ProtobufExprShims {
       Map(clazz.asInstanceOf[Class[_ <: Expression]] -> fromProtobufRule)
     } catch {
       case _: ClassNotFoundException => Map.empty
+      case e: Exception =>
+        logWarning(s"Failed to load $protobufDataToCatalystClassName: ${e.getMessage}")
+        Map.empty
     }
   }
 
@@ -693,8 +696,10 @@ object ProtobufExprShims {
 
         /**
          * Analyze which fields are actually required by downstream operations.
-         * Traverses parent FilterExec and ProjectExec nodes to collect
-         * struct field references, then returns the set of required top-level field names.
+         * Traverses parent plan nodes upward, collecting struct field references from
+         * ProjectExec, FilterExec, and transparent pass-through nodes (AggregateExec,
+         * SortExec, WindowExec, etc.), then returns the set of required top-level
+         * field names.
          *
          * @param allFieldNames All field names in the full schema
          * @return Set of field names that are actually required
@@ -711,6 +716,13 @@ object ProtobufExprShims {
           var safeToPrune = true
           val collectedExprs = mutable.ArrayBuffer[Expression]()
 
+          def advanceToParent(): Unit = {
+            currentMeta = currentMeta.get.parent match {
+              case Some(pm: SparkPlanMeta[_]) => Some(pm)
+              case _ => None
+            }
+          }
+
           while (currentMeta.isDefined && !foundProject && safeToPrune) {
             currentMeta.get.wrapped match {
               case p: ProjectExec =>
@@ -720,10 +732,22 @@ object ProtobufExprShims {
               case f: org.apache.spark.sql.execution.FilterExec =>
                 collectedExprs += f.condition
                 collectStructFieldReferences(f.condition, fieldReqs, holder)
-                currentMeta = currentMeta.get.parent match {
-                  case Some(pm: SparkPlanMeta[_]) => Some(pm)
-                  case _ => None
-                }
+                advanceToParent()
+              case a: org.apache.spark.sql.execution.aggregate.BaseAggregateExec =>
+                val exprs = a.aggregateExpressions ++ a.groupingExpressions
+                collectedExprs ++= exprs
+                exprs.foreach(collectStructFieldReferences(_, fieldReqs, holder))
+                advanceToParent()
+              case s: org.apache.spark.sql.execution.SortExec =>
+                val exprs = s.sortOrder
+                collectedExprs ++= exprs
+                exprs.foreach(collectStructFieldReferences(_, fieldReqs, holder))
+                advanceToParent()
+              case w: org.apache.spark.sql.execution.window.WindowExec =>
+                val exprs = w.windowExpression
+                collectedExprs ++= exprs
+                exprs.foreach(collectStructFieldReferences(_, fieldReqs, holder))
+                advanceToParent()
               case _ =>
                 safeToPrune = false
             }
@@ -938,7 +962,7 @@ object ProtobufExprShims {
             parent.flatMap { meta =>
               meta.wrapped match {
                 case alias: org.apache.spark.sql.catalyst.expressions
-                      .Alias if alias.child eq e =>
+                      .Alias if alias.child.semanticEquals(e) =>
                   Some(alias.exprId)
                 case _ => None
               }

From 303b4f8bd1143b999ba6dd79f9dc3fc31f60c65e Mon Sep 17 00:00:00 2001
From: Haoyang Li <haoyangl@nvidia.com>
Date: Fri, 6 Mar 2026 12:05:37 +0800
Subject: [PATCH 23/33] address comments

Signed-off-by: Haoyang Li <haoyangl@nvidia.com>
---
 .../com/nvidia/spark/rapids/shims/ProtobufExprShims.scala | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala
index b973641cf85..8f9da497d91 100644
--- a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala
+++ b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala
@@ -416,7 +416,11 @@ object ProtobufExprShims extends org.apache.spark.internal.Logging {
                   }
                   filteredFields.foreach { childSf =>
                     val childFd = PbReflect.findFieldByName(childMsgDesc, childSf.name)
-                    if (childFd != null) {
+                    if (childFd == null) {
+                      willNotWorkOnGpu(
+                        s"Nested field '${childSf.name}' not found in protobuf " +
+                          s"descriptor for message at '$path'")
+                    } else {
                       val childProtoTypeName = typeName(PbReflect.getFieldType(childFd))
                       val childFieldNumber = PbReflect.getFieldNumber(childFd)
                       val childIsRepeated = PbReflect.isRepeated(childFd)
@@ -907,6 +911,8 @@ object ProtobufExprShims extends org.apache.spark.internal.Logging {
               resolveFieldAccessChain(gasf.child) match {
                 case Some(parentPath) if parentPath.nonEmpty =>
                   registerPathRequirements(fieldReqs, parentPath, gasf.field.name)
+                case Some(parentPath) if parentPath.isEmpty =>
+                  fieldReqs(gasf.field.name) = None
                 case _ =>
                   gasf.children.foreach { child =>
                     collectStructFieldReferences(child, fieldReqs, hasDirectStructRefHolder)

From 981b75c89630d90ceee597532370bcc45785a2e2 Mon Sep 17 00:00:00 2001
From: Haoyang Li <haoyangl@nvidia.com>
Date: Fri, 6 Mar 2026 15:43:36 +0800
Subject: [PATCH 24/33] address comments

Signed-off-by: Haoyang Li <haoyangl@nvidia.com>
---
 integration_tests/src/main/python/data_gen.py |  4 +--
 .../src/main/python/protobuf_test.py          | 30 +++++++++++++++++--
 .../rapids/shims/ProtobufExprShims.scala      | 14 +++++----
 3 files changed, 38 insertions(+), 10 deletions(-)

diff --git a/integration_tests/src/main/python/data_gen.py b/integration_tests/src/main/python/data_gen.py
index b2d285b1af3..965c0e2d4c0 100644
--- a/integration_tests/src/main/python/data_gen.py
+++ b/integration_tests/src/main/python/data_gen.py
@@ -1000,7 +1000,7 @@ def _encode_protobuf_packed_repeated(field_number, spark_element_type, values, e
             packed_data += _encode_protobuf_uvarint(1 if v else 0)
         elif isinstance(spark_element_type, IntegerType):
             if encoding == 'fixed':
-                packed_data += struct.pack("<i", int(v))
+                packed_data += struct.pack("<I", int(v) & 0xFFFFFFFF)
             elif encoding == 'zigzag':
                 packed_data += _encode_protobuf_uvarint(_encode_protobuf_zigzag32(int(v)))
             else:
@@ -1008,7 +1008,7 @@ def _encode_protobuf_packed_repeated(field_number, spark_element_type, values, e
                 packed_data += _encode_protobuf_uvarint(u64)
         elif isinstance(spark_element_type, LongType):
             if encoding == 'fixed':
-                packed_data += struct.pack("<q", int(v))
+                packed_data += struct.pack("<Q", int(v) & 0xFFFFFFFFFFFFFFFF)
             elif encoding == 'zigzag':
                 packed_data += _encode_protobuf_uvarint(_encode_protobuf_zigzag64(int(v)))
             else:
diff --git a/integration_tests/src/main/python/protobuf_test.py b/integration_tests/src/main/python/protobuf_test.py
index 28940305a57..b0a9fd044a2 100644
--- a/integration_tests/src/main/python/protobuf_test.py
+++ b/integration_tests/src/main/python/protobuf_test.py
@@ -22,11 +22,12 @@
 from data_gen import (
     BooleanGen, IntegerGen, LongGen, FloatGen, DoubleGen, StringGen, BinaryGen,
     ProtobufMessageGen, PbScalar, PbNested, PbRepeated, PbRepeatedMessage,
-    encode_pb_message, gen_df, idfn
+    encode_pb_message, gen_df, idfn, _encode_protobuf_packed_repeated
 )
 from marks import ignore_order
 from spark_session import with_cpu_session, is_before_spark_340
 import pyspark.sql.functions as f
+from pyspark.sql.types import IntegerType, LongType
 
 pytestmark = [pytest.mark.premerge_ci_1]
 
@@ -137,11 +138,36 @@ def _call_from_protobuf(from_protobuf_fn, col, message_name,
         if options is not None:
             kw["options"] = options
         return from_protobuf_fn(col, message_name, **kw)
-    if options is not None and "options" in sig.parameters:
+    if options is not None:
         return from_protobuf_fn(col, message_name, desc_path, options)
     return from_protobuf_fn(col, message_name, desc_path)
 
 
+def test_call_from_protobuf_preserves_options_for_legacy_signature():
+    calls = []
+
+    def fake_from_protobuf(col, message_name, desc_path, *args):
+        calls.append((col, message_name, desc_path, args))
+        return "ok"
+
+    options = {"enums.as.ints": "true"}
+    result = _call_from_protobuf(
+        fake_from_protobuf, "col", "msg", "/tmp/test.desc", b"desc", options=options)
+
+    assert result == "ok"
+    assert calls == [("col", "msg", "/tmp/test.desc", (options,))]
+
+
+def test_encode_protobuf_packed_repeated_fixed_uses_unsigned_twos_complement():
+    i32_encoded = _encode_protobuf_packed_repeated(
+        1, IntegerType(), [0xFFFFFFFF], encoding='fixed')
+    i64_encoded = _encode_protobuf_packed_repeated(
+        1, LongType(), [0xFFFFFFFFFFFFFFFF], encoding='fixed')
+
+    assert i32_encoded == b"\x0a\x04" + struct.pack("<I", 0xFFFFFFFF)
+    assert i64_encoded == b"\x0a\x08" + struct.pack("<Q", 0xFFFFFFFFFFFFFFFF)
+
+
 def _build_simple_descriptor_set_bytes(spark):
     """
     Build a FileDescriptorSet for:
diff --git a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala
index 8f9da497d91..ff14f386fcc 100644
--- a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala
+++ b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala
@@ -404,7 +404,10 @@ object ProtobufExprShims extends org.apache.spark.internal.Logging {
                 pathPrefix: String): Unit = {
               val path = if (pathPrefix.isEmpty) fieldName else s"$pathPrefix.$fieldName"
               val fd = PbReflect.findFieldByName(parentMsgDesc, fieldName)
-              if (fd != null) {
+              if (fd == null) {
+                willNotWorkOnGpu(
+                  s"Nested field '$fieldName' not found in protobuf descriptor at '$path'")
+              } else {
                 try {
                   val childMsgDesc = PbReflect.getMessageType(fd)
                   val requiredChildren = nestedFieldRequirements.get(path)
@@ -716,7 +719,6 @@ object ProtobufExprShims extends org.apache.spark.internal.Logging {
           val holder = () => { hasDirectStructRef = true }
 
           var currentMeta: Option[SparkPlanMeta[_]] = findParentPlanMeta()
-          var foundProject = false
           var safeToPrune = true
           val collectedExprs = mutable.ArrayBuffer[Expression]()
 
@@ -727,12 +729,12 @@ object ProtobufExprShims extends org.apache.spark.internal.Logging {
             }
           }
 
-          while (currentMeta.isDefined && !foundProject && safeToPrune) {
+          while (currentMeta.isDefined && safeToPrune) {
             currentMeta.get.wrapped match {
               case p: ProjectExec =>
                 collectedExprs ++= p.projectList
                 p.projectList.foreach(collectStructFieldReferences(_, fieldReqs, holder))
-                foundProject = true
+                currentMeta = None
               case f: org.apache.spark.sql.execution.FilterExec =>
                 collectedExprs += f.condition
                 collectStructFieldReferences(f.condition, fieldReqs, holder)
@@ -757,7 +759,7 @@ object ProtobufExprShims extends org.apache.spark.internal.Logging {
             }
           }
 
-          if (!safeToPrune || !foundProject || hasDirectStructRef || fieldReqs.isEmpty) {
+          if (!safeToPrune || collectedExprs.isEmpty || hasDirectStructRef || fieldReqs.isEmpty) {
             targetExprsToRemap = Seq.empty
             allFieldNames
           } else {
@@ -937,7 +939,7 @@ object ProtobufExprShims extends org.apache.spark.internal.Logging {
          *    (when accessing from a downstream ProjectExec)
          */
         private def isProtobufStructReference(expr: Expression): Boolean = {
-          if (expr eq e) {
+          if ((expr eq e) || expr.semanticEquals(e)) {
             return true
           }
 

From 539af836c37a47bc26f42550f8fd41115cae59d9 Mon Sep 17 00:00:00 2001
From: Haoyang Li <haoyangl@nvidia.com>
Date: Wed, 11 Mar 2026 10:57:20 +0800
Subject: [PATCH 25/33] Batch merging after schema projection

Signed-off-by: Haoyang Li <haoyangl@nvidia.com>
---
 .../delta/common/DeltaProviderBase.scala      | 13 ++-
 .../src/main/python/protobuf_test.py          | 33 +++++++
 .../com/nvidia/spark/rapids/RapidsConf.scala  | 12 +++
 .../spark/rapids/basicPhysicalOperators.scala | 73 ++++++++++++--
 .../rapids/ProtobufBatchMergeSuite.scala      | 95 +++++++++++++++++++
 5 files changed, 215 insertions(+), 11 deletions(-)
 create mode 100644 sql-plugin/src/test/scala/com/nvidia/spark/rapids/ProtobufBatchMergeSuite.scala

diff --git a/delta-lake/common/src/main/delta-33x-40x/scala/com/nvidia/spark/rapids/delta/common/DeltaProviderBase.scala b/delta-lake/common/src/main/delta-33x-40x/scala/com/nvidia/spark/rapids/delta/common/DeltaProviderBase.scala
index 171d12c3dea..96e15da35b8 100644
--- a/delta-lake/common/src/main/delta-33x-40x/scala/com/nvidia/spark/rapids/delta/common/DeltaProviderBase.scala
+++ b/delta-lake/common/src/main/delta-33x-40x/scala/com/nvidia/spark/rapids/delta/common/DeltaProviderBase.scala
@@ -171,7 +171,7 @@ abstract class DeltaProviderBase extends DeltaIOProvider {
       //
       case dvRoot @ GpuProjectExec(outputList,
       dvFilter @ GpuFilterExec(condition,
-      dvFilterInput @ GpuProjectExec(inputList, fsse: GpuFileSourceScanExec, _)), _)
+      dvFilterInput @ GpuProjectExec(inputList, fsse: GpuFileSourceScanExec, _, _)), _, _)
         if condition.references.exists(_.name == IS_ROW_DELETED_COLUMN_NAME) &&
           !outputList.exists(_.name == "_metadata") && inputList.exists(_.name == "_metadata") =>
         dvRoot.withNewChildren(Seq(
@@ -242,7 +242,7 @@ object DVPredicatePushdown extends ShimPredicateHelper {
 
     def pruneIsRowDeletedColumn(plan: SparkPlan): SparkPlan = {
       plan.transformUp {
-        case project @ GpuProjectExec(projectList, _, _) =>
+        case project @ GpuProjectExec(projectList, _, _, _) =>
           val newProjList = projectList.filterNot(isRowDeletedColumnRef(_))
           project.copy(projectList = newProjList)
         case fsse: GpuFileSourceScanExec =>
@@ -293,11 +293,16 @@ object DVPredicatePushdown extends ShimPredicateHelper {
   def mergeIdenticalProjects(plan: SparkPlan): SparkPlan = {
     plan.transformUp {
       case p @ GpuProjectExec(projList1,
-      GpuProjectExec(projList2, child, enablePreSplit1), enablePreSplit2) =>
+      GpuProjectExec(projList2, child, enablePreSplit1, forcePostProjectCoalesce1),
+      enablePreSplit2, forcePostProjectCoalesce2) =>
         val projSet1 = projList1.map(_.exprId).toSet
         val projSet2 = projList2.map(_.exprId).toSet
         if (projSet1 == projSet2) {
-          GpuProjectExec(projList1, child, enablePreSplit1 && enablePreSplit2)
+          GpuProjectExec(
+            projList1,
+            child,
+            enablePreSplit1 && enablePreSplit2,
+            forcePostProjectCoalesce1 || forcePostProjectCoalesce2)
         } else {
           p
         }
diff --git a/integration_tests/src/main/python/protobuf_test.py b/integration_tests/src/main/python/protobuf_test.py
index b0a9fd044a2..5e3bc57da39 100644
--- a/integration_tests/src/main/python/protobuf_test.py
+++ b/integration_tests/src/main/python/protobuf_test.py
@@ -1542,6 +1542,39 @@ def run_on_spark(spark):
     assert_gpu_and_cpu_are_equal_collect(run_on_spark)
 
 
+@pytest.mark.skipif(is_before_spark_340(), reason="from_protobuf is Spark 3.4.0+")
+@ignore_order(local=True)
+def test_from_protobuf_nested_message_field_access_with_batch_merge(
+        spark_tmp_path, from_protobuf_fn):
+    """
+    Same as nested field access, but with protobuf post-project batch merge enabled.
+    This protects correctness when schema-projected protobuf output is coalesced.
+    """
+    desc_path, desc_bytes = _setup_protobuf_desc(
+        spark_tmp_path, "nested.desc", _build_nested_descriptor_set_bytes)
+    message_name = "test.WithNested"
+
+    data_gen = ProtobufMessageGen([
+        PbScalar("simple_int", 1, IntegerGen()),
+        PbScalar("simple_str", 2, StringGen(nullable=True)),
+        PbNested("nested_msg", 3, [PbScalar("x", 1, IntegerGen())]),
+        PbScalar("simple_long", 4, LongGen()),
+    ])
+
+    def run_on_spark(spark):
+        df = gen_df(spark, data_gen)
+        decoded = _call_from_protobuf(
+            from_protobuf_fn, f.col("bin"), message_name, desc_path, desc_bytes)
+        return df.select(
+            decoded.getField("simple_int").alias("simple_int"),
+            decoded.getField("nested_msg").getField("x").alias("nested_x"),
+        )
+
+    assert_gpu_and_cpu_are_equal_collect(
+        run_on_spark,
+        conf={"spark.rapids.sql.protobuf.batchMergeAfterProject.enabled": "true"})
+
+
 def _build_deep_nested_descriptor_set_bytes(spark):
     """
     Build a FileDescriptorSet for deeply nested message:
diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsConf.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsConf.scala
index 2257c4c0bce..256dd645594 100644
--- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsConf.scala
+++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsConf.scala
@@ -1591,6 +1591,15 @@ val GPU_COREDUMP_PIPE_PATTERN = conf("spark.rapids.gpu.coreDump.pipePattern")
     .booleanConf
     .createWithDefault(true)
 
+  val ENABLE_PROTOBUF_BATCH_MERGE_AFTER_PROJECT =
+    conf("spark.rapids.sql.protobuf.batchMergeAfterProject.enabled")
+      .doc("When set to true, allows a GPU Project containing a schema-pruned " +
+        "`from_protobuf` decode to request a post-project batch coalesce. This is intended " +
+        "to reduce tiny batches produced after protobuf schema projection.")
+      .internal()
+      .booleanConf
+      .createWithDefault(false)
+
   val ENABLE_ORC_FLOAT_TYPES_TO_STRING =
     conf("spark.rapids.sql.format.orc.floatTypesToString.enable")
     .doc("When reading an ORC file, the source data schemas(schemas of ORC file) may differ " +
@@ -3516,6 +3525,9 @@ class RapidsConf(conf: Map[String, String]) extends Logging {
 
   lazy val isCoalesceAfterExpandEnabled: Boolean = get(ENABLE_COALESCE_AFTER_EXPAND)
 
+  lazy val isProtobufBatchMergeAfterProjectEnabled: Boolean =
+    get(ENABLE_PROTOBUF_BATCH_MERGE_AFTER_PROJECT)
+
   lazy val multiThreadReadNumThreads: Int = {
     // Use the largest value set among all the options.
     val deprecatedConfs = Seq(
diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/basicPhysicalOperators.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/basicPhysicalOperators.scala
index a97f830fe3e..a808b31f1d0 100644
--- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/basicPhysicalOperators.scala
+++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/basicPhysicalOperators.scala
@@ -47,6 +47,48 @@ import org.apache.spark.sql.vectorized.{ColumnarBatch, ColumnVector}
 import org.apache.spark.unsafe.types.UTF8String
 import org.apache.spark.util.random.BernoulliCellSampler
 
+object GpuProjectExecMeta {
+  private def isProtobufDecodeExpr(expr: Expression): Boolean =
+    expr.getClass.getName.endsWith("ProtobufDataToCatalyst")
+
+  private def protobufAliasExprIds(plan: SparkPlan): Set[ExprId] = plan match {
+    case ProjectExec(projectList, _) =>
+      projectList.collect {
+        case alias: Alias if isProtobufDecodeExpr(alias.child) => alias.exprId
+      }.toSet
+    case _ => Set.empty
+  }
+
+  private def isRootedAtProtobufAlias(expr: Expression, protobufExprIds: Set[ExprId]): Boolean =
+    expr match {
+      case attr: AttributeReference =>
+        protobufExprIds.contains(attr.exprId)
+      case GetStructField(child, _, _) =>
+        isRootedAtProtobufAlias(child, protobufExprIds)
+      case gasf: GetArrayStructFields =>
+        isRootedAtProtobufAlias(gasf.child, protobufExprIds)
+      case _ =>
+        false
+    }
+
+  private def extractsFromProtobufAlias(
+      expr: Expression,
+      protobufExprIds: Set[ExprId]): Boolean = expr match {
+    case GetStructField(child, _, _) =>
+      isRootedAtProtobufAlias(child, protobufExprIds)
+    case gasf: GetArrayStructFields =>
+      isRootedAtProtobufAlias(gasf.child, protobufExprIds)
+    case other =>
+      other.children.exists(child => extractsFromProtobufAlias(child, protobufExprIds))
+  }
+
+  private[rapids] def shouldCoalesceAfterProject(plan: ProjectExec): Boolean = {
+    val protobufExprIds = protobufAliasExprIds(plan.child)
+    protobufExprIds.nonEmpty &&
+      plan.projectList.exists(expr => extractsFromProtobufAlias(expr, protobufExprIds))
+  }
+}
+
 class GpuProjectExecMeta(
     proj: ProjectExec,
     conf: RapidsConf,
@@ -57,11 +99,13 @@ class GpuProjectExecMeta(
     // Force list to avoid recursive Java serialization of lazy list Seq implementation
     val gpuExprs = childExprs.map(_.convertToGpu().asInstanceOf[NamedExpression]).toList
     val gpuChild = childPlans.head.convertIfNeeded()
+    val forcePostProjectCoalesce = conf.isProtobufBatchMergeAfterProjectEnabled &&
+      GpuProjectExecMeta.shouldCoalesceAfterProject(proj)
     if (conf.isProjectAstEnabled) {
       // cuDF requires return column is fixed width
       val allReturnTypesFixedWidth = gpuExprs.forall(e => GpuBatchUtils.isFixedWidth(e.dataType))
       if (allReturnTypesFixedWidth && childExprs.forall(_.canThisBeAst)) {
-        return GpuProjectAstExec(gpuExprs, gpuChild)
+        return GpuProjectAstExec(gpuExprs, gpuChild, forcePostProjectCoalesce)
       }
       // explain AST because this is optional and it is sometimes hard to debug
       if (conf.shouldExplain) {
@@ -76,7 +120,7 @@ class GpuProjectExecMeta(
         }
       }
     }
-    GpuProjectExec(gpuExprs, gpuChild)
+    GpuProjectExec(gpuExprs, gpuChild, forcePostProjectCoalesce = forcePostProjectCoalesce)
   }
 }
 
@@ -290,6 +334,7 @@ object GpuProjectExecLike {
 trait GpuProjectExecLike extends GpuPartitioningPreservingUnaryExecNode with GpuExec {
 
   def projectList: Seq[Expression]
+  def forcePostProjectCoalesce: Boolean
 
   override lazy val additionalMetrics: Map[String, GpuMetric] = Map(
     OP_TIME_LEGACY -> createNanoTimingMetric(DEBUG_LEVEL, DESCRIPTION_OP_TIME_LEGACY))
@@ -299,8 +344,16 @@ trait GpuProjectExecLike extends GpuPartitioningPreservingUnaryExecNode with Gpu
   override def doExecute(): RDD[InternalRow] =
     throw new IllegalStateException(s"Row-based execution should not occur for $this")
 
-  // The same as what feeds us
-  override def outputBatching: CoalesceGoal = GpuExec.outputBatching(child)
+  override def coalesceAfter: Boolean = forcePostProjectCoalesce
+
+  // Flagged protobuf projects intentionally drop the output batching guarantee so that
+  // the post-project coalesce inserted by transition rules is not optimized away.
+  override def outputBatching: CoalesceGoal =
+    if (forcePostProjectCoalesce) {
+      null
+    } else {
+      GpuExec.outputBatching(child)
+    }
 }
 
 /**
@@ -763,14 +816,19 @@ case class GpuProjectExec(
    //   immutable/List.scala#L516
    projectList: List[NamedExpression],
    child: SparkPlan,
-   enablePreSplit: Boolean = true) extends GpuProjectExecLike {
+   enablePreSplit: Boolean = true,
+   forcePostProjectCoalesce: Boolean = false) extends GpuProjectExecLike {
 
   override def output: Seq[Attribute] = projectList.map(_.toAttribute)
 
   override def outputBatching: CoalesceGoal = if (enablePreSplit) {
     // Pre-split will make sure the size of each output batch will not be larger
     // than the splitUntilSize.
-    TargetSize(PreProjectSplitIterator.getSplitUntilSize)
+    if (forcePostProjectCoalesce) {
+      super.outputBatching
+    } else {
+      TargetSize(PreProjectSplitIterator.getSplitUntilSize)
+    }
   } else {
     super.outputBatching
   }
@@ -822,7 +880,8 @@ case class GpuProjectAstExec(
     // serde: https://github.com/scala/scala/blob/2.12.x/src/library/scala/collection/
     //   immutable/List.scala#L516
     projectList: List[Expression],
-    child: SparkPlan
+    child: SparkPlan,
+    forcePostProjectCoalesce: Boolean = false
 ) extends GpuProjectExecLike {
 
   override def output: Seq[Attribute] = {
diff --git a/sql-plugin/src/test/scala/com/nvidia/spark/rapids/ProtobufBatchMergeSuite.scala b/sql-plugin/src/test/scala/com/nvidia/spark/rapids/ProtobufBatchMergeSuite.scala
new file mode 100644
index 00000000000..7d90237585b
--- /dev/null
+++ b/sql-plugin/src/test/scala/com/nvidia/spark/rapids/ProtobufBatchMergeSuite.scala
@@ -0,0 +1,95 @@
+/*
+ * Copyright (c) 2026, NVIDIA CORPORATION.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.nvidia.spark.rapids
+
+import org.scalatest.funsuite.AnyFunSuite
+
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.expressions.{Alias, AttributeReference, Expression, GetStructField, UnaryExpression}
+import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, ExprCode}
+import org.apache.spark.sql.execution.{LeafExecNode, ProjectExec, SparkPlan}
+import org.apache.spark.sql.types.{BinaryType, IntegerType, StructField, StructType}
+import org.apache.spark.sql.vectorized.ColumnarBatch
+
+class ProtobufBatchMergeSuite extends AnyFunSuite {
+
+  private case class DummyColumnarLeaf(output: Seq[AttributeReference]) extends LeafExecNode {
+    override protected def doExecute(): RDD[InternalRow] =
+      throw new UnsupportedOperationException("not needed for unit test")
+
+    override protected def doExecuteColumnar(): RDD[ColumnarBatch] =
+      throw new UnsupportedOperationException("not needed for unit test")
+
+    override def supportsColumnar: Boolean = true
+  }
+
+  private case class FakeProtobufDataToCatalyst(child: Expression) extends UnaryExpression {
+    override def dataType: StructType = StructType(Seq(
+      StructField("search_id", IntegerType, nullable = true),
+      StructField("nested", StructType(Seq(StructField("value", IntegerType, nullable = true))),
+        nullable = true)))
+    override def nullable: Boolean = true
+    override protected def withNewChildInternal(newChild: Expression): Expression = copy(newChild)
+    override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode =
+      throw new UnsupportedOperationException("codegen not needed in unit test")
+  }
+
+  test("project meta detects extractor project over protobuf child project") {
+    val binAttr = AttributeReference("bin", BinaryType)()
+    val childScan = DummyColumnarLeaf(Seq(binAttr))
+    val innerProject = ProjectExec(Seq(Alias(FakeProtobufDataToCatalyst(binAttr), "decoded")()), childScan)
+    val decodedAttr = innerProject.output.head.toAttribute
+    val outerProject = ProjectExec(Seq(
+      Alias(GetStructField(decodedAttr, 0, None), "search_id")(),
+      Alias(GetStructField(GetStructField(decodedAttr, 1, None), 0, None), "value")()), innerProject)
+
+    assert(GpuProjectExecMeta.shouldCoalesceAfterProject(outerProject))
+    assert(!GpuProjectExecMeta.shouldCoalesceAfterProject(innerProject))
+  }
+
+  test("protobuf batch merge config defaults off and can be enabled") {
+    val enabledConf = new RapidsConf(Map(
+      RapidsConf.ENABLE_PROTOBUF_BATCH_MERGE_AFTER_PROJECT.key -> "true"))
+
+    assert(!new RapidsConf(Map.empty[String, String]).isProtobufBatchMergeAfterProjectEnabled)
+    assert(enabledConf.isProtobufBatchMergeAfterProjectEnabled)
+  }
+
+  test("flagged gpu project drops output batching guarantee for post-project merge") {
+    val childAttr = AttributeReference("value", IntegerType)()
+    val child: SparkPlan = DummyColumnarLeaf(Seq(childAttr))
+
+    val unflaggedProject = GpuProjectExec(
+      projectList = child.output.map(a => Alias(a, a.name)()).toList,
+      child = child,
+      enablePreSplit = true,
+      forcePostProjectCoalesce = false)
+    val flaggedProject = GpuProjectExec(
+      projectList = child.output.map(a => Alias(a, a.name)()).toList,
+      child = child,
+      enablePreSplit = true,
+      forcePostProjectCoalesce = true)
+
+    assert(!unflaggedProject.coalesceAfter)
+    assert(flaggedProject.coalesceAfter)
+    assert(unflaggedProject.outputBatching.isInstanceOf[TargetSize])
+    assert(flaggedProject.outputBatching == null)
+    assert(CoalesceGoal.satisfies(unflaggedProject.outputBatching, TargetSize(1L)))
+    assert(!CoalesceGoal.satisfies(flaggedProject.outputBatching, TargetSize(1L)))
+  }
+}

From 19eb9833b0ccf787d540e2bafe2fde2979e459e5 Mon Sep 17 00:00:00 2001
From: Haoyang Li <haoyangl@nvidia.com>
Date: Wed, 11 Mar 2026 17:29:03 +0800
Subject: [PATCH 26/33] fix the enum bug

Signed-off-by: Haoyang Li <haoyangl@nvidia.com>
---
 .../src/main/python/protobuf_test.py          | 126 ++++++++++++++++++
 .../rapids/shims/ProtobufExprShims.scala      |  22 ++-
 .../rapids/ProtobufBatchMergeSuite.scala      |   7 +-
 3 files changed, 151 insertions(+), 4 deletions(-)

diff --git a/integration_tests/src/main/python/protobuf_test.py b/integration_tests/src/main/python/protobuf_test.py
index 5e3bc57da39..d178c8cb6f4 100644
--- a/integration_tests/src/main/python/protobuf_test.py
+++ b/integration_tests/src/main/python/protobuf_test.py
@@ -896,6 +896,132 @@ def run_on_spark(spark):
     assert_gpu_and_cpu_are_equal_collect(run_on_spark)
 
 
+def _build_nested_enum_default_struct_descriptor_set_bytes(spark):
+    """
+    Build a FileDescriptorSet for:
+      package test;
+      syntax = "proto2";
+      enum Language {
+        UNKNOWN_LANGUAGE = 0;
+        EN = 1;
+        ZH = 2;
+      }
+      enum CodeType {
+        UNKNOWN_CODE = 0;
+        UTF8 = 1;
+        GBK = 2;
+      }
+      message CommonWithEnumDefaults {
+        optional string logid = 1;
+        optional Language language = 2 [default = EN];
+        optional CodeType code_type = 3 [default = UTF8];
+      }
+      message OuterWithCommonEnumDefaults {
+        optional int32 id = 1;
+        optional CommonWithEnumDefaults common = 2;
+      }
+
+    Regression target:
+      In string mode, spark-protobuf surfaces enum defaults as EnumValueDescriptor.
+      GPU planning must not cast those defaults to Python/Scala String blindly.
+    """
+    D, fd = _new_proto2_file(spark, "nested_enum_defaults.proto")
+    label_opt = D.FieldDescriptorProto.Label.LABEL_OPTIONAL
+
+    language_enum = D.EnumDescriptorProto.newBuilder().setName("Language")
+    language_enum.addValue(
+        D.EnumValueDescriptorProto.newBuilder()
+            .setName("UNKNOWN_LANGUAGE").setNumber(0).build())
+    language_enum.addValue(
+        D.EnumValueDescriptorProto.newBuilder()
+            .setName("EN").setNumber(1).build())
+    language_enum.addValue(
+        D.EnumValueDescriptorProto.newBuilder()
+            .setName("ZH").setNumber(2).build())
+    fd.addEnumType(language_enum.build())
+
+    code_enum = D.EnumDescriptorProto.newBuilder().setName("CodeType")
+    code_enum.addValue(
+        D.EnumValueDescriptorProto.newBuilder()
+            .setName("UNKNOWN_CODE").setNumber(0).build())
+    code_enum.addValue(
+        D.EnumValueDescriptorProto.newBuilder()
+            .setName("UTF8").setNumber(1).build())
+    code_enum.addValue(
+        D.EnumValueDescriptorProto.newBuilder()
+            .setName("GBK").setNumber(2).build())
+    fd.addEnumType(code_enum.build())
+
+    common_msg = D.DescriptorProto.newBuilder().setName("CommonWithEnumDefaults")
+    common_msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("logid").setNumber(1).setLabel(label_opt)
+            .setType(D.FieldDescriptorProto.Type.TYPE_STRING).build())
+    common_msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("language").setNumber(2).setLabel(label_opt)
+            .setType(D.FieldDescriptorProto.Type.TYPE_ENUM)
+            .setTypeName(".test.Language")
+            .setDefaultValue("EN").build())
+    common_msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("code_type").setNumber(3).setLabel(label_opt)
+            .setType(D.FieldDescriptorProto.Type.TYPE_ENUM)
+            .setTypeName(".test.CodeType")
+            .setDefaultValue("UTF8").build())
+    fd.addMessageType(common_msg.build())
+
+    outer_msg = D.DescriptorProto.newBuilder().setName("OuterWithCommonEnumDefaults")
+    outer_msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("id").setNumber(1).setLabel(label_opt)
+            .setType(D.FieldDescriptorProto.Type.TYPE_INT32).build())
+    outer_msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("common").setNumber(2).setLabel(label_opt)
+            .setType(D.FieldDescriptorProto.Type.TYPE_MESSAGE)
+            .setTypeName(".test.CommonWithEnumDefaults").build())
+    fd.addMessageType(outer_msg.build())
+
+    fds = D.FileDescriptorSet.newBuilder().addFile(fd.build()).build()
+    return bytes(fds.toByteArray())
+
+
+@pytest.mark.skipif(is_before_spark_340(), reason="from_protobuf is Spark 3.4.0+")
+@ignore_order(local=True)
+def test_from_protobuf_nested_enum_defaults_string_mode(spark_tmp_path, from_protobuf_fn):
+    """Regression test: nested enum defaults in string mode must not crash GPU planning.
+
+    Selecting the whole nested struct forces GPU planning to visit all of its children,
+    including enum fields whose proto2 defaults are represented as EnumValueDescriptor
+    objects by spark-protobuf.
+    """
+    desc_path, desc_bytes = _setup_protobuf_desc(
+        spark_tmp_path, "nested_enum_defaults.desc",
+        _build_nested_enum_default_struct_descriptor_set_bytes)
+    message_name = "test.OuterWithCommonEnumDefaults"
+
+    common_with_logid = (_encode_tag(1, 2) + _encode_varint(5) + b"log-1")
+    row_logid_only = (_encode_tag(1, 0) + _encode_varint(1) +
+                      _encode_tag(2, 2) + _encode_varint(len(common_with_logid)) +
+                      common_with_logid)
+    row_empty_common = (_encode_tag(1, 0) + _encode_varint(2) +
+                        _encode_tag(2, 2) + _encode_varint(0))
+    row_no_common = _encode_tag(1, 0) + _encode_varint(3)
+
+    def run_on_spark(spark):
+        df = spark.createDataFrame(
+            [(row_logid_only,), (row_empty_common,), (row_no_common,)],
+            schema="bin binary")
+        decoded = _call_from_protobuf(
+            from_protobuf_fn, f.col("bin"), message_name, desc_path, desc_bytes)
+        return df.select(
+            decoded.getField("id").alias("id"),
+            decoded.getField("common").alias("common"))
+
+    assert_gpu_and_cpu_are_equal_collect(run_on_spark)
+
+
 def _build_repeated_enum_descriptor_set_bytes(spark):
     """
     message WithRepeatedEnum {
diff --git a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala
index ff14f386fcc..d3068a07a77 100644
--- a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala
+++ b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala
@@ -328,6 +328,14 @@ object ProtobufExprShims extends org.apache.spark.internal.Logging {
                     val intVal = defVal match {
                       case i: java.lang.Integer => i.longValue()
                       case l: java.lang.Long => l.longValue()
+                      case n: java.lang.Number => n.longValue()
+                      case ref: AnyRef =>
+                        // spark-protobuf may return enum defaults as EnumValueDescriptor.
+                        // In int mode, extract the numeric enum value reflectively.
+                        Try {
+                          ref.getClass.getMethod("getNumber")
+                            .invoke(ref).asInstanceOf[java.lang.Number].longValue()
+                        }.getOrElse(0L)
                       case _ => 0L
                     }
                     (intVal, 0.0, false, null: Array[Byte])
@@ -338,9 +346,19 @@ object ProtobufExprShims extends org.apache.spark.internal.Logging {
                     val d = defVal.asInstanceOf[java.lang.Double].doubleValue()
                     (0L, d, false, null: Array[Byte])
                   case StringType =>
-                    val str = defVal.asInstanceOf[String]
+                    val enumDefaultInt = defVal match {
+                      case ref: AnyRef =>
+                        Try {
+                          ref.getClass.getMethod("getNumber")
+                            .invoke(ref).asInstanceOf[java.lang.Number].longValue()
+                        }.getOrElse(0L)
+                      case _ => 0L
+                    }
+                    // spark-protobuf may surface enum defaults as EnumValueDescriptor
+                    // rather than String. `toString` yields the enum name in both cases.
+                    val str = if (defVal == null) null else defVal.toString
                     val bytes = if (str != null) str.getBytes("UTF-8") else null
-                    (0L, 0.0, false, bytes)
+                    (enumDefaultInt, 0.0, false, bytes)
                   case BinaryType =>
                     val bytes = Try {
                       val toByteArray = defVal.getClass.getMethod("toByteArray")
diff --git a/sql-plugin/src/test/scala/com/nvidia/spark/rapids/ProtobufBatchMergeSuite.scala b/sql-plugin/src/test/scala/com/nvidia/spark/rapids/ProtobufBatchMergeSuite.scala
index 7d90237585b..d8c7dd0b726 100644
--- a/sql-plugin/src/test/scala/com/nvidia/spark/rapids/ProtobufBatchMergeSuite.scala
+++ b/sql-plugin/src/test/scala/com/nvidia/spark/rapids/ProtobufBatchMergeSuite.scala
@@ -52,11 +52,14 @@ class ProtobufBatchMergeSuite extends AnyFunSuite {
   test("project meta detects extractor project over protobuf child project") {
     val binAttr = AttributeReference("bin", BinaryType)()
     val childScan = DummyColumnarLeaf(Seq(binAttr))
-    val innerProject = ProjectExec(Seq(Alias(FakeProtobufDataToCatalyst(binAttr), "decoded")()), childScan)
+    val innerProject = ProjectExec(
+      Seq(Alias(FakeProtobufDataToCatalyst(binAttr), "decoded")()),
+      childScan)
     val decodedAttr = innerProject.output.head.toAttribute
     val outerProject = ProjectExec(Seq(
       Alias(GetStructField(decodedAttr, 0, None), "search_id")(),
-      Alias(GetStructField(GetStructField(decodedAttr, 1, None), 0, None), "value")()), innerProject)
+      Alias(GetStructField(GetStructField(decodedAttr, 1, None), 0, None), "value")()),
+      innerProject)
 
     assert(GpuProjectExecMeta.shouldCoalesceAfterProject(outerProject))
     assert(!GpuProjectExecMeta.shouldCoalesceAfterProject(innerProject))

From 3059a50aa3b7806631dbd1c5933734bc4bc541b2 Mon Sep 17 00:00:00 2001
From: Haoyang Li <haoyangl@nvidia.com>
Date: Thu, 12 Mar 2026 10:04:32 +0800
Subject: [PATCH 27/33] reflection refactor

Signed-off-by: Haoyang Li <haoyangl@nvidia.com>
---
 .../src/main/python/protobuf_test.py          |  84 +++
 .../spark/sql/rapids/GpuFromProtobuf.scala    |   4 +-
 .../protobuf/ProtobufSchemaExtractor.scala    | 184 +++++
 .../rapids/protobuf/ProtobufSchemaModel.scala | 126 ++++
 .../protobuf/ProtobufSchemaValidator.scala    | 172 +++++
 .../rapids/shims/ProtobufExprShims.scala      | 680 +++---------------
 .../rapids/shims/SparkProtobufCompat.scala    | 297 ++++++++
 .../rapids/shims/ProtobufExprShimsSuite.scala | 207 ++++++
 8 files changed, 1164 insertions(+), 590 deletions(-)
 create mode 100644 sql-plugin/src/main/scala/org/apache/spark/sql/rapids/protobuf/ProtobufSchemaExtractor.scala
 create mode 100644 sql-plugin/src/main/scala/org/apache/spark/sql/rapids/protobuf/ProtobufSchemaModel.scala
 create mode 100644 sql-plugin/src/main/scala/org/apache/spark/sql/rapids/protobuf/ProtobufSchemaValidator.scala
 create mode 100644 sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/SparkProtobufCompat.scala
 create mode 100644 sql-plugin/src/test/scala/com/nvidia/spark/rapids/shims/ProtobufExprShimsSuite.scala

diff --git a/integration_tests/src/main/python/protobuf_test.py b/integration_tests/src/main/python/protobuf_test.py
index d178c8cb6f4..5fce5bd76c3 100644
--- a/integration_tests/src/main/python/protobuf_test.py
+++ b/integration_tests/src/main/python/protobuf_test.py
@@ -896,6 +896,41 @@ def run_on_spark(spark):
     assert_gpu_and_cpu_are_equal_collect(run_on_spark)
 
 
+@pytest.mark.skipif(is_before_spark_340(), reason="from_protobuf is Spark 3.4.0+")
+@ignore_order(local=True)
+def test_from_protobuf_nested_enum_invalid_permissive_nulls_sibling_fields(
+        spark_tmp_path, from_protobuf_fn):
+    """Invalid nested enums must null the full row, including sibling fields."""
+    desc_path, desc_bytes = _setup_protobuf_desc(
+        spark_tmp_path, "nested_enum.desc", _build_nested_enum_descriptor_set_bytes)
+    message_name = "test.WithNestedEnum"
+
+    row_valid = (_encode_tag(1, 0) + _encode_varint(1) +
+                 _encode_tag(2, 2) + _encode_varint(4) +
+                 _encode_tag(1, 0) + _encode_varint(1) +
+                 _encode_tag(2, 0) + _encode_varint(10) +
+                 _encode_tag(3, 2) + _encode_varint(2) + b"ok")
+    row_invalid = (_encode_tag(1, 0) + _encode_varint(2) +
+                   _encode_tag(2, 2) + _encode_varint(4) +
+                   _encode_tag(1, 0) + _encode_varint(999) +
+                   _encode_tag(2, 0) + _encode_varint(20) +
+                   _encode_tag(3, 2) + _encode_varint(3) + b"bad")
+
+    def run_on_spark(spark):
+        df = spark.createDataFrame([(0, row_valid), (1, row_invalid)], schema="idx int, bin binary")
+        decoded = _call_from_protobuf(
+            from_protobuf_fn, f.col("bin"), message_name, desc_path, desc_bytes,
+            options={"mode": "PERMISSIVE"})
+        return df.select(
+            f.col("idx"),
+            decoded.isNull().alias("decoded_is_null"),
+            decoded.getField("id").alias("id"),
+            decoded.getField("detail").getField("status").alias("status"),
+            decoded.getField("name").alias("name")).orderBy("idx")
+
+    assert_gpu_and_cpu_are_equal_collect(run_on_spark)
+
+
 def _build_nested_enum_default_struct_descriptor_set_bytes(spark):
     """
     Build a FileDescriptorSet for:
@@ -1090,6 +1125,36 @@ def run_on_spark(spark):
     assert_gpu_and_cpu_are_equal_collect(run_on_spark)
 
 
+@pytest.mark.skipif(is_before_spark_340(), reason="from_protobuf is Spark 3.4.0+")
+@ignore_order(local=True)
+def test_from_protobuf_repeated_enum_string_invalid_permissive_nulls_sibling_fields(
+        spark_tmp_path, from_protobuf_fn):
+    """Repeated enum string mode must null sibling fields when any enum value is invalid."""
+    desc_path, desc_bytes = _setup_protobuf_desc(
+        spark_tmp_path, "repeated_enum.desc", _build_repeated_enum_descriptor_set_bytes)
+    message_name = "test.WithRepeatedEnum"
+
+    row_valid = (_encode_tag(1, 0) + _encode_varint(1) +
+                 _encode_tag(2, 0) + _encode_varint(0) +
+                 _encode_tag(2, 0) + _encode_varint(2))
+    row_invalid = (_encode_tag(1, 0) + _encode_varint(2) +
+                   _encode_tag(2, 0) + _encode_varint(1) +
+                   _encode_tag(2, 0) + _encode_varint(99))
+
+    def run_on_spark(spark):
+        df = spark.createDataFrame([(0, row_valid), (1, row_invalid)], schema="idx int, bin binary")
+        decoded = _call_from_protobuf(
+            from_protobuf_fn, f.col("bin"), message_name, desc_path, desc_bytes,
+            options={"mode": "PERMISSIVE"})
+        return df.select(
+            f.col("idx"),
+            decoded.isNull().alias("decoded_is_null"),
+            decoded.getField("id").alias("id"),
+            decoded.getField("priority").alias("priority")).orderBy("idx")
+
+    assert_gpu_and_cpu_are_equal_collect(run_on_spark)
+
+
 def _build_required_field_descriptor_set_bytes(spark):
     """
     Build a FileDescriptorSet for a message with required fields (proto2):
@@ -1179,6 +1244,25 @@ def run_on_spark(spark):
     assert_gpu_and_cpu_are_equal_collect(run_on_spark)
 
 
+@pytest.mark.skipif(is_before_spark_340(), reason="from_protobuf is Spark 3.4.0+")
+@ignore_order(local=True)
+def test_from_protobuf_required_field_missing_failfast(spark_tmp_path, from_protobuf_fn):
+    """Required-field violations should fail consistently on CPU and GPU in FAILFAST mode."""
+    desc_path, desc_bytes = _setup_protobuf_desc(
+        spark_tmp_path, "required.desc", _build_required_field_descriptor_set_bytes)
+    message_name = "test.WithRequired"
+
+    missing_required_row = _encode_tag(2, 2) + _encode_varint(4) + b"oops"
+
+    def run_on_spark(spark):
+        df = spark.createDataFrame([(missing_required_row,)], schema="bin binary")
+        decoded = _call_from_protobuf(
+            from_protobuf_fn, f.col("bin"), message_name, desc_path, desc_bytes)
+        return df.select(decoded.alias("decoded")).collect()
+
+    assert_gpu_and_cpu_error(run_on_spark, conf={}, error_message="Malformed")
+
+
 def _build_default_value_descriptor_set_bytes(spark):
     """
     Build a FileDescriptorSet for a message with default values (proto2):
diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuFromProtobuf.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuFromProtobuf.scala
index 1bd7c7364c7..10d5116c40c 100644
--- a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuFromProtobuf.scala
+++ b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuFromProtobuf.scala
@@ -90,14 +90,14 @@ case class GpuFromProtobuf(
 
   override def nullable: Boolean = true
 
-  @transient private lazy val schema = new ProtobufSchemaDescriptor(
+  @transient private lazy val protobufSchema = new ProtobufSchemaDescriptor(
     fieldNumbers, parentIndices, depthLevels, wireTypes, outputTypeIds, encodings,
     isRepeated, isRequired, hasDefaultValue, defaultInts, defaultFloats, defaultBools,
     defaultStrings, enumValidValues, enumNames)
 
   override protected def doColumnar(input: GpuColumnVector): cudf.ColumnVector = {
     val jniResult = try {
-      Protobuf.decodeToStruct(input.getBase, schema, failOnErrors)
+      Protobuf.decodeToStruct(input.getBase, protobufSchema, failOnErrors)
     } catch {
       case e: CudfException if failOnErrors =>
         throw new org.apache.spark.SparkException("Malformed protobuf message", e)
diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/protobuf/ProtobufSchemaExtractor.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/protobuf/ProtobufSchemaExtractor.scala
new file mode 100644
index 00000000000..d3f0c0fc68f
--- /dev/null
+++ b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/protobuf/ProtobufSchemaExtractor.scala
@@ -0,0 +1,184 @@
+/*
+ * Copyright (c) 2026, NVIDIA CORPORATION.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.rapids.protobuf
+
+import scala.collection.mutable
+
+import com.nvidia.spark.rapids.jni.Protobuf.{WT_32BIT, WT_64BIT, WT_LEN, WT_VARINT}
+
+import org.apache.spark.sql.rapids.GpuFromProtobuf
+import org.apache.spark.sql.types._
+
+object ProtobufSchemaExtractor {
+  def analyzeAllFields(
+      schema: StructType,
+      msgDesc: ProtobufMessageDescriptor,
+      enumsAsInts: Boolean,
+      messageName: String): Either[String, Map[String, ProtobufFieldInfo]] = {
+    val result = mutable.Map[String, ProtobufFieldInfo]()
+
+    schema.fields.foreach { sf =>
+      val fd = msgDesc.findField(sf.name).getOrElse {
+        return Left(s"Protobuf field '${sf.name}' not found in message '$messageName'")
+      }
+      result(sf.name) = extractFieldInfo(sf, fd, enumsAsInts)
+    }
+
+    Right(result.toMap)
+  }
+
+  def extractFieldInfo(
+      sparkField: StructField,
+      fieldDescriptor: ProtobufFieldDescriptor,
+      enumsAsInts: Boolean): ProtobufFieldInfo = {
+    val (isSupported, unsupportedReason, encoding) =
+      checkFieldSupport(
+        sparkField.dataType,
+        fieldDescriptor.protoTypeName,
+        fieldDescriptor.isRepeated,
+        enumsAsInts)
+
+    ProtobufFieldInfo(
+      fieldNumber = fieldDescriptor.fieldNumber,
+      protoTypeName = fieldDescriptor.protoTypeName,
+      sparkType = sparkField.dataType,
+      encoding = encoding,
+      isSupported = isSupported,
+      unsupportedReason = unsupportedReason,
+      isRequired = fieldDescriptor.isRequired,
+      defaultValue = fieldDescriptor.defaultValue,
+      enumMetadata = fieldDescriptor.enumMetadata,
+      isRepeated = fieldDescriptor.isRepeated
+    )
+  }
+
+  def checkFieldSupport(
+      sparkType: DataType,
+      protoTypeName: String,
+      isRepeated: Boolean,
+      enumsAsInts: Boolean): (Boolean, Option[String], Int) = {
+
+    if (isRepeated) {
+      sparkType match {
+        case ArrayType(elementType, _) =>
+          elementType match {
+            case BooleanType | IntegerType | LongType | FloatType | DoubleType |
+                 StringType | BinaryType =>
+              return checkScalarEncoding(elementType, protoTypeName, enumsAsInts)
+            case _: StructType =>
+              return (true, None, GpuFromProtobuf.ENC_DEFAULT)
+            case _ =>
+              return (
+                false,
+                Some(s"unsupported repeated element type: $elementType"),
+                GpuFromProtobuf.ENC_DEFAULT)
+          }
+        case _ =>
+          return (
+            false,
+            Some(s"repeated field should map to ArrayType, got: $sparkType"),
+            GpuFromProtobuf.ENC_DEFAULT)
+      }
+    }
+
+    if (protoTypeName == "MESSAGE") {
+      sparkType match {
+        case _: StructType =>
+          return (true, None, GpuFromProtobuf.ENC_DEFAULT)
+        case _ =>
+          return (
+            false,
+            Some(s"nested message should map to StructType, got: $sparkType"),
+            GpuFromProtobuf.ENC_DEFAULT)
+      }
+    }
+
+    sparkType match {
+      case BooleanType | IntegerType | LongType | FloatType | DoubleType |
+           StringType | BinaryType =>
+      case other =>
+        return (
+          false,
+          Some(s"unsupported Spark type: $other"),
+          GpuFromProtobuf.ENC_DEFAULT)
+    }
+
+    checkScalarEncoding(sparkType, protoTypeName, enumsAsInts)
+  }
+
+  def checkScalarEncoding(
+      sparkType: DataType,
+      protoTypeName: String,
+      enumsAsInts: Boolean): (Boolean, Option[String], Int) = {
+    val encoding = (sparkType, protoTypeName) match {
+      case (BooleanType, "BOOL") => Some(GpuFromProtobuf.ENC_DEFAULT)
+      case (IntegerType, "INT32" | "UINT32") => Some(GpuFromProtobuf.ENC_DEFAULT)
+      case (IntegerType, "SINT32") => Some(GpuFromProtobuf.ENC_ZIGZAG)
+      case (IntegerType, "FIXED32" | "SFIXED32") => Some(GpuFromProtobuf.ENC_FIXED)
+      case (LongType, "INT64" | "UINT64") => Some(GpuFromProtobuf.ENC_DEFAULT)
+      case (LongType, "SINT64") => Some(GpuFromProtobuf.ENC_ZIGZAG)
+      case (LongType, "FIXED64" | "SFIXED64") => Some(GpuFromProtobuf.ENC_FIXED)
+      case (LongType, "INT32" | "UINT32" | "SINT32" | "FIXED32" | "SFIXED32") =>
+        val enc = protoTypeName match {
+          case "SINT32" => GpuFromProtobuf.ENC_ZIGZAG
+          case "FIXED32" | "SFIXED32" => GpuFromProtobuf.ENC_FIXED
+          case _ => GpuFromProtobuf.ENC_DEFAULT
+        }
+        Some(enc)
+      case (FloatType, "FLOAT") => Some(GpuFromProtobuf.ENC_DEFAULT)
+      case (DoubleType, "DOUBLE") => Some(GpuFromProtobuf.ENC_DEFAULT)
+      case (StringType, "STRING") => Some(GpuFromProtobuf.ENC_DEFAULT)
+      case (BinaryType, "BYTES") => Some(GpuFromProtobuf.ENC_DEFAULT)
+      case (IntegerType, "ENUM") if enumsAsInts => Some(GpuFromProtobuf.ENC_DEFAULT)
+      case (StringType, "ENUM") if !enumsAsInts => Some(GpuFromProtobuf.ENC_ENUM_STRING)
+      case _ => None
+    }
+
+    encoding match {
+      case Some(enc) => (true, None, enc)
+      case None =>
+        (false,
+          Some(s"type mismatch: Spark $sparkType vs Protobuf $protoTypeName"),
+          GpuFromProtobuf.ENC_DEFAULT)
+    }
+  }
+
+  def getWireType(protoTypeName: String, encoding: Int): Either[String, Int] = {
+    val wireType = protoTypeName match {
+      case "BOOL" | "INT32" | "UINT32" | "SINT32" | "INT64" | "UINT64" | "SINT64" | "ENUM" =>
+        if (encoding == GpuFromProtobuf.ENC_FIXED) {
+          if (protoTypeName.contains("64")) {
+            WT_64BIT
+          } else {
+            WT_32BIT
+          }
+        } else {
+          WT_VARINT
+        }
+      case "FIXED32" | "SFIXED32" | "FLOAT" =>
+        WT_32BIT
+      case "FIXED64" | "SFIXED64" | "DOUBLE" =>
+        WT_64BIT
+      case "STRING" | "BYTES" | "MESSAGE" =>
+        WT_LEN
+      case other =>
+        return Left(
+          s"Unknown protobuf type name '$other' - cannot determine wire type; falling back to CPU")
+    }
+    Right(wireType)
+  }
+}
diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/protobuf/ProtobufSchemaModel.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/protobuf/ProtobufSchemaModel.scala
new file mode 100644
index 00000000000..1e04505d5b3
--- /dev/null
+++ b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/protobuf/ProtobufSchemaModel.scala
@@ -0,0 +1,126 @@
+/*
+ * Copyright (c) 2026, NVIDIA CORPORATION.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.rapids.protobuf
+
+import org.apache.spark.sql.types.DataType
+
+sealed trait ProtobufDescriptorSource
+
+object ProtobufDescriptorSource {
+  final case class DescriptorPath(path: String) extends ProtobufDescriptorSource
+  final case class DescriptorBytes(bytes: Array[Byte]) extends ProtobufDescriptorSource
+}
+
+final case class ProtobufExprInfo(
+    messageName: String,
+    descriptorSource: ProtobufDescriptorSource,
+    options: Map[String, String])
+
+final case class ProtobufPlannerOptions(
+    enumsAsInts: Boolean,
+    failOnErrors: Boolean)
+
+sealed trait ProtobufDefaultValue
+
+object ProtobufDefaultValue {
+  final case class BoolValue(value: Boolean) extends ProtobufDefaultValue
+  final case class IntValue(value: Long) extends ProtobufDefaultValue
+  final case class FloatValue(value: Float) extends ProtobufDefaultValue
+  final case class DoubleValue(value: Double) extends ProtobufDefaultValue
+  final case class StringValue(value: String) extends ProtobufDefaultValue
+  final case class BinaryValue(value: Array[Byte]) extends ProtobufDefaultValue
+  final case class EnumValue(number: Int, name: String) extends ProtobufDefaultValue
+}
+
+final case class ProtobufEnumValue(number: Int, name: String)
+
+final case class ProtobufEnumMetadata(values: Seq[ProtobufEnumValue]) {
+  lazy val validValues: Array[Int] = values.map(_.number).toArray
+  lazy val orderedNames: Array[Array[Byte]] = values.map(_.name.getBytes("UTF-8")).toArray
+  lazy val namesByNumber: Map[Int, String] = values.map(v => v.number -> v.name).toMap
+
+  def enumDefault(number: Int): ProtobufDefaultValue.EnumValue = {
+    val name = namesByNumber.getOrElse(number, s"$number")
+    ProtobufDefaultValue.EnumValue(number, name)
+  }
+}
+
+trait ProtobufMessageDescriptor {
+  def syntax: String
+  def findField(name: String): Option[ProtobufFieldDescriptor]
+}
+
+trait ProtobufFieldDescriptor {
+  def name: String
+  def fieldNumber: Int
+  def protoTypeName: String
+  def isRepeated: Boolean
+  def isRequired: Boolean
+  def defaultValue: Option[ProtobufDefaultValue]
+  def enumMetadata: Option[ProtobufEnumMetadata]
+  def messageDescriptor: Option[ProtobufMessageDescriptor]
+}
+
+final case class ProtobufFieldInfo(
+    fieldNumber: Int,
+    protoTypeName: String,
+    sparkType: DataType,
+    encoding: Int,
+    isSupported: Boolean,
+    unsupportedReason: Option[String],
+    isRequired: Boolean,
+    defaultValue: Option[ProtobufDefaultValue],
+    enumMetadata: Option[ProtobufEnumMetadata],
+    isRepeated: Boolean = false) {
+  def hasDefaultValue: Boolean = defaultValue.isDefined
+}
+
+final case class FlattenedFieldDescriptor(
+    fieldNumber: Int,
+    parentIdx: Int,
+    depth: Int,
+    wireType: Int,
+    outputTypeId: Int,
+    encoding: Int,
+    isRepeated: Boolean,
+    isRequired: Boolean,
+    hasDefaultValue: Boolean,
+    defaultInt: Long,
+    defaultFloat: Double,
+    defaultBool: Boolean,
+    defaultString: Array[Byte],
+    enumValidValues: Array[Int],
+    enumNames: Array[Array[Byte]]
+)
+
+final case class FlattenedSchemaArrays(
+    fieldNumbers: Array[Int],
+    parentIndices: Array[Int],
+    depthLevels: Array[Int],
+    wireTypes: Array[Int],
+    outputTypeIds: Array[Int],
+    encodings: Array[Int],
+    isRepeated: Array[Boolean],
+    isRequired: Array[Boolean],
+    hasDefaultValue: Array[Boolean],
+    defaultInts: Array[Long],
+    defaultFloats: Array[Double],
+    defaultBools: Array[Boolean],
+    defaultStrings: Array[Array[Byte]],
+    enumValidValues: Array[Array[Int]],
+    enumNames: Array[Array[Array[Byte]]]
+)
diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/protobuf/ProtobufSchemaValidator.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/protobuf/ProtobufSchemaValidator.scala
new file mode 100644
index 00000000000..eb554336ff3
--- /dev/null
+++ b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/protobuf/ProtobufSchemaValidator.scala
@@ -0,0 +1,172 @@
+/*
+ * Copyright (c) 2026, NVIDIA CORPORATION.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.rapids.protobuf
+
+import org.apache.spark.sql.rapids.GpuFromProtobuf
+import org.apache.spark.sql.types._
+
+object ProtobufSchemaValidator {
+  private final case class JniDefaultValues(
+      defaultInt: Long,
+      defaultFloat: Double,
+      defaultBool: Boolean,
+      defaultString: Array[Byte])
+
+  def toFlattenedFieldDescriptor(
+      path: String,
+      field: StructField,
+      fieldInfo: ProtobufFieldInfo,
+      parentIdx: Int,
+      depth: Int,
+      outputTypeId: Int): Either[String, FlattenedFieldDescriptor] = {
+    validateFieldInfo(path, field, fieldInfo).flatMap { _ =>
+      ProtobufSchemaExtractor.getWireType(fieldInfo.protoTypeName, fieldInfo.encoding).map { wireType =>
+        val defaults = encodeDefaultValue(path, field.dataType, fieldInfo)
+        val enumValidValues = fieldInfo.enumMetadata.map(_.validValues).orNull
+        val enumNames =
+          if (fieldInfo.encoding == GpuFromProtobuf.ENC_ENUM_STRING) {
+            fieldInfo.enumMetadata.map(_.orderedNames).orNull
+          } else {
+            null
+          }
+
+        FlattenedFieldDescriptor(
+          fieldNumber = fieldInfo.fieldNumber,
+          parentIdx = parentIdx,
+          depth = depth,
+          wireType = wireType,
+          outputTypeId = outputTypeId,
+          encoding = fieldInfo.encoding,
+          isRepeated = fieldInfo.isRepeated,
+          isRequired = fieldInfo.isRequired,
+          hasDefaultValue = fieldInfo.hasDefaultValue,
+          defaultInt = defaults.defaultInt,
+          defaultFloat = defaults.defaultFloat,
+          defaultBool = defaults.defaultBool,
+          defaultString = defaults.defaultString,
+          enumValidValues = enumValidValues,
+          enumNames = enumNames
+        )
+      }
+    }
+  }
+
+  def validateFlattenedSchema(flatFields: Seq[FlattenedFieldDescriptor]): Either[String, Unit] = {
+    flatFields.zipWithIndex.foreach { case (field, idx) =>
+      if (field.parentIdx >= idx) {
+        return Left(s"Flattened protobuf schema has invalid parent index at position $idx")
+      }
+      if (field.parentIdx == -1 && field.depth != 0) {
+        return Left(s"Top-level protobuf field at position $idx must have depth 0")
+      }
+      if (field.parentIdx >= 0 && field.depth <= 0) {
+        return Left(s"Nested protobuf field at position $idx must have positive depth")
+      }
+      if (field.encoding == GpuFromProtobuf.ENC_ENUM_STRING) {
+        if (field.enumValidValues == null || field.enumNames == null) {
+          return Left(s"Enum-string field at position $idx is missing enum metadata")
+        }
+        if (field.enumValidValues.length != field.enumNames.length) {
+          return Left(s"Enum-string field at position $idx has mismatched enum metadata")
+        }
+      }
+    }
+    Right(())
+  }
+
+  def toFlattenedSchemaArrays(flatFields: Array[FlattenedFieldDescriptor]): FlattenedSchemaArrays = {
+    FlattenedSchemaArrays(
+      fieldNumbers = flatFields.map(_.fieldNumber),
+      parentIndices = flatFields.map(_.parentIdx),
+      depthLevels = flatFields.map(_.depth),
+      wireTypes = flatFields.map(_.wireType),
+      outputTypeIds = flatFields.map(_.outputTypeId),
+      encodings = flatFields.map(_.encoding),
+      isRepeated = flatFields.map(_.isRepeated),
+      isRequired = flatFields.map(_.isRequired),
+      hasDefaultValue = flatFields.map(_.hasDefaultValue),
+      defaultInts = flatFields.map(_.defaultInt),
+      defaultFloats = flatFields.map(_.defaultFloat),
+      defaultBools = flatFields.map(_.defaultBool),
+      defaultStrings = flatFields.map(_.defaultString),
+      enumValidValues = flatFields.map(_.enumValidValues),
+      enumNames = flatFields.map(_.enumNames)
+    )
+  }
+
+  private def validateFieldInfo(
+      path: String,
+      field: StructField,
+      fieldInfo: ProtobufFieldInfo): Either[String, Unit] = {
+    if (fieldInfo.isRepeated && fieldInfo.hasDefaultValue) {
+      return Left(s"Repeated protobuf field '$path' cannot carry a default value")
+    }
+
+    fieldInfo.enumMetadata match {
+      case Some(enumMeta) if enumMeta.values.isEmpty =>
+        return Left(s"Enum field '$path' is missing enum values")
+      case Some(_) if fieldInfo.protoTypeName != "ENUM" =>
+        return Left(s"Non-enum field '$path' should not carry enum metadata")
+      case None if fieldInfo.protoTypeName == "ENUM" =>
+        return Left(s"Enum field '$path' is missing enum metadata")
+      case _ =>
+    }
+
+    if (fieldInfo.encoding == GpuFromProtobuf.ENC_ENUM_STRING &&
+        fieldInfo.enumMetadata.isEmpty) {
+      return Left(s"Enum-string field '$path' is missing enum metadata")
+    }
+
+    Right(())
+  }
+
+  private def encodeDefaultValue(
+      path: String,
+      dataType: DataType,
+      fieldInfo: ProtobufFieldInfo): JniDefaultValues = {
+    val empty = JniDefaultValues(0L, 0.0, defaultBool = false, null)
+    fieldInfo.defaultValue match {
+      case None => empty
+      case Some(defaultValue) =>
+        val targetType = dataType match {
+          case ArrayType(elementType, _) => elementType
+          case other => other
+        }
+        (targetType, defaultValue) match {
+          case (BooleanType, ProtobufDefaultValue.BoolValue(value)) =>
+            empty.copy(defaultBool = value)
+          case (IntegerType | LongType, ProtobufDefaultValue.IntValue(value)) =>
+            empty.copy(defaultInt = value)
+          case (IntegerType | LongType, ProtobufDefaultValue.EnumValue(number, _)) =>
+            empty.copy(defaultInt = number.toLong)
+          case (FloatType, ProtobufDefaultValue.FloatValue(value)) =>
+            empty.copy(defaultFloat = value.toDouble)
+          case (DoubleType, ProtobufDefaultValue.DoubleValue(value)) =>
+            empty.copy(defaultFloat = value)
+          case (StringType, ProtobufDefaultValue.StringValue(value)) =>
+            empty.copy(defaultString = value.getBytes("UTF-8"))
+          case (StringType, ProtobufDefaultValue.EnumValue(number, name)) =>
+            empty.copy(defaultInt = number.toLong, defaultString = name.getBytes("UTF-8"))
+          case (BinaryType, ProtobufDefaultValue.BinaryValue(value)) =>
+            empty.copy(defaultString = value)
+          case _ =>
+            throw new IllegalStateException(
+              s"Incompatible default value for protobuf field '$path': $defaultValue")
+        }
+    }
+  }
+}
diff --git a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala
index d3068a07a77..5756c5a670b 100644
--- a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala
+++ b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala
@@ -36,64 +36,25 @@ spark-rapids-shim-json-lines ***/
 
 package com.nvidia.spark.rapids.shims
 
-import java.lang.ReflectiveOperationException
-
 import scala.collection.mutable
-import scala.util.Try
 
 import ai.rapids.cudf.DType
 import com.nvidia.spark.rapids._
-import com.nvidia.spark.rapids.jni.Protobuf.{WT_32BIT, WT_64BIT, WT_LEN, WT_VARINT}
 
 import org.apache.spark.sql.catalyst.expressions.{
   AttributeReference, Expression, GetArrayStructFields, GetStructField, UnaryExpression
 }
 import org.apache.spark.sql.execution.ProjectExec
 import org.apache.spark.sql.rapids.GpuFromProtobuf
+import org.apache.spark.sql.rapids.protobuf.{
+  FlattenedFieldDescriptor,
+  ProtobufFieldInfo,
+  ProtobufMessageDescriptor,
+  ProtobufSchemaExtractor,
+  ProtobufSchemaValidator
+}
 import org.apache.spark.sql.types._
 
-/**
- * Information about a protobuf field for schema projection support.
- */
-private[shims] case class ProtobufFieldInfo(
-    fieldNumber: Int,
-    protoTypeName: String,
-    sparkType: DataType,
-    encoding: Int,
-    isSupported: Boolean,
-    unsupportedReason: Option[String],
-    isRequired: Boolean,
-    hasDefaultValue: Boolean,
-    defaultValue: Option[Any],  // Stored as protobuf-java type, will be converted for JNI
-    // Valid enum values for ENUM fields (used for validation)
-    enumValues: Option[Set[Int]] = None,
-    // Enum value-name mapping for ENUM -> STRING decoding (enumsAsInts=false)
-    enumNames: Option[Map[Int, String]] = None,
-    isRepeated: Boolean = false  // Whether this is a repeated field
-)
-
-/**
- * Flattened field descriptor for nested protobuf schemas.
- * Used to represent a hierarchical schema as a linear array for GPU processing.
- */
-private[shims] case class FlattenedFieldDescriptor(
-    fieldNumber: Int,
-    parentIdx: Int,          // Index of parent field in flattened array (-1 for top-level)
-    depth: Int,              // Nesting depth (0 for top-level)
-    wireType: Int,           // Protobuf wire type
-    outputTypeId: Int,       // cudf type id for the output (element type for repeated)
-    encoding: Int,           // Encoding (default/fixed/zigzag)
-    isRepeated: Boolean,     // Whether this is a repeated field
-    isRequired: Boolean,     // Whether this field is required (proto2)
-    hasDefaultValue: Boolean,
-    defaultInt: Long,
-    defaultFloat: Double,
-    defaultBool: Boolean,
-    defaultString: Array[Byte],
-    enumValidValues: Array[Int],
-    enumNames: Array[Array[Byte]]
-)
-
 /**
  * Spark 3.4+ optional integration for spark-protobuf expressions.
  *
@@ -103,9 +64,6 @@ object ProtobufExprShims extends org.apache.spark.internal.Logging {
   private[this] val protobufDataToCatalystClassName =
     "org.apache.spark.sql.protobuf.ProtobufDataToCatalyst"
 
-  private[this] val sparkProtobufUtilsObjectClassName =
-    "org.apache.spark.sql.protobuf.utils.ProtobufUtils$"
-
   val PRUNED_ORDINAL_TAG =
     org.apache.spark.sql.rapids.GpuStructFieldOrdinalTag.PRUNED_ORDINAL_TAG
 
@@ -165,15 +123,13 @@ object ProtobufExprShims extends org.apache.spark.internal.Logging {
               return
           }
 
-          val options = getOptionsMap(e) match {
-            case Some(m) => m
-            case None =>
-              willNotWorkOnGpu(
-                "Cannot read from_protobuf options via reflection; falling back to CPU")
+          val exprInfo = SparkProtobufCompat.extractExprInfo(e) match {
+            case Right(info) => info
+            case Left(reason) =>
+              willNotWorkOnGpu(reason)
               return
           }
-          val supportedOptions = Set("enums.as.ints", "mode")
-          val unsupportedOptions = options.keys.filterNot(supportedOptions.contains)
+          val unsupportedOptions = SparkProtobufCompat.unsupportedOptions(exprInfo.options)
           if (unsupportedOptions.nonEmpty) {
             val keys = unsupportedOptions.mkString(",")
             willNotWorkOnGpu(
@@ -181,45 +137,27 @@ object ProtobufExprShims extends org.apache.spark.internal.Logging {
             return
           }
 
-          val enumsAsInts = Try(options.getOrElse("enums.as.ints", "false").toBoolean) match {
-            case scala.util.Success(v) => v
-            case scala.util.Failure(_) =>
-              willNotWorkOnGpu("Invalid value for from_protobuf option 'enums.as.ints': " +
-                s"'${options.getOrElse("enums.as.ints", "")}' (expected true/false)")
+          val plannerOptions = SparkProtobufCompat.parsePlannerOptions(exprInfo.options) match {
+            case Right(opts) => opts
+            case Left(reason) =>
+              willNotWorkOnGpu(reason)
               return
           }
-          failOnErrors = options.getOrElse("mode", "FAILFAST").equalsIgnoreCase("FAILFAST")
-          val messageName = getMessageName(e)
-
-          // Try to get descriptor: either file path (Spark 3.4.x) or binary bytes (Spark 3.5+)
-          val descFilePathOrBytes: Option[Either[String, Array[Byte]]] =
-            getDescFilePath(e).map(Left(_)).orElse {
-              getDescriptorBytes(e).map(Right(_))
-            }
-
-          if (descFilePathOrBytes.isEmpty) {
-            willNotWorkOnGpu(
-              "from_protobuf requires a descriptor set " +
-                "(descFilePath or binaryFileDescriptorSet)")
-            return
-          }
+          val enumsAsInts = plannerOptions.enumsAsInts
+          failOnErrors = plannerOptions.failOnErrors
+          val messageName = exprInfo.messageName
 
-          val msgDesc = try {
-            // Spark 3.4.x: buildDescriptor(messageName, descFilePath: Option[String])
-            // Spark 3.5+:  buildDescriptor(messageName, binaryFileDescriptorSet)
-            buildMessageDescriptorWithSparkProtobuf(messageName, descFilePathOrBytes.get)
-          } catch {
-            case t: Throwable =>
-              willNotWorkOnGpu(
-                s"Failed to resolve protobuf descriptor for message '$messageName': " +
-                  s"${t.getMessage}")
+          val msgDesc = SparkProtobufCompat.resolveMessageDescriptor(exprInfo) match {
+            case Right(desc) => desc
+            case Left(reason) =>
+              willNotWorkOnGpu(reason)
               return
           }
 
           // Reject proto3 descriptors — GPU decoder only supports proto2 semantics.
           // proto3 has different null/default-value behavior that the GPU path doesn't handle.
-          val protoSyntax = PbReflect.getFileSyntax(msgDesc)
-          if (protoSyntax == "PROTO3" || protoSyntax == "EDITIONS" || protoSyntax.isEmpty) {
+          val protoSyntax = msgDesc.syntax
+          if (!SparkProtobufCompat.isGpuSupportedProtoSyntax(protoSyntax)) {
             willNotWorkOnGpu(
               "proto3/editions syntax is not supported by the GPU protobuf decoder; " +
                 "only proto2 is supported. The query will fall back to CPU.")
@@ -227,12 +165,12 @@ object ProtobufExprShims extends org.apache.spark.internal.Logging {
           }
 
           // Step 1: Analyze all fields and build field info map
-          val allFieldsInfo = analyzeAllFields(fullSchema, msgDesc, enumsAsInts, messageName)
-          if (allFieldsInfo.isEmpty) {
-            // Error was already reported in analyzeAllFields
-            return
-          }
-          val fieldsInfoMap = allFieldsInfo.get
+          val fieldsInfoMap =
+            ProtobufSchemaExtractor.analyzeAllFields(fullSchema, msgDesc, enumsAsInts, messageName)
+              .fold({ reason =>
+                willNotWorkOnGpu(reason)
+                return
+              }, identity)
 
           // Step 2: Determine which fields are actually required by downstream operations
           val requiredFieldNames = analyzeRequiredFields(fieldsInfoMap.keySet)
@@ -288,7 +226,7 @@ object ProtobufExprShims extends org.apache.spark.internal.Logging {
                 info: ProtobufFieldInfo,
                 parentIdx: Int,
                 depth: Int,
-                nestedMsgDesc: AnyRef,
+                nestedMsgDesc: ProtobufMessageDescriptor,
                 pathPrefix: String = ""): Unit = {
 
               val currentIdx = flatFields.size
@@ -315,85 +253,17 @@ object ProtobufExprShims extends org.apache.spark.internal.Logging {
                     .getOrElse(DType.INT8.getTypeId.getNativeId)
               }
 
-              val wireType = getWireType(info.protoTypeName, info.encoding)
-
-              val hasDefault = info.hasDefaultValue && info.defaultValue.isDefined
-              val (defInt, defFloat, defBool, defString) = if (hasDefault) {
-                val defVal = info.defaultValue.get
-                sf.dataType match {
-                  case BooleanType =>
-                    val b = defVal.asInstanceOf[java.lang.Boolean].booleanValue()
-                    (0L, 0.0, b, null: Array[Byte])
-                  case IntegerType | LongType =>
-                    val intVal = defVal match {
-                      case i: java.lang.Integer => i.longValue()
-                      case l: java.lang.Long => l.longValue()
-                      case n: java.lang.Number => n.longValue()
-                      case ref: AnyRef =>
-                        // spark-protobuf may return enum defaults as EnumValueDescriptor.
-                        // In int mode, extract the numeric enum value reflectively.
-                        Try {
-                          ref.getClass.getMethod("getNumber")
-                            .invoke(ref).asInstanceOf[java.lang.Number].longValue()
-                        }.getOrElse(0L)
-                      case _ => 0L
-                    }
-                    (intVal, 0.0, false, null: Array[Byte])
-                  case FloatType =>
-                    val f = defVal.asInstanceOf[java.lang.Float].doubleValue()
-                    (0L, f, false, null: Array[Byte])
-                  case DoubleType =>
-                    val d = defVal.asInstanceOf[java.lang.Double].doubleValue()
-                    (0L, d, false, null: Array[Byte])
-                  case StringType =>
-                    val enumDefaultInt = defVal match {
-                      case ref: AnyRef =>
-                        Try {
-                          ref.getClass.getMethod("getNumber")
-                            .invoke(ref).asInstanceOf[java.lang.Number].longValue()
-                        }.getOrElse(0L)
-                      case _ => 0L
-                    }
-                    // spark-protobuf may surface enum defaults as EnumValueDescriptor
-                    // rather than String. `toString` yields the enum name in both cases.
-                    val str = if (defVal == null) null else defVal.toString
-                    val bytes = if (str != null) str.getBytes("UTF-8") else null
-                    (enumDefaultInt, 0.0, false, bytes)
-                  case BinaryType =>
-                    val bytes = Try {
-                      val toByteArray = defVal.getClass.getMethod("toByteArray")
-                      toByteArray.invoke(defVal).asInstanceOf[Array[Byte]]
-                    }.getOrElse(null: Array[Byte])
-                    (0L, 0.0, false, bytes)
-                  case _ => (0L, 0.0, false, null: Array[Byte])
-                }
-              } else {
-                (0L, 0.0, false, null: Array[Byte])
-              }
-
-              val enumValsArr = info.enumValues.map(_.toArray.sorted).orNull
-              val enumNamesArr = info.enumNames.map { nameMap =>
-                val sorted = nameMap.toSeq.sortBy(_._1)
-                sorted.map { case (_, enumName) => enumName.getBytes("UTF-8") }.toArray
-              }.orNull
-
-              flatFields += FlattenedFieldDescriptor(
-                fieldNumber = info.fieldNumber,
-                parentIdx = parentIdx,
-                depth = depth,
-                wireType = wireType,
-                outputTypeId = outputType,
-                encoding = info.encoding,
-                isRepeated = info.isRepeated,
-                isRequired = info.isRequired,
-                hasDefaultValue = hasDefault,
-                defaultInt = defInt,
-                defaultFloat = defFloat,
-                defaultBool = defBool,
-                defaultString = defString,
-                enumValidValues = enumValsArr,
-                enumNames = enumNamesArr
-              )
+              val path = if (pathPrefix.isEmpty) sf.name else s"$pathPrefix.${sf.name}"
+              ProtobufSchemaValidator.toFlattenedFieldDescriptor(
+                path,
+                sf,
+                info,
+                parentIdx,
+                depth,
+                outputType).fold({ reason =>
+                willNotWorkOnGpu(reason)
+                return
+              }, flatFields += _)
 
               // For nested struct types (including repeated message = ArrayType(StructType)), 
               // add child fields
@@ -415,91 +285,48 @@ object ProtobufExprShims extends org.apache.spark.internal.Logging {
             // lookup into nestedFieldRequirements.
             def addChildFieldsFromStruct(
                 st: StructType,
-                parentMsgDesc: AnyRef,
+                parentMsgDesc: ProtobufMessageDescriptor,
                 fieldName: String,
                 parentIdx: Int,
                 parentDepth: Int,
                 pathPrefix: String): Unit = {
               val path = if (pathPrefix.isEmpty) fieldName else s"$pathPrefix.$fieldName"
-              val fd = PbReflect.findFieldByName(parentMsgDesc, fieldName)
-              if (fd == null) {
+              val parentField = parentMsgDesc.findField(fieldName)
+              if (parentField.isEmpty) {
                 willNotWorkOnGpu(
                   s"Nested field '$fieldName' not found in protobuf descriptor at '$path'")
               } else {
-                try {
-                  val childMsgDesc = PbReflect.getMessageType(fd)
-                  val requiredChildren = nestedFieldRequirements.get(path)
-                  val filteredFields = requiredChildren match {
-                    case Some(Some(childNames)) =>
-                      st.fields.filter(f => childNames.contains(f.name))
-                    case _ =>
-                      st.fields
-                  }
-                  filteredFields.foreach { childSf =>
-                    val childFd = PbReflect.findFieldByName(childMsgDesc, childSf.name)
-                    if (childFd == null) {
-                      willNotWorkOnGpu(
-                        s"Nested field '${childSf.name}' not found in protobuf " +
-                          s"descriptor for message at '$path'")
-                    } else {
-                      val childProtoTypeName = typeName(PbReflect.getFieldType(childFd))
-                      val childFieldNumber = PbReflect.getFieldNumber(childFd)
-                      val childIsRepeated = PbReflect.isRepeated(childFd)
-                      val childIsRequired = PbReflect.isRequired(childFd)
-                      val childHasDefault = PbReflect.hasDefaultValue(childFd)
-                      val (childIsSupported, childUnsupportedReason, childEncoding) =
-                        checkFieldSupport(
-                          childSf.dataType, childProtoTypeName, childIsRepeated, enumsAsInts)
-
-                      if (!childIsSupported) {
-                        willNotWorkOnGpu(
-                          s"Nested field '${childSf.name}' at '$path': " +
-                            childUnsupportedReason.getOrElse("unsupported type"))
-                      } else {
-                        val (childEnumVals, childEnumNameMap): (Option[Set[Int]],
-                          Option[Map[Int, String]]) =
-                          if (childProtoTypeName == "ENUM") {
-                            Try {
-                              val pairs = PbReflect.getEnumValues(
-                                PbReflect.getEnumType(childFd))
-                              if (enumsAsInts) {
-                                (Some(pairs.map(_._1).toSet), None)
-                              } else {
-                                (Some(pairs.map(_._1).toSet), Some(pairs.toMap))
-                              }
-                            }.getOrElse((None, None))
+                parentField.get.messageDescriptor match {
+                  case Some(childMsgDesc) =>
+                    val requiredChildren = nestedFieldRequirements.get(path)
+                    val filteredFields = requiredChildren match {
+                      case Some(Some(childNames)) =>
+                        st.fields.filter(f => childNames.contains(f.name))
+                      case _ =>
+                        st.fields
+                    }
+                    filteredFields.foreach { childSf =>
+                      childMsgDesc.findField(childSf.name) match {
+                        case None =>
+                          willNotWorkOnGpu(
+                            s"Nested field '${childSf.name}' not found in protobuf " +
+                              s"descriptor for message at '$path'")
+                        case Some(childFd) =>
+                          val childInfo =
+                            ProtobufSchemaExtractor.extractFieldInfo(childSf, childFd, enumsAsInts)
+                          if (!childInfo.isSupported) {
+                            willNotWorkOnGpu(
+                              s"Nested field '${childSf.name}' at '$path': " +
+                                childInfo.unsupportedReason.getOrElse("unsupported type"))
                           } else {
-                            (None, None)
+                            addFieldWithChildren(
+                              childSf, childInfo, parentIdx, parentDepth + 1, childMsgDesc, path)
                           }
-
-                        val childDefaultVal =
-                          if (childHasDefault) PbReflect.getDefaultValue(childFd) else None
-
-                        val childInfo = ProtobufFieldInfo(
-                          fieldNumber = childFieldNumber,
-                          protoTypeName = childProtoTypeName,
-                          sparkType = childSf.dataType,
-                          encoding = childEncoding,
-                          isSupported = true,
-                          unsupportedReason = None,
-                          isRequired = childIsRequired,
-                          hasDefaultValue = childHasDefault,
-                          defaultValue = childDefaultVal,
-                          enumValues = childEnumVals,
-                          enumNames = childEnumNameMap,
-                          isRepeated = childIsRepeated
-                        )
-
-                        addFieldWithChildren(
-                          childSf, childInfo, parentIdx, parentDepth + 1, childMsgDesc, path)
                       }
                     }
-                  }
-                } catch {
-                  case ex: ReflectiveOperationException =>
+                  case None =>
                     willNotWorkOnGpu(
-                      s"Failed to reflect message type for nested field '$fieldName': " +
-                        ex.getMessage)
+                      s"Nested field '$fieldName' at '$path' did not resolve to a message type")
                 }
               }
             }
@@ -517,205 +344,26 @@ object ProtobufExprShims extends org.apache.spark.internal.Logging {
 
             // Populate flattened schema variables
             val flat = flatFields.toArray
-            flatFieldNumbers = flat.map(_.fieldNumber)
-            flatParentIndices = flat.map(_.parentIdx)
-            flatDepthLevels = flat.map(_.depth)
-            flatWireTypes = flat.map(_.wireType)
-            flatOutputTypeIds = flat.map(_.outputTypeId)
-            flatEncodings = flat.map(_.encoding)
-            flatIsRepeated = flat.map(_.isRepeated)
-            flatIsRequired = flat.map(_.isRequired)
-            flatHasDefaultValue = flat.map(_.hasDefaultValue)
-            flatDefaultInts = flat.map(_.defaultInt)
-            flatDefaultFloats = flat.map(_.defaultFloat)
-            flatDefaultBools = flat.map(_.defaultBool)
-            flatDefaultStrings = flat.map(_.defaultString)
-            flatEnumValidValues = flat.map(_.enumValidValues)
-            flatEnumNames = flat.map(_.enumNames)
-          }
-        }
-
-        /**
-         * Analyze all fields in the schema and build a map of field name to ProtobufFieldInfo.
-         * Returns None if there's an error that should abort processing.
-         */
-        private def analyzeAllFields(
-            schema: StructType,
-            msgDesc: AnyRef,
-            enumsAsInts: Boolean,
-            messageName: String): Option[Map[String, ProtobufFieldInfo]] = {
-          val result = mutable.Map[String, ProtobufFieldInfo]()
-
-          for (sf <- schema.fields) {
-            val fd = PbReflect.findFieldByName(msgDesc, sf.name)
-            if (fd == null) {
-              willNotWorkOnGpu(
-                s"Protobuf field '${sf.name}' not found in message '$messageName'")
-              return None
-            }
-
-            val isRepeated = PbReflect.isRepeated(fd)
-            val isFieldRequired = PbReflect.isRequired(fd)
-            val hasDefault = PbReflect.hasDefaultValue(fd)
-            val defaultVal = if (hasDefault) PbReflect.getDefaultValue(fd) else None
-
-            val protoTypeName = typeName(PbReflect.getFieldType(fd))
-            val fieldNumber = PbReflect.getFieldNumber(fd)
-
-            val (isSupported, unsupportedReason, encoding) =
-              checkFieldSupport(sf.dataType, protoTypeName, isRepeated, enumsAsInts)
-
-            val (enumVals, enumNameMap): (Option[Set[Int]], Option[Map[Int, String]]) =
-              if (protoTypeName == "ENUM") {
-                Try {
-                  val pairs = PbReflect.getEnumValues(PbReflect.getEnumType(fd))
-                  if (enumsAsInts) {
-                    (Some(pairs.map(_._1).toSet), None)
-                  } else {
-                    (Some(pairs.map(_._1).toSet), Some(pairs.toMap))
-                  }
-                }.getOrElse((None, None))
-              } else {
-                (None, None)
-              }
-
-            result(sf.name) = ProtobufFieldInfo(
-              fieldNumber = fieldNumber,
-              protoTypeName = protoTypeName,
-              sparkType = sf.dataType,
-              encoding = encoding,
-              isSupported = isSupported,
-              unsupportedReason = unsupportedReason,
-              isRequired = isFieldRequired,
-              hasDefaultValue = hasDefault,
-              defaultValue = defaultVal,
-              enumValues = enumVals,
-              enumNames = enumNameMap,
-              isRepeated = isRepeated
-            )
-          }
-
-          Some(result.toMap)
-        }
-
-        /**
-         * Check if a field type is supported and return encoding information.
-         * @return (isSupported, unsupportedReason, encoding)
-         */
-        private def checkFieldSupport(
-            sparkType: DataType,
-            protoTypeName: String,
-            isRepeated: Boolean,
-            enumsAsInts: Boolean): (Boolean, Option[String], Int) = {
-
-          // Handle repeated fields (arrays)
-          if (isRepeated) {
-            sparkType match {
-              case ArrayType(elementType, _) =>
-                // Check if element type is supported
-                elementType match {
-                  case BooleanType | IntegerType | LongType | FloatType | DoubleType |
-                       StringType | BinaryType =>
-                    // Supported repeated scalar - determine encoding from proto type
-                    return checkScalarEncoding(elementType, protoTypeName, enumsAsInts)
-                  case _: StructType =>
-                    // Repeated nested message (array of structs) - supported on GPU
-                    return (true, None, GpuFromProtobuf.ENC_DEFAULT)
-                  case _ =>
-                    return (false, Some(s"unsupported repeated element type: $elementType"),
-                      GpuFromProtobuf.ENC_DEFAULT)
-                }
-              case _ =>
-                return (false, Some(s"repeated field should map to ArrayType, got: $sparkType"),
-                  GpuFromProtobuf.ENC_DEFAULT)
-            }
-          }
-
-          // Handle nested messages (non-repeated)
-          if (protoTypeName == "MESSAGE") {
-            sparkType match {
-              case _: StructType =>
-                return (true, None, GpuFromProtobuf.ENC_DEFAULT)
-              case _ =>
-                return (false, Some(s"nested message should map to StructType, got: $sparkType"),
-                  GpuFromProtobuf.ENC_DEFAULT)
-            }
-          }
-
-          // Check Spark type is one of the supported simple types
-          sparkType match {
-            case BooleanType | IntegerType | LongType | FloatType | DoubleType |
-                 StringType | BinaryType =>
-              // Supported Spark type, continue to check encoding
-            case other =>
-              return (false, Some(s"unsupported Spark type: $other"), GpuFromProtobuf.ENC_DEFAULT)
-          }
-
-          checkScalarEncoding(sparkType, protoTypeName, enumsAsInts)
-        }
-
-        /**
-         * Determine encoding for scalar types.
-         */
-        private def checkScalarEncoding(
-            sparkType: DataType,
-            protoTypeName: String,
-            enumsAsInts: Boolean): (Boolean, Option[String], Int) = {
-
-          // Determine encoding based on Spark type and proto type combination
-          val encoding = (sparkType, protoTypeName) match {
-            case (BooleanType, "BOOL") => Some(GpuFromProtobuf.ENC_DEFAULT)
-            case (IntegerType, "INT32" | "UINT32") => Some(GpuFromProtobuf.ENC_DEFAULT)
-            case (IntegerType, "SINT32") => Some(GpuFromProtobuf.ENC_ZIGZAG)
-            case (IntegerType, "FIXED32" | "SFIXED32") => Some(GpuFromProtobuf.ENC_FIXED)
-            case (LongType, "INT64" | "UINT64") => Some(GpuFromProtobuf.ENC_DEFAULT)
-            case (LongType, "SINT64") => Some(GpuFromProtobuf.ENC_ZIGZAG)
-            case (LongType, "FIXED64" | "SFIXED64") => Some(GpuFromProtobuf.ENC_FIXED)
-            // Spark may upcast smaller integers to LongType
-            case (LongType, "INT32" | "UINT32" | "SINT32" | "FIXED32" | "SFIXED32") =>
-              val enc = protoTypeName match {
-                case "SINT32" => GpuFromProtobuf.ENC_ZIGZAG
-                case "FIXED32" | "SFIXED32" => GpuFromProtobuf.ENC_FIXED
-                case _ => GpuFromProtobuf.ENC_DEFAULT
-              }
-              Some(enc)
-            case (FloatType, "FLOAT") => Some(GpuFromProtobuf.ENC_DEFAULT)
-            case (DoubleType, "DOUBLE") => Some(GpuFromProtobuf.ENC_DEFAULT)
-            case (StringType, "STRING") => Some(GpuFromProtobuf.ENC_DEFAULT)
-            case (BinaryType, "BYTES") => Some(GpuFromProtobuf.ENC_DEFAULT)
-            case (IntegerType, "ENUM") if enumsAsInts => Some(GpuFromProtobuf.ENC_DEFAULT)
-            case (StringType, "ENUM") if !enumsAsInts => Some(GpuFromProtobuf.ENC_ENUM_STRING)
-            case _ => None
-          }
-
-          encoding match {
-            case Some(enc) => (true, None, enc)
-            case None =>
-              (false,
-                Some(s"type mismatch: Spark $sparkType vs Protobuf $protoTypeName"),
-                GpuFromProtobuf.ENC_DEFAULT)
-          }
-        }
-
-        /**
-         * Get wire type constant for a given protobuf type name and encoding.
-         */
-        private def getWireType(protoTypeName: String, encoding: Int): Int = {
-          protoTypeName match {
-            case "BOOL" | "INT32" | "UINT32" | "SINT32" | "INT64" | "UINT64" | "SINT64" | "ENUM" =>
-              if (encoding == GpuFromProtobuf.ENC_FIXED) {
-                if (protoTypeName.contains("64")) WT_64BIT else WT_32BIT
-              } else {
-                WT_VARINT
-              }
-            case "FIXED32" | "SFIXED32" | "FLOAT" => WT_32BIT
-            case "FIXED64" | "SFIXED64" | "DOUBLE" => WT_64BIT
-            case "STRING" | "BYTES" | "MESSAGE" => WT_LEN
-            case other =>
-              willNotWorkOnGpu(
-                s"Unknown protobuf type name '$other' - cannot determine wire type; " +
-                  "falling back to CPU")
-              WT_VARINT
+            ProtobufSchemaValidator.validateFlattenedSchema(flat).fold({ reason =>
+              willNotWorkOnGpu(reason)
+              return
+            }, identity)
+            val arrays = ProtobufSchemaValidator.toFlattenedSchemaArrays(flat)
+            flatFieldNumbers = arrays.fieldNumbers
+            flatParentIndices = arrays.parentIndices
+            flatDepthLevels = arrays.depthLevels
+            flatWireTypes = arrays.wireTypes
+            flatOutputTypeIds = arrays.outputTypeIds
+            flatEncodings = arrays.encodings
+            flatIsRepeated = arrays.isRepeated
+            flatIsRequired = arrays.isRequired
+            flatHasDefaultValue = arrays.hasDefaultValue
+            flatDefaultInts = arrays.defaultInts
+            flatDefaultFloats = arrays.defaultFloats
+            flatDefaultBools = arrays.defaultBools
+            flatDefaultStrings = arrays.defaultStrings
+            flatEnumValidValues = arrays.enumValidValues
+            flatEnumNames = arrays.enumNames
           }
         }
 
@@ -1112,148 +760,4 @@ object ProtobufExprShims extends org.apache.spark.internal.Logging {
     )
   }
 
-  private def getMessageName(e: Expression): String =
-    PbReflect.invoke0[String](e, "messageName")
-
-  private def getDescriptorBytes(e: Expression): Option[Array[Byte]] = {
-    val spark35Result = Try(PbReflect.invoke0[Option[Array[Byte]]](e, "binaryFileDescriptorSet"))
-      .toOption.flatten
-    spark35Result.orElse {
-      val direct = Try(PbReflect.invoke0[Array[Byte]](e, "binaryDescriptorSet")).toOption
-      direct.orElse {
-        Try(PbReflect.invoke0[Option[Array[Byte]]](e, "binaryDescriptorSet")).toOption.flatten
-      }
-    }
-  }
-
-  private def getDescFilePath(e: Expression): Option[String] =
-    Try(PbReflect.invoke0[Option[String]](e, "descFilePath")).toOption.flatten
-
-  /**
-   * Build message descriptor using Spark's ProtobufUtils.
-   * Supports both Spark 3.4.x (descFilePath: Option[String]) and
-   * Spark 3.5+ (binaryFileDescriptorSet: Option[Array[Byte]]).
-   *
-   * @param messageName The protobuf message name
-   * @param descFilePathOrBytes Either a file path (String) or binary descriptor bytes (Array[Byte])
-   */
-  private def buildMessageDescriptorWithSparkProtobuf(
-      messageName: String,
-      descFilePathOrBytes: Either[String, Array[Byte]]): AnyRef = {
-    val cls = ShimReflectionUtils.loadClass(sparkProtobufUtilsObjectClassName)
-    val module = cls.getField("MODULE$").get(null)
-
-    descFilePathOrBytes match {
-      case Left(filePath) =>
-        // Spark 3.4.x: buildDescriptor(messageName: String, descFilePath: Option[String])
-        val m = cls.getMethod("buildDescriptor", classOf[String], classOf[scala.Option[_]])
-        m.invoke(module, messageName, Some(filePath)).asInstanceOf[AnyRef]
-      case Right(bytes) =>
-        // Spark 3.5+: buildDescriptor(messageName, binaryFileDescriptorSet)
-        val m = cls.getMethod("buildDescriptor", classOf[String], classOf[scala.Option[_]])
-        m.invoke(module, messageName, Some(bytes)).asInstanceOf[AnyRef]
-    }
-  }
-
-  private def typeName(t: AnyRef): String = {
-    if (t == null) {
-      "null"
-    } else {
-      Try(PbReflect.invoke0[String](t, "name")).getOrElse(t.toString)
-    }
-  }
-
-  private def getOptionsMap(e: Expression): Option[Map[String, String]] = {
-    Try(PbReflect.invoke0[scala.collection.Map[String, String]](e, "options"))
-      .map(_.toMap)
-      .toOption
-  }
-
-  /**
-   * Cached reflection helper for protobuf-java descriptor APIs.
-   *
-   * All protobuf descriptor method calls go through this object so that:
-   *  1. java.lang.reflect.Method objects are cached (ConcurrentHashMap)
-   *  2. Missing methods produce a clear UnsupportedOperationException with the
-   *     class name and loaded protobuf-java version, instead of a raw
-   *     NoSuchMethodException.
-   */
-  private[shims] object PbReflect {
-    import java.lang.reflect.Method
-    private val cache = new java.util.concurrent.ConcurrentHashMap[String, Method]()
-
-    private def protobufJavaVersion: String = Try {
-      val rtCls = Class.forName("com.google.protobuf.RuntimeVersion")
-      val domain = rtCls.getField("DOMAIN").get(null)
-      val major = rtCls.getField("MAJOR").get(null)
-      val minor = rtCls.getField("MINOR").get(null)
-      val patch = rtCls.getField("PATCH").get(null)
-      s"$domain-$major.$minor.$patch"
-    }.getOrElse("unknown")
-
-    private def cached(cls: Class[_], name: String, paramTypes: Class[_]*): Method = {
-      val key = s"${cls.getName}#$name(${paramTypes.map(_.getName).mkString(",")})"
-      cache.computeIfAbsent(key, _ => {
-        try {
-          cls.getMethod(name, paramTypes: _*)
-        } catch {
-          case ex: NoSuchMethodException =>
-            throw new UnsupportedOperationException(
-              s"protobuf-java method not found: ${cls.getSimpleName}.$name " +
-                s"(protobuf-java version: $protobufJavaVersion). " +
-                s"This may indicate an incompatible protobuf-java library version.",
-              ex)
-        }
-      })
-    }
-
-    def invoke0[T](obj: AnyRef, method: String): T =
-      cached(obj.getClass, method).invoke(obj).asInstanceOf[T]
-
-    def invoke1[T](obj: AnyRef, method: String, arg0Cls: Class[_], arg0: AnyRef): T =
-      cached(obj.getClass, method, arg0Cls).invoke(obj, arg0).asInstanceOf[T]
-
-    // ---- Typed helpers for common Descriptor operations ----
-
-    def findFieldByName(msgDesc: AnyRef, name: String): AnyRef =
-      invoke1[AnyRef](msgDesc, "findFieldByName", classOf[String], name)
-
-    def getFieldNumber(fd: AnyRef): Int =
-      invoke0[java.lang.Integer](fd, "getNumber").intValue()
-
-    def getFieldType(fd: AnyRef): AnyRef = invoke0[AnyRef](fd, "getType")
-
-    def isRepeated(fd: AnyRef): Boolean =
-      Try(invoke0[java.lang.Boolean](fd, "isRepeated").booleanValue()).getOrElse(false)
-
-    def isRequired(fd: AnyRef): Boolean =
-      Try(invoke0[java.lang.Boolean](fd, "isRequired").booleanValue()).getOrElse(false)
-
-    def hasDefaultValue(fd: AnyRef): Boolean =
-      Try(invoke0[java.lang.Boolean](fd, "hasDefaultValue").booleanValue()).getOrElse(false)
-
-    def getDefaultValue(fd: AnyRef): Option[AnyRef] =
-      Try(Some(invoke0[AnyRef](fd, "getDefaultValue"))).getOrElse(None)
-
-    def getMessageType(fd: AnyRef): AnyRef = invoke0[AnyRef](fd, "getMessageType")
-
-    def getEnumType(fd: AnyRef): AnyRef = invoke0[AnyRef](fd, "getEnumType")
-
-    def getEnumValues(enumType: AnyRef): Seq[(Int, String)] = {
-      import scala.collection.JavaConverters._
-      val values = invoke0[java.util.List[_]](enumType, "getValues")
-      values.asScala.map { v =>
-        val ev = v.asInstanceOf[AnyRef]
-        val num = invoke0[java.lang.Integer](ev, "getNumber").intValue()
-        val name = invoke0[String](ev, "getName")
-        (num, name)
-      }.toSeq
-    }
-
-    def getFileSyntax(msgDesc: AnyRef): String = Try {
-      val fileDesc = invoke0[AnyRef](msgDesc, "getFile")
-      val syntaxObj = invoke0[AnyRef](fileDesc, "getSyntax")
-      typeName(syntaxObj)
-    }.getOrElse("")
-  }
 }
diff --git a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/SparkProtobufCompat.scala b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/SparkProtobufCompat.scala
new file mode 100644
index 00000000000..c1c3d3cfc1f
--- /dev/null
+++ b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/SparkProtobufCompat.scala
@@ -0,0 +1,297 @@
+/*
+ * Copyright (c) 2026, NVIDIA CORPORATION.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*** spark-rapids-shim-json-lines
+{"spark": "340"}
+{"spark": "341"}
+{"spark": "342"}
+{"spark": "343"}
+{"spark": "344"}
+{"spark": "350"}
+{"spark": "351"}
+{"spark": "352"}
+{"spark": "353"}
+{"spark": "354"}
+{"spark": "355"}
+{"spark": "356"}
+{"spark": "357"}
+{"spark": "400"}
+{"spark": "401"}
+{"spark": "402"}
+{"spark": "411"}
+spark-rapids-shim-json-lines ***/
+
+package com.nvidia.spark.rapids.shims
+
+import java.lang.reflect.Method
+
+import scala.util.Try
+
+import com.nvidia.spark.rapids.ShimReflectionUtils
+
+import org.apache.spark.internal.Logging
+import org.apache.spark.sql.catalyst.expressions.Expression
+import org.apache.spark.sql.rapids.protobuf._
+
+private[shims] object SparkProtobufCompat extends Logging {
+  private[this] val sparkProtobufUtilsObjectClassName =
+    "org.apache.spark.sql.protobuf.utils.ProtobufUtils$"
+
+  val SupportedOptions: Set[String] = Set("enums.as.ints", "mode")
+
+  def extractExprInfo(e: Expression): Either[String, ProtobufExprInfo] = {
+    for {
+      messageName <- reflectMessageName(e)
+      options <- reflectOptions(e)
+      descriptorSource <- reflectDescriptorSource(e)
+    } yield ProtobufExprInfo(messageName, descriptorSource, options)
+  }
+
+  def parsePlannerOptions(options: Map[String, String]): Either[String, ProtobufPlannerOptions] = {
+    val enumsAsInts = Try(options.getOrElse("enums.as.ints", "false").toBoolean).toEither.left.map { _ =>
+      "Invalid value for from_protobuf option 'enums.as.ints': " +
+        s"'${options.getOrElse("enums.as.ints", "")}' (expected true/false)"
+    }
+    enumsAsInts.map(v =>
+      ProtobufPlannerOptions(
+        enumsAsInts = v,
+        failOnErrors = options.getOrElse("mode", "FAILFAST").equalsIgnoreCase("FAILFAST")))
+  }
+
+  def unsupportedOptions(options: Map[String, String]): Seq[String] =
+    options.keys.filterNot(SupportedOptions.contains).toSeq.sorted
+
+  def isGpuSupportedProtoSyntax(syntax: String): Boolean =
+    syntax.nonEmpty && syntax != "PROTO3" && syntax != "EDITIONS"
+
+  def resolveMessageDescriptor(exprInfo: ProtobufExprInfo): Either[String, ProtobufMessageDescriptor] = {
+    Try(buildMessageDescriptor(exprInfo.messageName, exprInfo.descriptorSource)).toEither.left.map { t =>
+      s"Failed to resolve protobuf descriptor for message '${exprInfo.messageName}': ${t.getMessage}"
+    }.map(new ReflectiveMessageDescriptor(_))
+  }
+
+  private def reflectMessageName(e: Expression): Either[String, String] =
+    Try(PbReflect.invoke0[String](e, "messageName")).toEither.left.map { t =>
+      s"Cannot read from_protobuf messageName via reflection: ${t.getMessage}"
+    }
+
+  private def reflectOptions(e: Expression): Either[String, Map[String, String]] = {
+    Try(PbReflect.invoke0[scala.collection.Map[String, String]](e, "options"))
+      .map(_.toMap)
+      .toEither.left.map { _ =>
+        "Cannot read from_protobuf options via reflection; falling back to CPU"
+      }
+  }
+
+  private def reflectDescriptorSource(e: Expression): Either[String, ProtobufDescriptorSource] = {
+    reflectDescFilePath(e).map(ProtobufDescriptorSource.DescriptorPath).orElse(
+      reflectDescriptorBytes(e).map(ProtobufDescriptorSource.DescriptorBytes)).toRight(
+      "from_protobuf requires a descriptor set (descFilePath or binaryFileDescriptorSet)")
+  }
+
+  private def reflectDescFilePath(e: Expression): Option[String] =
+    Try(PbReflect.invoke0[Option[String]](e, "descFilePath")).toOption.flatten
+
+  private def reflectDescriptorBytes(e: Expression): Option[Array[Byte]] = {
+    val spark35Result = Try(PbReflect.invoke0[Option[Array[Byte]]](e, "binaryFileDescriptorSet"))
+      .toOption.flatten
+    spark35Result.orElse {
+      val direct = Try(PbReflect.invoke0[Array[Byte]](e, "binaryDescriptorSet")).toOption
+      direct.orElse {
+        Try(PbReflect.invoke0[Option[Array[Byte]]](e, "binaryDescriptorSet")).toOption.flatten
+      }
+    }
+  }
+
+  private def buildMessageDescriptor(
+      messageName: String,
+      descriptorSource: ProtobufDescriptorSource): AnyRef = {
+    val cls = ShimReflectionUtils.loadClass(sparkProtobufUtilsObjectClassName)
+    val module = cls.getField("MODULE$").get(null)
+    val buildMethod = cls.getMethod("buildDescriptor", classOf[String], classOf[scala.Option[_]])
+
+    descriptorSource match {
+      case ProtobufDescriptorSource.DescriptorPath(filePath) =>
+        buildMethod.invoke(module, messageName, Some(filePath)).asInstanceOf[AnyRef]
+      case ProtobufDescriptorSource.DescriptorBytes(bytes) =>
+        buildMethod.invoke(module, messageName, Some(bytes)).asInstanceOf[AnyRef]
+    }
+  }
+
+  private def typeName(t: AnyRef): String =
+    if (t == null) "null" else Try(PbReflect.invoke0[String](t, "name")).getOrElse(t.toString)
+
+  private final class ReflectiveMessageDescriptor(raw: AnyRef) extends ProtobufMessageDescriptor {
+    override lazy val syntax: String = PbReflect.getFileSyntax(raw, typeName)
+
+    override def findField(name: String): Option[ProtobufFieldDescriptor] =
+      Option(PbReflect.findFieldByName(raw, name)).map(new ReflectiveFieldDescriptor(_))
+  }
+
+  private final class ReflectiveFieldDescriptor(raw: AnyRef) extends ProtobufFieldDescriptor {
+    override lazy val name: String = PbReflect.invoke0[String](raw, "getName")
+    override lazy val fieldNumber: Int = PbReflect.getFieldNumber(raw)
+    override lazy val protoTypeName: String = typeName(PbReflect.getFieldType(raw))
+    override lazy val isRepeated: Boolean = PbReflect.isRepeated(raw)
+    override lazy val isRequired: Boolean = PbReflect.isRequired(raw)
+    override lazy val enumMetadata: Option[ProtobufEnumMetadata] =
+      if (protoTypeName == "ENUM") {
+        Some(ProtobufEnumMetadata(PbReflect.getEnumValues(PbReflect.getEnumType(raw))))
+      } else {
+        None
+      }
+    override lazy val defaultValue: Option[ProtobufDefaultValue] =
+      if (PbReflect.hasDefaultValue(raw)) {
+        PbReflect.getDefaultValue(raw).map(toDefaultValue(_, protoTypeName, enumMetadata))
+      } else {
+        None
+      }
+    override lazy val messageDescriptor: Option[ProtobufMessageDescriptor] =
+      if (protoTypeName == "MESSAGE") {
+        Some(new ReflectiveMessageDescriptor(PbReflect.getMessageType(raw)))
+      } else {
+        None
+      }
+  }
+
+  private def toDefaultValue(
+      rawDefault: AnyRef,
+      protoTypeName: String,
+      enumMetadata: Option[ProtobufEnumMetadata]): ProtobufDefaultValue = protoTypeName match {
+    case "BOOL" =>
+      ProtobufDefaultValue.BoolValue(rawDefault.asInstanceOf[java.lang.Boolean].booleanValue())
+    case "FLOAT" =>
+      ProtobufDefaultValue.FloatValue(rawDefault.asInstanceOf[java.lang.Float].floatValue())
+    case "DOUBLE" =>
+      ProtobufDefaultValue.DoubleValue(rawDefault.asInstanceOf[java.lang.Double].doubleValue())
+    case "STRING" =>
+      ProtobufDefaultValue.StringValue(if (rawDefault == null) null else rawDefault.toString)
+    case "BYTES" =>
+      ProtobufDefaultValue.BinaryValue(extractBytes(rawDefault))
+    case "ENUM" =>
+      val number = extractNumber(rawDefault).intValue()
+      enumMetadata.map(_.enumDefault(number))
+        .getOrElse(ProtobufDefaultValue.EnumValue(number, rawDefault.toString))
+    case "INT32" | "UINT32" | "SINT32" | "FIXED32" | "SFIXED32" |
+         "INT64" | "UINT64" | "SINT64" | "FIXED64" | "SFIXED64" =>
+      ProtobufDefaultValue.IntValue(extractNumber(rawDefault).longValue())
+    case other =>
+      throw new IllegalStateException(
+        s"Unsupported protobuf default value type '$other' for value ${rawDefault.toString}")
+  }
+
+  private def extractNumber(rawDefault: AnyRef): java.lang.Number = rawDefault match {
+    case n: java.lang.Number => n
+    case ref: AnyRef =>
+      Try {
+        ref.getClass.getMethod("getNumber").invoke(ref).asInstanceOf[java.lang.Number]
+      }.getOrElse {
+        throw new IllegalStateException(
+          s"Unsupported protobuf numeric default value class: ${ref.getClass.getName}")
+      }
+    case _ =>
+      throw new IllegalStateException("Unexpected protobuf numeric default value")
+  }
+
+  private def extractBytes(rawDefault: AnyRef): Array[Byte] = rawDefault match {
+    case bytes: Array[Byte] => bytes
+    case ref: AnyRef =>
+      Try {
+        ref.getClass.getMethod("toByteArray").invoke(ref).asInstanceOf[Array[Byte]]
+      }.getOrElse {
+        throw new IllegalStateException(
+          s"Unsupported protobuf bytes default value class: ${ref.getClass.getName}")
+      }
+    case _ =>
+      throw new IllegalStateException("Unexpected protobuf bytes default value")
+  }
+
+  private object PbReflect {
+    private val cache = new java.util.concurrent.ConcurrentHashMap[String, Method]()
+
+    private def protobufJavaVersion: String = Try {
+      val rtCls = Class.forName("com.google.protobuf.RuntimeVersion")
+      val domain = rtCls.getField("DOMAIN").get(null)
+      val major = rtCls.getField("MAJOR").get(null)
+      val minor = rtCls.getField("MINOR").get(null)
+      val patch = rtCls.getField("PATCH").get(null)
+      s"$domain-$major.$minor.$patch"
+    }.getOrElse("unknown")
+
+    private def cached(cls: Class[_], name: String, paramTypes: Class[_]*): Method = {
+      val key = s"${cls.getName}#$name(${paramTypes.map(_.getName).mkString(",")})"
+      cache.computeIfAbsent(key, _ => {
+        try {
+          cls.getMethod(name, paramTypes: _*)
+        } catch {
+          case ex: NoSuchMethodException =>
+            throw new UnsupportedOperationException(
+              s"protobuf-java method not found: ${cls.getSimpleName}.$name " +
+                s"(protobuf-java version: $protobufJavaVersion). " +
+                s"This may indicate an incompatible protobuf-java library version.",
+              ex)
+        }
+      })
+    }
+
+    def invoke0[T](obj: AnyRef, method: String): T =
+      cached(obj.getClass, method).invoke(obj).asInstanceOf[T]
+
+    def invoke1[T](obj: AnyRef, method: String, arg0Cls: Class[_], arg0: AnyRef): T =
+      cached(obj.getClass, method, arg0Cls).invoke(obj, arg0).asInstanceOf[T]
+
+    def findFieldByName(msgDesc: AnyRef, name: String): AnyRef =
+      invoke1[AnyRef](msgDesc, "findFieldByName", classOf[String], name)
+
+    def getFieldNumber(fd: AnyRef): Int =
+      invoke0[java.lang.Integer](fd, "getNumber").intValue()
+
+    def getFieldType(fd: AnyRef): AnyRef = invoke0[AnyRef](fd, "getType")
+
+    def isRepeated(fd: AnyRef): Boolean =
+      invoke0[java.lang.Boolean](fd, "isRepeated").booleanValue()
+
+    def isRequired(fd: AnyRef): Boolean =
+      invoke0[java.lang.Boolean](fd, "isRequired").booleanValue()
+
+    def hasDefaultValue(fd: AnyRef): Boolean =
+      invoke0[java.lang.Boolean](fd, "hasDefaultValue").booleanValue()
+
+    def getDefaultValue(fd: AnyRef): Option[AnyRef] =
+      Option(invoke0[AnyRef](fd, "getDefaultValue"))
+
+    def getMessageType(fd: AnyRef): AnyRef = invoke0[AnyRef](fd, "getMessageType")
+
+    def getEnumType(fd: AnyRef): AnyRef = invoke0[AnyRef](fd, "getEnumType")
+
+    def getEnumValues(enumType: AnyRef): Seq[ProtobufEnumValue] = {
+      import scala.collection.JavaConverters._
+      val values = invoke0[java.util.List[_]](enumType, "getValues")
+      values.asScala.map { v =>
+        val ev = v.asInstanceOf[AnyRef]
+        val num = invoke0[java.lang.Integer](ev, "getNumber").intValue()
+        val enumName = invoke0[String](ev, "getName")
+        ProtobufEnumValue(num, enumName)
+      }.toSeq
+    }
+
+    def getFileSyntax(msgDesc: AnyRef, typeNameFn: AnyRef => String): String = Try {
+      val fileDesc = invoke0[AnyRef](msgDesc, "getFile")
+      val syntaxObj = invoke0[AnyRef](fileDesc, "getSyntax")
+      typeNameFn(syntaxObj)
+    }.getOrElse("")
+  }
+}
diff --git a/sql-plugin/src/test/scala/com/nvidia/spark/rapids/shims/ProtobufExprShimsSuite.scala b/sql-plugin/src/test/scala/com/nvidia/spark/rapids/shims/ProtobufExprShimsSuite.scala
new file mode 100644
index 00000000000..26b2a063e30
--- /dev/null
+++ b/sql-plugin/src/test/scala/com/nvidia/spark/rapids/shims/ProtobufExprShimsSuite.scala
@@ -0,0 +1,207 @@
+/*
+ * Copyright (c) 2026, NVIDIA CORPORATION.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.nvidia.spark.rapids.shims
+
+import org.scalatest.funsuite.AnyFunSuite
+
+import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, ExprCode}
+import org.apache.spark.sql.catalyst.expressions.{Expression, UnaryExpression}
+import org.apache.spark.sql.rapids.GpuFromProtobuf
+import org.apache.spark.sql.rapids.protobuf._
+import org.apache.spark.sql.types._
+
+class ProtobufExprShimsSuite extends AnyFunSuite {
+  private val outputSchema = StructType(Seq(
+    StructField("id", IntegerType, nullable = true),
+    StructField("name", StringType, nullable = true)))
+
+  private case class FakeExprChild() extends Expression {
+    override def children: Seq[Expression] = Nil
+    override def nullable: Boolean = true
+    override def dataType: DataType = BinaryType
+    override def eval(input: org.apache.spark.sql.catalyst.InternalRow): Any = null
+    override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode =
+      throw new UnsupportedOperationException("not needed")
+    override protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]): Expression = {
+      assert(newChildren.isEmpty)
+      this
+    }
+  }
+
+  private abstract class FakeBaseProtobufExpr(childExpr: Expression) extends UnaryExpression {
+    override def child: Expression = childExpr
+    override def nullable: Boolean = true
+    override def dataType: DataType = outputSchema
+    override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode =
+      throw new UnsupportedOperationException("not needed")
+    override protected def withNewChildInternal(newChild: Expression): Expression = this
+  }
+
+  private case class FakePathProtobufExpr(override val child: Expression)
+      extends FakeBaseProtobufExpr(child) {
+    def messageName: String = "test.Message"
+    def descFilePath: Option[String] = Some("/tmp/test.desc")
+    def options: scala.collection.Map[String, String] = Map("mode" -> "FAILFAST")
+  }
+
+  private case class FakeBytesProtobufExpr(override val child: Expression)
+      extends FakeBaseProtobufExpr(child) {
+    def messageName: String = "test.Message"
+    def binaryDescriptorSet: Array[Byte] = Array[Byte](1, 2, 3)
+    def options: scala.collection.Map[String, String] =
+      Map("mode" -> "PERMISSIVE", "enums.as.ints" -> "true")
+  }
+
+  private case class FakeMissingOptionsExpr(override val child: Expression)
+      extends FakeBaseProtobufExpr(child) {
+    def messageName: String = "test.Message"
+    def descFilePath: Option[String] = Some("/tmp/test.desc")
+  }
+
+  private case class FakeMessageDescriptor(
+      syntax: String,
+      fields: Map[String, ProtobufFieldDescriptor]) extends ProtobufMessageDescriptor {
+    override def findField(name: String): Option[ProtobufFieldDescriptor] = fields.get(name)
+  }
+
+  private case class FakeFieldDescriptor(
+      name: String,
+      fieldNumber: Int,
+      protoTypeName: String,
+      isRepeated: Boolean = false,
+      isRequired: Boolean = false,
+      defaultValue: Option[ProtobufDefaultValue] = None,
+      enumMetadata: Option[ProtobufEnumMetadata] = None,
+      messageDescriptor: Option[ProtobufMessageDescriptor] = None) extends ProtobufFieldDescriptor
+
+  test("compat extracts descriptor path and options from legacy expression") {
+    val exprInfo = SparkProtobufCompat.extractExprInfo(FakePathProtobufExpr(FakeExprChild()))
+    assert(exprInfo.isRight)
+    val info = exprInfo.toOption.get
+    assert(info.messageName == "test.Message")
+    assert(info.options == Map("mode" -> "FAILFAST"))
+    assert(info.descriptorSource ==
+      ProtobufDescriptorSource.DescriptorPath("/tmp/test.desc"))
+  }
+
+  test("compat extracts binary descriptor source and planner options") {
+    val exprInfo = SparkProtobufCompat.extractExprInfo(FakeBytesProtobufExpr(FakeExprChild()))
+    assert(exprInfo.isRight)
+    val info = exprInfo.toOption.get
+    info.descriptorSource match {
+      case ProtobufDescriptorSource.DescriptorBytes(bytes) =>
+        assert(bytes.sameElements(Array[Byte](1, 2, 3)))
+      case other =>
+        fail(s"Unexpected descriptor source: $other")
+    }
+    val plannerOptions = SparkProtobufCompat.parsePlannerOptions(info.options)
+    assert(plannerOptions == Right(ProtobufPlannerOptions(enumsAsInts = true, failOnErrors = false)))
+  }
+
+  test("compat reports missing options accessor as cpu fallback reason") {
+    val exprInfo = SparkProtobufCompat.extractExprInfo(FakeMissingOptionsExpr(FakeExprChild()))
+    assert(exprInfo.left.toOption.exists(_.contains("Cannot read from_protobuf options via reflection")))
+  }
+
+  test("compat detects unsupported options and proto3 syntax") {
+    assert(SparkProtobufCompat.unsupportedOptions(Map("mode" -> "FAILFAST", "foo" -> "bar")) ==
+      Seq("foo"))
+    assert(!SparkProtobufCompat.isGpuSupportedProtoSyntax("PROTO3"))
+    assert(!SparkProtobufCompat.isGpuSupportedProtoSyntax("EDITIONS"))
+    assert(!SparkProtobufCompat.isGpuSupportedProtoSyntax(""))
+    assert(SparkProtobufCompat.isGpuSupportedProtoSyntax("PROTO2"))
+  }
+
+  test("extractor preserves typed enum defaults") {
+    val enumMeta = ProtobufEnumMetadata(Seq(
+      ProtobufEnumValue(0, "UNKNOWN"),
+      ProtobufEnumValue(1, "EN"),
+      ProtobufEnumValue(2, "ZH")))
+    val msgDesc = FakeMessageDescriptor(
+      syntax = "PROTO2",
+      fields = Map(
+        "language" -> FakeFieldDescriptor(
+          name = "language",
+          fieldNumber = 1,
+          protoTypeName = "ENUM",
+          defaultValue = Some(ProtobufDefaultValue.EnumValue(1, "EN")),
+          enumMetadata = Some(enumMeta))))
+    val schema = StructType(Seq(StructField("language", StringType, nullable = true)))
+
+    val infos = ProtobufSchemaExtractor.analyzeAllFields(
+      schema, msgDesc, enumsAsInts = false, "test.Message")
+
+    assert(infos.isRight)
+    assert(infos.toOption.get("language").defaultValue.contains(
+      ProtobufDefaultValue.EnumValue(1, "EN")))
+  }
+
+  test("validator encodes enum-string defaults into both numeric and string payloads") {
+    val enumMeta = ProtobufEnumMetadata(Seq(
+      ProtobufEnumValue(0, "UNKNOWN"),
+      ProtobufEnumValue(1, "EN")))
+    val info = ProtobufFieldInfo(
+      fieldNumber = 2,
+      protoTypeName = "ENUM",
+      sparkType = StringType,
+      encoding = GpuFromProtobuf.ENC_ENUM_STRING,
+      isSupported = true,
+      unsupportedReason = None,
+      isRequired = false,
+      defaultValue = Some(ProtobufDefaultValue.EnumValue(1, "EN")),
+      enumMetadata = Some(enumMeta),
+      isRepeated = false)
+
+    val flat = ProtobufSchemaValidator.toFlattenedFieldDescriptor(
+      path = "common.language",
+      field = StructField("language", StringType, nullable = true),
+      fieldInfo = info,
+      parentIdx = 0,
+      depth = 1,
+      outputTypeId = 6)
+
+    assert(flat.isRight)
+    assert(flat.toOption.get.defaultInt == 1L)
+    assert(new String(flat.toOption.get.defaultString, "UTF-8") == "EN")
+    assert(flat.toOption.get.enumValidValues.sameElements(Array(0, 1)))
+    assert(flat.toOption.get.enumNames.map(new String(_, "UTF-8")).sameElements(Array("UNKNOWN", "EN")))
+  }
+
+  test("validator rejects enum-string field without enum metadata") {
+    val info = ProtobufFieldInfo(
+      fieldNumber = 2,
+      protoTypeName = "ENUM",
+      sparkType = StringType,
+      encoding = GpuFromProtobuf.ENC_ENUM_STRING,
+      isSupported = true,
+      unsupportedReason = None,
+      isRequired = false,
+      defaultValue = Some(ProtobufDefaultValue.EnumValue(1, "EN")),
+      enumMetadata = None,
+      isRepeated = false)
+
+    val flat = ProtobufSchemaValidator.toFlattenedFieldDescriptor(
+      path = "common.language",
+      field = StructField("language", StringType, nullable = true),
+      fieldInfo = info,
+      parentIdx = 0,
+      depth = 1,
+      outputTypeId = 6)
+
+    assert(flat.left.toOption.exists(_.contains("missing enum metadata")))
+  }
+}

From 2c47aeefc58993ff4e2fa6e2c32bc89c5ab9967a Mon Sep 17 00:00:00 2001
From: Haoyang Li <haoyangl@nvidia.com>
Date: Thu, 12 Mar 2026 11:23:13 +0800
Subject: [PATCH 28/33] equals and hashcode

Signed-off-by: Haoyang Li <haoyangl@nvidia.com>
---
 .../spark/sql/rapids/GpuFromProtobuf.scala    | 53 ++++++++++++++++
 .../protobuf/ProtobufSchemaValidator.scala    |  7 ++-
 .../rapids/shims/SparkProtobufCompat.scala    | 28 ++++++---
 .../rapids/shims/ProtobufExprShimsSuite.scala | 62 +++++++++++++++++--
 4 files changed, 134 insertions(+), 16 deletions(-)

diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuFromProtobuf.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuFromProtobuf.scala
index 10d5116c40c..e9d40a4b868 100644
--- a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuFromProtobuf.scala
+++ b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuFromProtobuf.scala
@@ -16,6 +16,8 @@
 
 package org.apache.spark.sql.rapids
 
+import java.util.Arrays
+
 import ai.rapids.cudf
 import ai.rapids.cudf.{BinaryOp, CudfException, DType}
 import com.nvidia.spark.rapids.{GpuColumnVector, GpuUnaryExpression}
@@ -90,6 +92,51 @@ case class GpuFromProtobuf(
 
   override def nullable: Boolean = true
 
+  override def equals(other: Any): Boolean = other match {
+    case that: GpuFromProtobuf =>
+      decodedSchema == that.decodedSchema &&
+        Arrays.equals(fieldNumbers, that.fieldNumbers) &&
+        Arrays.equals(parentIndices, that.parentIndices) &&
+        Arrays.equals(depthLevels, that.depthLevels) &&
+        Arrays.equals(wireTypes, that.wireTypes) &&
+        Arrays.equals(outputTypeIds, that.outputTypeIds) &&
+        Arrays.equals(encodings, that.encodings) &&
+        Arrays.equals(isRepeated, that.isRepeated) &&
+        Arrays.equals(isRequired, that.isRequired) &&
+        Arrays.equals(hasDefaultValue, that.hasDefaultValue) &&
+        Arrays.equals(defaultInts, that.defaultInts) &&
+        Arrays.equals(defaultFloats, that.defaultFloats) &&
+        Arrays.equals(defaultBools, that.defaultBools) &&
+        GpuFromProtobuf.deepEquals(defaultStrings, that.defaultStrings) &&
+        GpuFromProtobuf.deepEquals(enumValidValues, that.enumValidValues) &&
+        GpuFromProtobuf.deepEquals(enumNames, that.enumNames) &&
+        failOnErrors == that.failOnErrors &&
+        child == that.child
+    case _ => false
+  }
+
+  override def hashCode(): Int = {
+    var result = decodedSchema.hashCode()
+    result = 31 * result + Arrays.hashCode(fieldNumbers)
+    result = 31 * result + Arrays.hashCode(parentIndices)
+    result = 31 * result + Arrays.hashCode(depthLevels)
+    result = 31 * result + Arrays.hashCode(wireTypes)
+    result = 31 * result + Arrays.hashCode(outputTypeIds)
+    result = 31 * result + Arrays.hashCode(encodings)
+    result = 31 * result + Arrays.hashCode(isRepeated)
+    result = 31 * result + Arrays.hashCode(isRequired)
+    result = 31 * result + Arrays.hashCode(hasDefaultValue)
+    result = 31 * result + Arrays.hashCode(defaultInts)
+    result = 31 * result + Arrays.hashCode(defaultFloats)
+    result = 31 * result + Arrays.hashCode(defaultBools)
+    result = 31 * result + GpuFromProtobuf.deepHashCode(defaultStrings)
+    result = 31 * result + GpuFromProtobuf.deepHashCode(enumValidValues)
+    result = 31 * result + GpuFromProtobuf.deepHashCode(enumNames)
+    result = 31 * result + failOnErrors.hashCode()
+    result = 31 * result + child.hashCode()
+    result
+  }
+
   @transient private lazy val protobufSchema = new ProtobufSchemaDescriptor(
     fieldNumbers, parentIndices, depthLevels, wireTypes, outputTypeIds, encodings,
     isRepeated, isRequired, hasDefaultValue, defaultInts, defaultFloats, defaultBools,
@@ -145,4 +192,10 @@ object GpuFromProtobuf {
    * Check if a Spark DataType is supported by the GPU protobuf decoder.
    */
   def isTypeSupported(dt: DataType): Boolean = sparkTypeToCudfIdOpt(dt).isDefined
+
+  private def deepEquals[T](left: Array[T], right: Array[T]): Boolean =
+    Arrays.deepEquals(left.asInstanceOf[Array[Object]], right.asInstanceOf[Array[Object]])
+
+  private def deepHashCode[T](arr: Array[T]): Int =
+    Arrays.deepHashCode(arr.asInstanceOf[Array[Object]])
 }
diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/protobuf/ProtobufSchemaValidator.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/protobuf/ProtobufSchemaValidator.scala
index eb554336ff3..ba7be5b34cf 100644
--- a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/protobuf/ProtobufSchemaValidator.scala
+++ b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/protobuf/ProtobufSchemaValidator.scala
@@ -34,7 +34,9 @@ object ProtobufSchemaValidator {
       depth: Int,
       outputTypeId: Int): Either[String, FlattenedFieldDescriptor] = {
     validateFieldInfo(path, field, fieldInfo).flatMap { _ =>
-      ProtobufSchemaExtractor.getWireType(fieldInfo.protoTypeName, fieldInfo.encoding).map { wireType =>
+      ProtobufSchemaExtractor
+        .getWireType(fieldInfo.protoTypeName, fieldInfo.encoding)
+        .map { wireType =>
         val defaults = encodeDefaultValue(path, field.dataType, fieldInfo)
         val enumValidValues = fieldInfo.enumMetadata.map(_.validValues).orNull
         val enumNames =
@@ -88,7 +90,8 @@ object ProtobufSchemaValidator {
     Right(())
   }
 
-  def toFlattenedSchemaArrays(flatFields: Array[FlattenedFieldDescriptor]): FlattenedSchemaArrays = {
+  def toFlattenedSchemaArrays(
+      flatFields: Array[FlattenedFieldDescriptor]): FlattenedSchemaArrays = {
     FlattenedSchemaArrays(
       fieldNumbers = flatFields.map(_.fieldNumber),
       parentIndices = flatFields.map(_.parentIdx),
diff --git a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/SparkProtobufCompat.scala b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/SparkProtobufCompat.scala
index c1c3d3cfc1f..9785187dc0b 100644
--- a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/SparkProtobufCompat.scala
+++ b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/SparkProtobufCompat.scala
@@ -60,11 +60,15 @@ private[shims] object SparkProtobufCompat extends Logging {
     } yield ProtobufExprInfo(messageName, descriptorSource, options)
   }
 
-  def parsePlannerOptions(options: Map[String, String]): Either[String, ProtobufPlannerOptions] = {
-    val enumsAsInts = Try(options.getOrElse("enums.as.ints", "false").toBoolean).toEither.left.map { _ =>
-      "Invalid value for from_protobuf option 'enums.as.ints': " +
-        s"'${options.getOrElse("enums.as.ints", "")}' (expected true/false)"
-    }
+  def parsePlannerOptions(
+      options: Map[String, String]): Either[String, ProtobufPlannerOptions] = {
+    val enumsAsInts = Try(options.getOrElse("enums.as.ints", "false").toBoolean)
+      .toEither
+      .left
+      .map { _ =>
+        "Invalid value for from_protobuf option 'enums.as.ints': " +
+          s"'${options.getOrElse("enums.as.ints", "")}' (expected true/false)"
+      }
     enumsAsInts.map(v =>
       ProtobufPlannerOptions(
         enumsAsInts = v,
@@ -77,10 +81,16 @@ private[shims] object SparkProtobufCompat extends Logging {
   def isGpuSupportedProtoSyntax(syntax: String): Boolean =
     syntax.nonEmpty && syntax != "PROTO3" && syntax != "EDITIONS"
 
-  def resolveMessageDescriptor(exprInfo: ProtobufExprInfo): Either[String, ProtobufMessageDescriptor] = {
-    Try(buildMessageDescriptor(exprInfo.messageName, exprInfo.descriptorSource)).toEither.left.map { t =>
-      s"Failed to resolve protobuf descriptor for message '${exprInfo.messageName}': ${t.getMessage}"
-    }.map(new ReflectiveMessageDescriptor(_))
+  def resolveMessageDescriptor(
+      exprInfo: ProtobufExprInfo): Either[String, ProtobufMessageDescriptor] = {
+    Try(buildMessageDescriptor(exprInfo.messageName, exprInfo.descriptorSource))
+      .toEither
+      .left
+      .map { t =>
+        s"Failed to resolve protobuf descriptor for message '${exprInfo.messageName}': " +
+          s"${t.getMessage}"
+      }
+      .map(new ReflectiveMessageDescriptor(_))
   }
 
   private def reflectMessageName(e: Expression): Either[String, String] =
diff --git a/sql-plugin/src/test/scala/com/nvidia/spark/rapids/shims/ProtobufExprShimsSuite.scala b/sql-plugin/src/test/scala/com/nvidia/spark/rapids/shims/ProtobufExprShimsSuite.scala
index 26b2a063e30..c38ab916f36 100644
--- a/sql-plugin/src/test/scala/com/nvidia/spark/rapids/shims/ProtobufExprShimsSuite.scala
+++ b/sql-plugin/src/test/scala/com/nvidia/spark/rapids/shims/ProtobufExprShimsSuite.scala
@@ -18,8 +18,8 @@ package com.nvidia.spark.rapids.shims
 
 import org.scalatest.funsuite.AnyFunSuite
 
-import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, ExprCode}
 import org.apache.spark.sql.catalyst.expressions.{Expression, UnaryExpression}
+import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, ExprCode}
 import org.apache.spark.sql.rapids.GpuFromProtobuf
 import org.apache.spark.sql.rapids.protobuf._
 import org.apache.spark.sql.types._
@@ -36,7 +36,8 @@ class ProtobufExprShimsSuite extends AnyFunSuite {
     override def eval(input: org.apache.spark.sql.catalyst.InternalRow): Any = null
     override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode =
       throw new UnsupportedOperationException("not needed")
-    override protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]): Expression = {
+    override protected def withNewChildrenInternal(
+        newChildren: IndexedSeq[Expression]): Expression = {
       assert(newChildren.isEmpty)
       this
     }
@@ -109,12 +110,14 @@ class ProtobufExprShimsSuite extends AnyFunSuite {
         fail(s"Unexpected descriptor source: $other")
     }
     val plannerOptions = SparkProtobufCompat.parsePlannerOptions(info.options)
-    assert(plannerOptions == Right(ProtobufPlannerOptions(enumsAsInts = true, failOnErrors = false)))
+    assert(plannerOptions ==
+      Right(ProtobufPlannerOptions(enumsAsInts = true, failOnErrors = false)))
   }
 
   test("compat reports missing options accessor as cpu fallback reason") {
     val exprInfo = SparkProtobufCompat.extractExprInfo(FakeMissingOptionsExpr(FakeExprChild()))
-    assert(exprInfo.left.toOption.exists(_.contains("Cannot read from_protobuf options via reflection")))
+    assert(exprInfo.left.toOption.exists(
+      _.contains("Cannot read from_protobuf options via reflection")))
   }
 
   test("compat detects unsupported options and proto3 syntax") {
@@ -178,7 +181,9 @@ class ProtobufExprShimsSuite extends AnyFunSuite {
     assert(flat.toOption.get.defaultInt == 1L)
     assert(new String(flat.toOption.get.defaultString, "UTF-8") == "EN")
     assert(flat.toOption.get.enumValidValues.sameElements(Array(0, 1)))
-    assert(flat.toOption.get.enumNames.map(new String(_, "UTF-8")).sameElements(Array("UNKNOWN", "EN")))
+    assert(flat.toOption.get.enumNames
+      .map(new String(_, "UTF-8"))
+      .sameElements(Array("UNKNOWN", "EN")))
   }
 
   test("validator rejects enum-string field without enum metadata") {
@@ -204,4 +209,51 @@ class ProtobufExprShimsSuite extends AnyFunSuite {
 
     assert(flat.left.toOption.exists(_.contains("missing enum metadata")))
   }
+
+  test("GpuFromProtobuf semantic equality is content-based for schema arrays") {
+    def emptyEnumNames: Array[Array[Byte]] = Array.empty[Array[Byte]]
+
+    val expr1 = GpuFromProtobuf(
+      decodedSchema = outputSchema,
+      fieldNumbers = Array(1, 2),
+      parentIndices = Array(-1, -1),
+      depthLevels = Array(0, 0),
+      wireTypes = Array(0, 2),
+      outputTypeIds = Array(3, 6),
+      encodings = Array(0, 0),
+      isRepeated = Array(false, false),
+      isRequired = Array(false, false),
+      hasDefaultValue = Array(false, false),
+      defaultInts = Array(0L, 0L),
+      defaultFloats = Array(0.0, 0.0),
+      defaultBools = Array(false, false),
+      defaultStrings = Array(Array.emptyByteArray, Array.emptyByteArray),
+      enumValidValues = Array(Array.emptyIntArray, Array.emptyIntArray),
+      enumNames = Array(emptyEnumNames, emptyEnumNames),
+      failOnErrors = true,
+      child = FakeExprChild())
+
+    val expr2 = GpuFromProtobuf(
+      decodedSchema = outputSchema,
+      fieldNumbers = Array(1, 2),
+      parentIndices = Array(-1, -1),
+      depthLevels = Array(0, 0),
+      wireTypes = Array(0, 2),
+      outputTypeIds = Array(3, 6),
+      encodings = Array(0, 0),
+      isRepeated = Array(false, false),
+      isRequired = Array(false, false),
+      hasDefaultValue = Array(false, false),
+      defaultInts = Array(0L, 0L),
+      defaultFloats = Array(0.0, 0.0),
+      defaultBools = Array(false, false),
+      defaultStrings = Array(Array.emptyByteArray, Array.emptyByteArray),
+      enumValidValues = Array(Array.emptyIntArray, Array.emptyIntArray),
+      enumNames = Array(emptyEnumNames.map(identity), emptyEnumNames.map(identity)),
+      failOnErrors = true,
+      child = FakeExprChild())
+
+    assert(expr1.semanticEquals(expr2))
+    assert(expr1.semanticHash() == expr2.semanticHash())
+  }
 }

From cea7594728a98af6d54788d9986bb369174bec8b Mon Sep 17 00:00:00 2001
From: Haoyang Li <haoyangl@nvidia.com>
Date: Thu, 12 Mar 2026 14:45:45 +0800
Subject: [PATCH 29/33] address comments

Signed-off-by: Haoyang Li <haoyangl@nvidia.com>
---
 .../src/main/python/protobuf_test.py          | 370 ++++++++++++++++++
 .../rapids/shims/ProtobufExprShims.scala      | 156 ++++----
 .../rapids/shims/SparkProtobufCompat.scala    |   7 +
 .../rapids/shims/ProtobufExprShimsSuite.scala |  34 ++
 4 files changed, 499 insertions(+), 68 deletions(-)

diff --git a/integration_tests/src/main/python/protobuf_test.py b/integration_tests/src/main/python/protobuf_test.py
index 5fce5bd76c3..8848220e7f2 100644
--- a/integration_tests/src/main/python/protobuf_test.py
+++ b/integration_tests/src/main/python/protobuf_test.py
@@ -1155,6 +1155,256 @@ def run_on_spark(spark):
     assert_gpu_and_cpu_are_equal_collect(run_on_spark)
 
 
+def _build_repeated_message_enum_descriptor_set_bytes(spark):
+    """
+    message ContainerWithPriorityItems {
+        optional int32 id = 1;
+        repeated ItemWithPriority items = 2;
+        optional string title = 3;
+    }
+    message ItemWithPriority {
+        optional Priority priority = 1;
+        optional int32 count = 2;
+    }
+    enum Priority { UNKNOWN = 0; FOO = 1; BAR = 2; }
+    """
+    D, fd = _new_proto2_file(spark, "repeated_message_enum.proto")
+    label_opt = D.FieldDescriptorProto.Label.LABEL_OPTIONAL
+    label_rep = D.FieldDescriptorProto.Label.LABEL_REPEATED
+
+    item_msg = D.DescriptorProto.newBuilder().setName("ItemWithPriority")
+    enum_type = D.EnumDescriptorProto.newBuilder().setName("Priority")
+    enum_type.addValue(D.EnumValueDescriptorProto.newBuilder().setName("UNKNOWN").setNumber(0).build())
+    enum_type.addValue(D.EnumValueDescriptorProto.newBuilder().setName("FOO").setNumber(1).build())
+    enum_type.addValue(D.EnumValueDescriptorProto.newBuilder().setName("BAR").setNumber(2).build())
+    item_msg.addEnumType(enum_type.build())
+    item_msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("priority").setNumber(1).setLabel(label_opt)
+            .setType(D.FieldDescriptorProto.Type.TYPE_ENUM)
+            .setTypeName(".test.ItemWithPriority.Priority").build())
+    item_msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("count").setNumber(2).setLabel(label_opt)
+            .setType(D.FieldDescriptorProto.Type.TYPE_INT32).build())
+    fd.addMessageType(item_msg.build())
+
+    container_msg = D.DescriptorProto.newBuilder().setName("ContainerWithPriorityItems")
+    container_msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("id").setNumber(1).setLabel(label_opt)
+            .setType(D.FieldDescriptorProto.Type.TYPE_INT32).build())
+    container_msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("items").setNumber(2).setLabel(label_rep)
+            .setType(D.FieldDescriptorProto.Type.TYPE_MESSAGE)
+            .setTypeName(".test.ItemWithPriority").build())
+    container_msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("title").setNumber(3).setLabel(label_opt)
+            .setType(D.FieldDescriptorProto.Type.TYPE_STRING).build())
+    fd.addMessageType(container_msg.build())
+
+    fds = D.FileDescriptorSet.newBuilder().addFile(fd.build()).build()
+    return bytes(fds.toByteArray())
+
+
+@pytest.mark.skipif(is_before_spark_340(), reason="from_protobuf is Spark 3.4.0+")
+@ignore_order(local=True)
+def test_from_protobuf_repeated_message_child_enum_string(
+        spark_tmp_path, from_protobuf_fn):
+    desc_path, desc_bytes = _setup_protobuf_desc(
+        spark_tmp_path, "repeated_message_enum.desc",
+        _build_repeated_message_enum_descriptor_set_bytes)
+    message_name = "test.ContainerWithPriorityItems"
+
+    item_foo = (_encode_tag(1, 0) + _encode_varint(1) +
+                _encode_tag(2, 0) + _encode_varint(10))
+    item_bar = (_encode_tag(1, 0) + _encode_varint(2) +
+                _encode_tag(2, 0) + _encode_varint(20))
+    row_with_items = (_encode_tag(1, 0) + _encode_varint(1) +
+                      _encode_tag(2, 2) + _encode_varint(len(item_foo)) + item_foo +
+                      _encode_tag(2, 2) + _encode_varint(len(item_bar)) + item_bar +
+                      _encode_tag(3, 2) + _encode_varint(5) + b"hello")
+    row_no_items = (_encode_tag(1, 0) + _encode_varint(2) +
+                    _encode_tag(3, 2) + _encode_varint(5) + b"empty")
+
+    def run_on_spark(spark):
+        df = spark.createDataFrame(
+            [(row_with_items,), (row_no_items,), (None,)], schema="bin binary")
+        decoded = _call_from_protobuf(
+            from_protobuf_fn, f.col("bin"), message_name, desc_path, desc_bytes)
+        return df.select(
+            decoded.getField("id").alias("id"),
+            decoded.getField("title").alias("title"),
+            decoded.getField("items").getField("priority").alias("priorities"))
+
+    assert_gpu_and_cpu_are_equal_collect(run_on_spark)
+
+
+@pytest.mark.skipif(is_before_spark_340(), reason="from_protobuf is Spark 3.4.0+")
+@ignore_order(local=True)
+def test_from_protobuf_repeated_message_child_enum_string_invalid_permissive(
+        spark_tmp_path, from_protobuf_fn):
+    desc_path, desc_bytes = _setup_protobuf_desc(
+        spark_tmp_path, "repeated_message_enum.desc",
+        _build_repeated_message_enum_descriptor_set_bytes)
+    message_name = "test.ContainerWithPriorityItems"
+
+    item_valid = (_encode_tag(1, 0) + _encode_varint(1) +
+                  _encode_tag(2, 0) + _encode_varint(10))
+    item_invalid = (_encode_tag(1, 0) + _encode_varint(99) +
+                    _encode_tag(2, 0) + _encode_varint(20))
+    row_valid = (_encode_tag(1, 0) + _encode_varint(1) +
+                 _encode_tag(2, 2) + _encode_varint(len(item_valid)) + item_valid +
+                 _encode_tag(3, 2) + _encode_varint(2) + b"ok")
+    row_invalid = (_encode_tag(1, 0) + _encode_varint(2) +
+                   _encode_tag(2, 2) + _encode_varint(len(item_invalid)) + item_invalid +
+                   _encode_tag(3, 2) + _encode_varint(3) + b"bad")
+
+    def run_on_spark(spark):
+        df = spark.createDataFrame(
+            [(0, row_valid), (1, row_invalid)], schema="idx int, bin binary")
+        decoded = _call_from_protobuf(
+            from_protobuf_fn, f.col("bin"), message_name, desc_path, desc_bytes,
+            options={"mode": "PERMISSIVE"})
+        return df.select(
+            f.col("idx"),
+            decoded.isNull().alias("decoded_is_null"),
+            decoded.getField("id").alias("id"),
+            decoded.getField("title").alias("title"),
+            decoded.getField("items").getField("priority").getItem(0).alias("priority0")
+        )
+
+    assert_gpu_and_cpu_are_equal_collect(run_on_spark)
+
+
+def _build_nested_repeated_enum_descriptor_set_bytes(spark):
+    """
+    message OuterWithNestedRepeatedEnum {
+        optional int32 id = 1;
+        optional InnerWithRepeatedPriority inner = 2;
+        optional string name = 3;
+    }
+    message InnerWithRepeatedPriority {
+        repeated Priority priority = 1;
+        optional int32 count = 2;
+    }
+    enum Priority { UNKNOWN = 0; FOO = 1; BAR = 2; }
+    """
+    D, fd = _new_proto2_file(spark, "nested_repeated_enum.proto")
+    label_opt = D.FieldDescriptorProto.Label.LABEL_OPTIONAL
+    label_rep = D.FieldDescriptorProto.Label.LABEL_REPEATED
+
+    inner_msg = D.DescriptorProto.newBuilder().setName("InnerWithRepeatedPriority")
+    enum_type = D.EnumDescriptorProto.newBuilder().setName("Priority")
+    enum_type.addValue(D.EnumValueDescriptorProto.newBuilder().setName("UNKNOWN").setNumber(0).build())
+    enum_type.addValue(D.EnumValueDescriptorProto.newBuilder().setName("FOO").setNumber(1).build())
+    enum_type.addValue(D.EnumValueDescriptorProto.newBuilder().setName("BAR").setNumber(2).build())
+    inner_msg.addEnumType(enum_type.build())
+    inner_msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("priority").setNumber(1).setLabel(label_rep)
+            .setType(D.FieldDescriptorProto.Type.TYPE_ENUM)
+            .setTypeName(".test.InnerWithRepeatedPriority.Priority").build())
+    inner_msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("count").setNumber(2).setLabel(label_opt)
+            .setType(D.FieldDescriptorProto.Type.TYPE_INT32).build())
+    fd.addMessageType(inner_msg.build())
+
+    outer_msg = D.DescriptorProto.newBuilder().setName("OuterWithNestedRepeatedEnum")
+    outer_msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("id").setNumber(1).setLabel(label_opt)
+            .setType(D.FieldDescriptorProto.Type.TYPE_INT32).build())
+    outer_msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("inner").setNumber(2).setLabel(label_opt)
+            .setType(D.FieldDescriptorProto.Type.TYPE_MESSAGE)
+            .setTypeName(".test.InnerWithRepeatedPriority").build())
+    outer_msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("name").setNumber(3).setLabel(label_opt)
+            .setType(D.FieldDescriptorProto.Type.TYPE_STRING).build())
+    fd.addMessageType(outer_msg.build())
+
+    fds = D.FileDescriptorSet.newBuilder().addFile(fd.build()).build()
+    return bytes(fds.toByteArray())
+
+
+@pytest.mark.skipif(is_before_spark_340(), reason="from_protobuf is Spark 3.4.0+")
+@ignore_order(local=True)
+def test_from_protobuf_nested_repeated_enum_string(
+        spark_tmp_path, from_protobuf_fn):
+    desc_path, desc_bytes = _setup_protobuf_desc(
+        spark_tmp_path, "nested_repeated_enum.desc",
+        _build_nested_repeated_enum_descriptor_set_bytes)
+    message_name = "test.OuterWithNestedRepeatedEnum"
+
+    inner = (_encode_tag(1, 0) + _encode_varint(0) +
+             _encode_tag(1, 0) + _encode_varint(2) +
+             _encode_tag(1, 0) + _encode_varint(1) +
+             _encode_tag(2, 0) + _encode_varint(7))
+    row_with_inner = (_encode_tag(1, 0) + _encode_varint(1) +
+                      _encode_tag(2, 2) + _encode_varint(len(inner)) + inner +
+                      _encode_tag(3, 2) + _encode_varint(5) + b"hello")
+    row_no_inner = (_encode_tag(1, 0) + _encode_varint(2) +
+                    _encode_tag(3, 2) + _encode_varint(4) + b"none")
+
+    def run_on_spark(spark):
+        df = spark.createDataFrame(
+            [(row_with_inner,), (row_no_inner,), (None,)], schema="bin binary")
+        decoded = _call_from_protobuf(
+            from_protobuf_fn, f.col("bin"), message_name, desc_path, desc_bytes)
+        return df.select(
+            decoded.getField("id").alias("id"),
+            decoded.getField("name").alias("name"),
+            decoded.getField("inner").getField("priority").alias("priorities"))
+
+    assert_gpu_and_cpu_are_equal_collect(run_on_spark)
+
+
+@pytest.mark.skipif(is_before_spark_340(), reason="from_protobuf is Spark 3.4.0+")
+@ignore_order(local=True)
+def test_from_protobuf_nested_repeated_enum_string_invalid_permissive(
+        spark_tmp_path, from_protobuf_fn):
+    desc_path, desc_bytes = _setup_protobuf_desc(
+        spark_tmp_path, "nested_repeated_enum.desc",
+        _build_nested_repeated_enum_descriptor_set_bytes)
+    message_name = "test.OuterWithNestedRepeatedEnum"
+
+    inner_valid = (_encode_tag(1, 0) + _encode_varint(1) +
+                   _encode_tag(1, 0) + _encode_varint(2) +
+                   _encode_tag(2, 0) + _encode_varint(7))
+    inner_invalid = (_encode_tag(1, 0) + _encode_varint(1) +
+                     _encode_tag(1, 0) + _encode_varint(99) +
+                     _encode_tag(2, 0) + _encode_varint(9))
+    row_valid = (_encode_tag(1, 0) + _encode_varint(1) +
+                 _encode_tag(2, 2) + _encode_varint(len(inner_valid)) + inner_valid +
+                 _encode_tag(3, 2) + _encode_varint(2) + b"ok")
+    row_invalid = (_encode_tag(1, 0) + _encode_varint(2) +
+                   _encode_tag(2, 2) + _encode_varint(len(inner_invalid)) + inner_invalid +
+                   _encode_tag(3, 2) + _encode_varint(3) + b"bad")
+
+    def run_on_spark(spark):
+        df = spark.createDataFrame(
+            [(0, row_valid), (1, row_invalid)], schema="idx int, bin binary")
+        decoded = _call_from_protobuf(
+            from_protobuf_fn, f.col("bin"), message_name, desc_path, desc_bytes,
+            options={"mode": "PERMISSIVE"})
+        return df.select(
+            f.col("idx"),
+            decoded.isNull().alias("decoded_is_null"),
+            decoded.getField("id").alias("id"),
+            decoded.getField("name").alias("name"),
+            decoded.getField("inner").getField("priority").getItem(0).alias("priority0"),
+            decoded.getField("inner").getField("priority").getItem(1).alias("priority1")
+        )
+
+    assert_gpu_and_cpu_are_equal_collect(run_on_spark)
+
+
 def _build_required_field_descriptor_set_bytes(spark):
     """
     Build a FileDescriptorSet for a message with required fields (proto2):
@@ -2641,6 +2891,126 @@ def _get_field_by_path(expr, path):
     return current
 
 
+@pytest.mark.skipif(is_before_spark_340(), reason="from_protobuf is Spark 3.4.0+")
+@ignore_order(local=True)
+def test_from_protobuf_projection_across_alias_project_boundary(
+        spark_tmp_path, from_protobuf_fn):
+    desc_path, desc_bytes = _setup_protobuf_desc(
+        spark_tmp_path, "schema_proj_alias.desc",
+        _build_schema_projection_descriptor_set_bytes)
+    message_name = "test.SchemaProj"
+
+    def run_on_spark(spark):
+        df = spark.createDataFrame([(row,) for row in _schema_proj_test_data], schema="bin binary")
+        decoded = _call_from_protobuf(
+            from_protobuf_fn, f.col("bin"), message_name, desc_path, desc_bytes)
+        aliased = df.select(decoded.alias("decoded"))
+        return aliased.select(
+            f.col("decoded").getField("detail").getField("a").alias("detail_a"),
+            f.col("decoded").getField("id").alias("id"))
+
+    assert_gpu_and_cpu_are_equal_collect(run_on_spark)
+
+
+@pytest.mark.skipif(is_before_spark_340(), reason="from_protobuf is Spark 3.4.0+")
+@ignore_order(local=True)
+def test_from_protobuf_projection_across_withcolumn_boundary(
+        spark_tmp_path, from_protobuf_fn):
+    desc_path, desc_bytes = _setup_protobuf_desc(
+        spark_tmp_path, "schema_proj_withcolumn.desc",
+        _build_schema_projection_descriptor_set_bytes)
+    message_name = "test.SchemaProj"
+
+    def run_on_spark(spark):
+        df = spark.createDataFrame([(row,) for row in _schema_proj_test_data], schema="bin binary")
+        with_decoded = df.withColumn(
+            "decoded",
+            _call_from_protobuf(from_protobuf_fn, f.col("bin"), message_name, desc_path, desc_bytes))
+        return with_decoded.select(
+            f.col("decoded").getField("items").getField("a").alias("items_a"),
+            f.col("decoded").getField("id").alias("id"))
+
+    assert_gpu_and_cpu_are_equal_collect(run_on_spark)
+
+
+def _build_dual_message_projection_descriptor_set_bytes(spark):
+    """
+    message BytesView {
+        optional int32 status = 1;
+        optional bytes payload = 2;
+    }
+    message NestedPayload {
+        optional int32 count = 1;
+    }
+    message NestedView {
+        optional int32 status = 1;
+        optional NestedPayload payload = 2;
+    }
+    """
+    D, fd = _new_proto2_file(spark, "dual_projection.proto")
+    label_opt = D.FieldDescriptorProto.Label.LABEL_OPTIONAL
+
+    nested_msg = D.DescriptorProto.newBuilder().setName("NestedPayload")
+    nested_msg.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("count").setNumber(1).setLabel(label_opt)
+            .setType(D.FieldDescriptorProto.Type.TYPE_INT32).build())
+    fd.addMessageType(nested_msg.build())
+
+    bytes_view = D.DescriptorProto.newBuilder().setName("BytesView")
+    bytes_view.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("status").setNumber(1).setLabel(label_opt)
+            .setType(D.FieldDescriptorProto.Type.TYPE_INT32).build())
+    bytes_view.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("payload").setNumber(2).setLabel(label_opt)
+            .setType(D.FieldDescriptorProto.Type.TYPE_BYTES).build())
+    fd.addMessageType(bytes_view.build())
+
+    nested_view = D.DescriptorProto.newBuilder().setName("NestedView")
+    nested_view.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("status").setNumber(1).setLabel(label_opt)
+            .setType(D.FieldDescriptorProto.Type.TYPE_INT32).build())
+    nested_view.addField(
+        D.FieldDescriptorProto.newBuilder()
+            .setName("payload").setNumber(2).setLabel(label_opt)
+            .setType(D.FieldDescriptorProto.Type.TYPE_MESSAGE)
+            .setTypeName(".test.NestedPayload").build())
+    fd.addMessageType(nested_view.build())
+
+    fds = D.FileDescriptorSet.newBuilder().addFile(fd.build()).build()
+    return bytes(fds.toByteArray())
+
+
+@pytest.mark.skipif(is_before_spark_340(), reason="from_protobuf is Spark 3.4.0+")
+@ignore_order(local=True)
+def test_from_protobuf_different_messages_same_binary_column_do_not_interfere(
+        spark_tmp_path, from_protobuf_fn):
+    desc_path, desc_bytes = _setup_protobuf_desc(
+        spark_tmp_path, "dual_projection.desc",
+        _build_dual_message_projection_descriptor_set_bytes)
+
+    payload_keep = _encode_tag(1, 0) + _encode_varint(7)
+    payload_drop = _encode_tag(1, 0) + _encode_varint(9)
+    row_keep = (_encode_tag(1, 0) + _encode_varint(1) +
+                _encode_tag(2, 2) + _encode_varint(len(payload_keep)) + payload_keep)
+    row_drop = (_encode_tag(1, 0) + _encode_varint(0) +
+                _encode_tag(2, 2) + _encode_varint(len(payload_drop)) + payload_drop)
+
+    def run_on_spark(spark):
+        df = spark.createDataFrame([(row_keep,), (row_drop,)], schema="bin binary")
+        bytes_view = _call_from_protobuf(
+            from_protobuf_fn, f.col("bin"), "test.BytesView", desc_path, desc_bytes)
+        nested_view = _call_from_protobuf(
+            from_protobuf_fn, f.col("bin"), "test.NestedView", desc_path, desc_bytes)
+        return df.filter(bytes_view.getField("status") == 1).select(
+            nested_view.getField("payload").getField("count").alias("payload_count"))
+
+    assert_gpu_and_cpu_are_equal_collect(run_on_spark)
+
+
 def _build_deep_nested_5_level_descriptor_set_bytes(spark):
     """
     Build a FileDescriptorSet for 5-level deep nesting:
diff --git a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala
index 5756c5a670b..d4fdfb85c26 100644
--- a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala
+++ b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala
@@ -381,6 +381,7 @@ object ProtobufExprShims extends org.apache.spark.internal.Logging {
 
         private def analyzeRequiredFields(allFieldNames: Set[String]): Set[String] = {
           val fieldReqs = mutable.Map[String, Option[Set[String]]]()
+          protobufOutputExprIds = Set.empty
           var hasDirectStructRef = false
           val holder = () => { hasDirectStructRef = true }
 
@@ -399,6 +400,12 @@ object ProtobufExprShims extends org.apache.spark.internal.Logging {
             currentMeta.get.wrapped match {
               case p: ProjectExec =>
                 collectedExprs ++= p.projectList
+                p.projectList.foreach {
+                  case alias: org.apache.spark.sql.catalyst.expressions.Alias
+                      if isProtobufStructReference(alias.child) =>
+                    protobufOutputExprIds += alias.exprId
+                  case _ =>
+                }
                 p.projectList.foreach(collectStructFieldReferences(_, fieldReqs, holder))
                 currentMeta = None
               case f: org.apache.spark.sql.execution.FilterExec =>
@@ -431,6 +438,12 @@ object ProtobufExprShims extends org.apache.spark.internal.Logging {
           } else {
             nestedFieldRequirements = fieldReqs.toMap
             targetExprsToRemap = collectedExprs.toSeq
+            val prunedFieldsMap = buildPrunedFieldsMap()
+            val topLevelIndices = fullSchema.fields.zipWithIndex.collect {
+              case (sf, idx) if fieldReqs.keySet.contains(sf.name) => idx
+            }
+            targetExprsToRemap.foreach(
+              registerPrunedOrdinals(_, prunedFieldsMap, topLevelIndices))
             fieldReqs.keySet.toSet
           }
         }
@@ -459,6 +472,8 @@ object ProtobufExprShims extends org.apache.spark.internal.Logging {
          * Top-level names (keys without dots) also determine which fields are decoded.
          */
         private var nestedFieldRequirements: Map[String, Option[Set[String]]] = Map.empty
+        private var protobufOutputExprIds: Set[
+          org.apache.spark.sql.catalyst.expressions.ExprId] = Set.empty
 
         private def getFieldName(ordinal: Int, nameOpt: Option[String],
             schema: StructType): String = {
@@ -587,6 +602,11 @@ object ProtobufExprShims extends org.apache.spark.internal.Logging {
                   }
               }
 
+            case alias: org.apache.spark.sql.catalyst.expressions.Alias =>
+              if (!isProtobufStructReference(alias.child)) {
+                collectStructFieldReferences(alias.child, fieldReqs, hasDirectStructRefHolder)
+              }
+
             case _ =>
               if (isProtobufStructReference(expr)) {
                 hasDirectStructRefHolder()
@@ -597,6 +617,66 @@ object ProtobufExprShims extends org.apache.spark.internal.Logging {
           }
         }
 
+        private def buildPrunedFieldsMap(): Map[String, Seq[String]] = {
+          nestedFieldRequirements.collect {
+            case (pathKey, Some(childNames)) =>
+              val pathParts = pathKey.split("\\.").toSeq
+              val childSchema = resolveSchemaAtPath(fullSchema, pathParts)
+              if (childSchema != null) {
+                val orderedNames = childSchema.fields
+                  .map(_.name)
+                  .filter(childNames.contains)
+                  .toSeq
+                pathKey -> orderedNames
+              } else {
+                pathKey -> childNames.toSeq
+              }
+          }
+        }
+
+        private def registerPrunedOrdinals(
+            expr: Expression,
+            prunedFieldsMap: Map[String, Seq[String]],
+            topLevelIndices: Seq[Int]): Unit = {
+          expr match {
+            case gsf @ GetStructField(childExpr, ordinal, nameOpt) =>
+              resolveFieldAccessChain(childExpr) match {
+                case Some(parentPath) if parentPath.nonEmpty =>
+                  val parentSchema = resolveSchemaAtPath(fullSchema, parentPath)
+                  if (parentSchema != null) {
+                    val pathKey = parentPath.mkString(".")
+                    val childName = getFieldName(ordinal, nameOpt, parentSchema)
+                    prunedFieldsMap.get(pathKey).foreach { orderedChildren =>
+                      val runtimeOrd = orderedChildren.indexOf(childName)
+                      if (runtimeOrd >= 0) {
+                        gsf.setTagValue(ProtobufExprShims.PRUNED_ORDINAL_TAG, runtimeOrd)
+                      }
+                    }
+                  }
+                case Some(parentPath) if parentPath.isEmpty =>
+                  val runtimeOrd = topLevelIndices.indexOf(ordinal)
+                  if (runtimeOrd >= 0) {
+                    gsf.setTagValue(ProtobufExprShims.PRUNED_ORDINAL_TAG, runtimeOrd)
+                  }
+                case _ =>
+              }
+            case gasf @ GetArrayStructFields(childExpr, field, _, _, _) =>
+              resolveFieldAccessChain(childExpr) match {
+                case Some(parentPath) if parentPath.nonEmpty =>
+                  val pathKey = parentPath.mkString(".")
+                  prunedFieldsMap.get(pathKey).foreach { orderedChildren =>
+                    val runtimeOrd = orderedChildren.indexOf(field.name)
+                    if (runtimeOrd >= 0) {
+                      gasf.setTagValue(ProtobufExprShims.PRUNED_ORDINAL_TAG, runtimeOrd)
+                    }
+                  }
+                case _ =>
+              }
+            case _ =>
+          }
+          expr.children.foreach(registerPrunedOrdinals(_, prunedFieldsMap, topLevelIndices))
+        }
+
         /**
          * Check if an expression references the output of a protobuf decode expression.
          * This can be either:
@@ -611,23 +691,17 @@ object ProtobufExprShims extends org.apache.spark.internal.Logging {
 
           // Catalyst may create duplicate ProtobufDataToCatalyst
           // instances for each GetStructField access. Match copies
-          // by class + identical input child so that
+          // by class + identical input child + identical decode
+          // semantics so that
           // analyzeRequiredFields detects all field accesses in one
           // pass, keeping schema projection correct.
-          //
-          // Known limitation: if two different from_protobuf calls
-          // (different messageName / descriptor) share the same binary
-          // input column within one ProjectExec, this check will
-          // incorrectly treat them as the same expression, potentially
-          // mixing up field requirements and ordinal remapping. This is
-          // acceptable because decoding the same bytes as two different
-          // message types is not a meaningful query.
           if (expr.getClass == e.getClass &&
               expr.children.nonEmpty &&
               e.children.nonEmpty &&
               ((expr.children.head eq e.children.head) ||
                 expr.children.head.semanticEquals(
-                  e.children.head))) {
+                  e.children.head)) &&
+              SparkProtobufCompat.sameDecodeSemantics(expr, e)) {
             return true
           }
 
@@ -644,69 +718,15 @@ object ProtobufExprShims extends org.apache.spark.internal.Logging {
 
           expr match {
             case attr: AttributeReference =>
-              protobufOutputExprId.exists(_ == attr.exprId)
+              protobufOutputExprIds.contains(attr.exprId) || protobufOutputExprId.exists(_ == attr.exprId)
             case _ => false
           }
         }
 
         override def convertToGpu(child: Expression): GpuExpression = {
-          val prunedFieldsMap: Map[String, Seq[String]] = nestedFieldRequirements.collect {
-            case (pathKey, Some(childNames)) =>
-              val pathParts = pathKey.split("\\.").toSeq
-              val childSchema = resolveSchemaAtPath(fullSchema, pathParts)
-              if (childSchema != null) {
-                val orderedNames = childSchema.fields
-                  .map(_.name)
-                  .filter(childNames.contains)
-                  .toSeq
-                pathKey -> orderedNames
-              } else {
-                pathKey -> childNames.toSeq
-              }
-          }
-
-          def registerExprs(expr: Expression): Unit = {
-            expr match {
-              case gsf @ GetStructField(childExpr, ordinal, nameOpt) =>
-                resolveFieldAccessChain(childExpr) match {
-                  case Some(parentPath) if parentPath.nonEmpty =>
-                    val parentSchema = resolveSchemaAtPath(fullSchema, parentPath)
-                    if (parentSchema != null) {
-                      val pathKey = parentPath.mkString(".")
-                      val childName = getFieldName(ordinal, nameOpt, parentSchema)
-                      prunedFieldsMap.get(pathKey).foreach { orderedChildren =>
-                        val runtimeOrd = orderedChildren.indexOf(childName)
-                        if (runtimeOrd >= 0) {
-                          gsf.setTagValue(ProtobufExprShims.PRUNED_ORDINAL_TAG, runtimeOrd)
-                        }
-                      }
-                    }
-                  case Some(parentPath) if parentPath.isEmpty =>
-                    val runtimeOrd = decodedTopLevelIndices.indexOf(ordinal)
-                    if (runtimeOrd >= 0) {
-                      gsf.setTagValue(ProtobufExprShims.PRUNED_ORDINAL_TAG, runtimeOrd)
-                    }
-                  case _ =>
-                }
-
-              case gasf @ GetArrayStructFields(childExpr, field, _, _, _) =>
-                resolveFieldAccessChain(childExpr) match {
-                  case Some(parentPath) if parentPath.nonEmpty =>
-                    val pathKey = parentPath.mkString(".")
-                    prunedFieldsMap.get(pathKey).foreach { orderedChildren =>
-                      val runtimeOrd = orderedChildren.indexOf(field.name)
-                      if (runtimeOrd >= 0) {
-                        gasf.setTagValue(ProtobufExprShims.PRUNED_ORDINAL_TAG, runtimeOrd)
-                      }
-                    }
-                  case _ =>
-                }
-              case _ =>
-            }
-            expr.children.foreach(registerExprs)
-          }
-
-          targetExprsToRemap.foreach(registerExprs)
+          val prunedFieldsMap = buildPrunedFieldsMap()
+          targetExprsToRemap.foreach(
+            registerPrunedOrdinals(_, prunedFieldsMap, decodedTopLevelIndices.toSeq))
 
           val decodedSchema = {
             def applyPruning(field: StructField, prefix: String): StructField = {
diff --git a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/SparkProtobufCompat.scala b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/SparkProtobufCompat.scala
index 9785187dc0b..1368198559f 100644
--- a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/SparkProtobufCompat.scala
+++ b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/SparkProtobufCompat.scala
@@ -60,6 +60,13 @@ private[shims] object SparkProtobufCompat extends Logging {
     } yield ProtobufExprInfo(messageName, descriptorSource, options)
   }
 
+  def sameDecodeSemantics(left: Expression, right: Expression): Boolean = {
+    (extractExprInfo(left), extractExprInfo(right)) match {
+      case (Right(leftInfo), Right(rightInfo)) => leftInfo == rightInfo
+      case _ => false
+    }
+  }
+
   def parsePlannerOptions(
       options: Map[String, String]): Either[String, ProtobufPlannerOptions] = {
     val enumsAsInts = Try(options.getOrElse("enums.as.ints", "false").toBoolean)
diff --git a/sql-plugin/src/test/scala/com/nvidia/spark/rapids/shims/ProtobufExprShimsSuite.scala b/sql-plugin/src/test/scala/com/nvidia/spark/rapids/shims/ProtobufExprShimsSuite.scala
index c38ab916f36..b0ec70abd1c 100644
--- a/sql-plugin/src/test/scala/com/nvidia/spark/rapids/shims/ProtobufExprShimsSuite.scala
+++ b/sql-plugin/src/test/scala/com/nvidia/spark/rapids/shims/ProtobufExprShimsSuite.scala
@@ -73,6 +73,27 @@ class ProtobufExprShimsSuite extends AnyFunSuite {
     def descFilePath: Option[String] = Some("/tmp/test.desc")
   }
 
+  private case class FakeDifferentMessageExpr(override val child: Expression)
+      extends FakeBaseProtobufExpr(child) {
+    def messageName: String = "test.OtherMessage"
+    def descFilePath: Option[String] = Some("/tmp/test.desc")
+    def options: scala.collection.Map[String, String] = Map("mode" -> "FAILFAST")
+  }
+
+  private case class FakeDifferentDescriptorExpr(override val child: Expression)
+      extends FakeBaseProtobufExpr(child) {
+    def messageName: String = "test.Message"
+    def descFilePath: Option[String] = Some("/tmp/other.desc")
+    def options: scala.collection.Map[String, String] = Map("mode" -> "FAILFAST")
+  }
+
+  private case class FakeDifferentOptionsExpr(override val child: Expression)
+      extends FakeBaseProtobufExpr(child) {
+    def messageName: String = "test.Message"
+    def descFilePath: Option[String] = Some("/tmp/test.desc")
+    def options: scala.collection.Map[String, String] = Map("mode" -> "PERMISSIVE")
+  }
+
   private case class FakeMessageDescriptor(
       syntax: String,
       fields: Map[String, ProtobufFieldDescriptor]) extends ProtobufMessageDescriptor {
@@ -114,6 +135,19 @@ class ProtobufExprShimsSuite extends AnyFunSuite {
       Right(ProtobufPlannerOptions(enumsAsInts = true, failOnErrors = false)))
   }
 
+  test("compat distinguishes decode semantics across message descriptor and options") {
+    val child = FakeExprChild()
+
+    assert(SparkProtobufCompat.sameDecodeSemantics(
+      FakePathProtobufExpr(child), FakePathProtobufExpr(child)))
+    assert(!SparkProtobufCompat.sameDecodeSemantics(
+      FakePathProtobufExpr(child), FakeDifferentMessageExpr(child)))
+    assert(!SparkProtobufCompat.sameDecodeSemantics(
+      FakePathProtobufExpr(child), FakeDifferentDescriptorExpr(child)))
+    assert(!SparkProtobufCompat.sameDecodeSemantics(
+      FakePathProtobufExpr(child), FakeDifferentOptionsExpr(child)))
+  }
+
   test("compat reports missing options accessor as cpu fallback reason") {
     val exprInfo = SparkProtobufCompat.extractExprInfo(FakeMissingOptionsExpr(FakeExprChild()))
     assert(exprInfo.left.toOption.exists(

From 9467ebc8f9401174458dde079e4a3be13b172bd3 Mon Sep 17 00:00:00 2001
From: Haoyang Li <haoyangl@nvidia.com>
Date: Thu, 12 Mar 2026 14:55:37 +0800
Subject: [PATCH 30/33] address comments

Signed-off-by: Haoyang Li <haoyangl@nvidia.com>
---
 .../spark/rapids/basicPhysicalOperators.scala | 15 ++--
 .../spark/sql/rapids/GpuFromProtobuf.scala    |  2 +
 .../protobuf/ProtobufSchemaExtractor.scala    | 35 ++++----
 .../rapids/protobuf/ProtobufSchemaModel.scala |  2 +-
 .../protobuf/ProtobufSchemaValidator.scala    | 80 ++++++++++---------
 .../rapids/shims/ProtobufExprShims.scala      | 23 +++---
 .../rapids/shims/SparkProtobufCompat.scala    | 14 ++--
 .../rapids/ProtobufBatchMergeSuite.scala      | 17 ++++
 .../rapids/shims/ProtobufExprShimsSuite.scala | 53 +++++++++++-
 9 files changed, 165 insertions(+), 76 deletions(-)

diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/basicPhysicalOperators.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/basicPhysicalOperators.scala
index a808b31f1d0..5db9f27ad0b 100644
--- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/basicPhysicalOperators.scala
+++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/basicPhysicalOperators.scala
@@ -59,14 +59,16 @@ object GpuProjectExecMeta {
     case _ => Set.empty
   }
 
-  private def isRootedAtProtobufAlias(expr: Expression, protobufExprIds: Set[ExprId]): Boolean =
+  private def isRootedAtProtobufDecode(expr: Expression, protobufExprIds: Set[ExprId]): Boolean =
     expr match {
       case attr: AttributeReference =>
         protobufExprIds.contains(attr.exprId)
+      case decode if isProtobufDecodeExpr(decode) =>
+        true
       case GetStructField(child, _, _) =>
-        isRootedAtProtobufAlias(child, protobufExprIds)
+        isRootedAtProtobufDecode(child, protobufExprIds)
       case gasf: GetArrayStructFields =>
-        isRootedAtProtobufAlias(gasf.child, protobufExprIds)
+        isRootedAtProtobufDecode(gasf.child, protobufExprIds)
       case _ =>
         false
     }
@@ -75,17 +77,16 @@ object GpuProjectExecMeta {
       expr: Expression,
       protobufExprIds: Set[ExprId]): Boolean = expr match {
     case GetStructField(child, _, _) =>
-      isRootedAtProtobufAlias(child, protobufExprIds)
+      isRootedAtProtobufDecode(child, protobufExprIds)
     case gasf: GetArrayStructFields =>
-      isRootedAtProtobufAlias(gasf.child, protobufExprIds)
+      isRootedAtProtobufDecode(gasf.child, protobufExprIds)
     case other =>
       other.children.exists(child => extractsFromProtobufAlias(child, protobufExprIds))
   }
 
   private[rapids] def shouldCoalesceAfterProject(plan: ProjectExec): Boolean = {
     val protobufExprIds = protobufAliasExprIds(plan.child)
-    protobufExprIds.nonEmpty &&
-      plan.projectList.exists(expr => extractsFromProtobufAlias(expr, protobufExprIds))
+    plan.projectList.exists(expr => extractsFromProtobufAlias(expr, protobufExprIds))
   }
 }
 
diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuFromProtobuf.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuFromProtobuf.scala
index e9d40a4b868..3cf6a290041 100644
--- a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuFromProtobuf.scala
+++ b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuFromProtobuf.scala
@@ -137,6 +137,8 @@ case class GpuFromProtobuf(
     result
   }
 
+  // ProtobufSchemaDescriptor is a pure-Java immutable holder for validated schema arrays.
+  // It does not own native resources, so task-scoped close hooks are not required here.
   @transient private lazy val protobufSchema = new ProtobufSchemaDescriptor(
     fieldNumbers, parentIndices, depthLevels, wireTypes, outputTypeIds, encodings,
     isRepeated, isRequired, hasDefaultValue, defaultInts, defaultFloats, defaultBools,
diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/protobuf/ProtobufSchemaExtractor.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/protobuf/ProtobufSchemaExtractor.scala
index d3f0c0fc68f..4fafb325da8 100644
--- a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/protobuf/ProtobufSchemaExtractor.scala
+++ b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/protobuf/ProtobufSchemaExtractor.scala
@@ -35,7 +35,12 @@ object ProtobufSchemaExtractor {
       val fd = msgDesc.findField(sf.name).getOrElse {
         return Left(s"Protobuf field '${sf.name}' not found in message '$messageName'")
       }
-      result(sf.name) = extractFieldInfo(sf, fd, enumsAsInts)
+      extractFieldInfo(sf, fd, enumsAsInts) match {
+        case Right(fieldInfo) =>
+          result(sf.name) = fieldInfo
+        case Left(reason) =>
+          return Left(reason)
+      }
     }
 
     Right(result.toMap)
@@ -44,7 +49,7 @@ object ProtobufSchemaExtractor {
   def extractFieldInfo(
       sparkField: StructField,
       fieldDescriptor: ProtobufFieldDescriptor,
-      enumsAsInts: Boolean): ProtobufFieldInfo = {
+      enumsAsInts: Boolean): Either[String, ProtobufFieldInfo] = {
     val (isSupported, unsupportedReason, encoding) =
       checkFieldSupport(
         sparkField.dataType,
@@ -52,18 +57,20 @@ object ProtobufSchemaExtractor {
         fieldDescriptor.isRepeated,
         enumsAsInts)
 
-    ProtobufFieldInfo(
-      fieldNumber = fieldDescriptor.fieldNumber,
-      protoTypeName = fieldDescriptor.protoTypeName,
-      sparkType = sparkField.dataType,
-      encoding = encoding,
-      isSupported = isSupported,
-      unsupportedReason = unsupportedReason,
-      isRequired = fieldDescriptor.isRequired,
-      defaultValue = fieldDescriptor.defaultValue,
-      enumMetadata = fieldDescriptor.enumMetadata,
-      isRepeated = fieldDescriptor.isRepeated
-    )
+    fieldDescriptor.defaultValueResult.map { defaultValue =>
+      ProtobufFieldInfo(
+        fieldNumber = fieldDescriptor.fieldNumber,
+        protoTypeName = fieldDescriptor.protoTypeName,
+        sparkType = sparkField.dataType,
+        encoding = encoding,
+        isSupported = isSupported,
+        unsupportedReason = unsupportedReason,
+        isRequired = fieldDescriptor.isRequired,
+        defaultValue = defaultValue,
+        enumMetadata = fieldDescriptor.enumMetadata,
+        isRepeated = fieldDescriptor.isRepeated
+      )
+    }
   }
 
   def checkFieldSupport(
diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/protobuf/ProtobufSchemaModel.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/protobuf/ProtobufSchemaModel.scala
index 1e04505d5b3..bf8969567e7 100644
--- a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/protobuf/ProtobufSchemaModel.scala
+++ b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/protobuf/ProtobufSchemaModel.scala
@@ -70,7 +70,7 @@ trait ProtobufFieldDescriptor {
   def protoTypeName: String
   def isRepeated: Boolean
   def isRequired: Boolean
-  def defaultValue: Option[ProtobufDefaultValue]
+  def defaultValueResult: Either[String, Option[ProtobufDefaultValue]]
   def enumMetadata: Option[ProtobufEnumMetadata]
   def messageDescriptor: Option[ProtobufMessageDescriptor]
 }
diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/protobuf/ProtobufSchemaValidator.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/protobuf/ProtobufSchemaValidator.scala
index ba7be5b34cf..627fed83296 100644
--- a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/protobuf/ProtobufSchemaValidator.scala
+++ b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/protobuf/ProtobufSchemaValidator.scala
@@ -36,34 +36,35 @@ object ProtobufSchemaValidator {
     validateFieldInfo(path, field, fieldInfo).flatMap { _ =>
       ProtobufSchemaExtractor
         .getWireType(fieldInfo.protoTypeName, fieldInfo.encoding)
-        .map { wireType =>
-        val defaults = encodeDefaultValue(path, field.dataType, fieldInfo)
-        val enumValidValues = fieldInfo.enumMetadata.map(_.validValues).orNull
-        val enumNames =
-          if (fieldInfo.encoding == GpuFromProtobuf.ENC_ENUM_STRING) {
-            fieldInfo.enumMetadata.map(_.orderedNames).orNull
-          } else {
-            null
-          }
+        .flatMap { wireType =>
+          encodeDefaultValue(path, field.dataType, fieldInfo).map { defaults =>
+            val enumValidValues = fieldInfo.enumMetadata.map(_.validValues).orNull
+            val enumNames =
+              if (fieldInfo.encoding == GpuFromProtobuf.ENC_ENUM_STRING) {
+                fieldInfo.enumMetadata.map(_.orderedNames).orNull
+              } else {
+                null
+              }
 
-        FlattenedFieldDescriptor(
-          fieldNumber = fieldInfo.fieldNumber,
-          parentIdx = parentIdx,
-          depth = depth,
-          wireType = wireType,
-          outputTypeId = outputTypeId,
-          encoding = fieldInfo.encoding,
-          isRepeated = fieldInfo.isRepeated,
-          isRequired = fieldInfo.isRequired,
-          hasDefaultValue = fieldInfo.hasDefaultValue,
-          defaultInt = defaults.defaultInt,
-          defaultFloat = defaults.defaultFloat,
-          defaultBool = defaults.defaultBool,
-          defaultString = defaults.defaultString,
-          enumValidValues = enumValidValues,
-          enumNames = enumNames
-        )
-      }
+            FlattenedFieldDescriptor(
+              fieldNumber = fieldInfo.fieldNumber,
+              parentIdx = parentIdx,
+              depth = depth,
+              wireType = wireType,
+              outputTypeId = outputTypeId,
+              encoding = fieldInfo.encoding,
+              isRepeated = fieldInfo.isRepeated,
+              isRequired = fieldInfo.isRequired,
+              hasDefaultValue = fieldInfo.hasDefaultValue,
+              defaultInt = defaults.defaultInt,
+              defaultFloat = defaults.defaultFloat,
+              defaultBool = defaults.defaultBool,
+              defaultString = defaults.defaultString,
+              enumValidValues = enumValidValues,
+              enumNames = enumNames
+            )
+          }
+        }
     }
   }
 
@@ -140,10 +141,10 @@ object ProtobufSchemaValidator {
   private def encodeDefaultValue(
       path: String,
       dataType: DataType,
-      fieldInfo: ProtobufFieldInfo): JniDefaultValues = {
+      fieldInfo: ProtobufFieldInfo): Either[String, JniDefaultValues] = {
     val empty = JniDefaultValues(0L, 0.0, defaultBool = false, null)
     fieldInfo.defaultValue match {
-      case None => empty
+      case None => Right(empty)
       case Some(defaultValue) =>
         val targetType = dataType match {
           case ArrayType(elementType, _) => elementType
@@ -151,24 +152,25 @@ object ProtobufSchemaValidator {
         }
         (targetType, defaultValue) match {
           case (BooleanType, ProtobufDefaultValue.BoolValue(value)) =>
-            empty.copy(defaultBool = value)
+            Right(empty.copy(defaultBool = value))
           case (IntegerType | LongType, ProtobufDefaultValue.IntValue(value)) =>
-            empty.copy(defaultInt = value)
+            Right(empty.copy(defaultInt = value))
           case (IntegerType | LongType, ProtobufDefaultValue.EnumValue(number, _)) =>
-            empty.copy(defaultInt = number.toLong)
+            Right(empty.copy(defaultInt = number.toLong))
           case (FloatType, ProtobufDefaultValue.FloatValue(value)) =>
-            empty.copy(defaultFloat = value.toDouble)
+            Right(empty.copy(defaultFloat = value.toDouble))
           case (DoubleType, ProtobufDefaultValue.DoubleValue(value)) =>
-            empty.copy(defaultFloat = value)
+            Right(empty.copy(defaultFloat = value))
           case (StringType, ProtobufDefaultValue.StringValue(value)) =>
-            empty.copy(defaultString = value.getBytes("UTF-8"))
+            Right(empty.copy(defaultString = value.getBytes("UTF-8")))
           case (StringType, ProtobufDefaultValue.EnumValue(number, name)) =>
-            empty.copy(defaultInt = number.toLong, defaultString = name.getBytes("UTF-8"))
+            Right(empty.copy(
+              defaultInt = number.toLong,
+              defaultString = name.getBytes("UTF-8")))
           case (BinaryType, ProtobufDefaultValue.BinaryValue(value)) =>
-            empty.copy(defaultString = value)
+            Right(empty.copy(defaultString = value))
           case _ =>
-            throw new IllegalStateException(
-              s"Incompatible default value for protobuf field '$path': $defaultValue")
+            Left(s"Incompatible default value for protobuf field '$path': $defaultValue")
         }
     }
   }
diff --git a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala
index d4fdfb85c26..4634d4da429 100644
--- a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala
+++ b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala
@@ -312,15 +312,20 @@ object ProtobufExprShims extends org.apache.spark.internal.Logging {
                             s"Nested field '${childSf.name}' not found in protobuf " +
                               s"descriptor for message at '$path'")
                         case Some(childFd) =>
-                          val childInfo =
-                            ProtobufSchemaExtractor.extractFieldInfo(childSf, childFd, enumsAsInts)
-                          if (!childInfo.isSupported) {
-                            willNotWorkOnGpu(
-                              s"Nested field '${childSf.name}' at '$path': " +
-                                childInfo.unsupportedReason.getOrElse("unsupported type"))
-                          } else {
-                            addFieldWithChildren(
-                              childSf, childInfo, parentIdx, parentDepth + 1, childMsgDesc, path)
+                          ProtobufSchemaExtractor
+                            .extractFieldInfo(childSf, childFd, enumsAsInts) match {
+                            case Left(reason) =>
+                              willNotWorkOnGpu(reason)
+                            case Right(childInfo) =>
+                              if (!childInfo.isSupported) {
+                                willNotWorkOnGpu(
+                                  s"Nested field '${childSf.name}' at '$path': " +
+                                    childInfo.unsupportedReason.getOrElse("unsupported type"))
+                              } else {
+                                addFieldWithChildren(
+                                  childSf, childInfo, parentIdx, parentDepth + 1, childMsgDesc,
+                                  path)
+                              }
                           }
                       }
                     }
diff --git a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/SparkProtobufCompat.scala b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/SparkProtobufCompat.scala
index 1368198559f..73ee429ceac 100644
--- a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/SparkProtobufCompat.scala
+++ b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/SparkProtobufCompat.scala
@@ -170,11 +170,15 @@ private[shims] object SparkProtobufCompat extends Logging {
       } else {
         None
       }
-    override lazy val defaultValue: Option[ProtobufDefaultValue] =
-      if (PbReflect.hasDefaultValue(raw)) {
-        PbReflect.getDefaultValue(raw).map(toDefaultValue(_, protoTypeName, enumMetadata))
-      } else {
-        None
+    override lazy val defaultValueResult: Either[String, Option[ProtobufDefaultValue]] =
+      Try {
+        if (PbReflect.hasDefaultValue(raw)) {
+          PbReflect.getDefaultValue(raw).map(toDefaultValue(_, protoTypeName, enumMetadata))
+        } else {
+          None
+        }
+      }.toEither.left.map { t =>
+        s"Failed to read protobuf default value for field '$name': ${t.getMessage}"
       }
     override lazy val messageDescriptor: Option[ProtobufMessageDescriptor] =
       if (protoTypeName == "MESSAGE") {
diff --git a/sql-plugin/src/test/scala/com/nvidia/spark/rapids/ProtobufBatchMergeSuite.scala b/sql-plugin/src/test/scala/com/nvidia/spark/rapids/ProtobufBatchMergeSuite.scala
index d8c7dd0b726..1f2d4492108 100644
--- a/sql-plugin/src/test/scala/com/nvidia/spark/rapids/ProtobufBatchMergeSuite.scala
+++ b/sql-plugin/src/test/scala/com/nvidia/spark/rapids/ProtobufBatchMergeSuite.scala
@@ -65,6 +65,23 @@ class ProtobufBatchMergeSuite extends AnyFunSuite {
     assert(!GpuProjectExecMeta.shouldCoalesceAfterProject(innerProject))
   }
 
+  test("project meta detects direct protobuf extraction in same project") {
+    val binAttr = AttributeReference("bin", BinaryType)()
+    val childScan = DummyColumnarLeaf(Seq(binAttr))
+    val directProject = ProjectExec(
+      Seq(
+        Alias(GetStructField(FakeProtobufDataToCatalyst(binAttr), 0, None), "search_id")(),
+        Alias(
+          GetStructField(
+            GetStructField(FakeProtobufDataToCatalyst(binAttr), 1, None),
+            0,
+            None),
+          "value")()),
+      childScan)
+
+    assert(GpuProjectExecMeta.shouldCoalesceAfterProject(directProject))
+  }
+
   test("protobuf batch merge config defaults off and can be enabled") {
     val enabledConf = new RapidsConf(Map(
       RapidsConf.ENABLE_PROTOBUF_BATCH_MERGE_AFTER_PROJECT.key -> "true"))
diff --git a/sql-plugin/src/test/scala/com/nvidia/spark/rapids/shims/ProtobufExprShimsSuite.scala b/sql-plugin/src/test/scala/com/nvidia/spark/rapids/shims/ProtobufExprShimsSuite.scala
index b0ec70abd1c..b375fd67f9b 100644
--- a/sql-plugin/src/test/scala/com/nvidia/spark/rapids/shims/ProtobufExprShimsSuite.scala
+++ b/sql-plugin/src/test/scala/com/nvidia/spark/rapids/shims/ProtobufExprShimsSuite.scala
@@ -107,8 +107,15 @@ class ProtobufExprShimsSuite extends AnyFunSuite {
       isRepeated: Boolean = false,
       isRequired: Boolean = false,
       defaultValue: Option[ProtobufDefaultValue] = None,
+      defaultValueError: Option[String] = None,
       enumMetadata: Option[ProtobufEnumMetadata] = None,
-      messageDescriptor: Option[ProtobufMessageDescriptor] = None) extends ProtobufFieldDescriptor
+      messageDescriptor: Option[ProtobufMessageDescriptor] = None) extends ProtobufFieldDescriptor {
+    override lazy val defaultValueResult: Either[String, Option[ProtobufDefaultValue]] =
+      defaultValueError match {
+        case Some(reason) => Left(reason)
+        case None => Right(defaultValue)
+      }
+  }
 
   test("compat extracts descriptor path and options from legacy expression") {
     val exprInfo = SparkProtobufCompat.extractExprInfo(FakePathProtobufExpr(FakeExprChild()))
@@ -187,6 +194,25 @@ class ProtobufExprShimsSuite extends AnyFunSuite {
       ProtobufDefaultValue.EnumValue(1, "EN")))
   }
 
+  test("extractor reports default value reflection failures as cpu fallback reason") {
+    val msgDesc = FakeMessageDescriptor(
+      syntax = "PROTO2",
+      fields = Map(
+        "id" -> FakeFieldDescriptor(
+          name = "id",
+          fieldNumber = 1,
+          protoTypeName = "INT32",
+          defaultValueError =
+            Some("Failed to read protobuf default value for field 'id': unsupported type"))))
+    val schema = StructType(Seq(StructField("id", IntegerType, nullable = true)))
+
+    val infos = ProtobufSchemaExtractor.analyzeAllFields(
+      schema, msgDesc, enumsAsInts = true, "test.Message")
+
+    assert(infos.left.toOption.exists(
+      _.contains("Failed to read protobuf default value for field 'id'")))
+  }
+
   test("validator encodes enum-string defaults into both numeric and string payloads") {
     val enumMeta = ProtobufEnumMetadata(Seq(
       ProtobufEnumValue(0, "UNKNOWN"),
@@ -244,6 +270,31 @@ class ProtobufExprShimsSuite extends AnyFunSuite {
     assert(flat.left.toOption.exists(_.contains("missing enum metadata")))
   }
 
+  test("validator returns Left for incompatible default type instead of throwing") {
+    val info = ProtobufFieldInfo(
+      fieldNumber = 3,
+      protoTypeName = "FLOAT",
+      sparkType = DoubleType,
+      encoding = GpuFromProtobuf.ENC_DEFAULT,
+      isSupported = true,
+      unsupportedReason = None,
+      isRequired = false,
+      defaultValue = Some(ProtobufDefaultValue.FloatValue(1.5f)),
+      enumMetadata = None,
+      isRepeated = false)
+
+    val flat = ProtobufSchemaValidator.toFlattenedFieldDescriptor(
+      path = "common.score",
+      field = StructField("score", DoubleType, nullable = true),
+      fieldInfo = info,
+      parentIdx = 0,
+      depth = 1,
+      outputTypeId = 6)
+
+    assert(flat.left.toOption.exists(
+      _.contains("Incompatible default value for protobuf field 'common.score'")))
+  }
+
   test("GpuFromProtobuf semantic equality is content-based for schema arrays") {
     def emptyEnumNames: Array[Array[Byte]] = Array.empty[Array[Byte]]
 

From c6477769fb38c94577490e8317d647f41b5e573d Mon Sep 17 00:00:00 2001
From: Haoyang Li <haoyangl@nvidia.com>
Date: Thu, 12 Mar 2026 15:23:33 +0800
Subject: [PATCH 31/33] address comments

Signed-off-by: Haoyang Li <haoyangl@nvidia.com>
---
 .../sql/rapids/complexTypeExtractors.scala    | 20 ++++++++-
 .../rapids/protobuf/ProtobufSchemaModel.scala | 11 ++++-
 .../rapids/shims/ProtobufExprShimsSuite.scala | 43 ++++++++++++++++++-
 3 files changed, 71 insertions(+), 3 deletions(-)

diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/complexTypeExtractors.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/complexTypeExtractors.scala
index b0d80fb6a8d..988c03b00b1 100644
--- a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/complexTypeExtractors.scala
+++ b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/complexTypeExtractors.scala
@@ -445,8 +445,26 @@ class GpuGetArrayStructFieldsMeta(
       GpuStructFieldOrdinalTag.PRUNED_ORDINAL_TAG).getOrElse(-1)
     val effectiveOrd =
       if (runtimeOrd >= 0) runtimeOrd else expr.ordinal
+    val effectiveNumFields =
+      GpuGetArrayStructFieldsMeta.effectiveNumFields(child, expr, runtimeOrd)
     GpuGetArrayStructFields(child, expr.field,
-      effectiveOrd, expr.numFields, expr.containsNull)
+      effectiveOrd, effectiveNumFields, expr.containsNull)
+  }
+}
+
+object GpuGetArrayStructFieldsMeta {
+  def effectiveNumFields(
+      child: Expression,
+      expr: GetArrayStructFields,
+      runtimeOrd: Int): Int = {
+    if (runtimeOrd >= 0) {
+      child.dataType match {
+        case ArrayType(st: StructType, _) => st.fields.length
+        case _ => expr.numFields
+      }
+    } else {
+      expr.numFields
+    }
   }
 }
 
diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/protobuf/ProtobufSchemaModel.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/protobuf/ProtobufSchemaModel.scala
index bf8969567e7..77c1cbdf8f3 100644
--- a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/protobuf/ProtobufSchemaModel.scala
+++ b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/protobuf/ProtobufSchemaModel.scala
@@ -16,13 +16,22 @@
 
 package org.apache.spark.sql.rapids.protobuf
 
+import java.util.Arrays
+
 import org.apache.spark.sql.types.DataType
 
 sealed trait ProtobufDescriptorSource
 
 object ProtobufDescriptorSource {
   final case class DescriptorPath(path: String) extends ProtobufDescriptorSource
-  final case class DescriptorBytes(bytes: Array[Byte]) extends ProtobufDescriptorSource
+  final case class DescriptorBytes(bytes: Array[Byte]) extends ProtobufDescriptorSource {
+    override def equals(other: Any): Boolean = other match {
+      case DescriptorBytes(otherBytes) => Arrays.equals(bytes, otherBytes)
+      case _ => false
+    }
+
+    override def hashCode(): Int = Arrays.hashCode(bytes)
+  }
 }
 
 final case class ProtobufExprInfo(
diff --git a/sql-plugin/src/test/scala/com/nvidia/spark/rapids/shims/ProtobufExprShimsSuite.scala b/sql-plugin/src/test/scala/com/nvidia/spark/rapids/shims/ProtobufExprShimsSuite.scala
index b375fd67f9b..b2d14205718 100644
--- a/sql-plugin/src/test/scala/com/nvidia/spark/rapids/shims/ProtobufExprShimsSuite.scala
+++ b/sql-plugin/src/test/scala/com/nvidia/spark/rapids/shims/ProtobufExprShimsSuite.scala
@@ -20,7 +20,12 @@ import org.scalatest.funsuite.AnyFunSuite
 
 import org.apache.spark.sql.catalyst.expressions.{Expression, UnaryExpression}
 import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, ExprCode}
-import org.apache.spark.sql.rapids.GpuFromProtobuf
+import org.apache.spark.sql.catalyst.expressions.GetArrayStructFields
+import org.apache.spark.sql.rapids.{
+  GpuFromProtobuf,
+  GpuGetArrayStructFieldsMeta,
+  GpuStructFieldOrdinalTag
+}
 import org.apache.spark.sql.rapids.protobuf._
 import org.apache.spark.sql.types._
 
@@ -94,6 +99,17 @@ class ProtobufExprShimsSuite extends AnyFunSuite {
     def options: scala.collection.Map[String, String] = Map("mode" -> "PERMISSIVE")
   }
 
+  private case class FakeTypedUnaryExpr(
+      dt: DataType,
+      override val child: Expression = FakeExprChild()) extends UnaryExpression {
+    override def nullable: Boolean = true
+    override def dataType: DataType = dt
+    override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode =
+      throw new UnsupportedOperationException("not needed")
+    override protected def withNewChildInternal(newChild: Expression): Expression = copy(child =
+      newChild)
+  }
+
   private case class FakeMessageDescriptor(
       syntax: String,
       fields: Map[String, ProtobufFieldDescriptor]) extends ProtobufMessageDescriptor {
@@ -147,6 +163,8 @@ class ProtobufExprShimsSuite extends AnyFunSuite {
 
     assert(SparkProtobufCompat.sameDecodeSemantics(
       FakePathProtobufExpr(child), FakePathProtobufExpr(child)))
+    assert(SparkProtobufCompat.sameDecodeSemantics(
+      FakeBytesProtobufExpr(child), FakeBytesProtobufExpr(child)))
     assert(!SparkProtobufCompat.sameDecodeSemantics(
       FakePathProtobufExpr(child), FakeDifferentMessageExpr(child)))
     assert(!SparkProtobufCompat.sameDecodeSemantics(
@@ -295,6 +313,29 @@ class ProtobufExprShimsSuite extends AnyFunSuite {
       _.contains("Incompatible default value for protobuf field 'common.score'")))
   }
 
+  test("array struct field meta uses pruned child field count after ordinal remap") {
+    val originalStruct = StructType(Seq(
+      StructField("a", IntegerType, nullable = true),
+      StructField("b", IntegerType, nullable = true),
+      StructField("c", IntegerType, nullable = true)))
+    val prunedStruct = StructType(Seq(StructField("b", IntegerType, nullable = true)))
+    val originalChild = FakeTypedUnaryExpr(ArrayType(originalStruct, containsNull = true))
+    val sparkExpr = GetArrayStructFields(
+      child = originalChild,
+      field = originalStruct.fields(1),
+      ordinal = 1,
+      numFields = originalStruct.fields.length,
+      containsNull = true)
+    sparkExpr.setTagValue(GpuStructFieldOrdinalTag.PRUNED_ORDINAL_TAG, 0)
+
+    val prunedChild = FakeTypedUnaryExpr(ArrayType(prunedStruct, containsNull = true))
+    val runtimeOrd = sparkExpr.getTagValue(GpuStructFieldOrdinalTag.PRUNED_ORDINAL_TAG).get
+
+    assert(runtimeOrd == 0)
+    assert(
+      GpuGetArrayStructFieldsMeta.effectiveNumFields(prunedChild, sparkExpr, runtimeOrd) == 1)
+  }
+
   test("GpuFromProtobuf semantic equality is content-based for schema arrays") {
     def emptyEnumNames: Array[Array[Byte]] = Array.empty[Array[Byte]]
 

From be7bdc661a6b4b51e1955ccfbf728d7e6fc1dd4a Mon Sep 17 00:00:00 2001
From: Haoyang Li <haoyangl@nvidia.com>
Date: Thu, 12 Mar 2026 16:17:34 +0800
Subject: [PATCH 32/33] address comments

Signed-off-by: Haoyang Li <haoyangl@nvidia.com>
---
 .../rapids/shims/SparkProtobufCompat.scala    | 30 +++++++++-
 .../rapids/shims/ProtobufExprShimsSuite.scala | 57 +++++++++++++++++++
 2 files changed, 85 insertions(+), 2 deletions(-)

diff --git a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/SparkProtobufCompat.scala b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/SparkProtobufCompat.scala
index 73ee429ceac..ef2ffc5ef57 100644
--- a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/SparkProtobufCompat.scala
+++ b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/SparkProtobufCompat.scala
@@ -37,6 +37,7 @@ spark-rapids-shim-json-lines ***/
 package com.nvidia.spark.rapids.shims
 
 import java.lang.reflect.Method
+import java.nio.file.{Files, Paths}
 
 import scala.util.Try
 
@@ -140,11 +141,36 @@ private[shims] object SparkProtobufCompat extends Logging {
     val module = cls.getField("MODULE$").get(null)
     val buildMethod = cls.getMethod("buildDescriptor", classOf[String], classOf[scala.Option[_]])
 
+    invokeBuildDescriptor(
+      buildMethod,
+      module,
+      messageName,
+      descriptorSource,
+      filePath => Files.readAllBytes(Paths.get(filePath)))
+  }
+
+  private[shims] def invokeBuildDescriptor(
+      buildMethod: Method,
+      module: AnyRef,
+      messageName: String,
+      descriptorSource: ProtobufDescriptorSource,
+      readDescriptorFile: String => Array[Byte]): AnyRef = {
     descriptorSource match {
-      case ProtobufDescriptorSource.DescriptorPath(filePath) =>
-        buildMethod.invoke(module, messageName, Some(filePath)).asInstanceOf[AnyRef]
       case ProtobufDescriptorSource.DescriptorBytes(bytes) =>
         buildMethod.invoke(module, messageName, Some(bytes)).asInstanceOf[AnyRef]
+      case ProtobufDescriptorSource.DescriptorPath(filePath) =>
+        try {
+          buildMethod.invoke(module, messageName, Some(filePath)).asInstanceOf[AnyRef]
+        } catch {
+          // Spark 3.5+ changed the descriptor payload from Option[String] to Option[Array[Byte]]
+          // while keeping the same erased JVM signature. Retry with file contents when the
+          // path-based invocation clearly hit that binary-descriptor variant.
+          case ex: java.lang.reflect.InvocationTargetException
+              if ex.getCause.isInstanceOf[ClassCastException] ||
+                  ex.getCause.isInstanceOf[MatchError] =>
+            buildMethod.invoke(
+              module, messageName, Some(readDescriptorFile(filePath))).asInstanceOf[AnyRef]
+        }
     }
   }
 
diff --git a/sql-plugin/src/test/scala/com/nvidia/spark/rapids/shims/ProtobufExprShimsSuite.scala b/sql-plugin/src/test/scala/com/nvidia/spark/rapids/shims/ProtobufExprShimsSuite.scala
index b2d14205718..581aac667be 100644
--- a/sql-plugin/src/test/scala/com/nvidia/spark/rapids/shims/ProtobufExprShimsSuite.scala
+++ b/sql-plugin/src/test/scala/com/nvidia/spark/rapids/shims/ProtobufExprShimsSuite.scala
@@ -110,6 +110,16 @@ class ProtobufExprShimsSuite extends AnyFunSuite {
       newChild)
   }
 
+  private object FakeSpark34ProtobufUtils {
+    def buildDescriptor(messageName: String, descFilePath: Option[String]): String =
+      s"$messageName:${descFilePath.getOrElse("none")}"
+  }
+
+  private object FakeSpark35ProtobufUtils {
+    def buildDescriptor(messageName: String, binaryFileDescriptorSet: Option[Array[Byte]]): String =
+      s"$messageName:${binaryFileDescriptorSet.map(_.mkString(",")).getOrElse("none")}"
+  }
+
   private case class FakeMessageDescriptor(
       syntax: String,
       fields: Map[String, ProtobufFieldDescriptor]) extends ProtobufMessageDescriptor {
@@ -158,6 +168,53 @@ class ProtobufExprShimsSuite extends AnyFunSuite {
       Right(ProtobufPlannerOptions(enumsAsInts = true, failOnErrors = false)))
   }
 
+  test("compat invokes Spark 3.4 descriptor builder with descriptor path") {
+    val buildMethod = FakeSpark34ProtobufUtils.getClass.getMethod(
+      "buildDescriptor", classOf[String], classOf[scala.Option[_]])
+
+    val result = SparkProtobufCompat.invokeBuildDescriptor(
+      buildMethod,
+      FakeSpark34ProtobufUtils,
+      "test.Message",
+      ProtobufDescriptorSource.DescriptorPath("/tmp/test.desc"),
+      _ => fail("path-to-bytes fallback should not be needed for Spark 3.4"))
+
+    assert(result == "test.Message:/tmp/test.desc")
+  }
+
+  test("compat retries descriptor path as bytes for Spark 3.5 descriptor builder") {
+    val buildMethod = FakeSpark35ProtobufUtils.getClass.getMethod(
+      "buildDescriptor", classOf[String], classOf[scala.Option[_]])
+    var readCalls = 0
+
+    val result = SparkProtobufCompat.invokeBuildDescriptor(
+      buildMethod,
+      FakeSpark35ProtobufUtils,
+      "test.Message",
+      ProtobufDescriptorSource.DescriptorPath("/tmp/test.desc"),
+      _ => {
+        readCalls += 1
+        Array[Byte](1, 2, 3)
+      })
+
+    assert(readCalls == 1)
+    assert(result == "test.Message:1,2,3")
+  }
+
+  test("compat passes bytes directly to Spark 3.5 descriptor builder") {
+    val buildMethod = FakeSpark35ProtobufUtils.getClass.getMethod(
+      "buildDescriptor", classOf[String], classOf[scala.Option[_]])
+
+    val result = SparkProtobufCompat.invokeBuildDescriptor(
+      buildMethod,
+      FakeSpark35ProtobufUtils,
+      "test.Message",
+      ProtobufDescriptorSource.DescriptorBytes(Array[Byte](4, 5, 6)),
+      _ => fail("binary descriptor source should not read a file"))
+
+    assert(result == "test.Message:4,5,6")
+  }
+
   test("compat distinguishes decode semantics across message descriptor and options") {
     val child = FakeExprChild()
 

From 16f2f6e6ffd9198560afb35cb5b64ac135d9f823 Mon Sep 17 00:00:00 2001
From: Haoyang Li <haoyangl@nvidia.com>
Date: Thu, 12 Mar 2026 16:55:18 +0800
Subject: [PATCH 33/33] address commmit

Signed-off-by: Haoyang Li <haoyangl@nvidia.com>
---
 integration_tests/src/main/python/data_gen.py | 17 --------
 .../protobuf/ProtobufSchemaExtractor.scala    | 40 +++++++++++++++----
 .../rapids/shims/ProtobufExprShims.scala      |  3 ++
 .../rapids/shims/ProtobufExprShimsSuite.scala | 17 ++++++--
 4 files changed, 48 insertions(+), 29 deletions(-)

diff --git a/integration_tests/src/main/python/data_gen.py b/integration_tests/src/main/python/data_gen.py
index 965c0e2d4c0..d6583e3fd00 100644
--- a/integration_tests/src/main/python/data_gen.py
+++ b/integration_tests/src/main/python/data_gen.py
@@ -953,23 +953,6 @@ def _encode_protobuf_field(field_number, spark_type, value, encoding='default'):
     else:
         raise ValueError("Unsupported type for protobuf simple generator: {}".format(spark_type))
 
-def _encode_protobuf_nested_message(field_number, child_fields, child_values):
-    """
-    Encode a nested protobuf message.
-    
-    child_fields: list of (field_name, field_number, spark_type, encoding)
-    child_values: dict mapping field_name to value
-    """
-    # First encode all child fields
-    child_encoded = b""
-    for (name, num, spark_type, enc) in child_fields:
-        if name in child_values and child_values[name] is not None:
-            child_encoded += _encode_protobuf_field(num, spark_type, child_values[name], enc)
-    
-    # Then wrap in length-delimited format
-    return (_encode_protobuf_key(field_number, _PROTOBUF_WIRE_LEN_DELIM) +
-            _encode_protobuf_uvarint(len(child_encoded)) + child_encoded)
-
 def _encode_protobuf_repeated_field(field_number, spark_element_type, values, encoding='default'):
     """
     Encode a repeated (non-packed) protobuf field.
diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/protobuf/ProtobufSchemaExtractor.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/protobuf/ProtobufSchemaExtractor.scala
index 4fafb325da8..a6617e1c89a 100644
--- a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/protobuf/ProtobufSchemaExtractor.scala
+++ b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/protobuf/ProtobufSchemaExtractor.scala
@@ -32,15 +32,21 @@ object ProtobufSchemaExtractor {
     val result = mutable.Map[String, ProtobufFieldInfo]()
 
     schema.fields.foreach { sf =>
-      val fd = msgDesc.findField(sf.name).getOrElse {
-        return Left(s"Protobuf field '${sf.name}' not found in message '$messageName'")
-      }
-      extractFieldInfo(sf, fd, enumsAsInts) match {
-        case Right(fieldInfo) =>
-          result(sf.name) = fieldInfo
-        case Left(reason) =>
-          return Left(reason)
+      val fieldInfo = msgDesc.findField(sf.name) match {
+        case None =>
+          unsupportedFieldInfo(
+            sf,
+            None,
+            s"Protobuf field '${sf.name}' not found in message '$messageName'")
+        case Some(fd) =>
+          extractFieldInfo(sf, fd, enumsAsInts) match {
+            case Right(info) =>
+              info
+            case Left(reason) =>
+              unsupportedFieldInfo(sf, Some(fd), reason)
+          }
       }
+      result(sf.name) = fieldInfo
     }
 
     Right(result.toMap)
@@ -73,6 +79,24 @@ object ProtobufSchemaExtractor {
     }
   }
 
+  private def unsupportedFieldInfo(
+      sparkField: StructField,
+      fieldDescriptor: Option[ProtobufFieldDescriptor],
+      reason: String): ProtobufFieldInfo = {
+    ProtobufFieldInfo(
+      fieldNumber = fieldDescriptor.map(_.fieldNumber).getOrElse(-1),
+      protoTypeName = fieldDescriptor.map(_.protoTypeName).getOrElse("UNKNOWN"),
+      sparkType = sparkField.dataType,
+      encoding = GpuFromProtobuf.ENC_DEFAULT,
+      isSupported = false,
+      unsupportedReason = Some(reason),
+      isRequired = fieldDescriptor.exists(_.isRequired),
+      defaultValue = None,
+      enumMetadata = fieldDescriptor.flatMap(_.enumMetadata),
+      isRepeated = fieldDescriptor.exists(_.isRepeated)
+    )
+  }
+
   def checkFieldSupport(
       sparkType: DataType,
       protoTypeName: String,
diff --git a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala
index 4634d4da429..44f67ffff10 100644
--- a/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala
+++ b/sql-plugin/src/main/spark340/scala/com/nvidia/spark/rapids/shims/ProtobufExprShims.scala
@@ -311,16 +311,19 @@ object ProtobufExprShims extends org.apache.spark.internal.Logging {
                           willNotWorkOnGpu(
                             s"Nested field '${childSf.name}' not found in protobuf " +
                               s"descriptor for message at '$path'")
+                          return
                         case Some(childFd) =>
                           ProtobufSchemaExtractor
                             .extractFieldInfo(childSf, childFd, enumsAsInts) match {
                             case Left(reason) =>
                               willNotWorkOnGpu(reason)
+                              return
                             case Right(childInfo) =>
                               if (!childInfo.isSupported) {
                                 willNotWorkOnGpu(
                                   s"Nested field '${childSf.name}' at '$path': " +
                                     childInfo.unsupportedReason.getOrElse("unsupported type"))
+                                return
                               } else {
                                 addFieldWithChildren(
                                   childSf, childInfo, parentIdx, parentDepth + 1, childMsgDesc,
diff --git a/sql-plugin/src/test/scala/com/nvidia/spark/rapids/shims/ProtobufExprShimsSuite.scala b/sql-plugin/src/test/scala/com/nvidia/spark/rapids/shims/ProtobufExprShimsSuite.scala
index 581aac667be..1ab2c16cdf6 100644
--- a/sql-plugin/src/test/scala/com/nvidia/spark/rapids/shims/ProtobufExprShimsSuite.scala
+++ b/sql-plugin/src/test/scala/com/nvidia/spark/rapids/shims/ProtobufExprShimsSuite.scala
@@ -269,22 +269,31 @@ class ProtobufExprShimsSuite extends AnyFunSuite {
       ProtobufDefaultValue.EnumValue(1, "EN")))
   }
 
-  test("extractor reports default value reflection failures as cpu fallback reason") {
+  test("extractor records reflection failures as unsupported field info") {
     val msgDesc = FakeMessageDescriptor(
       syntax = "PROTO2",
       fields = Map(
+        "ok" -> FakeFieldDescriptor(
+          name = "ok",
+          fieldNumber = 1,
+          protoTypeName = "INT32"),
         "id" -> FakeFieldDescriptor(
           name = "id",
-          fieldNumber = 1,
+          fieldNumber = 2,
           protoTypeName = "INT32",
           defaultValueError =
             Some("Failed to read protobuf default value for field 'id': unsupported type"))))
-    val schema = StructType(Seq(StructField("id", IntegerType, nullable = true)))
+    val schema = StructType(Seq(
+      StructField("ok", IntegerType, nullable = true),
+      StructField("id", IntegerType, nullable = true)))
 
     val infos = ProtobufSchemaExtractor.analyzeAllFields(
       schema, msgDesc, enumsAsInts = true, "test.Message")
 
-    assert(infos.left.toOption.exists(
+    assert(infos.isRight)
+    assert(infos.toOption.get("ok").isSupported)
+    assert(!infos.toOption.get("id").isSupported)
+    assert(infos.toOption.get("id").unsupportedReason.exists(
       _.contains("Failed to read protobuf default value for field 'id'")))
   }