dekaf: Implement e2e testing framework

.config/nextest.toml

@@ -4,11 +4,17 @@ serial-db-tests = { max-threads = 1 }
 
 [profile.default]
 slow-timeout = { period = "30s", terminate-after = 4 }
+# Exclude e2e tests from Dekaf by default because they require special setup.
+default-filter = 'not binary_id(/dekaf::e2e/)'
+
+[profile.dekaf-e2e]
+# Run only Dekaf e2e tests
+default-filter = 'binary_id(/dekaf::e2e/)'
 
 [[profile.default.overrides]]
 filter = 'package(agent) and (test(discovers::) + test(directives::) + test(integration_tests::))'
 test-group = 'serial-db-tests'
 
 [[profile.default.overrides]]
 filter = 'package(automations) and test(test_fibonacci_bench)'
-test-group = 'serial-db-tests'
+test-group = 'serial-db-tests'

.github/workflows/dekaf-test.yaml

@@ -0,0 +1,56 @@
+name: Dekaf Test
+permissions:
+    contents: read
+
+on:
+    pull_request:
+        branches: [master]
+        paths-ignore:
+            - "site/**"
+
+concurrency:
+    group: ${{ github.workflow }}-${{ github.ref }}
+    cancel-in-progress: true
+
+jobs:
+    dekaf-test:
+        name: Dekaf Test
+        runs-on: ubuntu-2404-large
+        steps:
+            - uses: actions/checkout@v5
+              with:
+                  submodules: true # Official JSON test data.
+                  lfs: true # Test fixtures.
+
+            - name: Install build tools
+              uses: jdx/mise-action@9dc7d5dd454262207dea3ab5a06a3df6afc8ff26 # v3.4.1
+            - run: rustup upgrade
+
+            - name: Cache Rust workspace
+              uses: Swatinem/rust-cache@f13886b937689c021905a6b90929199931d60db1 # v2.8.1
+              with:
+                  workspaces: ". -> ../../../cargo-target"
+
+            - name: Cache RocksDB
+              uses: actions/cache@v4
+              with:
+                  key: rocksdb-${{ runner.os }}-${{ runner.arch }}-${{ env.ROCKSDB_VERSION }}
+                  path: |
+                      ~/rocksdb-${{ env.ROCKSDB_VERSION }}/include/
+                      ~/rocksdb-${{ env.ROCKSDB_VERSION }}/lib/
+
+            - name: Cache/Restore Go workspace.
+              uses: actions/cache@v4
+              with:
+                  path: ~/go/pkg/mod
+                  key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+
+            - uses: mozilla-actions/sccache-action@v0.0.9
+            - run: echo 'SCCACHE_GHA_ENABLED=true' >> $GITHUB_ENV
+            - run: mise run build:rocksdb
+            - run: mise run ci:musl-dev
+            - run: mise run ci:gnu-dev
+            - run: mise run build:gazette
+            - run: mise run build:flowctl-go
+            - run: mise run local:stack
+            - run: mise run ci:dekaf-e2e

crates/dekaf/Cargo.toml

@@ -94,5 +94,7 @@ apache-avro = { workspace = true }
 insta = { workspace = true }
 rdkafka = { workspace = true }
 schema_registry_converter = { workspace = true }
+serde_yaml = { workspace = true }
 tempfile = { workspace = true }
 tracing-test = { workspace = true }
+uuid = { workspace = true, features = ["v4"] }