From 5cb33c9e9d9a2967a1e1cc753e3c308e9ef9e739 Mon Sep 17 00:00:00 2001 From: Brent Johnson Date: Sun, 1 Jun 2025 13:23:15 -0400 Subject: [PATCH 01/49] Update .gitignore --- .gitignore | 76 +++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 58 insertions(+), 18 deletions(-) diff --git a/.gitignore b/.gitignore index 524f096..09ea567 100644 --- a/.gitignore +++ b/.gitignore @@ -1,24 +1,64 @@ -# Compiled class file -*.class +# Gradle +.gradle/ +build/ +gradle-app.setting +!gradle-wrapper.jar +!gradle-wrapper.properties -# Log file +# IntelliJ IDEA +.idea/ +*.iml +*.ipr +*.iws + +# Eclipse +.project +.classpath +.settings/ +bin/ + +# NetBeans +nbproject/private/ +build/ +nbbuild/ +dist/ +nbdist/ +.nb-gradle/ + +# VS Code +.vscode/ + +# macOS +.DS_Store + +# Windows +Thumbs.db +ehthumbs.db + +# JMH +jmh-result.json + +# Logs *.log -# BlueJ files -*.ctxt +# Temporary files +*.tmp +*.temp -# Mobile Tools for Java (J2ME) -.mtj.tmp/ +# JVM Crash logs +hs_err_pid* -# Package Files # +# Maven (in case someone uses Maven) +target/ +pom.xml.tag +pom.xml.releaseBackup +pom.xml.versionsBackup +pom.xml.next +release.properties +dependency-reduced-pom.xml +buildNumber.properties +.mvn/timing.properties + +# JAR files (unless they're dependencies) *.jar -*.war -*.nar -*.ear -*.zip -*.tar.gz -*.rar - -# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml -hs_err_pid* -replay_pid* +!lombok.jar From 63313e4d81404b40ec649369286b47856574a002 Mon Sep 17 00:00:00 2001 From: expand3d <> Date: Sun, 1 Jun 2025 14:02:51 -0400 Subject: [PATCH 02/49] initial commit for imprint-java --- PROFILING.md | 106 +++++ gradle/wrapper/gradle-wrapper.properties | 7 + gradlew | 251 +++++++++++ gradlew.bat | 94 ++++ profile.sh | 98 +++++ settings.gradle | 1 + .../benchmark/ComparisonBenchmark.java | 344 +++++++++++++++ .../benchmark/FieldAccessBenchmark.java | 276 ++++++++++++ .../com/imprint/benchmark/MergeBenchmark.java | 163 +++++++ .../benchmark/SerializationBenchmark.java | 166 ++++++++ src/main/java/com/imprint/Constants.java | 10 + .../java/com/imprint/core/DirectoryEntry.java | 23 + src/main/java/com/imprint/core/Flags.java | 12 + src/main/java/com/imprint/core/Header.java | 13 + .../java/com/imprint/core/ImprintRecord.java | 365 ++++++++++++++++ .../imprint/core/ImprintRecordBuilder.java | 233 ++++++++++ .../java/com/imprint/core/ImprintWriter.java | 257 +++++++++++ src/main/java/com/imprint/core/SchemaId.java | 12 + .../java/com/imprint/error/ErrorType.java | 18 + .../com/imprint/error/ImprintException.java | 26 ++ src/main/java/com/imprint/types/MapKey.java | 163 +++++++ src/main/java/com/imprint/types/TypeCode.java | 48 +++ .../java/com/imprint/types/TypeHandler.java | 304 +++++++++++++ src/main/java/com/imprint/types/Value.java | 403 ++++++++++++++++++ src/main/java/com/imprint/util/VarInt.java | 118 +++++ .../imprint/ByteBufferIntegrationTest.java | 87 ++++ .../java/com/imprint/ComprehensiveTest.java | 208 +++++++++ .../java/com/imprint/IntegrationTest.java | 144 +++++++ .../com/imprint/benchmark/ProfilerTest.java | 226 ++++++++++ .../core/ImprintRecordBuilderTest.java | 234 ++++++++++ .../com/imprint/core/ImprintRecordTest.java | 232 ++++++++++ .../java/com/imprint/types/MapKeyTest.java | 91 ++++ .../com/imprint/types/TypeHandlerTest.java | 279 ++++++++++++ .../java/com/imprint/types/ValueTest.java | 123 ++++++ .../java/com/imprint/util/VarIntTest.java | 115 +++++ 35 files changed, 5250 insertions(+) create mode 100644 PROFILING.md create mode 100644 gradle/wrapper/gradle-wrapper.properties create mode 100644 gradlew create mode 100644 gradlew.bat create mode 100644 profile.sh create mode 100644 settings.gradle create mode 100644 src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java create mode 100644 src/jmh/java/com/imprint/benchmark/FieldAccessBenchmark.java create mode 100644 src/jmh/java/com/imprint/benchmark/MergeBenchmark.java create mode 100644 src/jmh/java/com/imprint/benchmark/SerializationBenchmark.java create mode 100644 src/main/java/com/imprint/Constants.java create mode 100644 src/main/java/com/imprint/core/DirectoryEntry.java create mode 100644 src/main/java/com/imprint/core/Flags.java create mode 100644 src/main/java/com/imprint/core/Header.java create mode 100644 src/main/java/com/imprint/core/ImprintRecord.java create mode 100644 src/main/java/com/imprint/core/ImprintRecordBuilder.java create mode 100644 src/main/java/com/imprint/core/ImprintWriter.java create mode 100644 src/main/java/com/imprint/core/SchemaId.java create mode 100644 src/main/java/com/imprint/error/ErrorType.java create mode 100644 src/main/java/com/imprint/error/ImprintException.java create mode 100644 src/main/java/com/imprint/types/MapKey.java create mode 100644 src/main/java/com/imprint/types/TypeCode.java create mode 100644 src/main/java/com/imprint/types/TypeHandler.java create mode 100644 src/main/java/com/imprint/types/Value.java create mode 100644 src/main/java/com/imprint/util/VarInt.java create mode 100644 src/test/java/com/imprint/ByteBufferIntegrationTest.java create mode 100644 src/test/java/com/imprint/ComprehensiveTest.java create mode 100644 src/test/java/com/imprint/IntegrationTest.java create mode 100644 src/test/java/com/imprint/benchmark/ProfilerTest.java create mode 100644 src/test/java/com/imprint/core/ImprintRecordBuilderTest.java create mode 100644 src/test/java/com/imprint/core/ImprintRecordTest.java create mode 100644 src/test/java/com/imprint/types/MapKeyTest.java create mode 100644 src/test/java/com/imprint/types/TypeHandlerTest.java create mode 100644 src/test/java/com/imprint/types/ValueTest.java create mode 100644 src/test/java/com/imprint/util/VarIntTest.java diff --git a/PROFILING.md b/PROFILING.md new file mode 100644 index 0000000..43f7b4e --- /dev/null +++ b/PROFILING.md @@ -0,0 +1,106 @@ +# Performance Profiling Guide + +This guide helps developers identify performance hotspots in the Imprint Java implementation. + +## Quick Start + +```bash +# Run field access profiling with async-profiler +./profile.sh profileFieldAccess asyncprofiler + +# Run memory allocation profiling with JFR +./profile.sh profileMemoryAllocation jfr +``` + +## Available Tests + +1. **`profileFieldAccess`** - Measures random field access patterns + - Focus: Binary search, TypeHandler dispatch, string decoding + - Good for: Optimizing read-heavy workloads + +2. **`profileSerialization`** - Tests record creation and serialization + - Focus: Object allocation, ByteBuffer operations, encoding + - Good for: Optimizing write-heavy workloads + +3. **`profileProjection`** - Simulates analytical field projection + - Focus: Bulk field access, string materialization + - Good for: Optimizing analytical workloads + +4. **`profileMemoryAllocation`** - Stress tests allocation patterns + - Focus: GC pressure, object lifecycle, string allocations + - Good for: Reducing memory footprint + +## Profiler Options + +### async-profiler (Recommended) +- **Setup**: Download from [async-profiler releases](https://github.com/jvm-profiling-tools/async-profiler/releases) +- **Output**: HTML flame graphs in `profiler-results/` +- **Best for**: CPU profiling, finding hot methods + +### Java Flight Recorder (JFR) +- **Setup**: Built into OpenJDK 11+ +- **Output**: `.jfr` files for Java Mission Control +- **Best for**: Memory profiling, GC analysis + +### VisualVM +- **Setup**: `jvisualvm` (usually pre-installed) +- **Output**: Real-time profiling UI +- **Best for**: Interactive profiling, heap dumps + +## Expected Hotspots + +Based on our optimizations, watch for: + +### CPU Hotspots +1. **Binary search** in `findDirectoryIndex()` - should be fast +2. **String decoding** in `StringBufferValue.getValue()` - lazy evaluation +3. **TypeHandler dispatch** - interface calls vs switch statements +4. **VarInt encoding/decoding** - variable-length integers +5. **ByteBuffer operations** - slicing and positioning + +### Memory Hotspots +1. **String allocations** during UTF-8 conversion +2. **Temporary objects** in binary search (should be eliminated) +3. **ByteBuffer slicing** (should be zero-copy) +4. **Array allocations** for BYTES values + +## Analyzing Results + +### async-profiler Flame Graphs +- **Wide bars** = high CPU usage (hotspots) +- **Deep stacks** = call overhead +- **Look for**: Red bars in `deserializeValue`, `findDirectoryIndex`, string operations + +### JFR Analysis +1. Open `.jfr` file in Java Mission Control +2. Check "Memory" tab for allocation hotspots +3. Check "Method Profiling" for CPU usage +4. Look at GC events for memory pressure + +### Memory Profiler Tips +- **Object allocation rate** should be low for zero-copy operations +- **String allocations** should be rare (lazy evaluation) +- **GC frequency** indicates allocation pressure + +## Performance Targets + +Based on our benchmarks: +- **Single field access**: < 50ns +- **Zero-copy operations**: < 30ns +- **String decoding**: Should be lazy, not in hot path +- **Binary search**: O(log n), ~10ns per comparison + +## Common Issues + +1. **High string allocation** → Enable lazy string decoding +2. **Object allocations in binary search** → Check DirectoryEntry creation +3. **ByteBuffer copying** → Ensure zero-copy slicing +4. **Switch statement overhead** → TypeHandler dispatch working? + +## Profiling Best Practices + +1. **Warm up JVM** - Run tests multiple times +2. **Use realistic data** - Match production patterns +3. **Profile different scenarios** - Read vs write heavy +4. **Check allocations** - Memory profiling reveals hidden costs +5. **Compare before/after** - Measure optimization impact \ No newline at end of file diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 0000000..37f853b --- /dev/null +++ b/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,7 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-8.13-bin.zip +networkTimeout=10000 +validateDistributionUrl=true +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/gradlew b/gradlew new file mode 100644 index 0000000..faf9300 --- /dev/null +++ b/gradlew @@ -0,0 +1,251 @@ +#!/bin/sh + +# +# Copyright © 2015-2021 the original authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# + +############################################################################## +# +# Gradle start up script for POSIX generated by Gradle. +# +# Important for running: +# +# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is +# noncompliant, but you have some other compliant shell such as ksh or +# bash, then to run this script, type that shell name before the whole +# command line, like: +# +# ksh Gradle +# +# Busybox and similar reduced shells will NOT work, because this script +# requires all of these POSIX shell features: +# * functions; +# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», +# «${var#prefix}», «${var%suffix}», and «$( cmd )»; +# * compound commands having a testable exit status, especially «case»; +# * various built-in commands including «command», «set», and «ulimit». +# +# Important for patching: +# +# (2) This script targets any POSIX shell, so it avoids extensions provided +# by Bash, Ksh, etc; in particular arrays are avoided. +# +# The "traditional" practice of packing multiple parameters into a +# space-separated string is a well documented source of bugs and security +# problems, so this is (mostly) avoided, by progressively accumulating +# options in "$@", and eventually passing that to Java. +# +# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, +# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; +# see the in-line comments for details. +# +# There are tweaks for specific operating systems such as AIX, CygWin, +# Darwin, MinGW, and NonStop. +# +# (3) This script is generated from the Groovy template +# https://github.com/gradle/gradle/blob/HEAD/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# within the Gradle project. +# +# You can find Gradle at https://github.com/gradle/gradle/. +# +############################################################################## + +# Attempt to set APP_HOME + +# Resolve links: $0 may be a link +app_path=$0 + +# Need this for daisy-chained symlinks. +while + APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path + [ -h "$app_path" ] +do + ls=$( ls -ld "$app_path" ) + link=${ls#*' -> '} + case $link in #( + /*) app_path=$link ;; #( + *) app_path=$APP_HOME$link ;; + esac +done + +# This is normally unused +# shellcheck disable=SC2034 +APP_BASE_NAME=${0##*/} +# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) +APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s\n' "$PWD" ) || exit + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD=maximum + +warn () { + echo "$*" +} >&2 + +die () { + echo + echo "$*" + echo + exit 1 +} >&2 + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "$( uname )" in #( + CYGWIN* ) cygwin=true ;; #( + Darwin* ) darwin=true ;; #( + MSYS* | MINGW* ) msys=true ;; #( + NONSTOP* ) nonstop=true ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD=$JAVA_HOME/jre/sh/java + else + JAVACMD=$JAVA_HOME/bin/java + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD=java + if ! command -v java >/dev/null 2>&1 + then + die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +fi + +# Increase the maximum file descriptors if we can. +if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then + case $MAX_FD in #( + max*) + # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + MAX_FD=$( ulimit -H -n ) || + warn "Could not query maximum file descriptor limit" + esac + case $MAX_FD in #( + '' | soft) :;; #( + *) + # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + ulimit -n "$MAX_FD" || + warn "Could not set maximum file descriptor limit to $MAX_FD" + esac +fi + +# Collect all arguments for the java command, stacking in reverse order: +# * args from the command line +# * the main class name +# * -classpath +# * -D...appname settings +# * --module-path (only if needed) +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. + +# For Cygwin or MSYS, switch paths to Windows format before running java +if "$cygwin" || "$msys" ; then + APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) + CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) + + JAVACMD=$( cygpath --unix "$JAVACMD" ) + + # Now convert the arguments - kludge to limit ourselves to /bin/sh + for arg do + if + case $arg in #( + -*) false ;; # don't mess with options #( + /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath + [ -e "$t" ] ;; #( + *) false ;; + esac + then + arg=$( cygpath --path --ignore --mixed "$arg" ) + fi + # Roll the args list around exactly as many times as the number of + # args, so each arg winds up back in the position where it started, but + # possibly modified. + # + # NB: a `for` loop captures its iteration list before it begins, so + # changing the positional parameters here affects neither the number of + # iterations, nor the values presented in `arg`. + shift # remove old arg + set -- "$@" "$arg" # push replacement arg + done +fi + + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Collect all arguments for the java command: +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, +# and any embedded shellness will be escaped. +# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be +# treated as '${Hostname}' itself on the command line. + +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -classpath "$CLASSPATH" \ + org.gradle.wrapper.GradleWrapperMain \ + "$@" + +# Stop when "xargs" is not available. +if ! command -v xargs >/dev/null 2>&1 +then + die "xargs is not available" +fi + +# Use "xargs" to parse quoted args. +# +# With -n1 it outputs one arg per line, with the quotes and backslashes removed. +# +# In Bash we could simply go: +# +# readarray ARGS < <( xargs -n1 <<<"$var" ) && +# set -- "${ARGS[@]}" "$@" +# +# but POSIX shell has neither arrays nor command substitution, so instead we +# post-process each arg (as a line of input to sed) to backslash-escape any +# character that might be a shell metacharacter, then use eval to reverse +# that process (while maintaining the separation between arguments), and wrap +# the whole thing up as a single "set" statement. +# +# This will of course break if any of these variables contains a newline or +# an unmatched quote. +# + +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | + tr '\n' ' ' + )" '"$@"' + +exec "$JAVACMD" "$@" diff --git a/gradlew.bat b/gradlew.bat new file mode 100644 index 0000000..9d21a21 --- /dev/null +++ b/gradlew.bat @@ -0,0 +1,94 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem +@rem SPDX-License-Identifier: Apache-2.0 +@rem + +@if "%DEBUG%"=="" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%"=="" set DIRNAME=. +@rem This is normally unused +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if %ERRORLEVEL% equ 0 goto execute + +echo. 1>&2 +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. 1>&2 +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 + +goto fail + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* + +:end +@rem End local scope for the variables with windows NT shell +if %ERRORLEVEL% equ 0 goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +set EXIT_CODE=%ERRORLEVEL% +if %EXIT_CODE% equ 0 set EXIT_CODE=1 +if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% +exit /b %EXIT_CODE% + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/profile.sh b/profile.sh new file mode 100644 index 0000000..1e618af --- /dev/null +++ b/profile.sh @@ -0,0 +1,98 @@ +#!/bin/bash + +# Profiling helper script for Imprint Java implementation +# +# Usage: +# ./profile.sh [test_method] [profiler] +# +# test_method: profileFieldAccess, profileSerialization, profileProjection, profileMemoryAllocation +# profiler: asyncprofiler, jfr, visualvm +# +# Examples: +# ./profile.sh profileFieldAccess asyncprofiler +# ./profile.sh profileSerialization jfr +# ./profile.sh profileMemoryAllocation + +set -e + +TEST_METHOD=${1:-profileFieldAccess} +PROFILER=${2:-asyncprofiler} + +echo "🔬 Starting profiling session for $TEST_METHOD using $PROFILER" + +# Enable the profiler test by removing @Disabled +sed -i 's/@Disabled.*/@Test/' src/test/java/com/imprint/benchmark/ProfilerTest.java + +case $PROFILER in + "asyncprofiler") + echo "📊 Using async-profiler (download from https://github.com/jvm-profiling-tools/async-profiler)" + echo " Will generate CPU profile in profiler-results/" + mkdir -p profiler-results + + # Run test in background and profile it + ./gradlew test --tests "*ProfilerTest.$TEST_METHOD" \ + -Dorg.gradle.jvmargs="-XX:+UnlockDiagnosticVMOptions -XX:+DebugNonSafepoints" & + + TEST_PID=$! + sleep 2 + + # Find the actual Java process (Gradle daemon) + JAVA_PID=$(pgrep -f "ProfilerTest.$TEST_METHOD" | head -1) + + if [ -n "$JAVA_PID" ]; then + echo " Profiling Java process $JAVA_PID" + if command -v async-profiler.jar >/dev/null 2>&1; then + java -jar async-profiler.jar -d 30 -f profiler-results/profile-$TEST_METHOD.html $JAVA_PID + else + echo " ⚠️ async-profiler.jar not found in PATH" + echo " 📥 Download from: https://github.com/jvm-profiling-tools/async-profiler/releases" + fi + fi + + wait $TEST_PID + ;; + + "jfr") + echo "📊 Using Java Flight Recorder" + mkdir -p profiler-results + + ./gradlew test --tests "*ProfilerTest.$TEST_METHOD" \ + -Dorg.gradle.jvmargs="-XX:+FlightRecorder -XX:StartFlightRecording=duration=60s,filename=profiler-results/profile-$TEST_METHOD.jfr,settings=profile" + + echo " 📂 JFR file saved to: profiler-results/profile-$TEST_METHOD.jfr" + echo " 🔍 Open with: jmc profiler-results/profile-$TEST_METHOD.jfr" + ;; + + "visualvm") + echo "📊 Using VisualVM" + echo " 1. Start VisualVM: jvisualvm" + echo " 2. Enable the ProfilerTest manually" + echo " 3. Run: ./gradlew test --tests '*ProfilerTest.$TEST_METHOD' --debug-jvm" + echo " 4. Attach VisualVM to the Gradle daemon process" + echo " 5. Start CPU/Memory profiling" + + read -p "Press Enter when VisualVM is ready..." + ./gradlew test --tests "*ProfilerTest.$TEST_METHOD" --debug-jvm + ;; + + *) + echo "❌ Unknown profiler: $PROFILER" + echo " Supported: asyncprofiler, jfr, visualvm" + exit 1 + ;; +esac + +# Restore @Disabled annotation +sed -i 's/@Test/@Disabled("Enable manually for profiling")/' src/test/java/com/imprint/benchmark/ProfilerTest.java + +echo "✅ Profiling complete!" +echo "" +echo "🔍 Key areas to examine:" +echo " • Object allocation hotspots (new, arrays, strings)" +echo " • ByteBuffer operations and slicing" +echo " • String UTF-8 encoding/decoding" +echo " • Binary search in directory lookup" +echo " • TypeHandler method dispatch" +echo " • VarInt encoding/decoding" +echo "" +echo "📊 Profile results in: profiler-results/" \ No newline at end of file diff --git a/settings.gradle b/settings.gradle new file mode 100644 index 0000000..0568c52 --- /dev/null +++ b/settings.gradle @@ -0,0 +1 @@ +rootProject.name = 'imprint-java' diff --git a/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java b/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java new file mode 100644 index 0000000..1293478 --- /dev/null +++ b/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java @@ -0,0 +1,344 @@ +package com.imprint.benchmark; + +import com.esotericsoftware.kryo.Kryo; +import com.esotericsoftware.kryo.io.Input; +import com.esotericsoftware.kryo.io.Output; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.imprint.core.ImprintRecord; +import com.imprint.core.ImprintWriter; +import com.imprint.core.SchemaId; +import com.imprint.types.MapKey; +import com.imprint.types.Value; +import org.openjdk.jmh.annotations.*; +import org.openjdk.jmh.infra.Blackhole; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.nio.ByteBuffer; +import java.util.*; +import java.util.concurrent.TimeUnit; + +/** + * Head-to-head benchmarks comparing Imprint against other serialization libraries. + * Tests the performance claims made in the documentation. + */ +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@State(Scope.Benchmark) +@Warmup(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS) +@Fork(1) +public class ComparisonBenchmark { + + // Test data + private TestRecord testData; + + // Serialized formats + private ByteBuffer imprintBytes; + private byte[] jacksonBytes; + private byte[] kryoBytes; + + // Library instances + private ObjectMapper jackson; + private Kryo kryo; + + @Setup + public void setup() throws Exception { + testData = createTestRecord(); + + // Initialize libraries + jackson = new ObjectMapper(); + kryo = new Kryo(); + kryo.register(TestRecord.class); + kryo.register(ArrayList.class); + kryo.register(HashMap.class); + + // Pre-serialize for deserialization benchmarks + imprintBytes = serializeWithImprint(testData); + jacksonBytes = serializeWithJackson(testData); + kryoBytes = serializeWithKryo(testData); + } + + // ===== SERIALIZATION BENCHMARKS ===== + + @Benchmark + public void serializeImprint(Blackhole bh) throws Exception { + ByteBuffer result = serializeWithImprint(testData); + bh.consume(result); + } + + @Benchmark + public void serializeJackson(Blackhole bh) throws Exception { + byte[] result = serializeWithJackson(testData); + bh.consume(result); + } + + @Benchmark + public void serializeKryo(Blackhole bh) throws Exception { + byte[] result = serializeWithKryo(testData); + bh.consume(result); + } + + // ===== DESERIALIZATION BENCHMARKS ===== + + @Benchmark + public void deserializeImprint(Blackhole bh) throws Exception { + ImprintRecord result = ImprintRecord.deserialize(imprintBytes.duplicate()); + bh.consume(result); + } + + @Benchmark + public void deserializeJackson(Blackhole bh) throws Exception { + TestRecord result = jackson.readValue(jacksonBytes, TestRecord.class); + bh.consume(result); + } + + @Benchmark + public void deserializeKryo(Blackhole bh) { + Input input = new Input(new ByteArrayInputStream(kryoBytes)); + TestRecord result = kryo.readObject(input, TestRecord.class); + input.close(); + bh.consume(result); + } + + // ===== FIELD ACCESS BENCHMARKS ===== + + @Benchmark + public void fieldAccessImprint(Blackhole bh) throws Exception { + ImprintRecord record = ImprintRecord.deserialize(imprintBytes.duplicate()); + + // Access multiple fields without full deserialization + var id = record.getValue(1); + var name = record.getValue(2); + var price = record.getValue(3); + var active = record.getValue(4); + var category = record.getValue(5); + + bh.consume(id); + bh.consume(name); + bh.consume(price); + bh.consume(active); + bh.consume(category); + } + + @Benchmark + public void fieldAccessJackson(Blackhole bh) throws Exception { + // Jackson requires full deserialization to access fields + TestRecord record = jackson.readValue(jacksonBytes, TestRecord.class); + + bh.consume(record.id); + bh.consume(record.name); + bh.consume(record.price); + bh.consume(record.active); + bh.consume(record.category); + } + + @Benchmark + public void fieldAccessKryo(Blackhole bh) { + // Kryo requires full deserialization to access fields + Input input = new Input(new ByteArrayInputStream(kryoBytes)); + TestRecord record = kryo.readObject(input, TestRecord.class); + input.close(); + + bh.consume(record.id); + bh.consume(record.name); + bh.consume(record.price); + bh.consume(record.active); + bh.consume(record.category); + } + + // ===== SIZE COMPARISON ===== + + @Benchmark + public void measureImprintSize(Blackhole bh) throws Exception { + ByteBuffer serialized = serializeWithImprint(testData); + bh.consume(serialized.remaining()); + } + + @Benchmark + public void measureJacksonSize(Blackhole bh) throws Exception { + byte[] serialized = serializeWithJackson(testData); + bh.consume(serialized.length); + } + + @Benchmark + public void measureKryoSize(Blackhole bh) throws Exception { + byte[] serialized = serializeWithKryo(testData); + bh.consume(serialized.length); + } + + // ===== MERGE SIMULATION BENCHMARKS ===== + + @Benchmark + public void mergeImprint(Blackhole bh) throws Exception { + // Simulate merge with Imprint (O(1) with proper API) + var record1 = serializeWithImprint(testData); + var record2 = serializeWithImprint(createTestRecord2()); + + // Current simulation - will be O(1) with actual merge API + var deserialized1 = ImprintRecord.deserialize(record1); + var deserialized2 = ImprintRecord.deserialize(record2); + var merged = simulateMerge(deserialized1, deserialized2); + + bh.consume(merged); + } + + @Benchmark + public void mergeJackson(Blackhole bh) throws Exception { + // Jackson merge requires full deserialization + merge + serialization + var record1 = jackson.readValue(jacksonBytes, TestRecord.class); + var record2 = jackson.readValue(serializeWithJackson(createTestRecord2()), TestRecord.class); + + var merged = mergeTestRecords(record1, record2); + byte[] result = jackson.writeValueAsBytes(merged); + + bh.consume(result); + } + + @Benchmark + public void mergeKryo(Blackhole bh) throws Exception { + // Kryo merge requires full deserialization + merge + serialization + Input input1 = new Input(new ByteArrayInputStream(kryoBytes)); + var record1 = kryo.readObject(input1, TestRecord.class); + input1.close(); + + Input input2 = new Input(new ByteArrayInputStream(serializeWithKryo(createTestRecord2()))); + var record2 = kryo.readObject(input2, TestRecord.class); + input2.close(); + + var merged = mergeTestRecords(record1, record2); + byte[] result = serializeWithKryo(merged); + + bh.consume(result); + } + + // ===== HELPER METHODS ===== + + private ByteBuffer serializeWithImprint(TestRecord data) throws Exception { + var writer = new ImprintWriter(new SchemaId(1, 0x12345678)); + + writer.addField(1, Value.fromInt32(data.id)); + writer.addField(2, Value.fromString(data.name)); + writer.addField(3, Value.fromFloat64(data.price)); + writer.addField(4, Value.fromBoolean(data.active)); + writer.addField(5, Value.fromString(data.category)); + + // Convert tags list + var tagValues = new ArrayList(); + for (String tag : data.tags) { + tagValues.add(Value.fromString(tag)); + } + writer.addField(6, Value.fromArray(tagValues)); + + // Convert metadata map + var metadataMap = new HashMap(); + for (var entry : data.metadata.entrySet()) { + metadataMap.put(MapKey.fromString(entry.getKey()), Value.fromString(entry.getValue())); + } + writer.addField(7, Value.fromMap(metadataMap)); + + return writer.build().serializeToBuffer(); + } + + private byte[] serializeWithJackson(TestRecord data) throws Exception { + return jackson.writeValueAsBytes(data); + } + + private byte[] serializeWithKryo(TestRecord data) { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + Output output = new Output(baos); + kryo.writeObject(output, data); + output.close(); + return baos.toByteArray(); + } + + private ImprintRecord simulateMerge(ImprintRecord first, ImprintRecord second) throws Exception { + var writer = new ImprintWriter(first.getHeader().getSchemaId()); + var usedFieldIds = new HashSet(); + + // Copy fields from first record (takes precedence) + copyFieldsToWriter(first, writer, usedFieldIds); + copyFieldsToWriter(second, writer, usedFieldIds); + + return writer.build(); + } + + private void copyFieldsToWriter(ImprintRecord record, ImprintWriter writer, Set usedFieldIds) throws Exception { + for (var entry : record.getDirectory()) { + int fieldId = entry.getId(); + if (!usedFieldIds.contains(fieldId)) { + var value = record.getValue(fieldId); + if (value.isPresent()) { + writer.addField(fieldId, value.get()); + usedFieldIds.add(fieldId); + } + } + } + } + + private TestRecord mergeTestRecords(TestRecord first, TestRecord second) { + // Simple merge logic - first record takes precedence + var merged = new TestRecord(); + merged.id = first.id; + merged.name = first.name != null ? first.name : second.name; + merged.price = first.price != 0.0 ? first.price : second.price; + merged.active = first.active; + merged.category = first.category != null ? first.category : second.category; + + merged.tags = new ArrayList<>(first.tags); + merged.tags.addAll(second.tags); + + merged.metadata = new HashMap<>(first.metadata); + merged.metadata.putAll(second.metadata); + + return merged; + } + + private TestRecord createTestRecord() { + var record = new TestRecord(); + record.id = 12345; + record.name = "Test Product"; + record.price = 99.99; + record.active = true; + record.category = "Electronics"; + + record.tags = Arrays.asList("popular", "trending", "bestseller"); + + record.metadata = new HashMap<>(); + record.metadata.put("manufacturer", "TechCorp"); + record.metadata.put("model", "TC-2024"); + record.metadata.put("warranty", "2 years"); + + return record; + } + + private TestRecord createTestRecord2() { + var record = new TestRecord(); + record.id = 67890; + record.name = "Test Product 2"; + record.price = 149.99; + record.active = false; + record.category = "Software"; + + record.tags = Arrays.asList("new", "premium"); + + record.metadata = new HashMap<>(); + record.metadata.put("vendor", "SoftCorp"); + record.metadata.put("version", "2.1"); + + return record; + } + + // Test data class for other serialization libraries + public static class TestRecord { + public int id; + public String name; + public double price; + public boolean active; + public String category; + public List tags = new ArrayList<>(); + public Map metadata = new HashMap<>(); + + public TestRecord() {} // Required for deserialization + } +} \ No newline at end of file diff --git a/src/jmh/java/com/imprint/benchmark/FieldAccessBenchmark.java b/src/jmh/java/com/imprint/benchmark/FieldAccessBenchmark.java new file mode 100644 index 0000000..f3abb7e --- /dev/null +++ b/src/jmh/java/com/imprint/benchmark/FieldAccessBenchmark.java @@ -0,0 +1,276 @@ +package com.imprint.benchmark; + +import com.imprint.core.ImprintRecord; +import com.imprint.core.ImprintWriter; +import com.imprint.core.SchemaId; +import com.imprint.types.MapKey; +import com.imprint.types.Value; +import org.openjdk.jmh.annotations.*; +import org.openjdk.jmh.infra.Blackhole; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.concurrent.TimeUnit; + +/** + * Benchmarks for ImprintRecord field access and projection operations. + * Tests the zero-copy field access performance claims. + */ +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@State(Scope.Benchmark) +@Warmup(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS) +@Fork(1) +public class FieldAccessBenchmark { + + private ImprintRecord sparseRecord; + private ImprintRecord denseRecord; + private ImprintRecord largeRecord; + + // Field IDs for testing different access patterns + private int[] firstFields; + private int[] middleFields; + private int[] lastFields; + private int[] randomFields; + private int[] allFields; + + @Setup + public void setup() throws Exception { + sparseRecord = createSparseRecord(); // Few fields, large field IDs + denseRecord = createDenseRecord(); // Many sequential fields + largeRecord = createLargeRecord(); // Large record with complex data + + // Setup field access patterns + firstFields = new int[]{1, 2, 3, 4, 5}; + middleFields = new int[]{45, 46, 47, 48, 49}; + lastFields = new int[]{95, 96, 97, 98, 99}; + randomFields = new int[]{7, 23, 41, 67, 89}; + allFields = new int[100]; + for (int i = 0; i < 100; i++) { + allFields[i] = i + 1; + } + } + + // ===== SINGLE FIELD ACCESS BENCHMARKS ===== + + @Benchmark + public void accessFirstField(Blackhole bh) throws Exception { + var value = denseRecord.getValue(1); + bh.consume(value); + } + + @Benchmark + public void accessMiddleField(Blackhole bh) throws Exception { + var value = denseRecord.getValue(50); + bh.consume(value); + } + + @Benchmark + public void accessLastField(Blackhole bh) throws Exception { + var value = denseRecord.getValue(100); + bh.consume(value); + } + + @Benchmark + public void accessNonExistentField(Blackhole bh) throws Exception { + var value = denseRecord.getValue(999); + bh.consume(value); + } + + // ===== MULTIPLE FIELD ACCESS PATTERNS ===== + + @Benchmark + public void accessFirstFields(Blackhole bh) throws Exception { + for (int fieldId : firstFields) { + var value = denseRecord.getValue(fieldId); + bh.consume(value); + } + } + + @Benchmark + public void accessMiddleFields(Blackhole bh) throws Exception { + for (int fieldId : middleFields) { + var value = denseRecord.getValue(fieldId); + bh.consume(value); + } + } + + @Benchmark + public void accessLastFields(Blackhole bh) throws Exception { + for (int fieldId : lastFields) { + var value = denseRecord.getValue(fieldId); + bh.consume(value); + } + } + + @Benchmark + public void accessRandomFields(Blackhole bh) throws Exception { + for (int fieldId : randomFields) { + var value = denseRecord.getValue(fieldId); + bh.consume(value); + } + } + + // ===== FIELD PROJECTION BENCHMARKS ===== + + @Benchmark + public void projectSmallSubset(Blackhole bh) throws Exception { + // Project 5 fields from a 100-field record + var projection = simulateProject(denseRecord, firstFields); + bh.consume(projection); + } + + @Benchmark + public void projectMediumSubset(Blackhole bh) throws Exception { + // Project 25 fields from a 100-field record + int[] fields = Arrays.copyOf(allFields, 25); + var projection = simulateProject(denseRecord, fields); + bh.consume(projection); + } + + @Benchmark + public void projectLargeSubset(Blackhole bh) throws Exception { + // Project 75 fields from a 100-field record + int[] fields = Arrays.copyOf(allFields, 75); + var projection = simulateProject(denseRecord, fields); + bh.consume(projection); + } + + @Benchmark + public void projectAllFields(Blackhole bh) throws Exception { + // Project all fields (should be nearly equivalent to full record) + var projection = simulateProject(denseRecord, allFields); + bh.consume(projection); + } + + // ===== RAW BYTES ACCESS BENCHMARKS ===== + + @Benchmark + public void getRawBytesFirstField(Blackhole bh) { + var rawBytes = denseRecord.getRawBytes(1); + bh.consume(rawBytes); + } + + @Benchmark + public void getRawBytesMiddleField(Blackhole bh) { + var rawBytes = denseRecord.getRawBytes(50); + bh.consume(rawBytes); + } + + @Benchmark + public void getRawBytesLastField(Blackhole bh) { + var rawBytes = denseRecord.getRawBytes(100); + bh.consume(rawBytes); + } + + // ===== SPARSE VS DENSE ACCESS PATTERNS ===== + + @Benchmark + public void accessSparseRecord(Blackhole bh) throws Exception { + // Access fields in sparse record (large field IDs, few fields) + var value1 = sparseRecord.getValue(1000); + var value2 = sparseRecord.getValue(5000); + var value3 = sparseRecord.getValue(10000); + bh.consume(value1); + bh.consume(value2); + bh.consume(value3); + } + + @Benchmark + public void accessDenseRecord(Blackhole bh) throws Exception { + // Access fields in dense record (sequential field IDs) + var value1 = denseRecord.getValue(1); + var value2 = denseRecord.getValue(2); + var value3 = denseRecord.getValue(3); + bh.consume(value1); + bh.consume(value2); + bh.consume(value3); + } + + // ===== HELPER METHODS ===== + + /** + * Simulates field projection by creating a new record with only specified fields. + * This should be replaced with actual project API when available. + */ + private ImprintRecord simulateProject(ImprintRecord source, int[] fieldIds) throws Exception { + var writer = new ImprintWriter(source.getHeader().getSchemaId()); + + for (int fieldId : fieldIds) { + var value = source.getValue(fieldId); + value.ifPresent(value1 -> writer.addField(fieldId, value1)); + } + + return writer.build(); + } + + private ImprintRecord createSparseRecord() throws Exception { + var writer = new ImprintWriter(new SchemaId(1, 0x12345678)); + + // Sparse record with large field IDs and few fields + writer.addField(1000, Value.fromString("sparse_field_1")); + writer.addField(5000, Value.fromInt32(42)); + writer.addField(10000, Value.fromFloat64(3.14159)); + writer.addField(15000, Value.fromBoolean(true)); + writer.addField(20000, Value.fromString("sparse_field_5")); + + return writer.build(); + } + + private ImprintRecord createDenseRecord() throws Exception { + var writer = new ImprintWriter(new SchemaId(2, 0x87654321)); + + // Dense record with 100 sequential fields + for (int i = 1; i <= 100; i++) { + switch (i % 5) { + case 0: + writer.addField(i, Value.fromString("string_field_" + i)); + break; + case 1: + writer.addField(i, Value.fromInt32(i * 10)); + break; + case 2: + writer.addField(i, Value.fromFloat64(i * 1.5)); + break; + case 3: + writer.addField(i, Value.fromBoolean(i % 2 == 0)); + break; + case 4: + writer.addField(i, Value.fromInt64(i * 1000L)); + break; + } + } + + return writer.build(); + } + + private ImprintRecord createLargeRecord() throws Exception { + var writer = new ImprintWriter(new SchemaId(3, 0x11223344)); + + // Large record with complex data types + writer.addField(1, Value.fromString("LargeRecord")); + + // Large array field + var largeArray = new ArrayList(); + for (int i = 0; i < 1000; i++) { + largeArray.add(Value.fromString("array_item_" + i)); + } + writer.addField(2, Value.fromArray(largeArray)); + + // Large map field + var largeMap = new HashMap(); + for (int i = 0; i < 100; i++) { + largeMap.put(MapKey.fromString("key_" + i), Value.fromString("map_value_" + i)); + } + writer.addField(3, Value.fromMap(largeMap)); + + // Many regular fields + for (int i = 4; i <= 50; i++) { + writer.addField(i, Value.fromString("large_record_field_" + i + "_with_substantial_content")); + } + + return writer.build(); + } +} \ No newline at end of file diff --git a/src/jmh/java/com/imprint/benchmark/MergeBenchmark.java b/src/jmh/java/com/imprint/benchmark/MergeBenchmark.java new file mode 100644 index 0000000..5c52908 --- /dev/null +++ b/src/jmh/java/com/imprint/benchmark/MergeBenchmark.java @@ -0,0 +1,163 @@ +package com.imprint.benchmark; + +import com.imprint.core.ImprintRecord; +import com.imprint.core.ImprintWriter; +import com.imprint.core.SchemaId; +import com.imprint.types.Value; +import org.openjdk.jmh.annotations.*; +import org.openjdk.jmh.infra.Blackhole; + +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +/** + * Benchmarks for ImprintRecord merge operations. + * NOTE: These benchmarks simulate merge operations until the actual merge API is implemented. + */ +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@State(Scope.Benchmark) +@Warmup(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS) +@Fork(1) +public class MergeBenchmark { + + private ImprintRecord productRecord; + private ImprintRecord orderRecord; + private ImprintRecord customerRecord; + + @Setup + public void setup() throws Exception { + productRecord = createProductRecord(); + orderRecord = createOrderRecord(); + customerRecord = createCustomerRecord(); + } + + // ===== SIMULATED MERGE BENCHMARKS ===== + // These will be replaced with actual merge API when implemented + + @Benchmark + public void mergeProductAndOrder(Blackhole bh) throws Exception { + // Simulate merge by creating a new record with fields from both + ImprintRecord result = simulateMerge(productRecord, orderRecord); + bh.consume(result); + } + + @Benchmark + public void mergeProductAndCustomer(Blackhole bh) throws Exception { + ImprintRecord result = simulateMerge(productRecord, customerRecord); + bh.consume(result); + } + + @Benchmark + public void mergeOrderAndCustomer(Blackhole bh) throws Exception { + ImprintRecord result = simulateMerge(orderRecord, customerRecord); + bh.consume(result); + } + + @Benchmark + public void mergeThreeRecords(Blackhole bh) throws Exception { + // Test merging multiple records + var temp = simulateMerge(productRecord, orderRecord); + ImprintRecord result = simulateMerge(temp, customerRecord); + bh.consume(result); + } + + // ===== MERGE CONFLICT HANDLING ===== + + @Benchmark + public void mergeWithConflicts(Blackhole bh) throws Exception { + // Create records with overlapping field IDs to test conflict resolution + var record1 = createRecordWithFields(1, 50, "record1_"); + var record2 = createRecordWithFields(25, 75, "record2_"); + + ImprintRecord result = simulateMerge(record1, record2); + bh.consume(result); + } + + // ===== HELPER METHODS ===== + + /** + * Simulates merge operation by manually copying fields. + * This should be replaced with actual merge API when available. + */ + private ImprintRecord simulateMerge(ImprintRecord first, ImprintRecord second) throws Exception { + var writer = new ImprintWriter(first.getHeader().getSchemaId()); + var usedFieldIds = new HashSet(); + + // Copy fields from first record (takes precedence) + copyFieldsToWriter(first, writer, usedFieldIds); + + // Copy non-conflicting fields from second record + copyFieldsToWriter(second, writer, usedFieldIds); + + return writer.build(); + } + + private void copyFieldsToWriter(ImprintRecord record, ImprintWriter writer, Set usedFieldIds) throws Exception { + for (var entry : record.getDirectory()) { + int fieldId = entry.getId(); + if (!usedFieldIds.contains(fieldId)) { + var value = record.getValue(fieldId); + if (value.isPresent()) { + writer.addField(fieldId, value.get()); + usedFieldIds.add(fieldId); + } + } + } + } + + private ImprintRecord createProductRecord() throws Exception { + var writer = new ImprintWriter(new SchemaId(1, 0x12345678)); + + writer.addField(1, Value.fromString("Product")); + writer.addField(2, Value.fromInt32(12345)); + writer.addField(3, Value.fromString("Laptop")); + writer.addField(4, Value.fromFloat64(999.99)); + writer.addField(5, Value.fromString("Electronics")); + writer.addField(6, Value.fromInt32(50)); // stock + writer.addField(7, Value.fromString("TechCorp")); + writer.addField(8, Value.fromBoolean(true)); // available + + return writer.build(); + } + + private ImprintRecord createOrderRecord() throws Exception { + var writer = new ImprintWriter(new SchemaId(2, 0x87654321)); + + writer.addField(10, Value.fromString("Order")); + writer.addField(11, Value.fromInt32(67890)); + writer.addField(12, Value.fromInt32(12345)); // product_id (overlaps with product) + writer.addField(13, Value.fromInt32(2)); // quantity + writer.addField(14, Value.fromFloat64(1999.98)); // total + writer.addField(15, Value.fromString("2024-01-15")); // order_date + writer.addField(16, Value.fromString("shipped")); // status + + return writer.build(); + } + + private ImprintRecord createCustomerRecord() throws Exception { + var writer = new ImprintWriter(new SchemaId(3, 0x11223344)); + + writer.addField(20, Value.fromString("Customer")); + writer.addField(21, Value.fromInt32(555)); + writer.addField(22, Value.fromString("John Doe")); + writer.addField(23, Value.fromString("john.doe@email.com")); + writer.addField(24, Value.fromString("123 Main St")); + writer.addField(25, Value.fromString("premium")); // tier + writer.addField(26, Value.fromBoolean(true)); // active + + return writer.build(); + } + + private ImprintRecord createRecordWithFields(int startId, int endId, String prefix) throws Exception { + var writer = new ImprintWriter(new SchemaId(1, 0x12345678)); + + for (int i = startId; i <= endId; i++) { + writer.addField(i, Value.fromString(prefix + "field_" + i)); + } + + return writer.build(); + } +} \ No newline at end of file diff --git a/src/jmh/java/com/imprint/benchmark/SerializationBenchmark.java b/src/jmh/java/com/imprint/benchmark/SerializationBenchmark.java new file mode 100644 index 0000000..2544b88 --- /dev/null +++ b/src/jmh/java/com/imprint/benchmark/SerializationBenchmark.java @@ -0,0 +1,166 @@ +package com.imprint.benchmark; + +import com.imprint.core.ImprintRecord; +import com.imprint.core.ImprintWriter; +import com.imprint.core.SchemaId; +import com.imprint.types.MapKey; +import com.imprint.types.Value; +import org.openjdk.jmh.annotations.*; +import org.openjdk.jmh.infra.Blackhole; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.concurrent.TimeUnit; + +/** + * Benchmarks for ImprintRecord serialization and deserialization operations. + */ +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@State(Scope.Benchmark) +@Warmup(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS) +@Fork(1) +public class SerializationBenchmark { + + private ImprintRecord smallRecord; + private ImprintRecord mediumRecord; + private ImprintRecord largeRecord; + + private ByteBuffer smallRecordBytes; + private ByteBuffer mediumRecordBytes; + private ByteBuffer largeRecordBytes; + + @Setup + public void setup() throws Exception { + // Create test records of varying sizes + smallRecord = createSmallRecord(); + mediumRecord = createMediumRecord(); + largeRecord = createLargeRecord(); + + // Pre-serialize for deserialization benchmarks + smallRecordBytes = smallRecord.serializeToBuffer(); + mediumRecordBytes = mediumRecord.serializeToBuffer(); + largeRecordBytes = largeRecord.serializeToBuffer(); + } + + // ===== SERIALIZATION BENCHMARKS ===== + + @Benchmark + public void serializeSmallRecord(Blackhole bh) { + ByteBuffer result = smallRecord.serializeToBuffer(); + bh.consume(result); + } + + @Benchmark + public void serializeMediumRecord(Blackhole bh) { + ByteBuffer result = mediumRecord.serializeToBuffer(); + bh.consume(result); + } + + @Benchmark + public void serializeLargeRecord(Blackhole bh) { + ByteBuffer result = largeRecord.serializeToBuffer(); + bh.consume(result); + } + + // ===== DESERIALIZATION BENCHMARKS ===== + + @Benchmark + public void deserializeSmallRecord(Blackhole bh) throws Exception { + ImprintRecord result = ImprintRecord.deserialize(smallRecordBytes.duplicate()); + bh.consume(result); + } + + @Benchmark + public void deserializeMediumRecord(Blackhole bh) throws Exception { + ImprintRecord result = ImprintRecord.deserialize(mediumRecordBytes.duplicate()); + bh.consume(result); + } + + @Benchmark + public void deserializeLargeRecord(Blackhole bh) throws Exception { + ImprintRecord result = ImprintRecord.deserialize(largeRecordBytes.duplicate()); + bh.consume(result); + } + + // ===== HELPER METHODS ===== + + private ImprintRecord createSmallRecord() throws Exception { + var writer = new ImprintWriter(new SchemaId(1, 0x12345678)); + + // Small record: ~10 fields, simple types + writer.addField(1, Value.fromString("Product")); + writer.addField(2, Value.fromInt32(12345)); + writer.addField(3, Value.fromFloat64(99.99)); + writer.addField(4, Value.fromBoolean(true)); + writer.addField(5, Value.fromString("Electronics")); + + return writer.build(); + } + + private ImprintRecord createMediumRecord() throws Exception { + var writer = new ImprintWriter(new SchemaId(1, 0x12345678)); + + // Medium record: ~50 fields, mixed types including arrays + writer.addField(1, Value.fromString("Product")); + writer.addField(2, Value.fromInt32(12345)); + writer.addField(3, Value.fromFloat64(99.99)); + writer.addField(4, Value.fromBoolean(true)); + writer.addField(5, Value.fromString("Electronics")); + + // Add array field + var tags = Arrays.asList( + Value.fromString("popular"), + Value.fromString("trending"), + Value.fromString("bestseller") + ); + writer.addField(6, Value.fromArray(tags)); + + // Add map field + var metadata = new HashMap(); + metadata.put(MapKey.fromString("manufacturer"), Value.fromString("TechCorp")); + metadata.put(MapKey.fromString("model"), Value.fromString("TC-2024")); + metadata.put(MapKey.fromString("year"), Value.fromInt32(2024)); + writer.addField(7, Value.fromMap(metadata)); + + // Add more fields for medium size + for (int i = 8; i <= 50; i++) { + writer.addField(i, Value.fromString("field_" + i + "_value")); + } + + return writer.build(); + } + + private ImprintRecord createLargeRecord() throws Exception { + var writer = new ImprintWriter(new SchemaId(1, 0x12345678)); + + // Large record: ~200 fields, complex nested structures + writer.addField(1, Value.fromString("LargeProduct")); + writer.addField(2, Value.fromInt32(12345)); + writer.addField(3, Value.fromFloat64(99.99)); + + // Large array + var largeArray = new ArrayList(); + for (int i = 0; i < 100; i++) { + largeArray.add(Value.fromString("item_" + i)); + } + writer.addField(4, Value.fromArray(largeArray)); + + // Large map + var largeMap = new HashMap(); + for (int i = 0; i < 50; i++) { + largeMap.put(MapKey.fromString("key_" + i), Value.fromString("value_" + i)); + } + writer.addField(5, Value.fromMap(largeMap)); + + // Many string fields + for (int i = 6; i <= 200; i++) { + writer.addField(i, Value.fromString("this_is_a_longer_field_value_for_field_" + i + "_to_increase_record_size")); + } + + return writer.build(); + } +} \ No newline at end of file diff --git a/src/main/java/com/imprint/Constants.java b/src/main/java/com/imprint/Constants.java new file mode 100644 index 0000000..78b91a0 --- /dev/null +++ b/src/main/java/com/imprint/Constants.java @@ -0,0 +1,10 @@ +package com.imprint; + +public final class Constants { + public static final byte MAGIC = 0x49; + public static final byte VERSION = 0x02; + public static final int HEADER_BYTES = 15; + public static final int DIR_ENTRY_BYTES = 7; + + private Constants() {} +} diff --git a/src/main/java/com/imprint/core/DirectoryEntry.java b/src/main/java/com/imprint/core/DirectoryEntry.java new file mode 100644 index 0000000..9556256 --- /dev/null +++ b/src/main/java/com/imprint/core/DirectoryEntry.java @@ -0,0 +1,23 @@ +package com.imprint.core; + +import com.imprint.types.TypeCode; +import lombok.Value; + +import java.util.Objects; + +/** + * A directory entry describing a single field in an Imprint record. + * Each entry has a fixed size of 7 bytes. + */ +@Value +public class DirectoryEntry { + short id; + TypeCode typeCode; + int offset; + + public DirectoryEntry(int id, TypeCode typeCode, int offset) { + this.id = (short) id; + this.typeCode = Objects.requireNonNull(typeCode, "TypeCode cannot be null"); + this.offset = offset; + } +} \ No newline at end of file diff --git a/src/main/java/com/imprint/core/Flags.java b/src/main/java/com/imprint/core/Flags.java new file mode 100644 index 0000000..ee6ffa7 --- /dev/null +++ b/src/main/java/com/imprint/core/Flags.java @@ -0,0 +1,12 @@ +package com.imprint.core; + +import lombok.Value; + +/** + * Bit flags for Imprint record header. + * Currently reserved for future use - field directory is always present. + */ +@Value +public class Flags { + byte value; +} \ No newline at end of file diff --git a/src/main/java/com/imprint/core/Header.java b/src/main/java/com/imprint/core/Header.java new file mode 100644 index 0000000..388d491 --- /dev/null +++ b/src/main/java/com/imprint/core/Header.java @@ -0,0 +1,13 @@ +package com.imprint.core; + +import lombok.Value; + +/** + * The header of an Imprint record. + */ +@Value +public class Header { + Flags flags; + SchemaId schemaId; + int payloadSize; +} \ No newline at end of file diff --git a/src/main/java/com/imprint/core/ImprintRecord.java b/src/main/java/com/imprint/core/ImprintRecord.java new file mode 100644 index 0000000..b7ed224 --- /dev/null +++ b/src/main/java/com/imprint/core/ImprintRecord.java @@ -0,0 +1,365 @@ + +package com.imprint.core; + +import com.imprint.Constants; +import com.imprint.error.ErrorType; +import com.imprint.error.ImprintException; +import com.imprint.types.MapKey; +import com.imprint.types.TypeCode; +import com.imprint.types.Value; +import com.imprint.util.VarInt; +import lombok.Getter; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.*; + +/** + * An Imprint record containing a header, optional field directory, and payload. + * Uses ByteBuffer for zero-copy operations to achieve low latency. + */ +@Getter +public final class ImprintRecord { + private final Header header; + private final List directory; + private final ByteBuffer payload; // Read-only view for zero-copy + + public ImprintRecord(Header header, List directory, ByteBuffer payload) { + this.header = Objects.requireNonNull(header, "Header cannot be null"); + this.directory = List.copyOf(Objects.requireNonNull(directory, "Directory cannot be null")); + this.payload = payload.asReadOnlyBuffer(); // Zero-copy read-only view + } + + /** + * Get a value by field ID, deserializing it on demand. + */ + public Optional getValue(int fieldId) throws ImprintException { + // Binary search for the field ID without allocation + int index = findDirectoryIndex(fieldId); + if (index < 0) return Optional.empty(); + + var entry = directory.get(index); + int startOffset = entry.getOffset(); + int endOffset = (index + 1 < directory.size()) ? + directory.get(index + 1).getOffset() : payload.remaining(); + + var valueBytes = payload.duplicate(); + valueBytes.position(startOffset).limit(endOffset); + var value = deserializeValue(entry.getTypeCode(), valueBytes.slice()); + return Optional.of(value); + } + + /** + * Get the raw bytes for a field without deserializing. + * Returns a zero-copy ByteBuffer view. + */ + public Optional getRawBytes(int fieldId) { + int index = findDirectoryIndex(fieldId); + if (index < 0) return Optional.empty(); + + var entry = directory.get(index); + int startOffset = entry.getOffset(); + int endOffset = (index + 1 < directory.size()) ? + directory.get(index + 1).getOffset() : payload.remaining(); + + var fieldBuffer = payload.duplicate(); + fieldBuffer.position(startOffset).limit(endOffset); + return Optional.of(fieldBuffer.slice().asReadOnlyBuffer()); + } + + /** + * Serialize this record to a ByteBuffer (zero-copy when possible). + */ + public ByteBuffer serializeToBuffer() { + var buffer = ByteBuffer.allocate(estimateSerializedSize()); + buffer.order(ByteOrder.LITTLE_ENDIAN); + + // Write header + serializeHeader(buffer); + + // Write directory (always present) + VarInt.encode(directory.size(), buffer); + for (var entry : directory) { + serializeDirectoryEntry(entry, buffer); + } + + // Write payload (shallow copy only) + var payloadCopy = payload.duplicate(); + buffer.put(payloadCopy); + + // Return read-only view of used portion + buffer.flip(); + return buffer.asReadOnlyBuffer(); + } + + /** + * Create a fluent builder for constructing ImprintRecord instances. + * + * @param schemaId the schema identifier for this record + * @return a new builder instance + */ + public static ImprintRecordBuilder builder(SchemaId schemaId) { + return new ImprintRecordBuilder(schemaId); + } + + /** + * Create a fluent builder for constructing ImprintRecord instances. + * + * @param fieldspaceId the fieldspace identifier + * @param schemaHash the schema hash + * @return a new builder instance + */ + public static ImprintRecordBuilder builder(int fieldspaceId, int schemaHash) { + return new ImprintRecordBuilder(new SchemaId(fieldspaceId, schemaHash)); + } + + /** + * Deserialize a record from bytes. + */ + public static ImprintRecord deserialize(byte[] bytes) throws ImprintException { + return deserialize(ByteBuffer.wrap(bytes)); + } + + /** + * Deserialize a record from a ByteBuffer (zero-copy when possible). + */ + public static ImprintRecord deserialize(ByteBuffer buffer) throws ImprintException { + buffer = buffer.duplicate().order(ByteOrder.LITTLE_ENDIAN); + + // Read header + var header = deserializeHeader(buffer); + + // Read directory (always present) + var directory = new ArrayList(); + VarInt.DecodeResult countResult = VarInt.decode(buffer); + int directoryCount = countResult.getValue(); + + for (int i = 0; i < directoryCount; i++) { + directory.add(deserializeDirectoryEntry(buffer)); + } + + // Read payload as ByteBuffer slice for zero-copy + var payload = buffer.slice(); + payload.limit(header.getPayloadSize()); + buffer.position(buffer.position() + header.getPayloadSize()); + + return new ImprintRecord(header, directory, payload); + } + + /** + * Binary search for field ID in directory without object allocation. + * Returns the index of the field if found, or a negative value if not found. + * + * @param fieldId the field ID to search for + * @return index if found, or negative insertion point - 1 if not found + */ + private int findDirectoryIndex(int fieldId) { + int low = 0; + int high = directory.size() - 1; + + while (low <= high) { + int mid = (low + high) >>> 1; // unsigned right shift to avoid overflow + int midFieldId = directory.get(mid).getId(); + + if (midFieldId < fieldId) { + low = mid + 1; + } else if (midFieldId > fieldId) { + high = mid - 1; + } else { + return mid; // field found + } + } + return -(low + 1); // field not found, return insertion point + } + + private int estimateSerializedSize() { + int size = Constants.HEADER_BYTES; // header + size += VarInt.encodedLength(directory.size()); // directory count + size += directory.size() * Constants.DIR_ENTRY_BYTES; // directory entries + size += payload.remaining(); // payload + return size; + } + + private void serializeHeader(ByteBuffer buffer) { + buffer.put(Constants.MAGIC); + buffer.put(Constants.VERSION); + buffer.put(header.getFlags().getValue()); + buffer.putInt(header.getSchemaId().getFieldspaceId()); + buffer.putInt(header.getSchemaId().getSchemaHash()); + buffer.putInt(header.getPayloadSize()); + } + + private static Header deserializeHeader(ByteBuffer buffer) throws ImprintException { + if (buffer.remaining() < Constants.HEADER_BYTES) { + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, + "Not enough bytes for header"); + } + + byte magic = buffer.get(); + if (magic != Constants.MAGIC) { + throw new ImprintException(ErrorType.INVALID_MAGIC, + "Invalid magic byte: expected 0x" + Integer.toHexString(Constants.MAGIC) + + ", got 0x" + Integer.toHexString(magic & 0xFF)); + } + + byte version = buffer.get(); + if (version != Constants.VERSION) { + throw new ImprintException(ErrorType.UNSUPPORTED_VERSION, + "Unsupported version: " + version); + } + + var flags = new Flags(buffer.get()); + int fieldspaceId = buffer.getInt(); + int schemaHash = buffer.getInt(); + int payloadSize = buffer.getInt(); + + return new Header(flags, new SchemaId(fieldspaceId, schemaHash), payloadSize); + } + + private void serializeDirectoryEntry(DirectoryEntry entry, ByteBuffer buffer) { + buffer.putShort(entry.getId()); + buffer.put(entry.getTypeCode().getCode()); + buffer.putInt(entry.getOffset()); + } + + private static DirectoryEntry deserializeDirectoryEntry(ByteBuffer buffer) throws ImprintException { + if (buffer.remaining() < Constants.DIR_ENTRY_BYTES) { + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, + "Not enough bytes for directory entry"); + } + + short id = buffer.getShort(); + var typeCode = TypeCode.fromByte(buffer.get()); + int offset = buffer.getInt(); + + return new DirectoryEntry(id, typeCode, offset); + } + + private Value deserializeValue(TypeCode typeCode, ByteBuffer buffer) throws ImprintException { + // Buffer is already positioned and limited correctly + buffer = buffer.duplicate().order(ByteOrder.LITTLE_ENDIAN); + + // Use TypeHandler for simple types + switch (typeCode) { + case NULL: + case BOOL: + case INT32: + case INT64: + case FLOAT32: + case FLOAT64: + case BYTES: + case STRING: + return typeCode.getHandler().deserialize(buffer); + + case ARRAY: + return deserializeArray(buffer); + + case MAP: + return deserializeMap(buffer); + + case ROW: + var remainingBuffer = buffer.slice(); + var nestedRecord = deserialize(remainingBuffer); + return Value.fromRow(nestedRecord); + + default: + throw new ImprintException(ErrorType.INVALID_TYPE_CODE, "Unknown type code: " + typeCode); + } + } + + private Value deserializeArray(ByteBuffer buffer) throws ImprintException { + VarInt.DecodeResult lengthResult = VarInt.decode(buffer); + int length = lengthResult.getValue(); + + if (length == 0) { + return Value.fromArray(Collections.emptyList()); + } + + var elementType = TypeCode.fromByte(buffer.get()); + var elements = new ArrayList(length); + + for (int i = 0; i < length; i++) { + var elementBytes = readValueBytes(elementType, buffer); + var element = deserializeValue(elementType, elementBytes); + elements.add(element); + } + + return Value.fromArray(elements); + } + + private Value deserializeMap(ByteBuffer buffer) throws ImprintException { + VarInt.DecodeResult lengthResult = VarInt.decode(buffer); + int length = lengthResult.getValue(); + + if (length == 0) { + return Value.fromMap(Collections.emptyMap()); + } + + var keyType = TypeCode.fromByte(buffer.get()); + var valueType = TypeCode.fromByte(buffer.get()); + var map = new HashMap(length); + + for (int i = 0; i < length; i++) { + // Read key + var keyBytes = readValueBytes(keyType, buffer); + var keyValue = deserializeValue(keyType, keyBytes); + var key = MapKey.fromValue(keyValue); + + // Read value + var valueBytes = readValueBytes(valueType, buffer); + var value = deserializeValue(valueType, valueBytes); + + map.put(key, value); + } + + return Value.fromMap(map); + } + + private ByteBuffer readValueBytes(TypeCode typeCode, ByteBuffer buffer) throws ImprintException { + // Use TypeHandler for simple types + switch (typeCode) { + case NULL: + case BOOL: + case INT32: + case INT64: + case FLOAT32: + case FLOAT64: + case BYTES: + case STRING: + return typeCode.getHandler().readValueBytes(buffer); + + case ARRAY: + case MAP: + case ROW: + // For complex types, return the entire remaining buffer for now + // The specific deserializer will handle parsing in the future + var remainingBuffer = buffer.slice(); + buffer.position(buffer.limit()); + return remainingBuffer.asReadOnlyBuffer(); + + default: + throw new ImprintException(ErrorType.INVALID_TYPE_CODE, "Unknown type code: " + typeCode); + } + } + + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (obj == null || getClass() != obj.getClass()) return false; + var that = (ImprintRecord) obj; + return header.equals(that.header) && + directory.equals(that.directory) && + payload.equals(that.payload); + } + + @Override + public int hashCode() { + return Objects.hash(header, directory, payload); + } + + @Override + public String toString() { + return String.format("ImprintRecord{header=%s, directorySize=%d, payloadSize=%d}", + header, directory.size(), payload.remaining()); + } +} \ No newline at end of file diff --git a/src/main/java/com/imprint/core/ImprintRecordBuilder.java b/src/main/java/com/imprint/core/ImprintRecordBuilder.java new file mode 100644 index 0000000..48b0998 --- /dev/null +++ b/src/main/java/com/imprint/core/ImprintRecordBuilder.java @@ -0,0 +1,233 @@ +package com.imprint.core; + +import com.imprint.error.ImprintException; +import com.imprint.types.MapKey; +import com.imprint.types.Value; + +import java.util.*; + +/** + * A fluent builder for creating ImprintRecord instances with type-safe, + * developer-friendly API that eliminates boilerplate Value.fromX() calls. + *

+ * Usage: + * var record = ImprintRecord.builder(schemaId) + * .field(1, 42) // int -> Int32Value + * .field(2, "hello") // String -> StringValue + * .field(3, 3.14) // double -> Float64Value + * .field(4, bytes) // byte[] -> BytesValue + * .field(5, true) // boolean -> BoolValue + * .nullField(6) // -> NullValue + * .build(); + */ +public final class ImprintRecordBuilder { + private final SchemaId schemaId; + private final Map fields = new TreeMap<>(); + + ImprintRecordBuilder(SchemaId schemaId) { + this.schemaId = Objects.requireNonNull(schemaId, "SchemaId cannot be null"); + } + + // Primitive types with automatic Value wrapping + public ImprintRecordBuilder field(int id, boolean value) { + return addField(id, Value.fromBoolean(value)); + } + + public ImprintRecordBuilder field(int id, int value) { + return addField(id, Value.fromInt32(value)); + } + + public ImprintRecordBuilder field(int id, long value) { + return addField(id, Value.fromInt64(value)); + } + + public ImprintRecordBuilder field(int id, float value) { + return addField(id, Value.fromFloat32(value)); + } + + public ImprintRecordBuilder field(int id, double value) { + return addField(id, Value.fromFloat64(value)); + } + + public ImprintRecordBuilder field(int id, String value) { + return addField(id, Value.fromString(value)); + } + + public ImprintRecordBuilder field(int id, byte[] value) { + return addField(id, Value.fromBytes(value)); + } + + // Collections with automatic conversion + public ImprintRecordBuilder field(int id, List values) { + var convertedValues = new ArrayList(values.size()); + for (var item : values) { + convertedValues.add(convertToValue(item)); + } + return addField(id, Value.fromArray(convertedValues)); + } + + public ImprintRecordBuilder field(int id, Map map) { + var convertedMap = new HashMap(map.size()); + for (var entry : map.entrySet()) { + var key = convertToMapKey(entry.getKey()); + var value = convertToValue(entry.getValue()); + convertedMap.put(key, value); + } + return addField(id, Value.fromMap(convertedMap)); + } + + // Nested records + public ImprintRecordBuilder field(int id, ImprintRecord nestedRecord) { + return addField(id, Value.fromRow(nestedRecord)); + } + + // Explicit null field + public ImprintRecordBuilder nullField(int id) { + return addField(id, Value.nullValue()); + } + + // Direct Value API (escape hatch for advanced usage) + public ImprintRecordBuilder field(int id, Value value) { + return addField(id, value); + } + + // Conditional field addition + public ImprintRecordBuilder fieldIf(boolean condition, int id, Object value) { + if (condition) { + return field(id, convertToValue(value)); + } + return this; + } + + public ImprintRecordBuilder fieldIfNotNull(int id, Object value) { + return fieldIf(value != null, id, value); + } + + // Bulk operations + public ImprintRecordBuilder fields(Map fieldsMap) { + for (var entry : fieldsMap.entrySet()) { + field(entry.getKey(), convertToValue(entry.getValue())); + } + return this; + } + + // Builder utilities + public boolean hasField(int id) { + return fields.containsKey(id); + } + + public int fieldCount() { + return fields.size(); + } + + public Set fieldIds() { + return new TreeSet<>(fields.keySet()); + } + + // Build the final record + public ImprintRecord build() throws ImprintException { + if (fields.isEmpty()) { + throw new ImprintException(com.imprint.error.ErrorType.SCHEMA_ERROR, + "Cannot build empty record - add at least one field"); + } + + var writer = new ImprintWriter(schemaId); + for (var entry : fields.entrySet()) { + writer.addField(entry.getKey(), entry.getValue()); + } + return writer.build(); + } + + // Internal helper methods + private ImprintRecordBuilder addField(int id, Value value) { + Objects.requireNonNull(value, "Value cannot be null - use nullField() for explicit null values"); + if (fields.containsKey(id)) { + throw new IllegalArgumentException("Field ID " + id + " already exists - field IDs must be unique"); + } + fields.put(id, value); + return this; + } + + private Value convertToValue(Object obj) { + if (obj == null) { + return Value.nullValue(); + } + + if (obj instanceof Value) { + return (Value) obj; + } + + // Auto-boxing conversion + if (obj instanceof Boolean) { + return Value.fromBoolean((Boolean) obj); + } + if (obj instanceof Integer) { + return Value.fromInt32((Integer) obj); + } + if (obj instanceof Long) { + return Value.fromInt64((Long) obj); + } + if (obj instanceof Float) { + return Value.fromFloat32((Float) obj); + } + if (obj instanceof Double) { + return Value.fromFloat64((Double) obj); + } + if (obj instanceof String) { + return Value.fromString((String) obj); + } + if (obj instanceof byte[]) { + return Value.fromBytes((byte[]) obj); + } + if (obj instanceof List) { + //test + @SuppressWarnings("unchecked") + List list = (List) obj; + var convertedValues = new ArrayList(list.size()); + for (var item : list) { + convertedValues.add(convertToValue(item)); + } + return Value.fromArray(convertedValues); + } + if (obj instanceof Map) { + @SuppressWarnings("unchecked") + Map map = (Map) obj; + var convertedMap = new HashMap(map.size()); + for (var entry : map.entrySet()) { + var key = convertToMapKey(entry.getKey()); + var value = convertToValue(entry.getValue()); + convertedMap.put(key, value); + } + return Value.fromMap(convertedMap); + } + if (obj instanceof ImprintRecord) { + return Value.fromRow((ImprintRecord) obj); + } + + throw new IllegalArgumentException("Cannot convert " + obj.getClass().getSimpleName() + + " to Imprint Value. Supported types: boolean, int, long, float, double, String, byte[], List, Map, ImprintRecord"); + } + + private MapKey convertToMapKey(Object obj) { + if (obj instanceof Integer) { + return MapKey.fromInt32((Integer) obj); + } + if (obj instanceof Long) { + return MapKey.fromInt64((Long) obj); + } + if (obj instanceof String) { + return MapKey.fromString((String) obj); + } + if (obj instanceof byte[]) { + return MapKey.fromBytes((byte[]) obj); + } + + throw new IllegalArgumentException("Invalid map key type: " + obj.getClass().getSimpleName() + + ". Map keys must be int, long, String, or byte[]"); + } + + @Override + public String toString() { + return String.format("ImprintRecordBuilder{schemaId=%s, fields=%d}", schemaId, fields.size()); + } +} \ No newline at end of file diff --git a/src/main/java/com/imprint/core/ImprintWriter.java b/src/main/java/com/imprint/core/ImprintWriter.java new file mode 100644 index 0000000..39ad9ea --- /dev/null +++ b/src/main/java/com/imprint/core/ImprintWriter.java @@ -0,0 +1,257 @@ +package com.imprint.core; + +import com.imprint.error.ErrorType; +import com.imprint.error.ImprintException; +import com.imprint.types.MapKey; +import com.imprint.types.Value; +import com.imprint.util.VarInt; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.util.*; + +/** + * A writer for constructing ImprintRecords by adding fields sequentially. + */ +public final class ImprintWriter { + private final SchemaId schemaId; + private final TreeMap fields; // keep fields in sorted order + + public ImprintWriter(SchemaId schemaId) { + this.schemaId = Objects.requireNonNull(schemaId, "SchemaId cannot be null"); + this.fields = new TreeMap<>(); + } + + /** + * Adds a field to the record being built. + */ + public ImprintWriter addField(int id, Value value) { + Objects.requireNonNull(value, "Value cannot be null"); + this.fields.put(id, value); + return this; + } + + /** + * Consumes the writer and builds an ImprintRecord. + */ + public ImprintRecord build() throws ImprintException { + var directory = new ArrayList(fields.size()); + var payloadBuffer = ByteBuffer.allocate(estimatePayloadSize()); + payloadBuffer.order(ByteOrder.LITTLE_ENDIAN); + + for (var entry : fields.entrySet()) { + int fieldId = entry.getKey(); + var value = entry.getValue(); + + directory.add(new DirectoryEntry(fieldId, value.getTypeCode(), payloadBuffer.position())); + serializeValue(value, payloadBuffer); + } + + // Create read-only view of the payload without copying + payloadBuffer.flip(); // limit = position, position = 0 + var payloadView = payloadBuffer.slice().asReadOnlyBuffer(); + + var header = new Header(new Flags((byte) 0), schemaId, payloadView.remaining()); + return new ImprintRecord(header, directory, payloadView); + } + + private int estimatePayloadSize() { + // More accurate estimation to reduce allocations + int estimatedSize = 0; + for (var value : fields.values()) { + estimatedSize += estimateValueSize(value); + } + // Add 25% buffer to reduce reallocations + return Math.max(estimatedSize + (estimatedSize / 4), fields.size() * 16); + } + + /** + * Estimates the serialized size in bytes for a given value. + * This method provides size estimates for payload buffer allocation, + * supporting both array-based and ByteBuffer-based value types. + * + * @param value the value to estimate size for + * @return estimated size in bytes including type-specific overhead + */ + private int estimateValueSize(Value value) { + // Use TypeHandler for simple types + switch (value.getTypeCode()) { + case NULL: + case BOOL: + case INT32: + case INT64: + case FLOAT32: + case FLOAT64: + case BYTES: + case STRING: + return value.getTypeCode().getHandler().estimateSize(value); + + case ARRAY: + List array = ((Value.ArrayValue) value).getValue(); + int arraySize = VarInt.encodedLength(array.size()) + 1; // length + type code + for (Value element : array) { + arraySize += estimateValueSize(element); + } + return arraySize; + + case MAP: + Map map = ((Value.MapValue) value).getValue(); + int mapSize = VarInt.encodedLength(map.size()) + 2; // length + 2 type codes + for (Map.Entry entry : map.entrySet()) { + mapSize += estimateMapKeySize(entry.getKey()); + mapSize += estimateValueSize(entry.getValue()); + } + return mapSize; + + case ROW: + // Estimate nested record size (rough approximation) + return 100; // Conservative estimate + + default: + return 32; // Default fallback + } + } + + private int estimateMapKeySize(MapKey key) { + switch (key.getTypeCode()) { + case INT32: return 4; + case INT64: return 8; + case BYTES: + byte[] bytes = ((MapKey.BytesKey) key).getValue(); + return VarInt.encodedLength(bytes.length) + bytes.length; + + case STRING: + var str = ((MapKey.StringKey) key).getValue(); + int utf8Length = str.getBytes(StandardCharsets.UTF_8).length; + return VarInt.encodedLength(utf8Length) + utf8Length; + + default: + return 16; // Default fallback + } + } + + private void serializeValue(Value value, ByteBuffer buffer) throws ImprintException { + switch (value.getTypeCode()) { + case NULL: + case BOOL: + case INT32: + case INT64: + case FLOAT32: + case FLOAT64: + case BYTES: + case STRING: + value.getTypeCode().getHandler().serialize(value, buffer); + break; + + case ARRAY: + serializeArray((Value.ArrayValue) value, buffer); + break; + + case MAP: + serializeMap((Value.MapValue) value, buffer); + break; + + case ROW: + Value.RowValue rowValue = (Value.RowValue) value; + var serializedRow = rowValue.getValue().serializeToBuffer(); + buffer.put(serializedRow); + break; + + default: + throw new ImprintException(ErrorType.SERIALIZATION_ERROR, + "Unknown type code: " + value.getTypeCode()); + } + } + + private void serializeArray(Value.ArrayValue arrayValue, ByteBuffer buffer) throws ImprintException { + var elements = arrayValue.getValue(); + VarInt.encode(elements.size(), buffer); + + if (elements.isEmpty()) return; + + // All elements must have the same type + var elementType = elements.get(0).getTypeCode(); + buffer.put(elementType.getCode()); + for (var element : elements) { + if (element.getTypeCode() != elementType) { + throw new ImprintException(ErrorType.SCHEMA_ERROR, + "Array elements must have same type code: " + + element.getTypeCode() + " != " + elementType); + } + serializeValue(element, buffer); + } + } + + private void serializeMap(Value.MapValue mapValue, ByteBuffer buffer) throws ImprintException { + var map = mapValue.getValue(); + VarInt.encode(map.size(), buffer); + + if (map.isEmpty()) { + return; + } + + // All keys and values must have consistent types + var iterator = map.entrySet().iterator(); + var first = iterator.next(); + var keyType = first.getKey().getTypeCode(); + var valueType = first.getValue().getTypeCode(); + + buffer.put(keyType.getCode()); + buffer.put(valueType.getCode()); + + // Serialize the first entry + serializeMapKey(first.getKey(), buffer); + serializeValue(first.getValue(), buffer); + + // Serialize remaining entries + while (iterator.hasNext()) { + var entry = iterator.next(); + if (entry.getKey().getTypeCode() != keyType) { + throw new ImprintException(ErrorType.SCHEMA_ERROR, + "Map keys must have same type code: " + + entry.getKey().getTypeCode() + " != " + keyType); + } + if (entry.getValue().getTypeCode() != valueType) { + throw new ImprintException(ErrorType.SCHEMA_ERROR, + "Map values must have same type code: " + + entry.getValue().getTypeCode() + " != " + valueType); + } + + serializeMapKey(entry.getKey(), buffer); + serializeValue(entry.getValue(), buffer); + } + } + + private void serializeMapKey(MapKey key, ByteBuffer buffer) throws ImprintException { + switch (key.getTypeCode()) { + case INT32: + MapKey.Int32Key int32Key = (MapKey.Int32Key) key; + buffer.putInt(int32Key.getValue()); + break; + + case INT64: + MapKey.Int64Key int64Key = (MapKey.Int64Key) key; + buffer.putLong(int64Key.getValue()); + break; + + case BYTES: + MapKey.BytesKey bytesKey = (MapKey.BytesKey) key; + byte[] bytes = bytesKey.getValue(); + VarInt.encode(bytes.length, buffer); + buffer.put(bytes); + break; + + case STRING: + MapKey.StringKey stringKey = (MapKey.StringKey) key; + byte[] stringBytes = stringKey.getValue().getBytes(StandardCharsets.UTF_8); + VarInt.encode(stringBytes.length, buffer); + buffer.put(stringBytes); + break; + + default: + throw new ImprintException(ErrorType.SERIALIZATION_ERROR, + "Invalid map key type: " + key.getTypeCode()); + } + } +} \ No newline at end of file diff --git a/src/main/java/com/imprint/core/SchemaId.java b/src/main/java/com/imprint/core/SchemaId.java new file mode 100644 index 0000000..cb03c1c --- /dev/null +++ b/src/main/java/com/imprint/core/SchemaId.java @@ -0,0 +1,12 @@ +package com.imprint.core; + +import lombok.Value; + +/** + * Schema identifier containing field-space ID and schema hash. + */ +@Value +public class SchemaId { + int fieldspaceId; + int schemaHash; +} \ No newline at end of file diff --git a/src/main/java/com/imprint/error/ErrorType.java b/src/main/java/com/imprint/error/ErrorType.java new file mode 100644 index 0000000..97b9772 --- /dev/null +++ b/src/main/java/com/imprint/error/ErrorType.java @@ -0,0 +1,18 @@ +package com.imprint.error; + +/** + * Types of errors that can occur in Imprint operations. + */ +public enum ErrorType { + INVALID_MAGIC, + UNSUPPORTED_VERSION, + BUFFER_UNDERFLOW, + FIELD_NOT_FOUND, + SCHEMA_ERROR, + INVALID_UTF8_STRING, + MALFORMED_VARINT, + TYPE_MISMATCH, + INVALID_TYPE_CODE, + SERIALIZATION_ERROR, + DESERIALIZATION_ERROR +} diff --git a/src/main/java/com/imprint/error/ImprintException.java b/src/main/java/com/imprint/error/ImprintException.java new file mode 100644 index 0000000..1b5a5d4 --- /dev/null +++ b/src/main/java/com/imprint/error/ImprintException.java @@ -0,0 +1,26 @@ +package com.imprint.error; + +import lombok.Getter; + +/** + * Exception thrown by Imprint operations. + */ +@Getter +public class ImprintException extends Exception { + private final ErrorType errorType; + + public ImprintException(ErrorType errorType, String message) { + super(message); + this.errorType = errorType; + } + + public ImprintException(ErrorType errorType, String message, Throwable cause) { + super(message, cause); + this.errorType = errorType; + } + + @Override + public String toString() { + return String.format("ImprintException{type=%s, message='%s'}", errorType, getMessage()); + } +} diff --git a/src/main/java/com/imprint/types/MapKey.java b/src/main/java/com/imprint/types/MapKey.java new file mode 100644 index 0000000..c0e0747 --- /dev/null +++ b/src/main/java/com/imprint/types/MapKey.java @@ -0,0 +1,163 @@ +package com.imprint.types; + +import com.imprint.error.ErrorType; +import com.imprint.error.ImprintException; +import lombok.EqualsAndHashCode; +import lombok.Getter; + +import java.util.Arrays; +import java.util.Objects; + +/** + * A subset of Value that's valid as a map key. + * Only Int32, Int64, Bytes, and String are valid map keys. + */ +public abstract class MapKey { + + public abstract TypeCode getTypeCode(); + public abstract boolean equals(Object obj); + public abstract int hashCode(); + public abstract String toString(); + + public static MapKey fromInt32(int value) { + return new Int32Key(value); + } + + public static MapKey fromInt64(long value) { + return new Int64Key(value); + } + + public static MapKey fromBytes(byte[] value) { + return new BytesKey(value); + } + + public static MapKey fromString(String value) { + return new StringKey(value); + } + + public static MapKey fromValue(Value value) throws ImprintException { + switch (value.getTypeCode()) { + case INT32: + return fromInt32(((Value.Int32Value) value).getValue()); + case INT64: + return fromInt64(((Value.Int64Value) value).getValue()); + case BYTES: + if (value instanceof Value.BytesBufferValue) { + return fromBytes(((Value.BytesBufferValue) value).getValue()); + } else { + return fromBytes(((Value.BytesValue) value).getValue()); + } + case STRING: + if (value instanceof Value.StringBufferValue) { + return fromString(((Value.StringBufferValue) value).getValue()); + } else { + return fromString(((Value.StringValue) value).getValue()); + } + default: + throw new ImprintException(ErrorType.TYPE_MISMATCH, + "Cannot convert " + value.getTypeCode() + " to MapKey"); + } + } + + public Value toValue() { + switch (getTypeCode()) { + case INT32: + return Value.fromInt32(((Int32Key) this).getValue()); + case INT64: + return Value.fromInt64(((Int64Key) this).getValue()); + case BYTES: + return Value.fromBytes(((BytesKey) this).getValue()); + case STRING: + return Value.fromString(((StringKey) this).getValue()); + default: + throw new IllegalStateException("Unknown MapKey type: " + getTypeCode()); + } + } + + @Getter + @EqualsAndHashCode(callSuper = false) + public static class Int32Key extends MapKey { + private final int value; + + public Int32Key(int value) { + this.value = value; + } + + @Override + public TypeCode getTypeCode() { return TypeCode.INT32; } + + @Override + public String toString() { + return String.valueOf(value); + } + } + + @Getter + @EqualsAndHashCode(callSuper = false) + public static class Int64Key extends MapKey { + private final long value; + + public Int64Key(long value) { + this.value = value; + } + + @Override + public TypeCode getTypeCode() { return TypeCode.INT64; } + + @Override + public String toString() { + return String.valueOf(value); + } + } + + public static class BytesKey extends MapKey { + private final byte[] value; + + public BytesKey(byte[] value) { + this.value = value.clone(); // defensive copy + } + + public byte[] getValue() { + return value.clone(); // defensive copy + } + + @Override + public TypeCode getTypeCode() { return TypeCode.BYTES; } + + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (obj == null || getClass() != obj.getClass()) return false; + BytesKey that = (BytesKey) obj; + return Arrays.equals(value, that.value); + } + + @Override + public int hashCode() { + return Arrays.hashCode(value); + } + + @Override + public String toString() { + return "bytes[" + value.length + "]"; + } + } + + @Getter + @EqualsAndHashCode(callSuper = false) + public static class StringKey extends MapKey { + private final String value; + + public StringKey(String value) { + this.value = Objects.requireNonNull(value, "String cannot be null"); + } + + @Override + public TypeCode getTypeCode() { return TypeCode.STRING; } + + @Override + public String toString() { + return "\"" + value + "\""; + } + } +} \ No newline at end of file diff --git a/src/main/java/com/imprint/types/TypeCode.java b/src/main/java/com/imprint/types/TypeCode.java new file mode 100644 index 0000000..6bf450d --- /dev/null +++ b/src/main/java/com/imprint/types/TypeCode.java @@ -0,0 +1,48 @@ +package com.imprint.types; + +import com.imprint.error.ImprintException; +import com.imprint.error.ErrorType; +import lombok.Getter; + +/** + * Type codes for Imprint values. + */ +public enum TypeCode { + NULL(0x0, TypeHandler.NULL), + BOOL(0x1, TypeHandler.BOOL), + INT32(0x2, TypeHandler.INT32), + INT64(0x3, TypeHandler.INT64), + FLOAT32(0x4, TypeHandler.FLOAT32), + FLOAT64(0x5, TypeHandler.FLOAT64), + BYTES(0x6, TypeHandler.BYTES), + STRING(0x7, TypeHandler.STRING), + ARRAY(0x8, null), // TODO: implement + MAP(0x9, null), // TODO: implement + ROW(0xA, null); // TODO: implement (basically a placeholder for user-defined type) + + @Getter + private final byte code; + private final TypeHandler handler; + + TypeCode(int code, TypeHandler handler) { + this.code = (byte) code; + this.handler = handler; + } + + public TypeHandler getHandler() { + if (handler == null) { + throw new UnsupportedOperationException("Handler not implemented for " + this); + } + return handler; + } + + public static TypeCode fromByte(byte code) throws ImprintException { + for (TypeCode type : values()) { + if (type.code == code) { + return type; + } + } + throw new ImprintException(ErrorType.INVALID_TYPE_CODE, + "Unknown type code: 0x" + Integer.toHexString(code & 0xFF)); + } +} \ No newline at end of file diff --git a/src/main/java/com/imprint/types/TypeHandler.java b/src/main/java/com/imprint/types/TypeHandler.java new file mode 100644 index 0000000..4b5830a --- /dev/null +++ b/src/main/java/com/imprint/types/TypeHandler.java @@ -0,0 +1,304 @@ +package com.imprint.types; + +import com.imprint.error.ImprintException; +import com.imprint.util.VarInt; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; + +/** + * Interface for handling type-specific serialization, deserialization, and size estimation. + * Note that primitives are potentially auto/un-boxed here which could impact performance slightly + * but having all the types in their own implementation helps keep things organized for now, especially + * for dealing with and testing more complex types in the future. + */ +public interface TypeHandler { + Value deserialize(ByteBuffer buffer) throws ImprintException; + void serialize(Value value, ByteBuffer buffer) throws ImprintException; + int estimateSize(Value value); + ByteBuffer readValueBytes(ByteBuffer buffer) throws ImprintException; + + // Static implementations for each type + TypeHandler NULL = new TypeHandler() { + @Override + public Value deserialize(ByteBuffer buffer) { + return Value.nullValue(); + } + + @Override + public void serialize(Value value, ByteBuffer buffer) { + // NULL values have no payload + } + + @Override + public int estimateSize(Value value) { + return 0; + } + + @Override + public ByteBuffer readValueBytes(ByteBuffer buffer) { + return ByteBuffer.allocate(0).asReadOnlyBuffer(); + } + }; + + TypeHandler BOOL = new TypeHandler() { + @Override + public Value deserialize(ByteBuffer buffer) throws ImprintException { + if (buffer.remaining() < 1) { + throw new ImprintException(com.imprint.error.ErrorType.BUFFER_UNDERFLOW, "Not enough bytes for bool"); + } + byte boolByte = buffer.get(); + if (boolByte == 0) return Value.fromBoolean(false); + if (boolByte == 1) return Value.fromBoolean(true); + throw new ImprintException(com.imprint.error.ErrorType.SCHEMA_ERROR, "Invalid boolean value: " + boolByte); + } + + @Override + public void serialize(Value value, ByteBuffer buffer) { + Value.BoolValue boolValue = (Value.BoolValue) value; + buffer.put((byte) (boolValue.getValue() ? 1 : 0)); + } + + @Override + public int estimateSize(Value value) { + return 1; + } + + @Override + public ByteBuffer readValueBytes(ByteBuffer buffer) { + var boolBuffer = buffer.slice(); + boolBuffer.limit(1); + buffer.position(buffer.position() + 1); + return boolBuffer.asReadOnlyBuffer(); + } + }; + + TypeHandler INT32 = new TypeHandler() { + @Override + public Value deserialize(ByteBuffer buffer) throws ImprintException { + if (buffer.remaining() < 4) { + throw new ImprintException(com.imprint.error.ErrorType.BUFFER_UNDERFLOW, "Not enough bytes for int32"); + } + return Value.fromInt32(buffer.getInt()); + } + + @Override + public void serialize(Value value, ByteBuffer buffer) { + Value.Int32Value int32Value = (Value.Int32Value) value; + buffer.putInt(int32Value.getValue()); + } + + @Override + public int estimateSize(Value value) { + return 4; + } + + @Override + public ByteBuffer readValueBytes(ByteBuffer buffer) { + var int32Buffer = buffer.slice(); + int32Buffer.limit(4); + buffer.position(buffer.position() + 4); + return int32Buffer.asReadOnlyBuffer(); + } + }; + + TypeHandler INT64 = new TypeHandler() { + @Override + public Value deserialize(ByteBuffer buffer) throws ImprintException { + if (buffer.remaining() < 8) { + throw new ImprintException(com.imprint.error.ErrorType.BUFFER_UNDERFLOW, "Not enough bytes for int64"); + } + return Value.fromInt64(buffer.getLong()); + } + + @Override + public void serialize(Value value, ByteBuffer buffer) { + Value.Int64Value int64Value = (Value.Int64Value) value; + buffer.putLong(int64Value.getValue()); + } + + @Override + public int estimateSize(Value value) { + return 8; + } + + @Override + public ByteBuffer readValueBytes(ByteBuffer buffer) { + var int64Buffer = buffer.slice(); + int64Buffer.limit(8); + buffer.position(buffer.position() + 8); + return int64Buffer.asReadOnlyBuffer(); + } + }; + + TypeHandler FLOAT32 = new TypeHandler() { + @Override + public Value deserialize(ByteBuffer buffer) throws ImprintException { + if (buffer.remaining() < 4) { + throw new ImprintException(com.imprint.error.ErrorType.BUFFER_UNDERFLOW, "Not enough bytes for float32"); + } + return Value.fromFloat32(buffer.getFloat()); + } + + @Override + public void serialize(Value value, ByteBuffer buffer) { + Value.Float32Value float32Value = (Value.Float32Value) value; + buffer.putFloat(float32Value.getValue()); + } + + @Override + public int estimateSize(Value value) { + return 4; + } + + @Override + public ByteBuffer readValueBytes(ByteBuffer buffer) { + var float32Buffer = buffer.slice(); + float32Buffer.limit(4); + buffer.position(buffer.position() + 4); + return float32Buffer.asReadOnlyBuffer(); + } + }; + + TypeHandler FLOAT64 = new TypeHandler() { + @Override + public Value deserialize(ByteBuffer buffer) throws ImprintException { + if (buffer.remaining() < 8) { + throw new ImprintException(com.imprint.error.ErrorType.BUFFER_UNDERFLOW, "Not enough bytes for float64"); + } + return Value.fromFloat64(buffer.getDouble()); + } + + @Override + public void serialize(Value value, ByteBuffer buffer) { + Value.Float64Value float64Value = (Value.Float64Value) value; + buffer.putDouble(float64Value.getValue()); + } + + @Override + public int estimateSize(Value value) { + return 8; + } + + @Override + public ByteBuffer readValueBytes(ByteBuffer buffer) { + var float64Buffer = buffer.slice(); + float64Buffer.limit(8); + buffer.position(buffer.position() + 8); + return float64Buffer.asReadOnlyBuffer(); + } + }; + + TypeHandler BYTES = new TypeHandler() { + @Override + public Value deserialize(ByteBuffer buffer) throws ImprintException { + VarInt.DecodeResult lengthResult = VarInt.decode(buffer); + int length = lengthResult.getValue(); + if (buffer.remaining() < length) { + throw new ImprintException(com.imprint.error.ErrorType.BUFFER_UNDERFLOW, "Not enough bytes for bytes value"); + } + var bytesView = buffer.slice(); + bytesView.limit(length); + buffer.position(buffer.position() + length); + return Value.fromBytesBuffer(bytesView.asReadOnlyBuffer()); + } + + @Override + public void serialize(Value value, ByteBuffer buffer) { + if (value instanceof Value.BytesBufferValue) { + Value.BytesBufferValue bufferValue = (Value.BytesBufferValue) value; + var bytesBuffer = bufferValue.getBuffer(); + VarInt.encode(bytesBuffer.remaining(), buffer); + buffer.put(bytesBuffer); + } else { + Value.BytesValue bytesValue = (Value.BytesValue) value; + byte[] bytes = bytesValue.getValue(); + VarInt.encode(bytes.length, buffer); + buffer.put(bytes); + } + } + + @Override + public int estimateSize(Value value) { + if (value instanceof Value.BytesBufferValue) { + Value.BytesBufferValue bufferValue = (Value.BytesBufferValue) value; + int length = bufferValue.getBuffer().remaining(); + return VarInt.encodedLength(length) + length; + } else { + byte[] bytes = ((Value.BytesValue) value).getValue(); + return VarInt.encodedLength(bytes.length) + bytes.length; + } + } + + @Override + public ByteBuffer readValueBytes(ByteBuffer buffer) throws ImprintException { + int originalPosition = buffer.position(); + VarInt.DecodeResult lengthResult = VarInt.decode(buffer); + int totalLength = lengthResult.getBytesRead() + lengthResult.getValue(); + buffer.position(originalPosition); + var valueBuffer = buffer.slice(); + valueBuffer.limit(totalLength); + buffer.position(buffer.position() + totalLength); + return valueBuffer.asReadOnlyBuffer(); + } + }; + + TypeHandler STRING = new TypeHandler() { + @Override + public Value deserialize(ByteBuffer buffer) throws ImprintException { + VarInt.DecodeResult strLengthResult = VarInt.decode(buffer); + int strLength = strLengthResult.getValue(); + if (buffer.remaining() < strLength) { + throw new ImprintException(com.imprint.error.ErrorType.BUFFER_UNDERFLOW, "Not enough bytes for string value"); + } + var stringBytesView = buffer.slice(); + stringBytesView.limit(strLength); + buffer.position(buffer.position() + strLength); + try { + return Value.fromStringBuffer(stringBytesView.asReadOnlyBuffer()); + } catch (Exception e) { + throw new ImprintException(com.imprint.error.ErrorType.INVALID_UTF8_STRING, "Invalid UTF-8 string"); + } + } + + @Override + public void serialize(Value value, ByteBuffer buffer) { + if (value instanceof Value.StringBufferValue) { + Value.StringBufferValue bufferValue = (Value.StringBufferValue) value; + var stringBuffer = bufferValue.getBuffer(); + VarInt.encode(stringBuffer.remaining(), buffer); + buffer.put(stringBuffer); + } else { + Value.StringValue stringValue = (Value.StringValue) value; + byte[] stringBytes = stringValue.getValue().getBytes(StandardCharsets.UTF_8); + VarInt.encode(stringBytes.length, buffer); + buffer.put(stringBytes); + } + } + + @Override + public int estimateSize(Value value) { + if (value instanceof Value.StringBufferValue) { + Value.StringBufferValue bufferValue = (Value.StringBufferValue) value; + int length = bufferValue.getBuffer().remaining(); + return VarInt.encodedLength(length) + length; + } else { + String str = ((Value.StringValue) value).getValue(); + int utf8Length = str.getBytes(StandardCharsets.UTF_8).length; + return VarInt.encodedLength(utf8Length) + utf8Length; + } + } + + @Override + public ByteBuffer readValueBytes(ByteBuffer buffer) throws ImprintException { + int originalPosition = buffer.position(); + VarInt.DecodeResult lengthResult = VarInt.decode(buffer); + int totalLength = lengthResult.getBytesRead() + lengthResult.getValue(); + buffer.position(originalPosition); + var valueBuffer = buffer.slice(); + valueBuffer.limit(totalLength); + buffer.position(buffer.position() + totalLength); + return valueBuffer.asReadOnlyBuffer(); + } + }; +} \ No newline at end of file diff --git a/src/main/java/com/imprint/types/Value.java b/src/main/java/com/imprint/types/Value.java new file mode 100644 index 0000000..4710ec5 --- /dev/null +++ b/src/main/java/com/imprint/types/Value.java @@ -0,0 +1,403 @@ +package com.imprint.types; + +import com.imprint.core.ImprintRecord; +import lombok.EqualsAndHashCode; +import lombok.Getter; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * A value that can be stored in an Imprint record. + */ +public abstract class Value { + + public abstract TypeCode getTypeCode(); + public abstract boolean equals(Object obj); + public abstract int hashCode(); + public abstract String toString(); + + // Factory methods + public static Value nullValue() { + return NullValue.INSTANCE; + } + + public static Value fromBoolean(boolean value) { + return new BoolValue(value); + } + + public static Value fromInt32(int value) { + return new Int32Value(value); + } + + public static Value fromInt64(long value) { + return new Int64Value(value); + } + + public static Value fromFloat32(float value) { + return new Float32Value(value); + } + + public static Value fromFloat64(double value) { + return new Float64Value(value); + } + + public static Value fromBytes(byte[] value) { + return new BytesValue(value); + } + + public static Value fromBytesBuffer(ByteBuffer value) { + return new BytesBufferValue(value); + } + + public static Value fromString(String value) { + return new StringValue(value); + } + + public static Value fromStringBuffer(ByteBuffer value) { + return new StringBufferValue(value); + } + + public static Value fromArray(List value) { + return new ArrayValue(value); + } + + public static Value fromMap(Map value) { + return new MapValue(value); + } + + public static Value fromRow(ImprintRecord value) { + return new RowValue(value); + } + + // Null Value + @EqualsAndHashCode(callSuper = false) + public static class NullValue extends Value { + public static final NullValue INSTANCE = new NullValue(); + + private NullValue() {} + + @Override + public TypeCode getTypeCode() { return TypeCode.NULL; } + + @Override + public String toString() { + return "null"; + } + } + + // Boolean Value + @Getter + @EqualsAndHashCode(callSuper = false) + public static class BoolValue extends Value { + private final boolean value; + + public BoolValue(boolean value) { + this.value = value; + } + + public boolean getValue() { return value; } + + @Override + public TypeCode getTypeCode() { return TypeCode.BOOL; } + + @Override + public String toString() { + return String.valueOf(value); + } + } + + // Int32 Value + @Getter + @EqualsAndHashCode(callSuper = false) + public static class Int32Value extends Value { + private final int value; + + public Int32Value(int value) { + this.value = value; + } + + @Override + public TypeCode getTypeCode() { return TypeCode.INT32; } + + @Override + public String toString() { + return String.valueOf(value); + } + } + + // Int64 Value + @Getter + @EqualsAndHashCode(callSuper = false) + public static class Int64Value extends Value { + private final long value; + + public Int64Value(long value) { + this.value = value; + } + + @Override + public TypeCode getTypeCode() { return TypeCode.INT64; } + + @Override + public String toString() { + return String.valueOf(value); + } + } + + // Float32 Value + @Getter + @EqualsAndHashCode(callSuper = false) + public static class Float32Value extends Value { + private final float value; + + public Float32Value(float value) { + this.value = value; + } + + @Override + public TypeCode getTypeCode() { return TypeCode.FLOAT32; } + + @Override + public String toString() { + return String.valueOf(value); + } + } + + // Float64 Value + @Getter + @EqualsAndHashCode(callSuper = false) + public static class Float64Value extends Value { + private final double value; + + public Float64Value(double value) { + this.value = value; + } + + @Override + public TypeCode getTypeCode() { return TypeCode.FLOAT64; } + + @Override + public String toString() { + return String.valueOf(value); + } + } + + // Bytes Value (array-based) + public static class BytesValue extends Value { + private final byte[] value; + + public BytesValue(byte[] value) { + this.value = value.clone(); // defensive copy + } + + public byte[] getValue() { + return value.clone(); // defensive copy + } + + @Override + public TypeCode getTypeCode() { return TypeCode.BYTES; } + + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (obj == null || getClass() != obj.getClass()) return false; + BytesValue that = (BytesValue) obj; + return Arrays.equals(value, that.value); + } + + @Override + public int hashCode() { + return Arrays.hashCode(value); + } + + @Override + public String toString() { + return "bytes[" + value.length + "]"; + } + } + + // Bytes Value (ByteBuffer-based, zero-copy) + public static class BytesBufferValue extends Value { + private final ByteBuffer value; + + public BytesBufferValue(ByteBuffer value) { + this.value = value.asReadOnlyBuffer(); // zero-copy read-only view + } + + public byte[] getValue() { + // Fallback to array when needed + byte[] array = new byte[value.remaining()]; + value.duplicate().get(array); + return array; + } + + public ByteBuffer getBuffer() { + return value.duplicate(); // zero-copy view + } + + @Override + public TypeCode getTypeCode() { return TypeCode.BYTES; } + + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (obj == null) return false; + if (obj instanceof BytesBufferValue) { + BytesBufferValue that = (BytesBufferValue) obj; + return value.equals(that.value); + } + if (obj instanceof BytesValue) { + BytesValue that = (BytesValue) obj; + return Arrays.equals(getValue(), that.getValue()); + } + return false; + } + + @Override + public int hashCode() { + return value.hashCode(); + } + + @Override + public String toString() { + return "bytes[" + value.remaining() + "]"; + } + } + + // String Value (String-based) + @Getter + @EqualsAndHashCode(callSuper = false) + public static class StringValue extends Value { + private final String value; + + public StringValue(String value) { + this.value = Objects.requireNonNull(value, "String cannot be null"); + } + + @Override + public TypeCode getTypeCode() { return TypeCode.STRING; } + + @Override + public String toString() { + return "\"" + value + "\""; + } + } + + // String Value (ByteBuffer-based, zero-copy) + public static class StringBufferValue extends Value { + private final ByteBuffer value; + private volatile String cachedString; // lazy decode + + public StringBufferValue(ByteBuffer value) { + this.value = value.asReadOnlyBuffer(); // zero-copy read-only view + } + + public String getValue() { + if (cachedString == null) { + synchronized (this) { + if (cachedString == null) { + var array = new byte[value.remaining()]; + value.duplicate().get(array); + cachedString = new String(array, StandardCharsets.UTF_8); + } + } + } + return cachedString; + } + + public ByteBuffer getBuffer() { + return value.duplicate(); // zero-copy view + } + + @Override + public TypeCode getTypeCode() { return TypeCode.STRING; } + + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (obj == null) return false; + if (obj instanceof StringBufferValue) { + StringBufferValue that = (StringBufferValue) obj; + return value.equals(that.value); + } + if (obj instanceof StringValue) { + StringValue that = (StringValue) obj; + return getValue().equals(that.getValue()); + } + return false; + } + + @Override + public int hashCode() { + return getValue().hashCode(); // Use string hash for consistency + } + + @Override + public String toString() { + return "\"" + getValue() + "\""; + } + } + + // Array Value + @Getter + @EqualsAndHashCode(callSuper = false) + public static class ArrayValue extends Value { + private final List value; + + public ArrayValue(List value) { + this.value = List.copyOf(Objects.requireNonNull(value, "Array cannot be null")); + } + + @Override + public TypeCode getTypeCode() { return TypeCode.ARRAY; } + + @Override + public String toString() { + return value.toString(); + } + } + + // Map Value + @Getter + @EqualsAndHashCode(callSuper = false) + public static class MapValue extends Value { + private final Map value; + + public MapValue(Map value) { + this.value = Map.copyOf(Objects.requireNonNull(value, "Map cannot be null")); + } + + @Override + public TypeCode getTypeCode() { return TypeCode.MAP; } + + @Override + public String toString() { + return value.toString(); + } + } + + // Row Value + @Getter + @EqualsAndHashCode(callSuper = false) + public static class RowValue extends Value { + private final ImprintRecord value; + + public RowValue(ImprintRecord value) { + this.value = Objects.requireNonNull(value, "Record cannot be null"); + } + + @Override + public TypeCode getTypeCode() { return TypeCode.ROW; } + + @Override + public String toString() { + return "Row{" + value + "}"; + } + } + +} \ No newline at end of file diff --git a/src/main/java/com/imprint/util/VarInt.java b/src/main/java/com/imprint/util/VarInt.java new file mode 100644 index 0000000..5c9a7e5 --- /dev/null +++ b/src/main/java/com/imprint/util/VarInt.java @@ -0,0 +1,118 @@ +package com.imprint.util; + +import com.imprint.error.ImprintException; +import com.imprint.error.ErrorType; +import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; +import java.nio.ByteBuffer; + +/** + * Utility class for encoding and decoding variable-length integers (VarInt). + * Supports encoding/decoding of 32-bit unsigned integers. + */ +public final class VarInt { + + private static final byte CONTINUATION_BIT = (byte) 0x80; + private static final byte SEGMENT_BITS = 0x7f; + private static final int MAX_VARINT_LEN = 5; // Enough for u32 + + private VarInt() {} // utility class + + + /** + * Encode a 32-bit unsigned integer as a VarInt into the given ByteBuffer. + * @param value the value to encode (treated as unsigned) + * @param buffer the buffer to write to + */ + public static void encode(int value, ByteBuffer buffer) { + // Convert to unsigned long for proper bit manipulation + long val = Integer.toUnsignedLong(value); + + // Encode at least one byte, then continue while value has more bits + do { + byte b = (byte) (val & SEGMENT_BITS); + val >>>= 7; + if (val != 0) { + b |= CONTINUATION_BIT; + } + buffer.put(b); + } while (val != 0); + } + + + /** + * Decode a VarInt from a ByteBuffer. + * @param buffer the buffer to decode from + * @return a DecodeResult containing the decoded value and number of bytes consumed + * @throws ImprintException if the VarInt is malformed + */ + public static DecodeResult decode(ByteBuffer buffer) throws ImprintException { + long result = 0; + int shift = 0; + int bytesRead = 0; + + while (true) { + if (bytesRead >= MAX_VARINT_LEN) { + throw new ImprintException(ErrorType.MALFORMED_VARINT, "VarInt too long"); + } + if (!buffer.hasRemaining()) { + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, + "Unexpected end of data while reading VarInt"); + } + + byte b = buffer.get(); + bytesRead++; + + // Check if adding these 7 bits would overflow + long segment = b & SEGMENT_BITS; + if (shift >= 32 || (shift == 28 && segment > 0xF)) { + throw new ImprintException(ErrorType.MALFORMED_VARINT, "VarInt overflow"); + } + + // Add the bottom 7 bits to the result + result |= segment << shift; + + // If the high bit is not set, this is the last byte + if ((b & CONTINUATION_BIT) == 0) { + break; + } + + shift += 7; + } + + return new DecodeResult((int) result, bytesRead); + } + + /** + * Calculate the number of bytes needed to encode the given value as a VarInt. + * @param value the value to encode (treated as unsigned) + * @return the number of bytes needed + */ + public static int encodedLength(int value) { + // Convert to unsigned long for proper bit manipulation + long val = Integer.toUnsignedLong(value); + int length = 1; + + // Count additional bytes needed for values >= 128 + while (val >= 0x80) { + val >>>= 7; + length++; + } + + return length; + } + + /** + * Result of a VarInt decode operation. + */ + @Getter + @AllArgsConstructor + @EqualsAndHashCode + @ToString + public static class DecodeResult { + private final int value; + private final int bytesRead; + } +} \ No newline at end of file diff --git a/src/test/java/com/imprint/ByteBufferIntegrationTest.java b/src/test/java/com/imprint/ByteBufferIntegrationTest.java new file mode 100644 index 0000000..9460cbf --- /dev/null +++ b/src/test/java/com/imprint/ByteBufferIntegrationTest.java @@ -0,0 +1,87 @@ +package com.imprint; + +import com.imprint.core.*; +import com.imprint.types.*; +import com.imprint.error.ImprintException; +import java.nio.ByteBuffer; +import java.util.*; + +/** + * Integration test to verify ByteBuffer functionality and zero-copy benefits. + */ +public class ByteBufferIntegrationTest { + + public static void main(String[] args) { + try { + testByteBufferFunctionality(); + testZeroCopy(); + System.out.println("All ByteBuffer integration tests passed!"); + } catch (Exception e) { + System.err.println("ByteBuffer integration test failed: " + e.getMessage()); + e.printStackTrace(); + System.exit(1); + } + } + + static void testByteBufferFunctionality() throws ImprintException { + System.out.println("Testing ByteBuffer functionality..."); + + SchemaId schemaId = new SchemaId(1, 0xdeadbeef); + ImprintWriter writer = new ImprintWriter(schemaId); + + writer.addField(1, Value.fromInt32(42)) + .addField(2, Value.fromString("zero-copy test")) + .addField(3, Value.fromBytes(new byte[]{1, 2, 3, 4, 5})); + + ImprintRecord record = writer.build(); + + // Test ByteBuffer serialization + ByteBuffer serializedBuffer = record.serializeToBuffer(); + assert serializedBuffer.isReadOnly() : "Serialized buffer should be read-only"; + + // Test deserialization from ByteBuffer + ImprintRecord deserialized = ImprintRecord.deserialize(serializedBuffer); + + assert deserialized.getValue(1).get().equals(Value.fromInt32(42)); + assert deserialized.getValue(2).get().equals(Value.fromString("zero-copy test")); + + // Test raw bytes access returns ByteBuffer + Optional rawBytes = deserialized.getRawBytes(1); + assert rawBytes.isPresent() : "Raw bytes should be present for field 1"; + assert rawBytes.get().isReadOnly() : "Raw bytes buffer should be read-only"; + + System.out.println("ByteBuffer functionality test passed"); + } + + static void testZeroCopy() { + System.out.println("Testing zero-copy"); + + // Create a large payload to demonstrate zero-copy benefits + byte[] largePayload = new byte[1024 * 1024]; // 1MB + Arrays.fill(largePayload, (byte) 0xAB); + + SchemaId schemaId = new SchemaId(2, 0xcafebabe); + ImprintWriter writer = new ImprintWriter(schemaId); + + try { + writer.addField(1, Value.fromBytes(largePayload)); + ImprintRecord record = writer.build(); + + // Test that getRawBytes returns a view, not a copy + Optional rawBytes = record.getRawBytes(1); + assert rawBytes.isPresent() : "Raw bytes should be present"; + + ByteBuffer rawBuffer = rawBytes.get(); + assert rawBuffer.isReadOnly() : "Raw buffer should be read-only"; + + // The buffer should be positioned at the start of the actual data + // (after the VarInt length prefix) + assert rawBuffer.remaining() > largePayload.length : "Buffer should include length prefix"; + + System.out.println("Zero-copy benefits test passed"); + + } catch (ImprintException e) { + throw new RuntimeException("Failed zero-copy test", e); + } + } +} \ No newline at end of file diff --git a/src/test/java/com/imprint/ComprehensiveTest.java b/src/test/java/com/imprint/ComprehensiveTest.java new file mode 100644 index 0000000..af7f0b0 --- /dev/null +++ b/src/test/java/com/imprint/ComprehensiveTest.java @@ -0,0 +1,208 @@ +package com.imprint; + +import com.imprint.core.*; +import com.imprint.types.*; +import com.imprint.error.ImprintException; +import com.imprint.util.VarInt; +import java.nio.ByteBuffer; +import java.util.*; + +/** + * Comprehensive test to verify all functionality works correctly. + */ +public class ComprehensiveTest { + + public static void main(String[] args) { + try { + testVarIntFunctionality(); + testValueTypes(); + testMapKeys(); + testComplexSerialization(); + testErrorHandling(); + testByteBufferPerformance(); + System.out.println("All comprehensive tests passed!"); + } catch (Exception e) { + System.err.println("Comprehensive test failed: " + e.getMessage()); + e.printStackTrace(); + System.exit(1); + } + } + + static void testVarIntFunctionality() throws ImprintException { + System.out.println("Testing VarInt functionality..."); + + // Test encoding/decoding of various values + int[] testValues = {0, 1, 127, 128, 16383, 16384, Integer.MAX_VALUE}; + + for (int value : testValues) { + ByteBuffer buffer = ByteBuffer.allocate(10); + VarInt.encode(value, buffer); + int encodedLength = buffer.position(); + + buffer.flip(); + VarInt.DecodeResult result = VarInt.decode(buffer); + + assert result.getValue() == value : "VarInt roundtrip failed for " + value; + assert result.getBytesRead() == encodedLength : "Bytes read mismatch for " + value; + } + + System.out.println("✓ VarInt functionality test passed"); + } + + static void testValueTypes() { + System.out.println("Testing Value types"); + + // Test all value types + Value nullVal = Value.nullValue(); + Value boolVal = Value.fromBoolean(true); + Value int32Val = Value.fromInt32(42); + Value int64Val = Value.fromInt64(123456789L); + Value float32Val = Value.fromFloat32(3.14f); + Value float64Val = Value.fromFloat64(2.718281828); + Value bytesVal = Value.fromBytes(new byte[]{1, 2, 3, 4}); + Value stringVal = Value.fromString("test"); + + // Test type codes + assert nullVal.getTypeCode() == TypeCode.NULL; + assert boolVal.getTypeCode() == TypeCode.BOOL; + assert int32Val.getTypeCode() == TypeCode.INT32; + assert int64Val.getTypeCode() == TypeCode.INT64; + assert float32Val.getTypeCode() == TypeCode.FLOAT32; + assert float64Val.getTypeCode() == TypeCode.FLOAT64; + assert bytesVal.getTypeCode() == TypeCode.BYTES; + assert stringVal.getTypeCode() == TypeCode.STRING; + + // Test value extraction + assert ((Value.BoolValue) boolVal).getValue(); + assert ((Value.Int32Value) int32Val).getValue() == 42; + assert ((Value.Int64Value) int64Val).getValue() == 123456789L; + assert ((Value.Float32Value) float32Val).getValue() == 3.14f; + assert ((Value.Float64Value) float64Val).getValue() == 2.718281828; + assert Arrays.equals(((Value.BytesValue) bytesVal).getValue(), new byte[]{1, 2, 3, 4}); + assert ((Value.StringValue) stringVal).getValue().equals("test"); + + System.out.println("✓ Value types test passed"); + } + + static void testMapKeys() throws ImprintException { + System.out.println("Testing MapKey functionality..."); + + MapKey int32Key = MapKey.fromInt32(42); + MapKey int64Key = MapKey.fromInt64(123L); + MapKey bytesKey = MapKey.fromBytes(new byte[]{1, 2, 3}); + MapKey stringKey = MapKey.fromString("test"); + + // Test conversion to/from Values + Value int32Value = int32Key.toValue(); + Value int64Value = int64Key.toValue(); + Value bytesValue = bytesKey.toValue(); + Value stringValue = stringKey.toValue(); + + assert MapKey.fromValue(int32Value).equals(int32Key); + assert MapKey.fromValue(int64Value).equals(int64Key); + assert MapKey.fromValue(bytesValue).equals(bytesKey); + assert MapKey.fromValue(stringValue).equals(stringKey); + + System.out.println("✓ MapKey functionality test passed"); + } + + static void testComplexSerialization() throws ImprintException { + System.out.println("Testing complex serialization..."); + + SchemaId schemaId = new SchemaId(1, 0xdeadbeef); + ImprintWriter writer = new ImprintWriter(schemaId); + + // Create complex nested structure + List array = Arrays.asList( + Value.fromInt32(1), + Value.fromInt32(2), + Value.fromInt32(3) + ); + + Map map = new HashMap<>(); + map.put(MapKey.fromString("key1"), Value.fromString("value1")); + map.put(MapKey.fromString("key2"), Value.fromString("value2")); + + writer.addField(1, Value.fromArray(array)) + .addField(2, Value.fromMap(map)) + .addField(3, Value.fromString("complex test")); + + ImprintRecord record = writer.build(); + + // Test ByteBuffer serialization + ByteBuffer serialized = record.serializeToBuffer(); + ImprintRecord deserialized = ImprintRecord.deserialize(serialized); + + // Verify complex structures + Value deserializedArray = deserialized.getValue(1).get(); + assert deserializedArray instanceof Value.ArrayValue; + List deserializedList = ((Value.ArrayValue) deserializedArray).getValue(); + assert deserializedList.size() == 3; + assert deserializedList.get(0).equals(Value.fromInt32(1)); + + Value deserializedMap = deserialized.getValue(2).get(); + assert deserializedMap instanceof Value.MapValue; + Map deserializedMapValue = ((Value.MapValue) deserializedMap).getValue(); + assert deserializedMapValue.size() == 2; + assert deserializedMapValue.get(MapKey.fromString("key1")).equals(Value.fromString("value1")); + + System.out.println("✓ Complex serialization test passed"); + } + + static void testErrorHandling() { + System.out.println("Testing error handling..."); + + try { + // Test invalid type code + TypeCode.fromByte((byte) 0xFF); + assert false : "Should have thrown exception for invalid type code"; + } catch (ImprintException e) { + assert e.getErrorType() == com.imprint.error.ErrorType.INVALID_TYPE_CODE; + } + + try { + // Test invalid magic byte + byte[] invalidData = new byte[15]; + invalidData[0] = 0x00; // wrong magic + ImprintRecord.deserialize(invalidData); + assert false : "Should have thrown exception for invalid magic"; + } catch (ImprintException e) { + assert e.getErrorType() == com.imprint.error.ErrorType.INVALID_MAGIC; + } + + System.out.println("✓ Error handling test passed"); + } + + static void testByteBufferPerformance() throws ImprintException { + System.out.println("Testing ByteBuffer performance benefits..."); + + // Create a record with moderate-sized data + byte[] testData = new byte[1024]; + for (int i = 0; i < testData.length; i++) { + testData[i] = (byte) (i % 256); + } + + SchemaId schemaId = new SchemaId(1, 0x12345678); + ImprintWriter writer = new ImprintWriter(schemaId); + writer.addField(1, Value.fromBytes(testData)) + .addField(2, Value.fromString("performance test")); + + ImprintRecord record = writer.build(); + + // Test that raw bytes access is zero-copy + Optional rawBytes = record.getRawBytes(1); + assert rawBytes.isPresent(); + assert rawBytes.get().isReadOnly(); + + // Test ByteBuffer serialization + ByteBuffer serialized = record.serializeToBuffer(); + assert serialized.isReadOnly(); + + // Verify deserialization works + ImprintRecord deserialized = ImprintRecord.deserialize(serialized); + Value retrievedBytes = deserialized.getValue(1).get(); + assert Arrays.equals(((Value.BytesValue) retrievedBytes).getValue(), testData); + + System.out.println("✓ ByteBuffer performance test passed"); + } +} \ No newline at end of file diff --git a/src/test/java/com/imprint/IntegrationTest.java b/src/test/java/com/imprint/IntegrationTest.java new file mode 100644 index 0000000..49cfce7 --- /dev/null +++ b/src/test/java/com/imprint/IntegrationTest.java @@ -0,0 +1,144 @@ +package com.imprint; + +import com.imprint.core.*; +import com.imprint.types.*; +import com.imprint.error.ImprintException; +import java.util.*; + +/** + * Integration test to verify the complete Java implementation works. + * This can be run as a simple main method without JUnit. + */ +public class IntegrationTest { + + public static void main(String[] args) { + try { + testBasicFunctionality(); + testArraysAndMaps(); + testNestedRecords(); + System.out.println("All integration tests passed!"); + } catch (Exception e) { + System.err.println("Integration test failed: " + e.getMessage()); + e.printStackTrace(); + System.exit(1); + } + } + + static void testBasicFunctionality() throws ImprintException { + System.out.println("Testing basic functionality..."); + + SchemaId schemaId = new SchemaId(1, 0xdeadbeef); + ImprintWriter writer = new ImprintWriter(schemaId); + + writer.addField(1, Value.fromInt32(42)) + .addField(2, Value.fromString("testing java imprint spec")) + .addField(3, Value.fromBoolean(true)) + .addField(4, Value.fromFloat64(3.14159)) + .addField(5, Value.fromBytes(new byte[]{1, 2, 3, 4})); + + ImprintRecord record = writer.build(); + + // Verify we can read values back + assert record.getValue(1).get().equals(Value.fromInt32(42)); + assert record.getValue(2).get().equals(Value.fromString("testing java imprint spec")); + assert record.getValue(3).get().equals(Value.fromBoolean(true)); + assert record.getValue(999).isEmpty(); // non-existent field + + // Test serialization round-trip + var buffer = record.serializeToBuffer(); + byte[] serialized = new byte[buffer.remaining()]; + buffer.get(serialized); + ImprintRecord deserialized = ImprintRecord.deserialize(serialized); + + assert deserialized.getValue(1).get().equals(Value.fromInt32(42)); + assert deserialized.getValue(2).get().equals(Value.fromString("testing java imprint spec")); + assert deserialized.getValue(3).get().equals(Value.fromBoolean(true)); + + System.out.println("✓ Basic functionality test passed"); + } + + static void testArraysAndMaps() throws ImprintException { + System.out.println("Testing arrays and maps..."); + + SchemaId schemaId = new SchemaId(2, 0xcafebabe); + ImprintWriter writer = new ImprintWriter(schemaId); + + // Create an array + List intArray = Arrays.asList( + Value.fromInt32(1), + Value.fromInt32(2), + Value.fromInt32(3) + ); + + // Create a map + Map stringToIntMap = new HashMap<>(); + stringToIntMap.put(MapKey.fromString("one"), Value.fromInt32(1)); + stringToIntMap.put(MapKey.fromString("two"), Value.fromInt32(2)); + + writer.addField(1, Value.fromArray(intArray)) + .addField(2, Value.fromMap(stringToIntMap)); + + ImprintRecord record = writer.build(); + + // Test serialization round-trip + var buffer = record.serializeToBuffer(); + byte[] serialized = new byte[buffer.remaining()]; + buffer.get(serialized); + ImprintRecord deserialized = ImprintRecord.deserialize(serialized); + + // Verify array + Value arrayValue = deserialized.getValue(1).get(); + assert arrayValue instanceof Value.ArrayValue; + List deserializedArray = ((Value.ArrayValue) arrayValue).getValue(); + assert deserializedArray.size() == 3; + assert deserializedArray.get(0).equals(Value.fromInt32(1)); + + // Verify map + Value mapValue = deserialized.getValue(2).get(); + assert mapValue instanceof Value.MapValue; + Map deserializedMap = ((Value.MapValue) mapValue).getValue(); + assert deserializedMap.size() == 2; + assert deserializedMap.get(MapKey.fromString("one")).equals(Value.fromInt32(1)); + + System.out.println("✓ Arrays and maps test passed"); + } + + static void testNestedRecords() throws ImprintException { + System.out.println("Testing nested records..."); + + // Create inner record + SchemaId innerSchemaId = new SchemaId(3, 0x12345678); + ImprintWriter innerWriter = new ImprintWriter(innerSchemaId); + innerWriter.addField(1, Value.fromString("nested data")) + .addField(2, Value.fromInt64(9876543210L)); + ImprintRecord innerRecord = innerWriter.build(); + + // Create outer record + SchemaId outerSchemaId = new SchemaId(4, 0x87654321); + ImprintWriter outerWriter = new ImprintWriter(outerSchemaId); + outerWriter.addField(1, Value.fromRow(innerRecord)) + .addField(2, Value.fromString("outer data")); + ImprintRecord outerRecord = outerWriter.build(); + + // Test serialization round-trip + var buffer = outerRecord.serializeToBuffer(); + byte[] serialized = new byte[buffer.remaining()]; + buffer.get(serialized); + ImprintRecord deserialized = ImprintRecord.deserialize(serialized); + + // Verify outer record + assert deserialized.getHeader().getSchemaId().getFieldspaceId() == 4; + assert deserialized.getValue(2).get().equals(Value.fromString("outer data")); + + // Verify nested record + Value rowValue = deserialized.getValue(1).get(); + assert rowValue instanceof Value.RowValue; + ImprintRecord nestedRecord = ((Value.RowValue) rowValue).getValue(); + + assert nestedRecord.getHeader().getSchemaId().getFieldspaceId() == 3; + assert nestedRecord.getValue(1).get().equals(Value.fromString("nested data")); + assert nestedRecord.getValue(2).get().equals(Value.fromInt64(9876543210L)); + + System.out.println("✓ Nested records test passed"); + } +} \ No newline at end of file diff --git a/src/test/java/com/imprint/benchmark/ProfilerTest.java b/src/test/java/com/imprint/benchmark/ProfilerTest.java new file mode 100644 index 0000000..233d993 --- /dev/null +++ b/src/test/java/com/imprint/benchmark/ProfilerTest.java @@ -0,0 +1,226 @@ +package com.imprint.benchmark; + +import com.imprint.core.*; +import com.imprint.types.Value; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Disabled; + +import java.util.Random; + +/** + * A test designed for profiling hotspots during development. + *

+ * To use with a profiler: + * 1. Remove @Disabled annotation + * 2. Run with JProfiler, VisualVM, or async-profiler: + * - JProfiler: Attach to test JVM + * - VisualVM: jvisualvm, attach to process + * - async-profiler: java -jar async-profiler.jar -d 30 -f profile.html + * 3. Look for hotspots in CPU sampling + *

+ * Key areas to examine: + * - Object allocation (memory profiling) + * - Method call frequency (CPU sampling) + * - GC pressure (memory profiling) + * - String operations and UTF-8 encoding + * - ByteBuffer operations + */ +@Disabled("Enable manually for profiling") +public class ProfilerTest { + + private static final int ITERATIONS = 1_000_000; + private static final int RECORD_SIZE = 20; + + @Test + void profileFieldAccess() throws Exception { + System.out.println("Starting profiler test - attach profiler now..."); + Thread.sleep(5000); // Give time to attach profiler + + // Create a representative record + var record = createTestRecord(); + + System.out.println("Beginning field access profiling..."); + long start = System.nanoTime(); + + // Simulate real-world access patterns + Random random = new Random(42); + int hits = 0; + + for (int i = 0; i < ITERATIONS; i++) { + // Random field access (hotspot) + int fieldId = random.nextInt(RECORD_SIZE) + 1; + var value = record.getValue(fieldId); + if (value.isPresent()) { + hits++; + + // Trigger string decoding (potential hotspot) + if (value.get().getTypeCode() == com.imprint.types.TypeCode.STRING) { + if (value.get() instanceof Value.StringBufferValue) { + ((Value.StringBufferValue) value.get()).getValue(); + } else { + ((Value.StringValue) value.get()).getValue(); + } + } + } + + // Some raw access (zero-copy path) + if (i % 10 == 0) { + record.getRawBytes(fieldId); + } + } + + long duration = System.nanoTime() - start; + System.out.printf("Completed %,d field accesses in %.2f ms (avg: %.1f ns/op, hits: %d)%n", + ITERATIONS, duration / 1_000_000.0, (double) duration / ITERATIONS, hits); + } + + @Test + void profileSerialization() throws Exception { + System.out.println("Starting serialization profiler test..."); + Thread.sleep(3000); + + var schemaId = new SchemaId(1, 0x12345678); + + System.out.println("Beginning serialization profiling..."); + long start = System.nanoTime(); + + // Create and serialize many records (allocation hotspot) + for (int i = 0; i < 100_000; i++) { + var writer = new ImprintWriter(schemaId); + + // Add various field types + writer.addField(1, Value.fromInt32(i)) + .addField(2, Value.fromString("test-string-" + i)) + .addField(3, Value.fromFloat64(i * 3.14159)) + .addField(4, Value.fromBytes(("bytes-" + i).getBytes())); + + var record = writer.build(); + var serialized = record.serializeToBuffer(); // Potential hotspot + + // Trigger some deserialization + if (i % 1000 == 0) { + var deserialized = ImprintRecord.deserialize(serialized); + deserialized.getValue(2); // String decoding hotspot + } + } + + long duration = System.nanoTime() - start; + System.out.printf("Completed serialization test in %.2f ms%n", duration / 1_000_000.0); + } + + @Test + void profileProjection() throws Exception { + System.out.println("Starting projection profiler test..."); + Thread.sleep(3000); + + var record = createLargeRecord(); + + System.out.println("Beginning projection profiling..."); + long start = System.nanoTime(); + + // Simulate analytical workload - project subset of fields repeatedly + for (int i = 0; i < 50_000; i++) { + // Project 10 fields out of 100 (common analytical pattern) + for (int fieldId = 1; fieldId <= 10; fieldId++) { + var value = record.getValue(fieldId); + if (value.isPresent()) { + // Force materialization of string values + if (value.get().getTypeCode() == com.imprint.types.TypeCode.STRING) { + if (value.get() instanceof Value.StringBufferValue) { + ((Value.StringBufferValue) value.get()).getValue(); + } + } + } + } + } + + long duration = System.nanoTime() - start; + System.out.printf("Completed projection test in %.2f ms%n", duration / 1_000_000.0); + } + + @Test + void profileMemoryAllocation() throws Exception { + System.out.println("Starting allocation profiler test..."); + Thread.sleep(3000); + + System.out.println("Beginning allocation profiling - watch for GC events..."); + + // Force allocation pressure to reveal GC hotspots + for (int batch = 0; batch < 1000; batch++) { + for (int i = 0; i < 1000; i++) { + var schemaId = new SchemaId(batch, i); + var writer = new ImprintWriter(schemaId); + + // Create strings of varying sizes (allocation pressure) + writer.addField(1, Value.fromString("small")) + .addField(2, Value.fromString("medium-length-string-" + i)) + .addField(3, Value.fromString("very-long-string-that-will-cause-more-allocation-pressure-" + batch + "-" + i)) + .addField(4, Value.fromBytes(new byte[100 + i % 1000])); // Varying byte arrays + + var record = writer.build(); + + // Some deserialization to trigger string decoding allocations + record.getValue(2); + record.getValue(3); + } + + if (batch % 100 == 0) { + System.out.printf("Completed batch %d/1000%n", batch); + } + } + + System.out.println("Allocation test complete - check GC logs and memory profiler"); + } + + private ImprintRecord createTestRecord() throws Exception { + var schemaId = new SchemaId(1, 0xdeadbeef); + var writer = new ImprintWriter(schemaId); + + for (int i = 1; i <= RECORD_SIZE; i++) { + switch (i % 4) { + case 0: + writer.addField(i, Value.fromInt32(i * 100)); + break; + case 1: + writer.addField(i, Value.fromString("field-value-" + i)); + break; + case 2: + writer.addField(i, Value.fromFloat64(i * 3.14159)); + break; + case 3: + writer.addField(i, Value.fromBytes(("bytes-" + i).getBytes())); + break; + } + } + + return writer.build(); + } + + private ImprintRecord createLargeRecord() throws Exception { + var schemaId = new SchemaId(2, 0xcafebabe); + var writer = new ImprintWriter(schemaId); + + // Create 100 fields with realistic data + for (int i = 1; i <= 100; i++) { + switch (i % 5) { + case 0: + writer.addField(i, Value.fromInt32(i)); + break; + case 1: + writer.addField(i, Value.fromString("user-name-" + i + "@example.com")); + break; + case 2: + writer.addField(i, Value.fromString("Some longer descriptive text for field " + i + " that might represent a comment or description")); + break; + case 3: + writer.addField(i, Value.fromFloat64(i * 2.718281828)); + break; + case 4: + writer.addField(i, Value.fromBytes(String.format("binary-data-%04d", i).getBytes())); + break; + } + } + + return writer.build(); + } +} \ No newline at end of file diff --git a/src/test/java/com/imprint/core/ImprintRecordBuilderTest.java b/src/test/java/com/imprint/core/ImprintRecordBuilderTest.java new file mode 100644 index 0000000..54dcfae --- /dev/null +++ b/src/test/java/com/imprint/core/ImprintRecordBuilderTest.java @@ -0,0 +1,234 @@ +package com.imprint.core; + +import com.imprint.error.ImprintException; +import com.imprint.types.Value; +import org.junit.jupiter.api.Test; + +import java.util.*; + +import static org.assertj.core.api.Assertions.*; + +class ImprintRecordBuilderTest { + + private static final SchemaId TEST_SCHEMA = new SchemaId(1, 0x12345678); + + @Test + void shouldCreateRecordWithPrimitiveTypes() throws ImprintException { + var record = ImprintRecord.builder(TEST_SCHEMA) + .field(1, true) + .field(2, 42) + .field(3, 123L) + .field(4, 3.14f) + .field(5, 2.718) + .field(6, "hello world") + .field(7, new byte[]{1, 2, 3}) + .nullField(8) + .build(); + + assertThat(record.getHeader().getSchemaId()).isEqualTo(TEST_SCHEMA); + assertThat(record.getDirectory()).hasSize(8); + + // Verify field values + assertThat(getFieldValue(record, 1, Value.BoolValue.class).getValue()).isTrue(); + assertThat(getFieldValue(record, 2, Value.Int32Value.class).getValue()).isEqualTo(42); + assertThat(getFieldValue(record, 3, Value.Int64Value.class).getValue()).isEqualTo(123L); + assertThat(getFieldValue(record, 4, Value.Float32Value.class).getValue()).isEqualTo(3.14f); + assertThat(getFieldValue(record, 5, Value.Float64Value.class).getValue()).isEqualTo(2.718); + assertThat(getStringValue(record, 6)).isEqualTo("hello world"); + assertThat(getBytesValue(record, 7)).isEqualTo(new byte[]{1, 2, 3}); + assertThat(record.getValue(8).get()).isInstanceOf(Value.NullValue.class); + } + + @Test + void shouldCreateRecordWithCollections() throws ImprintException { + var list = List.of(1, 2, 3); + var map = Map.of("key1", 100, "key2", 200); + + var record = ImprintRecord.builder(TEST_SCHEMA) + .field(1, list) + .field(2, map) + .build(); + + // Verify array + var arrayValue = getFieldValue(record, 1, Value.ArrayValue.class); + assertThat(arrayValue.getValue()).hasSize(3); + assertThat(((Value.Int32Value) arrayValue.getValue().get(0)).getValue()).isEqualTo(1); + assertThat(((Value.Int32Value) arrayValue.getValue().get(1)).getValue()).isEqualTo(2); + assertThat(((Value.Int32Value) arrayValue.getValue().get(2)).getValue()).isEqualTo(3); + + // Verify map + var mapValue = getFieldValue(record, 2, Value.MapValue.class); + assertThat(mapValue.getValue()).hasSize(2); + } + + @Test + void shouldCreateRecordWithNestedRecord() throws ImprintException { + var nestedRecord = ImprintRecord.builder(new SchemaId(2, 0x87654321)) + .field(1, "nested") + .field(2, 999) + .build(); + + var record = ImprintRecord.builder(TEST_SCHEMA) + .field(1, "parent") + .field(2, nestedRecord) + .build(); + + var rowValue = getFieldValue(record, 2, Value.RowValue.class); + var nested = rowValue.getValue(); + assertThat(getStringValue(nested, 1)).isEqualTo("nested"); + assertThat(getFieldValue(nested, 2, Value.Int32Value.class).getValue()).isEqualTo(999); + } + + @Test + void shouldSupportConditionalFields() throws ImprintException { + boolean includeOptional = true; + String optionalValue = "optional"; + + var record = ImprintRecord.builder(TEST_SCHEMA) + .field(1, "required") + .fieldIf(includeOptional, 2, optionalValue) + .fieldIfNotNull(3, null) // Should not add field + .fieldIfNotNull(4, "not null") // Should add field + .build(); + + assertThat(record.getDirectory()).hasSize(3); // Only fields 1, 2, 4 + assertThat(getStringValue(record, 1)).isEqualTo("required"); + assertThat(getStringValue(record, 2)).isEqualTo("optional"); + assertThat(record.getValue(3)).isEmpty(); // Not added + assertThat(getStringValue(record, 4)).isEqualTo("not null"); + } + + @Test + void shouldSupportBulkOperations() throws ImprintException { + var fieldsMap = Map.of( + 1, "bulk1", + 2, 42, + 3, true + ); + + var record = ImprintRecord.builder(TEST_SCHEMA) + .fields(fieldsMap) + .field(4, "additional") + .build(); + + assertThat(record.getDirectory()).hasSize(4); + assertThat(getStringValue(record, 1)).isEqualTo("bulk1"); + assertThat(getFieldValue(record, 2, Value.Int32Value.class).getValue()).isEqualTo(42); + assertThat(getFieldValue(record, 3, Value.BoolValue.class).getValue()).isTrue(); + assertThat(getStringValue(record, 4)).isEqualTo("additional"); + } + + @Test + void shouldProvideBuilderUtilities() { + var builder = ImprintRecord.builder(TEST_SCHEMA) + .field(1, "test") + .field(2, 42); + + assertThat(builder.hasField(1)).isTrue(); + assertThat(builder.hasField(3)).isFalse(); + assertThat(builder.fieldCount()).isEqualTo(2); + assertThat(builder.fieldIds()).containsExactly(1, 2); + } + + @Test + void shouldSupportAlternativeSchemaConstructor() throws ImprintException { + var record = ImprintRecord.builder(1, 0x12345678) + .field(1, "test") + .build(); + + assertThat(record.getHeader().getSchemaId().getFieldspaceId()).isEqualTo(1); + assertThat(record.getHeader().getSchemaId().getSchemaHash()).isEqualTo(0x12345678); + } + + @Test + void shouldRoundTripThroughSerialization() throws ImprintException { + var original = ImprintRecord.builder(TEST_SCHEMA) + .field(1, "test string") + .field(2, 42) + .field(3, 3.14159) + .field(4, true) + .field(5, new byte[]{0x01, 0x02, 0x03}) + .build(); + + var serialized = original.serializeToBuffer(); + var deserialized = ImprintRecord.deserialize(serialized); + + assertThat(deserialized.getHeader().getSchemaId()).isEqualTo(TEST_SCHEMA); + assertThat(getStringValue(deserialized, 1)).isEqualTo("test string"); + assertThat(getFieldValue(deserialized, 2, Value.Int32Value.class).getValue()).isEqualTo(42); + assertThat(getFieldValue(deserialized, 3, Value.Float64Value.class).getValue()).isEqualTo(3.14159); + assertThat(getFieldValue(deserialized, 4, Value.BoolValue.class).getValue()).isTrue(); + assertThat(getBytesValue(deserialized, 5)).isEqualTo(new byte[]{0x01, 0x02, 0x03}); + } + + // Error cases + + @Test + void shouldRejectDuplicateFieldIds() { + assertThatThrownBy(() -> + ImprintRecord.builder(TEST_SCHEMA) + .field(1, "first") + .field(1, "duplicate") // Same field ID + ).isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Field ID 1 already exists"); + } + + @Test + void shouldRejectEmptyRecord() { + assertThatThrownBy(() -> + ImprintRecord.builder(TEST_SCHEMA).build() + ).isInstanceOf(ImprintException.class) + .hasMessageContaining("Cannot build empty record"); + } + + @Test + void shouldRejectInvalidMapKeys() { + var mapWithInvalidKey = Map.of(3.14, "value"); // Double key not supported + + assertThatThrownBy(() -> + ImprintRecord.builder(TEST_SCHEMA) + .field(1, mapWithInvalidKey) + ).isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Invalid map key type: Double"); + } + + @Test + void shouldRejectNullValueWithoutExplicitNullField() { + assertThatThrownBy(() -> + ImprintRecord.builder(TEST_SCHEMA) + .field(1, (Value) null) + ).isInstanceOf(NullPointerException.class) + .hasMessageContaining("Value cannot be null - use nullField()"); + } + + // Helper methods for cleaner test assertions + + private T getFieldValue(ImprintRecord record, int fieldId, Class valueType) throws ImprintException { + var value = record.getValue(fieldId); + assertThat(value).isPresent(); + assertThat(value.get()).isInstanceOf(valueType); + return valueType.cast(value.get()); + } + + private String getStringValue(ImprintRecord record, int fieldId) throws ImprintException { + var value = record.getValue(fieldId).get(); + if (value instanceof Value.StringValue) { + return ((Value.StringValue) value).getValue(); + } else if (value instanceof Value.StringBufferValue) { + return ((Value.StringBufferValue) value).getValue(); + } else { + throw new AssertionError("Expected string value, got: " + value.getClass()); + } + } + + private byte[] getBytesValue(ImprintRecord record, int fieldId) throws ImprintException { + var value = record.getValue(fieldId).get(); + if (value instanceof Value.BytesValue) { + return ((Value.BytesValue) value).getValue(); + } else if (value instanceof Value.BytesBufferValue) { + return ((Value.BytesBufferValue) value).getValue(); + } else { + throw new AssertionError("Expected bytes value, got: " + value.getClass()); + } + } +} \ No newline at end of file diff --git a/src/test/java/com/imprint/core/ImprintRecordTest.java b/src/test/java/com/imprint/core/ImprintRecordTest.java new file mode 100644 index 0000000..0772580 --- /dev/null +++ b/src/test/java/com/imprint/core/ImprintRecordTest.java @@ -0,0 +1,232 @@ +package com.imprint.core; + +import com.imprint.error.ImprintException; +import com.imprint.error.ErrorType; +import com.imprint.types.Value; +import com.imprint.types.MapKey; +import org.junit.jupiter.api.Test; +import java.util.*; +import static org.assertj.core.api.Assertions.*; + +class ImprintRecordTest { + + // Helper method to extract string value from either StringValue or StringBufferValue + private String getStringValue(Value value) { + if (value instanceof Value.StringValue) { + return ((Value.StringValue) value).getValue(); + } else if (value instanceof Value.StringBufferValue) { + return ((Value.StringBufferValue) value).getValue(); + } else { + throw new IllegalArgumentException("Expected string value, got: " + value.getClass()); + } + } + + @Test + void shouldCreateSimpleRecord() throws ImprintException { + var schemaId = new SchemaId(1, 0xdeadbeef); + var writer = new ImprintWriter(schemaId); + + writer.addField(1, Value.fromInt32(42)) + .addField(2, Value.fromString("hello")); + + var record = writer.build(); + + assertThat(record.getHeader().getSchemaId()).isEqualTo(schemaId); + assertThat(record.getDirectory()).hasSize(2); + + Optional field1 = record.getValue(1); + Optional field2 = record.getValue(2); + + assertThat(field1).isPresent(); + assertThat(field1.get()).isInstanceOf(Value.Int32Value.class); + assertThat(((Value.Int32Value) field1.get()).getValue()).isEqualTo(42); + + assertThat(field2).isPresent(); + assertThat(field2.get().getTypeCode()).isEqualTo(com.imprint.types.TypeCode.STRING); + String stringValue = getStringValue(field2.get()); + assertThat(stringValue).isEqualTo("hello"); + + // Non-existent field should return empty + assertThat(record.getValue(999)).isEmpty(); + } + + @Test + void shouldRoundtripThroughSerialization() throws ImprintException { + var schemaId = new SchemaId(1, 0xdeadbeef); + var writer = new ImprintWriter(schemaId); + + writer.addField(1, Value.nullValue()) + .addField(2, Value.fromBoolean(true)) + .addField(3, Value.fromInt32(42)) + .addField(4, Value.fromInt64(123456789L)) + .addField(5, Value.fromFloat32(3.14f)) + .addField(6, Value.fromFloat64(2.718281828)) + .addField(7, Value.fromBytes(new byte[]{1, 2, 3, 4})) + .addField(8, Value.fromString("test string")); + + var original = writer.build(); + + // Serialize and deserialize + var buffer = original.serializeToBuffer(); + byte[] serialized = new byte[buffer.remaining()]; + buffer.get(serialized); + var deserialized = ImprintRecord.deserialize(serialized); + + // Verify metadata + assertThat(deserialized.getHeader().getSchemaId().getFieldspaceId()).isEqualTo(1); + assertThat(deserialized.getHeader().getSchemaId().getSchemaHash()).isEqualTo(0xdeadbeef); + assertThat(deserialized.getDirectory()).hasSize(8); + + // Verify all values + assertThat(deserialized.getValue(1)).contains(Value.nullValue()); + assertThat(deserialized.getValue(2)).contains(Value.fromBoolean(true)); + assertThat(deserialized.getValue(3)).contains(Value.fromInt32(42)); + assertThat(deserialized.getValue(4)).contains(Value.fromInt64(123456789L)); + assertThat(deserialized.getValue(5)).contains(Value.fromFloat32(3.14f)); + assertThat(deserialized.getValue(6)).contains(Value.fromFloat64(2.718281828)); + assertThat(deserialized.getValue(7)).contains(Value.fromBytes(new byte[]{1, 2, 3, 4})); + assertThat(deserialized.getValue(8)).contains(Value.fromString("test string")); + + // Non-existent field + assertThat(deserialized.getValue(999)).isEmpty(); + } + + @Test + void shouldHandleArrays() throws ImprintException { + var schemaId = new SchemaId(1, 0xdeadbeef); + var writer = new ImprintWriter(schemaId); + + List intArray = Arrays.asList( + Value.fromInt32(1), + Value.fromInt32(2), + Value.fromInt32(3) + ); + + writer.addField(1, Value.fromArray(intArray)); + ImprintRecord record = writer.build(); + + // Serialize and deserialize + var buffer = record.serializeToBuffer(); + byte[] serialized = new byte[buffer.remaining()]; + buffer.get(serialized); + var deserialized = ImprintRecord.deserialize(serialized); + + Optional arrayValue = deserialized.getValue(1); + assertThat(arrayValue).isPresent(); + assertThat(arrayValue.get()).isInstanceOf(Value.ArrayValue.class); + + List deserializedArray = ((Value.ArrayValue) arrayValue.get()).getValue(); + assertThat(deserializedArray).hasSize(3); + assertThat(deserializedArray.get(0)).isEqualTo(Value.fromInt32(1)); + assertThat(deserializedArray.get(1)).isEqualTo(Value.fromInt32(2)); + assertThat(deserializedArray.get(2)).isEqualTo(Value.fromInt32(3)); + } + + @Test + void shouldHandleMaps() throws ImprintException { + var schemaId = new SchemaId(1, 0xdeadbeef); + var writer = new ImprintWriter(schemaId); + + var map = new HashMap(); + map.put(MapKey.fromString("key1"), Value.fromInt32(1)); + map.put(MapKey.fromString("key2"), Value.fromInt32(2)); + + writer.addField(1, Value.fromMap(map)); + var record = writer.build(); + + // Serialize and deserialize + var buffer = record.serializeToBuffer(); + byte[] serialized = new byte[buffer.remaining()]; + buffer.get(serialized); + var deserialized = ImprintRecord.deserialize(serialized); + + Optional mapValue = deserialized.getValue(1); + assertThat(mapValue).isPresent(); + assertThat(mapValue.get()).isInstanceOf(Value.MapValue.class); + + Map deserializedMap = ((Value.MapValue) mapValue.get()).getValue(); + assertThat(deserializedMap).hasSize(2); + assertThat(deserializedMap.get(MapKey.fromString("key1"))).isEqualTo(Value.fromInt32(1)); + assertThat(deserializedMap.get(MapKey.fromString("key2"))).isEqualTo(Value.fromInt32(2)); + } + + @Test + void shouldHandleNestedRecords() throws ImprintException { + // Create inner record + var innerSchemaId = new SchemaId(2, 0xcafebabe); + var innerWriter = new ImprintWriter(innerSchemaId); + innerWriter.addField(1, Value.fromInt32(42)) + .addField(2, Value.fromString("nested")); + var innerRecord = innerWriter.build(); + + // Create outer record containing inner record + var outerSchemaId = new SchemaId(1, 0xdeadbeef); + var outerWriter = new ImprintWriter(outerSchemaId); + outerWriter.addField(1, Value.fromRow(innerRecord)) + .addField(2, Value.fromInt64(123L)); + var outerRecord = outerWriter.build(); + + // Serialize and deserialize + var buffer = outerRecord.serializeToBuffer(); + byte[] serialized = new byte[buffer.remaining()]; + buffer.get(serialized); + var deserialized = ImprintRecord.deserialize(serialized); + + // Verify outer record metadata + assertThat(deserialized.getHeader().getSchemaId().getFieldspaceId()).isEqualTo(1); + assertThat(deserialized.getHeader().getSchemaId().getSchemaHash()).isEqualTo(0xdeadbeef); + + // Verify nested record + Optional rowValue = deserialized.getValue(1); + assertThat(rowValue).isPresent(); + assertThat(rowValue.get()).isInstanceOf(Value.RowValue.class); + + var nestedRecord = ((Value.RowValue) rowValue.get()).getValue(); + assertThat(nestedRecord.getHeader().getSchemaId().getFieldspaceId()).isEqualTo(2); + assertThat(nestedRecord.getHeader().getSchemaId().getSchemaHash()).isEqualTo(0xcafebabe); + + assertThat(nestedRecord.getValue(1)).contains(Value.fromInt32(42)); + assertThat(nestedRecord.getValue(2)).contains(Value.fromString("nested")); + + // Verify outer record field + assertThat(deserialized.getValue(2)).contains(Value.fromInt64(123L)); + } + + @Test + void shouldRejectInvalidMagic() { + byte[] invalidData = new byte[15]; + invalidData[0] = 0x00; // wrong magic + + assertThatThrownBy(() -> ImprintRecord.deserialize(invalidData)) + .isInstanceOf(ImprintException.class) + .extracting("errorType") + .isEqualTo(ErrorType.INVALID_MAGIC); + } + + @Test + void shouldRejectUnsupportedVersion() { + byte[] invalidData = new byte[15]; + invalidData[0] = (byte) 0x49; // correct magic + invalidData[1] = (byte) 0xFF; // wrong version + + assertThatThrownBy(() -> ImprintRecord.deserialize(invalidData)) + .isInstanceOf(ImprintException.class) + .extracting("errorType") + .isEqualTo(ErrorType.UNSUPPORTED_VERSION); + } + + @Test + void shouldHandleDuplicateFieldIds() throws ImprintException { + var schemaId = new SchemaId(1, 0xdeadbeef); + var writer = new ImprintWriter(schemaId); + + // Add duplicate field IDs - last one should win + writer.addField(1, Value.fromInt32(42)) + .addField(1, Value.fromInt32(43)); + + var record = writer.build(); + + assertThat(record.getDirectory()).hasSize(1); + assertThat(record.getValue(1)).contains(Value.fromInt32(43)); + } +} \ No newline at end of file diff --git a/src/test/java/com/imprint/types/MapKeyTest.java b/src/test/java/com/imprint/types/MapKeyTest.java new file mode 100644 index 0000000..08f4180 --- /dev/null +++ b/src/test/java/com/imprint/types/MapKeyTest.java @@ -0,0 +1,91 @@ +package com.imprint.types; + +import com.imprint.error.ImprintException; +import com.imprint.error.ErrorType; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.*; + +class MapKeyTest { + + @Test + void shouldCreateMapKeysFromValues() throws ImprintException { + var int32Key = MapKey.fromValue(Value.fromInt32(42)); + var int64Key = MapKey.fromValue(Value.fromInt64(123L)); + var bytesKey = MapKey.fromValue(Value.fromBytes(new byte[]{1, 2, 3})); + var stringKey = MapKey.fromValue(Value.fromString("test")); + + assertThat(int32Key).isInstanceOf(MapKey.Int32Key.class); + assertThat(((MapKey.Int32Key) int32Key).getValue()).isEqualTo(42); + + assertThat(int64Key).isInstanceOf(MapKey.Int64Key.class); + assertThat(((MapKey.Int64Key) int64Key).getValue()).isEqualTo(123L); + + assertThat(bytesKey).isInstanceOf(MapKey.BytesKey.class); + assertThat(((MapKey.BytesKey) bytesKey).getValue()).containsExactly(1, 2, 3); + + assertThat(stringKey).isInstanceOf(MapKey.StringKey.class); + assertThat(((MapKey.StringKey) stringKey).getValue()).isEqualTo("test"); + } + + @Test + void shouldConvertBackToValues() { + var int32Key = MapKey.fromInt32(42); + var stringKey = MapKey.fromString("test"); + + var int32Value = int32Key.toValue(); + var stringValue = stringKey.toValue(); + + assertThat(int32Value).isInstanceOf(Value.Int32Value.class); + assertThat(((Value.Int32Value) int32Value).getValue()).isEqualTo(42); + + assertThat(stringValue).isInstanceOf(Value.StringValue.class); + assertThat(((Value.StringValue) stringValue).getValue()).isEqualTo("test"); + } + + @Test + void shouldRejectInvalidValueTypes() { + var boolValue = Value.fromBoolean(true); + var arrayValue = Value.fromArray(java.util.Collections.emptyList()); + + assertThatThrownBy(() -> MapKey.fromValue(boolValue)) + .isInstanceOf(ImprintException.class) + .extracting("errorType") + .isEqualTo(ErrorType.TYPE_MISMATCH); + + assertThatThrownBy(() -> MapKey.fromValue(arrayValue)) + .isInstanceOf(ImprintException.class) + .extracting("errorType") + .isEqualTo(ErrorType.TYPE_MISMATCH); + } + + @Test + void shouldHandleEqualityAndHashing() { + var key1 = MapKey.fromString("test"); + var key2 = MapKey.fromString("test"); + var key3 = MapKey.fromString("different"); + + assertThat(key1).isEqualTo(key2); + assertThat(key1).isNotEqualTo(key3); + assertThat(key1.hashCode()).isEqualTo(key2.hashCode()); + } + + @Test + void shouldDefensiveCopyBytes() { + byte[] original = {1, 2, 3}; + var bytesKey = MapKey.fromBytes(original); + + // Modify original array + original[0] = 99; + + // Key should be unchanged + assertThat(((MapKey.BytesKey) bytesKey).getValue()).containsExactly(1, 2, 3); + } + + @Test + void shouldHaveCorrectTypeCodes() { + assertThat(MapKey.fromInt32(1).getTypeCode()).isEqualTo(TypeCode.INT32); + assertThat(MapKey.fromInt64(1L).getTypeCode()).isEqualTo(TypeCode.INT64); + assertThat(MapKey.fromBytes(new byte[]{1}).getTypeCode()).isEqualTo(TypeCode.BYTES); + assertThat(MapKey.fromString("test").getTypeCode()).isEqualTo(TypeCode.STRING); + } +} \ No newline at end of file diff --git a/src/test/java/com/imprint/types/TypeHandlerTest.java b/src/test/java/com/imprint/types/TypeHandlerTest.java new file mode 100644 index 0000000..9a4ae85 --- /dev/null +++ b/src/test/java/com/imprint/types/TypeHandlerTest.java @@ -0,0 +1,279 @@ +package com.imprint.types; + +import com.imprint.error.ImprintException; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +import static org.assertj.core.api.Assertions.*; + +/** + * Tests for individual TypeHandler implementations. + * Validates serialization, deserialization, and size estimation for each type. + */ +class TypeHandlerTest { + + @Test + void testNullHandler() throws ImprintException { + var handler = TypeHandler.NULL; + var value = Value.nullValue(); + + // Size estimation + assertThat(handler.estimateSize(value)).isEqualTo(0); + + // Serialization + var buffer = ByteBuffer.allocate(10); + handler.serialize(value, buffer); + assertThat(buffer.position()).isEqualTo(0); // NULL writes nothing + + // Deserialization + buffer.flip(); + var deserialized = handler.deserialize(buffer); + assertThat(deserialized).isEqualTo(value); + + // readValueBytes + buffer.clear(); + var valueBytes = handler.readValueBytes(buffer); + assertThat(valueBytes.remaining()).isEqualTo(0); + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void testBoolHandler(boolean testValue) throws ImprintException { + var handler = TypeHandler.BOOL; + var value = Value.fromBoolean(testValue); + + // Size estimation + assertThat(handler.estimateSize(value)).isEqualTo(1); + + // Round-trip test + var buffer = ByteBuffer.allocate(10); + handler.serialize(value, buffer); + assertThat(buffer.position()).isEqualTo(1); + + buffer.flip(); + var deserialized = handler.deserialize(buffer); + assertThat(deserialized).isEqualTo(value); + assertThat(((Value.BoolValue) deserialized).getValue()).isEqualTo(testValue); + } + + @ParameterizedTest + @ValueSource(ints = {0, 1, -1, Integer.MAX_VALUE, Integer.MIN_VALUE, 42, -42}) + void testInt32Handler(int testValue) throws ImprintException { + var handler = TypeHandler.INT32; + var value = Value.fromInt32(testValue); + + // Size estimation + assertThat(handler.estimateSize(value)).isEqualTo(4); + + // Round-trip test + var buffer = ByteBuffer.allocate(10).order(ByteOrder.LITTLE_ENDIAN); + handler.serialize(value, buffer); + assertThat(buffer.position()).isEqualTo(4); + + buffer.flip(); + var deserialized = handler.deserialize(buffer); + assertThat(deserialized).isEqualTo(value); + assertThat(((Value.Int32Value) deserialized).getValue()).isEqualTo(testValue); + } + + @ParameterizedTest + @ValueSource(longs = {0L, 1L, -1L, Long.MAX_VALUE, Long.MIN_VALUE, 123456789L}) + void testInt64Handler(long testValue) throws ImprintException { + var handler = TypeHandler.INT64; + var value = Value.fromInt64(testValue); + + // Size estimation + assertThat(handler.estimateSize(value)).isEqualTo(8); + + // Round-trip test + var buffer = ByteBuffer.allocate(20).order(ByteOrder.LITTLE_ENDIAN); + handler.serialize(value, buffer); + assertThat(buffer.position()).isEqualTo(8); + + buffer.flip(); + var deserialized = handler.deserialize(buffer); + assertThat(deserialized).isEqualTo(value); + assertThat(((Value.Int64Value) deserialized).getValue()).isEqualTo(testValue); + } + + @ParameterizedTest + @ValueSource(floats = {0.0f, 1.0f, -1.0f, Float.MAX_VALUE, Float.MIN_VALUE, 3.14159f, Float.NaN, Float.POSITIVE_INFINITY}) + void testFloat32Handler(float testValue) throws ImprintException { + var handler = TypeHandler.FLOAT32; + var value = Value.fromFloat32(testValue); + + // Size estimation + assertThat(handler.estimateSize(value)).isEqualTo(4); + + // Round-trip test + var buffer = ByteBuffer.allocate(10).order(ByteOrder.LITTLE_ENDIAN); + handler.serialize(value, buffer); + assertThat(buffer.position()).isEqualTo(4); + + buffer.flip(); + var deserialized = handler.deserialize(buffer); + assertThat(deserialized).isEqualTo(value); + + float deserializedValue = ((Value.Float32Value) deserialized).getValue(); + if (Float.isNaN(testValue)) { + assertThat(deserializedValue).isNaN(); + } else { + assertThat(deserializedValue).isEqualTo(testValue); + } + } + + @ParameterizedTest + @ValueSource(doubles = {0.0, 1.0, -1.0, Double.MAX_VALUE, Double.MIN_VALUE, Math.PI, Double.NaN, Double.POSITIVE_INFINITY}) + void testFloat64Handler(double testValue) throws ImprintException { + var handler = TypeHandler.FLOAT64; + var value = Value.fromFloat64(testValue); + + // Size estimation + assertThat(handler.estimateSize(value)).isEqualTo(8); + + // Round-trip test + var buffer = ByteBuffer.allocate(20).order(ByteOrder.LITTLE_ENDIAN); + handler.serialize(value, buffer); + assertThat(buffer.position()).isEqualTo(8); + + buffer.flip(); + var deserialized = handler.deserialize(buffer); + assertThat(deserialized).isEqualTo(value); + + double deserializedValue = ((Value.Float64Value) deserialized).getValue(); + if (Double.isNaN(testValue)) { + assertThat(deserializedValue).isNaN(); + } else { + assertThat(deserializedValue).isEqualTo(testValue); + } + } + + @ParameterizedTest + @ValueSource(strings = {"", "hello", "世界", "a very long string that exceeds typical buffer sizes and contains unicode: 🚀🎉", "null\0bytes"}) + void testStringHandler(String testValue) throws ImprintException { + var handler = TypeHandler.STRING; + var value = Value.fromString(testValue); + + byte[] utf8Bytes = testValue.getBytes(java.nio.charset.StandardCharsets.UTF_8); + int expectedSize = com.imprint.util.VarInt.encodedLength(utf8Bytes.length) + utf8Bytes.length; + + // Size estimation + assertThat(handler.estimateSize(value)).isEqualTo(expectedSize); + + // Round-trip test + var buffer = ByteBuffer.allocate(expectedSize + 20).order(ByteOrder.LITTLE_ENDIAN); + handler.serialize(value, buffer); + + buffer.flip(); + var deserialized = handler.deserialize(buffer); + + // Should return StringBufferValue (zero-copy implementation) + assertThat(deserialized).isInstanceOf(Value.StringBufferValue.class); + + String deserializedString; + if (deserialized instanceof Value.StringBufferValue) { + deserializedString = ((Value.StringBufferValue) deserialized).getValue(); + } else { + deserializedString = ((Value.StringValue) deserialized).getValue(); + } + + assertThat(deserializedString).isEqualTo(testValue); + } + + @Test + void testBytesHandlerWithArrayValue() throws ImprintException { + var handler = TypeHandler.BYTES; + byte[] testBytes = {0, 1, 2, (byte) 0xFF, 42, 127, -128}; + var value = Value.fromBytes(testBytes); + + int expectedSize = com.imprint.util.VarInt.encodedLength(testBytes.length) + testBytes.length; + + // Size estimation + assertThat(handler.estimateSize(value)).isEqualTo(expectedSize); + + // Round-trip test + var buffer = ByteBuffer.allocate(expectedSize + 20).order(ByteOrder.LITTLE_ENDIAN); + handler.serialize(value, buffer); + + buffer.flip(); + var deserialized = handler.deserialize(buffer); + + // Should return BytesBufferValue (zero-copy implementation) + assertThat(deserialized).isInstanceOf(Value.BytesBufferValue.class); + + byte[] deserializedBytes = ((Value.BytesBufferValue) deserialized).getValue(); + assertThat(deserializedBytes).isEqualTo(testBytes); + } + + @Test + void testBytesHandlerWithBufferValue() throws ImprintException { + var handler = TypeHandler.BYTES; + byte[] testBytes = {10, 20, 30, 40}; + var bufferValue = Value.fromBytesBuffer(ByteBuffer.wrap(testBytes).asReadOnlyBuffer()); + + int expectedSize = com.imprint.util.VarInt.encodedLength(testBytes.length) + testBytes.length; + + // Size estimation + assertThat(handler.estimateSize(bufferValue)).isEqualTo(expectedSize); + + // Round-trip test + var buffer = ByteBuffer.allocate(expectedSize + 20).order(ByteOrder.LITTLE_ENDIAN); + handler.serialize(bufferValue, buffer); + + buffer.flip(); + var deserialized = handler.deserialize(buffer); + + byte[] deserializedBytes = ((Value.BytesBufferValue) deserialized).getValue(); + assertThat(deserializedBytes).isEqualTo(testBytes); + } + + @Test + void testStringHandlerWithBufferValue() throws ImprintException { + var handler = TypeHandler.STRING; + String testString = "zero-copy string test"; + byte[] utf8Bytes = testString.getBytes(java.nio.charset.StandardCharsets.UTF_8); + var bufferValue = Value.fromStringBuffer(ByteBuffer.wrap(utf8Bytes).asReadOnlyBuffer()); + + int expectedSize = com.imprint.util.VarInt.encodedLength(utf8Bytes.length) + utf8Bytes.length; + + // Size estimation + assertThat(handler.estimateSize(bufferValue)).isEqualTo(expectedSize); + + // Round-trip test + var buffer = ByteBuffer.allocate(expectedSize + 20).order(ByteOrder.LITTLE_ENDIAN); + handler.serialize(bufferValue, buffer); + + buffer.flip(); + var deserialized = handler.deserialize(buffer); + + String deserializedString = ((Value.StringBufferValue) deserialized).getValue(); + assertThat(deserializedString).isEqualTo(testString); + } + + @Test + void testBoolHandlerInvalidValue() { + var handler = TypeHandler.BOOL; + var buffer = ByteBuffer.allocate(10); + buffer.put((byte) 2); // Invalid boolean value + buffer.flip(); + + assertThatThrownBy(() -> handler.deserialize(buffer)) + .isInstanceOf(ImprintException.class) + .hasMessageContaining("Invalid boolean value: 2"); + } + + @Test + void testHandlerBufferUnderflow() { + // Test that handlers properly detect buffer underflow + var int32Handler = TypeHandler.INT32; + var buffer = ByteBuffer.allocate(2); // Too small for int32 + + assertThatThrownBy(() -> int32Handler.deserialize(buffer)) + .isInstanceOf(ImprintException.class) + .hasMessageContaining("Not enough bytes for int32"); + } +} \ No newline at end of file diff --git a/src/test/java/com/imprint/types/ValueTest.java b/src/test/java/com/imprint/types/ValueTest.java new file mode 100644 index 0000000..9dd99c9 --- /dev/null +++ b/src/test/java/com/imprint/types/ValueTest.java @@ -0,0 +1,123 @@ +package com.imprint.types; + +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class ValueTest { + + @Test + void shouldCreateNullValue() { + Value value = Value.nullValue(); + + assertThat(value).isInstanceOf(Value.NullValue.class); + assertThat(value.getTypeCode()).isEqualTo(TypeCode.NULL); + assertThat(value.toString()).isEqualTo("null"); + } + + @Test + void shouldCreateBooleanValues() { + Value trueValue = Value.fromBoolean(true); + Value falseValue = Value.fromBoolean(false); + + assertThat(trueValue).isInstanceOf(Value.BoolValue.class); + assertThat(((Value.BoolValue) trueValue).getValue()).isTrue(); + assertThat(trueValue.getTypeCode()).isEqualTo(TypeCode.BOOL); + + assertThat(falseValue).isInstanceOf(Value.BoolValue.class); + assertThat(((Value.BoolValue) falseValue).getValue()).isFalse(); + assertThat(falseValue.getTypeCode()).isEqualTo(TypeCode.BOOL); + } + + @Test + void shouldCreateNumericValues() { + var int32 = Value.fromInt32(42); + var int64 = Value.fromInt64(123456789L); + var float32 = Value.fromFloat32(3.14f); + var float64 = Value.fromFloat64(2.718281828); + + assertThat(int32.getTypeCode()).isEqualTo(TypeCode.INT32); + assertThat(((Value.Int32Value) int32).getValue()).isEqualTo(42); + + assertThat(int64.getTypeCode()).isEqualTo(TypeCode.INT64); + assertThat(((Value.Int64Value) int64).getValue()).isEqualTo(123456789L); + + assertThat(float32.getTypeCode()).isEqualTo(TypeCode.FLOAT32); + assertThat(((Value.Float32Value) float32).getValue()).isEqualTo(3.14f); + + assertThat(float64.getTypeCode()).isEqualTo(TypeCode.FLOAT64); + assertThat(((Value.Float64Value) float64).getValue()).isEqualTo(2.718281828); + } + + @Test + void shouldCreateBytesAndStringValues() { + byte[] bytes = {1, 2, 3, 4}; + var bytesValue = Value.fromBytes(bytes); + var stringValue = Value.fromString("hello"); + + assertThat(bytesValue.getTypeCode()).isEqualTo(TypeCode.BYTES); + assertThat(((Value.BytesValue) bytesValue).getValue()).isEqualTo(bytes); + + assertThat(stringValue.getTypeCode()).isEqualTo(TypeCode.STRING); + assertThat(((Value.StringValue) stringValue).getValue()).isEqualTo("hello"); + } + + @Test + void shouldCreateArrayValues() { + List elements = Arrays.asList( + Value.fromInt32(1), + Value.fromInt32(2), + Value.fromInt32(3) + ); + Value arrayValue = Value.fromArray(elements); + + assertThat(arrayValue.getTypeCode()).isEqualTo(TypeCode.ARRAY); + assertThat(((Value.ArrayValue) arrayValue).getValue()).isEqualTo(elements); + } + + @Test + void shouldCreateMapValues() { + var map = new HashMap(); + map.put(MapKey.fromString("key1"), Value.fromInt32(1)); + map.put(MapKey.fromString("key2"), Value.fromInt32(2)); + + Value mapValue = Value.fromMap(map); + + assertThat(mapValue.getTypeCode()).isEqualTo(TypeCode.MAP); + assertThat(((Value.MapValue) mapValue).getValue()).isEqualTo(map); + } + + @Test + void shouldHandleEqualityCorrectly() { + var int1 = Value.fromInt32(42); + var int2 = Value.fromInt32(42); + var int3 = Value.fromInt32(43); + + assertThat(int1).isEqualTo(int2); + assertThat(int1).isNotEqualTo(int3); + assertThat(int1.hashCode()).isEqualTo(int2.hashCode()); + } + + @Test + void shouldDefensiveCopyArrays() { + byte[] original = {1, 2, 3}; + var bytesValue = Value.fromBytes(original); + + // Modify original array + original[0] = 99; + + // Value should be unchanged + assertThat(((Value.BytesValue) bytesValue).getValue()).containsExactly(1, 2, 3); + } + + @Test + void shouldRejectNullString() { + assertThatThrownBy(() -> Value.fromString(null)) + .isInstanceOf(NullPointerException.class); + } +} \ No newline at end of file diff --git a/src/test/java/com/imprint/util/VarIntTest.java b/src/test/java/com/imprint/util/VarIntTest.java new file mode 100644 index 0000000..677afb7 --- /dev/null +++ b/src/test/java/com/imprint/util/VarIntTest.java @@ -0,0 +1,115 @@ +package com.imprint.util; + +import com.imprint.error.ImprintException; +import com.imprint.error.ErrorType; +import org.junit.jupiter.api.Test; +import java.nio.ByteBuffer; +import static org.assertj.core.api.Assertions.*; + +class VarIntTest { + + @Test + void shouldRoundtripCommonValues() throws ImprintException { + int[] testCases = { + 0, 1, 127, 128, 16383, 16384, 2097151, 2097152, + 268435455, 268435456, -1 // -1 as unsigned is 0xFFFFFFFF + }; + + for (int value : testCases) { + ByteBuffer buffer = ByteBuffer.allocate(10); + VarInt.encode(value, buffer); + int encodedLength = buffer.position(); + + buffer.flip(); + VarInt.DecodeResult result = VarInt.decode(buffer); + + assertThat(result.getValue()).isEqualTo(value); + assertThat(result.getBytesRead()).isEqualTo(encodedLength); + } + } + + @Test + void shouldEncodeKnownValuesCorrectly() { + // Test cases with known encodings + assertEncodedBytes(0, 0x00); + assertEncodedBytes(1, 0x01); + assertEncodedBytes(127, 0x7f); + assertEncodedBytes(128, 0x80, 0x01); + assertEncodedBytes(16383, 0xff, 0x7f); + assertEncodedBytes(16384, 0x80, 0x80, 0x01); + } + + private void assertEncodedBytes(int value, int... expectedBytes) { + ByteBuffer buffer = ByteBuffer.allocate(10); + VarInt.encode(value, buffer); + buffer.flip(); + + byte[] actual = new byte[buffer.remaining()]; + buffer.get(actual); + + byte[] expected = new byte[expectedBytes.length]; + for (int i = 0; i < expectedBytes.length; i++) { + expected[i] = (byte) expectedBytes[i]; + } + + assertThat(actual).containsExactly(expected); + } + + @Test + void shouldWorkWithByteBuffer() throws ImprintException { + ByteBuffer buffer = ByteBuffer.allocate(10); + VarInt.encode(16384, buffer); + + buffer.flip(); + VarInt.DecodeResult result = VarInt.decode(buffer); + + assertThat(result.getValue()).isEqualTo(16384); + assertThat(result.getBytesRead()).isEqualTo(3); + } + + @Test + void shouldCalculateEncodedLength() { + assertThat(VarInt.encodedLength(0)).isEqualTo(1); + assertThat(VarInt.encodedLength(127)).isEqualTo(1); + assertThat(VarInt.encodedLength(128)).isEqualTo(2); + assertThat(VarInt.encodedLength(16383)).isEqualTo(2); + assertThat(VarInt.encodedLength(16384)).isEqualTo(3); + assertThat(VarInt.encodedLength(-1)).isEqualTo(5); // max u32 + } + + @Test + void shouldHandleBufferUnderflow() { + ByteBuffer buffer = ByteBuffer.allocate(1); + buffer.put((byte) 0x80); // incomplete varint + buffer.flip(); + + assertThatThrownBy(() -> VarInt.decode(buffer)) + .isInstanceOf(ImprintException.class) + .extracting("errorType") + .isEqualTo(ErrorType.BUFFER_UNDERFLOW); + } + + @Test + void shouldHandleOverlongEncoding() { + ByteBuffer buffer = ByteBuffer.allocate(10); + buffer.put(new byte[]{(byte) 0x80, (byte) 0x80, (byte) 0x80, (byte) 0x80, (byte) 0x80, 0x01}); + buffer.flip(); + + assertThatThrownBy(() -> VarInt.decode(buffer)) + .isInstanceOf(ImprintException.class) + .extracting("errorType") + .isEqualTo(ErrorType.MALFORMED_VARINT); + } + + @Test + void shouldHandleOverflow() { + ByteBuffer buffer = ByteBuffer.allocate(10); + buffer.put(new byte[]{(byte) 0x80, (byte) 0x80, (byte) 0x80, (byte) 0x80, 0x10}); + buffer.flip(); + + assertThatThrownBy(() -> VarInt.decode(buffer)) + .isInstanceOf(ImprintException.class) + .extracting("errorType") + .isEqualTo(ErrorType.MALFORMED_VARINT); + } +} \ No newline at end of file From dd4fdbc01cd78e4345a3c928d67466bae92b91f4 Mon Sep 17 00:00:00 2001 From: expanded-for-real Date: Sun, 1 Jun 2025 14:02:51 -0400 Subject: [PATCH 03/49] initial commit for imprint-java --- PROFILING.md | 106 +++++ gradle/wrapper/gradle-wrapper.properties | 7 + gradlew | 251 +++++++++++ gradlew.bat | 94 ++++ profile.sh | 98 +++++ settings.gradle | 1 + .../benchmark/ComparisonBenchmark.java | 344 +++++++++++++++ .../benchmark/FieldAccessBenchmark.java | 276 ++++++++++++ .../com/imprint/benchmark/MergeBenchmark.java | 163 +++++++ .../benchmark/SerializationBenchmark.java | 166 ++++++++ src/main/java/com/imprint/Constants.java | 10 + .../java/com/imprint/core/DirectoryEntry.java | 23 + src/main/java/com/imprint/core/Flags.java | 12 + src/main/java/com/imprint/core/Header.java | 13 + .../java/com/imprint/core/ImprintRecord.java | 365 ++++++++++++++++ .../imprint/core/ImprintRecordBuilder.java | 233 ++++++++++ .../java/com/imprint/core/ImprintWriter.java | 257 +++++++++++ src/main/java/com/imprint/core/SchemaId.java | 12 + .../java/com/imprint/error/ErrorType.java | 18 + .../com/imprint/error/ImprintException.java | 26 ++ src/main/java/com/imprint/types/MapKey.java | 163 +++++++ src/main/java/com/imprint/types/TypeCode.java | 48 +++ .../java/com/imprint/types/TypeHandler.java | 304 +++++++++++++ src/main/java/com/imprint/types/Value.java | 403 ++++++++++++++++++ src/main/java/com/imprint/util/VarInt.java | 118 +++++ .../imprint/ByteBufferIntegrationTest.java | 87 ++++ .../java/com/imprint/ComprehensiveTest.java | 208 +++++++++ .../java/com/imprint/IntegrationTest.java | 144 +++++++ .../com/imprint/benchmark/ProfilerTest.java | 226 ++++++++++ .../core/ImprintRecordBuilderTest.java | 234 ++++++++++ .../com/imprint/core/ImprintRecordTest.java | 232 ++++++++++ .../java/com/imprint/types/MapKeyTest.java | 91 ++++ .../com/imprint/types/TypeHandlerTest.java | 279 ++++++++++++ .../java/com/imprint/types/ValueTest.java | 123 ++++++ .../java/com/imprint/util/VarIntTest.java | 115 +++++ 35 files changed, 5250 insertions(+) create mode 100644 PROFILING.md create mode 100644 gradle/wrapper/gradle-wrapper.properties create mode 100644 gradlew create mode 100644 gradlew.bat create mode 100644 profile.sh create mode 100644 settings.gradle create mode 100644 src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java create mode 100644 src/jmh/java/com/imprint/benchmark/FieldAccessBenchmark.java create mode 100644 src/jmh/java/com/imprint/benchmark/MergeBenchmark.java create mode 100644 src/jmh/java/com/imprint/benchmark/SerializationBenchmark.java create mode 100644 src/main/java/com/imprint/Constants.java create mode 100644 src/main/java/com/imprint/core/DirectoryEntry.java create mode 100644 src/main/java/com/imprint/core/Flags.java create mode 100644 src/main/java/com/imprint/core/Header.java create mode 100644 src/main/java/com/imprint/core/ImprintRecord.java create mode 100644 src/main/java/com/imprint/core/ImprintRecordBuilder.java create mode 100644 src/main/java/com/imprint/core/ImprintWriter.java create mode 100644 src/main/java/com/imprint/core/SchemaId.java create mode 100644 src/main/java/com/imprint/error/ErrorType.java create mode 100644 src/main/java/com/imprint/error/ImprintException.java create mode 100644 src/main/java/com/imprint/types/MapKey.java create mode 100644 src/main/java/com/imprint/types/TypeCode.java create mode 100644 src/main/java/com/imprint/types/TypeHandler.java create mode 100644 src/main/java/com/imprint/types/Value.java create mode 100644 src/main/java/com/imprint/util/VarInt.java create mode 100644 src/test/java/com/imprint/ByteBufferIntegrationTest.java create mode 100644 src/test/java/com/imprint/ComprehensiveTest.java create mode 100644 src/test/java/com/imprint/IntegrationTest.java create mode 100644 src/test/java/com/imprint/benchmark/ProfilerTest.java create mode 100644 src/test/java/com/imprint/core/ImprintRecordBuilderTest.java create mode 100644 src/test/java/com/imprint/core/ImprintRecordTest.java create mode 100644 src/test/java/com/imprint/types/MapKeyTest.java create mode 100644 src/test/java/com/imprint/types/TypeHandlerTest.java create mode 100644 src/test/java/com/imprint/types/ValueTest.java create mode 100644 src/test/java/com/imprint/util/VarIntTest.java diff --git a/PROFILING.md b/PROFILING.md new file mode 100644 index 0000000..43f7b4e --- /dev/null +++ b/PROFILING.md @@ -0,0 +1,106 @@ +# Performance Profiling Guide + +This guide helps developers identify performance hotspots in the Imprint Java implementation. + +## Quick Start + +```bash +# Run field access profiling with async-profiler +./profile.sh profileFieldAccess asyncprofiler + +# Run memory allocation profiling with JFR +./profile.sh profileMemoryAllocation jfr +``` + +## Available Tests + +1. **`profileFieldAccess`** - Measures random field access patterns + - Focus: Binary search, TypeHandler dispatch, string decoding + - Good for: Optimizing read-heavy workloads + +2. **`profileSerialization`** - Tests record creation and serialization + - Focus: Object allocation, ByteBuffer operations, encoding + - Good for: Optimizing write-heavy workloads + +3. **`profileProjection`** - Simulates analytical field projection + - Focus: Bulk field access, string materialization + - Good for: Optimizing analytical workloads + +4. **`profileMemoryAllocation`** - Stress tests allocation patterns + - Focus: GC pressure, object lifecycle, string allocations + - Good for: Reducing memory footprint + +## Profiler Options + +### async-profiler (Recommended) +- **Setup**: Download from [async-profiler releases](https://github.com/jvm-profiling-tools/async-profiler/releases) +- **Output**: HTML flame graphs in `profiler-results/` +- **Best for**: CPU profiling, finding hot methods + +### Java Flight Recorder (JFR) +- **Setup**: Built into OpenJDK 11+ +- **Output**: `.jfr` files for Java Mission Control +- **Best for**: Memory profiling, GC analysis + +### VisualVM +- **Setup**: `jvisualvm` (usually pre-installed) +- **Output**: Real-time profiling UI +- **Best for**: Interactive profiling, heap dumps + +## Expected Hotspots + +Based on our optimizations, watch for: + +### CPU Hotspots +1. **Binary search** in `findDirectoryIndex()` - should be fast +2. **String decoding** in `StringBufferValue.getValue()` - lazy evaluation +3. **TypeHandler dispatch** - interface calls vs switch statements +4. **VarInt encoding/decoding** - variable-length integers +5. **ByteBuffer operations** - slicing and positioning + +### Memory Hotspots +1. **String allocations** during UTF-8 conversion +2. **Temporary objects** in binary search (should be eliminated) +3. **ByteBuffer slicing** (should be zero-copy) +4. **Array allocations** for BYTES values + +## Analyzing Results + +### async-profiler Flame Graphs +- **Wide bars** = high CPU usage (hotspots) +- **Deep stacks** = call overhead +- **Look for**: Red bars in `deserializeValue`, `findDirectoryIndex`, string operations + +### JFR Analysis +1. Open `.jfr` file in Java Mission Control +2. Check "Memory" tab for allocation hotspots +3. Check "Method Profiling" for CPU usage +4. Look at GC events for memory pressure + +### Memory Profiler Tips +- **Object allocation rate** should be low for zero-copy operations +- **String allocations** should be rare (lazy evaluation) +- **GC frequency** indicates allocation pressure + +## Performance Targets + +Based on our benchmarks: +- **Single field access**: < 50ns +- **Zero-copy operations**: < 30ns +- **String decoding**: Should be lazy, not in hot path +- **Binary search**: O(log n), ~10ns per comparison + +## Common Issues + +1. **High string allocation** → Enable lazy string decoding +2. **Object allocations in binary search** → Check DirectoryEntry creation +3. **ByteBuffer copying** → Ensure zero-copy slicing +4. **Switch statement overhead** → TypeHandler dispatch working? + +## Profiling Best Practices + +1. **Warm up JVM** - Run tests multiple times +2. **Use realistic data** - Match production patterns +3. **Profile different scenarios** - Read vs write heavy +4. **Check allocations** - Memory profiling reveals hidden costs +5. **Compare before/after** - Measure optimization impact \ No newline at end of file diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 0000000..37f853b --- /dev/null +++ b/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,7 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-8.13-bin.zip +networkTimeout=10000 +validateDistributionUrl=true +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/gradlew b/gradlew new file mode 100644 index 0000000..faf9300 --- /dev/null +++ b/gradlew @@ -0,0 +1,251 @@ +#!/bin/sh + +# +# Copyright © 2015-2021 the original authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# + +############################################################################## +# +# Gradle start up script for POSIX generated by Gradle. +# +# Important for running: +# +# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is +# noncompliant, but you have some other compliant shell such as ksh or +# bash, then to run this script, type that shell name before the whole +# command line, like: +# +# ksh Gradle +# +# Busybox and similar reduced shells will NOT work, because this script +# requires all of these POSIX shell features: +# * functions; +# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», +# «${var#prefix}», «${var%suffix}», and «$( cmd )»; +# * compound commands having a testable exit status, especially «case»; +# * various built-in commands including «command», «set», and «ulimit». +# +# Important for patching: +# +# (2) This script targets any POSIX shell, so it avoids extensions provided +# by Bash, Ksh, etc; in particular arrays are avoided. +# +# The "traditional" practice of packing multiple parameters into a +# space-separated string is a well documented source of bugs and security +# problems, so this is (mostly) avoided, by progressively accumulating +# options in "$@", and eventually passing that to Java. +# +# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, +# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; +# see the in-line comments for details. +# +# There are tweaks for specific operating systems such as AIX, CygWin, +# Darwin, MinGW, and NonStop. +# +# (3) This script is generated from the Groovy template +# https://github.com/gradle/gradle/blob/HEAD/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# within the Gradle project. +# +# You can find Gradle at https://github.com/gradle/gradle/. +# +############################################################################## + +# Attempt to set APP_HOME + +# Resolve links: $0 may be a link +app_path=$0 + +# Need this for daisy-chained symlinks. +while + APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path + [ -h "$app_path" ] +do + ls=$( ls -ld "$app_path" ) + link=${ls#*' -> '} + case $link in #( + /*) app_path=$link ;; #( + *) app_path=$APP_HOME$link ;; + esac +done + +# This is normally unused +# shellcheck disable=SC2034 +APP_BASE_NAME=${0##*/} +# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) +APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s\n' "$PWD" ) || exit + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD=maximum + +warn () { + echo "$*" +} >&2 + +die () { + echo + echo "$*" + echo + exit 1 +} >&2 + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "$( uname )" in #( + CYGWIN* ) cygwin=true ;; #( + Darwin* ) darwin=true ;; #( + MSYS* | MINGW* ) msys=true ;; #( + NONSTOP* ) nonstop=true ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD=$JAVA_HOME/jre/sh/java + else + JAVACMD=$JAVA_HOME/bin/java + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD=java + if ! command -v java >/dev/null 2>&1 + then + die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +fi + +# Increase the maximum file descriptors if we can. +if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then + case $MAX_FD in #( + max*) + # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + MAX_FD=$( ulimit -H -n ) || + warn "Could not query maximum file descriptor limit" + esac + case $MAX_FD in #( + '' | soft) :;; #( + *) + # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + ulimit -n "$MAX_FD" || + warn "Could not set maximum file descriptor limit to $MAX_FD" + esac +fi + +# Collect all arguments for the java command, stacking in reverse order: +# * args from the command line +# * the main class name +# * -classpath +# * -D...appname settings +# * --module-path (only if needed) +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. + +# For Cygwin or MSYS, switch paths to Windows format before running java +if "$cygwin" || "$msys" ; then + APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) + CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) + + JAVACMD=$( cygpath --unix "$JAVACMD" ) + + # Now convert the arguments - kludge to limit ourselves to /bin/sh + for arg do + if + case $arg in #( + -*) false ;; # don't mess with options #( + /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath + [ -e "$t" ] ;; #( + *) false ;; + esac + then + arg=$( cygpath --path --ignore --mixed "$arg" ) + fi + # Roll the args list around exactly as many times as the number of + # args, so each arg winds up back in the position where it started, but + # possibly modified. + # + # NB: a `for` loop captures its iteration list before it begins, so + # changing the positional parameters here affects neither the number of + # iterations, nor the values presented in `arg`. + shift # remove old arg + set -- "$@" "$arg" # push replacement arg + done +fi + + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Collect all arguments for the java command: +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, +# and any embedded shellness will be escaped. +# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be +# treated as '${Hostname}' itself on the command line. + +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -classpath "$CLASSPATH" \ + org.gradle.wrapper.GradleWrapperMain \ + "$@" + +# Stop when "xargs" is not available. +if ! command -v xargs >/dev/null 2>&1 +then + die "xargs is not available" +fi + +# Use "xargs" to parse quoted args. +# +# With -n1 it outputs one arg per line, with the quotes and backslashes removed. +# +# In Bash we could simply go: +# +# readarray ARGS < <( xargs -n1 <<<"$var" ) && +# set -- "${ARGS[@]}" "$@" +# +# but POSIX shell has neither arrays nor command substitution, so instead we +# post-process each arg (as a line of input to sed) to backslash-escape any +# character that might be a shell metacharacter, then use eval to reverse +# that process (while maintaining the separation between arguments), and wrap +# the whole thing up as a single "set" statement. +# +# This will of course break if any of these variables contains a newline or +# an unmatched quote. +# + +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | + tr '\n' ' ' + )" '"$@"' + +exec "$JAVACMD" "$@" diff --git a/gradlew.bat b/gradlew.bat new file mode 100644 index 0000000..9d21a21 --- /dev/null +++ b/gradlew.bat @@ -0,0 +1,94 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem +@rem SPDX-License-Identifier: Apache-2.0 +@rem + +@if "%DEBUG%"=="" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%"=="" set DIRNAME=. +@rem This is normally unused +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if %ERRORLEVEL% equ 0 goto execute + +echo. 1>&2 +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. 1>&2 +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 + +goto fail + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* + +:end +@rem End local scope for the variables with windows NT shell +if %ERRORLEVEL% equ 0 goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +set EXIT_CODE=%ERRORLEVEL% +if %EXIT_CODE% equ 0 set EXIT_CODE=1 +if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% +exit /b %EXIT_CODE% + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/profile.sh b/profile.sh new file mode 100644 index 0000000..1e618af --- /dev/null +++ b/profile.sh @@ -0,0 +1,98 @@ +#!/bin/bash + +# Profiling helper script for Imprint Java implementation +# +# Usage: +# ./profile.sh [test_method] [profiler] +# +# test_method: profileFieldAccess, profileSerialization, profileProjection, profileMemoryAllocation +# profiler: asyncprofiler, jfr, visualvm +# +# Examples: +# ./profile.sh profileFieldAccess asyncprofiler +# ./profile.sh profileSerialization jfr +# ./profile.sh profileMemoryAllocation + +set -e + +TEST_METHOD=${1:-profileFieldAccess} +PROFILER=${2:-asyncprofiler} + +echo "🔬 Starting profiling session for $TEST_METHOD using $PROFILER" + +# Enable the profiler test by removing @Disabled +sed -i 's/@Disabled.*/@Test/' src/test/java/com/imprint/benchmark/ProfilerTest.java + +case $PROFILER in + "asyncprofiler") + echo "📊 Using async-profiler (download from https://github.com/jvm-profiling-tools/async-profiler)" + echo " Will generate CPU profile in profiler-results/" + mkdir -p profiler-results + + # Run test in background and profile it + ./gradlew test --tests "*ProfilerTest.$TEST_METHOD" \ + -Dorg.gradle.jvmargs="-XX:+UnlockDiagnosticVMOptions -XX:+DebugNonSafepoints" & + + TEST_PID=$! + sleep 2 + + # Find the actual Java process (Gradle daemon) + JAVA_PID=$(pgrep -f "ProfilerTest.$TEST_METHOD" | head -1) + + if [ -n "$JAVA_PID" ]; then + echo " Profiling Java process $JAVA_PID" + if command -v async-profiler.jar >/dev/null 2>&1; then + java -jar async-profiler.jar -d 30 -f profiler-results/profile-$TEST_METHOD.html $JAVA_PID + else + echo " ⚠️ async-profiler.jar not found in PATH" + echo " 📥 Download from: https://github.com/jvm-profiling-tools/async-profiler/releases" + fi + fi + + wait $TEST_PID + ;; + + "jfr") + echo "📊 Using Java Flight Recorder" + mkdir -p profiler-results + + ./gradlew test --tests "*ProfilerTest.$TEST_METHOD" \ + -Dorg.gradle.jvmargs="-XX:+FlightRecorder -XX:StartFlightRecording=duration=60s,filename=profiler-results/profile-$TEST_METHOD.jfr,settings=profile" + + echo " 📂 JFR file saved to: profiler-results/profile-$TEST_METHOD.jfr" + echo " 🔍 Open with: jmc profiler-results/profile-$TEST_METHOD.jfr" + ;; + + "visualvm") + echo "📊 Using VisualVM" + echo " 1. Start VisualVM: jvisualvm" + echo " 2. Enable the ProfilerTest manually" + echo " 3. Run: ./gradlew test --tests '*ProfilerTest.$TEST_METHOD' --debug-jvm" + echo " 4. Attach VisualVM to the Gradle daemon process" + echo " 5. Start CPU/Memory profiling" + + read -p "Press Enter when VisualVM is ready..." + ./gradlew test --tests "*ProfilerTest.$TEST_METHOD" --debug-jvm + ;; + + *) + echo "❌ Unknown profiler: $PROFILER" + echo " Supported: asyncprofiler, jfr, visualvm" + exit 1 + ;; +esac + +# Restore @Disabled annotation +sed -i 's/@Test/@Disabled("Enable manually for profiling")/' src/test/java/com/imprint/benchmark/ProfilerTest.java + +echo "✅ Profiling complete!" +echo "" +echo "🔍 Key areas to examine:" +echo " • Object allocation hotspots (new, arrays, strings)" +echo " • ByteBuffer operations and slicing" +echo " • String UTF-8 encoding/decoding" +echo " • Binary search in directory lookup" +echo " • TypeHandler method dispatch" +echo " • VarInt encoding/decoding" +echo "" +echo "📊 Profile results in: profiler-results/" \ No newline at end of file diff --git a/settings.gradle b/settings.gradle new file mode 100644 index 0000000..0568c52 --- /dev/null +++ b/settings.gradle @@ -0,0 +1 @@ +rootProject.name = 'imprint-java' diff --git a/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java b/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java new file mode 100644 index 0000000..1293478 --- /dev/null +++ b/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java @@ -0,0 +1,344 @@ +package com.imprint.benchmark; + +import com.esotericsoftware.kryo.Kryo; +import com.esotericsoftware.kryo.io.Input; +import com.esotericsoftware.kryo.io.Output; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.imprint.core.ImprintRecord; +import com.imprint.core.ImprintWriter; +import com.imprint.core.SchemaId; +import com.imprint.types.MapKey; +import com.imprint.types.Value; +import org.openjdk.jmh.annotations.*; +import org.openjdk.jmh.infra.Blackhole; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.nio.ByteBuffer; +import java.util.*; +import java.util.concurrent.TimeUnit; + +/** + * Head-to-head benchmarks comparing Imprint against other serialization libraries. + * Tests the performance claims made in the documentation. + */ +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@State(Scope.Benchmark) +@Warmup(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS) +@Fork(1) +public class ComparisonBenchmark { + + // Test data + private TestRecord testData; + + // Serialized formats + private ByteBuffer imprintBytes; + private byte[] jacksonBytes; + private byte[] kryoBytes; + + // Library instances + private ObjectMapper jackson; + private Kryo kryo; + + @Setup + public void setup() throws Exception { + testData = createTestRecord(); + + // Initialize libraries + jackson = new ObjectMapper(); + kryo = new Kryo(); + kryo.register(TestRecord.class); + kryo.register(ArrayList.class); + kryo.register(HashMap.class); + + // Pre-serialize for deserialization benchmarks + imprintBytes = serializeWithImprint(testData); + jacksonBytes = serializeWithJackson(testData); + kryoBytes = serializeWithKryo(testData); + } + + // ===== SERIALIZATION BENCHMARKS ===== + + @Benchmark + public void serializeImprint(Blackhole bh) throws Exception { + ByteBuffer result = serializeWithImprint(testData); + bh.consume(result); + } + + @Benchmark + public void serializeJackson(Blackhole bh) throws Exception { + byte[] result = serializeWithJackson(testData); + bh.consume(result); + } + + @Benchmark + public void serializeKryo(Blackhole bh) throws Exception { + byte[] result = serializeWithKryo(testData); + bh.consume(result); + } + + // ===== DESERIALIZATION BENCHMARKS ===== + + @Benchmark + public void deserializeImprint(Blackhole bh) throws Exception { + ImprintRecord result = ImprintRecord.deserialize(imprintBytes.duplicate()); + bh.consume(result); + } + + @Benchmark + public void deserializeJackson(Blackhole bh) throws Exception { + TestRecord result = jackson.readValue(jacksonBytes, TestRecord.class); + bh.consume(result); + } + + @Benchmark + public void deserializeKryo(Blackhole bh) { + Input input = new Input(new ByteArrayInputStream(kryoBytes)); + TestRecord result = kryo.readObject(input, TestRecord.class); + input.close(); + bh.consume(result); + } + + // ===== FIELD ACCESS BENCHMARKS ===== + + @Benchmark + public void fieldAccessImprint(Blackhole bh) throws Exception { + ImprintRecord record = ImprintRecord.deserialize(imprintBytes.duplicate()); + + // Access multiple fields without full deserialization + var id = record.getValue(1); + var name = record.getValue(2); + var price = record.getValue(3); + var active = record.getValue(4); + var category = record.getValue(5); + + bh.consume(id); + bh.consume(name); + bh.consume(price); + bh.consume(active); + bh.consume(category); + } + + @Benchmark + public void fieldAccessJackson(Blackhole bh) throws Exception { + // Jackson requires full deserialization to access fields + TestRecord record = jackson.readValue(jacksonBytes, TestRecord.class); + + bh.consume(record.id); + bh.consume(record.name); + bh.consume(record.price); + bh.consume(record.active); + bh.consume(record.category); + } + + @Benchmark + public void fieldAccessKryo(Blackhole bh) { + // Kryo requires full deserialization to access fields + Input input = new Input(new ByteArrayInputStream(kryoBytes)); + TestRecord record = kryo.readObject(input, TestRecord.class); + input.close(); + + bh.consume(record.id); + bh.consume(record.name); + bh.consume(record.price); + bh.consume(record.active); + bh.consume(record.category); + } + + // ===== SIZE COMPARISON ===== + + @Benchmark + public void measureImprintSize(Blackhole bh) throws Exception { + ByteBuffer serialized = serializeWithImprint(testData); + bh.consume(serialized.remaining()); + } + + @Benchmark + public void measureJacksonSize(Blackhole bh) throws Exception { + byte[] serialized = serializeWithJackson(testData); + bh.consume(serialized.length); + } + + @Benchmark + public void measureKryoSize(Blackhole bh) throws Exception { + byte[] serialized = serializeWithKryo(testData); + bh.consume(serialized.length); + } + + // ===== MERGE SIMULATION BENCHMARKS ===== + + @Benchmark + public void mergeImprint(Blackhole bh) throws Exception { + // Simulate merge with Imprint (O(1) with proper API) + var record1 = serializeWithImprint(testData); + var record2 = serializeWithImprint(createTestRecord2()); + + // Current simulation - will be O(1) with actual merge API + var deserialized1 = ImprintRecord.deserialize(record1); + var deserialized2 = ImprintRecord.deserialize(record2); + var merged = simulateMerge(deserialized1, deserialized2); + + bh.consume(merged); + } + + @Benchmark + public void mergeJackson(Blackhole bh) throws Exception { + // Jackson merge requires full deserialization + merge + serialization + var record1 = jackson.readValue(jacksonBytes, TestRecord.class); + var record2 = jackson.readValue(serializeWithJackson(createTestRecord2()), TestRecord.class); + + var merged = mergeTestRecords(record1, record2); + byte[] result = jackson.writeValueAsBytes(merged); + + bh.consume(result); + } + + @Benchmark + public void mergeKryo(Blackhole bh) throws Exception { + // Kryo merge requires full deserialization + merge + serialization + Input input1 = new Input(new ByteArrayInputStream(kryoBytes)); + var record1 = kryo.readObject(input1, TestRecord.class); + input1.close(); + + Input input2 = new Input(new ByteArrayInputStream(serializeWithKryo(createTestRecord2()))); + var record2 = kryo.readObject(input2, TestRecord.class); + input2.close(); + + var merged = mergeTestRecords(record1, record2); + byte[] result = serializeWithKryo(merged); + + bh.consume(result); + } + + // ===== HELPER METHODS ===== + + private ByteBuffer serializeWithImprint(TestRecord data) throws Exception { + var writer = new ImprintWriter(new SchemaId(1, 0x12345678)); + + writer.addField(1, Value.fromInt32(data.id)); + writer.addField(2, Value.fromString(data.name)); + writer.addField(3, Value.fromFloat64(data.price)); + writer.addField(4, Value.fromBoolean(data.active)); + writer.addField(5, Value.fromString(data.category)); + + // Convert tags list + var tagValues = new ArrayList(); + for (String tag : data.tags) { + tagValues.add(Value.fromString(tag)); + } + writer.addField(6, Value.fromArray(tagValues)); + + // Convert metadata map + var metadataMap = new HashMap(); + for (var entry : data.metadata.entrySet()) { + metadataMap.put(MapKey.fromString(entry.getKey()), Value.fromString(entry.getValue())); + } + writer.addField(7, Value.fromMap(metadataMap)); + + return writer.build().serializeToBuffer(); + } + + private byte[] serializeWithJackson(TestRecord data) throws Exception { + return jackson.writeValueAsBytes(data); + } + + private byte[] serializeWithKryo(TestRecord data) { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + Output output = new Output(baos); + kryo.writeObject(output, data); + output.close(); + return baos.toByteArray(); + } + + private ImprintRecord simulateMerge(ImprintRecord first, ImprintRecord second) throws Exception { + var writer = new ImprintWriter(first.getHeader().getSchemaId()); + var usedFieldIds = new HashSet(); + + // Copy fields from first record (takes precedence) + copyFieldsToWriter(first, writer, usedFieldIds); + copyFieldsToWriter(second, writer, usedFieldIds); + + return writer.build(); + } + + private void copyFieldsToWriter(ImprintRecord record, ImprintWriter writer, Set usedFieldIds) throws Exception { + for (var entry : record.getDirectory()) { + int fieldId = entry.getId(); + if (!usedFieldIds.contains(fieldId)) { + var value = record.getValue(fieldId); + if (value.isPresent()) { + writer.addField(fieldId, value.get()); + usedFieldIds.add(fieldId); + } + } + } + } + + private TestRecord mergeTestRecords(TestRecord first, TestRecord second) { + // Simple merge logic - first record takes precedence + var merged = new TestRecord(); + merged.id = first.id; + merged.name = first.name != null ? first.name : second.name; + merged.price = first.price != 0.0 ? first.price : second.price; + merged.active = first.active; + merged.category = first.category != null ? first.category : second.category; + + merged.tags = new ArrayList<>(first.tags); + merged.tags.addAll(second.tags); + + merged.metadata = new HashMap<>(first.metadata); + merged.metadata.putAll(second.metadata); + + return merged; + } + + private TestRecord createTestRecord() { + var record = new TestRecord(); + record.id = 12345; + record.name = "Test Product"; + record.price = 99.99; + record.active = true; + record.category = "Electronics"; + + record.tags = Arrays.asList("popular", "trending", "bestseller"); + + record.metadata = new HashMap<>(); + record.metadata.put("manufacturer", "TechCorp"); + record.metadata.put("model", "TC-2024"); + record.metadata.put("warranty", "2 years"); + + return record; + } + + private TestRecord createTestRecord2() { + var record = new TestRecord(); + record.id = 67890; + record.name = "Test Product 2"; + record.price = 149.99; + record.active = false; + record.category = "Software"; + + record.tags = Arrays.asList("new", "premium"); + + record.metadata = new HashMap<>(); + record.metadata.put("vendor", "SoftCorp"); + record.metadata.put("version", "2.1"); + + return record; + } + + // Test data class for other serialization libraries + public static class TestRecord { + public int id; + public String name; + public double price; + public boolean active; + public String category; + public List tags = new ArrayList<>(); + public Map metadata = new HashMap<>(); + + public TestRecord() {} // Required for deserialization + } +} \ No newline at end of file diff --git a/src/jmh/java/com/imprint/benchmark/FieldAccessBenchmark.java b/src/jmh/java/com/imprint/benchmark/FieldAccessBenchmark.java new file mode 100644 index 0000000..f3abb7e --- /dev/null +++ b/src/jmh/java/com/imprint/benchmark/FieldAccessBenchmark.java @@ -0,0 +1,276 @@ +package com.imprint.benchmark; + +import com.imprint.core.ImprintRecord; +import com.imprint.core.ImprintWriter; +import com.imprint.core.SchemaId; +import com.imprint.types.MapKey; +import com.imprint.types.Value; +import org.openjdk.jmh.annotations.*; +import org.openjdk.jmh.infra.Blackhole; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.concurrent.TimeUnit; + +/** + * Benchmarks for ImprintRecord field access and projection operations. + * Tests the zero-copy field access performance claims. + */ +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@State(Scope.Benchmark) +@Warmup(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS) +@Fork(1) +public class FieldAccessBenchmark { + + private ImprintRecord sparseRecord; + private ImprintRecord denseRecord; + private ImprintRecord largeRecord; + + // Field IDs for testing different access patterns + private int[] firstFields; + private int[] middleFields; + private int[] lastFields; + private int[] randomFields; + private int[] allFields; + + @Setup + public void setup() throws Exception { + sparseRecord = createSparseRecord(); // Few fields, large field IDs + denseRecord = createDenseRecord(); // Many sequential fields + largeRecord = createLargeRecord(); // Large record with complex data + + // Setup field access patterns + firstFields = new int[]{1, 2, 3, 4, 5}; + middleFields = new int[]{45, 46, 47, 48, 49}; + lastFields = new int[]{95, 96, 97, 98, 99}; + randomFields = new int[]{7, 23, 41, 67, 89}; + allFields = new int[100]; + for (int i = 0; i < 100; i++) { + allFields[i] = i + 1; + } + } + + // ===== SINGLE FIELD ACCESS BENCHMARKS ===== + + @Benchmark + public void accessFirstField(Blackhole bh) throws Exception { + var value = denseRecord.getValue(1); + bh.consume(value); + } + + @Benchmark + public void accessMiddleField(Blackhole bh) throws Exception { + var value = denseRecord.getValue(50); + bh.consume(value); + } + + @Benchmark + public void accessLastField(Blackhole bh) throws Exception { + var value = denseRecord.getValue(100); + bh.consume(value); + } + + @Benchmark + public void accessNonExistentField(Blackhole bh) throws Exception { + var value = denseRecord.getValue(999); + bh.consume(value); + } + + // ===== MULTIPLE FIELD ACCESS PATTERNS ===== + + @Benchmark + public void accessFirstFields(Blackhole bh) throws Exception { + for (int fieldId : firstFields) { + var value = denseRecord.getValue(fieldId); + bh.consume(value); + } + } + + @Benchmark + public void accessMiddleFields(Blackhole bh) throws Exception { + for (int fieldId : middleFields) { + var value = denseRecord.getValue(fieldId); + bh.consume(value); + } + } + + @Benchmark + public void accessLastFields(Blackhole bh) throws Exception { + for (int fieldId : lastFields) { + var value = denseRecord.getValue(fieldId); + bh.consume(value); + } + } + + @Benchmark + public void accessRandomFields(Blackhole bh) throws Exception { + for (int fieldId : randomFields) { + var value = denseRecord.getValue(fieldId); + bh.consume(value); + } + } + + // ===== FIELD PROJECTION BENCHMARKS ===== + + @Benchmark + public void projectSmallSubset(Blackhole bh) throws Exception { + // Project 5 fields from a 100-field record + var projection = simulateProject(denseRecord, firstFields); + bh.consume(projection); + } + + @Benchmark + public void projectMediumSubset(Blackhole bh) throws Exception { + // Project 25 fields from a 100-field record + int[] fields = Arrays.copyOf(allFields, 25); + var projection = simulateProject(denseRecord, fields); + bh.consume(projection); + } + + @Benchmark + public void projectLargeSubset(Blackhole bh) throws Exception { + // Project 75 fields from a 100-field record + int[] fields = Arrays.copyOf(allFields, 75); + var projection = simulateProject(denseRecord, fields); + bh.consume(projection); + } + + @Benchmark + public void projectAllFields(Blackhole bh) throws Exception { + // Project all fields (should be nearly equivalent to full record) + var projection = simulateProject(denseRecord, allFields); + bh.consume(projection); + } + + // ===== RAW BYTES ACCESS BENCHMARKS ===== + + @Benchmark + public void getRawBytesFirstField(Blackhole bh) { + var rawBytes = denseRecord.getRawBytes(1); + bh.consume(rawBytes); + } + + @Benchmark + public void getRawBytesMiddleField(Blackhole bh) { + var rawBytes = denseRecord.getRawBytes(50); + bh.consume(rawBytes); + } + + @Benchmark + public void getRawBytesLastField(Blackhole bh) { + var rawBytes = denseRecord.getRawBytes(100); + bh.consume(rawBytes); + } + + // ===== SPARSE VS DENSE ACCESS PATTERNS ===== + + @Benchmark + public void accessSparseRecord(Blackhole bh) throws Exception { + // Access fields in sparse record (large field IDs, few fields) + var value1 = sparseRecord.getValue(1000); + var value2 = sparseRecord.getValue(5000); + var value3 = sparseRecord.getValue(10000); + bh.consume(value1); + bh.consume(value2); + bh.consume(value3); + } + + @Benchmark + public void accessDenseRecord(Blackhole bh) throws Exception { + // Access fields in dense record (sequential field IDs) + var value1 = denseRecord.getValue(1); + var value2 = denseRecord.getValue(2); + var value3 = denseRecord.getValue(3); + bh.consume(value1); + bh.consume(value2); + bh.consume(value3); + } + + // ===== HELPER METHODS ===== + + /** + * Simulates field projection by creating a new record with only specified fields. + * This should be replaced with actual project API when available. + */ + private ImprintRecord simulateProject(ImprintRecord source, int[] fieldIds) throws Exception { + var writer = new ImprintWriter(source.getHeader().getSchemaId()); + + for (int fieldId : fieldIds) { + var value = source.getValue(fieldId); + value.ifPresent(value1 -> writer.addField(fieldId, value1)); + } + + return writer.build(); + } + + private ImprintRecord createSparseRecord() throws Exception { + var writer = new ImprintWriter(new SchemaId(1, 0x12345678)); + + // Sparse record with large field IDs and few fields + writer.addField(1000, Value.fromString("sparse_field_1")); + writer.addField(5000, Value.fromInt32(42)); + writer.addField(10000, Value.fromFloat64(3.14159)); + writer.addField(15000, Value.fromBoolean(true)); + writer.addField(20000, Value.fromString("sparse_field_5")); + + return writer.build(); + } + + private ImprintRecord createDenseRecord() throws Exception { + var writer = new ImprintWriter(new SchemaId(2, 0x87654321)); + + // Dense record with 100 sequential fields + for (int i = 1; i <= 100; i++) { + switch (i % 5) { + case 0: + writer.addField(i, Value.fromString("string_field_" + i)); + break; + case 1: + writer.addField(i, Value.fromInt32(i * 10)); + break; + case 2: + writer.addField(i, Value.fromFloat64(i * 1.5)); + break; + case 3: + writer.addField(i, Value.fromBoolean(i % 2 == 0)); + break; + case 4: + writer.addField(i, Value.fromInt64(i * 1000L)); + break; + } + } + + return writer.build(); + } + + private ImprintRecord createLargeRecord() throws Exception { + var writer = new ImprintWriter(new SchemaId(3, 0x11223344)); + + // Large record with complex data types + writer.addField(1, Value.fromString("LargeRecord")); + + // Large array field + var largeArray = new ArrayList(); + for (int i = 0; i < 1000; i++) { + largeArray.add(Value.fromString("array_item_" + i)); + } + writer.addField(2, Value.fromArray(largeArray)); + + // Large map field + var largeMap = new HashMap(); + for (int i = 0; i < 100; i++) { + largeMap.put(MapKey.fromString("key_" + i), Value.fromString("map_value_" + i)); + } + writer.addField(3, Value.fromMap(largeMap)); + + // Many regular fields + for (int i = 4; i <= 50; i++) { + writer.addField(i, Value.fromString("large_record_field_" + i + "_with_substantial_content")); + } + + return writer.build(); + } +} \ No newline at end of file diff --git a/src/jmh/java/com/imprint/benchmark/MergeBenchmark.java b/src/jmh/java/com/imprint/benchmark/MergeBenchmark.java new file mode 100644 index 0000000..5c52908 --- /dev/null +++ b/src/jmh/java/com/imprint/benchmark/MergeBenchmark.java @@ -0,0 +1,163 @@ +package com.imprint.benchmark; + +import com.imprint.core.ImprintRecord; +import com.imprint.core.ImprintWriter; +import com.imprint.core.SchemaId; +import com.imprint.types.Value; +import org.openjdk.jmh.annotations.*; +import org.openjdk.jmh.infra.Blackhole; + +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +/** + * Benchmarks for ImprintRecord merge operations. + * NOTE: These benchmarks simulate merge operations until the actual merge API is implemented. + */ +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@State(Scope.Benchmark) +@Warmup(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS) +@Fork(1) +public class MergeBenchmark { + + private ImprintRecord productRecord; + private ImprintRecord orderRecord; + private ImprintRecord customerRecord; + + @Setup + public void setup() throws Exception { + productRecord = createProductRecord(); + orderRecord = createOrderRecord(); + customerRecord = createCustomerRecord(); + } + + // ===== SIMULATED MERGE BENCHMARKS ===== + // These will be replaced with actual merge API when implemented + + @Benchmark + public void mergeProductAndOrder(Blackhole bh) throws Exception { + // Simulate merge by creating a new record with fields from both + ImprintRecord result = simulateMerge(productRecord, orderRecord); + bh.consume(result); + } + + @Benchmark + public void mergeProductAndCustomer(Blackhole bh) throws Exception { + ImprintRecord result = simulateMerge(productRecord, customerRecord); + bh.consume(result); + } + + @Benchmark + public void mergeOrderAndCustomer(Blackhole bh) throws Exception { + ImprintRecord result = simulateMerge(orderRecord, customerRecord); + bh.consume(result); + } + + @Benchmark + public void mergeThreeRecords(Blackhole bh) throws Exception { + // Test merging multiple records + var temp = simulateMerge(productRecord, orderRecord); + ImprintRecord result = simulateMerge(temp, customerRecord); + bh.consume(result); + } + + // ===== MERGE CONFLICT HANDLING ===== + + @Benchmark + public void mergeWithConflicts(Blackhole bh) throws Exception { + // Create records with overlapping field IDs to test conflict resolution + var record1 = createRecordWithFields(1, 50, "record1_"); + var record2 = createRecordWithFields(25, 75, "record2_"); + + ImprintRecord result = simulateMerge(record1, record2); + bh.consume(result); + } + + // ===== HELPER METHODS ===== + + /** + * Simulates merge operation by manually copying fields. + * This should be replaced with actual merge API when available. + */ + private ImprintRecord simulateMerge(ImprintRecord first, ImprintRecord second) throws Exception { + var writer = new ImprintWriter(first.getHeader().getSchemaId()); + var usedFieldIds = new HashSet(); + + // Copy fields from first record (takes precedence) + copyFieldsToWriter(first, writer, usedFieldIds); + + // Copy non-conflicting fields from second record + copyFieldsToWriter(second, writer, usedFieldIds); + + return writer.build(); + } + + private void copyFieldsToWriter(ImprintRecord record, ImprintWriter writer, Set usedFieldIds) throws Exception { + for (var entry : record.getDirectory()) { + int fieldId = entry.getId(); + if (!usedFieldIds.contains(fieldId)) { + var value = record.getValue(fieldId); + if (value.isPresent()) { + writer.addField(fieldId, value.get()); + usedFieldIds.add(fieldId); + } + } + } + } + + private ImprintRecord createProductRecord() throws Exception { + var writer = new ImprintWriter(new SchemaId(1, 0x12345678)); + + writer.addField(1, Value.fromString("Product")); + writer.addField(2, Value.fromInt32(12345)); + writer.addField(3, Value.fromString("Laptop")); + writer.addField(4, Value.fromFloat64(999.99)); + writer.addField(5, Value.fromString("Electronics")); + writer.addField(6, Value.fromInt32(50)); // stock + writer.addField(7, Value.fromString("TechCorp")); + writer.addField(8, Value.fromBoolean(true)); // available + + return writer.build(); + } + + private ImprintRecord createOrderRecord() throws Exception { + var writer = new ImprintWriter(new SchemaId(2, 0x87654321)); + + writer.addField(10, Value.fromString("Order")); + writer.addField(11, Value.fromInt32(67890)); + writer.addField(12, Value.fromInt32(12345)); // product_id (overlaps with product) + writer.addField(13, Value.fromInt32(2)); // quantity + writer.addField(14, Value.fromFloat64(1999.98)); // total + writer.addField(15, Value.fromString("2024-01-15")); // order_date + writer.addField(16, Value.fromString("shipped")); // status + + return writer.build(); + } + + private ImprintRecord createCustomerRecord() throws Exception { + var writer = new ImprintWriter(new SchemaId(3, 0x11223344)); + + writer.addField(20, Value.fromString("Customer")); + writer.addField(21, Value.fromInt32(555)); + writer.addField(22, Value.fromString("John Doe")); + writer.addField(23, Value.fromString("john.doe@email.com")); + writer.addField(24, Value.fromString("123 Main St")); + writer.addField(25, Value.fromString("premium")); // tier + writer.addField(26, Value.fromBoolean(true)); // active + + return writer.build(); + } + + private ImprintRecord createRecordWithFields(int startId, int endId, String prefix) throws Exception { + var writer = new ImprintWriter(new SchemaId(1, 0x12345678)); + + for (int i = startId; i <= endId; i++) { + writer.addField(i, Value.fromString(prefix + "field_" + i)); + } + + return writer.build(); + } +} \ No newline at end of file diff --git a/src/jmh/java/com/imprint/benchmark/SerializationBenchmark.java b/src/jmh/java/com/imprint/benchmark/SerializationBenchmark.java new file mode 100644 index 0000000..2544b88 --- /dev/null +++ b/src/jmh/java/com/imprint/benchmark/SerializationBenchmark.java @@ -0,0 +1,166 @@ +package com.imprint.benchmark; + +import com.imprint.core.ImprintRecord; +import com.imprint.core.ImprintWriter; +import com.imprint.core.SchemaId; +import com.imprint.types.MapKey; +import com.imprint.types.Value; +import org.openjdk.jmh.annotations.*; +import org.openjdk.jmh.infra.Blackhole; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.concurrent.TimeUnit; + +/** + * Benchmarks for ImprintRecord serialization and deserialization operations. + */ +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@State(Scope.Benchmark) +@Warmup(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS) +@Fork(1) +public class SerializationBenchmark { + + private ImprintRecord smallRecord; + private ImprintRecord mediumRecord; + private ImprintRecord largeRecord; + + private ByteBuffer smallRecordBytes; + private ByteBuffer mediumRecordBytes; + private ByteBuffer largeRecordBytes; + + @Setup + public void setup() throws Exception { + // Create test records of varying sizes + smallRecord = createSmallRecord(); + mediumRecord = createMediumRecord(); + largeRecord = createLargeRecord(); + + // Pre-serialize for deserialization benchmarks + smallRecordBytes = smallRecord.serializeToBuffer(); + mediumRecordBytes = mediumRecord.serializeToBuffer(); + largeRecordBytes = largeRecord.serializeToBuffer(); + } + + // ===== SERIALIZATION BENCHMARKS ===== + + @Benchmark + public void serializeSmallRecord(Blackhole bh) { + ByteBuffer result = smallRecord.serializeToBuffer(); + bh.consume(result); + } + + @Benchmark + public void serializeMediumRecord(Blackhole bh) { + ByteBuffer result = mediumRecord.serializeToBuffer(); + bh.consume(result); + } + + @Benchmark + public void serializeLargeRecord(Blackhole bh) { + ByteBuffer result = largeRecord.serializeToBuffer(); + bh.consume(result); + } + + // ===== DESERIALIZATION BENCHMARKS ===== + + @Benchmark + public void deserializeSmallRecord(Blackhole bh) throws Exception { + ImprintRecord result = ImprintRecord.deserialize(smallRecordBytes.duplicate()); + bh.consume(result); + } + + @Benchmark + public void deserializeMediumRecord(Blackhole bh) throws Exception { + ImprintRecord result = ImprintRecord.deserialize(mediumRecordBytes.duplicate()); + bh.consume(result); + } + + @Benchmark + public void deserializeLargeRecord(Blackhole bh) throws Exception { + ImprintRecord result = ImprintRecord.deserialize(largeRecordBytes.duplicate()); + bh.consume(result); + } + + // ===== HELPER METHODS ===== + + private ImprintRecord createSmallRecord() throws Exception { + var writer = new ImprintWriter(new SchemaId(1, 0x12345678)); + + // Small record: ~10 fields, simple types + writer.addField(1, Value.fromString("Product")); + writer.addField(2, Value.fromInt32(12345)); + writer.addField(3, Value.fromFloat64(99.99)); + writer.addField(4, Value.fromBoolean(true)); + writer.addField(5, Value.fromString("Electronics")); + + return writer.build(); + } + + private ImprintRecord createMediumRecord() throws Exception { + var writer = new ImprintWriter(new SchemaId(1, 0x12345678)); + + // Medium record: ~50 fields, mixed types including arrays + writer.addField(1, Value.fromString("Product")); + writer.addField(2, Value.fromInt32(12345)); + writer.addField(3, Value.fromFloat64(99.99)); + writer.addField(4, Value.fromBoolean(true)); + writer.addField(5, Value.fromString("Electronics")); + + // Add array field + var tags = Arrays.asList( + Value.fromString("popular"), + Value.fromString("trending"), + Value.fromString("bestseller") + ); + writer.addField(6, Value.fromArray(tags)); + + // Add map field + var metadata = new HashMap(); + metadata.put(MapKey.fromString("manufacturer"), Value.fromString("TechCorp")); + metadata.put(MapKey.fromString("model"), Value.fromString("TC-2024")); + metadata.put(MapKey.fromString("year"), Value.fromInt32(2024)); + writer.addField(7, Value.fromMap(metadata)); + + // Add more fields for medium size + for (int i = 8; i <= 50; i++) { + writer.addField(i, Value.fromString("field_" + i + "_value")); + } + + return writer.build(); + } + + private ImprintRecord createLargeRecord() throws Exception { + var writer = new ImprintWriter(new SchemaId(1, 0x12345678)); + + // Large record: ~200 fields, complex nested structures + writer.addField(1, Value.fromString("LargeProduct")); + writer.addField(2, Value.fromInt32(12345)); + writer.addField(3, Value.fromFloat64(99.99)); + + // Large array + var largeArray = new ArrayList(); + for (int i = 0; i < 100; i++) { + largeArray.add(Value.fromString("item_" + i)); + } + writer.addField(4, Value.fromArray(largeArray)); + + // Large map + var largeMap = new HashMap(); + for (int i = 0; i < 50; i++) { + largeMap.put(MapKey.fromString("key_" + i), Value.fromString("value_" + i)); + } + writer.addField(5, Value.fromMap(largeMap)); + + // Many string fields + for (int i = 6; i <= 200; i++) { + writer.addField(i, Value.fromString("this_is_a_longer_field_value_for_field_" + i + "_to_increase_record_size")); + } + + return writer.build(); + } +} \ No newline at end of file diff --git a/src/main/java/com/imprint/Constants.java b/src/main/java/com/imprint/Constants.java new file mode 100644 index 0000000..78b91a0 --- /dev/null +++ b/src/main/java/com/imprint/Constants.java @@ -0,0 +1,10 @@ +package com.imprint; + +public final class Constants { + public static final byte MAGIC = 0x49; + public static final byte VERSION = 0x02; + public static final int HEADER_BYTES = 15; + public static final int DIR_ENTRY_BYTES = 7; + + private Constants() {} +} diff --git a/src/main/java/com/imprint/core/DirectoryEntry.java b/src/main/java/com/imprint/core/DirectoryEntry.java new file mode 100644 index 0000000..9556256 --- /dev/null +++ b/src/main/java/com/imprint/core/DirectoryEntry.java @@ -0,0 +1,23 @@ +package com.imprint.core; + +import com.imprint.types.TypeCode; +import lombok.Value; + +import java.util.Objects; + +/** + * A directory entry describing a single field in an Imprint record. + * Each entry has a fixed size of 7 bytes. + */ +@Value +public class DirectoryEntry { + short id; + TypeCode typeCode; + int offset; + + public DirectoryEntry(int id, TypeCode typeCode, int offset) { + this.id = (short) id; + this.typeCode = Objects.requireNonNull(typeCode, "TypeCode cannot be null"); + this.offset = offset; + } +} \ No newline at end of file diff --git a/src/main/java/com/imprint/core/Flags.java b/src/main/java/com/imprint/core/Flags.java new file mode 100644 index 0000000..ee6ffa7 --- /dev/null +++ b/src/main/java/com/imprint/core/Flags.java @@ -0,0 +1,12 @@ +package com.imprint.core; + +import lombok.Value; + +/** + * Bit flags for Imprint record header. + * Currently reserved for future use - field directory is always present. + */ +@Value +public class Flags { + byte value; +} \ No newline at end of file diff --git a/src/main/java/com/imprint/core/Header.java b/src/main/java/com/imprint/core/Header.java new file mode 100644 index 0000000..388d491 --- /dev/null +++ b/src/main/java/com/imprint/core/Header.java @@ -0,0 +1,13 @@ +package com.imprint.core; + +import lombok.Value; + +/** + * The header of an Imprint record. + */ +@Value +public class Header { + Flags flags; + SchemaId schemaId; + int payloadSize; +} \ No newline at end of file diff --git a/src/main/java/com/imprint/core/ImprintRecord.java b/src/main/java/com/imprint/core/ImprintRecord.java new file mode 100644 index 0000000..b7ed224 --- /dev/null +++ b/src/main/java/com/imprint/core/ImprintRecord.java @@ -0,0 +1,365 @@ + +package com.imprint.core; + +import com.imprint.Constants; +import com.imprint.error.ErrorType; +import com.imprint.error.ImprintException; +import com.imprint.types.MapKey; +import com.imprint.types.TypeCode; +import com.imprint.types.Value; +import com.imprint.util.VarInt; +import lombok.Getter; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.*; + +/** + * An Imprint record containing a header, optional field directory, and payload. + * Uses ByteBuffer for zero-copy operations to achieve low latency. + */ +@Getter +public final class ImprintRecord { + private final Header header; + private final List directory; + private final ByteBuffer payload; // Read-only view for zero-copy + + public ImprintRecord(Header header, List directory, ByteBuffer payload) { + this.header = Objects.requireNonNull(header, "Header cannot be null"); + this.directory = List.copyOf(Objects.requireNonNull(directory, "Directory cannot be null")); + this.payload = payload.asReadOnlyBuffer(); // Zero-copy read-only view + } + + /** + * Get a value by field ID, deserializing it on demand. + */ + public Optional getValue(int fieldId) throws ImprintException { + // Binary search for the field ID without allocation + int index = findDirectoryIndex(fieldId); + if (index < 0) return Optional.empty(); + + var entry = directory.get(index); + int startOffset = entry.getOffset(); + int endOffset = (index + 1 < directory.size()) ? + directory.get(index + 1).getOffset() : payload.remaining(); + + var valueBytes = payload.duplicate(); + valueBytes.position(startOffset).limit(endOffset); + var value = deserializeValue(entry.getTypeCode(), valueBytes.slice()); + return Optional.of(value); + } + + /** + * Get the raw bytes for a field without deserializing. + * Returns a zero-copy ByteBuffer view. + */ + public Optional getRawBytes(int fieldId) { + int index = findDirectoryIndex(fieldId); + if (index < 0) return Optional.empty(); + + var entry = directory.get(index); + int startOffset = entry.getOffset(); + int endOffset = (index + 1 < directory.size()) ? + directory.get(index + 1).getOffset() : payload.remaining(); + + var fieldBuffer = payload.duplicate(); + fieldBuffer.position(startOffset).limit(endOffset); + return Optional.of(fieldBuffer.slice().asReadOnlyBuffer()); + } + + /** + * Serialize this record to a ByteBuffer (zero-copy when possible). + */ + public ByteBuffer serializeToBuffer() { + var buffer = ByteBuffer.allocate(estimateSerializedSize()); + buffer.order(ByteOrder.LITTLE_ENDIAN); + + // Write header + serializeHeader(buffer); + + // Write directory (always present) + VarInt.encode(directory.size(), buffer); + for (var entry : directory) { + serializeDirectoryEntry(entry, buffer); + } + + // Write payload (shallow copy only) + var payloadCopy = payload.duplicate(); + buffer.put(payloadCopy); + + // Return read-only view of used portion + buffer.flip(); + return buffer.asReadOnlyBuffer(); + } + + /** + * Create a fluent builder for constructing ImprintRecord instances. + * + * @param schemaId the schema identifier for this record + * @return a new builder instance + */ + public static ImprintRecordBuilder builder(SchemaId schemaId) { + return new ImprintRecordBuilder(schemaId); + } + + /** + * Create a fluent builder for constructing ImprintRecord instances. + * + * @param fieldspaceId the fieldspace identifier + * @param schemaHash the schema hash + * @return a new builder instance + */ + public static ImprintRecordBuilder builder(int fieldspaceId, int schemaHash) { + return new ImprintRecordBuilder(new SchemaId(fieldspaceId, schemaHash)); + } + + /** + * Deserialize a record from bytes. + */ + public static ImprintRecord deserialize(byte[] bytes) throws ImprintException { + return deserialize(ByteBuffer.wrap(bytes)); + } + + /** + * Deserialize a record from a ByteBuffer (zero-copy when possible). + */ + public static ImprintRecord deserialize(ByteBuffer buffer) throws ImprintException { + buffer = buffer.duplicate().order(ByteOrder.LITTLE_ENDIAN); + + // Read header + var header = deserializeHeader(buffer); + + // Read directory (always present) + var directory = new ArrayList(); + VarInt.DecodeResult countResult = VarInt.decode(buffer); + int directoryCount = countResult.getValue(); + + for (int i = 0; i < directoryCount; i++) { + directory.add(deserializeDirectoryEntry(buffer)); + } + + // Read payload as ByteBuffer slice for zero-copy + var payload = buffer.slice(); + payload.limit(header.getPayloadSize()); + buffer.position(buffer.position() + header.getPayloadSize()); + + return new ImprintRecord(header, directory, payload); + } + + /** + * Binary search for field ID in directory without object allocation. + * Returns the index of the field if found, or a negative value if not found. + * + * @param fieldId the field ID to search for + * @return index if found, or negative insertion point - 1 if not found + */ + private int findDirectoryIndex(int fieldId) { + int low = 0; + int high = directory.size() - 1; + + while (low <= high) { + int mid = (low + high) >>> 1; // unsigned right shift to avoid overflow + int midFieldId = directory.get(mid).getId(); + + if (midFieldId < fieldId) { + low = mid + 1; + } else if (midFieldId > fieldId) { + high = mid - 1; + } else { + return mid; // field found + } + } + return -(low + 1); // field not found, return insertion point + } + + private int estimateSerializedSize() { + int size = Constants.HEADER_BYTES; // header + size += VarInt.encodedLength(directory.size()); // directory count + size += directory.size() * Constants.DIR_ENTRY_BYTES; // directory entries + size += payload.remaining(); // payload + return size; + } + + private void serializeHeader(ByteBuffer buffer) { + buffer.put(Constants.MAGIC); + buffer.put(Constants.VERSION); + buffer.put(header.getFlags().getValue()); + buffer.putInt(header.getSchemaId().getFieldspaceId()); + buffer.putInt(header.getSchemaId().getSchemaHash()); + buffer.putInt(header.getPayloadSize()); + } + + private static Header deserializeHeader(ByteBuffer buffer) throws ImprintException { + if (buffer.remaining() < Constants.HEADER_BYTES) { + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, + "Not enough bytes for header"); + } + + byte magic = buffer.get(); + if (magic != Constants.MAGIC) { + throw new ImprintException(ErrorType.INVALID_MAGIC, + "Invalid magic byte: expected 0x" + Integer.toHexString(Constants.MAGIC) + + ", got 0x" + Integer.toHexString(magic & 0xFF)); + } + + byte version = buffer.get(); + if (version != Constants.VERSION) { + throw new ImprintException(ErrorType.UNSUPPORTED_VERSION, + "Unsupported version: " + version); + } + + var flags = new Flags(buffer.get()); + int fieldspaceId = buffer.getInt(); + int schemaHash = buffer.getInt(); + int payloadSize = buffer.getInt(); + + return new Header(flags, new SchemaId(fieldspaceId, schemaHash), payloadSize); + } + + private void serializeDirectoryEntry(DirectoryEntry entry, ByteBuffer buffer) { + buffer.putShort(entry.getId()); + buffer.put(entry.getTypeCode().getCode()); + buffer.putInt(entry.getOffset()); + } + + private static DirectoryEntry deserializeDirectoryEntry(ByteBuffer buffer) throws ImprintException { + if (buffer.remaining() < Constants.DIR_ENTRY_BYTES) { + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, + "Not enough bytes for directory entry"); + } + + short id = buffer.getShort(); + var typeCode = TypeCode.fromByte(buffer.get()); + int offset = buffer.getInt(); + + return new DirectoryEntry(id, typeCode, offset); + } + + private Value deserializeValue(TypeCode typeCode, ByteBuffer buffer) throws ImprintException { + // Buffer is already positioned and limited correctly + buffer = buffer.duplicate().order(ByteOrder.LITTLE_ENDIAN); + + // Use TypeHandler for simple types + switch (typeCode) { + case NULL: + case BOOL: + case INT32: + case INT64: + case FLOAT32: + case FLOAT64: + case BYTES: + case STRING: + return typeCode.getHandler().deserialize(buffer); + + case ARRAY: + return deserializeArray(buffer); + + case MAP: + return deserializeMap(buffer); + + case ROW: + var remainingBuffer = buffer.slice(); + var nestedRecord = deserialize(remainingBuffer); + return Value.fromRow(nestedRecord); + + default: + throw new ImprintException(ErrorType.INVALID_TYPE_CODE, "Unknown type code: " + typeCode); + } + } + + private Value deserializeArray(ByteBuffer buffer) throws ImprintException { + VarInt.DecodeResult lengthResult = VarInt.decode(buffer); + int length = lengthResult.getValue(); + + if (length == 0) { + return Value.fromArray(Collections.emptyList()); + } + + var elementType = TypeCode.fromByte(buffer.get()); + var elements = new ArrayList(length); + + for (int i = 0; i < length; i++) { + var elementBytes = readValueBytes(elementType, buffer); + var element = deserializeValue(elementType, elementBytes); + elements.add(element); + } + + return Value.fromArray(elements); + } + + private Value deserializeMap(ByteBuffer buffer) throws ImprintException { + VarInt.DecodeResult lengthResult = VarInt.decode(buffer); + int length = lengthResult.getValue(); + + if (length == 0) { + return Value.fromMap(Collections.emptyMap()); + } + + var keyType = TypeCode.fromByte(buffer.get()); + var valueType = TypeCode.fromByte(buffer.get()); + var map = new HashMap(length); + + for (int i = 0; i < length; i++) { + // Read key + var keyBytes = readValueBytes(keyType, buffer); + var keyValue = deserializeValue(keyType, keyBytes); + var key = MapKey.fromValue(keyValue); + + // Read value + var valueBytes = readValueBytes(valueType, buffer); + var value = deserializeValue(valueType, valueBytes); + + map.put(key, value); + } + + return Value.fromMap(map); + } + + private ByteBuffer readValueBytes(TypeCode typeCode, ByteBuffer buffer) throws ImprintException { + // Use TypeHandler for simple types + switch (typeCode) { + case NULL: + case BOOL: + case INT32: + case INT64: + case FLOAT32: + case FLOAT64: + case BYTES: + case STRING: + return typeCode.getHandler().readValueBytes(buffer); + + case ARRAY: + case MAP: + case ROW: + // For complex types, return the entire remaining buffer for now + // The specific deserializer will handle parsing in the future + var remainingBuffer = buffer.slice(); + buffer.position(buffer.limit()); + return remainingBuffer.asReadOnlyBuffer(); + + default: + throw new ImprintException(ErrorType.INVALID_TYPE_CODE, "Unknown type code: " + typeCode); + } + } + + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (obj == null || getClass() != obj.getClass()) return false; + var that = (ImprintRecord) obj; + return header.equals(that.header) && + directory.equals(that.directory) && + payload.equals(that.payload); + } + + @Override + public int hashCode() { + return Objects.hash(header, directory, payload); + } + + @Override + public String toString() { + return String.format("ImprintRecord{header=%s, directorySize=%d, payloadSize=%d}", + header, directory.size(), payload.remaining()); + } +} \ No newline at end of file diff --git a/src/main/java/com/imprint/core/ImprintRecordBuilder.java b/src/main/java/com/imprint/core/ImprintRecordBuilder.java new file mode 100644 index 0000000..48b0998 --- /dev/null +++ b/src/main/java/com/imprint/core/ImprintRecordBuilder.java @@ -0,0 +1,233 @@ +package com.imprint.core; + +import com.imprint.error.ImprintException; +import com.imprint.types.MapKey; +import com.imprint.types.Value; + +import java.util.*; + +/** + * A fluent builder for creating ImprintRecord instances with type-safe, + * developer-friendly API that eliminates boilerplate Value.fromX() calls. + *

+ * Usage: + * var record = ImprintRecord.builder(schemaId) + * .field(1, 42) // int -> Int32Value + * .field(2, "hello") // String -> StringValue + * .field(3, 3.14) // double -> Float64Value + * .field(4, bytes) // byte[] -> BytesValue + * .field(5, true) // boolean -> BoolValue + * .nullField(6) // -> NullValue + * .build(); + */ +public final class ImprintRecordBuilder { + private final SchemaId schemaId; + private final Map fields = new TreeMap<>(); + + ImprintRecordBuilder(SchemaId schemaId) { + this.schemaId = Objects.requireNonNull(schemaId, "SchemaId cannot be null"); + } + + // Primitive types with automatic Value wrapping + public ImprintRecordBuilder field(int id, boolean value) { + return addField(id, Value.fromBoolean(value)); + } + + public ImprintRecordBuilder field(int id, int value) { + return addField(id, Value.fromInt32(value)); + } + + public ImprintRecordBuilder field(int id, long value) { + return addField(id, Value.fromInt64(value)); + } + + public ImprintRecordBuilder field(int id, float value) { + return addField(id, Value.fromFloat32(value)); + } + + public ImprintRecordBuilder field(int id, double value) { + return addField(id, Value.fromFloat64(value)); + } + + public ImprintRecordBuilder field(int id, String value) { + return addField(id, Value.fromString(value)); + } + + public ImprintRecordBuilder field(int id, byte[] value) { + return addField(id, Value.fromBytes(value)); + } + + // Collections with automatic conversion + public ImprintRecordBuilder field(int id, List values) { + var convertedValues = new ArrayList(values.size()); + for (var item : values) { + convertedValues.add(convertToValue(item)); + } + return addField(id, Value.fromArray(convertedValues)); + } + + public ImprintRecordBuilder field(int id, Map map) { + var convertedMap = new HashMap(map.size()); + for (var entry : map.entrySet()) { + var key = convertToMapKey(entry.getKey()); + var value = convertToValue(entry.getValue()); + convertedMap.put(key, value); + } + return addField(id, Value.fromMap(convertedMap)); + } + + // Nested records + public ImprintRecordBuilder field(int id, ImprintRecord nestedRecord) { + return addField(id, Value.fromRow(nestedRecord)); + } + + // Explicit null field + public ImprintRecordBuilder nullField(int id) { + return addField(id, Value.nullValue()); + } + + // Direct Value API (escape hatch for advanced usage) + public ImprintRecordBuilder field(int id, Value value) { + return addField(id, value); + } + + // Conditional field addition + public ImprintRecordBuilder fieldIf(boolean condition, int id, Object value) { + if (condition) { + return field(id, convertToValue(value)); + } + return this; + } + + public ImprintRecordBuilder fieldIfNotNull(int id, Object value) { + return fieldIf(value != null, id, value); + } + + // Bulk operations + public ImprintRecordBuilder fields(Map fieldsMap) { + for (var entry : fieldsMap.entrySet()) { + field(entry.getKey(), convertToValue(entry.getValue())); + } + return this; + } + + // Builder utilities + public boolean hasField(int id) { + return fields.containsKey(id); + } + + public int fieldCount() { + return fields.size(); + } + + public Set fieldIds() { + return new TreeSet<>(fields.keySet()); + } + + // Build the final record + public ImprintRecord build() throws ImprintException { + if (fields.isEmpty()) { + throw new ImprintException(com.imprint.error.ErrorType.SCHEMA_ERROR, + "Cannot build empty record - add at least one field"); + } + + var writer = new ImprintWriter(schemaId); + for (var entry : fields.entrySet()) { + writer.addField(entry.getKey(), entry.getValue()); + } + return writer.build(); + } + + // Internal helper methods + private ImprintRecordBuilder addField(int id, Value value) { + Objects.requireNonNull(value, "Value cannot be null - use nullField() for explicit null values"); + if (fields.containsKey(id)) { + throw new IllegalArgumentException("Field ID " + id + " already exists - field IDs must be unique"); + } + fields.put(id, value); + return this; + } + + private Value convertToValue(Object obj) { + if (obj == null) { + return Value.nullValue(); + } + + if (obj instanceof Value) { + return (Value) obj; + } + + // Auto-boxing conversion + if (obj instanceof Boolean) { + return Value.fromBoolean((Boolean) obj); + } + if (obj instanceof Integer) { + return Value.fromInt32((Integer) obj); + } + if (obj instanceof Long) { + return Value.fromInt64((Long) obj); + } + if (obj instanceof Float) { + return Value.fromFloat32((Float) obj); + } + if (obj instanceof Double) { + return Value.fromFloat64((Double) obj); + } + if (obj instanceof String) { + return Value.fromString((String) obj); + } + if (obj instanceof byte[]) { + return Value.fromBytes((byte[]) obj); + } + if (obj instanceof List) { + //test + @SuppressWarnings("unchecked") + List list = (List) obj; + var convertedValues = new ArrayList(list.size()); + for (var item : list) { + convertedValues.add(convertToValue(item)); + } + return Value.fromArray(convertedValues); + } + if (obj instanceof Map) { + @SuppressWarnings("unchecked") + Map map = (Map) obj; + var convertedMap = new HashMap(map.size()); + for (var entry : map.entrySet()) { + var key = convertToMapKey(entry.getKey()); + var value = convertToValue(entry.getValue()); + convertedMap.put(key, value); + } + return Value.fromMap(convertedMap); + } + if (obj instanceof ImprintRecord) { + return Value.fromRow((ImprintRecord) obj); + } + + throw new IllegalArgumentException("Cannot convert " + obj.getClass().getSimpleName() + + " to Imprint Value. Supported types: boolean, int, long, float, double, String, byte[], List, Map, ImprintRecord"); + } + + private MapKey convertToMapKey(Object obj) { + if (obj instanceof Integer) { + return MapKey.fromInt32((Integer) obj); + } + if (obj instanceof Long) { + return MapKey.fromInt64((Long) obj); + } + if (obj instanceof String) { + return MapKey.fromString((String) obj); + } + if (obj instanceof byte[]) { + return MapKey.fromBytes((byte[]) obj); + } + + throw new IllegalArgumentException("Invalid map key type: " + obj.getClass().getSimpleName() + + ". Map keys must be int, long, String, or byte[]"); + } + + @Override + public String toString() { + return String.format("ImprintRecordBuilder{schemaId=%s, fields=%d}", schemaId, fields.size()); + } +} \ No newline at end of file diff --git a/src/main/java/com/imprint/core/ImprintWriter.java b/src/main/java/com/imprint/core/ImprintWriter.java new file mode 100644 index 0000000..39ad9ea --- /dev/null +++ b/src/main/java/com/imprint/core/ImprintWriter.java @@ -0,0 +1,257 @@ +package com.imprint.core; + +import com.imprint.error.ErrorType; +import com.imprint.error.ImprintException; +import com.imprint.types.MapKey; +import com.imprint.types.Value; +import com.imprint.util.VarInt; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.util.*; + +/** + * A writer for constructing ImprintRecords by adding fields sequentially. + */ +public final class ImprintWriter { + private final SchemaId schemaId; + private final TreeMap fields; // keep fields in sorted order + + public ImprintWriter(SchemaId schemaId) { + this.schemaId = Objects.requireNonNull(schemaId, "SchemaId cannot be null"); + this.fields = new TreeMap<>(); + } + + /** + * Adds a field to the record being built. + */ + public ImprintWriter addField(int id, Value value) { + Objects.requireNonNull(value, "Value cannot be null"); + this.fields.put(id, value); + return this; + } + + /** + * Consumes the writer and builds an ImprintRecord. + */ + public ImprintRecord build() throws ImprintException { + var directory = new ArrayList(fields.size()); + var payloadBuffer = ByteBuffer.allocate(estimatePayloadSize()); + payloadBuffer.order(ByteOrder.LITTLE_ENDIAN); + + for (var entry : fields.entrySet()) { + int fieldId = entry.getKey(); + var value = entry.getValue(); + + directory.add(new DirectoryEntry(fieldId, value.getTypeCode(), payloadBuffer.position())); + serializeValue(value, payloadBuffer); + } + + // Create read-only view of the payload without copying + payloadBuffer.flip(); // limit = position, position = 0 + var payloadView = payloadBuffer.slice().asReadOnlyBuffer(); + + var header = new Header(new Flags((byte) 0), schemaId, payloadView.remaining()); + return new ImprintRecord(header, directory, payloadView); + } + + private int estimatePayloadSize() { + // More accurate estimation to reduce allocations + int estimatedSize = 0; + for (var value : fields.values()) { + estimatedSize += estimateValueSize(value); + } + // Add 25% buffer to reduce reallocations + return Math.max(estimatedSize + (estimatedSize / 4), fields.size() * 16); + } + + /** + * Estimates the serialized size in bytes for a given value. + * This method provides size estimates for payload buffer allocation, + * supporting both array-based and ByteBuffer-based value types. + * + * @param value the value to estimate size for + * @return estimated size in bytes including type-specific overhead + */ + private int estimateValueSize(Value value) { + // Use TypeHandler for simple types + switch (value.getTypeCode()) { + case NULL: + case BOOL: + case INT32: + case INT64: + case FLOAT32: + case FLOAT64: + case BYTES: + case STRING: + return value.getTypeCode().getHandler().estimateSize(value); + + case ARRAY: + List array = ((Value.ArrayValue) value).getValue(); + int arraySize = VarInt.encodedLength(array.size()) + 1; // length + type code + for (Value element : array) { + arraySize += estimateValueSize(element); + } + return arraySize; + + case MAP: + Map map = ((Value.MapValue) value).getValue(); + int mapSize = VarInt.encodedLength(map.size()) + 2; // length + 2 type codes + for (Map.Entry entry : map.entrySet()) { + mapSize += estimateMapKeySize(entry.getKey()); + mapSize += estimateValueSize(entry.getValue()); + } + return mapSize; + + case ROW: + // Estimate nested record size (rough approximation) + return 100; // Conservative estimate + + default: + return 32; // Default fallback + } + } + + private int estimateMapKeySize(MapKey key) { + switch (key.getTypeCode()) { + case INT32: return 4; + case INT64: return 8; + case BYTES: + byte[] bytes = ((MapKey.BytesKey) key).getValue(); + return VarInt.encodedLength(bytes.length) + bytes.length; + + case STRING: + var str = ((MapKey.StringKey) key).getValue(); + int utf8Length = str.getBytes(StandardCharsets.UTF_8).length; + return VarInt.encodedLength(utf8Length) + utf8Length; + + default: + return 16; // Default fallback + } + } + + private void serializeValue(Value value, ByteBuffer buffer) throws ImprintException { + switch (value.getTypeCode()) { + case NULL: + case BOOL: + case INT32: + case INT64: + case FLOAT32: + case FLOAT64: + case BYTES: + case STRING: + value.getTypeCode().getHandler().serialize(value, buffer); + break; + + case ARRAY: + serializeArray((Value.ArrayValue) value, buffer); + break; + + case MAP: + serializeMap((Value.MapValue) value, buffer); + break; + + case ROW: + Value.RowValue rowValue = (Value.RowValue) value; + var serializedRow = rowValue.getValue().serializeToBuffer(); + buffer.put(serializedRow); + break; + + default: + throw new ImprintException(ErrorType.SERIALIZATION_ERROR, + "Unknown type code: " + value.getTypeCode()); + } + } + + private void serializeArray(Value.ArrayValue arrayValue, ByteBuffer buffer) throws ImprintException { + var elements = arrayValue.getValue(); + VarInt.encode(elements.size(), buffer); + + if (elements.isEmpty()) return; + + // All elements must have the same type + var elementType = elements.get(0).getTypeCode(); + buffer.put(elementType.getCode()); + for (var element : elements) { + if (element.getTypeCode() != elementType) { + throw new ImprintException(ErrorType.SCHEMA_ERROR, + "Array elements must have same type code: " + + element.getTypeCode() + " != " + elementType); + } + serializeValue(element, buffer); + } + } + + private void serializeMap(Value.MapValue mapValue, ByteBuffer buffer) throws ImprintException { + var map = mapValue.getValue(); + VarInt.encode(map.size(), buffer); + + if (map.isEmpty()) { + return; + } + + // All keys and values must have consistent types + var iterator = map.entrySet().iterator(); + var first = iterator.next(); + var keyType = first.getKey().getTypeCode(); + var valueType = first.getValue().getTypeCode(); + + buffer.put(keyType.getCode()); + buffer.put(valueType.getCode()); + + // Serialize the first entry + serializeMapKey(first.getKey(), buffer); + serializeValue(first.getValue(), buffer); + + // Serialize remaining entries + while (iterator.hasNext()) { + var entry = iterator.next(); + if (entry.getKey().getTypeCode() != keyType) { + throw new ImprintException(ErrorType.SCHEMA_ERROR, + "Map keys must have same type code: " + + entry.getKey().getTypeCode() + " != " + keyType); + } + if (entry.getValue().getTypeCode() != valueType) { + throw new ImprintException(ErrorType.SCHEMA_ERROR, + "Map values must have same type code: " + + entry.getValue().getTypeCode() + " != " + valueType); + } + + serializeMapKey(entry.getKey(), buffer); + serializeValue(entry.getValue(), buffer); + } + } + + private void serializeMapKey(MapKey key, ByteBuffer buffer) throws ImprintException { + switch (key.getTypeCode()) { + case INT32: + MapKey.Int32Key int32Key = (MapKey.Int32Key) key; + buffer.putInt(int32Key.getValue()); + break; + + case INT64: + MapKey.Int64Key int64Key = (MapKey.Int64Key) key; + buffer.putLong(int64Key.getValue()); + break; + + case BYTES: + MapKey.BytesKey bytesKey = (MapKey.BytesKey) key; + byte[] bytes = bytesKey.getValue(); + VarInt.encode(bytes.length, buffer); + buffer.put(bytes); + break; + + case STRING: + MapKey.StringKey stringKey = (MapKey.StringKey) key; + byte[] stringBytes = stringKey.getValue().getBytes(StandardCharsets.UTF_8); + VarInt.encode(stringBytes.length, buffer); + buffer.put(stringBytes); + break; + + default: + throw new ImprintException(ErrorType.SERIALIZATION_ERROR, + "Invalid map key type: " + key.getTypeCode()); + } + } +} \ No newline at end of file diff --git a/src/main/java/com/imprint/core/SchemaId.java b/src/main/java/com/imprint/core/SchemaId.java new file mode 100644 index 0000000..cb03c1c --- /dev/null +++ b/src/main/java/com/imprint/core/SchemaId.java @@ -0,0 +1,12 @@ +package com.imprint.core; + +import lombok.Value; + +/** + * Schema identifier containing field-space ID and schema hash. + */ +@Value +public class SchemaId { + int fieldspaceId; + int schemaHash; +} \ No newline at end of file diff --git a/src/main/java/com/imprint/error/ErrorType.java b/src/main/java/com/imprint/error/ErrorType.java new file mode 100644 index 0000000..97b9772 --- /dev/null +++ b/src/main/java/com/imprint/error/ErrorType.java @@ -0,0 +1,18 @@ +package com.imprint.error; + +/** + * Types of errors that can occur in Imprint operations. + */ +public enum ErrorType { + INVALID_MAGIC, + UNSUPPORTED_VERSION, + BUFFER_UNDERFLOW, + FIELD_NOT_FOUND, + SCHEMA_ERROR, + INVALID_UTF8_STRING, + MALFORMED_VARINT, + TYPE_MISMATCH, + INVALID_TYPE_CODE, + SERIALIZATION_ERROR, + DESERIALIZATION_ERROR +} diff --git a/src/main/java/com/imprint/error/ImprintException.java b/src/main/java/com/imprint/error/ImprintException.java new file mode 100644 index 0000000..1b5a5d4 --- /dev/null +++ b/src/main/java/com/imprint/error/ImprintException.java @@ -0,0 +1,26 @@ +package com.imprint.error; + +import lombok.Getter; + +/** + * Exception thrown by Imprint operations. + */ +@Getter +public class ImprintException extends Exception { + private final ErrorType errorType; + + public ImprintException(ErrorType errorType, String message) { + super(message); + this.errorType = errorType; + } + + public ImprintException(ErrorType errorType, String message, Throwable cause) { + super(message, cause); + this.errorType = errorType; + } + + @Override + public String toString() { + return String.format("ImprintException{type=%s, message='%s'}", errorType, getMessage()); + } +} diff --git a/src/main/java/com/imprint/types/MapKey.java b/src/main/java/com/imprint/types/MapKey.java new file mode 100644 index 0000000..c0e0747 --- /dev/null +++ b/src/main/java/com/imprint/types/MapKey.java @@ -0,0 +1,163 @@ +package com.imprint.types; + +import com.imprint.error.ErrorType; +import com.imprint.error.ImprintException; +import lombok.EqualsAndHashCode; +import lombok.Getter; + +import java.util.Arrays; +import java.util.Objects; + +/** + * A subset of Value that's valid as a map key. + * Only Int32, Int64, Bytes, and String are valid map keys. + */ +public abstract class MapKey { + + public abstract TypeCode getTypeCode(); + public abstract boolean equals(Object obj); + public abstract int hashCode(); + public abstract String toString(); + + public static MapKey fromInt32(int value) { + return new Int32Key(value); + } + + public static MapKey fromInt64(long value) { + return new Int64Key(value); + } + + public static MapKey fromBytes(byte[] value) { + return new BytesKey(value); + } + + public static MapKey fromString(String value) { + return new StringKey(value); + } + + public static MapKey fromValue(Value value) throws ImprintException { + switch (value.getTypeCode()) { + case INT32: + return fromInt32(((Value.Int32Value) value).getValue()); + case INT64: + return fromInt64(((Value.Int64Value) value).getValue()); + case BYTES: + if (value instanceof Value.BytesBufferValue) { + return fromBytes(((Value.BytesBufferValue) value).getValue()); + } else { + return fromBytes(((Value.BytesValue) value).getValue()); + } + case STRING: + if (value instanceof Value.StringBufferValue) { + return fromString(((Value.StringBufferValue) value).getValue()); + } else { + return fromString(((Value.StringValue) value).getValue()); + } + default: + throw new ImprintException(ErrorType.TYPE_MISMATCH, + "Cannot convert " + value.getTypeCode() + " to MapKey"); + } + } + + public Value toValue() { + switch (getTypeCode()) { + case INT32: + return Value.fromInt32(((Int32Key) this).getValue()); + case INT64: + return Value.fromInt64(((Int64Key) this).getValue()); + case BYTES: + return Value.fromBytes(((BytesKey) this).getValue()); + case STRING: + return Value.fromString(((StringKey) this).getValue()); + default: + throw new IllegalStateException("Unknown MapKey type: " + getTypeCode()); + } + } + + @Getter + @EqualsAndHashCode(callSuper = false) + public static class Int32Key extends MapKey { + private final int value; + + public Int32Key(int value) { + this.value = value; + } + + @Override + public TypeCode getTypeCode() { return TypeCode.INT32; } + + @Override + public String toString() { + return String.valueOf(value); + } + } + + @Getter + @EqualsAndHashCode(callSuper = false) + public static class Int64Key extends MapKey { + private final long value; + + public Int64Key(long value) { + this.value = value; + } + + @Override + public TypeCode getTypeCode() { return TypeCode.INT64; } + + @Override + public String toString() { + return String.valueOf(value); + } + } + + public static class BytesKey extends MapKey { + private final byte[] value; + + public BytesKey(byte[] value) { + this.value = value.clone(); // defensive copy + } + + public byte[] getValue() { + return value.clone(); // defensive copy + } + + @Override + public TypeCode getTypeCode() { return TypeCode.BYTES; } + + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (obj == null || getClass() != obj.getClass()) return false; + BytesKey that = (BytesKey) obj; + return Arrays.equals(value, that.value); + } + + @Override + public int hashCode() { + return Arrays.hashCode(value); + } + + @Override + public String toString() { + return "bytes[" + value.length + "]"; + } + } + + @Getter + @EqualsAndHashCode(callSuper = false) + public static class StringKey extends MapKey { + private final String value; + + public StringKey(String value) { + this.value = Objects.requireNonNull(value, "String cannot be null"); + } + + @Override + public TypeCode getTypeCode() { return TypeCode.STRING; } + + @Override + public String toString() { + return "\"" + value + "\""; + } + } +} \ No newline at end of file diff --git a/src/main/java/com/imprint/types/TypeCode.java b/src/main/java/com/imprint/types/TypeCode.java new file mode 100644 index 0000000..6bf450d --- /dev/null +++ b/src/main/java/com/imprint/types/TypeCode.java @@ -0,0 +1,48 @@ +package com.imprint.types; + +import com.imprint.error.ImprintException; +import com.imprint.error.ErrorType; +import lombok.Getter; + +/** + * Type codes for Imprint values. + */ +public enum TypeCode { + NULL(0x0, TypeHandler.NULL), + BOOL(0x1, TypeHandler.BOOL), + INT32(0x2, TypeHandler.INT32), + INT64(0x3, TypeHandler.INT64), + FLOAT32(0x4, TypeHandler.FLOAT32), + FLOAT64(0x5, TypeHandler.FLOAT64), + BYTES(0x6, TypeHandler.BYTES), + STRING(0x7, TypeHandler.STRING), + ARRAY(0x8, null), // TODO: implement + MAP(0x9, null), // TODO: implement + ROW(0xA, null); // TODO: implement (basically a placeholder for user-defined type) + + @Getter + private final byte code; + private final TypeHandler handler; + + TypeCode(int code, TypeHandler handler) { + this.code = (byte) code; + this.handler = handler; + } + + public TypeHandler getHandler() { + if (handler == null) { + throw new UnsupportedOperationException("Handler not implemented for " + this); + } + return handler; + } + + public static TypeCode fromByte(byte code) throws ImprintException { + for (TypeCode type : values()) { + if (type.code == code) { + return type; + } + } + throw new ImprintException(ErrorType.INVALID_TYPE_CODE, + "Unknown type code: 0x" + Integer.toHexString(code & 0xFF)); + } +} \ No newline at end of file diff --git a/src/main/java/com/imprint/types/TypeHandler.java b/src/main/java/com/imprint/types/TypeHandler.java new file mode 100644 index 0000000..4b5830a --- /dev/null +++ b/src/main/java/com/imprint/types/TypeHandler.java @@ -0,0 +1,304 @@ +package com.imprint.types; + +import com.imprint.error.ImprintException; +import com.imprint.util.VarInt; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; + +/** + * Interface for handling type-specific serialization, deserialization, and size estimation. + * Note that primitives are potentially auto/un-boxed here which could impact performance slightly + * but having all the types in their own implementation helps keep things organized for now, especially + * for dealing with and testing more complex types in the future. + */ +public interface TypeHandler { + Value deserialize(ByteBuffer buffer) throws ImprintException; + void serialize(Value value, ByteBuffer buffer) throws ImprintException; + int estimateSize(Value value); + ByteBuffer readValueBytes(ByteBuffer buffer) throws ImprintException; + + // Static implementations for each type + TypeHandler NULL = new TypeHandler() { + @Override + public Value deserialize(ByteBuffer buffer) { + return Value.nullValue(); + } + + @Override + public void serialize(Value value, ByteBuffer buffer) { + // NULL values have no payload + } + + @Override + public int estimateSize(Value value) { + return 0; + } + + @Override + public ByteBuffer readValueBytes(ByteBuffer buffer) { + return ByteBuffer.allocate(0).asReadOnlyBuffer(); + } + }; + + TypeHandler BOOL = new TypeHandler() { + @Override + public Value deserialize(ByteBuffer buffer) throws ImprintException { + if (buffer.remaining() < 1) { + throw new ImprintException(com.imprint.error.ErrorType.BUFFER_UNDERFLOW, "Not enough bytes for bool"); + } + byte boolByte = buffer.get(); + if (boolByte == 0) return Value.fromBoolean(false); + if (boolByte == 1) return Value.fromBoolean(true); + throw new ImprintException(com.imprint.error.ErrorType.SCHEMA_ERROR, "Invalid boolean value: " + boolByte); + } + + @Override + public void serialize(Value value, ByteBuffer buffer) { + Value.BoolValue boolValue = (Value.BoolValue) value; + buffer.put((byte) (boolValue.getValue() ? 1 : 0)); + } + + @Override + public int estimateSize(Value value) { + return 1; + } + + @Override + public ByteBuffer readValueBytes(ByteBuffer buffer) { + var boolBuffer = buffer.slice(); + boolBuffer.limit(1); + buffer.position(buffer.position() + 1); + return boolBuffer.asReadOnlyBuffer(); + } + }; + + TypeHandler INT32 = new TypeHandler() { + @Override + public Value deserialize(ByteBuffer buffer) throws ImprintException { + if (buffer.remaining() < 4) { + throw new ImprintException(com.imprint.error.ErrorType.BUFFER_UNDERFLOW, "Not enough bytes for int32"); + } + return Value.fromInt32(buffer.getInt()); + } + + @Override + public void serialize(Value value, ByteBuffer buffer) { + Value.Int32Value int32Value = (Value.Int32Value) value; + buffer.putInt(int32Value.getValue()); + } + + @Override + public int estimateSize(Value value) { + return 4; + } + + @Override + public ByteBuffer readValueBytes(ByteBuffer buffer) { + var int32Buffer = buffer.slice(); + int32Buffer.limit(4); + buffer.position(buffer.position() + 4); + return int32Buffer.asReadOnlyBuffer(); + } + }; + + TypeHandler INT64 = new TypeHandler() { + @Override + public Value deserialize(ByteBuffer buffer) throws ImprintException { + if (buffer.remaining() < 8) { + throw new ImprintException(com.imprint.error.ErrorType.BUFFER_UNDERFLOW, "Not enough bytes for int64"); + } + return Value.fromInt64(buffer.getLong()); + } + + @Override + public void serialize(Value value, ByteBuffer buffer) { + Value.Int64Value int64Value = (Value.Int64Value) value; + buffer.putLong(int64Value.getValue()); + } + + @Override + public int estimateSize(Value value) { + return 8; + } + + @Override + public ByteBuffer readValueBytes(ByteBuffer buffer) { + var int64Buffer = buffer.slice(); + int64Buffer.limit(8); + buffer.position(buffer.position() + 8); + return int64Buffer.asReadOnlyBuffer(); + } + }; + + TypeHandler FLOAT32 = new TypeHandler() { + @Override + public Value deserialize(ByteBuffer buffer) throws ImprintException { + if (buffer.remaining() < 4) { + throw new ImprintException(com.imprint.error.ErrorType.BUFFER_UNDERFLOW, "Not enough bytes for float32"); + } + return Value.fromFloat32(buffer.getFloat()); + } + + @Override + public void serialize(Value value, ByteBuffer buffer) { + Value.Float32Value float32Value = (Value.Float32Value) value; + buffer.putFloat(float32Value.getValue()); + } + + @Override + public int estimateSize(Value value) { + return 4; + } + + @Override + public ByteBuffer readValueBytes(ByteBuffer buffer) { + var float32Buffer = buffer.slice(); + float32Buffer.limit(4); + buffer.position(buffer.position() + 4); + return float32Buffer.asReadOnlyBuffer(); + } + }; + + TypeHandler FLOAT64 = new TypeHandler() { + @Override + public Value deserialize(ByteBuffer buffer) throws ImprintException { + if (buffer.remaining() < 8) { + throw new ImprintException(com.imprint.error.ErrorType.BUFFER_UNDERFLOW, "Not enough bytes for float64"); + } + return Value.fromFloat64(buffer.getDouble()); + } + + @Override + public void serialize(Value value, ByteBuffer buffer) { + Value.Float64Value float64Value = (Value.Float64Value) value; + buffer.putDouble(float64Value.getValue()); + } + + @Override + public int estimateSize(Value value) { + return 8; + } + + @Override + public ByteBuffer readValueBytes(ByteBuffer buffer) { + var float64Buffer = buffer.slice(); + float64Buffer.limit(8); + buffer.position(buffer.position() + 8); + return float64Buffer.asReadOnlyBuffer(); + } + }; + + TypeHandler BYTES = new TypeHandler() { + @Override + public Value deserialize(ByteBuffer buffer) throws ImprintException { + VarInt.DecodeResult lengthResult = VarInt.decode(buffer); + int length = lengthResult.getValue(); + if (buffer.remaining() < length) { + throw new ImprintException(com.imprint.error.ErrorType.BUFFER_UNDERFLOW, "Not enough bytes for bytes value"); + } + var bytesView = buffer.slice(); + bytesView.limit(length); + buffer.position(buffer.position() + length); + return Value.fromBytesBuffer(bytesView.asReadOnlyBuffer()); + } + + @Override + public void serialize(Value value, ByteBuffer buffer) { + if (value instanceof Value.BytesBufferValue) { + Value.BytesBufferValue bufferValue = (Value.BytesBufferValue) value; + var bytesBuffer = bufferValue.getBuffer(); + VarInt.encode(bytesBuffer.remaining(), buffer); + buffer.put(bytesBuffer); + } else { + Value.BytesValue bytesValue = (Value.BytesValue) value; + byte[] bytes = bytesValue.getValue(); + VarInt.encode(bytes.length, buffer); + buffer.put(bytes); + } + } + + @Override + public int estimateSize(Value value) { + if (value instanceof Value.BytesBufferValue) { + Value.BytesBufferValue bufferValue = (Value.BytesBufferValue) value; + int length = bufferValue.getBuffer().remaining(); + return VarInt.encodedLength(length) + length; + } else { + byte[] bytes = ((Value.BytesValue) value).getValue(); + return VarInt.encodedLength(bytes.length) + bytes.length; + } + } + + @Override + public ByteBuffer readValueBytes(ByteBuffer buffer) throws ImprintException { + int originalPosition = buffer.position(); + VarInt.DecodeResult lengthResult = VarInt.decode(buffer); + int totalLength = lengthResult.getBytesRead() + lengthResult.getValue(); + buffer.position(originalPosition); + var valueBuffer = buffer.slice(); + valueBuffer.limit(totalLength); + buffer.position(buffer.position() + totalLength); + return valueBuffer.asReadOnlyBuffer(); + } + }; + + TypeHandler STRING = new TypeHandler() { + @Override + public Value deserialize(ByteBuffer buffer) throws ImprintException { + VarInt.DecodeResult strLengthResult = VarInt.decode(buffer); + int strLength = strLengthResult.getValue(); + if (buffer.remaining() < strLength) { + throw new ImprintException(com.imprint.error.ErrorType.BUFFER_UNDERFLOW, "Not enough bytes for string value"); + } + var stringBytesView = buffer.slice(); + stringBytesView.limit(strLength); + buffer.position(buffer.position() + strLength); + try { + return Value.fromStringBuffer(stringBytesView.asReadOnlyBuffer()); + } catch (Exception e) { + throw new ImprintException(com.imprint.error.ErrorType.INVALID_UTF8_STRING, "Invalid UTF-8 string"); + } + } + + @Override + public void serialize(Value value, ByteBuffer buffer) { + if (value instanceof Value.StringBufferValue) { + Value.StringBufferValue bufferValue = (Value.StringBufferValue) value; + var stringBuffer = bufferValue.getBuffer(); + VarInt.encode(stringBuffer.remaining(), buffer); + buffer.put(stringBuffer); + } else { + Value.StringValue stringValue = (Value.StringValue) value; + byte[] stringBytes = stringValue.getValue().getBytes(StandardCharsets.UTF_8); + VarInt.encode(stringBytes.length, buffer); + buffer.put(stringBytes); + } + } + + @Override + public int estimateSize(Value value) { + if (value instanceof Value.StringBufferValue) { + Value.StringBufferValue bufferValue = (Value.StringBufferValue) value; + int length = bufferValue.getBuffer().remaining(); + return VarInt.encodedLength(length) + length; + } else { + String str = ((Value.StringValue) value).getValue(); + int utf8Length = str.getBytes(StandardCharsets.UTF_8).length; + return VarInt.encodedLength(utf8Length) + utf8Length; + } + } + + @Override + public ByteBuffer readValueBytes(ByteBuffer buffer) throws ImprintException { + int originalPosition = buffer.position(); + VarInt.DecodeResult lengthResult = VarInt.decode(buffer); + int totalLength = lengthResult.getBytesRead() + lengthResult.getValue(); + buffer.position(originalPosition); + var valueBuffer = buffer.slice(); + valueBuffer.limit(totalLength); + buffer.position(buffer.position() + totalLength); + return valueBuffer.asReadOnlyBuffer(); + } + }; +} \ No newline at end of file diff --git a/src/main/java/com/imprint/types/Value.java b/src/main/java/com/imprint/types/Value.java new file mode 100644 index 0000000..4710ec5 --- /dev/null +++ b/src/main/java/com/imprint/types/Value.java @@ -0,0 +1,403 @@ +package com.imprint.types; + +import com.imprint.core.ImprintRecord; +import lombok.EqualsAndHashCode; +import lombok.Getter; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * A value that can be stored in an Imprint record. + */ +public abstract class Value { + + public abstract TypeCode getTypeCode(); + public abstract boolean equals(Object obj); + public abstract int hashCode(); + public abstract String toString(); + + // Factory methods + public static Value nullValue() { + return NullValue.INSTANCE; + } + + public static Value fromBoolean(boolean value) { + return new BoolValue(value); + } + + public static Value fromInt32(int value) { + return new Int32Value(value); + } + + public static Value fromInt64(long value) { + return new Int64Value(value); + } + + public static Value fromFloat32(float value) { + return new Float32Value(value); + } + + public static Value fromFloat64(double value) { + return new Float64Value(value); + } + + public static Value fromBytes(byte[] value) { + return new BytesValue(value); + } + + public static Value fromBytesBuffer(ByteBuffer value) { + return new BytesBufferValue(value); + } + + public static Value fromString(String value) { + return new StringValue(value); + } + + public static Value fromStringBuffer(ByteBuffer value) { + return new StringBufferValue(value); + } + + public static Value fromArray(List value) { + return new ArrayValue(value); + } + + public static Value fromMap(Map value) { + return new MapValue(value); + } + + public static Value fromRow(ImprintRecord value) { + return new RowValue(value); + } + + // Null Value + @EqualsAndHashCode(callSuper = false) + public static class NullValue extends Value { + public static final NullValue INSTANCE = new NullValue(); + + private NullValue() {} + + @Override + public TypeCode getTypeCode() { return TypeCode.NULL; } + + @Override + public String toString() { + return "null"; + } + } + + // Boolean Value + @Getter + @EqualsAndHashCode(callSuper = false) + public static class BoolValue extends Value { + private final boolean value; + + public BoolValue(boolean value) { + this.value = value; + } + + public boolean getValue() { return value; } + + @Override + public TypeCode getTypeCode() { return TypeCode.BOOL; } + + @Override + public String toString() { + return String.valueOf(value); + } + } + + // Int32 Value + @Getter + @EqualsAndHashCode(callSuper = false) + public static class Int32Value extends Value { + private final int value; + + public Int32Value(int value) { + this.value = value; + } + + @Override + public TypeCode getTypeCode() { return TypeCode.INT32; } + + @Override + public String toString() { + return String.valueOf(value); + } + } + + // Int64 Value + @Getter + @EqualsAndHashCode(callSuper = false) + public static class Int64Value extends Value { + private final long value; + + public Int64Value(long value) { + this.value = value; + } + + @Override + public TypeCode getTypeCode() { return TypeCode.INT64; } + + @Override + public String toString() { + return String.valueOf(value); + } + } + + // Float32 Value + @Getter + @EqualsAndHashCode(callSuper = false) + public static class Float32Value extends Value { + private final float value; + + public Float32Value(float value) { + this.value = value; + } + + @Override + public TypeCode getTypeCode() { return TypeCode.FLOAT32; } + + @Override + public String toString() { + return String.valueOf(value); + } + } + + // Float64 Value + @Getter + @EqualsAndHashCode(callSuper = false) + public static class Float64Value extends Value { + private final double value; + + public Float64Value(double value) { + this.value = value; + } + + @Override + public TypeCode getTypeCode() { return TypeCode.FLOAT64; } + + @Override + public String toString() { + return String.valueOf(value); + } + } + + // Bytes Value (array-based) + public static class BytesValue extends Value { + private final byte[] value; + + public BytesValue(byte[] value) { + this.value = value.clone(); // defensive copy + } + + public byte[] getValue() { + return value.clone(); // defensive copy + } + + @Override + public TypeCode getTypeCode() { return TypeCode.BYTES; } + + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (obj == null || getClass() != obj.getClass()) return false; + BytesValue that = (BytesValue) obj; + return Arrays.equals(value, that.value); + } + + @Override + public int hashCode() { + return Arrays.hashCode(value); + } + + @Override + public String toString() { + return "bytes[" + value.length + "]"; + } + } + + // Bytes Value (ByteBuffer-based, zero-copy) + public static class BytesBufferValue extends Value { + private final ByteBuffer value; + + public BytesBufferValue(ByteBuffer value) { + this.value = value.asReadOnlyBuffer(); // zero-copy read-only view + } + + public byte[] getValue() { + // Fallback to array when needed + byte[] array = new byte[value.remaining()]; + value.duplicate().get(array); + return array; + } + + public ByteBuffer getBuffer() { + return value.duplicate(); // zero-copy view + } + + @Override + public TypeCode getTypeCode() { return TypeCode.BYTES; } + + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (obj == null) return false; + if (obj instanceof BytesBufferValue) { + BytesBufferValue that = (BytesBufferValue) obj; + return value.equals(that.value); + } + if (obj instanceof BytesValue) { + BytesValue that = (BytesValue) obj; + return Arrays.equals(getValue(), that.getValue()); + } + return false; + } + + @Override + public int hashCode() { + return value.hashCode(); + } + + @Override + public String toString() { + return "bytes[" + value.remaining() + "]"; + } + } + + // String Value (String-based) + @Getter + @EqualsAndHashCode(callSuper = false) + public static class StringValue extends Value { + private final String value; + + public StringValue(String value) { + this.value = Objects.requireNonNull(value, "String cannot be null"); + } + + @Override + public TypeCode getTypeCode() { return TypeCode.STRING; } + + @Override + public String toString() { + return "\"" + value + "\""; + } + } + + // String Value (ByteBuffer-based, zero-copy) + public static class StringBufferValue extends Value { + private final ByteBuffer value; + private volatile String cachedString; // lazy decode + + public StringBufferValue(ByteBuffer value) { + this.value = value.asReadOnlyBuffer(); // zero-copy read-only view + } + + public String getValue() { + if (cachedString == null) { + synchronized (this) { + if (cachedString == null) { + var array = new byte[value.remaining()]; + value.duplicate().get(array); + cachedString = new String(array, StandardCharsets.UTF_8); + } + } + } + return cachedString; + } + + public ByteBuffer getBuffer() { + return value.duplicate(); // zero-copy view + } + + @Override + public TypeCode getTypeCode() { return TypeCode.STRING; } + + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (obj == null) return false; + if (obj instanceof StringBufferValue) { + StringBufferValue that = (StringBufferValue) obj; + return value.equals(that.value); + } + if (obj instanceof StringValue) { + StringValue that = (StringValue) obj; + return getValue().equals(that.getValue()); + } + return false; + } + + @Override + public int hashCode() { + return getValue().hashCode(); // Use string hash for consistency + } + + @Override + public String toString() { + return "\"" + getValue() + "\""; + } + } + + // Array Value + @Getter + @EqualsAndHashCode(callSuper = false) + public static class ArrayValue extends Value { + private final List value; + + public ArrayValue(List value) { + this.value = List.copyOf(Objects.requireNonNull(value, "Array cannot be null")); + } + + @Override + public TypeCode getTypeCode() { return TypeCode.ARRAY; } + + @Override + public String toString() { + return value.toString(); + } + } + + // Map Value + @Getter + @EqualsAndHashCode(callSuper = false) + public static class MapValue extends Value { + private final Map value; + + public MapValue(Map value) { + this.value = Map.copyOf(Objects.requireNonNull(value, "Map cannot be null")); + } + + @Override + public TypeCode getTypeCode() { return TypeCode.MAP; } + + @Override + public String toString() { + return value.toString(); + } + } + + // Row Value + @Getter + @EqualsAndHashCode(callSuper = false) + public static class RowValue extends Value { + private final ImprintRecord value; + + public RowValue(ImprintRecord value) { + this.value = Objects.requireNonNull(value, "Record cannot be null"); + } + + @Override + public TypeCode getTypeCode() { return TypeCode.ROW; } + + @Override + public String toString() { + return "Row{" + value + "}"; + } + } + +} \ No newline at end of file diff --git a/src/main/java/com/imprint/util/VarInt.java b/src/main/java/com/imprint/util/VarInt.java new file mode 100644 index 0000000..5c9a7e5 --- /dev/null +++ b/src/main/java/com/imprint/util/VarInt.java @@ -0,0 +1,118 @@ +package com.imprint.util; + +import com.imprint.error.ImprintException; +import com.imprint.error.ErrorType; +import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; +import java.nio.ByteBuffer; + +/** + * Utility class for encoding and decoding variable-length integers (VarInt). + * Supports encoding/decoding of 32-bit unsigned integers. + */ +public final class VarInt { + + private static final byte CONTINUATION_BIT = (byte) 0x80; + private static final byte SEGMENT_BITS = 0x7f; + private static final int MAX_VARINT_LEN = 5; // Enough for u32 + + private VarInt() {} // utility class + + + /** + * Encode a 32-bit unsigned integer as a VarInt into the given ByteBuffer. + * @param value the value to encode (treated as unsigned) + * @param buffer the buffer to write to + */ + public static void encode(int value, ByteBuffer buffer) { + // Convert to unsigned long for proper bit manipulation + long val = Integer.toUnsignedLong(value); + + // Encode at least one byte, then continue while value has more bits + do { + byte b = (byte) (val & SEGMENT_BITS); + val >>>= 7; + if (val != 0) { + b |= CONTINUATION_BIT; + } + buffer.put(b); + } while (val != 0); + } + + + /** + * Decode a VarInt from a ByteBuffer. + * @param buffer the buffer to decode from + * @return a DecodeResult containing the decoded value and number of bytes consumed + * @throws ImprintException if the VarInt is malformed + */ + public static DecodeResult decode(ByteBuffer buffer) throws ImprintException { + long result = 0; + int shift = 0; + int bytesRead = 0; + + while (true) { + if (bytesRead >= MAX_VARINT_LEN) { + throw new ImprintException(ErrorType.MALFORMED_VARINT, "VarInt too long"); + } + if (!buffer.hasRemaining()) { + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, + "Unexpected end of data while reading VarInt"); + } + + byte b = buffer.get(); + bytesRead++; + + // Check if adding these 7 bits would overflow + long segment = b & SEGMENT_BITS; + if (shift >= 32 || (shift == 28 && segment > 0xF)) { + throw new ImprintException(ErrorType.MALFORMED_VARINT, "VarInt overflow"); + } + + // Add the bottom 7 bits to the result + result |= segment << shift; + + // If the high bit is not set, this is the last byte + if ((b & CONTINUATION_BIT) == 0) { + break; + } + + shift += 7; + } + + return new DecodeResult((int) result, bytesRead); + } + + /** + * Calculate the number of bytes needed to encode the given value as a VarInt. + * @param value the value to encode (treated as unsigned) + * @return the number of bytes needed + */ + public static int encodedLength(int value) { + // Convert to unsigned long for proper bit manipulation + long val = Integer.toUnsignedLong(value); + int length = 1; + + // Count additional bytes needed for values >= 128 + while (val >= 0x80) { + val >>>= 7; + length++; + } + + return length; + } + + /** + * Result of a VarInt decode operation. + */ + @Getter + @AllArgsConstructor + @EqualsAndHashCode + @ToString + public static class DecodeResult { + private final int value; + private final int bytesRead; + } +} \ No newline at end of file diff --git a/src/test/java/com/imprint/ByteBufferIntegrationTest.java b/src/test/java/com/imprint/ByteBufferIntegrationTest.java new file mode 100644 index 0000000..9460cbf --- /dev/null +++ b/src/test/java/com/imprint/ByteBufferIntegrationTest.java @@ -0,0 +1,87 @@ +package com.imprint; + +import com.imprint.core.*; +import com.imprint.types.*; +import com.imprint.error.ImprintException; +import java.nio.ByteBuffer; +import java.util.*; + +/** + * Integration test to verify ByteBuffer functionality and zero-copy benefits. + */ +public class ByteBufferIntegrationTest { + + public static void main(String[] args) { + try { + testByteBufferFunctionality(); + testZeroCopy(); + System.out.println("All ByteBuffer integration tests passed!"); + } catch (Exception e) { + System.err.println("ByteBuffer integration test failed: " + e.getMessage()); + e.printStackTrace(); + System.exit(1); + } + } + + static void testByteBufferFunctionality() throws ImprintException { + System.out.println("Testing ByteBuffer functionality..."); + + SchemaId schemaId = new SchemaId(1, 0xdeadbeef); + ImprintWriter writer = new ImprintWriter(schemaId); + + writer.addField(1, Value.fromInt32(42)) + .addField(2, Value.fromString("zero-copy test")) + .addField(3, Value.fromBytes(new byte[]{1, 2, 3, 4, 5})); + + ImprintRecord record = writer.build(); + + // Test ByteBuffer serialization + ByteBuffer serializedBuffer = record.serializeToBuffer(); + assert serializedBuffer.isReadOnly() : "Serialized buffer should be read-only"; + + // Test deserialization from ByteBuffer + ImprintRecord deserialized = ImprintRecord.deserialize(serializedBuffer); + + assert deserialized.getValue(1).get().equals(Value.fromInt32(42)); + assert deserialized.getValue(2).get().equals(Value.fromString("zero-copy test")); + + // Test raw bytes access returns ByteBuffer + Optional rawBytes = deserialized.getRawBytes(1); + assert rawBytes.isPresent() : "Raw bytes should be present for field 1"; + assert rawBytes.get().isReadOnly() : "Raw bytes buffer should be read-only"; + + System.out.println("ByteBuffer functionality test passed"); + } + + static void testZeroCopy() { + System.out.println("Testing zero-copy"); + + // Create a large payload to demonstrate zero-copy benefits + byte[] largePayload = new byte[1024 * 1024]; // 1MB + Arrays.fill(largePayload, (byte) 0xAB); + + SchemaId schemaId = new SchemaId(2, 0xcafebabe); + ImprintWriter writer = new ImprintWriter(schemaId); + + try { + writer.addField(1, Value.fromBytes(largePayload)); + ImprintRecord record = writer.build(); + + // Test that getRawBytes returns a view, not a copy + Optional rawBytes = record.getRawBytes(1); + assert rawBytes.isPresent() : "Raw bytes should be present"; + + ByteBuffer rawBuffer = rawBytes.get(); + assert rawBuffer.isReadOnly() : "Raw buffer should be read-only"; + + // The buffer should be positioned at the start of the actual data + // (after the VarInt length prefix) + assert rawBuffer.remaining() > largePayload.length : "Buffer should include length prefix"; + + System.out.println("Zero-copy benefits test passed"); + + } catch (ImprintException e) { + throw new RuntimeException("Failed zero-copy test", e); + } + } +} \ No newline at end of file diff --git a/src/test/java/com/imprint/ComprehensiveTest.java b/src/test/java/com/imprint/ComprehensiveTest.java new file mode 100644 index 0000000..af7f0b0 --- /dev/null +++ b/src/test/java/com/imprint/ComprehensiveTest.java @@ -0,0 +1,208 @@ +package com.imprint; + +import com.imprint.core.*; +import com.imprint.types.*; +import com.imprint.error.ImprintException; +import com.imprint.util.VarInt; +import java.nio.ByteBuffer; +import java.util.*; + +/** + * Comprehensive test to verify all functionality works correctly. + */ +public class ComprehensiveTest { + + public static void main(String[] args) { + try { + testVarIntFunctionality(); + testValueTypes(); + testMapKeys(); + testComplexSerialization(); + testErrorHandling(); + testByteBufferPerformance(); + System.out.println("All comprehensive tests passed!"); + } catch (Exception e) { + System.err.println("Comprehensive test failed: " + e.getMessage()); + e.printStackTrace(); + System.exit(1); + } + } + + static void testVarIntFunctionality() throws ImprintException { + System.out.println("Testing VarInt functionality..."); + + // Test encoding/decoding of various values + int[] testValues = {0, 1, 127, 128, 16383, 16384, Integer.MAX_VALUE}; + + for (int value : testValues) { + ByteBuffer buffer = ByteBuffer.allocate(10); + VarInt.encode(value, buffer); + int encodedLength = buffer.position(); + + buffer.flip(); + VarInt.DecodeResult result = VarInt.decode(buffer); + + assert result.getValue() == value : "VarInt roundtrip failed for " + value; + assert result.getBytesRead() == encodedLength : "Bytes read mismatch for " + value; + } + + System.out.println("✓ VarInt functionality test passed"); + } + + static void testValueTypes() { + System.out.println("Testing Value types"); + + // Test all value types + Value nullVal = Value.nullValue(); + Value boolVal = Value.fromBoolean(true); + Value int32Val = Value.fromInt32(42); + Value int64Val = Value.fromInt64(123456789L); + Value float32Val = Value.fromFloat32(3.14f); + Value float64Val = Value.fromFloat64(2.718281828); + Value bytesVal = Value.fromBytes(new byte[]{1, 2, 3, 4}); + Value stringVal = Value.fromString("test"); + + // Test type codes + assert nullVal.getTypeCode() == TypeCode.NULL; + assert boolVal.getTypeCode() == TypeCode.BOOL; + assert int32Val.getTypeCode() == TypeCode.INT32; + assert int64Val.getTypeCode() == TypeCode.INT64; + assert float32Val.getTypeCode() == TypeCode.FLOAT32; + assert float64Val.getTypeCode() == TypeCode.FLOAT64; + assert bytesVal.getTypeCode() == TypeCode.BYTES; + assert stringVal.getTypeCode() == TypeCode.STRING; + + // Test value extraction + assert ((Value.BoolValue) boolVal).getValue(); + assert ((Value.Int32Value) int32Val).getValue() == 42; + assert ((Value.Int64Value) int64Val).getValue() == 123456789L; + assert ((Value.Float32Value) float32Val).getValue() == 3.14f; + assert ((Value.Float64Value) float64Val).getValue() == 2.718281828; + assert Arrays.equals(((Value.BytesValue) bytesVal).getValue(), new byte[]{1, 2, 3, 4}); + assert ((Value.StringValue) stringVal).getValue().equals("test"); + + System.out.println("✓ Value types test passed"); + } + + static void testMapKeys() throws ImprintException { + System.out.println("Testing MapKey functionality..."); + + MapKey int32Key = MapKey.fromInt32(42); + MapKey int64Key = MapKey.fromInt64(123L); + MapKey bytesKey = MapKey.fromBytes(new byte[]{1, 2, 3}); + MapKey stringKey = MapKey.fromString("test"); + + // Test conversion to/from Values + Value int32Value = int32Key.toValue(); + Value int64Value = int64Key.toValue(); + Value bytesValue = bytesKey.toValue(); + Value stringValue = stringKey.toValue(); + + assert MapKey.fromValue(int32Value).equals(int32Key); + assert MapKey.fromValue(int64Value).equals(int64Key); + assert MapKey.fromValue(bytesValue).equals(bytesKey); + assert MapKey.fromValue(stringValue).equals(stringKey); + + System.out.println("✓ MapKey functionality test passed"); + } + + static void testComplexSerialization() throws ImprintException { + System.out.println("Testing complex serialization..."); + + SchemaId schemaId = new SchemaId(1, 0xdeadbeef); + ImprintWriter writer = new ImprintWriter(schemaId); + + // Create complex nested structure + List array = Arrays.asList( + Value.fromInt32(1), + Value.fromInt32(2), + Value.fromInt32(3) + ); + + Map map = new HashMap<>(); + map.put(MapKey.fromString("key1"), Value.fromString("value1")); + map.put(MapKey.fromString("key2"), Value.fromString("value2")); + + writer.addField(1, Value.fromArray(array)) + .addField(2, Value.fromMap(map)) + .addField(3, Value.fromString("complex test")); + + ImprintRecord record = writer.build(); + + // Test ByteBuffer serialization + ByteBuffer serialized = record.serializeToBuffer(); + ImprintRecord deserialized = ImprintRecord.deserialize(serialized); + + // Verify complex structures + Value deserializedArray = deserialized.getValue(1).get(); + assert deserializedArray instanceof Value.ArrayValue; + List deserializedList = ((Value.ArrayValue) deserializedArray).getValue(); + assert deserializedList.size() == 3; + assert deserializedList.get(0).equals(Value.fromInt32(1)); + + Value deserializedMap = deserialized.getValue(2).get(); + assert deserializedMap instanceof Value.MapValue; + Map deserializedMapValue = ((Value.MapValue) deserializedMap).getValue(); + assert deserializedMapValue.size() == 2; + assert deserializedMapValue.get(MapKey.fromString("key1")).equals(Value.fromString("value1")); + + System.out.println("✓ Complex serialization test passed"); + } + + static void testErrorHandling() { + System.out.println("Testing error handling..."); + + try { + // Test invalid type code + TypeCode.fromByte((byte) 0xFF); + assert false : "Should have thrown exception for invalid type code"; + } catch (ImprintException e) { + assert e.getErrorType() == com.imprint.error.ErrorType.INVALID_TYPE_CODE; + } + + try { + // Test invalid magic byte + byte[] invalidData = new byte[15]; + invalidData[0] = 0x00; // wrong magic + ImprintRecord.deserialize(invalidData); + assert false : "Should have thrown exception for invalid magic"; + } catch (ImprintException e) { + assert e.getErrorType() == com.imprint.error.ErrorType.INVALID_MAGIC; + } + + System.out.println("✓ Error handling test passed"); + } + + static void testByteBufferPerformance() throws ImprintException { + System.out.println("Testing ByteBuffer performance benefits..."); + + // Create a record with moderate-sized data + byte[] testData = new byte[1024]; + for (int i = 0; i < testData.length; i++) { + testData[i] = (byte) (i % 256); + } + + SchemaId schemaId = new SchemaId(1, 0x12345678); + ImprintWriter writer = new ImprintWriter(schemaId); + writer.addField(1, Value.fromBytes(testData)) + .addField(2, Value.fromString("performance test")); + + ImprintRecord record = writer.build(); + + // Test that raw bytes access is zero-copy + Optional rawBytes = record.getRawBytes(1); + assert rawBytes.isPresent(); + assert rawBytes.get().isReadOnly(); + + // Test ByteBuffer serialization + ByteBuffer serialized = record.serializeToBuffer(); + assert serialized.isReadOnly(); + + // Verify deserialization works + ImprintRecord deserialized = ImprintRecord.deserialize(serialized); + Value retrievedBytes = deserialized.getValue(1).get(); + assert Arrays.equals(((Value.BytesValue) retrievedBytes).getValue(), testData); + + System.out.println("✓ ByteBuffer performance test passed"); + } +} \ No newline at end of file diff --git a/src/test/java/com/imprint/IntegrationTest.java b/src/test/java/com/imprint/IntegrationTest.java new file mode 100644 index 0000000..49cfce7 --- /dev/null +++ b/src/test/java/com/imprint/IntegrationTest.java @@ -0,0 +1,144 @@ +package com.imprint; + +import com.imprint.core.*; +import com.imprint.types.*; +import com.imprint.error.ImprintException; +import java.util.*; + +/** + * Integration test to verify the complete Java implementation works. + * This can be run as a simple main method without JUnit. + */ +public class IntegrationTest { + + public static void main(String[] args) { + try { + testBasicFunctionality(); + testArraysAndMaps(); + testNestedRecords(); + System.out.println("All integration tests passed!"); + } catch (Exception e) { + System.err.println("Integration test failed: " + e.getMessage()); + e.printStackTrace(); + System.exit(1); + } + } + + static void testBasicFunctionality() throws ImprintException { + System.out.println("Testing basic functionality..."); + + SchemaId schemaId = new SchemaId(1, 0xdeadbeef); + ImprintWriter writer = new ImprintWriter(schemaId); + + writer.addField(1, Value.fromInt32(42)) + .addField(2, Value.fromString("testing java imprint spec")) + .addField(3, Value.fromBoolean(true)) + .addField(4, Value.fromFloat64(3.14159)) + .addField(5, Value.fromBytes(new byte[]{1, 2, 3, 4})); + + ImprintRecord record = writer.build(); + + // Verify we can read values back + assert record.getValue(1).get().equals(Value.fromInt32(42)); + assert record.getValue(2).get().equals(Value.fromString("testing java imprint spec")); + assert record.getValue(3).get().equals(Value.fromBoolean(true)); + assert record.getValue(999).isEmpty(); // non-existent field + + // Test serialization round-trip + var buffer = record.serializeToBuffer(); + byte[] serialized = new byte[buffer.remaining()]; + buffer.get(serialized); + ImprintRecord deserialized = ImprintRecord.deserialize(serialized); + + assert deserialized.getValue(1).get().equals(Value.fromInt32(42)); + assert deserialized.getValue(2).get().equals(Value.fromString("testing java imprint spec")); + assert deserialized.getValue(3).get().equals(Value.fromBoolean(true)); + + System.out.println("✓ Basic functionality test passed"); + } + + static void testArraysAndMaps() throws ImprintException { + System.out.println("Testing arrays and maps..."); + + SchemaId schemaId = new SchemaId(2, 0xcafebabe); + ImprintWriter writer = new ImprintWriter(schemaId); + + // Create an array + List intArray = Arrays.asList( + Value.fromInt32(1), + Value.fromInt32(2), + Value.fromInt32(3) + ); + + // Create a map + Map stringToIntMap = new HashMap<>(); + stringToIntMap.put(MapKey.fromString("one"), Value.fromInt32(1)); + stringToIntMap.put(MapKey.fromString("two"), Value.fromInt32(2)); + + writer.addField(1, Value.fromArray(intArray)) + .addField(2, Value.fromMap(stringToIntMap)); + + ImprintRecord record = writer.build(); + + // Test serialization round-trip + var buffer = record.serializeToBuffer(); + byte[] serialized = new byte[buffer.remaining()]; + buffer.get(serialized); + ImprintRecord deserialized = ImprintRecord.deserialize(serialized); + + // Verify array + Value arrayValue = deserialized.getValue(1).get(); + assert arrayValue instanceof Value.ArrayValue; + List deserializedArray = ((Value.ArrayValue) arrayValue).getValue(); + assert deserializedArray.size() == 3; + assert deserializedArray.get(0).equals(Value.fromInt32(1)); + + // Verify map + Value mapValue = deserialized.getValue(2).get(); + assert mapValue instanceof Value.MapValue; + Map deserializedMap = ((Value.MapValue) mapValue).getValue(); + assert deserializedMap.size() == 2; + assert deserializedMap.get(MapKey.fromString("one")).equals(Value.fromInt32(1)); + + System.out.println("✓ Arrays and maps test passed"); + } + + static void testNestedRecords() throws ImprintException { + System.out.println("Testing nested records..."); + + // Create inner record + SchemaId innerSchemaId = new SchemaId(3, 0x12345678); + ImprintWriter innerWriter = new ImprintWriter(innerSchemaId); + innerWriter.addField(1, Value.fromString("nested data")) + .addField(2, Value.fromInt64(9876543210L)); + ImprintRecord innerRecord = innerWriter.build(); + + // Create outer record + SchemaId outerSchemaId = new SchemaId(4, 0x87654321); + ImprintWriter outerWriter = new ImprintWriter(outerSchemaId); + outerWriter.addField(1, Value.fromRow(innerRecord)) + .addField(2, Value.fromString("outer data")); + ImprintRecord outerRecord = outerWriter.build(); + + // Test serialization round-trip + var buffer = outerRecord.serializeToBuffer(); + byte[] serialized = new byte[buffer.remaining()]; + buffer.get(serialized); + ImprintRecord deserialized = ImprintRecord.deserialize(serialized); + + // Verify outer record + assert deserialized.getHeader().getSchemaId().getFieldspaceId() == 4; + assert deserialized.getValue(2).get().equals(Value.fromString("outer data")); + + // Verify nested record + Value rowValue = deserialized.getValue(1).get(); + assert rowValue instanceof Value.RowValue; + ImprintRecord nestedRecord = ((Value.RowValue) rowValue).getValue(); + + assert nestedRecord.getHeader().getSchemaId().getFieldspaceId() == 3; + assert nestedRecord.getValue(1).get().equals(Value.fromString("nested data")); + assert nestedRecord.getValue(2).get().equals(Value.fromInt64(9876543210L)); + + System.out.println("✓ Nested records test passed"); + } +} \ No newline at end of file diff --git a/src/test/java/com/imprint/benchmark/ProfilerTest.java b/src/test/java/com/imprint/benchmark/ProfilerTest.java new file mode 100644 index 0000000..233d993 --- /dev/null +++ b/src/test/java/com/imprint/benchmark/ProfilerTest.java @@ -0,0 +1,226 @@ +package com.imprint.benchmark; + +import com.imprint.core.*; +import com.imprint.types.Value; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Disabled; + +import java.util.Random; + +/** + * A test designed for profiling hotspots during development. + *

+ * To use with a profiler: + * 1. Remove @Disabled annotation + * 2. Run with JProfiler, VisualVM, or async-profiler: + * - JProfiler: Attach to test JVM + * - VisualVM: jvisualvm, attach to process + * - async-profiler: java -jar async-profiler.jar -d 30 -f profile.html + * 3. Look for hotspots in CPU sampling + *

+ * Key areas to examine: + * - Object allocation (memory profiling) + * - Method call frequency (CPU sampling) + * - GC pressure (memory profiling) + * - String operations and UTF-8 encoding + * - ByteBuffer operations + */ +@Disabled("Enable manually for profiling") +public class ProfilerTest { + + private static final int ITERATIONS = 1_000_000; + private static final int RECORD_SIZE = 20; + + @Test + void profileFieldAccess() throws Exception { + System.out.println("Starting profiler test - attach profiler now..."); + Thread.sleep(5000); // Give time to attach profiler + + // Create a representative record + var record = createTestRecord(); + + System.out.println("Beginning field access profiling..."); + long start = System.nanoTime(); + + // Simulate real-world access patterns + Random random = new Random(42); + int hits = 0; + + for (int i = 0; i < ITERATIONS; i++) { + // Random field access (hotspot) + int fieldId = random.nextInt(RECORD_SIZE) + 1; + var value = record.getValue(fieldId); + if (value.isPresent()) { + hits++; + + // Trigger string decoding (potential hotspot) + if (value.get().getTypeCode() == com.imprint.types.TypeCode.STRING) { + if (value.get() instanceof Value.StringBufferValue) { + ((Value.StringBufferValue) value.get()).getValue(); + } else { + ((Value.StringValue) value.get()).getValue(); + } + } + } + + // Some raw access (zero-copy path) + if (i % 10 == 0) { + record.getRawBytes(fieldId); + } + } + + long duration = System.nanoTime() - start; + System.out.printf("Completed %,d field accesses in %.2f ms (avg: %.1f ns/op, hits: %d)%n", + ITERATIONS, duration / 1_000_000.0, (double) duration / ITERATIONS, hits); + } + + @Test + void profileSerialization() throws Exception { + System.out.println("Starting serialization profiler test..."); + Thread.sleep(3000); + + var schemaId = new SchemaId(1, 0x12345678); + + System.out.println("Beginning serialization profiling..."); + long start = System.nanoTime(); + + // Create and serialize many records (allocation hotspot) + for (int i = 0; i < 100_000; i++) { + var writer = new ImprintWriter(schemaId); + + // Add various field types + writer.addField(1, Value.fromInt32(i)) + .addField(2, Value.fromString("test-string-" + i)) + .addField(3, Value.fromFloat64(i * 3.14159)) + .addField(4, Value.fromBytes(("bytes-" + i).getBytes())); + + var record = writer.build(); + var serialized = record.serializeToBuffer(); // Potential hotspot + + // Trigger some deserialization + if (i % 1000 == 0) { + var deserialized = ImprintRecord.deserialize(serialized); + deserialized.getValue(2); // String decoding hotspot + } + } + + long duration = System.nanoTime() - start; + System.out.printf("Completed serialization test in %.2f ms%n", duration / 1_000_000.0); + } + + @Test + void profileProjection() throws Exception { + System.out.println("Starting projection profiler test..."); + Thread.sleep(3000); + + var record = createLargeRecord(); + + System.out.println("Beginning projection profiling..."); + long start = System.nanoTime(); + + // Simulate analytical workload - project subset of fields repeatedly + for (int i = 0; i < 50_000; i++) { + // Project 10 fields out of 100 (common analytical pattern) + for (int fieldId = 1; fieldId <= 10; fieldId++) { + var value = record.getValue(fieldId); + if (value.isPresent()) { + // Force materialization of string values + if (value.get().getTypeCode() == com.imprint.types.TypeCode.STRING) { + if (value.get() instanceof Value.StringBufferValue) { + ((Value.StringBufferValue) value.get()).getValue(); + } + } + } + } + } + + long duration = System.nanoTime() - start; + System.out.printf("Completed projection test in %.2f ms%n", duration / 1_000_000.0); + } + + @Test + void profileMemoryAllocation() throws Exception { + System.out.println("Starting allocation profiler test..."); + Thread.sleep(3000); + + System.out.println("Beginning allocation profiling - watch for GC events..."); + + // Force allocation pressure to reveal GC hotspots + for (int batch = 0; batch < 1000; batch++) { + for (int i = 0; i < 1000; i++) { + var schemaId = new SchemaId(batch, i); + var writer = new ImprintWriter(schemaId); + + // Create strings of varying sizes (allocation pressure) + writer.addField(1, Value.fromString("small")) + .addField(2, Value.fromString("medium-length-string-" + i)) + .addField(3, Value.fromString("very-long-string-that-will-cause-more-allocation-pressure-" + batch + "-" + i)) + .addField(4, Value.fromBytes(new byte[100 + i % 1000])); // Varying byte arrays + + var record = writer.build(); + + // Some deserialization to trigger string decoding allocations + record.getValue(2); + record.getValue(3); + } + + if (batch % 100 == 0) { + System.out.printf("Completed batch %d/1000%n", batch); + } + } + + System.out.println("Allocation test complete - check GC logs and memory profiler"); + } + + private ImprintRecord createTestRecord() throws Exception { + var schemaId = new SchemaId(1, 0xdeadbeef); + var writer = new ImprintWriter(schemaId); + + for (int i = 1; i <= RECORD_SIZE; i++) { + switch (i % 4) { + case 0: + writer.addField(i, Value.fromInt32(i * 100)); + break; + case 1: + writer.addField(i, Value.fromString("field-value-" + i)); + break; + case 2: + writer.addField(i, Value.fromFloat64(i * 3.14159)); + break; + case 3: + writer.addField(i, Value.fromBytes(("bytes-" + i).getBytes())); + break; + } + } + + return writer.build(); + } + + private ImprintRecord createLargeRecord() throws Exception { + var schemaId = new SchemaId(2, 0xcafebabe); + var writer = new ImprintWriter(schemaId); + + // Create 100 fields with realistic data + for (int i = 1; i <= 100; i++) { + switch (i % 5) { + case 0: + writer.addField(i, Value.fromInt32(i)); + break; + case 1: + writer.addField(i, Value.fromString("user-name-" + i + "@example.com")); + break; + case 2: + writer.addField(i, Value.fromString("Some longer descriptive text for field " + i + " that might represent a comment or description")); + break; + case 3: + writer.addField(i, Value.fromFloat64(i * 2.718281828)); + break; + case 4: + writer.addField(i, Value.fromBytes(String.format("binary-data-%04d", i).getBytes())); + break; + } + } + + return writer.build(); + } +} \ No newline at end of file diff --git a/src/test/java/com/imprint/core/ImprintRecordBuilderTest.java b/src/test/java/com/imprint/core/ImprintRecordBuilderTest.java new file mode 100644 index 0000000..54dcfae --- /dev/null +++ b/src/test/java/com/imprint/core/ImprintRecordBuilderTest.java @@ -0,0 +1,234 @@ +package com.imprint.core; + +import com.imprint.error.ImprintException; +import com.imprint.types.Value; +import org.junit.jupiter.api.Test; + +import java.util.*; + +import static org.assertj.core.api.Assertions.*; + +class ImprintRecordBuilderTest { + + private static final SchemaId TEST_SCHEMA = new SchemaId(1, 0x12345678); + + @Test + void shouldCreateRecordWithPrimitiveTypes() throws ImprintException { + var record = ImprintRecord.builder(TEST_SCHEMA) + .field(1, true) + .field(2, 42) + .field(3, 123L) + .field(4, 3.14f) + .field(5, 2.718) + .field(6, "hello world") + .field(7, new byte[]{1, 2, 3}) + .nullField(8) + .build(); + + assertThat(record.getHeader().getSchemaId()).isEqualTo(TEST_SCHEMA); + assertThat(record.getDirectory()).hasSize(8); + + // Verify field values + assertThat(getFieldValue(record, 1, Value.BoolValue.class).getValue()).isTrue(); + assertThat(getFieldValue(record, 2, Value.Int32Value.class).getValue()).isEqualTo(42); + assertThat(getFieldValue(record, 3, Value.Int64Value.class).getValue()).isEqualTo(123L); + assertThat(getFieldValue(record, 4, Value.Float32Value.class).getValue()).isEqualTo(3.14f); + assertThat(getFieldValue(record, 5, Value.Float64Value.class).getValue()).isEqualTo(2.718); + assertThat(getStringValue(record, 6)).isEqualTo("hello world"); + assertThat(getBytesValue(record, 7)).isEqualTo(new byte[]{1, 2, 3}); + assertThat(record.getValue(8).get()).isInstanceOf(Value.NullValue.class); + } + + @Test + void shouldCreateRecordWithCollections() throws ImprintException { + var list = List.of(1, 2, 3); + var map = Map.of("key1", 100, "key2", 200); + + var record = ImprintRecord.builder(TEST_SCHEMA) + .field(1, list) + .field(2, map) + .build(); + + // Verify array + var arrayValue = getFieldValue(record, 1, Value.ArrayValue.class); + assertThat(arrayValue.getValue()).hasSize(3); + assertThat(((Value.Int32Value) arrayValue.getValue().get(0)).getValue()).isEqualTo(1); + assertThat(((Value.Int32Value) arrayValue.getValue().get(1)).getValue()).isEqualTo(2); + assertThat(((Value.Int32Value) arrayValue.getValue().get(2)).getValue()).isEqualTo(3); + + // Verify map + var mapValue = getFieldValue(record, 2, Value.MapValue.class); + assertThat(mapValue.getValue()).hasSize(2); + } + + @Test + void shouldCreateRecordWithNestedRecord() throws ImprintException { + var nestedRecord = ImprintRecord.builder(new SchemaId(2, 0x87654321)) + .field(1, "nested") + .field(2, 999) + .build(); + + var record = ImprintRecord.builder(TEST_SCHEMA) + .field(1, "parent") + .field(2, nestedRecord) + .build(); + + var rowValue = getFieldValue(record, 2, Value.RowValue.class); + var nested = rowValue.getValue(); + assertThat(getStringValue(nested, 1)).isEqualTo("nested"); + assertThat(getFieldValue(nested, 2, Value.Int32Value.class).getValue()).isEqualTo(999); + } + + @Test + void shouldSupportConditionalFields() throws ImprintException { + boolean includeOptional = true; + String optionalValue = "optional"; + + var record = ImprintRecord.builder(TEST_SCHEMA) + .field(1, "required") + .fieldIf(includeOptional, 2, optionalValue) + .fieldIfNotNull(3, null) // Should not add field + .fieldIfNotNull(4, "not null") // Should add field + .build(); + + assertThat(record.getDirectory()).hasSize(3); // Only fields 1, 2, 4 + assertThat(getStringValue(record, 1)).isEqualTo("required"); + assertThat(getStringValue(record, 2)).isEqualTo("optional"); + assertThat(record.getValue(3)).isEmpty(); // Not added + assertThat(getStringValue(record, 4)).isEqualTo("not null"); + } + + @Test + void shouldSupportBulkOperations() throws ImprintException { + var fieldsMap = Map.of( + 1, "bulk1", + 2, 42, + 3, true + ); + + var record = ImprintRecord.builder(TEST_SCHEMA) + .fields(fieldsMap) + .field(4, "additional") + .build(); + + assertThat(record.getDirectory()).hasSize(4); + assertThat(getStringValue(record, 1)).isEqualTo("bulk1"); + assertThat(getFieldValue(record, 2, Value.Int32Value.class).getValue()).isEqualTo(42); + assertThat(getFieldValue(record, 3, Value.BoolValue.class).getValue()).isTrue(); + assertThat(getStringValue(record, 4)).isEqualTo("additional"); + } + + @Test + void shouldProvideBuilderUtilities() { + var builder = ImprintRecord.builder(TEST_SCHEMA) + .field(1, "test") + .field(2, 42); + + assertThat(builder.hasField(1)).isTrue(); + assertThat(builder.hasField(3)).isFalse(); + assertThat(builder.fieldCount()).isEqualTo(2); + assertThat(builder.fieldIds()).containsExactly(1, 2); + } + + @Test + void shouldSupportAlternativeSchemaConstructor() throws ImprintException { + var record = ImprintRecord.builder(1, 0x12345678) + .field(1, "test") + .build(); + + assertThat(record.getHeader().getSchemaId().getFieldspaceId()).isEqualTo(1); + assertThat(record.getHeader().getSchemaId().getSchemaHash()).isEqualTo(0x12345678); + } + + @Test + void shouldRoundTripThroughSerialization() throws ImprintException { + var original = ImprintRecord.builder(TEST_SCHEMA) + .field(1, "test string") + .field(2, 42) + .field(3, 3.14159) + .field(4, true) + .field(5, new byte[]{0x01, 0x02, 0x03}) + .build(); + + var serialized = original.serializeToBuffer(); + var deserialized = ImprintRecord.deserialize(serialized); + + assertThat(deserialized.getHeader().getSchemaId()).isEqualTo(TEST_SCHEMA); + assertThat(getStringValue(deserialized, 1)).isEqualTo("test string"); + assertThat(getFieldValue(deserialized, 2, Value.Int32Value.class).getValue()).isEqualTo(42); + assertThat(getFieldValue(deserialized, 3, Value.Float64Value.class).getValue()).isEqualTo(3.14159); + assertThat(getFieldValue(deserialized, 4, Value.BoolValue.class).getValue()).isTrue(); + assertThat(getBytesValue(deserialized, 5)).isEqualTo(new byte[]{0x01, 0x02, 0x03}); + } + + // Error cases + + @Test + void shouldRejectDuplicateFieldIds() { + assertThatThrownBy(() -> + ImprintRecord.builder(TEST_SCHEMA) + .field(1, "first") + .field(1, "duplicate") // Same field ID + ).isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Field ID 1 already exists"); + } + + @Test + void shouldRejectEmptyRecord() { + assertThatThrownBy(() -> + ImprintRecord.builder(TEST_SCHEMA).build() + ).isInstanceOf(ImprintException.class) + .hasMessageContaining("Cannot build empty record"); + } + + @Test + void shouldRejectInvalidMapKeys() { + var mapWithInvalidKey = Map.of(3.14, "value"); // Double key not supported + + assertThatThrownBy(() -> + ImprintRecord.builder(TEST_SCHEMA) + .field(1, mapWithInvalidKey) + ).isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Invalid map key type: Double"); + } + + @Test + void shouldRejectNullValueWithoutExplicitNullField() { + assertThatThrownBy(() -> + ImprintRecord.builder(TEST_SCHEMA) + .field(1, (Value) null) + ).isInstanceOf(NullPointerException.class) + .hasMessageContaining("Value cannot be null - use nullField()"); + } + + // Helper methods for cleaner test assertions + + private T getFieldValue(ImprintRecord record, int fieldId, Class valueType) throws ImprintException { + var value = record.getValue(fieldId); + assertThat(value).isPresent(); + assertThat(value.get()).isInstanceOf(valueType); + return valueType.cast(value.get()); + } + + private String getStringValue(ImprintRecord record, int fieldId) throws ImprintException { + var value = record.getValue(fieldId).get(); + if (value instanceof Value.StringValue) { + return ((Value.StringValue) value).getValue(); + } else if (value instanceof Value.StringBufferValue) { + return ((Value.StringBufferValue) value).getValue(); + } else { + throw new AssertionError("Expected string value, got: " + value.getClass()); + } + } + + private byte[] getBytesValue(ImprintRecord record, int fieldId) throws ImprintException { + var value = record.getValue(fieldId).get(); + if (value instanceof Value.BytesValue) { + return ((Value.BytesValue) value).getValue(); + } else if (value instanceof Value.BytesBufferValue) { + return ((Value.BytesBufferValue) value).getValue(); + } else { + throw new AssertionError("Expected bytes value, got: " + value.getClass()); + } + } +} \ No newline at end of file diff --git a/src/test/java/com/imprint/core/ImprintRecordTest.java b/src/test/java/com/imprint/core/ImprintRecordTest.java new file mode 100644 index 0000000..0772580 --- /dev/null +++ b/src/test/java/com/imprint/core/ImprintRecordTest.java @@ -0,0 +1,232 @@ +package com.imprint.core; + +import com.imprint.error.ImprintException; +import com.imprint.error.ErrorType; +import com.imprint.types.Value; +import com.imprint.types.MapKey; +import org.junit.jupiter.api.Test; +import java.util.*; +import static org.assertj.core.api.Assertions.*; + +class ImprintRecordTest { + + // Helper method to extract string value from either StringValue or StringBufferValue + private String getStringValue(Value value) { + if (value instanceof Value.StringValue) { + return ((Value.StringValue) value).getValue(); + } else if (value instanceof Value.StringBufferValue) { + return ((Value.StringBufferValue) value).getValue(); + } else { + throw new IllegalArgumentException("Expected string value, got: " + value.getClass()); + } + } + + @Test + void shouldCreateSimpleRecord() throws ImprintException { + var schemaId = new SchemaId(1, 0xdeadbeef); + var writer = new ImprintWriter(schemaId); + + writer.addField(1, Value.fromInt32(42)) + .addField(2, Value.fromString("hello")); + + var record = writer.build(); + + assertThat(record.getHeader().getSchemaId()).isEqualTo(schemaId); + assertThat(record.getDirectory()).hasSize(2); + + Optional field1 = record.getValue(1); + Optional field2 = record.getValue(2); + + assertThat(field1).isPresent(); + assertThat(field1.get()).isInstanceOf(Value.Int32Value.class); + assertThat(((Value.Int32Value) field1.get()).getValue()).isEqualTo(42); + + assertThat(field2).isPresent(); + assertThat(field2.get().getTypeCode()).isEqualTo(com.imprint.types.TypeCode.STRING); + String stringValue = getStringValue(field2.get()); + assertThat(stringValue).isEqualTo("hello"); + + // Non-existent field should return empty + assertThat(record.getValue(999)).isEmpty(); + } + + @Test + void shouldRoundtripThroughSerialization() throws ImprintException { + var schemaId = new SchemaId(1, 0xdeadbeef); + var writer = new ImprintWriter(schemaId); + + writer.addField(1, Value.nullValue()) + .addField(2, Value.fromBoolean(true)) + .addField(3, Value.fromInt32(42)) + .addField(4, Value.fromInt64(123456789L)) + .addField(5, Value.fromFloat32(3.14f)) + .addField(6, Value.fromFloat64(2.718281828)) + .addField(7, Value.fromBytes(new byte[]{1, 2, 3, 4})) + .addField(8, Value.fromString("test string")); + + var original = writer.build(); + + // Serialize and deserialize + var buffer = original.serializeToBuffer(); + byte[] serialized = new byte[buffer.remaining()]; + buffer.get(serialized); + var deserialized = ImprintRecord.deserialize(serialized); + + // Verify metadata + assertThat(deserialized.getHeader().getSchemaId().getFieldspaceId()).isEqualTo(1); + assertThat(deserialized.getHeader().getSchemaId().getSchemaHash()).isEqualTo(0xdeadbeef); + assertThat(deserialized.getDirectory()).hasSize(8); + + // Verify all values + assertThat(deserialized.getValue(1)).contains(Value.nullValue()); + assertThat(deserialized.getValue(2)).contains(Value.fromBoolean(true)); + assertThat(deserialized.getValue(3)).contains(Value.fromInt32(42)); + assertThat(deserialized.getValue(4)).contains(Value.fromInt64(123456789L)); + assertThat(deserialized.getValue(5)).contains(Value.fromFloat32(3.14f)); + assertThat(deserialized.getValue(6)).contains(Value.fromFloat64(2.718281828)); + assertThat(deserialized.getValue(7)).contains(Value.fromBytes(new byte[]{1, 2, 3, 4})); + assertThat(deserialized.getValue(8)).contains(Value.fromString("test string")); + + // Non-existent field + assertThat(deserialized.getValue(999)).isEmpty(); + } + + @Test + void shouldHandleArrays() throws ImprintException { + var schemaId = new SchemaId(1, 0xdeadbeef); + var writer = new ImprintWriter(schemaId); + + List intArray = Arrays.asList( + Value.fromInt32(1), + Value.fromInt32(2), + Value.fromInt32(3) + ); + + writer.addField(1, Value.fromArray(intArray)); + ImprintRecord record = writer.build(); + + // Serialize and deserialize + var buffer = record.serializeToBuffer(); + byte[] serialized = new byte[buffer.remaining()]; + buffer.get(serialized); + var deserialized = ImprintRecord.deserialize(serialized); + + Optional arrayValue = deserialized.getValue(1); + assertThat(arrayValue).isPresent(); + assertThat(arrayValue.get()).isInstanceOf(Value.ArrayValue.class); + + List deserializedArray = ((Value.ArrayValue) arrayValue.get()).getValue(); + assertThat(deserializedArray).hasSize(3); + assertThat(deserializedArray.get(0)).isEqualTo(Value.fromInt32(1)); + assertThat(deserializedArray.get(1)).isEqualTo(Value.fromInt32(2)); + assertThat(deserializedArray.get(2)).isEqualTo(Value.fromInt32(3)); + } + + @Test + void shouldHandleMaps() throws ImprintException { + var schemaId = new SchemaId(1, 0xdeadbeef); + var writer = new ImprintWriter(schemaId); + + var map = new HashMap(); + map.put(MapKey.fromString("key1"), Value.fromInt32(1)); + map.put(MapKey.fromString("key2"), Value.fromInt32(2)); + + writer.addField(1, Value.fromMap(map)); + var record = writer.build(); + + // Serialize and deserialize + var buffer = record.serializeToBuffer(); + byte[] serialized = new byte[buffer.remaining()]; + buffer.get(serialized); + var deserialized = ImprintRecord.deserialize(serialized); + + Optional mapValue = deserialized.getValue(1); + assertThat(mapValue).isPresent(); + assertThat(mapValue.get()).isInstanceOf(Value.MapValue.class); + + Map deserializedMap = ((Value.MapValue) mapValue.get()).getValue(); + assertThat(deserializedMap).hasSize(2); + assertThat(deserializedMap.get(MapKey.fromString("key1"))).isEqualTo(Value.fromInt32(1)); + assertThat(deserializedMap.get(MapKey.fromString("key2"))).isEqualTo(Value.fromInt32(2)); + } + + @Test + void shouldHandleNestedRecords() throws ImprintException { + // Create inner record + var innerSchemaId = new SchemaId(2, 0xcafebabe); + var innerWriter = new ImprintWriter(innerSchemaId); + innerWriter.addField(1, Value.fromInt32(42)) + .addField(2, Value.fromString("nested")); + var innerRecord = innerWriter.build(); + + // Create outer record containing inner record + var outerSchemaId = new SchemaId(1, 0xdeadbeef); + var outerWriter = new ImprintWriter(outerSchemaId); + outerWriter.addField(1, Value.fromRow(innerRecord)) + .addField(2, Value.fromInt64(123L)); + var outerRecord = outerWriter.build(); + + // Serialize and deserialize + var buffer = outerRecord.serializeToBuffer(); + byte[] serialized = new byte[buffer.remaining()]; + buffer.get(serialized); + var deserialized = ImprintRecord.deserialize(serialized); + + // Verify outer record metadata + assertThat(deserialized.getHeader().getSchemaId().getFieldspaceId()).isEqualTo(1); + assertThat(deserialized.getHeader().getSchemaId().getSchemaHash()).isEqualTo(0xdeadbeef); + + // Verify nested record + Optional rowValue = deserialized.getValue(1); + assertThat(rowValue).isPresent(); + assertThat(rowValue.get()).isInstanceOf(Value.RowValue.class); + + var nestedRecord = ((Value.RowValue) rowValue.get()).getValue(); + assertThat(nestedRecord.getHeader().getSchemaId().getFieldspaceId()).isEqualTo(2); + assertThat(nestedRecord.getHeader().getSchemaId().getSchemaHash()).isEqualTo(0xcafebabe); + + assertThat(nestedRecord.getValue(1)).contains(Value.fromInt32(42)); + assertThat(nestedRecord.getValue(2)).contains(Value.fromString("nested")); + + // Verify outer record field + assertThat(deserialized.getValue(2)).contains(Value.fromInt64(123L)); + } + + @Test + void shouldRejectInvalidMagic() { + byte[] invalidData = new byte[15]; + invalidData[0] = 0x00; // wrong magic + + assertThatThrownBy(() -> ImprintRecord.deserialize(invalidData)) + .isInstanceOf(ImprintException.class) + .extracting("errorType") + .isEqualTo(ErrorType.INVALID_MAGIC); + } + + @Test + void shouldRejectUnsupportedVersion() { + byte[] invalidData = new byte[15]; + invalidData[0] = (byte) 0x49; // correct magic + invalidData[1] = (byte) 0xFF; // wrong version + + assertThatThrownBy(() -> ImprintRecord.deserialize(invalidData)) + .isInstanceOf(ImprintException.class) + .extracting("errorType") + .isEqualTo(ErrorType.UNSUPPORTED_VERSION); + } + + @Test + void shouldHandleDuplicateFieldIds() throws ImprintException { + var schemaId = new SchemaId(1, 0xdeadbeef); + var writer = new ImprintWriter(schemaId); + + // Add duplicate field IDs - last one should win + writer.addField(1, Value.fromInt32(42)) + .addField(1, Value.fromInt32(43)); + + var record = writer.build(); + + assertThat(record.getDirectory()).hasSize(1); + assertThat(record.getValue(1)).contains(Value.fromInt32(43)); + } +} \ No newline at end of file diff --git a/src/test/java/com/imprint/types/MapKeyTest.java b/src/test/java/com/imprint/types/MapKeyTest.java new file mode 100644 index 0000000..08f4180 --- /dev/null +++ b/src/test/java/com/imprint/types/MapKeyTest.java @@ -0,0 +1,91 @@ +package com.imprint.types; + +import com.imprint.error.ImprintException; +import com.imprint.error.ErrorType; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.*; + +class MapKeyTest { + + @Test + void shouldCreateMapKeysFromValues() throws ImprintException { + var int32Key = MapKey.fromValue(Value.fromInt32(42)); + var int64Key = MapKey.fromValue(Value.fromInt64(123L)); + var bytesKey = MapKey.fromValue(Value.fromBytes(new byte[]{1, 2, 3})); + var stringKey = MapKey.fromValue(Value.fromString("test")); + + assertThat(int32Key).isInstanceOf(MapKey.Int32Key.class); + assertThat(((MapKey.Int32Key) int32Key).getValue()).isEqualTo(42); + + assertThat(int64Key).isInstanceOf(MapKey.Int64Key.class); + assertThat(((MapKey.Int64Key) int64Key).getValue()).isEqualTo(123L); + + assertThat(bytesKey).isInstanceOf(MapKey.BytesKey.class); + assertThat(((MapKey.BytesKey) bytesKey).getValue()).containsExactly(1, 2, 3); + + assertThat(stringKey).isInstanceOf(MapKey.StringKey.class); + assertThat(((MapKey.StringKey) stringKey).getValue()).isEqualTo("test"); + } + + @Test + void shouldConvertBackToValues() { + var int32Key = MapKey.fromInt32(42); + var stringKey = MapKey.fromString("test"); + + var int32Value = int32Key.toValue(); + var stringValue = stringKey.toValue(); + + assertThat(int32Value).isInstanceOf(Value.Int32Value.class); + assertThat(((Value.Int32Value) int32Value).getValue()).isEqualTo(42); + + assertThat(stringValue).isInstanceOf(Value.StringValue.class); + assertThat(((Value.StringValue) stringValue).getValue()).isEqualTo("test"); + } + + @Test + void shouldRejectInvalidValueTypes() { + var boolValue = Value.fromBoolean(true); + var arrayValue = Value.fromArray(java.util.Collections.emptyList()); + + assertThatThrownBy(() -> MapKey.fromValue(boolValue)) + .isInstanceOf(ImprintException.class) + .extracting("errorType") + .isEqualTo(ErrorType.TYPE_MISMATCH); + + assertThatThrownBy(() -> MapKey.fromValue(arrayValue)) + .isInstanceOf(ImprintException.class) + .extracting("errorType") + .isEqualTo(ErrorType.TYPE_MISMATCH); + } + + @Test + void shouldHandleEqualityAndHashing() { + var key1 = MapKey.fromString("test"); + var key2 = MapKey.fromString("test"); + var key3 = MapKey.fromString("different"); + + assertThat(key1).isEqualTo(key2); + assertThat(key1).isNotEqualTo(key3); + assertThat(key1.hashCode()).isEqualTo(key2.hashCode()); + } + + @Test + void shouldDefensiveCopyBytes() { + byte[] original = {1, 2, 3}; + var bytesKey = MapKey.fromBytes(original); + + // Modify original array + original[0] = 99; + + // Key should be unchanged + assertThat(((MapKey.BytesKey) bytesKey).getValue()).containsExactly(1, 2, 3); + } + + @Test + void shouldHaveCorrectTypeCodes() { + assertThat(MapKey.fromInt32(1).getTypeCode()).isEqualTo(TypeCode.INT32); + assertThat(MapKey.fromInt64(1L).getTypeCode()).isEqualTo(TypeCode.INT64); + assertThat(MapKey.fromBytes(new byte[]{1}).getTypeCode()).isEqualTo(TypeCode.BYTES); + assertThat(MapKey.fromString("test").getTypeCode()).isEqualTo(TypeCode.STRING); + } +} \ No newline at end of file diff --git a/src/test/java/com/imprint/types/TypeHandlerTest.java b/src/test/java/com/imprint/types/TypeHandlerTest.java new file mode 100644 index 0000000..9a4ae85 --- /dev/null +++ b/src/test/java/com/imprint/types/TypeHandlerTest.java @@ -0,0 +1,279 @@ +package com.imprint.types; + +import com.imprint.error.ImprintException; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +import static org.assertj.core.api.Assertions.*; + +/** + * Tests for individual TypeHandler implementations. + * Validates serialization, deserialization, and size estimation for each type. + */ +class TypeHandlerTest { + + @Test + void testNullHandler() throws ImprintException { + var handler = TypeHandler.NULL; + var value = Value.nullValue(); + + // Size estimation + assertThat(handler.estimateSize(value)).isEqualTo(0); + + // Serialization + var buffer = ByteBuffer.allocate(10); + handler.serialize(value, buffer); + assertThat(buffer.position()).isEqualTo(0); // NULL writes nothing + + // Deserialization + buffer.flip(); + var deserialized = handler.deserialize(buffer); + assertThat(deserialized).isEqualTo(value); + + // readValueBytes + buffer.clear(); + var valueBytes = handler.readValueBytes(buffer); + assertThat(valueBytes.remaining()).isEqualTo(0); + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void testBoolHandler(boolean testValue) throws ImprintException { + var handler = TypeHandler.BOOL; + var value = Value.fromBoolean(testValue); + + // Size estimation + assertThat(handler.estimateSize(value)).isEqualTo(1); + + // Round-trip test + var buffer = ByteBuffer.allocate(10); + handler.serialize(value, buffer); + assertThat(buffer.position()).isEqualTo(1); + + buffer.flip(); + var deserialized = handler.deserialize(buffer); + assertThat(deserialized).isEqualTo(value); + assertThat(((Value.BoolValue) deserialized).getValue()).isEqualTo(testValue); + } + + @ParameterizedTest + @ValueSource(ints = {0, 1, -1, Integer.MAX_VALUE, Integer.MIN_VALUE, 42, -42}) + void testInt32Handler(int testValue) throws ImprintException { + var handler = TypeHandler.INT32; + var value = Value.fromInt32(testValue); + + // Size estimation + assertThat(handler.estimateSize(value)).isEqualTo(4); + + // Round-trip test + var buffer = ByteBuffer.allocate(10).order(ByteOrder.LITTLE_ENDIAN); + handler.serialize(value, buffer); + assertThat(buffer.position()).isEqualTo(4); + + buffer.flip(); + var deserialized = handler.deserialize(buffer); + assertThat(deserialized).isEqualTo(value); + assertThat(((Value.Int32Value) deserialized).getValue()).isEqualTo(testValue); + } + + @ParameterizedTest + @ValueSource(longs = {0L, 1L, -1L, Long.MAX_VALUE, Long.MIN_VALUE, 123456789L}) + void testInt64Handler(long testValue) throws ImprintException { + var handler = TypeHandler.INT64; + var value = Value.fromInt64(testValue); + + // Size estimation + assertThat(handler.estimateSize(value)).isEqualTo(8); + + // Round-trip test + var buffer = ByteBuffer.allocate(20).order(ByteOrder.LITTLE_ENDIAN); + handler.serialize(value, buffer); + assertThat(buffer.position()).isEqualTo(8); + + buffer.flip(); + var deserialized = handler.deserialize(buffer); + assertThat(deserialized).isEqualTo(value); + assertThat(((Value.Int64Value) deserialized).getValue()).isEqualTo(testValue); + } + + @ParameterizedTest + @ValueSource(floats = {0.0f, 1.0f, -1.0f, Float.MAX_VALUE, Float.MIN_VALUE, 3.14159f, Float.NaN, Float.POSITIVE_INFINITY}) + void testFloat32Handler(float testValue) throws ImprintException { + var handler = TypeHandler.FLOAT32; + var value = Value.fromFloat32(testValue); + + // Size estimation + assertThat(handler.estimateSize(value)).isEqualTo(4); + + // Round-trip test + var buffer = ByteBuffer.allocate(10).order(ByteOrder.LITTLE_ENDIAN); + handler.serialize(value, buffer); + assertThat(buffer.position()).isEqualTo(4); + + buffer.flip(); + var deserialized = handler.deserialize(buffer); + assertThat(deserialized).isEqualTo(value); + + float deserializedValue = ((Value.Float32Value) deserialized).getValue(); + if (Float.isNaN(testValue)) { + assertThat(deserializedValue).isNaN(); + } else { + assertThat(deserializedValue).isEqualTo(testValue); + } + } + + @ParameterizedTest + @ValueSource(doubles = {0.0, 1.0, -1.0, Double.MAX_VALUE, Double.MIN_VALUE, Math.PI, Double.NaN, Double.POSITIVE_INFINITY}) + void testFloat64Handler(double testValue) throws ImprintException { + var handler = TypeHandler.FLOAT64; + var value = Value.fromFloat64(testValue); + + // Size estimation + assertThat(handler.estimateSize(value)).isEqualTo(8); + + // Round-trip test + var buffer = ByteBuffer.allocate(20).order(ByteOrder.LITTLE_ENDIAN); + handler.serialize(value, buffer); + assertThat(buffer.position()).isEqualTo(8); + + buffer.flip(); + var deserialized = handler.deserialize(buffer); + assertThat(deserialized).isEqualTo(value); + + double deserializedValue = ((Value.Float64Value) deserialized).getValue(); + if (Double.isNaN(testValue)) { + assertThat(deserializedValue).isNaN(); + } else { + assertThat(deserializedValue).isEqualTo(testValue); + } + } + + @ParameterizedTest + @ValueSource(strings = {"", "hello", "世界", "a very long string that exceeds typical buffer sizes and contains unicode: 🚀🎉", "null\0bytes"}) + void testStringHandler(String testValue) throws ImprintException { + var handler = TypeHandler.STRING; + var value = Value.fromString(testValue); + + byte[] utf8Bytes = testValue.getBytes(java.nio.charset.StandardCharsets.UTF_8); + int expectedSize = com.imprint.util.VarInt.encodedLength(utf8Bytes.length) + utf8Bytes.length; + + // Size estimation + assertThat(handler.estimateSize(value)).isEqualTo(expectedSize); + + // Round-trip test + var buffer = ByteBuffer.allocate(expectedSize + 20).order(ByteOrder.LITTLE_ENDIAN); + handler.serialize(value, buffer); + + buffer.flip(); + var deserialized = handler.deserialize(buffer); + + // Should return StringBufferValue (zero-copy implementation) + assertThat(deserialized).isInstanceOf(Value.StringBufferValue.class); + + String deserializedString; + if (deserialized instanceof Value.StringBufferValue) { + deserializedString = ((Value.StringBufferValue) deserialized).getValue(); + } else { + deserializedString = ((Value.StringValue) deserialized).getValue(); + } + + assertThat(deserializedString).isEqualTo(testValue); + } + + @Test + void testBytesHandlerWithArrayValue() throws ImprintException { + var handler = TypeHandler.BYTES; + byte[] testBytes = {0, 1, 2, (byte) 0xFF, 42, 127, -128}; + var value = Value.fromBytes(testBytes); + + int expectedSize = com.imprint.util.VarInt.encodedLength(testBytes.length) + testBytes.length; + + // Size estimation + assertThat(handler.estimateSize(value)).isEqualTo(expectedSize); + + // Round-trip test + var buffer = ByteBuffer.allocate(expectedSize + 20).order(ByteOrder.LITTLE_ENDIAN); + handler.serialize(value, buffer); + + buffer.flip(); + var deserialized = handler.deserialize(buffer); + + // Should return BytesBufferValue (zero-copy implementation) + assertThat(deserialized).isInstanceOf(Value.BytesBufferValue.class); + + byte[] deserializedBytes = ((Value.BytesBufferValue) deserialized).getValue(); + assertThat(deserializedBytes).isEqualTo(testBytes); + } + + @Test + void testBytesHandlerWithBufferValue() throws ImprintException { + var handler = TypeHandler.BYTES; + byte[] testBytes = {10, 20, 30, 40}; + var bufferValue = Value.fromBytesBuffer(ByteBuffer.wrap(testBytes).asReadOnlyBuffer()); + + int expectedSize = com.imprint.util.VarInt.encodedLength(testBytes.length) + testBytes.length; + + // Size estimation + assertThat(handler.estimateSize(bufferValue)).isEqualTo(expectedSize); + + // Round-trip test + var buffer = ByteBuffer.allocate(expectedSize + 20).order(ByteOrder.LITTLE_ENDIAN); + handler.serialize(bufferValue, buffer); + + buffer.flip(); + var deserialized = handler.deserialize(buffer); + + byte[] deserializedBytes = ((Value.BytesBufferValue) deserialized).getValue(); + assertThat(deserializedBytes).isEqualTo(testBytes); + } + + @Test + void testStringHandlerWithBufferValue() throws ImprintException { + var handler = TypeHandler.STRING; + String testString = "zero-copy string test"; + byte[] utf8Bytes = testString.getBytes(java.nio.charset.StandardCharsets.UTF_8); + var bufferValue = Value.fromStringBuffer(ByteBuffer.wrap(utf8Bytes).asReadOnlyBuffer()); + + int expectedSize = com.imprint.util.VarInt.encodedLength(utf8Bytes.length) + utf8Bytes.length; + + // Size estimation + assertThat(handler.estimateSize(bufferValue)).isEqualTo(expectedSize); + + // Round-trip test + var buffer = ByteBuffer.allocate(expectedSize + 20).order(ByteOrder.LITTLE_ENDIAN); + handler.serialize(bufferValue, buffer); + + buffer.flip(); + var deserialized = handler.deserialize(buffer); + + String deserializedString = ((Value.StringBufferValue) deserialized).getValue(); + assertThat(deserializedString).isEqualTo(testString); + } + + @Test + void testBoolHandlerInvalidValue() { + var handler = TypeHandler.BOOL; + var buffer = ByteBuffer.allocate(10); + buffer.put((byte) 2); // Invalid boolean value + buffer.flip(); + + assertThatThrownBy(() -> handler.deserialize(buffer)) + .isInstanceOf(ImprintException.class) + .hasMessageContaining("Invalid boolean value: 2"); + } + + @Test + void testHandlerBufferUnderflow() { + // Test that handlers properly detect buffer underflow + var int32Handler = TypeHandler.INT32; + var buffer = ByteBuffer.allocate(2); // Too small for int32 + + assertThatThrownBy(() -> int32Handler.deserialize(buffer)) + .isInstanceOf(ImprintException.class) + .hasMessageContaining("Not enough bytes for int32"); + } +} \ No newline at end of file diff --git a/src/test/java/com/imprint/types/ValueTest.java b/src/test/java/com/imprint/types/ValueTest.java new file mode 100644 index 0000000..9dd99c9 --- /dev/null +++ b/src/test/java/com/imprint/types/ValueTest.java @@ -0,0 +1,123 @@ +package com.imprint.types; + +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class ValueTest { + + @Test + void shouldCreateNullValue() { + Value value = Value.nullValue(); + + assertThat(value).isInstanceOf(Value.NullValue.class); + assertThat(value.getTypeCode()).isEqualTo(TypeCode.NULL); + assertThat(value.toString()).isEqualTo("null"); + } + + @Test + void shouldCreateBooleanValues() { + Value trueValue = Value.fromBoolean(true); + Value falseValue = Value.fromBoolean(false); + + assertThat(trueValue).isInstanceOf(Value.BoolValue.class); + assertThat(((Value.BoolValue) trueValue).getValue()).isTrue(); + assertThat(trueValue.getTypeCode()).isEqualTo(TypeCode.BOOL); + + assertThat(falseValue).isInstanceOf(Value.BoolValue.class); + assertThat(((Value.BoolValue) falseValue).getValue()).isFalse(); + assertThat(falseValue.getTypeCode()).isEqualTo(TypeCode.BOOL); + } + + @Test + void shouldCreateNumericValues() { + var int32 = Value.fromInt32(42); + var int64 = Value.fromInt64(123456789L); + var float32 = Value.fromFloat32(3.14f); + var float64 = Value.fromFloat64(2.718281828); + + assertThat(int32.getTypeCode()).isEqualTo(TypeCode.INT32); + assertThat(((Value.Int32Value) int32).getValue()).isEqualTo(42); + + assertThat(int64.getTypeCode()).isEqualTo(TypeCode.INT64); + assertThat(((Value.Int64Value) int64).getValue()).isEqualTo(123456789L); + + assertThat(float32.getTypeCode()).isEqualTo(TypeCode.FLOAT32); + assertThat(((Value.Float32Value) float32).getValue()).isEqualTo(3.14f); + + assertThat(float64.getTypeCode()).isEqualTo(TypeCode.FLOAT64); + assertThat(((Value.Float64Value) float64).getValue()).isEqualTo(2.718281828); + } + + @Test + void shouldCreateBytesAndStringValues() { + byte[] bytes = {1, 2, 3, 4}; + var bytesValue = Value.fromBytes(bytes); + var stringValue = Value.fromString("hello"); + + assertThat(bytesValue.getTypeCode()).isEqualTo(TypeCode.BYTES); + assertThat(((Value.BytesValue) bytesValue).getValue()).isEqualTo(bytes); + + assertThat(stringValue.getTypeCode()).isEqualTo(TypeCode.STRING); + assertThat(((Value.StringValue) stringValue).getValue()).isEqualTo("hello"); + } + + @Test + void shouldCreateArrayValues() { + List elements = Arrays.asList( + Value.fromInt32(1), + Value.fromInt32(2), + Value.fromInt32(3) + ); + Value arrayValue = Value.fromArray(elements); + + assertThat(arrayValue.getTypeCode()).isEqualTo(TypeCode.ARRAY); + assertThat(((Value.ArrayValue) arrayValue).getValue()).isEqualTo(elements); + } + + @Test + void shouldCreateMapValues() { + var map = new HashMap(); + map.put(MapKey.fromString("key1"), Value.fromInt32(1)); + map.put(MapKey.fromString("key2"), Value.fromInt32(2)); + + Value mapValue = Value.fromMap(map); + + assertThat(mapValue.getTypeCode()).isEqualTo(TypeCode.MAP); + assertThat(((Value.MapValue) mapValue).getValue()).isEqualTo(map); + } + + @Test + void shouldHandleEqualityCorrectly() { + var int1 = Value.fromInt32(42); + var int2 = Value.fromInt32(42); + var int3 = Value.fromInt32(43); + + assertThat(int1).isEqualTo(int2); + assertThat(int1).isNotEqualTo(int3); + assertThat(int1.hashCode()).isEqualTo(int2.hashCode()); + } + + @Test + void shouldDefensiveCopyArrays() { + byte[] original = {1, 2, 3}; + var bytesValue = Value.fromBytes(original); + + // Modify original array + original[0] = 99; + + // Value should be unchanged + assertThat(((Value.BytesValue) bytesValue).getValue()).containsExactly(1, 2, 3); + } + + @Test + void shouldRejectNullString() { + assertThatThrownBy(() -> Value.fromString(null)) + .isInstanceOf(NullPointerException.class); + } +} \ No newline at end of file diff --git a/src/test/java/com/imprint/util/VarIntTest.java b/src/test/java/com/imprint/util/VarIntTest.java new file mode 100644 index 0000000..677afb7 --- /dev/null +++ b/src/test/java/com/imprint/util/VarIntTest.java @@ -0,0 +1,115 @@ +package com.imprint.util; + +import com.imprint.error.ImprintException; +import com.imprint.error.ErrorType; +import org.junit.jupiter.api.Test; +import java.nio.ByteBuffer; +import static org.assertj.core.api.Assertions.*; + +class VarIntTest { + + @Test + void shouldRoundtripCommonValues() throws ImprintException { + int[] testCases = { + 0, 1, 127, 128, 16383, 16384, 2097151, 2097152, + 268435455, 268435456, -1 // -1 as unsigned is 0xFFFFFFFF + }; + + for (int value : testCases) { + ByteBuffer buffer = ByteBuffer.allocate(10); + VarInt.encode(value, buffer); + int encodedLength = buffer.position(); + + buffer.flip(); + VarInt.DecodeResult result = VarInt.decode(buffer); + + assertThat(result.getValue()).isEqualTo(value); + assertThat(result.getBytesRead()).isEqualTo(encodedLength); + } + } + + @Test + void shouldEncodeKnownValuesCorrectly() { + // Test cases with known encodings + assertEncodedBytes(0, 0x00); + assertEncodedBytes(1, 0x01); + assertEncodedBytes(127, 0x7f); + assertEncodedBytes(128, 0x80, 0x01); + assertEncodedBytes(16383, 0xff, 0x7f); + assertEncodedBytes(16384, 0x80, 0x80, 0x01); + } + + private void assertEncodedBytes(int value, int... expectedBytes) { + ByteBuffer buffer = ByteBuffer.allocate(10); + VarInt.encode(value, buffer); + buffer.flip(); + + byte[] actual = new byte[buffer.remaining()]; + buffer.get(actual); + + byte[] expected = new byte[expectedBytes.length]; + for (int i = 0; i < expectedBytes.length; i++) { + expected[i] = (byte) expectedBytes[i]; + } + + assertThat(actual).containsExactly(expected); + } + + @Test + void shouldWorkWithByteBuffer() throws ImprintException { + ByteBuffer buffer = ByteBuffer.allocate(10); + VarInt.encode(16384, buffer); + + buffer.flip(); + VarInt.DecodeResult result = VarInt.decode(buffer); + + assertThat(result.getValue()).isEqualTo(16384); + assertThat(result.getBytesRead()).isEqualTo(3); + } + + @Test + void shouldCalculateEncodedLength() { + assertThat(VarInt.encodedLength(0)).isEqualTo(1); + assertThat(VarInt.encodedLength(127)).isEqualTo(1); + assertThat(VarInt.encodedLength(128)).isEqualTo(2); + assertThat(VarInt.encodedLength(16383)).isEqualTo(2); + assertThat(VarInt.encodedLength(16384)).isEqualTo(3); + assertThat(VarInt.encodedLength(-1)).isEqualTo(5); // max u32 + } + + @Test + void shouldHandleBufferUnderflow() { + ByteBuffer buffer = ByteBuffer.allocate(1); + buffer.put((byte) 0x80); // incomplete varint + buffer.flip(); + + assertThatThrownBy(() -> VarInt.decode(buffer)) + .isInstanceOf(ImprintException.class) + .extracting("errorType") + .isEqualTo(ErrorType.BUFFER_UNDERFLOW); + } + + @Test + void shouldHandleOverlongEncoding() { + ByteBuffer buffer = ByteBuffer.allocate(10); + buffer.put(new byte[]{(byte) 0x80, (byte) 0x80, (byte) 0x80, (byte) 0x80, (byte) 0x80, 0x01}); + buffer.flip(); + + assertThatThrownBy(() -> VarInt.decode(buffer)) + .isInstanceOf(ImprintException.class) + .extracting("errorType") + .isEqualTo(ErrorType.MALFORMED_VARINT); + } + + @Test + void shouldHandleOverflow() { + ByteBuffer buffer = ByteBuffer.allocate(10); + buffer.put(new byte[]{(byte) 0x80, (byte) 0x80, (byte) 0x80, (byte) 0x80, 0x10}); + buffer.flip(); + + assertThatThrownBy(() -> VarInt.decode(buffer)) + .isInstanceOf(ImprintException.class) + .extracting("errorType") + .isEqualTo(ErrorType.MALFORMED_VARINT); + } +} \ No newline at end of file From bce1d13dc70c6b1e95bf93d66548d693ab3073bb Mon Sep 17 00:00:00 2001 From: expand3d <> Date: Sun, 1 Jun 2025 20:33:38 -0400 Subject: [PATCH 04/49] Add GitHub Actions CI workflow for automated testing --- .github/workflows/ci.yml | 95 ++++++++++++++++++++++++++++++++++++++++ build.gradle | 75 +++++++++++++++++++++++++++++++ 2 files changed, 170 insertions(+) create mode 100644 .github/workflows/ci.yml create mode 100644 build.gradle diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..0650d7c --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,95 @@ +name: CI + +on: + push: + branches: [ main, dev ] + pull_request: + branches: [ main, dev ] + +jobs: + test: + runs-on: ubuntu-latest + + strategy: + matrix: + java-version: [11, 17, 21] + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up JDK ${{ matrix.java-version }} + uses: actions/setup-java@v4 + with: + java-version: ${{ matrix.java-version }} + distribution: 'temurin' + + - name: Cache Gradle dependencies + uses: actions/cache@v3 + with: + path: | + ~/.gradle/caches + ~/.gradle/wrapper + key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} + restore-keys: | + ${{ runner.os }}-gradle- + + - name: Make gradlew executable + run: chmod +x ./gradlew + + - name: Run tests + run: ./gradlew test + + - name: Run build + run: ./gradlew build + + - name: Upload test results + uses: actions/upload-artifact@v3 + if: always() + with: + name: test-results-java-${{ matrix.java-version }} + path: build/test-results/test/ + + - name: Upload build reports + uses: actions/upload-artifact@v3 + if: always() + with: + name: build-reports-java-${{ matrix.java-version }} + path: build/reports/ + + benchmark: + runs-on: ubuntu-latest + needs: test + if: github.event_name == 'pull_request' + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up JDK 17 + uses: actions/setup-java@v4 + with: + java-version: 17 + distribution: 'temurin' + + - name: Cache Gradle dependencies + uses: actions/cache@v3 + with: + path: | + ~/.gradle/caches + ~/.gradle/wrapper + key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} + restore-keys: | + ${{ runner.os }}-gradle- + + - name: Make gradlew executable + run: chmod +x ./gradlew + + - name: Run quick benchmark + run: ./gradlew jmh -Pjmh.fork=1 -Pjmh.warmupIterations=1 -Pjmh.iterations=1 + + - name: Upload benchmark results + uses: actions/upload-artifact@v3 + with: + name: benchmark-results + path: build/results/jmh/ \ No newline at end of file diff --git a/build.gradle b/build.gradle new file mode 100644 index 0000000..9262297 --- /dev/null +++ b/build.gradle @@ -0,0 +1,75 @@ +plugins { + id 'java-library' + id 'maven-publish' + id 'me.champeau.jmh' version '0.7.2' +} + +group = 'com.imprint' +version = '0.1.0-SNAPSHOT' + +java { + toolchain { + languageVersion = JavaLanguageVersion.of(11) + } + withJavadocJar() + withSourcesJar() +} + +repositories { + mavenCentral() +} + +dependencies { + // Lombok for reducing boilerplate + compileOnly 'org.projectlombok:lombok:1.18.30' + annotationProcessor 'org.projectlombok:lombok:1.18.30' + + // Test dependencies + testImplementation 'org.junit.jupiter:junit-jupiter:5.10.0' + testImplementation 'org.assertj:assertj-core:3.24.2' + testImplementation 'org.mockito:mockito-core:5.5.0' + + // Lombok for tests + testCompileOnly 'org.projectlombok:lombok:1.18.30' + testAnnotationProcessor 'org.projectlombok:lombok:1.18.30' + + // Performance testing with JMH + jmhImplementation 'org.openjdk.jmh:jmh-core:1.37' + jmhAnnotationProcessor 'org.openjdk.jmh:jmh-generator-annprocess:1.37' + + // Competitor libraries for benchmarking + jmhImplementation 'com.google.protobuf:protobuf-java:3.25.1' + jmhImplementation 'org.apache.avro:avro:1.11.3' + jmhImplementation 'com.fasterxml.jackson.core:jackson-databind:2.16.0' + jmhImplementation 'com.google.flatbuffers:flatbuffers-java:23.5.26' + jmhImplementation 'com.esotericsoftware:kryo:5.4.0' +} + +test { + useJUnitPlatform() + + // Enable detailed test output + testLogging { + events "passed", "skipped", "failed" + } +} + +// JMH configuration +jmh { + fork = 1 + warmupIterations = 3 + iterations = 3 + resultFormat = 'JSON' + includeTests = false +} + +compileJava { + options.compilerArgs << '-Xlint:unchecked' + options.deprecation = true +} + +javadoc { + if(JavaVersion.current().isJava9Compatible()) { + options.addBooleanOption('html5', true) + } +} From 72c468f1e052007e521864f0fc975e62e6c90bb0 Mon Sep 17 00:00:00 2001 From: expand3d <> Date: Sun, 1 Jun 2025 20:36:51 -0400 Subject: [PATCH 05/49] Update GitHub Actions workflow to use upload-artifact@v4 --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0650d7c..6e5a2a0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,14 +44,14 @@ jobs: run: ./gradlew build - name: Upload test results - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: always() with: name: test-results-java-${{ matrix.java-version }} path: build/test-results/test/ - name: Upload build reports - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: always() with: name: build-reports-java-${{ matrix.java-version }} @@ -89,7 +89,7 @@ jobs: run: ./gradlew jmh -Pjmh.fork=1 -Pjmh.warmupIterations=1 -Pjmh.iterations=1 - name: Upload benchmark results - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: benchmark-results path: build/results/jmh/ \ No newline at end of file From 468d68218bdc087c1be39f627fecbe799ae37e3a Mon Sep 17 00:00:00 2001 From: expand3d <> Date: Sun, 1 Jun 2025 20:42:37 -0400 Subject: [PATCH 06/49] Add Gradle wrapper validation to CI workflow --- .github/workflows/ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6e5a2a0..18842f6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,6 +37,9 @@ jobs: - name: Make gradlew executable run: chmod +x ./gradlew + - name: Validate Gradle wrapper + uses: gradle/wrapper-validation-action@v1 + - name: Run tests run: ./gradlew test @@ -85,6 +88,9 @@ jobs: - name: Make gradlew executable run: chmod +x ./gradlew + - name: Validate Gradle wrapper + uses: gradle/wrapper-validation-action@v1 + - name: Run quick benchmark run: ./gradlew jmh -Pjmh.fork=1 -Pjmh.warmupIterations=1 -Pjmh.iterations=1 From cf05b13275ac8878ef47f33713a8c396466c59e2 Mon Sep 17 00:00:00 2001 From: expand3d <> Date: Sun, 1 Jun 2025 20:45:02 -0400 Subject: [PATCH 07/49] Fix gitignore to include gradle-wrapper.jar for CI --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 09ea567..54c84dc 100644 --- a/.gitignore +++ b/.gitignore @@ -61,4 +61,5 @@ buildNumber.properties # JAR files (unless they're dependencies) *.jar +!gradle/wrapper/gradle-wrapper.jar !lombok.jar From d0d798388c77c1f33cacf410218753b179c000e9 Mon Sep 17 00:00:00 2001 From: expand3d <> Date: Sun, 1 Jun 2025 20:48:18 -0400 Subject: [PATCH 08/49] Force add gradle-wrapper.jar to repository --- gradle/wrapper/gradle-wrapper.jar | Bin 0 -> 43705 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 gradle/wrapper/gradle-wrapper.jar diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 0000000000000000000000000000000000000000..9bbc975c742b298b441bfb90dbc124400a3751b9 GIT binary patch literal 43705 zcma&Obx`DOvL%eWOXJW;V64viP??$)@wHcsJ68)>bJS6*&iHnskXE8MjvIPVl|FrmV}Npeql07fCw6`pw`0s zGauF(<*@v{3t!qoUU*=j)6;|-(yg@jvDx&fV^trtZt27?4Tkn729qrItVh@PMwG5$ z+oXHSPM??iHZ!cVP~gYact-CwV`}~Q+R}PPNRy+T-geK+>fHrijpllon_F4N{@b-} z1M0=a!VbVmJM8Xk@NRv)m&aRYN}FSJ{LS;}2ArQ5baSjfy40l@T5)1r-^0fAU6f_} zzScst%$Nd-^ElV~H0TetQhMc%S{}Q4lssln=|;LG?Ulo}*mhg8YvBAUY7YFdXs~vv zv~{duzVw%C#GxkBwX=TYp1Dh*Uaum2?RmsvPaLlzO^fIJ`L?&OV?Y&kKj~^kWC`Ly zfL-}J^4a0Ojuz9O{jUbIS;^JatJ5+YNNHe}6nG9Yd6P-lJiK2ms)A^xq^H2fKrTF) zp!6=`Ece~57>^9(RA4OB9;f1FAhV%zVss%#rDq$9ZW3N2cXC7dMz;|UcRFecBm`DA z1pCO!#6zKp#@mx{2>Qcme8y$Qg_gnA%(`Vtg3ccwgb~D(&@y8#Jg8nNYW*-P{_M#E zZ|wCsQoO1(iIKd-2B9xzI}?l#Q@G5d$m1Lfh0q;iS5FDQ&9_2X-H)VDKA*fa{b(sV zL--krNCXibi1+*C2;4qVjb0KWUVGjjRT{A}Q*!cFmj0tRip2ra>WYJ>ZK4C|V~RYs z6;~+*)5F^x^aQqk9tjh)L;DOLlD8j+0<>kHc8MN|68PxQV`tJFbgxSfq-}b(_h`luA0&;Vk<@51i0 z_cu6{_*=vlvYbKjDawLw+t^H?OV00_73Cn3goU5?})UYFuoSX6Xqw;TKcrsc|r# z$sMWYl@cs#SVopO$hpHZ)cdU-+Ui%z&Sa#lMI~zWW@vE%QDh@bTe0&V9nL>4Et9`N zGT8(X{l@A~loDx}BDz`m6@tLv@$mTlVJ;4MGuj!;9Y=%;;_kj#o8n5tX%@M)2I@}u z_{I!^7N1BxW9`g&Z+K#lZ@7_dXdsqp{W9_`)zgZ=sD~%WS5s$`7z#XR!Lfy(4se(m zR@a3twgMs19!-c4jh`PfpJOSU;vShBKD|I0@rmv_x|+ogqslnLLOepJpPMOxhRb*i zGHkwf#?ylQ@k9QJL?!}MY4i7joSzMcEhrDKJH&?2v{-tgCqJe+Y0njl7HYff z{&~M;JUXVR$qM1FPucIEY(IBAuCHC@^~QG6O!dAjzQBxDOR~lJEr4KS9R*idQ^p{D zS#%NQADGbAH~6wAt}(1=Uff-1O#ITe)31zCL$e9~{w)gx)g>?zFE{Bc9nJT6xR!i8 z)l)~9&~zSZTHk{?iQL^MQo$wLi}`B*qnvUy+Y*jEraZMnEhuj`Fu+>b5xD1_Tp z)8|wedv42#3AZUL7x&G@p@&zcUvPkvg=YJS6?1B7ZEXr4b>M+9Gli$gK-Sgh{O@>q7TUg+H zNJj`6q#O@>4HpPJEHvNij`sYW&u%#=215HKNg;C!0#hH1vlO5+dFq9& zS)8{5_%hz?#D#wn&nm@aB?1_|@kpA@{%jYcs{K%$a4W{k@F zPyTav?jb;F(|GaZhm6&M#g|`ckO+|mCtAU)5_(hn&Ogd z9Ku}orOMu@K^Ac>eRh3+0-y^F`j^noa*OkS3p^tLV`TY$F$cPXZJ48!xz1d7%vfA( zUx2+sDPqHfiD-_wJDb38K^LtpN2B0w=$A10z%F9f_P2aDX63w7zDG5CekVQJGy18I zB!tI`6rZr7TK10L(8bpiaQ>S@b7r_u@lh^vakd0e6USWw7W%d_Ob%M!a`K>#I3r-w zo2^+9Y)Sb?P9)x0iA#^ns+Kp{JFF|$09jb6ZS2}_<-=$?^#IUo5;g`4ICZknr!_aJ zd73%QP^e-$%Xjt|28xM}ftD|V@76V_qvNu#?Mt*A-OV{E4_zC4Ymo|(cb+w^`Wv== z>)c%_U0w`d$^`lZQp@midD89ta_qTJW~5lRrIVwjRG_9aRiQGug%f3p@;*%Y@J5uQ|#dJ+P{Omc`d2VR)DXM*=ukjVqIpkb<9gn9{*+&#p)Ek zN=4zwNWHF~=GqcLkd!q0p(S2_K=Q`$whZ}r@ec_cb9hhg9a z6CE=1n8Q;hC?;ujo0numJBSYY6)GTq^=kB~`-qE*h%*V6-ip=c4+Yqs*7C@@b4YAi zuLjsmD!5M7r7d5ZPe>4$;iv|zq=9=;B$lI|xuAJwi~j~^Wuv!Qj2iEPWjh9Z&#+G>lZQpZ@(xfBrhc{rlLwOC;optJZDj4Xfu3$u6rt_=YY0~lxoy~fq=*L_&RmD7dZWBUmY&12S;(Ui^y zBpHR0?Gk|`U&CooNm_(kkO~pK+cC%uVh^cnNn)MZjF@l{_bvn4`Jc}8QwC5_)k$zs zM2qW1Zda%bIgY^3NcfL)9ug`05r5c%8ck)J6{fluBQhVE>h+IA&Kb}~$55m-^c1S3 zJMXGlOk+01qTQUFlh5Jc3xq|7McY$nCs$5=`8Y;|il#Ypb{O9}GJZD8!kYh{TKqs@ z-mQn1K4q$yGeyMcryHQgD6Ra<6^5V(>6_qg`3uxbl|T&cJVA*M_+OC#>w(xL`RoPQ zf1ZCI3G%;o-x>RzO!mc}K!XX{1rih0$~9XeczHgHdPfL}4IPi~5EV#ZcT9 zdgkB3+NPbybS-d;{8%bZW^U+x@Ak+uw;a5JrZH!WbNvl!b~r4*vs#he^bqz`W93PkZna2oYO9dBrKh2QCWt{dGOw)%Su%1bIjtp4dKjZ^ zWfhb$M0MQiDa4)9rkip9DaH0_tv=XxNm>6MKeWv>`KNk@QVkp$Lhq_~>M6S$oliq2 zU6i7bK;TY)m>-}X7hDTie>cc$J|`*}t=MAMfWIALRh2=O{L57{#fA_9LMnrV(HrN6 zG0K_P5^#$eKt{J|#l~U0WN_3)p^LLY(XEqes0OvI?3)GTNY&S13X+9`6PLVFRf8K) z9x@c|2T72+-KOm|kZ@j4EDDec>03FdgQlJ!&FbUQQH+nU^=U3Jyrgu97&#-W4C*;_ z(WacjhBDp@&Yon<9(BWPb;Q?Kc0gR5ZH~aRNkPAWbDY!FiYVSu!~Ss^9067|JCrZk z-{Rn2KEBR|Wti_iy) zXnh2wiU5Yz2L!W{{_#LwNWXeNPHkF=jjXmHC@n*oiz zIoM~Wvo^T@@t!QQW?Ujql-GBOlnB|HjN@x~K8z)c(X}%%5Zcux09vC8=@tvgY>czq z3D(U&FiETaN9aP}FDP3ZSIXIffq>M3{~eTB{uauL07oYiM=~K(XA{SN!rJLyXeC+Y zOdeebgHOc2aCIgC=8>-Q>zfuXV*=a&gp{l#E@K|{qft@YtO>xaF>O7sZz%8);e86? z+jJlFB{0fu6%8ew^_<+v>>%6eB8|t*_v7gb{x=vLLQYJKo;p7^o9!9A1)fZZ8i#ZU z<|E?bZakjkEV8xGi?n+{Xh3EgFKdM^;4D;5fHmc04PI>6oU>>WuLy6jgpPhf8$K4M zjJo*MbN0rZbZ!5DmoC^@hbqXiP^1l7I5;Wtp2i9Jkh+KtDJoXP0O8qmN;Sp(+%upX zAxXs*qlr(ck+-QG_mMx?hQNXVV~LT{$Q$ShX+&x?Q7v z@8t|UDylH6@RZ?WsMVd3B0z5zf50BP6U<&X_}+y3uJ0c5OD}+J&2T8}A%2Hu#Nt_4 zoOoTI$A!hQ<2pk5wfZDv+7Z{yo+Etqry=$!*pvYyS+kA4xnJ~3b~TBmA8Qd){w_bE zqDaLIjnU8m$wG#&T!}{e0qmHHipA{$j`%KN{&#_Kmjd&#X-hQN+ju$5Ms$iHj4r?) z&5m8tI}L$ih&95AjQ9EDfPKSmMj-@j?Q+h~C3<|Lg2zVtfKz=ft{YaQ1i6Om&EMll zzov%MsjSg=u^%EfnO+W}@)O6u0LwoX709h3Cxdc2Rwgjd%LLTChQvHZ+y<1q6kbJXj3_pq1&MBE{8 zd;aFotyW>4WHB{JSD8Z9M@jBitC1RF;!B8;Rf-B4nOiVbGlh9w51(8WjL&e{_iXN( zAvuMDIm_>L?rJPxc>S`bqC|W$njA0MKWa?V$u6mN@PLKYqak!bR!b%c^ze(M`ec(x zv500337YCT4gO3+9>oVIJLv$pkf`01S(DUM+4u!HQob|IFHJHm#>eb#eB1X5;bMc| z>QA4Zv}$S?fWg~31?Lr(C>MKhZg>gplRm`2WZ--iw%&&YlneQYY|PXl;_4*>vkp;I z$VYTZq|B*(3(y17#@ud@o)XUZPYN*rStQg5U1Sm2gM}7hf_G<>*T%6ebK*tF(kbJc zNPH4*xMnJNgw!ff{YXrhL&V$6`ylY={qT_xg9znQWw9>PlG~IbhnpsG_94Kk_(V-o&v7#F znra%uD-}KOX2dkak**hJnZZQyp#ERyyV^lNe!Qrg=VHiyr7*%j#PMvZMuYNE8o;JM zGrnDWmGGy)(UX{rLzJ*QEBd(VwMBXnJ@>*F8eOFy|FK*Vi0tYDw;#E zu#6eS;%Nm2KY+7dHGT3m{TM7sl=z8|V0e!DzEkY-RG8vTWDdSQFE|?+&FYA146@|y zV(JP>LWL;TSL6rao@W5fWqM1-xr$gRci#RQV2DX-x4@`w{uEUgoH4G|`J%H!N?*Qn zy~rjzuf(E7E!A9R2bSF|{{U(zO+;e29K_dGmC^p7MCP!=Bzq@}&AdF5=rtCwka zTT1A?5o}i*sXCsRXBt)`?nOL$zxuP3i*rm3Gmbmr6}9HCLvL*45d|(zP;q&(v%}S5yBmRVdYQQ24zh z6qL2<2>StU$_Ft29IyF!6=!@;tW=o8vNzVy*hh}XhZhUbxa&;9~woye<_YmkUZ)S?PW{7t; zmr%({tBlRLx=ffLd60`e{PQR3NUniWN2W^~7Sy~MPJ>A#!6PLnlw7O0(`=PgA}JLZ ztqhiNcKvobCcBel2 z-N82?4-()eGOisnWcQ9Wp23|ybG?*g!2j#>m3~0__IX1o%dG4b;VF@^B+mRgKx|ij zWr5G4jiRy}5n*(qu!W`y54Y*t8g`$YrjSunUmOsqykYB4-D(*(A~?QpuFWh;)A;5= zPl|=x+-w&H9B7EZGjUMqXT}MkcSfF}bHeRFLttu!vHD{Aq)3HVhvtZY^&-lxYb2%` zDXk7>V#WzPfJs6u{?ZhXpsMdm3kZscOc<^P&e&684Rc1-d=+=VOB)NR;{?0NjTl~D z1MXak$#X4{VNJyD$b;U~Q@;zlGoPc@ny!u7Pe;N2l4;i8Q=8>R3H{>HU(z z%hV2?rSinAg6&wuv1DmXok`5@a3@H0BrqsF~L$pRYHNEXXuRIWom0l zR9hrZpn1LoYc+G@q@VsFyMDNX;>_Vf%4>6$Y@j;KSK#g)TZRmjJxB!_NmUMTY(cAV zmewn7H{z`M3^Z& z2O$pWlDuZHAQJ{xjA}B;fuojAj8WxhO}_9>qd0|p0nBXS6IIRMX|8Qa!YDD{9NYYK z%JZrk2!Ss(Ra@NRW<7U#%8SZdWMFDU@;q<}%F{|6n#Y|?FaBgV$7!@|=NSVoxlJI4G-G(rn}bh|?mKkaBF$-Yr zA;t0r?^5Nz;u6gwxURapQ0$(-su(S+24Ffmx-aP(@8d>GhMtC5x*iEXIKthE*mk$` zOj!Uri|EAb4>03C1xaC#(q_I<;t}U7;1JqISVHz3tO{) zD(Yu@=>I9FDmDtUiWt81;BeaU{_=es^#QI7>uYl@e$$lGeZ~Q(f$?^3>$<<{n`Bn$ zn8bamZlL@6r^RZHV_c5WV7m2(G6X|OI!+04eAnNA5=0v1Z3lxml2#p~Zo57ri;4>;#16sSXXEK#QlH>=b$inEH0`G#<_ zvp;{+iY)BgX$R!`HmB{S&1TrS=V;*5SB$7*&%4rf_2wQS2ed2E%Wtz@y$4ecq4w<) z-?1vz_&u>s?BMrCQG6t9;t&gvYz;@K@$k!Zi=`tgpw*v-#U1Pxy%S9%52`uf$XMv~ zU}7FR5L4F<#9i%$P=t29nX9VBVv)-y7S$ZW;gmMVBvT$BT8d}B#XV^@;wXErJ-W2A zA=JftQRL>vNO(!n4mcd3O27bHYZD!a0kI)6b4hzzL9)l-OqWn)a~{VP;=Uo|D~?AY z#8grAAASNOkFMbRDdlqVUfB;GIS-B-_YXNlT_8~a|LvRMVXf!<^uy;)d$^OR(u)!) zHHH=FqJF-*BXif9uP~`SXlt0pYx|W&7jQnCbjy|8b-i>NWb@!6bx;1L&$v&+!%9BZ z0nN-l`&}xvv|wwxmC-ZmoFT_B#BzgQZxtm|4N+|;+(YW&Jtj^g!)iqPG++Z%x0LmqnF875%Ry&2QcCamx!T@FgE@H zN39P6e#I5y6Yl&K4eUP{^biV`u9{&CiCG#U6xgGRQr)zew;Z%x+ z-gC>y%gvx|dM=OrO`N@P+h2klPtbYvjS!mNnk4yE0+I&YrSRi?F^plh}hIp_+OKd#o7ID;b;%*c0ES z!J))9D&YufGIvNVwT|qsGWiZAwFODugFQ$VsNS%gMi8OJ#i${a4!E3<-4Jj<9SdSY z&xe|D0V1c`dZv+$8>(}RE|zL{E3 z-$5Anhp#7}oO(xm#}tF+W=KE*3(xxKxhBt-uuJP}`_K#0A< zE%rhMg?=b$ot^i@BhE3&)bNBpt1V*O`g?8hhcsV-n#=|9wGCOYt8`^#T&H7{U`yt2 z{l9Xl5CVsE=`)w4A^%PbIR6uG_5Ww9k`=q<@t9Bu662;o{8PTjDBzzbY#tL;$wrpjONqZ{^Ds4oanFm~uyPm#y1Ll3(H57YDWk9TlC zq;kebC!e=`FU&q2ojmz~GeLxaJHfs0#F%c(i+~gg$#$XOHIi@1mA72g2pFEdZSvp}m0zgQb5u2?tSRp#oo!bp`FP}< zaK4iuMpH+Jg{bb7n9N6eR*NZfgL7QiLxI zk6{uKr>xxJ42sR%bJ%m8QgrL|fzo9@?9eQiMW8O`j3teoO_R8cXPe_XiLnlYkE3U4 zN!^F)Z4ZWcA8gekEPLtFqX-Q~)te`LZnJK_pgdKs)Dp50 zdUq)JjlJeELskKg^6KY!sIou-HUnSFRsqG^lsHuRs`Z{f(Ti9eyd3cwu*Kxp?Ws7l z3cN>hGPXTnQK@qBgqz(n*qdJ2wbafELi?b90fK~+#XIkFGU4+HihnWq;{{)1J zv*Txl@GlnIMOjzjA1z%g?GsB2(6Zb-8fooT*8b0KF2CdsIw}~Hir$d3TdVHRx1m3c z4C3#h@1Xi@{t4zge-#B6jo*ChO%s-R%+9%-E|y<*4;L>$766RiygaLR?X%izyqMXA zb|N=Z-0PSFeH;W6aQ3(5VZWVC>5Ibgi&cj*c%_3=o#VyUJv* zM&bjyFOzlaFq;ZW(q?|yyi|_zS%oIuH^T*MZ6NNXBj;&yM3eQ7!CqXY?`7+*+GN47 zNR#%*ZH<^x{(0@hS8l{seisY~IE*)BD+R6^OJX}<2HRzo^fC$n>#yTOAZbk4%=Bei=JEe=o$jm`or0YDw*G?d> z=i$eEL7^}_?UI^9$;1Tn9b>$KOM@NAnvWrcru)r`?LodV%lz55O3y(%FqN;cKgj7t zlJ7BmLTQ*NDX#uelGbCY>k+&H*iSK?x-{w;f5G%%!^e4QT9z<_0vHbXW^MLR} zeC*jezrU|{*_F`I0mi)9=sUj^G03i@MjXx@ePv@(Udt2CCXVOJhRh4yp~fpn>ssHZ z?k(C>2uOMWKW5FVsBo#Nk!oqYbL`?#i~#!{3w^qmCto05uS|hKkT+iPrC-}hU_nbL zO622#mJupB21nChpime}&M1+whF2XM?prT-Vv)|EjWYK(yGYwJLRRMCkx;nMSpu?0 zNwa*{0n+Yg6=SR3-S&;vq=-lRqN`s9~#)OOaIcy3GZ&~l4g@2h| zThAN#=dh{3UN7Xil;nb8@%)wx5t!l z0RSe_yJQ+_y#qEYy$B)m2yDlul^|m9V2Ia$1CKi6Q19~GTbzqk*{y4;ew=_B4V8zw zScDH&QedBl&M*-S+bH}@IZUSkUfleyM45G>CnYY{hx8J9q}ME?Iv%XK`#DJRNmAYt zk2uY?A*uyBA=nlYjkcNPMGi*552=*Q>%l?gDK_XYh*Rya_c)ve{=ps`QYE0n!n!)_$TrGi_}J|>1v}(VE7I~aP-wns#?>Y zu+O7`5kq32zM4mAQpJ50vJsUDT_^s&^k-llQMy9!@wRnxw@~kXV6{;z_wLu3i=F3m z&eVsJmuauY)8(<=pNUM5!!fQ4uA6hBkJoElL1asWNkYE#qaP?a+biwWw~vB48PRS7 zY;DSHvgbIB$)!uJU)xA!yLE*kP0owzYo`v@wfdux#~f!dv#uNc_$SF@Qq9#3q5R zfuQnPPN_(z;#X#nRHTV>TWL_Q%}5N-a=PhkQ^GL+$=QYfoDr2JO-zo#j;mCsZVUQ) zJ96e^OqdLW6b-T@CW@eQg)EgIS9*k`xr$1yDa1NWqQ|gF^2pn#dP}3NjfRYx$pTrb zwGrf8=bQAjXx*8?du*?rlH2x~^pXjiEmj^XwQo{`NMonBN=Q@Y21!H)D( zA~%|VhiTjaRQ%|#Q9d*K4j~JDXOa4wmHb0L)hn*;Eq#*GI}@#ux4}bt+olS(M4$>c z=v8x74V_5~xH$sP+LZCTrMxi)VC%(Dg!2)KvW|Wwj@pwmH6%8zd*x0rUUe$e(Z%AW z@Q{4LL9#(A-9QaY2*+q8Yq2P`pbk3!V3mJkh3uH~uN)+p?67d(r|Vo0CebgR#u}i? zBxa^w%U|7QytN%L9bKaeYhwdg7(z=AoMeP0)M3XZA)NnyqL%D_x-(jXp&tp*`%Qsx z6}=lGr;^m1<{;e=QQZ!FNxvLcvJVGPkJ63at5%*`W?46!6|5FHYV0qhizSMT>Zoe8 zsJ48kb2@=*txGRe;?~KhZgr-ZZ&c0rNV7eK+h$I-UvQ=552@psVrvj#Ys@EU4p8`3 zsNqJu-o=#@9N!Pq`}<=|((u)>^r0k^*%r<{YTMm+mOPL>EoSREuQc-e2~C#ZQ&Xve zZ}OUzmE4{N-7cqhJiUoO_V#(nHX11fdfVZJT>|6CJGX5RQ+Ng$Nq9xs-C86-)~`>p zW--X53J`O~vS{WWjsAuGq{K#8f#2iz` zzSSNIf6;?5sXrHig%X(}0q^Y=eYwvh{TWK-fT>($8Ex>!vo_oGFw#ncr{vmERi^m7lRi%8Imph})ZopLoIWt*eFWSPuBK zu>;Pu2B#+e_W|IZ0_Q9E9(s@0>C*1ft`V{*UWz^K<0Ispxi@4umgGXW!j%7n+NC~* zBDhZ~k6sS44(G}*zg||X#9Weto;u*Ty;fP!+v*7be%cYG|yEOBomch#m8Np!Sw`L)q+T` zmrTMf2^}7j=RPwgpO9@eXfb{Q>GW#{X=+xt`AwTl!=TgYm)aS2x5*`FSUaaP_I{Xi zA#irF%G33Bw>t?^1YqX%czv|JF0+@Pzi%!KJ?z!u$A`Catug*tYPO`_Zho5iip0@! z;`rR0-|Ao!YUO3yaujlSQ+j-@*{m9dHLtve!sY1Xq_T2L3&=8N;n!!Eb8P0Z^p4PL zQDdZ?An2uzbIakOpC|d@=xEA}v-srucnX3Ym{~I#Ghl~JZU(a~Ppo9Gy1oZH&Wh%y zI=KH_s!Lm%lAY&`_KGm*Ht)j*C{-t}Nn71drvS!o|I|g>ZKjE3&Mq0TCs6}W;p>%M zQ(e!h*U~b;rsZ1OPigud>ej=&hRzs@b>>sq6@Yjhnw?M26YLnDH_Wt#*7S$-BtL08 zVyIKBm$}^vp?ILpIJetMkW1VtIc&7P3z0M|{y5gA!Yi5x4}UNz5C0Wdh02!h zNS>923}vrkzl07CX`hi)nj-B?#n?BJ2Vk0zOGsF<~{Fo7OMCN_85daxhk*pO}x_8;-h>}pcw26V6CqR-=x2vRL?GB#y%tYqi;J}kvxaz}*iFO6YO0ha6!fHU9#UI2Nv z_(`F#QU1B+P;E!t#Lb)^KaQYYSewj4L!_w$RH%@IL-M($?DV@lGj%3ZgVdHe^q>n(x zyd5PDpGbvR-&p*eU9$#e5#g3-W_Z@loCSz}f~{94>k6VRG`e5lI=SE0AJ7Z_+=nnE zTuHEW)W|a8{fJS>2TaX zuRoa=LCP~kP)kx4L+OqTjtJOtXiF=y;*eUFgCn^Y@`gtyp?n14PvWF=zhNGGsM{R- z^DsGxtoDtx+g^hZi@E2Y(msb-hm{dWiHdoQvdX88EdM>^DS#f}&kCGpPFDu*KjEpv$FZtLpeT>@)mf|z#ZWEsueeW~hF78Hu zfY9a+Gp?<)s{Poh_qdcSATV2oZJo$OH~K@QzE2kCADZ@xX(; z)0i=kcAi%nvlsYagvUp(z0>3`39iKG9WBDu3z)h38p|hLGdD+Khk394PF3qkX!02H z#rNE`T~P9vwNQ_pNe0toMCRCBHuJUmNUl)KFn6Gu2je+p>{<9^oZ4Gfb!)rLZ3CR3 z-o&b;Bh>51JOt=)$-9+Z!P}c@cKev_4F1ZZGs$I(A{*PoK!6j@ZJrAt zv2LxN#p1z2_0Ox|Q8PVblp9N${kXkpsNVa^tNWhof)8x8&VxywcJz#7&P&d8vvxn` zt75mu>yV=Dl#SuiV!^1BPh5R)`}k@Nr2+s8VGp?%Le>+fa{3&(XYi~{k{ z-u4#CgYIdhp~GxLC+_wT%I*)tm4=w;ErgmAt<5i6c~)7JD2olIaK8by{u-!tZWT#RQddptXRfEZxmfpt|@bs<*uh?Y_< zD>W09Iy4iM@@80&!e^~gj!N`3lZwosC!!ydvJtc0nH==K)v#ta_I}4Tar|;TLb|+) zSF(;=?$Z0?ZFdG6>Qz)6oPM}y1&zx_Mf`A&chb znSERvt9%wdPDBIU(07X+CY74u`J{@SSgesGy~)!Mqr#yV6$=w-dO;C`JDmv=YciTH zvcrN1kVvq|(3O)NNdth>X?ftc`W2X|FGnWV%s})+uV*bw>aoJ#0|$pIqK6K0Lw!@- z3pkPbzd`ljS=H2Bt0NYe)u+%kU%DWwWa>^vKo=lzDZHr>ruL5Ky&#q7davj-_$C6J z>V8D-XJ}0cL$8}Xud{T_{19#W5y}D9HT~$&YY-@=Th219U+#nT{tu=d|B)3K`pL53 zf7`I*|L@^dPEIDJkI3_oA9vsH7n7O}JaR{G~8 zfi$?kmKvu20(l`dV7=0S43VwVKvtF!7njv1Q{Ju#ysj=|dASq&iTE8ZTbd-iiu|2& zmll%Ee1|M?n9pf~?_tdQ<7%JA53!ulo1b^h#s|Su2S4r{TH7BRB3iIOiX5|vc^;5( zKfE1+ah18YA9o1EPT(AhBtve5(%GMbspXV)|1wf5VdvzeYt8GVGt0e*3|ELBhwRaO zE|yMhl;Bm?8Ju3-;DNnxM3Roelg`^!S%e({t)jvYtJCKPqN`LmMg^V&S z$9OIFLF$%Py~{l?#ReyMzpWixvm(n(Y^Am*#>atEZ8#YD&?>NUU=zLxOdSh0m6mL? z_twklB0SjM!3+7U^>-vV=KyQZI-6<(EZiwmNBzGy;Sjc#hQk%D;bay$v#zczt%mFCHL*817X4R;E$~N5(N$1Tv{VZh7d4mhu?HgkE>O+^-C*R@ zR0ima8PsEV*WFvz`NaB+lhX3&LUZcWWJJrG7ZjQrOWD%_jxv=)`cbCk zMgelcftZ%1-p9u!I-Zf_LLz{hcn5NRbxkWby@sj2XmYfAV?iw^0?hM<$&ZDctdC`; zsL|C-7d;w$z2Gt0@hsltNlytoPnK&$>ksr(=>!7}Vk#;)Hp)LuA7(2(Hh(y3LcxRY zim!`~j6`~B+sRBv4 z<#B{@38kH;sLB4eH2+8IPWklhd25r5j2VR}YK$lpZ%7eVF5CBr#~=kUp`i zlb+>Z%i%BJH}5dmfg1>h7U5Q(-F{1d=aHDbMv9TugohX5lq#szPAvPE|HaokMQIi_ zTcTNsO53(oX=hg2w!XA&+qP}nwr$(C)pgG8emS@Mf7m0&*kiA!wPLS`88c=aD$niJ zp?3j%NI^uy|5*MzF`k4hFbsyQZ@wu!*IY+U&&9PwumdmyfL(S0#!2RFfmtzD3m9V7 zsNOw9RQofl-XBfKBF^~~{oUVouka#r3EqRf=SnleD=r1Hm@~`y8U7R)w16fgHvK-6?-TFth)f3WlklbZh+}0 zx*}7oDF4U^1tX4^$qd%987I}g;+o0*$Gsd=J>~Uae~XY6UtbdF)J8TzJXoSrqHVC) zJ@pMgE#;zmuz?N2MIC+{&)tx=7A%$yq-{GAzyz zLzZLf=%2Jqy8wGHD;>^x57VG)sDZxU+EMfe0L{@1DtxrFOp)=zKY1i%HUf~Dro#8} zUw_Mj10K7iDsX}+fThqhb@&GI7PwONx!5z;`yLmB_92z0sBd#HiqTzDvAsTdx+%W{ z2YL#U=9r!@3pNXMp_nvximh+@HV3psUaVa-lOBekVuMf1RUd26~P*|MLouQrb}XM-bEw(UgQxMI6M&l3Nha z{MBcV=tl(b_4}oFdAo}WX$~$Mj-z70FowdoB{TN|h2BdYs?$imcj{IQpEf9q z)rzpttc0?iwopSmEoB&V!1aoZqEWEeO-MKMx(4iK7&Fhc(94c zdy}SOnSCOHX+A8q@i>gB@mQ~Anv|yiUsW!bO9hb&5JqTfDit9X6xDEz*mQEiNu$ay zwqkTV%WLat|Ar+xCOfYs0UQNM`sdsnn*zJr>5T=qOU4#Z(d90!IL76DaHIZeWKyE1 zqwN%9+~lPf2d7)vN2*Q?En?DEPcM+GQwvA<#;X3v=fqsxmjYtLJpc3)A8~*g(KqFx zZEnqqruFDnEagXUM>TC7ngwKMjc2Gx%#Ll#=N4qkOuK|;>4%=0Xl7k`E69@QJ-*Vq zk9p5!+Ek#bjuPa<@Xv7ku4uiWo|_wy)6tIr`aO!)h>m5zaMS-@{HGIXJ0UilA7*I} z?|NZ!Tp8@o-lnyde*H+@8IHME8VTQOGh96&XX3E+}OB zA>VLAGW+urF&J{H{9Gj3&u+Gyn?JAVW84_XBeGs1;mm?2SQm9^!3UE@(_FiMwgkJI zZ*caE={wMm`7>9R?z3Ewg!{PdFDrbzCmz=RF<@(yQJ_A6?PCd_MdUf5vv6G#9Mf)i#G z($OxDT~8RNZ>1R-vw|nN699a}MQN4gJE_9gA-0%>a?Q<9;f3ymgoi$OI!=aE6Elw z2I`l!qe-1J$T$X&x9Zz#;3!P$I);jdOgYY1nqny-k=4|Q4F!mkqACSN`blRji>z1` zc8M57`~1lgL+Ha%@V9_G($HFBXH%k;Swyr>EsQvg%6rNi){Tr&+NAMga2;@85531V z_h+h{jdB&-l+%aY{$oy2hQfx`d{&?#psJ78iXrhrO)McOFt-o80(W^LKM{Zw93O}m z;}G!51qE?hi=Gk2VRUL2kYOBRuAzktql%_KYF4>944&lJKfbr+uo@)hklCHkC=i)E zE*%WbWr@9zoNjumq|kT<9Hm*%&ahcQ)|TCjp@uymEU!&mqqgS;d|v)QlBsE0Jw|+^ zFi9xty2hOk?rlGYT3)Q7i4k65@$RJ-d<38o<`}3KsOR}t8sAShiVWevR8z^Si4>dS z)$&ILfZ9?H#H&lumngpj7`|rKQQ`|tmMmFR+y-9PP`;-425w+#PRKKnx7o-Rw8;}*Ctyw zKh~1oJ5+0hNZ79!1fb(t7IqD8*O1I_hM;o*V~vd_LKqu7c_thyLalEF8Y3oAV=ODv z$F_m(Z>ucO(@?+g_vZ`S9+=~Msu6W-V5I-V6h7->50nQ@+TELlpl{SIfYYNvS6T6D z`9cq=at#zEZUmTfTiM3*vUamr!OB~g$#?9$&QiwDMbSaEmciWf3O2E8?oE0ApScg38hb&iN%K+kvRt#d))-tr^ zD+%!d`i!OOE3in0Q_HzNXE!JcZ<0;cu6P_@;_TIyMZ@Wv!J z)HSXAYKE%-oBk`Ye@W3ShYu-bfCAZ}1|J16hFnLy z?Bmg2_kLhlZ*?`5R8(1%Y?{O?xT)IMv{-)VWa9#1pKH|oVRm4!lLmls=u}Lxs44@g^Zwa0Z_h>Rk<(_mHN47=Id4oba zQ-=qXGz^cNX(b*=NT0<^23+hpS&#OXzzVO@$Z2)D`@oS=#(s+eQ@+FSQcpXD@9npp zlxNC&q-PFU6|!;RiM`?o&Sj&)<4xG3#ozRyQxcW4=EE;E)wcZ&zUG*5elg;{9!j}I z9slay#_bb<)N!IKO16`n3^@w=Y%duKA-{8q``*!w9SW|SRbxcNl50{k&CsV@b`5Xg zWGZ1lX)zs_M65Yt&lO%mG0^IFxzE_CL_6$rDFc&#xX5EXEKbV8E2FOAt>Ka@e0aHQ zMBf>J$FLrCGL@$VgPKSbRkkqo>sOXmU!Yx+Dp7E3SRfT`v~!mjU3qj-*!!YjgI*^) z+*05x78FVnVwSGKr^A|FW*0B|HYgc{c;e3Ld}z4rMI7hVBKaiJRL_e$rxDW^8!nGLdJ<7ex9dFoyj|EkODflJ#Xl`j&bTO%=$v)c+gJsLK_%H3}A_} z6%rfG?a7+k7Bl(HW;wQ7BwY=YFMSR3J43?!;#~E&)-RV_L!|S%XEPYl&#`s!LcF>l zn&K8eemu&CJp2hOHJKaYU#hxEutr+O161ze&=j3w12)UKS%+LAwbjqR8sDoZHnD=m0(p62!zg zxt!Sj65S?6WPmm zL&U9c`6G}T`irf=NcOiZ!V)qhnvMNOPjVkyO2^CGJ+dKTnNAPa?!AxZEpO7yL_LkB zWpolpaDfSaO-&Uv=dj7`03^BT3_HJOAjn~X;wz-}03kNs@D^()_{*BD|0mII!J>5p z1h06PTyM#3BWzAz1FPewjtrQfvecWhkRR=^gKeFDe$rmaYAo!np6iuio3>$w?az$E zwGH|zy@OgvuXok}C)o1_&N6B3P7ZX&-yimXc1hAbXr!K&vclCL%hjVF$yHpK6i_Wa z*CMg1RAH1(EuuA01@lA$sMfe*s@9- z$jNWqM;a%d3?(>Hzp*MiOUM*?8eJ$=(0fYFis!YA;0m8s^Q=M0Hx4ai3eLn%CBm14 zOb8lfI!^UAu_RkuHmKA-8gx8Z;##oCpZV{{NlNSe<i;9!MfIN!&;JI-{|n{(A19|s z9oiGesENcLf@NN^9R0uIrgg(46r%kjR{0SbnjBqPq()wDJ@LC2{kUu_j$VR=l`#RdaRe zxx;b7bu+@IntWaV$si1_nrQpo*IWGLBhhMS13qH zTy4NpK<-3aVc;M)5v(8JeksSAGQJ%6(PXGnQ-g^GQPh|xCop?zVXlFz>42%rbP@jg z)n)% zM9anq5(R=uo4tq~W7wES$g|Ko z1iNIw@-{x@xKxSXAuTx@SEcw(%E49+JJCpT(y=d+n9PO0Gv1SmHkYbcxPgDHF}4iY zkXU4rkqkwVBz<{mcv~A0K|{zpX}aJcty9s(u-$je2&=1u(e#Q~UA{gA!f;0EAaDzdQ=}x7g(9gWrWYe~ zV98=VkHbI!5Rr;+SM;*#tOgYNlfr7;nLU~MD^jSdSpn@gYOa$TQPv+e8DyJ&>aInB zDk>JmjH=}<4H4N4z&QeFx>1VPY8GU&^1c&71T*@2#dINft%ibtY(bAm%<2YwPL?J0Mt{ z7l7BR718o5=v|jB!<7PDBafdL>?cCdVmKC;)MCOobo5edt%RTWiReAMaIU5X9h`@El0sR&Z z7Ed+FiyA+QAyWn zf7=%(8XpcS*C4^-L24TBUu%0;@s!Nzy{e95qjgkzElf0#ou`sYng<}wG1M|L? zKl6ITA1X9mt6o@S(#R3B{uwJI8O$&<3{+A?T~t>Kapx6#QJDol6%?i-{b1aRu?&9B z*W@$T*o&IQ&5Kc*4LK_)MK-f&Ys^OJ9FfE?0SDbAPd(RB)Oju#S(LK)?EVandS1qb#KR;OP|86J?;TqI%E8`vszd&-kS%&~;1Als=NaLzRNnj4q=+ zu5H#z)BDKHo1EJTC?Cd_oq0qEqNAF8PwU7fK!-WwVEp4~4g z3SEmE3-$ddli))xY9KN$lxEIfyLzup@utHn=Q{OCoz9?>u%L^JjClW$M8OB`txg4r6Q-6UlVx3tR%%Z!VMb6#|BKRL`I))#g zij8#9gk|p&Iwv+4s+=XRDW7VQrI(+9>DikEq!_6vIX8$>poDjSYIPcju%=qluSS&j zI-~+ztl1f71O-B+s7Hf>AZ#}DNSf`7C7*)%(Xzf|ps6Dr7IOGSR417xsU=Rxb z1pgk9vv${17h7mZ{)*R{mc%R=!i}8EFV9pl8V=nXCZruBff`$cqN3tpB&RK^$yH!A8RL zJ5KltH$&5%xC7pLZD}6wjD2-uq3&XL8CM$@V9jqalF{mvZ)c4Vn?xXbvkB(q%xbSdjoXJXanVN@I;8I`)XlBX@6BjuQKD28Jrg05} z^ImmK-Ux*QMn_A|1ionE#AurP8Vi?x)7jG?v#YyVe_9^up@6^t_Zy^T1yKW*t* z&Z0+0Eo(==98ig=^`he&G^K$I!F~1l~gq}%o5#pR6?T+ zLmZu&_ekx%^nys<^tC@)s$kD`^r8)1^tUazRkWEYPw0P)=%cqnyeFo3nW zyV$^0DXPKn5^QiOtOi4MIX^#3wBPJjenU#2OIAgCHPKXv$OY=e;yf7+_vI7KcjKq% z?RVzC24ekYp2lEhIE^J$l&wNX0<}1Poir8PjM`m#zwk-AL0w6WvltT}*JN8WFmtP_ z6#rK7$6S!nS!}PSFTG6AF7giGJw5%A%14ECde3x95(%>&W3zUF!8x5%*h-zk8b@Bz zh`7@ixoCVCZ&$$*YUJpur90Yg0X-P82>c~NMzDy7@Ed|6(#`;{)%t7#Yb>*DBiXC3 zUFq(UDFjrgOsc%0KJ_L;WQKF0q!MINpQzSsqwv?#Wg+-NO; z84#4nk$+3C{2f#}TrRhin=Erdfs77TqBSvmxm0P?01Tn@V(}gI_ltHRzQKPyvQ2=M zX#i1-a(>FPaESNx+wZ6J{^m_q3i})1n~JG80c<%-Ky!ZdTs8cn{qWY%x%X^27-Or_ z`KjiUE$OG9K4lWS16+?aak__C*)XA{ z6HmS*8#t_3dl}4;7ZZgn4|Tyy1lOEM1~6Qgl(|BgfQF{Mfjktch zB5kc~4NeehRYO%)3Z!FFHhUVVcV@uEX$eft5Qn&V3g;}hScW_d)K_h5i)vxjKCxcf zL>XlZ^*pQNuX*RJQn)b6;blT3<7@Ap)55)aK3n-H08GIx65W zO9B%gE%`!fyT`)hKjm-&=on)l&!i-QH+mXQ&lbXg0d|F{Ac#U;6b$pqQcpqWSgAPo zmr$gOoE*0r#7J=cu1$5YZE%uylM!i3L{;GW{ae9uy)+EaV>GqW6QJ)*B2)-W`|kLL z)EeeBtpgm;79U_1;Ni5!c^0RbG8yZ0W98JiG~TC8rjFRjGc6Zi8BtoC);q1@8h7UV zFa&LRzYsq%6d!o5-yrqyjXi>jg&c8bu}{Bz9F2D(B%nnuVAz74zmBGv)PAdFXS2(A z=Z?uupM2f-ar0!A)C6l2o8a|+uT*~huH)!h3i!&$ zr>76mt|lwexD(W_+5R{e@2SwR15lGxsnEy|gbS-s5?U}l*kcfQlfnQKo5=LZXizrL zM=0ty+$#f_qGGri-*t@LfGS?%7&LigUIU#JXvwEdJZvIgPCWFBTPT`@Re5z%%tRDO zkMlJCoqf2A=hkU7Ih=IxmPF~fEL90)u76nfFRQwe{m7b&Ww$pnk~$4Lx#s9|($Cvt ze|p{Xozhb^g1MNh-PqS_dLY|Fex4|rhM#lmzq&mhebD$5P>M$eqLoV|z=VQY{)7&sR#tW zl(S1i!!Rrg7kv+V@EL51PGpm511he%MbX2-Jl+DtyYA(0gZyZQjPZP@`SAH{n&25@ zd)emg(p2T3$A!Nmzo|%=z%AhLX)W4hsZNFhmd4<1l6?b3&Fg)G(Zh%J{Cf8Q;?_++ zgO7O<(-)H|Es@QqUgcXNJEfC-BCB~#dhi6ADVZtL!)Mx|u7>ukD052z!QZ5UC-+rd zYXWNRpCmdM{&?M9OMa;OiN{Y#0+F>lBQ=W@M;OXq;-7v3niC$pM8p!agNmq7F04;| z@s-_98JJB&s`Pr6o$KZ=8}qO*7m6SMp7kVmmh$jfnG{r@O(auI7Z^jj!x}NTLS9>k zdo}&Qc2m4Ws3)5qFw#<$h=g%+QUKiYog33bE)e4*H~6tfd42q+|FT5+vmr6Y$6HGC zV!!q>B`1Ho|6E|D<2tYE;4`8WRfm2#AVBBn%_W)mi(~x@g;uyQV3_)~!#A6kmFy0p zY~#!R1%h5E{5;rehP%-#kjMLt*{g((o@0-9*8lKVu+t~CtnOxuaMgo2ssI6@kX09{ zkn~q8Gx<6T)l}7tWYS#q0&~x|-3ho@l}qIr79qOJQcm&Kfr7H54=BQto0)vd1A_*V z)8b2{xa5O^u95~TS=HcJF5b9gMV%&M6uaj<>E zPNM~qGjJ~xbg%QTy#(hPtfc46^nN=Y_GmPYY_hTL{q`W3NedZyRL^kgU@Q$_KMAjEzz*eip`3u6AhPDcWXzR=Io5EtZRPme>#K9 z4lN&87i%YYjoCKN_z9YK+{fJu{yrriba#oGM|2l$ir017UH86Eoig3x+;bz32R*;n zt)Eyg#PhQbbGr^naCv0?H<=@+Poz)Xw*3Gn00qdSL|zGiyYKOA0CP%qk=rBAlt~hr zEvd3Z4nfW%g|c`_sfK$z8fWsXTQm@@eI-FpLGrW<^PIjYw)XC-xFk+M<6>MfG;WJr zuN}7b;p^`uc0j(73^=XJcw;|D4B(`)Flm|qEbB?>qBBv2V?`mWA?Q3yRdLkK7b}y& z+!3!JBI{+&`~;%Pj#n&&y+<;IQzw5SvqlbC+V=kLZLAHOQb zS{{8E&JXy1p|B&$K!T*GKtSV^{|Uk;`oE*F;?@q1dX|>|KWb@|Dy*lbGV0Gx;gpA$ z*N16`v*gQ?6Skw(f^|SL;;^ox6jf2AQ$Zl?gvEV&H|-ep*hIS@0TmGu1X1ZmEPY&f zKCrV{UgRAiNU*=+Uw%gjIQhTAC@67m)6(_D+N>)(^gK74F%M2NUpWpho}aq|Kxh$3 zz#DWOmQV4Lg&}`XTU41Z|P~5;wN2c?2L{a=)Xi~!m#*=22c~&AW zgG#yc!_p##fI&E{xQD9l#^x|9`wSyCMxXe<3^kDIkS0N>=oAz7b`@M>aT?e$IGZR; zS;I{gnr4cS^u$#>D(sjkh^T6_$s=*o%vNLC5+6J=HA$&0v6(Y1lm|RDn&v|^CTV{= zjVrg_S}WZ|k=zzp>DX08AtfT@LhW&}!rv^);ds7|mKc5^zge_Li>FTNFoA8dbk@K$ zuuzmDQRL1leikp%m}2_`A7*7=1p2!HBlj0KjPC|WT?5{_aa%}rQ+9MqcfXI0NtjvXz1U)|H>0{6^JpHspI4MfXjV%1Tc1O!tdvd{!IpO+@ z!nh()i-J3`AXow^MP!oVLVhVW&!CDaQxlD9b|Zsc%IzsZ@d~OfMvTFXoEQg9Nj|_L zI+^=(GK9!FGck+y8!KF!nzw8ZCX>?kQr=p@7EL_^;2Mlu1e7@ixfZQ#pqpyCJ```(m;la2NpJNoLQR};i4E;hd+|QBL@GdQy(Cc zTSgZ)4O~hXj86x<7&ho5ePzDrVD`XL7{7PjjNM1|6d5>*1hFPY!E(XDMA+AS;_%E~ z(dOs)vy29&I`5_yEw0x{8Adg%wvmoW&Q;x?5`HJFB@KtmS+o0ZFkE@f)v>YYh-z&m z#>ze?@JK4oE7kFRFD%MPC@x$^p{aW}*CH9Y_(oJ~St#(2)4e-b34D>VG6giMGFA83 zpZTHM2I*c8HE}5G;?Y7RXMA2k{Y?RxHb2 zZFQv?!*Kr_q;jt3`{?B5Wf}_a7`roT&m1BN9{;5Vqo6JPh*gnN(gj}#=A$-F(SRJj zUih_ce0f%K19VLXi5(VBGOFbc(YF zLvvOJl+W<}>_6_4O?LhD>MRGlrk;~J{S#Q;Q9F^;Cu@>EgZAH=-5fp02(VND(v#7n zK-`CfxEdonk!!65?3Ry(s$=|CvNV}u$5YpUf?9kZl8h@M!AMR7RG<9#=`_@qF@})d ztJDH>=F!5I+h!4#^DN6C$pd6^)_;0Bz7|#^edb9_qFg&eI}x{Roovml5^Yf5;=ehZ zGqz-x{I`J$ejkmGTFipKrUbv-+1S_Yga=)I2ZsO16_ye@!%&Op^6;#*Bm;=I^#F;? z27Sz-pXm4x-ykSW*3`)y4$89wy6dNOP$(@VYuPfb97XPDTY2FE{Z+{6=}LLA23mAc zskjZJ05>b)I7^SfVc)LnKW(&*(kP*jBnj>jtph`ZD@&30362cnQpZW8juUWcDnghc zy|tN1T6m?R7E8iyrL%)53`ymXX~_;#r${G`4Q(&7=m7b#jN%wdLlS0lb~r9RMdSuU zJ{~>>zGA5N`^QmrzaqDJ(=9y*?@HZyE!yLFONJO!8q5Up#2v>fR6CkquE$PEcvw5q zC8FZX!15JgSn{Gqft&>A9r0e#be^C<%)psE*nyW^e>tsc8s4Q}OIm})rOhuc{3o)g1r>Q^w5mas) zDlZQyjQefhl0PmH%cK05*&v{-M1QCiK=rAP%c#pdCq_StgDW}mmw$S&K6ASE=`u4+ z5wcmtrP27nAlQCc4qazffZoFV7*l2=Va}SVJD6CgRY^=5Ul=VYLGqR7H^LHA;H^1g}ekn=4K8SPRCT+pel*@jUXnLz+AIePjz@mUsslCN2 z({jl?BWf&DS+FlE5Xwp%5zXC7{!C=k9oQLP5B;sLQxd`pg+B@qPRqZ6FU(k~QkQu{ zF~5P=kLhs+D}8qqa|CQo2=cv$wkqAzBRmz_HL9(HRBj&73T@+B{(zZahlkkJ>EQmQ zenp59dy+L;sSWYde!z_W+I~-+2Xnm;c;wI_wH=RTgxpMlCW@;Us*0}L74J#E z8XbDWJGpBscw?W$&ZxZNxUq(*DKDwNzW7_}AIw$HF6Ix|;AJ3t6lN=v(c9=?n9;Y0 zK9A0uW4Ib9|Mp-itnzS#5in=Ny+XhGO8#(1_H4%Z6yEBciBiHfn*h;^r9gWb^$UB4 zJtN8^++GfT`1!WfQt#3sXGi-p<~gIVdMM<#ZZ0e_kdPG%Q5s20NNt3Jj^t$(?5cJ$ zGZ#FT(Lt>-0fP4b5V3az4_byF12k%}Spc$WsRydi&H|9H5u1RbfPC#lq=z#a9W(r1 z!*}KST!Yhsem0tO#r!z`znSL-=NnP~f(pw-sE+Z$e7i7t9nBP^5ts1~WFmW+j+<@7 zIh@^zKO{1%Lpx^$w8-S+T_59v;%N;EZtJzcfN%&@(Ux5 z@YzX^MwbbXESD*d(&qT7-eOHD6iaH-^N>p2sVdq&(`C$;?#mgBANIc5$r| z^A$r)@c{Z}N%sbfo?T`tTHz9-YpiMW?6>kr&W9t$Cuk{q^g1<$I~L zo++o2!!$;|U93cI#p4hyc!_Mv2QKXxv419}Ej#w#%N+YIBDdnn8;35!f2QZkUG?8O zpP47Wf9rnoI^^!9!dy~XsZ&!DU4bVTAi3Fc<9$_krGR&3TI=Az9uMgYU5dd~ksx+} zP+bs9y+NgEL>c@l>H1R%@>5SWg2k&@QZL(qNUI4XwDl6(=!Q^U%o984{|0e|mR$p+ z9BcwttR#7?As?@Q{+j?K6H7R71PuiA^Dl$=f47nUKL|koCwutc_P<-m{|Al3C~o7w z=4S=}s5LcJFT1zjS)+10X_r$74`K78pz!nGGH%JV%w75!YSIt#hT7}}K>+@{{a+Im z5p#6%^X*txY?}|T17xWW*sa^?G2QHt#@tlcw0GIcy;|NR2vaCBDvn=`h)1il7E5Rx z%)mA4$`$OZx)NF5vXZnaJ1)*cA6ryx6Ll~t!LzhxvcTedxT;>JS&e=?-&DXUPaQ2~ zH*69ezE`hgV{K-|0z|m~ld}=X^-Ob={wpex&}*+Rz{gx)G}gn!C_VN{UN=>^EV=Xc zr$-HO09cW&p4^M}V3yBjTP_xrVcc8iU_^Y-JD~(bgw*@GXGB1gYKz5DWO+O`>})|N zWrC)MR93yA)3{&27-M)TJB6Ml3~?zZg#mYsF=#OSTaw&K z@hBftpt+2l@)YK@|3DvTjl(8wZtpLp9Ik!6G$CSL_idZ$Ti?R)4toe8bb)l|)lNb}?K;O2K9vyn1QG zd=v#y-Ld49UVkmfRU>Egc+(Y$^-;6vW;3Lcu*6~etz}0|@+b|+!UCal)DEYGLbHWJ zll5Wi^$Y<6@S%^y%hdjRh6&{!z1Py|lZ|q&Wub3l41uN2zEF8E&5H5?PL*&V}?*a}Lp% zCYi{ghjpRNT^^B+_U59No50Ghih5qn(W5`RkrsDWr{~A1dgtv{sRkH4RU2^A{jb&0 zxVRnrm|u<;$iI;M6A>$POP)TWGU-gSjAERk*EGmVT(aw$!XUSe~7Ql-oRA54^4V(JWS6Q1mG?!vZ zx+pE!FEtvqr|Xrcb3oR`%LHFLmU_&{=p%mGy6MRe2Yz_5WJ8p@IgU2 zdVvvhhQtiQkChK%*&PsiPCBL9oDOoJX8!$S(V>R}+1M}wzK*U*A{KJ`r=lM;mPrKU zQDqqN(W*u-5-?$(SIk<6A0E}34y&@-IVC%S!a1F4kz<3bIKjlyD)ooO_7ftl%S_(6w`!vX&1PZ!K`@D@L6JR)6zO@Dl!YF{RY}d3HZ7?Q5E>w=$ ze)H_)48Ds*Ov4?zoGb2fe3}{!5Ooc|KCIni1o)(Gj+CO?`*7jsV`hIv@8J(22o4Q? zu?Bvi)zDG(me?7XKeL|iF9ZRgZdT*}Ffsl62Cu;{Gv9j6dO zPt*H2GqC)-C`V`ceuu=tM{7!2yTEj=*5+T~5DYiZ)Hy)*PARYI6R2lZXoOj;v8M4W z*O-NX(7_~Q&A3>Oaw&1lBH_H%SwmISX-i3)HfHvBOeVwTT{LUM3}ZuZmg<(>)KE;d zbs2!0v6>J;1nQ0UJkUxnkE@Ibi~Q}M=-=Rk;hcOnxO$luOKEVxZc|!XECgex(2`}T z3Y;Q_6rL)e+SrOZhQj5_e}Lv>w7n*Pep$yWZNQl>ubBgb_NIWWDn3kNpn+MPQXV;8 zV|_Ba5jsQ(w&Ey^IM|@|y!AqcJ#3m0#Q6_qvgCG~eoF#mnGmbO(;DP+bW%_aOs1R_ z@9p#7X2UA^--#Nwx_Hvk2l1`eO{P*#j@q2UELtH|Uh6hxR`h_847wIJo0=5CQQ`6it|%a-I$^&a@we1rc&*;QIu5Ck^?) zx*5eSd*mG#=6Hi(5!;5uUi&{HfnT1S8X-)?gE5CZ6KWoqM5|CyrULmuFBKOU8SOp* z{IB1$OCcq`S-k*xs;4fmhKsIGZ;GYAY*%(@875NxhMq|j*m4CNLI(Vho|N|F);!E0cS5y^$H^Izje?z}oTgyr`9x9G&rlJZw&uqIoBMtz zzhU0(9;w02?m#0!)cFi*r+8YvooQ;(s2lLVvyLqAE%Xqe!vtWbIs!l1Bpp(FIht-Z zPn#CN-2C|J*GhA2fuHqYQ2mJiXlGTzD}mkr2;ia8Wp}h^;OS7+N^Mw|en!1${vN6 z-x{8N*4UekA~`IV2&K-GzhAqau|}d*pEQ$1MH$cFi03OG^1NetZ_jW^STaEzr&Xho zB452St%v3ez2#TFm~`gZh$vi=in+y2d!z<{OZ~Kty-5bQ;0O=k_ESi8Nx9{*T`LJy6jqR>&|+>OZ;+=0hA04 zE25t^sE9HG)3^KKR_A5WDkqispweP9!I-@dCO&N!JrD@i{WBHnfQ z95o8;d$`AFnca3;N-0iX-CmbbAp5yQ!GoH;h7Cn?m{ammZJI8igP{U73lFnl2&gCs zqJ4(Vo~^j`{zOAzScL5B_Sm?Mjtek1d(A6X5ObcZi$;aOYy|g$}BY z$GEP3#i60Ju_&3SHzryH!gUFwC9-295u??cf+aYRQ1$+!rc#42YNattd6mZEFI@?C zqFM>6+zxEunIHDZ>{Z15u##>N(28Dw!>G(k*dB{NHvip@aP}f`@=Q;!o;zRMWo{Cx zo?kyzh8n7#f1g0&g>Cd>O-2g?uPwy8sy8hZbHSsXPmU;@l=HL=zm7mN(=@*|D$i+u zs~TllkCTvD$f&-#b9B?}#Lg*-ibK13R_a$RyoN3m5`10tdhAq{+VW)K#Bht-ra1*J z+n$N%V>u0rVtx`aKJDwXXrxaD7nS<>$=c82v7@KVx^S@vT;h=SZE37K>iahpx3;VDzEr9GY=2(%uaqM;^76eSP0QLzo4sI z>p_Eei*T$K;|qK`sq;?Hesp}(@VvX2Q4sAMYAJ}b&d$htDMC{FG-$o4k9ApECi1$a zXdamjiOGKHBh(4M<3(2x6n-CrmZMCknkQxdSS!qlis#I}btfX;J`JU3RlvtLdrymP zG0ZzrsGXVFiq+Wk1=BFay&9ZiCE#(`h~CL+c-Hs@iGTU@YxM%vlg;)`Tf~IknA^02 zXkN#Txo6aR{j$wP5T#|UH#5AP2{rSY8p?jKFv zG3kn3y`FaV!*Jq%m39_TQEhD>M@l*bhEPGe1{ft3q#K5AknT=F2_=T^l#ou5ln@D# z5Tzs(kRG@qNDa~HLNvfv7Z0g=bSlb?`QAx|Gfoni|iHJ%K0cy z;~Nsaa+{8HP_qrb{nj+xzkdYhSI@W4N_1`z(eSGIkbDP)!Ko|M%}Rqp(~KI2hl~eE zvJ!j4m6iwMgKy>fkCLC)`M$z9EV}B+sq1}}kVf$(ig0pWTY?rHz1Sm=4srTGNb^JG z=2$9wz-C@aZZZ2!HY#HNejqZRmE=pN(D$Kui$NpfhU`!y_s{@MIxiJdHb1|{6xb`> zE74_@QtgtG{4=3P1$^vn&m}7Aw8!1DnT$2thO#~44wl(N#ao8S0@t@m+Z!KD2CfK; z)n5DAPKV_etmH1aLDK$?`;sL91iVt$D z*SG}=-LIAg(*+JON!-5ivqOMQ1S!OQUgHglDsKik&Mwg;vva523`JwQH6SRz9eTY# zTIi23145~kc3r1mSWC_RzD%hs$S#!pkI9!BU80jJCJcwo*FZolQG$q`8C1d9pP@ND zG^&-ZraIvhg_FDVSfKGwkcI=avIan%2sK4coUs~Nr8jC*&!G0#?}_^s3r-c}-uAqi zM-Lw>Y}I``T;IS%Y|qH;s{F*ZefM!4{I5awr!K+T@uPd*Vu*iPWI}>(-D{zxsN>LG z=@747a_Rb2>q?y8xYf?dq2HM5tFO8Y5e4N;Y=xy8yAhI zsm>oy%R5;7)7T3V_b2%`aH^tNlsQpFxIFW#iV#8?{6{^cGr{A0@1bA)|K z>MMTuZD(pd2t|7vmHtywGXb%%=)S<`OG~}U+jm#xd%H8 z$v8-C%F?ah3$;hn?{G3(LT!SgvCVi$vwsZssAQvUwT`Q%qSw!LSd!(I!64w1=%Sc1Mck)q1@pZ@)=SY zoX}d+L3-RA|c?G3_BQNm&( z!i$AZ7cI(z7q|e9VM##6T3Xorj1JG(9os$;(I$y%mBy(#8{|3l4|x*oBAQL^XhZ0g zy1FR1teRrpKq{uLAibTLx#n({qwjlkOvR{OdSAeT5ah4-sNN)n4Clg1T9lzF)&yj; zyal1%+s4n1IG;^VPWJ;#olpk8Z42Gj-tjFeQ&PlxB)`oCNoUYKj4U$AeG8rYiD{pK zndDf&2;2;)D|KvOZP+e7fcPU9k4M2sfhr@vC~Ly0?S-4dz)ZGAYpCsAhChgbxLd4g zhTrbIPkO5SEp_kD>Ha0m12h5n3s;mE8kn515&nzSf+^D= zyE{JnJ;43l&BH55CL<=W%CF;6iUI)V5C*6!`**KqvzR2=Fj*3Y4`HYwx}TYD445(K z-QtXwtL?m*(F=LVH*H4oM>dXHBW=38q_dZ-_Vr&qpEPxd9Fs95P5W~@Z|Rt+WZP6l zPSQ}~Dh4V?Pp1g&Hk*Px?lm16C@X6M29Vrk%Rw@E||E-v~$ zb_E~{z<}#8i`Mx9mkqtd#Z1lZ-E_J8I+2oumc#x1)jdvh{W76NKm6x-RYpM~v!P8$ zw3e|YVf|}Hse9~oC@N7^j}Fi$hNpyaYnu1}bdXsD=^oI*%WKvbme|BI}$G3>smu#6y)ls|j? zF7Bhu9Z)j)C;3cZb+I>0stSK^WLOYV^U{pUYkgv>?+Nt^5j*CUB=eGw-CvU&40>y~ zGoHLXxY^7k5Xgv62{iQy|5jJQuq0|LU`}lE@flQ2Z*Zn*VWcQjm4FTb>LSVox^S4q zLn`LfS@mrjKCmg$nb^af?d?0&$aX6#2u(JyzIJvuJ*lwPrh|0~aEnSACCTezSdG%h zmSQg`17j@$Iq)r1&?+eR@1nlX|H`<}_!?BQSF&N+QQnvEAqZe+mIFui!0V49R?|9*$ zv!K1A01{8xq;L()Tv*Qk0-$Oj6+vCT*TUD{HvxO@3JjxBwM!4g3ydy&eaJw4CoQBF zJtULJ!YxgNR7_Ls%LmogyI7uIs=!B&?=MYY^yX+v;j@D_xGeZg>eZk0C;4e|HRNSi z6KlD9>q=3v-$4Zik&^ZDhNm1X)+7LCH1k!s+T3tn zUn@={1U&NJLq@K?~w|(=Y<4W{ucX}FdRr6pLw(l2$iK)At%t3gYBMlJz#(K0Nqm;=KAML!&MMSNz=%k=j*zh77r34Rs37iCY` z=_kva_41bdrj(b=4Wc5MO0~q^z#pIWJ>)vDSgIQF=3JVJe1iDy%h)8oNy{s_r&;m` zL{DYKSB_5xRb9xKNOS{qAY3qv5sSXVrrf%~*q5HO|CQ&lbKMePa$M5D{vlJcoGrCZ zD?fKbZN$6rWwz)w7`9h4DAmh1ij2}EO|bO#A9L0_RW6l*$sPPUJrUbhLC75L9%W5iO$Iw5~Yut-qBeu~hF|xD7-eQ%l z412vpq_;t%^F*pYDk%Q35c-erK|6Ve=FxQbAv~ikZ4c9$Y4;ee#ciOD9{yRqf55Qk zumv}#+JciT|Gj$uFOxBUze)=?l{B}qaC0_7m`t82<$K53!4Xvi9Tr)ADp3Off?O8o zVDG0Yx|tfn@r((m?Nxrh(b0DGjg)$;DfO&$6uY;4&F!4jnxkhP}Y3x zS?WFFt>=HWzqlQhffVfvM$Ta8Sg*r3j!Eo&rUOW7SCL2~lG7<+XZ;+{&8h5g8ElI+P>>yR2U%S93NN!Xhm|C682t6ysH-=o1=Bd*N*VlnG%l+KZFtjG`UkL;%65qn0UYQ`h zh0{9jDQx(`aBe7J0Aj3Z)4}`A|4OMM0a;?{j}qkYwi)~O8$9D}ITiMH2buiU>ixYp zhL${nwj6X($*OwmpVG`y5b6v45tX*J8?og}Qju6eJ9H}`X87iEd%BUo7<`2q(HJx+ zMR}d-J4oAf{V1W^a2~`M-YAdZ81dd4o6NPO{cmZaAS@RS4ir#Sr zfFZO-VIL|VN<%nEXr2` z$0FK2L#8O_f1w~c@G70JrB@N}r(gJ!Vmkk6{r68w!o$qO?HrFcjeU0_3F5;*!E2%( zTx>4?gP8w z1B?3UVZmz^%d_dIps>>0{cB~mp3{9UoPR6uQFecVq&} zY{ebB?AlPAD_}(ll{fK99;Wh1cgRbnw)maD^F>*J!R}eHM*W0VYN1TADWMy9H=$00 z5bHY${oDgwX7(W9LZw?}{!8(_{JB~Xkje6{0x4fgC4kUmpfJ+LT1DYD*TWu4#h{Y7 zFLronmc=hS=W=j1ar3r1JNjQoWo2hMWsqW*e?TF%#&{GpsaLp}iN~$)ar+7Ti}E&X z-nq~+Gkp(`qF0F_4A22>VZn-x>I$?PDZSeG8h_ifoWf^DxIb5%T7UytYo3}F|4#RC zUHpg$=)qVqD~=m(!~?XwocuxU1u}9qhhM7d^eqmJPi_e-!IO`*{u7A zbu*?L$Mbj-X9n3G2>+Kc#l`@d8}Xb9{l*IN{#M*d;s+3Pdr8FO$EBELR=8{ zd?LJbSv9fI`{OqTH)5{b?WulgMb)psp+W|@cSp=jtl-&5C}9lw@*0H+gEW(}mAWNz zf{~U;;N}|wdSaphgqnH{FWUy!{y3^=AC*c?RJ5Eb<^ zCgH_v7^axIUVmHSFL^zlj2R$zow$|y#7>%#U7d#Vp_ezcp3lefMyd5ES=q$>4pWyA zp_Zso^^NP~lu2=S6nD(3Z5u=Uy&B&F1i$J*3;3KhEkD_lgscHGR*;T;U!9vgQa(hI}oh9IzEf_PU_8F+i77t-~gDX z490Sb)LyVZmf18N6w{+37$aO<2!Av0 ztLaPOv^J<2@p{WnMiDudoghX_`luFZt_4eNU}*~cF5i%eEcNLs;D>QVIwr8mH;=dc z09`}JV;aaF;13@&iS(w>Jc=k~|d_1hcpM(l|O zu>!@}me%isTT$xT#hNUvh(ATd0wT4fbv=6htcHNEZIw9%E6wlYmwfu2{j0kh1y=$;Yf!|NldgB9ul zB{dbE&LfRnr8ITm@;-68wo#VV?8lG3ed&9k1}QBS3}WGV9%26?A1rBkkDR9Z3o+g+ z)eQg8BY3y(Dh5&z?VLLNdDV`C=muUvCPpGg!oYxIgOI3^%4>5d7jTh~ni!Fg2;fhx z(*c%H6Je84kmQh;5tC3*l~7khLxK-e|Cz?FLh!yYe7g|*LwqU?2wv^_ZyKT$fYVkGJo@AK0$+ml?}zJeB~deT2WL1vz}dxB z)y??t!}%M@)u$_IyW~)6u1SttJ!awd6N5lx|xBrmyrBh>tb&D*=C+Z3nPfq$1%WgY0bY*?PZ#Hk|=xn zGM#0*w4CaB^y0G(J4q=;5NeM@m-P}#mv7QZNF)M!dK^w{mk_!n0`+Y3PQutu-%NBt zzgPXug?JLEbUL{e_dk;Vd896&yPe(hliVK!lj%5+@BKdcrEZ2Nc_*i@ve*2lB>u~{ zFozd2FM|_0+nAGR4TLNHanQn_Oeb!JrUcvzJ?7p9TTNB}ocO3j$7ij!li8#k6 z@2tSd1>K03K9A#_-MIq)S;T#oE^;>U$)&}okIvDf3lm?kI{d80$>~xKUoS!%q1Pi?WpsUUt(tI ztjNjY*y&Rm9(S(DC2GuPHBJs@5M{RGm`c1z<6nwyN^)rMo-AS{M2$oM9|y%fM|}G~ DHx0+F literal 0 HcmV?d00001 From f2cdd1b912feff0dcfb1f0b6820863e583b81dd9 Mon Sep 17 00:00:00 2001 From: expand3d <> Date: Sun, 1 Jun 2025 20:48:43 -0400 Subject: [PATCH 09/49] Update wrapper validation action to v3 --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 18842f6..2e906c6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,7 +38,7 @@ jobs: run: chmod +x ./gradlew - name: Validate Gradle wrapper - uses: gradle/wrapper-validation-action@v1 + uses: gradle/wrapper-validation-action@v3 - name: Run tests run: ./gradlew test @@ -89,7 +89,7 @@ jobs: run: chmod +x ./gradlew - name: Validate Gradle wrapper - uses: gradle/wrapper-validation-action@v1 + uses: gradle/wrapper-validation-action@v3 - name: Run quick benchmark run: ./gradlew jmh -Pjmh.fork=1 -Pjmh.warmupIterations=1 -Pjmh.iterations=1 From 57c824964a18f804cffdf458a746b20e0fcd68ae Mon Sep 17 00:00:00 2001 From: expand3d <> Date: Sun, 1 Jun 2025 20:53:03 -0400 Subject: [PATCH 10/49] Fix Javadoc syntax errors and disable strict Javadoc checking --- build.gradle | 2 ++ .../com/imprint/core/ImprintRecordBuilder.java | 14 ++++++++------ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/build.gradle b/build.gradle index 9262297..7e85806 100644 --- a/build.gradle +++ b/build.gradle @@ -72,4 +72,6 @@ javadoc { if(JavaVersion.current().isJava9Compatible()) { options.addBooleanOption('html5', true) } + // Don't fail build on missing javadoc + options.addStringOption('Xdoclint:none', '-quiet') } diff --git a/src/main/java/com/imprint/core/ImprintRecordBuilder.java b/src/main/java/com/imprint/core/ImprintRecordBuilder.java index 48b0998..202bd2a 100644 --- a/src/main/java/com/imprint/core/ImprintRecordBuilder.java +++ b/src/main/java/com/imprint/core/ImprintRecordBuilder.java @@ -11,14 +11,16 @@ * developer-friendly API that eliminates boilerplate Value.fromX() calls. *

* Usage: + * 

  *   var record = ImprintRecord.builder(schemaId)
- *       .field(1, 42)              // int -> Int32Value  
- *       .field(2, "hello")         // String -> StringValue
- *       .field(3, 3.14)            // double -> Float64Value
- *       .field(4, bytes)           // byte[] -> BytesValue
- *       .field(5, true)            // boolean -> BoolValue
- *       .nullField(6)              // -> NullValue
+ *       .field(1, 42)              // int to Int32Value  
+ *       .field(2, "hello")         // String to StringValue
+ *       .field(3, 3.14)            // double to Float64Value
+ *       .field(4, bytes)           // byte[] to BytesValue
+ *       .field(5, true)            // boolean to BoolValue
+ *       .nullField(6)              // to NullValue
  *       .build();
+ *
*/ public final class ImprintRecordBuilder { private final SchemaId schemaId; From edb30578f35d65522df2778eea62a9d0e35e0268 Mon Sep 17 00:00:00 2001 From: expand3d <> Date: Sun, 1 Jun 2025 23:33:47 -0400 Subject: [PATCH 11/49] Add JMH benchmark .bat and .sh for full suite benchmarking and performance tracking; add comprehensive String benchmark --- .gitignore | 3 + benchmark-results/README.md | 65 +++++ build.gradle | 1 + run-benchmarks.bat | 89 +++++++ run-benchmarks.sh | 148 +++++++++++ .../imprint/benchmark/StringBenchmark.java | 248 ++++++++++++++++++ 6 files changed, 554 insertions(+) create mode 100644 benchmark-results/README.md create mode 100644 run-benchmarks.bat create mode 100644 run-benchmarks.sh create mode 100644 src/jmh/java/com/imprint/benchmark/StringBenchmark.java diff --git a/.gitignore b/.gitignore index 54c84dc..3f1edb4 100644 --- a/.gitignore +++ b/.gitignore @@ -63,3 +63,6 @@ buildNumber.properties *.jar !gradle/wrapper/gradle-wrapper.jar !lombok.jar + +# Benchmark Results (keep timestamped results in repo for tracking) +# benchmark-results/ - Commented out to keep results in repo diff --git a/benchmark-results/README.md b/benchmark-results/README.md new file mode 100644 index 0000000..dc193d7 --- /dev/null +++ b/benchmark-results/README.md @@ -0,0 +1,65 @@ +# Benchmark Results + +This directory contains historical benchmark results for the Imprint Java implementation. + +## Files + +- `*.json` - Raw JMH benchmark results in JSON format +- `*-summary-*.txt` - Human-readable summaries of benchmark runs +- `system-info-*.txt` - System information for each benchmark run +- `overall-summary-*.txt` - Complete benchmark overview + +## Running Benchmarks + +### All Benchmarks +```bash +# Unix/Linux/macOS +./run-benchmarks.sh + +# Windows +run-benchmarks.bat +``` + +### Specific Benchmark Categories +```bash +# String performance tests +./gradlew jmh -Pjmh.include=StringBenchmark + +# Serialization tests +./gradlew jmh -Pjmh.include=".*[Ss]erial.*" + +# Field access tests +./gradlew jmh -Pjmh.include=".*[Aa]ccess.*" + +# Comparison tests (vs other formats) +./gradlew jmh -Pjmh.include=ComparisonBenchmark +``` + +## Analyzing Results + +### Online Visualization +Upload JSON files to [JMH Visualizer](https://jmh.morethan.io/) for interactive charts and analysis. + +### Command Line Analysis +```bash +# View benchmark names and scores +jq -r '.[] | select(.benchmark) | "\(.benchmark): \(.primaryMetric.score) \(.primaryMetric.scoreUnit)"' results.json + +# Find fastest operations +jq -r '.[] | select(.benchmark) | "\(.benchmark): \(.primaryMetric.score)"' results.json | sort -k2 -n + +# Compare specific benchmarks +jq '.[] | select(.benchmark | contains("String"))' results.json +``` + +## Performance Tracking + +Results are timestamped and committed to track performance changes over time. Compare results between commits to identify performance regressions or improvements. + +## System Requirements + +For consistent results: +- Run on dedicated hardware when possible +- Close unnecessary applications +- Run multiple times and compare results +- Note system configuration in commit messages \ No newline at end of file diff --git a/build.gradle b/build.gradle index 7e85806..2606710 100644 --- a/build.gradle +++ b/build.gradle @@ -61,6 +61,7 @@ jmh { iterations = 3 resultFormat = 'JSON' includeTests = false + resultsFile = file("${projectDir}/benchmark-results/jmh-results-${new Date().format('yyyy-MM-dd-HHmmss')}.json") } compileJava { diff --git a/run-benchmarks.bat b/run-benchmarks.bat new file mode 100644 index 0000000..750722d --- /dev/null +++ b/run-benchmarks.bat @@ -0,0 +1,89 @@ +@echo off +REM Benchmark execution script for Imprint Java implementation (Windows) +REM This script runs all benchmark suites and saves results with timestamps + +setlocal enabledelayedexpansion + +for /f "tokens=2 delims==" %%a in ('wmic OS Get localdatetime /value') do set "dt=%%a" +set "TIMESTAMP=%dt:~0,4%-%dt:~4,2%-%dt:~6,2%-%dt:~8,2%%dt:~10,2%%dt:~12,2%" +set "RESULTS_DIR=.\benchmark-results" +set "SYSTEM_INFO_FILE=%RESULTS_DIR%\system-info-%TIMESTAMP%.txt" + +echo 🏃 Running Imprint Java Benchmarks - %TIMESTAMP% +echo ================================================ + +REM Ensure results directory exists +if not exist "%RESULTS_DIR%" mkdir "%RESULTS_DIR%" + +REM Capture system information +echo 📊 Capturing system information... +( + echo Benchmark Run: %TIMESTAMP% + echo ============================== + echo. + echo System Information: + echo - Date: %date% %time% + echo - OS: %OS% + echo - Processor: %PROCESSOR_IDENTIFIER% + echo - Java Version: + java -version 2^>^&1 + echo. + echo - Gradle Version: + call gradlew --version + echo. + echo - Git Commit: + for /f %%i in ('git rev-parse HEAD') do echo - Hash: %%i + for /f %%i in ('git rev-parse --abbrev-ref HEAD') do echo - Branch: %%i + for /f "tokens=*" %%i in ('git log -1 --format^=%%cd') do echo - Date: %%i + for /f "tokens=*" %%i in ('git log -1 --format^=%%s') do echo - Message: %%i + echo. +) > "%SYSTEM_INFO_FILE%" + +echo ✅ System info saved to: %SYSTEM_INFO_FILE% + +echo 🚀 Starting benchmark execution... +echo. + +REM Function to run complete benchmark suite +echo 🔄 Running complete benchmark suite... +set "complete_output=%RESULTS_DIR%\complete-benchmarks-%TIMESTAMP%.json" +call gradlew jmh -Pjmh.resultsFile="%complete_output%" --console=plain + +if exist "%complete_output%" ( + echo ✅ Complete benchmark suite completed: %complete_output% + + REM Generate overall summary + set "overall_summary=%RESULTS_DIR%\overall-summary-%TIMESTAMP%.txt" + ( + echo Complete Imprint Java Benchmark Results - %TIMESTAMP% + echo ================================================== + echo. + echo Benchmark execution completed successfully. + echo Results saved to: %complete_output% + echo. + echo To analyze results: + echo - Use JMH Visualizer: https://jmh.morethan.io/ + echo - Import JSON into analysis tools + echo - Use jq for command-line analysis + echo. + ) > "!overall_summary!" + + echo 📊 Overall summary: !overall_summary! +) else ( + echo ❌ Complete benchmark suite failed +) + +echo. +echo 🎉 Benchmark execution completed! +echo 📁 All results saved in: %RESULTS_DIR% +echo 📄 Files created: +dir "%RESULTS_DIR%\*%TIMESTAMP%*" 2>nul + +echo. +echo 💡 To view results: +echo - JSON files can be analyzed with jq or imported into analysis tools +echo - Visit https://jmh.morethan.io/ to visualize results +echo - Summary files provide human-readable overviews +echo - System info file contains environment details for reproducibility + +pause \ No newline at end of file diff --git a/run-benchmarks.sh b/run-benchmarks.sh new file mode 100644 index 0000000..571ab5f --- /dev/null +++ b/run-benchmarks.sh @@ -0,0 +1,148 @@ +#!/bin/bash + +# Benchmark execution script for Imprint Java implementation +# This script runs all benchmark suites and saves results with timestamps + +set -e + +TIMESTAMP=$(date '+%Y-%m-%d-%H%M%S') +RESULTS_DIR="./benchmark-results" +SYSTEM_INFO_FILE="$RESULTS_DIR/system-info-$TIMESTAMP.txt" + +echo "🏃 Running Imprint Java Benchmarks - $TIMESTAMP" +echo "================================================" + +# Ensure results directory exists +mkdir -p "$RESULTS_DIR" + +# Capture system information +echo "📊 Capturing system information..." +{ + echo "Benchmark Run: $TIMESTAMP" + echo "==============================" + echo "" + echo "System Information:" + echo "- Date: $(date)" + echo "- OS: $(uname -a)" + echo "- Java Version:" + java -version 2>&1 | sed 's/^/ /' + echo "" + echo "- Gradle Version:" + ./gradlew --version | sed 's/^/ /' + echo "" + echo "- Git Commit:" + echo " - Hash: $(git rev-parse HEAD)" + echo " - Branch: $(git rev-parse --abbrev-ref HEAD)" + echo " - Date: $(git log -1 --format=%cd)" + echo " - Message: $(git log -1 --format=%s)" + echo "" + echo "Hardware Information:" + if command -v lscpu &> /dev/null; then + echo "- CPU:" + lscpu | grep -E "Model name|Architecture|CPU\(s\)|Thread|Core" | sed 's/^/ /' + fi + if command -v free &> /dev/null; then + echo "- Memory:" + free -h | sed 's/^/ /' + fi + echo "" +} > "$SYSTEM_INFO_FILE" + +echo "✅ System info saved to: $SYSTEM_INFO_FILE" + +# Function to run a specific benchmark suite +run_benchmark_suite() { + local suite_name=$1 + local suite_pattern=$2 + local output_file="$RESULTS_DIR/${suite_name}-$TIMESTAMP.json" + + echo "🔄 Running $suite_name benchmarks..." + echo " Pattern: $suite_pattern" + echo " Output: $output_file" + + ./gradlew jmh \ + -Pjmh.include="$suite_pattern" \ + -Pjmh.resultsFile="$output_file" \ + --console=plain + + if [ -f "$output_file" ]; then + echo "✅ $suite_name completed: $output_file" + + # Generate a human-readable summary + local summary_file="$RESULTS_DIR/${suite_name}-summary-$TIMESTAMP.txt" + { + echo "$suite_name Benchmark Summary - $TIMESTAMP" + echo "=======================================" + echo "" + echo "Top 10 Fastest Operations:" + jq -r '.[] | select(.benchmark) | "\(.benchmark): \(.primaryMetric.score | tonumber | . * 1000000 | floor / 1000000) \(.primaryMetric.scoreUnit)"' "$output_file" | sort -k2 -n | head -10 + echo "" + echo "Top 10 Slowest Operations:" + jq -r '.[] | select(.benchmark) | "\(.benchmark): \(.primaryMetric.score | tonumber | . * 1000000 | floor / 1000000) \(.primaryMetric.scoreUnit)"' "$output_file" | sort -k2 -nr | head -10 + echo "" + } > "$summary_file" 2>/dev/null || echo "⚠️ Could not generate summary (jq not available)" + else + echo "❌ $suite_name failed - no output file generated" + fi + echo "" +} + +# Run all benchmark suites +echo "🚀 Starting benchmark execution..." +echo "" + +# 1. Serialization benchmarks +run_benchmark_suite "serialization" ".*[Ss]erial.*" + +# 2. Field access benchmarks +run_benchmark_suite "field-access" ".*[Aa]ccess.*" + +# 3. Merge benchmarks +run_benchmark_suite "merge" ".*[Mm]erge.*" + +# 4. String benchmarks +run_benchmark_suite "string" ".*String.*" + +# 5. Comparison benchmarks (vs other formats) +run_benchmark_suite "comparison" ".*Comparison.*" + +# 6. Complete benchmark run (all benchmarks) +echo "🔄 Running complete benchmark suite..." +complete_output="$RESULTS_DIR/complete-benchmarks-$TIMESTAMP.json" +./gradlew jmh \ + -Pjmh.resultsFile="$complete_output" \ + --console=plain + +if [ -f "$complete_output" ]; then + echo "✅ Complete benchmark suite completed: $complete_output" + + # Generate overall summary + overall_summary="$RESULTS_DIR/overall-summary-$TIMESTAMP.txt" + { + echo "Complete Imprint Java Benchmark Results - $TIMESTAMP" + echo "==================================================" + echo "" + echo "Total Benchmarks Run: $(jq '[.[] | select(.benchmark)] | length' "$complete_output" 2>/dev/null || echo "Unknown")" + echo "" + echo "Performance Overview:" + echo "--------------------" + jq -r '.[] | select(.benchmark) | "\(.benchmark): \(.primaryMetric.score | tonumber | . * 1000000 | floor / 1000000) \(.primaryMetric.scoreUnit)"' "$complete_output" 2>/dev/null | sort -k2 -n || echo "Could not generate overview" + echo "" + } > "$overall_summary" 2>/dev/null + + echo "📊 Overall summary: $overall_summary" +else + echo "❌ Complete benchmark suite failed" +fi + +echo "" +echo "🎉 Benchmark execution completed!" +echo "📁 All results saved in: $RESULTS_DIR" +echo "📄 Files created:" +ls -la "$RESULTS_DIR"/*"$TIMESTAMP"* 2>/dev/null || echo " No files with timestamp $TIMESTAMP found" + +echo "" +echo "💡 To view results:" +echo " - JSON files can be analyzed with jq or imported into analysis tools" +echo " - Summary files provide human-readable overviews" +echo " - System info file contains environment details for reproducibility" \ No newline at end of file diff --git a/src/jmh/java/com/imprint/benchmark/StringBenchmark.java b/src/jmh/java/com/imprint/benchmark/StringBenchmark.java new file mode 100644 index 0000000..1891251 --- /dev/null +++ b/src/jmh/java/com/imprint/benchmark/StringBenchmark.java @@ -0,0 +1,248 @@ +package com.imprint.benchmark; + +import com.imprint.core.ImprintRecord; +import com.imprint.core.SchemaId; +import com.imprint.types.Value; +import org.openjdk.jmh.annotations.*; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; + +import java.nio.ByteBuffer; +import java.util.concurrent.TimeUnit; + +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@State(Scope.Benchmark) +@Fork(1) +@Warmup(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS) +public class StringBenchmark { + + private static final SchemaId SCHEMA_ID = new SchemaId(1, 42); + + // Small strings (typical field names, short values) + private String smallString5; + private String smallString20; + private String smallString50; + + // Medium strings (typical text content) + private String mediumString500; + private String mediumString2K; + + // Large strings (document content, JSON payloads) + private String largeString10K; + private String largeString100K; + private String largeString1M; + + // Pre-serialized records for deserialization benchmarks + private byte[] serializedSmall5; + private byte[] serializedSmall20; + private byte[] serializedSmall50; + private byte[] serializedMedium500; + private byte[] serializedMedium2K; + private byte[] serializedLarge10K; + private byte[] serializedLarge100K; + private byte[] serializedLarge1M; + + @Setup + public void setup() throws Exception { + // Generate strings of different sizes + smallString5 = generateString(5); + smallString20 = generateString(20); + smallString50 = generateString(50); + mediumString500 = generateString(500); + mediumString2K = generateString(2 * 1024); + largeString10K = generateString(10 * 1024); + largeString100K = generateString(100 * 1024); + largeString1M = generateString(1024 * 1024); + + // Pre-serialize records for deserialization benchmarks + serializedSmall5 = bufferToArray(createStringRecord(smallString5).serializeToBuffer()); + serializedSmall20 = bufferToArray(createStringRecord(smallString20).serializeToBuffer()); + serializedSmall50 = bufferToArray(createStringRecord(smallString50).serializeToBuffer()); + serializedMedium500 = bufferToArray(createStringRecord(mediumString500).serializeToBuffer()); + serializedMedium2K = bufferToArray(createStringRecord(mediumString2K).serializeToBuffer()); + serializedLarge10K = bufferToArray(createStringRecord(largeString10K).serializeToBuffer()); + serializedLarge100K = bufferToArray(createStringRecord(largeString100K).serializeToBuffer()); + serializedLarge1M = bufferToArray(createStringRecord(largeString1M).serializeToBuffer()); + } + + private String generateString(int length) { + StringBuilder sb = new StringBuilder(length); + String chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789 "; + for (int i = 0; i < length; i++) { + sb.append(chars.charAt(i % chars.length())); + } + return sb.toString(); + } + + private ImprintRecord createStringRecord(String value) throws Exception { + return ImprintRecord.builder(SCHEMA_ID) + .field(1, value) + .build(); + } + + private String extractString(Value value) { + if (value instanceof Value.StringValue) { + return ((Value.StringValue) value).getValue(); + } else if (value instanceof Value.StringBufferValue) { + return ((Value.StringBufferValue) value).getValue(); + } + return null; + } + + private byte[] bufferToArray(ByteBuffer buffer) { + byte[] array = new byte[buffer.remaining()]; + buffer.duplicate().get(array); + return array; + } + + // Serialization benchmarks + + @Benchmark + public ByteBuffer serializeSmallString5() throws Exception { + return createStringRecord(smallString5).serializeToBuffer(); + } + + @Benchmark + public ByteBuffer serializeSmallString20() throws Exception { + return createStringRecord(smallString20).serializeToBuffer(); + } + + @Benchmark + public ByteBuffer serializeSmallString50() throws Exception { + return createStringRecord(smallString50).serializeToBuffer(); + } + + @Benchmark + public ByteBuffer serializeMediumString500() throws Exception { + return createStringRecord(mediumString500).serializeToBuffer(); + } + + @Benchmark + public ByteBuffer serializeMediumString2K() throws Exception { + return createStringRecord(mediumString2K).serializeToBuffer(); + } + + @Benchmark + public ByteBuffer serializeLargeString10K() throws Exception { + return createStringRecord(largeString10K).serializeToBuffer(); + } + + @Benchmark + public ByteBuffer serializeLargeString100K() throws Exception { + return createStringRecord(largeString100K).serializeToBuffer(); + } + + @Benchmark + public ByteBuffer serializeLargeString1M() throws Exception { + return createStringRecord(largeString1M).serializeToBuffer(); + } + + // Deserialization benchmarks + + @Benchmark + public ImprintRecord deserializeSmallString5() throws Exception { + return ImprintRecord.deserialize(serializedSmall5); + } + + @Benchmark + public ImprintRecord deserializeSmallString20() throws Exception { + return ImprintRecord.deserialize(serializedSmall20); + } + + @Benchmark + public ImprintRecord deserializeSmallString50() throws Exception { + return ImprintRecord.deserialize(serializedSmall50); + } + + @Benchmark + public ImprintRecord deserializeMediumString500() throws Exception { + return ImprintRecord.deserialize(serializedMedium500); + } + + @Benchmark + public ImprintRecord deserializeMediumString2K() throws Exception { + return ImprintRecord.deserialize(serializedMedium2K); + } + + @Benchmark + public ImprintRecord deserializeLargeString10K() throws Exception { + return ImprintRecord.deserialize(serializedLarge10K); + } + + @Benchmark + public ImprintRecord deserializeLargeString100K() throws Exception { + return ImprintRecord.deserialize(serializedLarge100K); + } + + @Benchmark + public ImprintRecord deserializeLargeString1M() throws Exception { + return ImprintRecord.deserialize(serializedLarge1M); + } + + // String access benchmarks + + @Benchmark + public String accessSmallString5() throws Exception { + ImprintRecord record = ImprintRecord.deserialize(serializedSmall5); + return record.getValue(1).map(this::extractString).orElse(null); + } + + @Benchmark + public String accessMediumString500() throws Exception { + ImprintRecord record = ImprintRecord.deserialize(serializedMedium500); + return record.getValue(1).map(this::extractString).orElse(null); + } + + @Benchmark + public String accessLargeString100K() throws Exception { + ImprintRecord record = ImprintRecord.deserialize(serializedLarge100K); + return record.getValue(1).map(this::extractString).orElse(null); + } + + // Raw bytes access benchmarks (zero-copy) + + @Benchmark + public ByteBuffer getRawBytesSmallString5() throws Exception { + ImprintRecord record = ImprintRecord.deserialize(serializedSmall5); + return record.getRawBytes(1).orElse(null); + } + + @Benchmark + public ByteBuffer getRawBytesMediumString500() throws Exception { + ImprintRecord record = ImprintRecord.deserialize(serializedMedium500); + return record.getRawBytes(1).orElse(null); + } + + @Benchmark + public ByteBuffer getRawBytesLargeString100K() throws Exception { + ImprintRecord record = ImprintRecord.deserialize(serializedLarge100K); + return record.getRawBytes(1).orElse(null); + } + + // Size measurement benchmarks + + @Benchmark + public int measureSmallString5Size() throws Exception { + return createStringRecord(smallString5).serializeToBuffer().remaining(); + } + + @Benchmark + public int measureMediumString500Size() throws Exception { + return createStringRecord(mediumString500).serializeToBuffer().remaining(); + } + + @Benchmark + public int measureLargeString100KSize() throws Exception { + return createStringRecord(largeString100K).serializeToBuffer().remaining(); + } + + public static void main(String[] args) throws Exception { + Options opt = new OptionsBuilder() + .include(StringBenchmark.class.getSimpleName()) + .build(); + new Runner(opt).run(); + } +} \ No newline at end of file From 2853e3fd18b85eda7e14c25407670ecc8f22a5b2 Mon Sep 17 00:00:00 2001 From: expand3d <> Date: Sun, 1 Jun 2025 23:44:13 -0400 Subject: [PATCH 12/49] fix map serialization error in benchmark test and streamline ci file to remove a bunch of stuff --- .github/workflows/ci.yml | 64 +------------------ .../benchmark/SerializationBenchmark.java | 4 +- 2 files changed, 4 insertions(+), 64 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2e906c6..0ad6f6b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,7 +25,7 @@ jobs: distribution: 'temurin' - name: Cache Gradle dependencies - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: | ~/.gradle/caches @@ -34,68 +34,8 @@ jobs: restore-keys: | ${{ runner.os }}-gradle- - - name: Make gradlew executable - run: chmod +x ./gradlew - - - name: Validate Gradle wrapper - uses: gradle/wrapper-validation-action@v3 - - name: Run tests run: ./gradlew test - name: Run build - run: ./gradlew build - - - name: Upload test results - uses: actions/upload-artifact@v4 - if: always() - with: - name: test-results-java-${{ matrix.java-version }} - path: build/test-results/test/ - - - name: Upload build reports - uses: actions/upload-artifact@v4 - if: always() - with: - name: build-reports-java-${{ matrix.java-version }} - path: build/reports/ - - benchmark: - runs-on: ubuntu-latest - needs: test - if: github.event_name == 'pull_request' - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up JDK 17 - uses: actions/setup-java@v4 - with: - java-version: 17 - distribution: 'temurin' - - - name: Cache Gradle dependencies - uses: actions/cache@v3 - with: - path: | - ~/.gradle/caches - ~/.gradle/wrapper - key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} - restore-keys: | - ${{ runner.os }}-gradle- - - - name: Make gradlew executable - run: chmod +x ./gradlew - - - name: Validate Gradle wrapper - uses: gradle/wrapper-validation-action@v3 - - - name: Run quick benchmark - run: ./gradlew jmh -Pjmh.fork=1 -Pjmh.warmupIterations=1 -Pjmh.iterations=1 - - - name: Upload benchmark results - uses: actions/upload-artifact@v4 - with: - name: benchmark-results - path: build/results/jmh/ \ No newline at end of file + run: ./gradlew build \ No newline at end of file diff --git a/src/jmh/java/com/imprint/benchmark/SerializationBenchmark.java b/src/jmh/java/com/imprint/benchmark/SerializationBenchmark.java index 2544b88..3275843 100644 --- a/src/jmh/java/com/imprint/benchmark/SerializationBenchmark.java +++ b/src/jmh/java/com/imprint/benchmark/SerializationBenchmark.java @@ -119,11 +119,11 @@ private ImprintRecord createMediumRecord() throws Exception { ); writer.addField(6, Value.fromArray(tags)); - // Add map field + // Add map field (all string values for consistency) var metadata = new HashMap(); metadata.put(MapKey.fromString("manufacturer"), Value.fromString("TechCorp")); metadata.put(MapKey.fromString("model"), Value.fromString("TC-2024")); - metadata.put(MapKey.fromString("year"), Value.fromInt32(2024)); + metadata.put(MapKey.fromString("year"), Value.fromString("2024")); writer.addField(7, Value.fromMap(metadata)); // Add more fields for medium size From 3a5a113ff093933da07e4c9c5fded1c8109b7fca Mon Sep 17 00:00:00 2001 From: expand3d <> Date: Sun, 1 Jun 2025 23:45:45 -0400 Subject: [PATCH 13/49] Add execute permissions back for gradlew in CI --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0ad6f6b..d4c8bde 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,6 +34,9 @@ jobs: restore-keys: | ${{ runner.os }}-gradle- + - name: Make gradlew executable + run: chmod +x ./gradlew + - name: Run tests run: ./gradlew test From 50a288b19e0eb142f7dcd1b8d0d57b334b540b07 Mon Sep 17 00:00:00 2001 From: expand3d <> Date: Mon, 2 Jun 2025 01:18:43 -0400 Subject: [PATCH 14/49] Add some more string based performance benchmarks and try to make string deserialization a bit faster --- .../imprint/benchmark/StringBenchmark.java | 32 ++++++++++ .../java/com/imprint/core/ImprintRecord.java | 4 +- .../java/com/imprint/types/TypeHandler.java | 9 ++- src/main/java/com/imprint/types/Value.java | 58 +++++++++++++------ 4 files changed, 78 insertions(+), 25 deletions(-) diff --git a/src/jmh/java/com/imprint/benchmark/StringBenchmark.java b/src/jmh/java/com/imprint/benchmark/StringBenchmark.java index 1891251..dda7f1c 100644 --- a/src/jmh/java/com/imprint/benchmark/StringBenchmark.java +++ b/src/jmh/java/com/imprint/benchmark/StringBenchmark.java @@ -44,6 +44,10 @@ public class StringBenchmark { private byte[] serializedLarge10K; private byte[] serializedLarge100K; private byte[] serializedLarge1M; + + private ImprintRecord preDeserializedSmall5; + private ImprintRecord preDeserializedMedium500; + private ImprintRecord preDeserializedLarge100K; @Setup public void setup() throws Exception { @@ -66,6 +70,10 @@ public void setup() throws Exception { serializedLarge10K = bufferToArray(createStringRecord(largeString10K).serializeToBuffer()); serializedLarge100K = bufferToArray(createStringRecord(largeString100K).serializeToBuffer()); serializedLarge1M = bufferToArray(createStringRecord(largeString1M).serializeToBuffer()); + + preDeserializedSmall5 = ImprintRecord.deserialize(serializedSmall5); + preDeserializedMedium500 = ImprintRecord.deserialize(serializedMedium500); + preDeserializedLarge100K = ImprintRecord.deserialize(serializedLarge100K); } private String generateString(int length) { @@ -238,6 +246,30 @@ public int measureMediumString500Size() throws Exception { public int measureLargeString100KSize() throws Exception { return createStringRecord(largeString100K).serializeToBuffer().remaining(); } + + // Pure string access benchmarks (no record deserialization overhead) + @Benchmark + public String pureStringAccessSmall5() throws Exception { + return preDeserializedSmall5.getValue(1).map(this::extractString).orElse(null); + } + + @Benchmark + public String pureStringAccessMedium500() throws Exception { + return preDeserializedMedium500.getValue(1).map(this::extractString).orElse(null); + } + + @Benchmark + public String pureStringAccessLarge100K() throws Exception { + return preDeserializedLarge100K.getValue(1).map(this::extractString).orElse(null); + } + + // Test cached vs uncached access + @Benchmark + public String cachedStringAccessSmall5() throws Exception { + // Second access should hit cache + preDeserializedSmall5.getValue(1).map(this::extractString).orElse(null); + return preDeserializedSmall5.getValue(1).map(this::extractString).orElse(null); + } public static void main(String[] args) throws Exception { Options opt = new OptionsBuilder() diff --git a/src/main/java/com/imprint/core/ImprintRecord.java b/src/main/java/com/imprint/core/ImprintRecord.java index b7ed224..d9d5659 100644 --- a/src/main/java/com/imprint/core/ImprintRecord.java +++ b/src/main/java/com/imprint/core/ImprintRecord.java @@ -114,14 +114,14 @@ public static ImprintRecordBuilder builder(int fieldspaceId, int schemaHash) { } /** - * Deserialize a record from bytes. + * Deserialize a record from bytes through an array backed ByteBuffer. */ public static ImprintRecord deserialize(byte[] bytes) throws ImprintException { return deserialize(ByteBuffer.wrap(bytes)); } /** - * Deserialize a record from a ByteBuffer (zero-copy when possible). + * Deserialize a record from a ByteBuffer. */ public static ImprintRecord deserialize(ByteBuffer buffer) throws ImprintException { buffer = buffer.duplicate().order(ByteOrder.LITTLE_ENDIAN); diff --git a/src/main/java/com/imprint/types/TypeHandler.java b/src/main/java/com/imprint/types/TypeHandler.java index 4b5830a..07ef7a1 100644 --- a/src/main/java/com/imprint/types/TypeHandler.java +++ b/src/main/java/com/imprint/types/TypeHandler.java @@ -4,7 +4,6 @@ import com.imprint.util.VarInt; import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; /** * Interface for handling type-specific serialization, deserialization, and size estimation. @@ -270,7 +269,7 @@ public void serialize(Value value, ByteBuffer buffer) { buffer.put(stringBuffer); } else { Value.StringValue stringValue = (Value.StringValue) value; - byte[] stringBytes = stringValue.getValue().getBytes(StandardCharsets.UTF_8); + byte[] stringBytes = stringValue.getUtf8Bytes(); // Use cached UTF-8 bytes VarInt.encode(stringBytes.length, buffer); buffer.put(stringBytes); } @@ -283,9 +282,9 @@ public int estimateSize(Value value) { int length = bufferValue.getBuffer().remaining(); return VarInt.encodedLength(length) + length; } else { - String str = ((Value.StringValue) value).getValue(); - int utf8Length = str.getBytes(StandardCharsets.UTF_8).length; - return VarInt.encodedLength(utf8Length) + utf8Length; + Value.StringValue stringValue = (Value.StringValue) value; + byte[] utf8Bytes = stringValue.getUtf8Bytes(); // Use cached UTF-8 bytes + return VarInt.encodedLength(utf8Bytes.length) + utf8Bytes.length; } } diff --git a/src/main/java/com/imprint/types/Value.java b/src/main/java/com/imprint/types/Value.java index 4710ec5..64eae91 100644 --- a/src/main/java/com/imprint/types/Value.java +++ b/src/main/java/com/imprint/types/Value.java @@ -270,15 +270,26 @@ public String toString() { } // String Value (String-based) - @Getter @EqualsAndHashCode(callSuper = false) public static class StringValue extends Value { + @Getter private final String value; + private volatile byte[] cachedUtf8Bytes; // Cache UTF-8 encoding public StringValue(String value) { this.value = Objects.requireNonNull(value, "String cannot be null"); } + public byte[] getUtf8Bytes() { + byte[] cached = cachedUtf8Bytes; + if (cached == null) { + // Multiple threads may compute this - that's OK since it's idempotent + cached = value.getBytes(StandardCharsets.UTF_8); + cachedUtf8Bytes = cached; // Benign race - last writer wins + } + return cached; // Return our computed value, not re-read from volatile field + } + @Override public TypeCode getTypeCode() { return TypeCode.STRING; } @@ -288,35 +299,46 @@ public String toString() { } } - // String Value (ByteBuffer-based, zero-copy) + // String Value (ByteBuffer-based) public static class StringBufferValue extends Value { private final ByteBuffer value; private volatile String cachedString; // lazy decode - + public StringBufferValue(ByteBuffer value) { this.value = value.asReadOnlyBuffer(); // zero-copy read-only view } - + public String getValue() { - if (cachedString == null) { - synchronized (this) { - if (cachedString == null) { - var array = new byte[value.remaining()]; - value.duplicate().get(array); - cachedString = new String(array, StandardCharsets.UTF_8); - } - } + String result = cachedString; + if (result == null) { + // Simple, fast decoding - no thread-local overhead + result = decodeUtf8(); + cachedString = result; } - return cachedString; + return result; } - + + private String decodeUtf8() { + // Fast path: zero-copy for array-backed ByteBuffers + if (value.hasArray() && value.arrayOffset() == 0) { + return new String(value.array(), value.position(), + value.remaining(), StandardCharsets.UTF_8); + } + + // Fallback path - should be impossible since deserialize uses wrap() to create an array-backed ByteBuffer. + // Allocation required for direct ByteBuffers since Java's String API doesn't provide ByteBuffer constructors + var array = new byte[value.remaining()]; + value.duplicate().get(array); + return new String(array, StandardCharsets.UTF_8); + } + public ByteBuffer getBuffer() { return value.duplicate(); // zero-copy view } - + @Override public TypeCode getTypeCode() { return TypeCode.STRING; } - + @Override public boolean equals(Object obj) { if (this == obj) return true; @@ -331,12 +353,12 @@ public boolean equals(Object obj) { } return false; } - + @Override public int hashCode() { return getValue().hashCode(); // Use string hash for consistency } - + @Override public String toString() { return "\"" + getValue() + "\""; From 43cab28d2b532be7b03e74533a7e1c1c1d7fdc8c Mon Sep 17 00:00:00 2001 From: Brent Johnson Date: Tue, 3 Jun 2025 18:28:49 -0400 Subject: [PATCH 15/49] second main commit to address initial commits A full list of enhancements can be found here - https://github.com/imprint-serde/imprint-java/issues/3 --- .../benchmark/ComparisonBenchmark.java | 69 ++-- .../benchmark/FieldAccessBenchmark.java | 4 +- .../com/imprint/benchmark/MergeBenchmark.java | 4 +- .../imprint/benchmark/StringBenchmark.java | 67 ++- src/main/java/com/imprint/Constants.java | 4 +- .../java/com/imprint/core/ImprintRecord.java | 168 ++------ .../imprint/core/ImprintRecordBuilder.java | 19 +- .../java/com/imprint/core/ImprintWriter.java | 155 +------ src/main/java/com/imprint/core/SchemaId.java | 2 +- src/main/java/com/imprint/types/TypeCode.java | 4 +- .../java/com/imprint/types/TypeHandler.java | 386 ++++++++++++++++-- src/main/java/com/imprint/types/Value.java | 47 ++- src/main/java/com/imprint/util/VarInt.java | 20 + .../imprint/ByteBufferIntegrationTest.java | 157 +++++-- .../java/com/imprint/ComprehensiveTest.java | 208 ---------- .../java/com/imprint/IntegrationTest.java | 32 +- .../com/imprint/benchmark/ProfilerTest.java | 18 +- .../core/ImprintRecordBuilderTest.java | 234 ----------- .../com/imprint/core/ImprintRecordTest.java | 76 ++-- .../java/com/imprint/types/ValueTest.java | 115 +++++- 20 files changed, 871 insertions(+), 918 deletions(-) delete mode 100644 src/test/java/com/imprint/ComprehensiveTest.java delete mode 100644 src/test/java/com/imprint/core/ImprintRecordBuilderTest.java diff --git a/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java b/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java index 1293478..152bb6d 100644 --- a/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java +++ b/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java @@ -74,7 +74,7 @@ public void serializeJackson(Blackhole bh) throws Exception { } @Benchmark - public void serializeKryo(Blackhole bh) throws Exception { + public void serializeKryo(Blackhole bh) { byte[] result = serializeWithKryo(testData); bh.consume(result); } @@ -102,49 +102,36 @@ public void deserializeKryo(Blackhole bh) { } // ===== FIELD ACCESS BENCHMARKS ===== + // Tests accessing a single field near the end of a large record + // This showcases Imprint's O(1) directory lookup vs sequential deserialization @Benchmark - public void fieldAccessImprint(Blackhole bh) throws Exception { + public void singleFieldAccessImprint(Blackhole bh) throws Exception { ImprintRecord record = ImprintRecord.deserialize(imprintBytes.duplicate()); - // Access multiple fields without full deserialization - var id = record.getValue(1); - var name = record.getValue(2); - var price = record.getValue(3); - var active = record.getValue(4); - var category = record.getValue(5); - - bh.consume(id); - bh.consume(name); - bh.consume(price); - bh.consume(active); - bh.consume(category); + // Access field 15 directly via directory lookup - O(1) + var field15 = record.getValue(15); + bh.consume(field15); } @Benchmark - public void fieldAccessJackson(Blackhole bh) throws Exception { - // Jackson requires full deserialization to access fields + public void singleFieldAccessJackson(Blackhole bh) throws Exception { + // Jackson must deserialize entire object to access any field TestRecord record = jackson.readValue(jacksonBytes, TestRecord.class); - bh.consume(record.id); - bh.consume(record.name); - bh.consume(record.price); - bh.consume(record.active); - bh.consume(record.category); + // Access field15 equivalent (extraData[4]) after full deserialization + bh.consume(record.extraData.get(4)); } @Benchmark - public void fieldAccessKryo(Blackhole bh) { - // Kryo requires full deserialization to access fields + public void singleFieldAccessKryo(Blackhole bh) { + // Kryo must deserialize entire object to access any field Input input = new Input(new ByteArrayInputStream(kryoBytes)); TestRecord record = kryo.readObject(input, TestRecord.class); input.close(); - bh.consume(record.id); - bh.consume(record.name); - bh.consume(record.price); - bh.consume(record.active); - bh.consume(record.category); + // Access field15 equivalent (extraData[4]) after full deserialization + bh.consume(record.extraData.get(4)); } // ===== SIZE COMPARISON ===== @@ -162,7 +149,7 @@ public void measureJacksonSize(Blackhole bh) throws Exception { } @Benchmark - public void measureKryoSize(Blackhole bh) throws Exception { + public void measureKryoSize(Blackhole bh) { byte[] serialized = serializeWithKryo(testData); bh.consume(serialized.length); } @@ -196,7 +183,7 @@ public void mergeJackson(Blackhole bh) throws Exception { } @Benchmark - public void mergeKryo(Blackhole bh) throws Exception { + public void mergeKryo(Blackhole bh) { // Kryo merge requires full deserialization + merge + serialization Input input1 = new Input(new ByteArrayInputStream(kryoBytes)); var record1 = kryo.readObject(input1, TestRecord.class); @@ -237,6 +224,11 @@ private ByteBuffer serializeWithImprint(TestRecord data) throws Exception { } writer.addField(7, Value.fromMap(metadataMap)); + // Add extra fields (8-20) to create a larger record + for (int i = 0; i < data.extraData.size(); i++) { + writer.addField(8 + i, Value.fromString(data.extraData.get(i))); + } + return writer.build().serializeToBuffer(); } @@ -268,8 +260,8 @@ private void copyFieldsToWriter(ImprintRecord record, ImprintWriter writer, Set< int fieldId = entry.getId(); if (!usedFieldIds.contains(fieldId)) { var value = record.getValue(fieldId); - if (value.isPresent()) { - writer.addField(fieldId, value.get()); + if (value != null) { + writer.addField(fieldId, value); usedFieldIds.add(fieldId); } } @@ -309,6 +301,12 @@ var record = new TestRecord(); record.metadata.put("model", "TC-2024"); record.metadata.put("warranty", "2 years"); + // Add extra data to create a larger record (fields 8-20) + record.extraData = new ArrayList<>(); + for (int i = 0; i < 13; i++) { + record.extraData.add("extraField" + i + "_value_" + (1000 + i)); + } + return record; } @@ -326,6 +324,12 @@ var record = new TestRecord(); record.metadata.put("vendor", "SoftCorp"); record.metadata.put("version", "2.1"); + // Add extra data to match the structure + record.extraData = new ArrayList<>(); + for (int i = 0; i < 13; i++) { + record.extraData.add("extraField" + i + "_value2_" + (2000 + i)); + } + return record; } @@ -338,6 +342,7 @@ public static class TestRecord { public String category; public List tags = new ArrayList<>(); public Map metadata = new HashMap<>(); + public List extraData = new ArrayList<>(); // Fields 8-20 for large record test public TestRecord() {} // Required for deserialization } diff --git a/src/jmh/java/com/imprint/benchmark/FieldAccessBenchmark.java b/src/jmh/java/com/imprint/benchmark/FieldAccessBenchmark.java index f3abb7e..1ead21f 100644 --- a/src/jmh/java/com/imprint/benchmark/FieldAccessBenchmark.java +++ b/src/jmh/java/com/imprint/benchmark/FieldAccessBenchmark.java @@ -200,7 +200,9 @@ private ImprintRecord simulateProject(ImprintRecord source, int[] fieldIds) thro for (int fieldId : fieldIds) { var value = source.getValue(fieldId); - value.ifPresent(value1 -> writer.addField(fieldId, value1)); + if (value != null) { + writer.addField(fieldId, value); + } } return writer.build(); diff --git a/src/jmh/java/com/imprint/benchmark/MergeBenchmark.java b/src/jmh/java/com/imprint/benchmark/MergeBenchmark.java index 5c52908..f93092a 100644 --- a/src/jmh/java/com/imprint/benchmark/MergeBenchmark.java +++ b/src/jmh/java/com/imprint/benchmark/MergeBenchmark.java @@ -100,8 +100,8 @@ private void copyFieldsToWriter(ImprintRecord record, ImprintWriter writer, Set< int fieldId = entry.getId(); if (!usedFieldIds.contains(fieldId)) { var value = record.getValue(fieldId); - if (value.isPresent()) { - writer.addField(fieldId, value.get()); + if (value != null) { + writer.addField(fieldId, value); usedFieldIds.add(fieldId); } } diff --git a/src/jmh/java/com/imprint/benchmark/StringBenchmark.java b/src/jmh/java/com/imprint/benchmark/StringBenchmark.java index dda7f1c..e156c8c 100644 --- a/src/jmh/java/com/imprint/benchmark/StringBenchmark.java +++ b/src/jmh/java/com/imprint/benchmark/StringBenchmark.java @@ -5,7 +5,6 @@ import com.imprint.types.Value; import org.openjdk.jmh.annotations.*; import org.openjdk.jmh.runner.Runner; -import org.openjdk.jmh.runner.options.Options; import org.openjdk.jmh.runner.options.OptionsBuilder; import java.nio.ByteBuffer; @@ -15,8 +14,8 @@ @OutputTimeUnit(TimeUnit.NANOSECONDS) @State(Scope.Benchmark) @Fork(1) -@Warmup(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS) -@Measurement(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS) +@Warmup(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 10, time = 1, timeUnit = TimeUnit.SECONDS) public class StringBenchmark { private static final SchemaId SCHEMA_ID = new SchemaId(1, 42); @@ -195,19 +194,22 @@ public ImprintRecord deserializeLargeString1M() throws Exception { @Benchmark public String accessSmallString5() throws Exception { ImprintRecord record = ImprintRecord.deserialize(serializedSmall5); - return record.getValue(1).map(this::extractString).orElse(null); + Value value = record.getValue(1); + return value != null ? extractString(value) : null; } @Benchmark public String accessMediumString500() throws Exception { ImprintRecord record = ImprintRecord.deserialize(serializedMedium500); - return record.getValue(1).map(this::extractString).orElse(null); + Value value = record.getValue(1); + return value != null ? extractString(value) : null; } @Benchmark public String accessLargeString100K() throws Exception { ImprintRecord record = ImprintRecord.deserialize(serializedLarge100K); - return record.getValue(1).map(this::extractString).orElse(null); + Value value = record.getValue(1); + return value != null ? extractString(value) : null; } // Raw bytes access benchmarks (zero-copy) @@ -215,19 +217,19 @@ public String accessLargeString100K() throws Exception { @Benchmark public ByteBuffer getRawBytesSmallString5() throws Exception { ImprintRecord record = ImprintRecord.deserialize(serializedSmall5); - return record.getRawBytes(1).orElse(null); + return record.getRawBytes(1); } @Benchmark public ByteBuffer getRawBytesMediumString500() throws Exception { ImprintRecord record = ImprintRecord.deserialize(serializedMedium500); - return record.getRawBytes(1).orElse(null); + return record.getRawBytes(1); } @Benchmark public ByteBuffer getRawBytesLargeString100K() throws Exception { ImprintRecord record = ImprintRecord.deserialize(serializedLarge100K); - return record.getRawBytes(1).orElse(null); + return record.getRawBytes(1); } // Size measurement benchmarks @@ -250,30 +252,63 @@ public int measureLargeString100KSize() throws Exception { // Pure string access benchmarks (no record deserialization overhead) @Benchmark public String pureStringAccessSmall5() throws Exception { - return preDeserializedSmall5.getValue(1).map(this::extractString).orElse(null); + Value value = preDeserializedSmall5.getValue(1); + return value != null ? extractString(value) : null; } @Benchmark public String pureStringAccessMedium500() throws Exception { - return preDeserializedMedium500.getValue(1).map(this::extractString).orElse(null); + Value value = preDeserializedMedium500.getValue(1); + return value != null ? extractString(value) : null; } @Benchmark public String pureStringAccessLarge100K() throws Exception { - return preDeserializedLarge100K.getValue(1).map(this::extractString).orElse(null); + Value value = preDeserializedLarge100K.getValue(1); + return value != null ? extractString(value) : null; } // Test cached vs uncached access @Benchmark public String cachedStringAccessSmall5() throws Exception { // Second access should hit cache - preDeserializedSmall5.getValue(1).map(this::extractString).orElse(null); - return preDeserializedSmall5.getValue(1).map(this::extractString).orElse(null); + Value value1 = preDeserializedSmall5.getValue(1); + String result1 = value1 != null ? extractString(value1) : null; + Value value2 = preDeserializedSmall5.getValue(1); + return value2 != null ? extractString(value2) : null; } public static void main(String[] args) throws Exception { - Options opt = new OptionsBuilder() - .include(StringBenchmark.class.getSimpleName()) + runDeserializationOnly(); + } + + public static void runAll() throws Exception { + var opt = new OptionsBuilder() + .include(StringBenchmark.class.getSimpleName()) + .build(); + new Runner(opt).run(); + } + + /** + * Run only string deserialization benchmarks to measure the impact of + * ThreadLocal buffer pool optimization and fast/fallback path performance. + */ + public static void runDeserializationOnly() throws Exception { + var opt = new OptionsBuilder() + .include(StringBenchmark.class.getSimpleName() + ".*deserialize.*") // Only deserialize methods + .forks(0) // Run in same JVM to avoid serialization issues + .build(); + new Runner(opt).run(); + } + + /** + * Run only pure string access benchmarks (no record deserialization overhead) + * to isolate string decode performance with ThreadLocal buffer optimization. + */ + public static void runStringAccessOnly() throws Exception { + var opt = new OptionsBuilder() + .include(StringBenchmark.class.getSimpleName() + ".*(pureStringAccess|cachedStringAccess).*") // Only pure string access methods + .forks(0) // Run in same JVM to avoid serialization issues .build(); new Runner(opt).run(); } diff --git a/src/main/java/com/imprint/Constants.java b/src/main/java/com/imprint/Constants.java index 78b91a0..3c84a28 100644 --- a/src/main/java/com/imprint/Constants.java +++ b/src/main/java/com/imprint/Constants.java @@ -2,9 +2,7 @@ public final class Constants { public static final byte MAGIC = 0x49; - public static final byte VERSION = 0x02; + public static final byte VERSION = 0x01; public static final int HEADER_BYTES = 15; public static final int DIR_ENTRY_BYTES = 7; - - private Constants() {} } diff --git a/src/main/java/com/imprint/core/ImprintRecord.java b/src/main/java/com/imprint/core/ImprintRecord.java index d9d5659..1915707 100644 --- a/src/main/java/com/imprint/core/ImprintRecord.java +++ b/src/main/java/com/imprint/core/ImprintRecord.java @@ -1,10 +1,8 @@ - package com.imprint.core; import com.imprint.Constants; import com.imprint.error.ErrorType; import com.imprint.error.ImprintException; -import com.imprint.types.MapKey; import com.imprint.types.TypeCode; import com.imprint.types.Value; import com.imprint.util.VarInt; @@ -15,8 +13,12 @@ import java.util.*; /** - * An Imprint record containing a header, optional field directory, and payload. + * An Imprint record containing a header, field directory, and payload. * Uses ByteBuffer for zero-copy operations to achieve low latency. + * + *

Performance Note: All ByteBuffers should be array-backed + * (hasArray() == true) for optimal zero-copy performance. Direct buffers + * may cause performance degradation.

*/ @Getter public final class ImprintRecord { @@ -24,6 +26,11 @@ public final class ImprintRecord { private final List directory; private final ByteBuffer payload; // Read-only view for zero-copy + /** + * Creates a new ImprintRecord. + * + * @param payload the payload buffer. Should be array-backed for optimal performance. + */ public ImprintRecord(Header header, List directory, ByteBuffer payload) { this.header = Objects.requireNonNull(header, "Header cannot be null"); this.directory = List.copyOf(Objects.requireNonNull(directory, "Directory cannot be null")); @@ -32,39 +39,44 @@ public ImprintRecord(Header header, List directory, ByteBuffer p /** * Get a value by field ID, deserializing it on demand. + * Returns null if the field is not found. */ - public Optional getValue(int fieldId) throws ImprintException { - // Binary search for the field ID without allocation - int index = findDirectoryIndex(fieldId); - if (index < 0) return Optional.empty(); + public Value getValue(int fieldId) throws ImprintException { + var fieldBuffer = getFieldBuffer(fieldId); + if (fieldBuffer == null) return null; - var entry = directory.get(index); - int startOffset = entry.getOffset(); - int endOffset = (index + 1 < directory.size()) ? - directory.get(index + 1).getOffset() : payload.remaining(); - - var valueBytes = payload.duplicate(); - valueBytes.position(startOffset).limit(endOffset); - var value = deserializeValue(entry.getTypeCode(), valueBytes.slice()); - return Optional.of(value); + var entry = directory.get(findDirectoryIndex(fieldId)); + return deserializeValue(entry.getTypeCode(), fieldBuffer); } /** * Get the raw bytes for a field without deserializing. - * Returns a zero-copy ByteBuffer view. + * Returns a zero-copy ByteBuffer view, or null if field not found. + */ + public ByteBuffer getRawBytes(int fieldId) { + var fieldBuffer = getFieldBuffer(fieldId); + return fieldBuffer != null ? fieldBuffer.asReadOnlyBuffer() : null; + } + + /** + * Get a ByteBuffer view of a field's data. + * Returns null if the field is not found. */ - public Optional getRawBytes(int fieldId) { + private ByteBuffer getFieldBuffer(int fieldId) { int index = findDirectoryIndex(fieldId); - if (index < 0) return Optional.empty(); - + if (index < 0) return null; + var entry = directory.get(index); int startOffset = entry.getOffset(); int endOffset = (index + 1 < directory.size()) ? - directory.get(index + 1).getOffset() : payload.remaining(); + directory.get(index + 1).getOffset() : payload.limit(); var fieldBuffer = payload.duplicate(); + if (startOffset > payload.limit() || endOffset > payload.limit() || startOffset > endOffset) { + return null; + } fieldBuffer.position(startOffset).limit(endOffset); - return Optional.of(fieldBuffer.slice().asReadOnlyBuffer()); + return fieldBuffer.slice(); } /** @@ -122,6 +134,9 @@ public static ImprintRecord deserialize(byte[] bytes) throws ImprintException { /** * Deserialize a record from a ByteBuffer. + * + * @param buffer the buffer to deserialize from. Must be array-backed + * (buffer.hasArray() == true) for optimal zero-copy performance. */ public static ImprintRecord deserialize(ByteBuffer buffer) throws ImprintException { buffer = buffer.duplicate().order(ByteOrder.LITTLE_ENDIAN); @@ -172,7 +187,7 @@ private int findDirectoryIndex(int fieldId) { return -(low + 1); // field not found, return insertion point } - private int estimateSerializedSize() { + public int estimateSerializedSize() { int size = Constants.HEADER_BYTES; // header size += VarInt.encodedLength(directory.size()); // directory count size += directory.size() * Constants.DIR_ENTRY_BYTES; // directory entries @@ -184,7 +199,7 @@ private void serializeHeader(ByteBuffer buffer) { buffer.put(Constants.MAGIC); buffer.put(Constants.VERSION); buffer.put(header.getFlags().getValue()); - buffer.putInt(header.getSchemaId().getFieldspaceId()); + buffer.putInt(header.getSchemaId().getFieldSpaceId()); buffer.putInt(header.getSchemaId().getSchemaHash()); buffer.putInt(header.getPayloadSize()); } @@ -236,10 +251,9 @@ private static DirectoryEntry deserializeDirectoryEntry(ByteBuffer buffer) throw } private Value deserializeValue(TypeCode typeCode, ByteBuffer buffer) throws ImprintException { - // Buffer is already positioned and limited correctly - buffer = buffer.duplicate().order(ByteOrder.LITTLE_ENDIAN); + ByteBuffer valueSpecificBuffer = buffer.duplicate(); + valueSpecificBuffer.order(ByteOrder.LITTLE_ENDIAN); - // Use TypeHandler for simple types switch (typeCode) { case NULL: case BOOL: @@ -249,17 +263,11 @@ private Value deserializeValue(TypeCode typeCode, ByteBuffer buffer) throws Impr case FLOAT64: case BYTES: case STRING: - return typeCode.getHandler().deserialize(buffer); - case ARRAY: - return deserializeArray(buffer); - case MAP: - return deserializeMap(buffer); - + return typeCode.getHandler().deserialize(valueSpecificBuffer); case ROW: - var remainingBuffer = buffer.slice(); - var nestedRecord = deserialize(remainingBuffer); + var nestedRecord = deserialize(valueSpecificBuffer); return Value.fromRow(nestedRecord); default: @@ -267,96 +275,6 @@ private Value deserializeValue(TypeCode typeCode, ByteBuffer buffer) throws Impr } } - private Value deserializeArray(ByteBuffer buffer) throws ImprintException { - VarInt.DecodeResult lengthResult = VarInt.decode(buffer); - int length = lengthResult.getValue(); - - if (length == 0) { - return Value.fromArray(Collections.emptyList()); - } - - var elementType = TypeCode.fromByte(buffer.get()); - var elements = new ArrayList(length); - - for (int i = 0; i < length; i++) { - var elementBytes = readValueBytes(elementType, buffer); - var element = deserializeValue(elementType, elementBytes); - elements.add(element); - } - - return Value.fromArray(elements); - } - - private Value deserializeMap(ByteBuffer buffer) throws ImprintException { - VarInt.DecodeResult lengthResult = VarInt.decode(buffer); - int length = lengthResult.getValue(); - - if (length == 0) { - return Value.fromMap(Collections.emptyMap()); - } - - var keyType = TypeCode.fromByte(buffer.get()); - var valueType = TypeCode.fromByte(buffer.get()); - var map = new HashMap(length); - - for (int i = 0; i < length; i++) { - // Read key - var keyBytes = readValueBytes(keyType, buffer); - var keyValue = deserializeValue(keyType, keyBytes); - var key = MapKey.fromValue(keyValue); - - // Read value - var valueBytes = readValueBytes(valueType, buffer); - var value = deserializeValue(valueType, valueBytes); - - map.put(key, value); - } - - return Value.fromMap(map); - } - - private ByteBuffer readValueBytes(TypeCode typeCode, ByteBuffer buffer) throws ImprintException { - // Use TypeHandler for simple types - switch (typeCode) { - case NULL: - case BOOL: - case INT32: - case INT64: - case FLOAT32: - case FLOAT64: - case BYTES: - case STRING: - return typeCode.getHandler().readValueBytes(buffer); - - case ARRAY: - case MAP: - case ROW: - // For complex types, return the entire remaining buffer for now - // The specific deserializer will handle parsing in the future - var remainingBuffer = buffer.slice(); - buffer.position(buffer.limit()); - return remainingBuffer.asReadOnlyBuffer(); - - default: - throw new ImprintException(ErrorType.INVALID_TYPE_CODE, "Unknown type code: " + typeCode); - } - } - - @Override - public boolean equals(Object obj) { - if (this == obj) return true; - if (obj == null || getClass() != obj.getClass()) return false; - var that = (ImprintRecord) obj; - return header.equals(that.header) && - directory.equals(that.directory) && - payload.equals(that.payload); - } - - @Override - public int hashCode() { - return Objects.hash(header, directory, payload); - } - @Override public String toString() { return String.format("ImprintRecord{header=%s, directorySize=%d, payloadSize=%d}", diff --git a/src/main/java/com/imprint/core/ImprintRecordBuilder.java b/src/main/java/com/imprint/core/ImprintRecordBuilder.java index 202bd2a..8c04bf5 100644 --- a/src/main/java/com/imprint/core/ImprintRecordBuilder.java +++ b/src/main/java/com/imprint/core/ImprintRecordBuilder.java @@ -10,11 +10,16 @@ * A fluent builder for creating ImprintRecord instances with type-safe, * developer-friendly API that eliminates boilerplate Value.fromX() calls. *

+ * Field IDs can be overwritten - calling field() with the same ID multiple times + * will replace the previous value. This allows for flexible builder patterns and + * conditional field updates. + *

* Usage: * 

  *   var record = ImprintRecord.builder(schemaId)
  *       .field(1, 42)              // int to Int32Value  
  *       .field(2, "hello")         // String to StringValue
+ *       .field(1, 100)             // overwrites field 1 with new value
  *       .field(3, 3.14)            // double to Float64Value
  *       .field(4, bytes)           // byte[] to BytesValue
  *       .field(5, true)            // boolean to BoolValue
@@ -22,6 +27,7 @@
  *       .build();
  *
*/ +@SuppressWarnings("unused") public final class ImprintRecordBuilder { private final SchemaId schemaId; private final Map fields = new TreeMap<>(); @@ -141,12 +147,17 @@ public ImprintRecord build() throws ImprintException { } // Internal helper methods + /** + * Adds or overwrites a field in the record being built. + * If a field with the given ID already exists, it will be replaced. + * + * @param id the field ID + * @param value the field value (cannot be null - use nullField() for explicit nulls) + * @return this builder for method chaining + */ private ImprintRecordBuilder addField(int id, Value value) { Objects.requireNonNull(value, "Value cannot be null - use nullField() for explicit null values"); - if (fields.containsKey(id)) { - throw new IllegalArgumentException("Field ID " + id + " already exists - field IDs must be unique"); - } - fields.put(id, value); + fields.put(id, value); // TreeMap.put() overwrites existing values return this; } diff --git a/src/main/java/com/imprint/core/ImprintWriter.java b/src/main/java/com/imprint/core/ImprintWriter.java index 39ad9ea..b1d5f53 100644 --- a/src/main/java/com/imprint/core/ImprintWriter.java +++ b/src/main/java/com/imprint/core/ImprintWriter.java @@ -2,14 +2,13 @@ import com.imprint.error.ErrorType; import com.imprint.error.ImprintException; -import com.imprint.types.MapKey; import com.imprint.types.Value; -import com.imprint.util.VarInt; import java.nio.ByteBuffer; import java.nio.ByteOrder; -import java.nio.charset.StandardCharsets; -import java.util.*; +import java.util.ArrayList; +import java.util.Objects; +import java.util.TreeMap; /** * A writer for constructing ImprintRecords by adding fields sequentially. @@ -56,7 +55,7 @@ public ImprintRecord build() throws ImprintException { return new ImprintRecord(header, directory, payloadView); } - private int estimatePayloadSize() { + private int estimatePayloadSize() throws ImprintException { // More accurate estimation to reduce allocations int estimatedSize = 0; for (var value : fields.values()) { @@ -74,7 +73,7 @@ private int estimatePayloadSize() { * @param value the value to estimate size for * @return estimated size in bytes including type-specific overhead */ - private int estimateValueSize(Value value) { + private int estimateValueSize(Value value) throws ImprintException { // Use TypeHandler for simple types switch (value.getTypeCode()) { case NULL: @@ -85,52 +84,20 @@ private int estimateValueSize(Value value) { case FLOAT64: case BYTES: case STRING: - return value.getTypeCode().getHandler().estimateSize(value); - case ARRAY: - List array = ((Value.ArrayValue) value).getValue(); - int arraySize = VarInt.encodedLength(array.size()) + 1; // length + type code - for (Value element : array) { - arraySize += estimateValueSize(element); - } - return arraySize; - case MAP: - Map map = ((Value.MapValue) value).getValue(); - int mapSize = VarInt.encodedLength(map.size()) + 2; // length + 2 type codes - for (Map.Entry entry : map.entrySet()) { - mapSize += estimateMapKeySize(entry.getKey()); - mapSize += estimateValueSize(entry.getValue()); - } - return mapSize; + return value.getTypeCode().getHandler().estimateSize(value); case ROW: - // Estimate nested record size (rough approximation) - return 100; // Conservative estimate + Value.RowValue rowValue = (Value.RowValue) value; + return rowValue.getValue().estimateSerializedSize(); default: - return 32; // Default fallback + throw new ImprintException(ErrorType.SERIALIZATION_ERROR, "Unknown type code: " + value.getTypeCode()); } } - - private int estimateMapKeySize(MapKey key) { - switch (key.getTypeCode()) { - case INT32: return 4; - case INT64: return 8; - case BYTES: - byte[] bytes = ((MapKey.BytesKey) key).getValue(); - return VarInt.encodedLength(bytes.length) + bytes.length; - case STRING: - var str = ((MapKey.StringKey) key).getValue(); - int utf8Length = str.getBytes(StandardCharsets.UTF_8).length; - return VarInt.encodedLength(utf8Length) + utf8Length; - default: - return 16; // Default fallback - } - } - private void serializeValue(Value value, ByteBuffer buffer) throws ImprintException { switch (value.getTypeCode()) { case NULL: @@ -141,17 +108,11 @@ private void serializeValue(Value value, ByteBuffer buffer) throws ImprintExcept case FLOAT64: case BYTES: case STRING: - value.getTypeCode().getHandler().serialize(value, buffer); - break; - case ARRAY: - serializeArray((Value.ArrayValue) value, buffer); - break; - case MAP: - serializeMap((Value.MapValue) value, buffer); + value.getTypeCode().getHandler().serialize(value, buffer); break; - + //TODO eliminate this switch entirely by implementing a ROW TypeHandler case ROW: Value.RowValue rowValue = (Value.RowValue) value; var serializedRow = rowValue.getValue().serializeToBuffer(); @@ -159,99 +120,7 @@ private void serializeValue(Value value, ByteBuffer buffer) throws ImprintExcept break; default: - throw new ImprintException(ErrorType.SERIALIZATION_ERROR, - "Unknown type code: " + value.getTypeCode()); - } - } - - private void serializeArray(Value.ArrayValue arrayValue, ByteBuffer buffer) throws ImprintException { - var elements = arrayValue.getValue(); - VarInt.encode(elements.size(), buffer); - - if (elements.isEmpty()) return; - - // All elements must have the same type - var elementType = elements.get(0).getTypeCode(); - buffer.put(elementType.getCode()); - for (var element : elements) { - if (element.getTypeCode() != elementType) { - throw new ImprintException(ErrorType.SCHEMA_ERROR, - "Array elements must have same type code: " + - element.getTypeCode() + " != " + elementType); - } - serializeValue(element, buffer); - } - } - - private void serializeMap(Value.MapValue mapValue, ByteBuffer buffer) throws ImprintException { - var map = mapValue.getValue(); - VarInt.encode(map.size(), buffer); - - if (map.isEmpty()) { - return; - } - - // All keys and values must have consistent types - var iterator = map.entrySet().iterator(); - var first = iterator.next(); - var keyType = first.getKey().getTypeCode(); - var valueType = first.getValue().getTypeCode(); - - buffer.put(keyType.getCode()); - buffer.put(valueType.getCode()); - - // Serialize the first entry - serializeMapKey(first.getKey(), buffer); - serializeValue(first.getValue(), buffer); - - // Serialize remaining entries - while (iterator.hasNext()) { - var entry = iterator.next(); - if (entry.getKey().getTypeCode() != keyType) { - throw new ImprintException(ErrorType.SCHEMA_ERROR, - "Map keys must have same type code: " + - entry.getKey().getTypeCode() + " != " + keyType); - } - if (entry.getValue().getTypeCode() != valueType) { - throw new ImprintException(ErrorType.SCHEMA_ERROR, - "Map values must have same type code: " + - entry.getValue().getTypeCode() + " != " + valueType); - } - - serializeMapKey(entry.getKey(), buffer); - serializeValue(entry.getValue(), buffer); - } - } - - private void serializeMapKey(MapKey key, ByteBuffer buffer) throws ImprintException { - switch (key.getTypeCode()) { - case INT32: - MapKey.Int32Key int32Key = (MapKey.Int32Key) key; - buffer.putInt(int32Key.getValue()); - break; - - case INT64: - MapKey.Int64Key int64Key = (MapKey.Int64Key) key; - buffer.putLong(int64Key.getValue()); - break; - - case BYTES: - MapKey.BytesKey bytesKey = (MapKey.BytesKey) key; - byte[] bytes = bytesKey.getValue(); - VarInt.encode(bytes.length, buffer); - buffer.put(bytes); - break; - - case STRING: - MapKey.StringKey stringKey = (MapKey.StringKey) key; - byte[] stringBytes = stringKey.getValue().getBytes(StandardCharsets.UTF_8); - VarInt.encode(stringBytes.length, buffer); - buffer.put(stringBytes); - break; - - default: - throw new ImprintException(ErrorType.SERIALIZATION_ERROR, - "Invalid map key type: " + key.getTypeCode()); + throw new ImprintException(ErrorType.SERIALIZATION_ERROR, "Unknown type code: " + value.getTypeCode()); } } } \ No newline at end of file diff --git a/src/main/java/com/imprint/core/SchemaId.java b/src/main/java/com/imprint/core/SchemaId.java index cb03c1c..b6dae3b 100644 --- a/src/main/java/com/imprint/core/SchemaId.java +++ b/src/main/java/com/imprint/core/SchemaId.java @@ -7,6 +7,6 @@ */ @Value public class SchemaId { - int fieldspaceId; + int fieldSpaceId; int schemaHash; } \ No newline at end of file diff --git a/src/main/java/com/imprint/types/TypeCode.java b/src/main/java/com/imprint/types/TypeCode.java index 6bf450d..a81b199 100644 --- a/src/main/java/com/imprint/types/TypeCode.java +++ b/src/main/java/com/imprint/types/TypeCode.java @@ -16,8 +16,8 @@ public enum TypeCode { FLOAT64(0x5, TypeHandler.FLOAT64), BYTES(0x6, TypeHandler.BYTES), STRING(0x7, TypeHandler.STRING), - ARRAY(0x8, null), // TODO: implement - MAP(0x9, null), // TODO: implement + ARRAY(0x8, TypeHandler.ARRAY), + MAP(0x9, TypeHandler.MAP), ROW(0xA, null); // TODO: implement (basically a placeholder for user-defined type) @Getter diff --git a/src/main/java/com/imprint/types/TypeHandler.java b/src/main/java/com/imprint/types/TypeHandler.java index 07ef7a1..573aed3 100644 --- a/src/main/java/com/imprint/types/TypeHandler.java +++ b/src/main/java/com/imprint/types/TypeHandler.java @@ -1,9 +1,11 @@ package com.imprint.types; +import com.imprint.error.ErrorType; import com.imprint.error.ImprintException; import com.imprint.util.VarInt; import java.nio.ByteBuffer; +import java.util.*; /** * Interface for handling type-specific serialization, deserialization, and size estimation. @@ -14,9 +16,52 @@ public interface TypeHandler { Value deserialize(ByteBuffer buffer) throws ImprintException; void serialize(Value value, ByteBuffer buffer) throws ImprintException; - int estimateSize(Value value); + int estimateSize(Value value) throws ImprintException; ByteBuffer readValueBytes(ByteBuffer buffer) throws ImprintException; + // Helper method to eliminate duplication in ARRAY/MAP readValueBytes + static ByteBuffer readComplexValueBytes(ByteBuffer buffer, String typeName, + ComplexValueMeasurer measurer) throws ImprintException { + int initialPosition = buffer.position(); + ByteBuffer tempBuffer = buffer.duplicate(); + tempBuffer.order(buffer.order()); + + VarInt.DecodeResult lengthResult = VarInt.decode(tempBuffer); + int numElements = lengthResult.getValue(); + int varIntLength = tempBuffer.position() - initialPosition; + + if (numElements == 0) { + if (buffer.remaining() < varIntLength) { + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, + "Not enough bytes for empty " + typeName + " VarInt. Needed: " + + varIntLength + ", available: " + buffer.remaining()); + } + ByteBuffer valueSlice = buffer.slice(); + valueSlice.limit(varIntLength); + buffer.position(initialPosition + varIntLength); + return valueSlice.asReadOnlyBuffer(); + } + + int dataLength = measurer.measureDataLength(tempBuffer, numElements); + int totalLength = varIntLength + dataLength; + + if (buffer.remaining() < totalLength) { + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, + "Not enough bytes for " + typeName + " value. Needed: " + totalLength + + ", available: " + buffer.remaining() + " at position " + initialPosition); + } + + ByteBuffer valueSlice = buffer.slice(); + valueSlice.limit(totalLength); + buffer.position(initialPosition + totalLength); + return valueSlice.asReadOnlyBuffer(); + } + + @FunctionalInterface + interface ComplexValueMeasurer { + int measureDataLength(ByteBuffer tempBuffer, int numElements) throws ImprintException; + } + // Static implementations for each type TypeHandler NULL = new TypeHandler() { @Override @@ -54,7 +99,7 @@ public Value deserialize(ByteBuffer buffer) throws ImprintException { @Override public void serialize(Value value, ByteBuffer buffer) { - Value.BoolValue boolValue = (Value.BoolValue) value; + var boolValue = (Value.BoolValue) value; buffer.put((byte) (boolValue.getValue() ? 1 : 0)); } @@ -76,14 +121,14 @@ public ByteBuffer readValueBytes(ByteBuffer buffer) { @Override public Value deserialize(ByteBuffer buffer) throws ImprintException { if (buffer.remaining() < 4) { - throw new ImprintException(com.imprint.error.ErrorType.BUFFER_UNDERFLOW, "Not enough bytes for int32"); + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, "Not enough bytes for int32"); } return Value.fromInt32(buffer.getInt()); } @Override public void serialize(Value value, ByteBuffer buffer) { - Value.Int32Value int32Value = (Value.Int32Value) value; + var int32Value = (Value.Int32Value) value; buffer.putInt(int32Value.getValue()); } @@ -105,7 +150,7 @@ public ByteBuffer readValueBytes(ByteBuffer buffer) { @Override public Value deserialize(ByteBuffer buffer) throws ImprintException { if (buffer.remaining() < 8) { - throw new ImprintException(com.imprint.error.ErrorType.BUFFER_UNDERFLOW, "Not enough bytes for int64"); + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, "Not enough bytes for int64"); } return Value.fromInt64(buffer.getLong()); } @@ -134,14 +179,14 @@ public ByteBuffer readValueBytes(ByteBuffer buffer) { @Override public Value deserialize(ByteBuffer buffer) throws ImprintException { if (buffer.remaining() < 4) { - throw new ImprintException(com.imprint.error.ErrorType.BUFFER_UNDERFLOW, "Not enough bytes for float32"); + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, "Not enough bytes for float32"); } return Value.fromFloat32(buffer.getFloat()); } @Override public void serialize(Value value, ByteBuffer buffer) { - Value.Float32Value float32Value = (Value.Float32Value) value; + var float32Value = (Value.Float32Value) value; buffer.putFloat(float32Value.getValue()); } @@ -163,14 +208,14 @@ public ByteBuffer readValueBytes(ByteBuffer buffer) { @Override public Value deserialize(ByteBuffer buffer) throws ImprintException { if (buffer.remaining() < 8) { - throw new ImprintException(com.imprint.error.ErrorType.BUFFER_UNDERFLOW, "Not enough bytes for float64"); + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, "Not enough bytes for float64"); } return Value.fromFloat64(buffer.getDouble()); } @Override public void serialize(Value value, ByteBuffer buffer) { - Value.Float64Value float64Value = (Value.Float64Value) value; + var float64Value = (Value.Float64Value) value; buffer.putDouble(float64Value.getValue()); } @@ -194,7 +239,7 @@ public Value deserialize(ByteBuffer buffer) throws ImprintException { VarInt.DecodeResult lengthResult = VarInt.decode(buffer); int length = lengthResult.getValue(); if (buffer.remaining() < length) { - throw new ImprintException(com.imprint.error.ErrorType.BUFFER_UNDERFLOW, "Not enough bytes for bytes value"); + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, "Not enough bytes for bytes value data after VarInt. Slice from readValueBytes is too short. Needed: " + length + ", available: " + buffer.remaining()); } var bytesView = buffer.slice(); bytesView.limit(length); @@ -231,14 +276,25 @@ public int estimateSize(Value value) { @Override public ByteBuffer readValueBytes(ByteBuffer buffer) throws ImprintException { - int originalPosition = buffer.position(); - VarInt.DecodeResult lengthResult = VarInt.decode(buffer); - int totalLength = lengthResult.getBytesRead() + lengthResult.getValue(); - buffer.position(originalPosition); - var valueBuffer = buffer.slice(); - valueBuffer.limit(totalLength); - buffer.position(buffer.position() + totalLength); - return valueBuffer.asReadOnlyBuffer(); + int initialPos = buffer.position(); + ByteBuffer tempMeasureBuffer = buffer.duplicate(); + VarInt.DecodeResult dr = VarInt.decode(tempMeasureBuffer); + + int varIntByteLength = tempMeasureBuffer.position() - initialPos; + int payloadByteLength = dr.getValue(); + int totalValueLength = varIntByteLength + payloadByteLength; + + if (buffer.remaining() < totalValueLength) { + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, + "Not enough bytes for VarInt-prefixed data. Needed: " + totalValueLength + + ", available: " + buffer.remaining() + " at position " + initialPos); + } + + ByteBuffer resultSlice = buffer.slice(); + resultSlice.limit(totalValueLength); + + buffer.position(initialPos + totalValueLength); + return resultSlice.asReadOnlyBuffer(); } }; @@ -248,28 +304,28 @@ public Value deserialize(ByteBuffer buffer) throws ImprintException { VarInt.DecodeResult strLengthResult = VarInt.decode(buffer); int strLength = strLengthResult.getValue(); if (buffer.remaining() < strLength) { - throw new ImprintException(com.imprint.error.ErrorType.BUFFER_UNDERFLOW, "Not enough bytes for string value"); + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, "Not enough bytes for string value data after VarInt. Slice from readValueBytes is too short. Needed: " + strLength + ", available: " + buffer.remaining()); } var stringBytesView = buffer.slice(); stringBytesView.limit(strLength); buffer.position(buffer.position() + strLength); try { - return Value.fromStringBuffer(stringBytesView.asReadOnlyBuffer()); + return Value.fromStringBuffer(stringBytesView); } catch (Exception e) { - throw new ImprintException(com.imprint.error.ErrorType.INVALID_UTF8_STRING, "Invalid UTF-8 string"); + throw new ImprintException(ErrorType.INVALID_UTF8_STRING, "Invalid UTF-8 string or buffer issue: " + e.getMessage()); } } @Override public void serialize(Value value, ByteBuffer buffer) { if (value instanceof Value.StringBufferValue) { - Value.StringBufferValue bufferValue = (Value.StringBufferValue) value; + var bufferValue = (Value.StringBufferValue) value; var stringBuffer = bufferValue.getBuffer(); VarInt.encode(stringBuffer.remaining(), buffer); buffer.put(stringBuffer); } else { - Value.StringValue stringValue = (Value.StringValue) value; - byte[] stringBytes = stringValue.getUtf8Bytes(); // Use cached UTF-8 bytes + var stringValue = (Value.StringValue) value; + byte[] stringBytes = stringValue.getUtf8Bytes(); VarInt.encode(stringBytes.length, buffer); buffer.put(stringBytes); } @@ -283,21 +339,287 @@ public int estimateSize(Value value) { return VarInt.encodedLength(length) + length; } else { Value.StringValue stringValue = (Value.StringValue) value; - byte[] utf8Bytes = stringValue.getUtf8Bytes(); // Use cached UTF-8 bytes + byte[] utf8Bytes = stringValue.getUtf8Bytes(); return VarInt.encodedLength(utf8Bytes.length) + utf8Bytes.length; } } @Override public ByteBuffer readValueBytes(ByteBuffer buffer) throws ImprintException { - int originalPosition = buffer.position(); + int initialPos = buffer.position(); + ByteBuffer tempMeasureBuffer = buffer.duplicate(); + VarInt.DecodeResult dr = VarInt.decode(tempMeasureBuffer); + + int varIntByteLength = tempMeasureBuffer.position() - initialPos; + int payloadByteLength = dr.getValue(); + int totalValueLength = varIntByteLength + payloadByteLength; + + if (buffer.remaining() < totalValueLength) { + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, + "Not enough bytes for VarInt-prefixed string. Needed: " + totalValueLength + + ", available: " + buffer.remaining() + " at position " + initialPos); + } + + ByteBuffer resultSlice = buffer.slice(); + resultSlice.limit(totalValueLength); + + buffer.position(initialPos + totalValueLength); + return resultSlice.asReadOnlyBuffer(); + } + }; + + TypeHandler ARRAY = new TypeHandler() { + @Override + public Value deserialize(ByteBuffer buffer) throws ImprintException { VarInt.DecodeResult lengthResult = VarInt.decode(buffer); - int totalLength = lengthResult.getBytesRead() + lengthResult.getValue(); - buffer.position(originalPosition); - var valueBuffer = buffer.slice(); - valueBuffer.limit(totalLength); - buffer.position(buffer.position() + totalLength); - return valueBuffer.asReadOnlyBuffer(); + int length = lengthResult.getValue(); + + if (length == 0) { + return Value.fromArray(Collections.emptyList()); + } + + if (buffer.remaining() < 1) { + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, "Not enough bytes for ARRAY element type code."); + } + var elementType = TypeCode.fromByte(buffer.get()); + var elements = new ArrayList(length); + var elementHandler = elementType.getHandler(); + + for (int i = 0; i < length; i++) { + var elementValueBytes = elementHandler.readValueBytes(buffer); + elementValueBytes.order(buffer.order()); + var element = elementHandler.deserialize(elementValueBytes); + elements.add(element); + } + + return Value.fromArray(elements); + } + + @Override + public void serialize(Value value, ByteBuffer buffer) throws ImprintException { + var arrayValue = (Value.ArrayValue) value; + var elements = arrayValue.getValue(); + VarInt.encode(elements.size(), buffer); + + if (elements.isEmpty()) return; + + var elementType = elements.get(0).getTypeCode(); + buffer.put(elementType.getCode()); + var elementHandler = elementType.getHandler(); + for (var element : elements) { + if (element.getTypeCode() != elementType) { + throw new ImprintException(ErrorType.SCHEMA_ERROR, + "Array elements must have same type code: " + + element.getTypeCode() + " != " + elementType); + } + elementHandler.serialize(element, buffer); + } + } + + @Override + public int estimateSize(Value value) throws ImprintException { + var arrayValue = (Value.ArrayValue) value; + var elements = arrayValue.getValue(); + int sizeOfLength = VarInt.encodedLength(elements.size()); + if (elements.isEmpty()) { + return sizeOfLength; + } + int sizeOfElementTypeCode = 1; + int arraySize = sizeOfLength + sizeOfElementTypeCode; + var elementHandler = elements.get(0).getTypeCode().getHandler(); + for (var element : elements) { + arraySize += elementHandler.estimateSize(element); + } + return arraySize; + } + + @Override + public ByteBuffer readValueBytes(ByteBuffer buffer) throws ImprintException { + return readComplexValueBytes(buffer, "ARRAY", (tempBuffer, numElements) -> { + if (tempBuffer.remaining() < 1) { + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, + "Not enough bytes for ARRAY element type code in temp buffer during measurement."); + } + byte elementTypeCodeByte = tempBuffer.get(); + int typeCodeLength = 1; + + TypeHandler elementHandler = TypeCode.fromByte(elementTypeCodeByte).getHandler(); + int elementsDataLength = 0; + for (int i = 0; i < numElements; i++) { + int elementStartPos = tempBuffer.position(); + elementHandler.readValueBytes(tempBuffer); + elementsDataLength += (tempBuffer.position() - elementStartPos); + } + + return typeCodeLength + elementsDataLength; + }); + } + }; + + TypeHandler MAP = new TypeHandler() { + @Override + public Value deserialize(ByteBuffer buffer) throws ImprintException { + VarInt.DecodeResult lengthResult = VarInt.decode(buffer); + int length = lengthResult.getValue(); + + if (length == 0) { + return Value.fromMap(Collections.emptyMap()); + } + + if (buffer.remaining() < 2) { + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, "Not enough bytes for MAP key/value type codes."); + } + var keyType = TypeCode.fromByte(buffer.get()); + var valueType = TypeCode.fromByte(buffer.get()); + var map = new HashMap(length); + + var keyHandler = keyType.getHandler(); + var valueHandler = valueType.getHandler(); + + for (int i = 0; i < length; i++) { + var keyBytes = keyHandler.readValueBytes(buffer); + keyBytes.order(buffer.order()); + var keyValue = keyHandler.deserialize(keyBytes); + var key = MapKey.fromValue(keyValue); + + var valueBytes = valueHandler.readValueBytes(buffer); + valueBytes.order(buffer.order()); + var mapInternalValue = valueHandler.deserialize(valueBytes); + + map.put(key, mapInternalValue); + } + + return Value.fromMap(map); + } + + @Override + public void serialize(Value value, ByteBuffer buffer) throws ImprintException { + var mapValue = (Value.MapValue) value; + var map = mapValue.getValue(); + VarInt.encode(map.size(), buffer); + + if (map.isEmpty()) { + return; + } + + var iterator = map.entrySet().iterator(); + var first = iterator.next(); + var keyType = first.getKey().getTypeCode(); + var valueType = first.getValue().getTypeCode(); + + buffer.put(keyType.getCode()); + buffer.put(valueType.getCode()); + + serializeMapKey(first.getKey(), buffer); + first.getValue().getTypeCode().getHandler().serialize(first.getValue(), buffer); + + while (iterator.hasNext()) { + var entry = iterator.next(); + if (entry.getKey().getTypeCode() != keyType) { + throw new ImprintException(ErrorType.SCHEMA_ERROR, + "Map keys must have same type code: " + + entry.getKey().getTypeCode() + " != " + keyType); + } + if (entry.getValue().getTypeCode() != valueType) { + throw new ImprintException(ErrorType.SCHEMA_ERROR, + "Map values must have same type code: " + + entry.getValue().getTypeCode() + " != " + valueType); + } + + serializeMapKey(entry.getKey(), buffer); + entry.getValue().getTypeCode().getHandler().serialize(entry.getValue(), buffer); + } + } + + @Override + public int estimateSize(Value value) throws ImprintException { + var mapValue = (Value.MapValue) value; + var map = mapValue.getValue(); + int sizeOfLength = VarInt.encodedLength(map.size()); + if (map.isEmpty()) { + return sizeOfLength; + } + int sizeOfTypeCodes = 2; + int mapSize = sizeOfLength + sizeOfTypeCodes; + + for (var entry : map.entrySet()) { + mapSize += estimateMapKeySize(entry.getKey()); + mapSize += entry.getValue().getTypeCode().getHandler().estimateSize(entry.getValue()); + } + return mapSize; + } + + @Override + public ByteBuffer readValueBytes(ByteBuffer buffer) throws ImprintException { + return readComplexValueBytes(buffer, "MAP", (tempBuffer, numEntries) -> { + if (tempBuffer.remaining() < 2) { + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, + "Not enough bytes for MAP key/value type codes in temp buffer during measurement."); + } + byte keyTypeCodeByte = tempBuffer.get(); + byte valueTypeCodeByte = tempBuffer.get(); + int typeCodesLength = 2; + int entriesDataLength = 0; + for (int i = 0; i < numEntries; i++) { + int entryStartPos = tempBuffer.position(); + TypeCode.fromByte(keyTypeCodeByte).getHandler().readValueBytes(tempBuffer); + TypeCode.fromByte(valueTypeCodeByte).getHandler().readValueBytes(tempBuffer); + entriesDataLength += (tempBuffer.position() - entryStartPos); + } + + return typeCodesLength + entriesDataLength; + }); + } + + private void serializeMapKey(MapKey key, ByteBuffer buffer) throws ImprintException { + switch (key.getTypeCode()) { + case INT32: + MapKey.Int32Key int32Key = (MapKey.Int32Key) key; + buffer.putInt(int32Key.getValue()); + break; + + case INT64: + MapKey.Int64Key int64Key = (MapKey.Int64Key) key; + buffer.putLong(int64Key.getValue()); + break; + + case BYTES: + MapKey.BytesKey bytesKey = (MapKey.BytesKey) key; + byte[] bytes = bytesKey.getValue(); + VarInt.encode(bytes.length, buffer); + buffer.put(bytes); + break; + + case STRING: + MapKey.StringKey stringKey = (MapKey.StringKey) key; + byte[] stringBytes = stringKey.getValue().getBytes(java.nio.charset.StandardCharsets.UTF_8); + VarInt.encode(stringBytes.length, buffer); + buffer.put(stringBytes); + break; + + default: + throw new ImprintException(ErrorType.SERIALIZATION_ERROR, + "Invalid map key type: " + key.getTypeCode()); + } + } + + private int estimateMapKeySize(MapKey key) throws ImprintException { + switch (key.getTypeCode()) { + case INT32: return 4; + case INT64: return 8; + case BYTES: + byte[] bytes = ((MapKey.BytesKey) key).getValue(); + return VarInt.encodedLength(bytes.length) + bytes.length; + + case STRING: + var str = ((MapKey.StringKey) key).getValue(); + int utf8Length = str.getBytes(java.nio.charset.StandardCharsets.UTF_8).length; + return VarInt.encodedLength(utf8Length) + utf8Length; + + default: + throw new ImprintException(ErrorType.SERIALIZATION_ERROR, + "Invalid map key type: " + key.getTypeCode()); + } } }; } \ No newline at end of file diff --git a/src/main/java/com/imprint/types/Value.java b/src/main/java/com/imprint/types/Value.java index 64eae91..7f3bbb9 100644 --- a/src/main/java/com/imprint/types/Value.java +++ b/src/main/java/com/imprint/types/Value.java @@ -205,9 +205,16 @@ public byte[] getValue() { @Override public boolean equals(Object obj) { if (this == obj) return true; - if (obj == null || getClass() != obj.getClass()) return false; - BytesValue that = (BytesValue) obj; - return Arrays.equals(value, that.value); + if (obj == null) return false; + if (obj instanceof BytesValue) { + BytesValue that = (BytesValue) obj; + return Arrays.equals(value, that.value); + } + if (obj instanceof BytesBufferValue) { + BytesBufferValue that = (BytesBufferValue) obj; + return Arrays.equals(value, that.getValue()); + } + return false; } @Override @@ -270,7 +277,6 @@ public String toString() { } // String Value (String-based) - @EqualsAndHashCode(callSuper = false) public static class StringValue extends Value { @Getter private final String value; @@ -281,11 +287,11 @@ public StringValue(String value) { } public byte[] getUtf8Bytes() { - byte[] cached = cachedUtf8Bytes; + var cached = cachedUtf8Bytes; if (cached == null) { // Multiple threads may compute this - that's OK since it's idempotent cached = value.getBytes(StandardCharsets.UTF_8); - cachedUtf8Bytes = cached; // Benign race - last writer wins + cachedUtf8Bytes = cached; } return cached; // Return our computed value, not re-read from volatile field } @@ -293,6 +299,26 @@ public byte[] getUtf8Bytes() { @Override public TypeCode getTypeCode() { return TypeCode.STRING; } + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (obj == null) return false; + if (obj instanceof StringValue) { + StringValue that = (StringValue) obj; + return value.equals(that.value); + } + if (obj instanceof StringBufferValue) { + StringBufferValue that = (StringBufferValue) obj; + return value.equals(that.getValue()); + } + return false; + } + + @Override + public int hashCode() { + return value.hashCode(); + } + @Override public String toString() { return "\"" + value + "\""; @@ -320,13 +346,14 @@ public String getValue() { private String decodeUtf8() { // Fast path: zero-copy for array-backed ByteBuffers - if (value.hasArray() && value.arrayOffset() == 0) { - return new String(value.array(), value.position(), + if (value.hasArray()) { + return new String(value.array(), value.arrayOffset() + value.position(), value.remaining(), StandardCharsets.UTF_8); } - // Fallback path - should be impossible since deserialize uses wrap() to create an array-backed ByteBuffer. - // Allocation required for direct ByteBuffers since Java's String API doesn't provide ByteBuffer constructors + // Fallback path for non-array-backed ByteBuffers (e.g., direct buffers). + // Allocation is required here as Java's String(byte[],...) constructor needs a heap array. + // Data is copied from the ByteBuffer to a new byte array. var array = new byte[value.remaining()]; value.duplicate().get(array); return new String(array, StandardCharsets.UTF_8); diff --git a/src/main/java/com/imprint/util/VarInt.java b/src/main/java/com/imprint/util/VarInt.java index 5c9a7e5..f4c22f2 100644 --- a/src/main/java/com/imprint/util/VarInt.java +++ b/src/main/java/com/imprint/util/VarInt.java @@ -104,6 +104,26 @@ public static int encodedLength(int value) { return length; } + /** + * Read VarInt-prefixed data from a ByteBuffer. + * The data format is: VarInt(length) + data(length bytes). + * Returns a read-only ByteBuffer containing the entire VarInt + data. + * + * @param buffer the buffer to read from + * @return a read-only ByteBuffer view of the VarInt + data + * @throws ImprintException if the VarInt is malformed or buffer underflow + */ + public static ByteBuffer readVarIntPrefixedBytes(ByteBuffer buffer) throws ImprintException { + int originalPosition = buffer.position(); + VarInt.DecodeResult lengthResult = VarInt.decode(buffer); + int totalLength = lengthResult.getBytesRead() + lengthResult.getValue(); + buffer.position(originalPosition); + var valueBuffer = buffer.slice(); + valueBuffer.limit(totalLength); + buffer.position(buffer.position() + totalLength); + return valueBuffer.asReadOnlyBuffer(); + } + /** * Result of a VarInt decode operation. */ diff --git a/src/test/java/com/imprint/ByteBufferIntegrationTest.java b/src/test/java/com/imprint/ByteBufferIntegrationTest.java index 9460cbf..56ec3b0 100644 --- a/src/test/java/com/imprint/ByteBufferIntegrationTest.java +++ b/src/test/java/com/imprint/ByteBufferIntegrationTest.java @@ -1,20 +1,25 @@ package com.imprint; -import com.imprint.core.*; -import com.imprint.types.*; +import com.imprint.core.ImprintRecord; +import com.imprint.core.ImprintWriter; +import com.imprint.core.SchemaId; import com.imprint.error.ImprintException; +import com.imprint.types.MapKey; +import com.imprint.types.Value; + import java.nio.ByteBuffer; import java.util.*; /** * Integration test to verify ByteBuffer functionality and zero-copy benefits. */ -public class ByteBufferIntegrationTest { - +class ByteBufferIntegrationTest { + public static void main(String[] args) { try { testByteBufferFunctionality(); testZeroCopy(); + testArrayBackedBuffers(); System.out.println("All ByteBuffer integration tests passed!"); } catch (Exception e) { System.err.println("ByteBuffer integration test failed: " + e.getMessage()); @@ -22,66 +27,154 @@ public static void main(String[] args) { System.exit(1); } } - + static void testByteBufferFunctionality() throws ImprintException { System.out.println("Testing ByteBuffer functionality..."); - + SchemaId schemaId = new SchemaId(1, 0xdeadbeef); ImprintWriter writer = new ImprintWriter(schemaId); - + writer.addField(1, Value.fromInt32(42)) .addField(2, Value.fromString("zero-copy test")) .addField(3, Value.fromBytes(new byte[]{1, 2, 3, 4, 5})); - + ImprintRecord record = writer.build(); - + // Test ByteBuffer serialization ByteBuffer serializedBuffer = record.serializeToBuffer(); assert serializedBuffer.isReadOnly() : "Serialized buffer should be read-only"; - + // Test deserialization from ByteBuffer ImprintRecord deserialized = ImprintRecord.deserialize(serializedBuffer); - - assert deserialized.getValue(1).get().equals(Value.fromInt32(42)); - assert deserialized.getValue(2).get().equals(Value.fromString("zero-copy test")); - + + assert Objects.equals(deserialized.getValue(1), Value.fromInt32(42)); + assert Objects.equals(deserialized.getValue(2), Value.fromString("zero-copy test")); + // Test raw bytes access returns ByteBuffer - Optional rawBytes = deserialized.getRawBytes(1); - assert rawBytes.isPresent() : "Raw bytes should be present for field 1"; - assert rawBytes.get().isReadOnly() : "Raw bytes buffer should be read-only"; - + var rawBytes = deserialized.getRawBytes(1); + assert rawBytes != null : "Raw bytes should be present for field 1"; + assert rawBytes.isReadOnly() : "Raw bytes buffer should be read-only"; + System.out.println("ByteBuffer functionality test passed"); } - + static void testZeroCopy() { System.out.println("Testing zero-copy"); - + // Create a large payload to demonstrate zero-copy benefits byte[] largePayload = new byte[1024 * 1024]; // 1MB Arrays.fill(largePayload, (byte) 0xAB); - + SchemaId schemaId = new SchemaId(2, 0xcafebabe); ImprintWriter writer = new ImprintWriter(schemaId); - + try { writer.addField(1, Value.fromBytes(largePayload)); ImprintRecord record = writer.build(); - + // Test that getRawBytes returns a view, not a copy - Optional rawBytes = record.getRawBytes(1); - assert rawBytes.isPresent() : "Raw bytes should be present"; - - ByteBuffer rawBuffer = rawBytes.get(); - assert rawBuffer.isReadOnly() : "Raw buffer should be read-only"; - + var rawBytes = record.getRawBytes(1); + assert rawBytes !=null : "Raw bytes should be present"; + assert rawBytes.isReadOnly() : "Raw buffer should be read-only"; + // The buffer should be positioned at the start of the actual data // (after the VarInt length prefix) - assert rawBuffer.remaining() > largePayload.length : "Buffer should include length prefix"; - + assert rawBytes.remaining() > largePayload.length : "Buffer should include length prefix"; + System.out.println("Zero-copy benefits test passed"); - + } catch (ImprintException e) { throw new RuntimeException("Failed zero-copy test", e); } } + + static void testArrayBackedBuffers() throws ImprintException { + System.out.println("Testing array-backed buffers for zero-copy performance..."); + + // Test serialized buffers are array-backed + SchemaId schemaId = new SchemaId(1, 0xdeadbeef); + ImprintWriter writer = new ImprintWriter(schemaId); + + writer.addField(1, Value.fromInt32(42)) + .addField(2, Value.fromString("test string")) + .addField(3, Value.fromBytes(new byte[]{1, 2, 3, 4})) + .addField(4, Value.fromBoolean(true)); + + ImprintRecord record = writer.build(); + + // Test that serializeToBuffer() returns array-backed buffer + ByteBuffer serializedBuffer = record.serializeToBuffer(); + assert serializedBuffer.hasArray() : "Serialized buffer should be array-backed for zero-copy performance"; + + // Test that the internal payload is array-backed + assert record.getPayload().hasArray() : "Record payload should be array-backed for zero-copy performance"; + + // Test deserialized buffers are array-backed + byte[] bytes = new byte[serializedBuffer.remaining()]; + serializedBuffer.get(bytes); + ImprintRecord deserialized = ImprintRecord.deserialize(bytes); + + assert deserialized.getPayload().hasArray() : "Deserialized record payload should be array-backed"; + + // Test that getRawBytes() returns array-backed buffers + ByteBuffer rawBytes1 = deserialized.getRawBytes(1); + ByteBuffer rawBytes2 = deserialized.getRawBytes(2); + + assert rawBytes1 != null && rawBytes1.hasArray() : "Raw bytes buffer for int field should be array-backed"; + assert rawBytes2 != null && rawBytes2.hasArray() : "Raw bytes buffer for string field should be array-backed"; + + // Test complex types use array-backed buffers + ImprintWriter complexWriter = new ImprintWriter(new SchemaId(2, 0xcafebabe)); + + // Create array value + List arrayValues = Arrays.asList( + Value.fromInt32(1), + Value.fromInt32(2), + Value.fromInt32(3) + ); + + // Create map value + Map mapValues = new HashMap<>(); + mapValues.put(MapKey.fromString("key1"), Value.fromString("value1")); + mapValues.put(MapKey.fromString("key2"), Value.fromString("value2")); + + complexWriter.addField(1, Value.fromArray(arrayValues)) + .addField(2, Value.fromMap(mapValues)); + + ImprintRecord complexRecord = complexWriter.build(); + + assert complexRecord.getPayload().hasArray() : "Record with complex types should use array-backed payload"; + + ByteBuffer complexSerialized = complexRecord.serializeToBuffer(); + assert complexSerialized.hasArray() : "Serialized buffer with complex types should be array-backed"; + + // Test nested records use array-backed buffers + ImprintWriter innerWriter = new ImprintWriter(new SchemaId(3, 0x12345678)); + innerWriter.addField(1, Value.fromString("nested data")); + ImprintRecord innerRecord = innerWriter.build(); + + ImprintWriter outerWriter = new ImprintWriter(new SchemaId(4, 0x87654321)); + outerWriter.addField(1, Value.fromRow(innerRecord)); + ImprintRecord outerRecord = outerWriter.build(); + + assert innerRecord.getPayload().hasArray() : "Inner record payload should be array-backed"; + assert outerRecord.getPayload().hasArray() : "Outer record payload should be array-backed"; + + ByteBuffer nestedSerialized = outerRecord.serializeToBuffer(); + assert nestedSerialized.hasArray() : "Serialized nested record should be array-backed"; + + // Test builder pattern uses array-backed buffers + ImprintRecord builderRecord = ImprintRecord.builder(1, 0xabcdef00) + .field(1, "test string") + .field(2, 42) + .field(3, new byte[]{1, 2, 3}) + .build(); + + assert builderRecord.getPayload().hasArray() : "Builder-created record should use array-backed payload"; + + ByteBuffer builderSerialized = builderRecord.serializeToBuffer(); + assert builderSerialized.hasArray() : "Builder-created serialized buffer should be array-backed"; + + System.out.println("✓ Array-backed buffers test passed"); + } } \ No newline at end of file diff --git a/src/test/java/com/imprint/ComprehensiveTest.java b/src/test/java/com/imprint/ComprehensiveTest.java deleted file mode 100644 index af7f0b0..0000000 --- a/src/test/java/com/imprint/ComprehensiveTest.java +++ /dev/null @@ -1,208 +0,0 @@ -package com.imprint; - -import com.imprint.core.*; -import com.imprint.types.*; -import com.imprint.error.ImprintException; -import com.imprint.util.VarInt; -import java.nio.ByteBuffer; -import java.util.*; - -/** - * Comprehensive test to verify all functionality works correctly. - */ -public class ComprehensiveTest { - - public static void main(String[] args) { - try { - testVarIntFunctionality(); - testValueTypes(); - testMapKeys(); - testComplexSerialization(); - testErrorHandling(); - testByteBufferPerformance(); - System.out.println("All comprehensive tests passed!"); - } catch (Exception e) { - System.err.println("Comprehensive test failed: " + e.getMessage()); - e.printStackTrace(); - System.exit(1); - } - } - - static void testVarIntFunctionality() throws ImprintException { - System.out.println("Testing VarInt functionality..."); - - // Test encoding/decoding of various values - int[] testValues = {0, 1, 127, 128, 16383, 16384, Integer.MAX_VALUE}; - - for (int value : testValues) { - ByteBuffer buffer = ByteBuffer.allocate(10); - VarInt.encode(value, buffer); - int encodedLength = buffer.position(); - - buffer.flip(); - VarInt.DecodeResult result = VarInt.decode(buffer); - - assert result.getValue() == value : "VarInt roundtrip failed for " + value; - assert result.getBytesRead() == encodedLength : "Bytes read mismatch for " + value; - } - - System.out.println("✓ VarInt functionality test passed"); - } - - static void testValueTypes() { - System.out.println("Testing Value types"); - - // Test all value types - Value nullVal = Value.nullValue(); - Value boolVal = Value.fromBoolean(true); - Value int32Val = Value.fromInt32(42); - Value int64Val = Value.fromInt64(123456789L); - Value float32Val = Value.fromFloat32(3.14f); - Value float64Val = Value.fromFloat64(2.718281828); - Value bytesVal = Value.fromBytes(new byte[]{1, 2, 3, 4}); - Value stringVal = Value.fromString("test"); - - // Test type codes - assert nullVal.getTypeCode() == TypeCode.NULL; - assert boolVal.getTypeCode() == TypeCode.BOOL; - assert int32Val.getTypeCode() == TypeCode.INT32; - assert int64Val.getTypeCode() == TypeCode.INT64; - assert float32Val.getTypeCode() == TypeCode.FLOAT32; - assert float64Val.getTypeCode() == TypeCode.FLOAT64; - assert bytesVal.getTypeCode() == TypeCode.BYTES; - assert stringVal.getTypeCode() == TypeCode.STRING; - - // Test value extraction - assert ((Value.BoolValue) boolVal).getValue(); - assert ((Value.Int32Value) int32Val).getValue() == 42; - assert ((Value.Int64Value) int64Val).getValue() == 123456789L; - assert ((Value.Float32Value) float32Val).getValue() == 3.14f; - assert ((Value.Float64Value) float64Val).getValue() == 2.718281828; - assert Arrays.equals(((Value.BytesValue) bytesVal).getValue(), new byte[]{1, 2, 3, 4}); - assert ((Value.StringValue) stringVal).getValue().equals("test"); - - System.out.println("✓ Value types test passed"); - } - - static void testMapKeys() throws ImprintException { - System.out.println("Testing MapKey functionality..."); - - MapKey int32Key = MapKey.fromInt32(42); - MapKey int64Key = MapKey.fromInt64(123L); - MapKey bytesKey = MapKey.fromBytes(new byte[]{1, 2, 3}); - MapKey stringKey = MapKey.fromString("test"); - - // Test conversion to/from Values - Value int32Value = int32Key.toValue(); - Value int64Value = int64Key.toValue(); - Value bytesValue = bytesKey.toValue(); - Value stringValue = stringKey.toValue(); - - assert MapKey.fromValue(int32Value).equals(int32Key); - assert MapKey.fromValue(int64Value).equals(int64Key); - assert MapKey.fromValue(bytesValue).equals(bytesKey); - assert MapKey.fromValue(stringValue).equals(stringKey); - - System.out.println("✓ MapKey functionality test passed"); - } - - static void testComplexSerialization() throws ImprintException { - System.out.println("Testing complex serialization..."); - - SchemaId schemaId = new SchemaId(1, 0xdeadbeef); - ImprintWriter writer = new ImprintWriter(schemaId); - - // Create complex nested structure - List array = Arrays.asList( - Value.fromInt32(1), - Value.fromInt32(2), - Value.fromInt32(3) - ); - - Map map = new HashMap<>(); - map.put(MapKey.fromString("key1"), Value.fromString("value1")); - map.put(MapKey.fromString("key2"), Value.fromString("value2")); - - writer.addField(1, Value.fromArray(array)) - .addField(2, Value.fromMap(map)) - .addField(3, Value.fromString("complex test")); - - ImprintRecord record = writer.build(); - - // Test ByteBuffer serialization - ByteBuffer serialized = record.serializeToBuffer(); - ImprintRecord deserialized = ImprintRecord.deserialize(serialized); - - // Verify complex structures - Value deserializedArray = deserialized.getValue(1).get(); - assert deserializedArray instanceof Value.ArrayValue; - List deserializedList = ((Value.ArrayValue) deserializedArray).getValue(); - assert deserializedList.size() == 3; - assert deserializedList.get(0).equals(Value.fromInt32(1)); - - Value deserializedMap = deserialized.getValue(2).get(); - assert deserializedMap instanceof Value.MapValue; - Map deserializedMapValue = ((Value.MapValue) deserializedMap).getValue(); - assert deserializedMapValue.size() == 2; - assert deserializedMapValue.get(MapKey.fromString("key1")).equals(Value.fromString("value1")); - - System.out.println("✓ Complex serialization test passed"); - } - - static void testErrorHandling() { - System.out.println("Testing error handling..."); - - try { - // Test invalid type code - TypeCode.fromByte((byte) 0xFF); - assert false : "Should have thrown exception for invalid type code"; - } catch (ImprintException e) { - assert e.getErrorType() == com.imprint.error.ErrorType.INVALID_TYPE_CODE; - } - - try { - // Test invalid magic byte - byte[] invalidData = new byte[15]; - invalidData[0] = 0x00; // wrong magic - ImprintRecord.deserialize(invalidData); - assert false : "Should have thrown exception for invalid magic"; - } catch (ImprintException e) { - assert e.getErrorType() == com.imprint.error.ErrorType.INVALID_MAGIC; - } - - System.out.println("✓ Error handling test passed"); - } - - static void testByteBufferPerformance() throws ImprintException { - System.out.println("Testing ByteBuffer performance benefits..."); - - // Create a record with moderate-sized data - byte[] testData = new byte[1024]; - for (int i = 0; i < testData.length; i++) { - testData[i] = (byte) (i % 256); - } - - SchemaId schemaId = new SchemaId(1, 0x12345678); - ImprintWriter writer = new ImprintWriter(schemaId); - writer.addField(1, Value.fromBytes(testData)) - .addField(2, Value.fromString("performance test")); - - ImprintRecord record = writer.build(); - - // Test that raw bytes access is zero-copy - Optional rawBytes = record.getRawBytes(1); - assert rawBytes.isPresent(); - assert rawBytes.get().isReadOnly(); - - // Test ByteBuffer serialization - ByteBuffer serialized = record.serializeToBuffer(); - assert serialized.isReadOnly(); - - // Verify deserialization works - ImprintRecord deserialized = ImprintRecord.deserialize(serialized); - Value retrievedBytes = deserialized.getValue(1).get(); - assert Arrays.equals(((Value.BytesValue) retrievedBytes).getValue(), testData); - - System.out.println("✓ ByteBuffer performance test passed"); - } -} \ No newline at end of file diff --git a/src/test/java/com/imprint/IntegrationTest.java b/src/test/java/com/imprint/IntegrationTest.java index 49cfce7..565b7cd 100644 --- a/src/test/java/com/imprint/IntegrationTest.java +++ b/src/test/java/com/imprint/IntegrationTest.java @@ -39,10 +39,10 @@ static void testBasicFunctionality() throws ImprintException { ImprintRecord record = writer.build(); // Verify we can read values back - assert record.getValue(1).get().equals(Value.fromInt32(42)); - assert record.getValue(2).get().equals(Value.fromString("testing java imprint spec")); - assert record.getValue(3).get().equals(Value.fromBoolean(true)); - assert record.getValue(999).isEmpty(); // non-existent field + assert Objects.equals(record.getValue(1), Value.fromInt32(42)); + assert Objects.equals(record.getValue(2), Value.fromString("testing java imprint spec")); + assert Objects.equals(record.getValue(3), Value.fromBoolean(true)); + assert record.getValue(999) == null; // non-existent field // Test serialization round-trip var buffer = record.serializeToBuffer(); @@ -50,11 +50,11 @@ static void testBasicFunctionality() throws ImprintException { buffer.get(serialized); ImprintRecord deserialized = ImprintRecord.deserialize(serialized); - assert deserialized.getValue(1).get().equals(Value.fromInt32(42)); - assert deserialized.getValue(2).get().equals(Value.fromString("testing java imprint spec")); - assert deserialized.getValue(3).get().equals(Value.fromBoolean(true)); + assert Objects.equals(deserialized.getValue(1), Value.fromInt32(42)); + assert Objects.equals(deserialized.getValue(2), Value.fromString("testing java imprint spec")); + assert Objects.equals(deserialized.getValue(3), Value.fromBoolean(true)); - System.out.println("✓ Basic functionality test passed"); + System.out.println("Basic functionality test passed"); } static void testArraysAndMaps() throws ImprintException { @@ -87,14 +87,14 @@ static void testArraysAndMaps() throws ImprintException { ImprintRecord deserialized = ImprintRecord.deserialize(serialized); // Verify array - Value arrayValue = deserialized.getValue(1).get(); + Value arrayValue = deserialized.getValue(1); assert arrayValue instanceof Value.ArrayValue; List deserializedArray = ((Value.ArrayValue) arrayValue).getValue(); assert deserializedArray.size() == 3; assert deserializedArray.get(0).equals(Value.fromInt32(1)); // Verify map - Value mapValue = deserialized.getValue(2).get(); + Value mapValue = deserialized.getValue(2); assert mapValue instanceof Value.MapValue; Map deserializedMap = ((Value.MapValue) mapValue).getValue(); assert deserializedMap.size() == 2; @@ -127,17 +127,17 @@ static void testNestedRecords() throws ImprintException { ImprintRecord deserialized = ImprintRecord.deserialize(serialized); // Verify outer record - assert deserialized.getHeader().getSchemaId().getFieldspaceId() == 4; - assert deserialized.getValue(2).get().equals(Value.fromString("outer data")); + assert deserialized.getHeader().getSchemaId().getFieldSpaceId() == 4; + assert Objects.equals(deserialized.getValue(2), Value.fromString("outer data")); // Verify nested record - Value rowValue = deserialized.getValue(1).get(); + Value rowValue = deserialized.getValue(1); assert rowValue instanceof Value.RowValue; ImprintRecord nestedRecord = ((Value.RowValue) rowValue).getValue(); - assert nestedRecord.getHeader().getSchemaId().getFieldspaceId() == 3; - assert nestedRecord.getValue(1).get().equals(Value.fromString("nested data")); - assert nestedRecord.getValue(2).get().equals(Value.fromInt64(9876543210L)); + assert nestedRecord.getHeader().getSchemaId().getFieldSpaceId() == 3; + assert Objects.equals(nestedRecord.getValue(1), Value.fromString("nested data")); + assert Objects.equals(nestedRecord.getValue(2), Value.fromInt64(9876543210L)); System.out.println("✓ Nested records test passed"); } diff --git a/src/test/java/com/imprint/benchmark/ProfilerTest.java b/src/test/java/com/imprint/benchmark/ProfilerTest.java index 233d993..5b531a9 100644 --- a/src/test/java/com/imprint/benchmark/ProfilerTest.java +++ b/src/test/java/com/imprint/benchmark/ProfilerTest.java @@ -50,15 +50,15 @@ var record = createTestRecord(); // Random field access (hotspot) int fieldId = random.nextInt(RECORD_SIZE) + 1; var value = record.getValue(fieldId); - if (value.isPresent()) { + if (value != null) { hits++; // Trigger string decoding (potential hotspot) - if (value.get().getTypeCode() == com.imprint.types.TypeCode.STRING) { - if (value.get() instanceof Value.StringBufferValue) { - ((Value.StringBufferValue) value.get()).getValue(); + if (value.getTypeCode() == com.imprint.types.TypeCode.STRING) { + if (value instanceof Value.StringBufferValue) { + ((Value.StringBufferValue) value).getValue(); } else { - ((Value.StringValue) value.get()).getValue(); + ((Value.StringValue) value).getValue(); } } } @@ -123,11 +123,11 @@ var record = createLargeRecord(); // Project 10 fields out of 100 (common analytical pattern) for (int fieldId = 1; fieldId <= 10; fieldId++) { var value = record.getValue(fieldId); - if (value.isPresent()) { + if (value != null) { // Force materialization of string values - if (value.get().getTypeCode() == com.imprint.types.TypeCode.STRING) { - if (value.get() instanceof Value.StringBufferValue) { - ((Value.StringBufferValue) value.get()).getValue(); + if (value.getTypeCode() == com.imprint.types.TypeCode.STRING) { + if (value instanceof Value.StringBufferValue) { + ((Value.StringBufferValue) value).getValue(); } } } diff --git a/src/test/java/com/imprint/core/ImprintRecordBuilderTest.java b/src/test/java/com/imprint/core/ImprintRecordBuilderTest.java deleted file mode 100644 index 54dcfae..0000000 --- a/src/test/java/com/imprint/core/ImprintRecordBuilderTest.java +++ /dev/null @@ -1,234 +0,0 @@ -package com.imprint.core; - -import com.imprint.error.ImprintException; -import com.imprint.types.Value; -import org.junit.jupiter.api.Test; - -import java.util.*; - -import static org.assertj.core.api.Assertions.*; - -class ImprintRecordBuilderTest { - - private static final SchemaId TEST_SCHEMA = new SchemaId(1, 0x12345678); - - @Test - void shouldCreateRecordWithPrimitiveTypes() throws ImprintException { - var record = ImprintRecord.builder(TEST_SCHEMA) - .field(1, true) - .field(2, 42) - .field(3, 123L) - .field(4, 3.14f) - .field(5, 2.718) - .field(6, "hello world") - .field(7, new byte[]{1, 2, 3}) - .nullField(8) - .build(); - - assertThat(record.getHeader().getSchemaId()).isEqualTo(TEST_SCHEMA); - assertThat(record.getDirectory()).hasSize(8); - - // Verify field values - assertThat(getFieldValue(record, 1, Value.BoolValue.class).getValue()).isTrue(); - assertThat(getFieldValue(record, 2, Value.Int32Value.class).getValue()).isEqualTo(42); - assertThat(getFieldValue(record, 3, Value.Int64Value.class).getValue()).isEqualTo(123L); - assertThat(getFieldValue(record, 4, Value.Float32Value.class).getValue()).isEqualTo(3.14f); - assertThat(getFieldValue(record, 5, Value.Float64Value.class).getValue()).isEqualTo(2.718); - assertThat(getStringValue(record, 6)).isEqualTo("hello world"); - assertThat(getBytesValue(record, 7)).isEqualTo(new byte[]{1, 2, 3}); - assertThat(record.getValue(8).get()).isInstanceOf(Value.NullValue.class); - } - - @Test - void shouldCreateRecordWithCollections() throws ImprintException { - var list = List.of(1, 2, 3); - var map = Map.of("key1", 100, "key2", 200); - - var record = ImprintRecord.builder(TEST_SCHEMA) - .field(1, list) - .field(2, map) - .build(); - - // Verify array - var arrayValue = getFieldValue(record, 1, Value.ArrayValue.class); - assertThat(arrayValue.getValue()).hasSize(3); - assertThat(((Value.Int32Value) arrayValue.getValue().get(0)).getValue()).isEqualTo(1); - assertThat(((Value.Int32Value) arrayValue.getValue().get(1)).getValue()).isEqualTo(2); - assertThat(((Value.Int32Value) arrayValue.getValue().get(2)).getValue()).isEqualTo(3); - - // Verify map - var mapValue = getFieldValue(record, 2, Value.MapValue.class); - assertThat(mapValue.getValue()).hasSize(2); - } - - @Test - void shouldCreateRecordWithNestedRecord() throws ImprintException { - var nestedRecord = ImprintRecord.builder(new SchemaId(2, 0x87654321)) - .field(1, "nested") - .field(2, 999) - .build(); - - var record = ImprintRecord.builder(TEST_SCHEMA) - .field(1, "parent") - .field(2, nestedRecord) - .build(); - - var rowValue = getFieldValue(record, 2, Value.RowValue.class); - var nested = rowValue.getValue(); - assertThat(getStringValue(nested, 1)).isEqualTo("nested"); - assertThat(getFieldValue(nested, 2, Value.Int32Value.class).getValue()).isEqualTo(999); - } - - @Test - void shouldSupportConditionalFields() throws ImprintException { - boolean includeOptional = true; - String optionalValue = "optional"; - - var record = ImprintRecord.builder(TEST_SCHEMA) - .field(1, "required") - .fieldIf(includeOptional, 2, optionalValue) - .fieldIfNotNull(3, null) // Should not add field - .fieldIfNotNull(4, "not null") // Should add field - .build(); - - assertThat(record.getDirectory()).hasSize(3); // Only fields 1, 2, 4 - assertThat(getStringValue(record, 1)).isEqualTo("required"); - assertThat(getStringValue(record, 2)).isEqualTo("optional"); - assertThat(record.getValue(3)).isEmpty(); // Not added - assertThat(getStringValue(record, 4)).isEqualTo("not null"); - } - - @Test - void shouldSupportBulkOperations() throws ImprintException { - var fieldsMap = Map.of( - 1, "bulk1", - 2, 42, - 3, true - ); - - var record = ImprintRecord.builder(TEST_SCHEMA) - .fields(fieldsMap) - .field(4, "additional") - .build(); - - assertThat(record.getDirectory()).hasSize(4); - assertThat(getStringValue(record, 1)).isEqualTo("bulk1"); - assertThat(getFieldValue(record, 2, Value.Int32Value.class).getValue()).isEqualTo(42); - assertThat(getFieldValue(record, 3, Value.BoolValue.class).getValue()).isTrue(); - assertThat(getStringValue(record, 4)).isEqualTo("additional"); - } - - @Test - void shouldProvideBuilderUtilities() { - var builder = ImprintRecord.builder(TEST_SCHEMA) - .field(1, "test") - .field(2, 42); - - assertThat(builder.hasField(1)).isTrue(); - assertThat(builder.hasField(3)).isFalse(); - assertThat(builder.fieldCount()).isEqualTo(2); - assertThat(builder.fieldIds()).containsExactly(1, 2); - } - - @Test - void shouldSupportAlternativeSchemaConstructor() throws ImprintException { - var record = ImprintRecord.builder(1, 0x12345678) - .field(1, "test") - .build(); - - assertThat(record.getHeader().getSchemaId().getFieldspaceId()).isEqualTo(1); - assertThat(record.getHeader().getSchemaId().getSchemaHash()).isEqualTo(0x12345678); - } - - @Test - void shouldRoundTripThroughSerialization() throws ImprintException { - var original = ImprintRecord.builder(TEST_SCHEMA) - .field(1, "test string") - .field(2, 42) - .field(3, 3.14159) - .field(4, true) - .field(5, new byte[]{0x01, 0x02, 0x03}) - .build(); - - var serialized = original.serializeToBuffer(); - var deserialized = ImprintRecord.deserialize(serialized); - - assertThat(deserialized.getHeader().getSchemaId()).isEqualTo(TEST_SCHEMA); - assertThat(getStringValue(deserialized, 1)).isEqualTo("test string"); - assertThat(getFieldValue(deserialized, 2, Value.Int32Value.class).getValue()).isEqualTo(42); - assertThat(getFieldValue(deserialized, 3, Value.Float64Value.class).getValue()).isEqualTo(3.14159); - assertThat(getFieldValue(deserialized, 4, Value.BoolValue.class).getValue()).isTrue(); - assertThat(getBytesValue(deserialized, 5)).isEqualTo(new byte[]{0x01, 0x02, 0x03}); - } - - // Error cases - - @Test - void shouldRejectDuplicateFieldIds() { - assertThatThrownBy(() -> - ImprintRecord.builder(TEST_SCHEMA) - .field(1, "first") - .field(1, "duplicate") // Same field ID - ).isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("Field ID 1 already exists"); - } - - @Test - void shouldRejectEmptyRecord() { - assertThatThrownBy(() -> - ImprintRecord.builder(TEST_SCHEMA).build() - ).isInstanceOf(ImprintException.class) - .hasMessageContaining("Cannot build empty record"); - } - - @Test - void shouldRejectInvalidMapKeys() { - var mapWithInvalidKey = Map.of(3.14, "value"); // Double key not supported - - assertThatThrownBy(() -> - ImprintRecord.builder(TEST_SCHEMA) - .field(1, mapWithInvalidKey) - ).isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("Invalid map key type: Double"); - } - - @Test - void shouldRejectNullValueWithoutExplicitNullField() { - assertThatThrownBy(() -> - ImprintRecord.builder(TEST_SCHEMA) - .field(1, (Value) null) - ).isInstanceOf(NullPointerException.class) - .hasMessageContaining("Value cannot be null - use nullField()"); - } - - // Helper methods for cleaner test assertions - - private T getFieldValue(ImprintRecord record, int fieldId, Class valueType) throws ImprintException { - var value = record.getValue(fieldId); - assertThat(value).isPresent(); - assertThat(value.get()).isInstanceOf(valueType); - return valueType.cast(value.get()); - } - - private String getStringValue(ImprintRecord record, int fieldId) throws ImprintException { - var value = record.getValue(fieldId).get(); - if (value instanceof Value.StringValue) { - return ((Value.StringValue) value).getValue(); - } else if (value instanceof Value.StringBufferValue) { - return ((Value.StringBufferValue) value).getValue(); - } else { - throw new AssertionError("Expected string value, got: " + value.getClass()); - } - } - - private byte[] getBytesValue(ImprintRecord record, int fieldId) throws ImprintException { - var value = record.getValue(fieldId).get(); - if (value instanceof Value.BytesValue) { - return ((Value.BytesValue) value).getValue(); - } else if (value instanceof Value.BytesBufferValue) { - return ((Value.BytesBufferValue) value).getValue(); - } else { - throw new AssertionError("Expected bytes value, got: " + value.getClass()); - } - } -} \ No newline at end of file diff --git a/src/test/java/com/imprint/core/ImprintRecordTest.java b/src/test/java/com/imprint/core/ImprintRecordTest.java index 0772580..3e37473 100644 --- a/src/test/java/com/imprint/core/ImprintRecordTest.java +++ b/src/test/java/com/imprint/core/ImprintRecordTest.java @@ -34,20 +34,20 @@ var record = writer.build(); assertThat(record.getHeader().getSchemaId()).isEqualTo(schemaId); assertThat(record.getDirectory()).hasSize(2); - Optional field1 = record.getValue(1); - Optional field2 = record.getValue(2); + Value field1 = record.getValue(1); + Value field2 = record.getValue(2); - assertThat(field1).isPresent(); - assertThat(field1.get()).isInstanceOf(Value.Int32Value.class); - assertThat(((Value.Int32Value) field1.get()).getValue()).isEqualTo(42); + assertThat(field1).isNotNull(); + assertThat(field1).isInstanceOf(Value.Int32Value.class); + assertThat(((Value.Int32Value) field1).getValue()).isEqualTo(42); - assertThat(field2).isPresent(); - assertThat(field2.get().getTypeCode()).isEqualTo(com.imprint.types.TypeCode.STRING); - String stringValue = getStringValue(field2.get()); + assertThat(field2).isNotNull(); + assertThat(field2.getTypeCode()).isEqualTo(com.imprint.types.TypeCode.STRING); + String stringValue = getStringValue(field2); assertThat(stringValue).isEqualTo("hello"); - // Non-existent field should return empty - assertThat(record.getValue(999)).isEmpty(); + // Non-existent field should return null + assertThat(record.getValue(999)).isNull(); } @Test @@ -73,22 +73,22 @@ void shouldRoundtripThroughSerialization() throws ImprintException { var deserialized = ImprintRecord.deserialize(serialized); // Verify metadata - assertThat(deserialized.getHeader().getSchemaId().getFieldspaceId()).isEqualTo(1); + assertThat(deserialized.getHeader().getSchemaId().getFieldSpaceId()).isEqualTo(1); assertThat(deserialized.getHeader().getSchemaId().getSchemaHash()).isEqualTo(0xdeadbeef); assertThat(deserialized.getDirectory()).hasSize(8); // Verify all values - assertThat(deserialized.getValue(1)).contains(Value.nullValue()); - assertThat(deserialized.getValue(2)).contains(Value.fromBoolean(true)); - assertThat(deserialized.getValue(3)).contains(Value.fromInt32(42)); - assertThat(deserialized.getValue(4)).contains(Value.fromInt64(123456789L)); - assertThat(deserialized.getValue(5)).contains(Value.fromFloat32(3.14f)); - assertThat(deserialized.getValue(6)).contains(Value.fromFloat64(2.718281828)); - assertThat(deserialized.getValue(7)).contains(Value.fromBytes(new byte[]{1, 2, 3, 4})); - assertThat(deserialized.getValue(8)).contains(Value.fromString("test string")); + assertThat(deserialized.getValue(1)).isEqualTo(Value.nullValue()); + assertThat(deserialized.getValue(2)).isEqualTo(Value.fromBoolean(true)); + assertThat(deserialized.getValue(3)).isEqualTo(Value.fromInt32(42)); + assertThat(deserialized.getValue(4)).isEqualTo(Value.fromInt64(123456789L)); + assertThat(deserialized.getValue(5)).isEqualTo(Value.fromFloat32(3.14f)); + assertThat(deserialized.getValue(6)).isEqualTo(Value.fromFloat64(2.718281828)); + assertThat(deserialized.getValue(7)).isEqualTo(Value.fromBytes(new byte[]{1, 2, 3, 4})); + assertThat(deserialized.getValue(8)).isEqualTo(Value.fromString("test string")); // Non-existent field - assertThat(deserialized.getValue(999)).isEmpty(); + assertThat(deserialized.getValue(999)).isNull(); } @Test @@ -111,11 +111,11 @@ void shouldHandleArrays() throws ImprintException { buffer.get(serialized); var deserialized = ImprintRecord.deserialize(serialized); - Optional arrayValue = deserialized.getValue(1); - assertThat(arrayValue).isPresent(); - assertThat(arrayValue.get()).isInstanceOf(Value.ArrayValue.class); + Value arrayValue = deserialized.getValue(1); + assertThat(arrayValue).isNotNull(); + assertThat(arrayValue).isInstanceOf(Value.ArrayValue.class); - List deserializedArray = ((Value.ArrayValue) arrayValue.get()).getValue(); + List deserializedArray = ((Value.ArrayValue) arrayValue).getValue(); assertThat(deserializedArray).hasSize(3); assertThat(deserializedArray.get(0)).isEqualTo(Value.fromInt32(1)); assertThat(deserializedArray.get(1)).isEqualTo(Value.fromInt32(2)); @@ -140,11 +140,11 @@ var record = writer.build(); buffer.get(serialized); var deserialized = ImprintRecord.deserialize(serialized); - Optional mapValue = deserialized.getValue(1); - assertThat(mapValue).isPresent(); - assertThat(mapValue.get()).isInstanceOf(Value.MapValue.class); + Value mapValue = deserialized.getValue(1); + assertThat(mapValue).isNotNull(); + assertThat(mapValue).isInstanceOf(Value.MapValue.class); - Map deserializedMap = ((Value.MapValue) mapValue.get()).getValue(); + Map deserializedMap = ((Value.MapValue) mapValue).getValue(); assertThat(deserializedMap).hasSize(2); assertThat(deserializedMap.get(MapKey.fromString("key1"))).isEqualTo(Value.fromInt32(1)); assertThat(deserializedMap.get(MapKey.fromString("key2"))).isEqualTo(Value.fromInt32(2)); @@ -173,23 +173,23 @@ void shouldHandleNestedRecords() throws ImprintException { var deserialized = ImprintRecord.deserialize(serialized); // Verify outer record metadata - assertThat(deserialized.getHeader().getSchemaId().getFieldspaceId()).isEqualTo(1); + assertThat(deserialized.getHeader().getSchemaId().getFieldSpaceId()).isEqualTo(1); assertThat(deserialized.getHeader().getSchemaId().getSchemaHash()).isEqualTo(0xdeadbeef); // Verify nested record - Optional rowValue = deserialized.getValue(1); - assertThat(rowValue).isPresent(); - assertThat(rowValue.get()).isInstanceOf(Value.RowValue.class); + Value rowValue = deserialized.getValue(1); + assertThat(rowValue).isNotNull(); + assertThat(rowValue).isInstanceOf(Value.RowValue.class); - var nestedRecord = ((Value.RowValue) rowValue.get()).getValue(); - assertThat(nestedRecord.getHeader().getSchemaId().getFieldspaceId()).isEqualTo(2); + var nestedRecord = ((Value.RowValue) rowValue).getValue(); + assertThat(nestedRecord.getHeader().getSchemaId().getFieldSpaceId()).isEqualTo(2); assertThat(nestedRecord.getHeader().getSchemaId().getSchemaHash()).isEqualTo(0xcafebabe); - assertThat(nestedRecord.getValue(1)).contains(Value.fromInt32(42)); - assertThat(nestedRecord.getValue(2)).contains(Value.fromString("nested")); + assertThat(nestedRecord.getValue(1)).isEqualTo(Value.fromInt32(42)); + assertThat(nestedRecord.getValue(2)).isEqualTo(Value.fromString("nested")); // Verify outer record field - assertThat(deserialized.getValue(2)).contains(Value.fromInt64(123L)); + assertThat(deserialized.getValue(2)).isEqualTo(Value.fromInt64(123L)); } @Test @@ -227,6 +227,6 @@ void shouldHandleDuplicateFieldIds() throws ImprintException { var record = writer.build(); assertThat(record.getDirectory()).hasSize(1); - assertThat(record.getValue(1)).contains(Value.fromInt32(43)); + assertThat(record.getValue(1)).isEqualTo(Value.fromInt32(43)); } } \ No newline at end of file diff --git a/src/test/java/com/imprint/types/ValueTest.java b/src/test/java/com/imprint/types/ValueTest.java index 9dd99c9..c400a44 100644 --- a/src/test/java/com/imprint/types/ValueTest.java +++ b/src/test/java/com/imprint/types/ValueTest.java @@ -2,6 +2,8 @@ import org.junit.jupiter.api.Test; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.HashMap; import java.util.List; @@ -104,20 +106,113 @@ void shouldHandleEqualityCorrectly() { } @Test - void shouldDefensiveCopyArrays() { - byte[] original = {1, 2, 3}; - var bytesValue = Value.fromBytes(original); + void shouldRejectNullString() { + assertThatThrownBy(() -> Value.fromString(null)) + .isInstanceOf(NullPointerException.class); + } + + @Test + void shouldCreateStringBufferValue() { + String testString = "hello world"; + byte[] utf8Bytes = testString.getBytes(StandardCharsets.UTF_8); + ByteBuffer buffer = ByteBuffer.wrap(utf8Bytes); - // Modify original array - original[0] = 99; + Value stringBufferValue = Value.fromStringBuffer(buffer); - // Value should be unchanged - assertThat(((Value.BytesValue) bytesValue).getValue()).containsExactly(1, 2, 3); + assertThat(stringBufferValue).isInstanceOf(Value.StringBufferValue.class); + assertThat(stringBufferValue.getTypeCode()).isEqualTo(TypeCode.STRING); + assertThat(((Value.StringBufferValue) stringBufferValue).getValue()).isEqualTo(testString); } @Test - void shouldRejectNullString() { - assertThatThrownBy(() -> Value.fromString(null)) - .isInstanceOf(NullPointerException.class); + void shouldCreateBytesBufferValue() { + byte[] testBytes = {1, 2, 3, 4, 5}; + ByteBuffer buffer = ByteBuffer.wrap(testBytes); + + Value bytesBufferValue = Value.fromBytesBuffer(buffer); + + assertThat(bytesBufferValue).isInstanceOf(Value.BytesBufferValue.class); + assertThat(bytesBufferValue.getTypeCode()).isEqualTo(TypeCode.BYTES); + assertThat(((Value.BytesBufferValue) bytesBufferValue).getValue()).isEqualTo(testBytes); + } + + @Test + void shouldHandleStringBufferValueFastPath() { + // Array-backed buffer with arrayOffset() == 0 should use fast path + String testString = "fast path test"; + byte[] utf8Bytes = testString.getBytes(StandardCharsets.UTF_8); + ByteBuffer buffer = ByteBuffer.wrap(utf8Bytes); + + Value stringBufferValue = Value.fromStringBuffer(buffer); + + // Should work correctly regardless of path taken + assertThat(((Value.StringBufferValue) stringBufferValue).getValue()).isEqualTo(testString); + } + + @Test + void shouldHandleStringBufferValueFallbackPath() { + // Sliced buffer will have non-zero arrayOffset, forcing fallback path + String testString = "fallback path test"; + byte[] utf8Bytes = testString.getBytes(StandardCharsets.UTF_8); + ByteBuffer buffer = ByteBuffer.wrap(utf8Bytes); + ByteBuffer sliced = buffer.slice(); // This may break arrayOffset() == 0 + + Value stringBufferValue = Value.fromStringBuffer(sliced); + + // Should work correctly regardless of path taken + assertThat(((Value.StringBufferValue) stringBufferValue).getValue()).isEqualTo(testString); + } + + @Test + void shouldHandleLargeStringWithoutCaching() { + // Create string > 1KB to test the no-cache path + String largeString = "x".repeat(2000); + byte[] utf8Bytes = largeString.getBytes(StandardCharsets.UTF_8); + ByteBuffer buffer = ByteBuffer.wrap(utf8Bytes).slice(); // Force fallback path + + Value stringBufferValue = Value.fromStringBuffer(buffer); + + assertThat(((Value.StringBufferValue) stringBufferValue).getValue()).isEqualTo(largeString); + } + + @Test + void shouldCacheStringDecoding() { + String testString = "cache test"; + byte[] utf8Bytes = testString.getBytes(StandardCharsets.UTF_8); + ByteBuffer buffer = ByteBuffer.wrap(utf8Bytes); + + Value.StringBufferValue stringBufferValue = (Value.StringBufferValue) Value.fromStringBuffer(buffer); + + // First call should decode and cache + String result1 = stringBufferValue.getValue(); + // Second call should return cached value + String result2 = stringBufferValue.getValue(); + + assertThat(result1).isEqualTo(testString); + assertThat(result2).isEqualTo(testString); + assertThat(result1).isSameAs(result2); // Should be same object reference due to caching + } + + @Test + void shouldHandleStringValueEquality() { + String testString = "equality test"; + + Value stringValue = Value.fromString(testString); + Value stringBufferValue = Value.fromStringBuffer(ByteBuffer.wrap(testString.getBytes(StandardCharsets.UTF_8))); + + assertThat(stringValue).isEqualTo(stringBufferValue); + assertThat(stringBufferValue).isEqualTo(stringValue); + assertThat(stringValue.hashCode()).isEqualTo(stringBufferValue.hashCode()); + } + + @Test + void shouldHandleBytesValueEquality() { + byte[] testBytes = {1, 2, 3, 4, 5}; + + Value bytesValue = Value.fromBytes(testBytes); + Value bytesBufferValue = Value.fromBytesBuffer(ByteBuffer.wrap(testBytes)); + + assertThat(bytesValue).isEqualTo(bytesBufferValue); + assertThat(bytesBufferValue).isEqualTo(bytesValue); } } \ No newline at end of file From fdb8a56ee7e81ba635a349031e28b64e1ab91a44 Mon Sep 17 00:00:00 2001 From: expand3d <> Date: Tue, 3 Jun 2025 18:44:27 -0400 Subject: [PATCH 16/49] additional cleanup to address concerns in https://github.com/imprint-serde/imprint-java/issues/3 --- .../java/com/imprint/core/ImprintRecord.java | 283 +++++++++++--- .../imprint/core/ImprintRecordBuilder.java | 73 ++-- .../java/com/imprint/error/ErrorType.java | 3 +- .../java/com/imprint/types/TypeHandler.java | 5 +- src/main/java/com/imprint/util/VarInt.java | 27 +- .../imprint/ByteBufferIntegrationTest.java | 180 --------- .../java/com/imprint/IntegrationTest.java | 368 +++++++++++++----- 7 files changed, 526 insertions(+), 413 deletions(-) delete mode 100644 src/test/java/com/imprint/ByteBufferIntegrationTest.java diff --git a/src/main/java/com/imprint/core/ImprintRecord.java b/src/main/java/com/imprint/core/ImprintRecord.java index 1915707..d667039 100644 --- a/src/main/java/com/imprint/core/ImprintRecord.java +++ b/src/main/java/com/imprint/core/ImprintRecord.java @@ -3,6 +3,7 @@ import com.imprint.Constants; import com.imprint.error.ErrorType; import com.imprint.error.ImprintException; +import com.imprint.types.MapKey; import com.imprint.types.TypeCode; import com.imprint.types.Value; import com.imprint.util.VarInt; @@ -15,7 +16,7 @@ /** * An Imprint record containing a header, field directory, and payload. * Uses ByteBuffer for zero-copy operations to achieve low latency. - * + * *

Performance Note: All ByteBuffers should be array-backed * (hasArray() == true) for optimal zero-copy performance. Direct buffers * may cause performance degradation.

@@ -25,10 +26,10 @@ public final class ImprintRecord { private final Header header; private final List directory; private final ByteBuffer payload; // Read-only view for zero-copy - + /** * Creates a new ImprintRecord. - * + * * @param payload the payload buffer. Should be array-backed for optimal performance. */ public ImprintRecord(Header header, List directory, ByteBuffer payload) { @@ -40,15 +41,29 @@ public ImprintRecord(Header header, List directory, ByteBuffer p /** * Get a value by field ID, deserializing it on demand. * Returns null if the field is not found. + * Note: If the field exists and is an explicit NULL type, this will return Value.NullValue.INSTANCE. */ public Value getValue(int fieldId) throws ImprintException { var fieldBuffer = getFieldBuffer(fieldId); - if (fieldBuffer == null) return null; - - var entry = directory.get(findDirectoryIndex(fieldId)); + if (fieldBuffer == null) { + // If getFieldBuffer returns null, it means the fieldId was not in the directory, + // or an issue occurred slicing the payload (e.g., bad offsets). + return null; + } + + // findDirectoryIndex should not be negative here if fieldBuffer is not null, + // but a defensive check or ensuring findDirectoryIndex is called once is good. + // For simplicity, assume getFieldBuffer implies a valid index. + int directoryIndex = findDirectoryIndex(fieldId); + if (directoryIndex < 0) { + // This case should ideally be caught by getFieldBuffer returning null. + // If it happens, indicates an inconsistency. + throw new ImprintException(ErrorType.INTERNAL_ERROR, "Field ID " + fieldId + " found buffer but not in directory."); + } + var entry = directory.get(directoryIndex); return deserializeValue(entry.getTypeCode(), fieldBuffer); } - + /** * Get the raw bytes for a field without deserializing. * Returns a zero-copy ByteBuffer view, or null if field not found. @@ -57,7 +72,7 @@ public ByteBuffer getRawBytes(int fieldId) { var fieldBuffer = getFieldBuffer(fieldId); return fieldBuffer != null ? fieldBuffer.asReadOnlyBuffer() : null; } - + /** * Get a ByteBuffer view of a field's data. * Returns null if the field is not found. @@ -65,12 +80,12 @@ public ByteBuffer getRawBytes(int fieldId) { private ByteBuffer getFieldBuffer(int fieldId) { int index = findDirectoryIndex(fieldId); if (index < 0) return null; - + var entry = directory.get(index); int startOffset = entry.getOffset(); - int endOffset = (index + 1 < directory.size()) ? - directory.get(index + 1).getOffset() : payload.limit(); - + int endOffset = (index + 1 < directory.size()) ? + directory.get(index + 1).getOffset() : payload.limit(); + var fieldBuffer = payload.duplicate(); if (startOffset > payload.limit() || endOffset > payload.limit() || startOffset > endOffset) { return null; @@ -78,104 +93,106 @@ private ByteBuffer getFieldBuffer(int fieldId) { fieldBuffer.position(startOffset).limit(endOffset); return fieldBuffer.slice(); } - + /** - * Serialize this record to a ByteBuffer (zero-copy when possible). + * Serialize this record to a ByteBuffer. + * The returned buffer will be array-backed. */ public ByteBuffer serializeToBuffer() { var buffer = ByteBuffer.allocate(estimateSerializedSize()); buffer.order(ByteOrder.LITTLE_ENDIAN); - + // Write header serializeHeader(buffer); - + // Write directory (always present) VarInt.encode(directory.size(), buffer); for (var entry : directory) { serializeDirectoryEntry(entry, buffer); } - + // Write payload (shallow copy only) var payloadCopy = payload.duplicate(); buffer.put(payloadCopy); - - // Return read-only view of used portion + + // Prepare buffer for reading buffer.flip(); - return buffer.asReadOnlyBuffer(); + return buffer; } - + /** * Create a fluent builder for constructing ImprintRecord instances. - * + * * @param schemaId the schema identifier for this record * @return a new builder instance */ public static ImprintRecordBuilder builder(SchemaId schemaId) { return new ImprintRecordBuilder(schemaId); } - + /** * Create a fluent builder for constructing ImprintRecord instances. - * + * * @param fieldspaceId the fieldspace identifier * @param schemaHash the schema hash * @return a new builder instance */ + @SuppressWarnings("unused") public static ImprintRecordBuilder builder(int fieldspaceId, int schemaHash) { return new ImprintRecordBuilder(new SchemaId(fieldspaceId, schemaHash)); } - + /** * Deserialize a record from bytes through an array backed ByteBuffer. */ public static ImprintRecord deserialize(byte[] bytes) throws ImprintException { return deserialize(ByteBuffer.wrap(bytes)); } - + /** * Deserialize a record from a ByteBuffer. - * - * @param buffer the buffer to deserialize from. Must be array-backed + * + * @param buffer the buffer to deserialize from. Must be array-backed * (buffer.hasArray() == true) for optimal zero-copy performance. */ public static ImprintRecord deserialize(ByteBuffer buffer) throws ImprintException { buffer = buffer.duplicate().order(ByteOrder.LITTLE_ENDIAN); - + // Read header var header = deserializeHeader(buffer); - + // Read directory (always present) var directory = new ArrayList(); VarInt.DecodeResult countResult = VarInt.decode(buffer); int directoryCount = countResult.getValue(); - + for (int i = 0; i < directoryCount; i++) { directory.add(deserializeDirectoryEntry(buffer)); } - + // Read payload as ByteBuffer slice for zero-copy var payload = buffer.slice(); payload.limit(header.getPayloadSize()); buffer.position(buffer.position() + header.getPayloadSize()); - + return new ImprintRecord(header, directory, payload); } - + /** * Binary search for field ID in directory without object allocation. * Returns the index of the field if found, or a negative value if not found. - * + * * @param fieldId the field ID to search for * @return index if found, or negative insertion point - 1 if not found */ private int findDirectoryIndex(int fieldId) { int low = 0; int high = directory.size() - 1; - + while (low <= high) { int mid = (low + high) >>> 1; // unsigned right shift to avoid overflow int midFieldId = directory.get(mid).getId(); - + if (midFieldId < fieldId) { low = mid + 1; } else if (midFieldId > fieldId) { @@ -186,7 +203,7 @@ private int findDirectoryIndex(int fieldId) { } return -(low + 1); // field not found, return insertion point } - + public int estimateSerializedSize() { int size = Constants.HEADER_BYTES; // header size += VarInt.encodedLength(directory.size()); // directory count @@ -194,7 +211,7 @@ public int estimateSerializedSize() { size += payload.remaining(); // payload return size; } - + private void serializeHeader(ByteBuffer buffer) { buffer.put(Constants.MAGIC); buffer.put(Constants.VERSION); @@ -203,57 +220,57 @@ private void serializeHeader(ByteBuffer buffer) { buffer.putInt(header.getSchemaId().getSchemaHash()); buffer.putInt(header.getPayloadSize()); } - + private static Header deserializeHeader(ByteBuffer buffer) throws ImprintException { if (buffer.remaining() < Constants.HEADER_BYTES) { - throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, - "Not enough bytes for header"); + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, + "Not enough bytes for header"); } - + byte magic = buffer.get(); if (magic != Constants.MAGIC) { - throw new ImprintException(ErrorType.INVALID_MAGIC, - "Invalid magic byte: expected 0x" + Integer.toHexString(Constants.MAGIC) + - ", got 0x" + Integer.toHexString(magic & 0xFF)); + throw new ImprintException(ErrorType.INVALID_MAGIC, + "Invalid magic byte: expected 0x" + Integer.toHexString(Constants.MAGIC) + + ", got 0x" + Integer.toHexString(magic & 0xFF)); } - + byte version = buffer.get(); if (version != Constants.VERSION) { - throw new ImprintException(ErrorType.UNSUPPORTED_VERSION, - "Unsupported version: " + version); + throw new ImprintException(ErrorType.UNSUPPORTED_VERSION, + "Unsupported version: " + version); } - + var flags = new Flags(buffer.get()); int fieldspaceId = buffer.getInt(); int schemaHash = buffer.getInt(); int payloadSize = buffer.getInt(); - + return new Header(flags, new SchemaId(fieldspaceId, schemaHash), payloadSize); } - + private void serializeDirectoryEntry(DirectoryEntry entry, ByteBuffer buffer) { buffer.putShort(entry.getId()); buffer.put(entry.getTypeCode().getCode()); buffer.putInt(entry.getOffset()); } - + private static DirectoryEntry deserializeDirectoryEntry(ByteBuffer buffer) throws ImprintException { if (buffer.remaining() < Constants.DIR_ENTRY_BYTES) { - throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, - "Not enough bytes for directory entry"); + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, + "Not enough bytes for directory entry"); } - + short id = buffer.getShort(); var typeCode = TypeCode.fromByte(buffer.get()); int offset = buffer.getInt(); - + return new DirectoryEntry(id, typeCode, offset); } - + private Value deserializeValue(TypeCode typeCode, ByteBuffer buffer) throws ImprintException { ByteBuffer valueSpecificBuffer = buffer.duplicate(); valueSpecificBuffer.order(ByteOrder.LITTLE_ENDIAN); - + switch (typeCode) { case NULL: case BOOL: @@ -274,10 +291,152 @@ private Value deserializeValue(TypeCode typeCode, ByteBuffer buffer) throws Impr throw new ImprintException(ErrorType.INVALID_TYPE_CODE, "Unknown type code: " + typeCode); } } - + @Override public String toString() { - return String.format("ImprintRecord{header=%s, directorySize=%d, payloadSize=%d}", - header, directory.size(), payload.remaining()); + return String.format("ImprintRecord{header=%s, directorySize=%d, payloadSize=%d}", + header, directory.size(), payload.remaining()); + } + + private T getTypedValueOrThrow(int fieldId, TypeCode expectedTypeCode, Class expectedValueClass, String expectedTypeName) throws ImprintException { + Value value = getValue(fieldId); + + if (value == null) { + throw new ImprintException(ErrorType.FIELD_NOT_FOUND, + "Field " + fieldId + " not found, cannot retrieve as " + expectedTypeName + "."); + } + + if (value.getTypeCode() == TypeCode.NULL) { + throw new ImprintException(ErrorType.TYPE_MISMATCH, + "Field " + fieldId + " is NULL, cannot retrieve as " + expectedTypeName + "."); + } + + if (value.getTypeCode() == expectedTypeCode && expectedValueClass.isInstance(value)) { + return expectedValueClass.cast(value); + } + + throw new ImprintException(ErrorType.TYPE_MISMATCH, + "Field " + fieldId + " is of type " + value.getTypeCode() + ", expected " + expectedTypeName + "."); + } + + /** + * Retrieves the boolean value for the given field ID. + * @throws ImprintException if the field is not found, is null, or is not of type BOOL. + */ + public boolean getBoolean(int fieldId) throws ImprintException { + return getTypedValueOrThrow(fieldId, TypeCode.BOOL, Value.BoolValue.class, "boolean").getValue(); + } + + /** + * Retrieves the int (int32) value for the given field ID. + * @throws ImprintException if the field is not found, is null, or is not of type INT32. + */ + public int getInt32(int fieldId) throws ImprintException { + return getTypedValueOrThrow(fieldId, TypeCode.INT32, Value.Int32Value.class, "int32").getValue(); + } + + /** + * Retrieves the long (int64) value for the given field ID. + * @throws ImprintException if the field is not found, is null, or is not of type INT64. + */ + public long getInt64(int fieldId) throws ImprintException { + return getTypedValueOrThrow(fieldId, TypeCode.INT64, Value.Int64Value.class, "int64").getValue(); + } + + /** + * Retrieves the float (float32) value for the given field ID. + * @throws ImprintException if the field is not found, is null, or is not of type FLOAT32. + */ + public float getFloat32(int fieldId) throws ImprintException { + return getTypedValueOrThrow(fieldId, TypeCode.FLOAT32, Value.Float32Value.class, "float32").getValue(); + } + + /** + * Retrieves the double (float64) value for the given field ID. + * @throws ImprintException if the field is not found, is null, or is not of type FLOAT64. + */ + public double getFloat64(int fieldId) throws ImprintException { + return getTypedValueOrThrow(fieldId, TypeCode.FLOAT64, Value.Float64Value.class, "float64").getValue(); + } + + /** + * Retrieves the String value for the given field ID. + * @throws ImprintException if the field is not found, is null, or is not of type STRING. + */ + public String getString(int fieldId) throws ImprintException { + Value value = getValue(fieldId); + + if (value == null) { + throw new ImprintException(ErrorType.FIELD_NOT_FOUND, + "Field " + fieldId + " not found, cannot retrieve as String."); + } + if (value.getTypeCode() == TypeCode.NULL) { + throw new ImprintException(ErrorType.TYPE_MISMATCH, + "Field " + fieldId + " is NULL, cannot retrieve as String."); + } + + if (value instanceof Value.StringValue) { + return ((Value.StringValue) value).getValue(); + } + if (value instanceof Value.StringBufferValue) { + return ((Value.StringBufferValue) value).getValue(); + } + + throw new ImprintException(ErrorType.TYPE_MISMATCH, + "Field " + fieldId + " is of type " + value.getTypeCode() + ", expected STRING."); + } + + /** + * Retrieves the byte array (byte[]) value for the given field ID. + * Note: This may involve a defensive copy depending on the underlying Value type. + * @throws ImprintException if the field is not found, is null, or is not of type BYTES. + */ + public byte[] getBytes(int fieldId) throws ImprintException { + Value value = getValue(fieldId); + + if (value == null) { + throw new ImprintException(ErrorType.FIELD_NOT_FOUND, + "Field " + fieldId + " not found, cannot retrieve as byte[]."); + } + if (value.getTypeCode() == TypeCode.NULL) { + throw new ImprintException(ErrorType.TYPE_MISMATCH, + "Field " + fieldId + " is NULL, cannot retrieve as byte[]."); + } + + if (value instanceof Value.BytesValue) { + return ((Value.BytesValue) value).getValue(); // getValue() in BytesValue returns a clone + } + if (value instanceof Value.BytesBufferValue) { + return ((Value.BytesBufferValue) value).getValue(); // getValue() in BytesBufferValue creates a new array + } + + throw new ImprintException(ErrorType.TYPE_MISMATCH, + "Field " + fieldId + " is of type " + value.getTypeCode() + ", expected BYTES."); + } + + /** + * Retrieves the List for the given field ID. + * The list itself is a copy; modifications to it will not affect the record. + * @throws ImprintException if the field is not found, is null, or is not of type ARRAY. + */ + public List getArray(int fieldId) throws ImprintException { + return getTypedValueOrThrow(fieldId, TypeCode.ARRAY, Value.ArrayValue.class, "ARRAY").getValue(); + } + + /** + * Retrieves the Map for the given field ID. + * The map itself is a copy; modifications to it will not affect the record. + * @throws ImprintException if the field is not found, is null, or is not of type MAP. + */ + public Map getMap(int fieldId) throws ImprintException { + return getTypedValueOrThrow(fieldId, TypeCode.MAP, Value.MapValue.class, "MAP").getValue(); + } + + /** + * Retrieves the nested ImprintRecord for the given field ID. + * @throws ImprintException if the field is not found, is null, or is not of type ROW. + */ + public ImprintRecord getRow(int fieldId) throws ImprintException { + return getTypedValueOrThrow(fieldId, TypeCode.ROW, Value.RowValue.class, "ROW").getValue(); } } \ No newline at end of file diff --git a/src/main/java/com/imprint/core/ImprintRecordBuilder.java b/src/main/java/com/imprint/core/ImprintRecordBuilder.java index 8c04bf5..51a3525 100644 --- a/src/main/java/com/imprint/core/ImprintRecordBuilder.java +++ b/src/main/java/com/imprint/core/ImprintRecordBuilder.java @@ -31,40 +31,40 @@ public final class ImprintRecordBuilder { private final SchemaId schemaId; private final Map fields = new TreeMap<>(); - + ImprintRecordBuilder(SchemaId schemaId) { this.schemaId = Objects.requireNonNull(schemaId, "SchemaId cannot be null"); } - + // Primitive types with automatic Value wrapping public ImprintRecordBuilder field(int id, boolean value) { return addField(id, Value.fromBoolean(value)); } - + public ImprintRecordBuilder field(int id, int value) { return addField(id, Value.fromInt32(value)); } - + public ImprintRecordBuilder field(int id, long value) { return addField(id, Value.fromInt64(value)); } - + public ImprintRecordBuilder field(int id, float value) { return addField(id, Value.fromFloat32(value)); } - + public ImprintRecordBuilder field(int id, double value) { return addField(id, Value.fromFloat64(value)); } - + public ImprintRecordBuilder field(int id, String value) { return addField(id, Value.fromString(value)); } - + public ImprintRecordBuilder field(int id, byte[] value) { return addField(id, Value.fromBytes(value)); } - + // Collections with automatic conversion public ImprintRecordBuilder field(int id, List values) { var convertedValues = new ArrayList(values.size()); @@ -73,7 +73,7 @@ public ImprintRecordBuilder field(int id, List values) { } return addField(id, Value.fromArray(convertedValues)); } - + public ImprintRecordBuilder field(int id, Map map) { var convertedMap = new HashMap(map.size()); for (var entry : map.entrySet()) { @@ -83,22 +83,22 @@ public ImprintRecordBuilder field(int id, Map fieldsMap) { for (var entry : fieldsMap.entrySet()) { @@ -118,58 +118,53 @@ public ImprintRecordBuilder fields(Map fieldsMap) { } return this; } - + // Builder utilities public boolean hasField(int id) { return fields.containsKey(id); } - + public int fieldCount() { return fields.size(); } - + public Set fieldIds() { return new TreeSet<>(fields.keySet()); } - + // Build the final record public ImprintRecord build() throws ImprintException { - if (fields.isEmpty()) { - throw new ImprintException(com.imprint.error.ErrorType.SCHEMA_ERROR, - "Cannot build empty record - add at least one field"); - } - var writer = new ImprintWriter(schemaId); for (var entry : fields.entrySet()) { writer.addField(entry.getKey(), entry.getValue()); } return writer.build(); } - + // Internal helper methods /** * Adds or overwrites a field in the record being built. * If a field with the given ID already exists, it will be replaced. - * + * * @param id the field ID * @param value the field value (cannot be null - use nullField() for explicit nulls) * @return this builder for method chaining */ private ImprintRecordBuilder addField(int id, Value value) { Objects.requireNonNull(value, "Value cannot be null - use nullField() for explicit null values"); - fields.put(id, value); // TreeMap.put() overwrites existing values + fields.put(id, value); return this; } - + private Value convertToValue(Object obj) { if (obj == null) { return Value.nullValue(); } - + if (obj instanceof Value) { return (Value) obj; } - + // Auto-boxing conversion if (obj instanceof Boolean) { return Value.fromBoolean((Boolean) obj); @@ -216,11 +211,11 @@ private Value convertToValue(Object obj) { if (obj instanceof ImprintRecord) { return Value.fromRow((ImprintRecord) obj); } - - throw new IllegalArgumentException("Cannot convert " + obj.getClass().getSimpleName() + - " to Imprint Value. Supported types: boolean, int, long, float, double, String, byte[], List, Map, ImprintRecord"); + + throw new IllegalArgumentException("Cannot convert " + obj.getClass().getSimpleName() + + " to Imprint Value. Supported types: boolean, int, long, float, double, String, byte[], List, Map, ImprintRecord"); } - + private MapKey convertToMapKey(Object obj) { if (obj instanceof Integer) { return MapKey.fromInt32((Integer) obj); @@ -234,11 +229,11 @@ private MapKey convertToMapKey(Object obj) { if (obj instanceof byte[]) { return MapKey.fromBytes((byte[]) obj); } - - throw new IllegalArgumentException("Invalid map key type: " + obj.getClass().getSimpleName() + - ". Map keys must be int, long, String, or byte[]"); + + throw new IllegalArgumentException("Invalid map key type: " + obj.getClass().getSimpleName() + + ". Map keys must be int, long, String, or byte[]"); } - + @Override public String toString() { return String.format("ImprintRecordBuilder{schemaId=%s, fields=%d}", schemaId, fields.size()); diff --git a/src/main/java/com/imprint/error/ErrorType.java b/src/main/java/com/imprint/error/ErrorType.java index 97b9772..49784ef 100644 --- a/src/main/java/com/imprint/error/ErrorType.java +++ b/src/main/java/com/imprint/error/ErrorType.java @@ -14,5 +14,6 @@ public enum ErrorType { TYPE_MISMATCH, INVALID_TYPE_CODE, SERIALIZATION_ERROR, - DESERIALIZATION_ERROR + DESERIALIZATION_ERROR, + INTERNAL_ERROR } diff --git a/src/main/java/com/imprint/types/TypeHandler.java b/src/main/java/com/imprint/types/TypeHandler.java index 573aed3..d21403b 100644 --- a/src/main/java/com/imprint/types/TypeHandler.java +++ b/src/main/java/com/imprint/types/TypeHandler.java @@ -20,8 +20,7 @@ public interface TypeHandler { ByteBuffer readValueBytes(ByteBuffer buffer) throws ImprintException; // Helper method to eliminate duplication in ARRAY/MAP readValueBytes - static ByteBuffer readComplexValueBytes(ByteBuffer buffer, String typeName, - ComplexValueMeasurer measurer) throws ImprintException { + static ByteBuffer readComplexValueBytes(ByteBuffer buffer, String typeName, BufferViewer measurer) throws ImprintException { int initialPosition = buffer.position(); ByteBuffer tempBuffer = buffer.duplicate(); tempBuffer.order(buffer.order()); @@ -58,7 +57,7 @@ static ByteBuffer readComplexValueBytes(ByteBuffer buffer, String typeName, } @FunctionalInterface - interface ComplexValueMeasurer { + interface BufferViewer { int measureDataLength(ByteBuffer tempBuffer, int numElements) throws ImprintException; } diff --git a/src/main/java/com/imprint/util/VarInt.java b/src/main/java/com/imprint/util/VarInt.java index f4c22f2..75bd132 100644 --- a/src/main/java/com/imprint/util/VarInt.java +++ b/src/main/java/com/imprint/util/VarInt.java @@ -6,20 +6,21 @@ import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.ToString; +import lombok.experimental.UtilityClass; + import java.nio.ByteBuffer; /** * Utility class for encoding and decoding variable-length integers (VarInt). * Supports encoding/decoding of 32-bit unsigned integers. */ +@UtilityClass public final class VarInt { private static final byte CONTINUATION_BIT = (byte) 0x80; private static final byte SEGMENT_BITS = 0x7f; private static final int MAX_VARINT_LEN = 5; // Enough for u32 - private VarInt() {} // utility class - /** * Encode a 32-bit unsigned integer as a VarInt into the given ByteBuffer. @@ -103,27 +104,7 @@ public static int encodedLength(int value) { return length; } - - /** - * Read VarInt-prefixed data from a ByteBuffer. - * The data format is: VarInt(length) + data(length bytes). - * Returns a read-only ByteBuffer containing the entire VarInt + data. - * - * @param buffer the buffer to read from - * @return a read-only ByteBuffer view of the VarInt + data - * @throws ImprintException if the VarInt is malformed or buffer underflow - */ - public static ByteBuffer readVarIntPrefixedBytes(ByteBuffer buffer) throws ImprintException { - int originalPosition = buffer.position(); - VarInt.DecodeResult lengthResult = VarInt.decode(buffer); - int totalLength = lengthResult.getBytesRead() + lengthResult.getValue(); - buffer.position(originalPosition); - var valueBuffer = buffer.slice(); - valueBuffer.limit(totalLength); - buffer.position(buffer.position() + totalLength); - return valueBuffer.asReadOnlyBuffer(); - } - + /** * Result of a VarInt decode operation. */ diff --git a/src/test/java/com/imprint/ByteBufferIntegrationTest.java b/src/test/java/com/imprint/ByteBufferIntegrationTest.java deleted file mode 100644 index 56ec3b0..0000000 --- a/src/test/java/com/imprint/ByteBufferIntegrationTest.java +++ /dev/null @@ -1,180 +0,0 @@ -package com.imprint; - -import com.imprint.core.ImprintRecord; -import com.imprint.core.ImprintWriter; -import com.imprint.core.SchemaId; -import com.imprint.error.ImprintException; -import com.imprint.types.MapKey; -import com.imprint.types.Value; - -import java.nio.ByteBuffer; -import java.util.*; - -/** - * Integration test to verify ByteBuffer functionality and zero-copy benefits. - */ -class ByteBufferIntegrationTest { - - public static void main(String[] args) { - try { - testByteBufferFunctionality(); - testZeroCopy(); - testArrayBackedBuffers(); - System.out.println("All ByteBuffer integration tests passed!"); - } catch (Exception e) { - System.err.println("ByteBuffer integration test failed: " + e.getMessage()); - e.printStackTrace(); - System.exit(1); - } - } - - static void testByteBufferFunctionality() throws ImprintException { - System.out.println("Testing ByteBuffer functionality..."); - - SchemaId schemaId = new SchemaId(1, 0xdeadbeef); - ImprintWriter writer = new ImprintWriter(schemaId); - - writer.addField(1, Value.fromInt32(42)) - .addField(2, Value.fromString("zero-copy test")) - .addField(3, Value.fromBytes(new byte[]{1, 2, 3, 4, 5})); - - ImprintRecord record = writer.build(); - - // Test ByteBuffer serialization - ByteBuffer serializedBuffer = record.serializeToBuffer(); - assert serializedBuffer.isReadOnly() : "Serialized buffer should be read-only"; - - // Test deserialization from ByteBuffer - ImprintRecord deserialized = ImprintRecord.deserialize(serializedBuffer); - - assert Objects.equals(deserialized.getValue(1), Value.fromInt32(42)); - assert Objects.equals(deserialized.getValue(2), Value.fromString("zero-copy test")); - - // Test raw bytes access returns ByteBuffer - var rawBytes = deserialized.getRawBytes(1); - assert rawBytes != null : "Raw bytes should be present for field 1"; - assert rawBytes.isReadOnly() : "Raw bytes buffer should be read-only"; - - System.out.println("ByteBuffer functionality test passed"); - } - - static void testZeroCopy() { - System.out.println("Testing zero-copy"); - - // Create a large payload to demonstrate zero-copy benefits - byte[] largePayload = new byte[1024 * 1024]; // 1MB - Arrays.fill(largePayload, (byte) 0xAB); - - SchemaId schemaId = new SchemaId(2, 0xcafebabe); - ImprintWriter writer = new ImprintWriter(schemaId); - - try { - writer.addField(1, Value.fromBytes(largePayload)); - ImprintRecord record = writer.build(); - - // Test that getRawBytes returns a view, not a copy - var rawBytes = record.getRawBytes(1); - assert rawBytes !=null : "Raw bytes should be present"; - assert rawBytes.isReadOnly() : "Raw buffer should be read-only"; - - // The buffer should be positioned at the start of the actual data - // (after the VarInt length prefix) - assert rawBytes.remaining() > largePayload.length : "Buffer should include length prefix"; - - System.out.println("Zero-copy benefits test passed"); - - } catch (ImprintException e) { - throw new RuntimeException("Failed zero-copy test", e); - } - } - - static void testArrayBackedBuffers() throws ImprintException { - System.out.println("Testing array-backed buffers for zero-copy performance..."); - - // Test serialized buffers are array-backed - SchemaId schemaId = new SchemaId(1, 0xdeadbeef); - ImprintWriter writer = new ImprintWriter(schemaId); - - writer.addField(1, Value.fromInt32(42)) - .addField(2, Value.fromString("test string")) - .addField(3, Value.fromBytes(new byte[]{1, 2, 3, 4})) - .addField(4, Value.fromBoolean(true)); - - ImprintRecord record = writer.build(); - - // Test that serializeToBuffer() returns array-backed buffer - ByteBuffer serializedBuffer = record.serializeToBuffer(); - assert serializedBuffer.hasArray() : "Serialized buffer should be array-backed for zero-copy performance"; - - // Test that the internal payload is array-backed - assert record.getPayload().hasArray() : "Record payload should be array-backed for zero-copy performance"; - - // Test deserialized buffers are array-backed - byte[] bytes = new byte[serializedBuffer.remaining()]; - serializedBuffer.get(bytes); - ImprintRecord deserialized = ImprintRecord.deserialize(bytes); - - assert deserialized.getPayload().hasArray() : "Deserialized record payload should be array-backed"; - - // Test that getRawBytes() returns array-backed buffers - ByteBuffer rawBytes1 = deserialized.getRawBytes(1); - ByteBuffer rawBytes2 = deserialized.getRawBytes(2); - - assert rawBytes1 != null && rawBytes1.hasArray() : "Raw bytes buffer for int field should be array-backed"; - assert rawBytes2 != null && rawBytes2.hasArray() : "Raw bytes buffer for string field should be array-backed"; - - // Test complex types use array-backed buffers - ImprintWriter complexWriter = new ImprintWriter(new SchemaId(2, 0xcafebabe)); - - // Create array value - List arrayValues = Arrays.asList( - Value.fromInt32(1), - Value.fromInt32(2), - Value.fromInt32(3) - ); - - // Create map value - Map mapValues = new HashMap<>(); - mapValues.put(MapKey.fromString("key1"), Value.fromString("value1")); - mapValues.put(MapKey.fromString("key2"), Value.fromString("value2")); - - complexWriter.addField(1, Value.fromArray(arrayValues)) - .addField(2, Value.fromMap(mapValues)); - - ImprintRecord complexRecord = complexWriter.build(); - - assert complexRecord.getPayload().hasArray() : "Record with complex types should use array-backed payload"; - - ByteBuffer complexSerialized = complexRecord.serializeToBuffer(); - assert complexSerialized.hasArray() : "Serialized buffer with complex types should be array-backed"; - - // Test nested records use array-backed buffers - ImprintWriter innerWriter = new ImprintWriter(new SchemaId(3, 0x12345678)); - innerWriter.addField(1, Value.fromString("nested data")); - ImprintRecord innerRecord = innerWriter.build(); - - ImprintWriter outerWriter = new ImprintWriter(new SchemaId(4, 0x87654321)); - outerWriter.addField(1, Value.fromRow(innerRecord)); - ImprintRecord outerRecord = outerWriter.build(); - - assert innerRecord.getPayload().hasArray() : "Inner record payload should be array-backed"; - assert outerRecord.getPayload().hasArray() : "Outer record payload should be array-backed"; - - ByteBuffer nestedSerialized = outerRecord.serializeToBuffer(); - assert nestedSerialized.hasArray() : "Serialized nested record should be array-backed"; - - // Test builder pattern uses array-backed buffers - ImprintRecord builderRecord = ImprintRecord.builder(1, 0xabcdef00) - .field(1, "test string") - .field(2, 42) - .field(3, new byte[]{1, 2, 3}) - .build(); - - assert builderRecord.getPayload().hasArray() : "Builder-created record should use array-backed payload"; - - ByteBuffer builderSerialized = builderRecord.serializeToBuffer(); - assert builderSerialized.hasArray() : "Builder-created serialized buffer should be array-backed"; - - System.out.println("✓ Array-backed buffers test passed"); - } -} \ No newline at end of file diff --git a/src/test/java/com/imprint/IntegrationTest.java b/src/test/java/com/imprint/IntegrationTest.java index 565b7cd..76efcc5 100644 --- a/src/test/java/com/imprint/IntegrationTest.java +++ b/src/test/java/com/imprint/IntegrationTest.java @@ -2,143 +2,301 @@ import com.imprint.core.*; import com.imprint.types.*; +import com.imprint.error.ErrorType; import com.imprint.error.ImprintException; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.DisplayName; + import java.util.*; +import static org.junit.jupiter.api.Assertions.*; + /** - * Integration test to verify the complete Java implementation works. - * This can be run as a simple main method without JUnit. + * Integration tests for Imprint core functionality. */ public class IntegrationTest { - - public static void main(String[] args) { - try { - testBasicFunctionality(); - testArraysAndMaps(); - testNestedRecords(); - System.out.println("All integration tests passed!"); - } catch (Exception e) { - System.err.println("Integration test failed: " + e.getMessage()); - e.printStackTrace(); - System.exit(1); - } - } - - static void testBasicFunctionality() throws ImprintException { - System.out.println("Testing basic functionality..."); - + + // Removed main method, individual methods are now JUnit tests. + + @Test + @DisplayName("Basic functionality: create, serialize, deserialize primitive types") + void testBasicFunctionality() throws ImprintException { + System.out.println("Testing basic functionality..."); // Keep for now if desired, or remove + SchemaId schemaId = new SchemaId(1, 0xdeadbeef); - ImprintWriter writer = new ImprintWriter(schemaId); - - writer.addField(1, Value.fromInt32(42)) - .addField(2, Value.fromString("testing java imprint spec")) - .addField(3, Value.fromBoolean(true)) - .addField(4, Value.fromFloat64(3.14159)) - .addField(5, Value.fromBytes(new byte[]{1, 2, 3, 4})); - - ImprintRecord record = writer.build(); - - // Verify we can read values back - assert Objects.equals(record.getValue(1), Value.fromInt32(42)); - assert Objects.equals(record.getValue(2), Value.fromString("testing java imprint spec")); - assert Objects.equals(record.getValue(3), Value.fromBoolean(true)); - assert record.getValue(999) == null; // non-existent field - + // Using ImprintRecordBuilder for consistency with other tests + ImprintRecord record = ImprintRecord.builder(schemaId) + .field(1, 42) + .field(2, "testing java imprint spec") + .field(3, true) + .field(4, 3.14159) // double + .field(5, new byte[]{1, 2, 3, 4}) + .build(); + + // Verify we can read values back using ergonomic getters + assertEquals(42, record.getInt32(1)); + assertEquals("testing java imprint spec", record.getString(2)); + assertTrue(record.getBoolean(3)); + assertEquals(3.14159, record.getFloat64(4)); + assertArrayEquals(new byte[]{1,2,3,4}, record.getBytes(5)); + + assertNull(record.getValue(999), "Non-existent field should return null from getValue()"); + assertThrows(ImprintException.class, () -> record.getInt32(999), "Accessing non-existent field with getInt32 should throw"); + // Test serialization round-trip var buffer = record.serializeToBuffer(); byte[] serialized = new byte[buffer.remaining()]; buffer.get(serialized); ImprintRecord deserialized = ImprintRecord.deserialize(serialized); - - assert Objects.equals(deserialized.getValue(1), Value.fromInt32(42)); - assert Objects.equals(deserialized.getValue(2), Value.fromString("testing java imprint spec")); - assert Objects.equals(deserialized.getValue(3), Value.fromBoolean(true)); - + + assertEquals(42, deserialized.getInt32(1)); + assertEquals("testing java imprint spec", deserialized.getString(2)); + assertTrue(deserialized.getBoolean(3)); + assertEquals(3.14159, deserialized.getFloat64(4)); + assertArrayEquals(new byte[]{1,2,3,4}, deserialized.getBytes(5)); + System.out.println("Basic functionality test passed"); } - - static void testArraysAndMaps() throws ImprintException { + + @Test + @DisplayName("Collections: create, serialize, deserialize arrays and maps") + void testArraysAndMaps() throws ImprintException { System.out.println("Testing arrays and maps..."); - + SchemaId schemaId = new SchemaId(2, 0xcafebabe); - ImprintWriter writer = new ImprintWriter(schemaId); - - // Create an array - List intArray = Arrays.asList( - Value.fromInt32(1), - Value.fromInt32(2), - Value.fromInt32(3) - ); - + + // Create an array using builder for convenience + List sourceIntList = Arrays.asList(1, 2, 3); + // Create a map - Map stringToIntMap = new HashMap<>(); - stringToIntMap.put(MapKey.fromString("one"), Value.fromInt32(1)); - stringToIntMap.put(MapKey.fromString("two"), Value.fromInt32(2)); - - writer.addField(1, Value.fromArray(intArray)) - .addField(2, Value.fromMap(stringToIntMap)); - - ImprintRecord record = writer.build(); - + Map sourceStringToIntMap = new HashMap<>(); + sourceStringToIntMap.put("one", 1); + sourceStringToIntMap.put("two", 2); + + ImprintRecord record = ImprintRecord.builder(schemaId) + .field(1, sourceIntList) // Builder converts List to List + .field(2, sourceStringToIntMap) // Builder converts Map + .build(); + // Test serialization round-trip var buffer = record.serializeToBuffer(); byte[] serialized = new byte[buffer.remaining()]; buffer.get(serialized); ImprintRecord deserialized = ImprintRecord.deserialize(serialized); - + // Verify array - Value arrayValue = deserialized.getValue(1); - assert arrayValue instanceof Value.ArrayValue; - List deserializedArray = ((Value.ArrayValue) arrayValue).getValue(); - assert deserializedArray.size() == 3; - assert deserializedArray.get(0).equals(Value.fromInt32(1)); - + List deserializedArray = deserialized.getArray(1); + assertNotNull(deserializedArray); + assertEquals(3, deserializedArray.size()); + assertEquals(Value.fromInt32(1), deserializedArray.get(0)); + assertEquals(Value.fromInt32(2), deserializedArray.get(1)); + assertEquals(Value.fromInt32(3), deserializedArray.get(2)); + // Verify map - Value mapValue = deserialized.getValue(2); - assert mapValue instanceof Value.MapValue; - Map deserializedMap = ((Value.MapValue) mapValue).getValue(); - assert deserializedMap.size() == 2; - assert deserializedMap.get(MapKey.fromString("one")).equals(Value.fromInt32(1)); - - System.out.println("✓ Arrays and maps test passed"); + Map deserializedMap = deserialized.getMap(2); + assertNotNull(deserializedMap); + assertEquals(2, deserializedMap.size()); + assertEquals(Value.fromInt32(1), deserializedMap.get(MapKey.fromString("one"))); + assertEquals(Value.fromInt32(2), deserializedMap.get(MapKey.fromString("two"))); + + System.out.println("Arrays and maps test passed"); } - - static void testNestedRecords() throws ImprintException { + + @Test + @DisplayName("Nested Records: create, serialize, deserialize records within records") + void testNestedRecords() throws ImprintException { System.out.println("Testing nested records..."); - - // Create inner record + SchemaId innerSchemaId = new SchemaId(3, 0x12345678); - ImprintWriter innerWriter = new ImprintWriter(innerSchemaId); - innerWriter.addField(1, Value.fromString("nested data")) - .addField(2, Value.fromInt64(9876543210L)); - ImprintRecord innerRecord = innerWriter.build(); - - // Create outer record + ImprintRecord innerRecord = ImprintRecord.builder(innerSchemaId) + .field(1, "nested data") + .field(2, 9876543210L) + .build(); + SchemaId outerSchemaId = new SchemaId(4, 0x87654321); - ImprintWriter outerWriter = new ImprintWriter(outerSchemaId); - outerWriter.addField(1, Value.fromRow(innerRecord)) - .addField(2, Value.fromString("outer data")); - ImprintRecord outerRecord = outerWriter.build(); - - // Test serialization round-trip + ImprintRecord outerRecord = ImprintRecord.builder(outerSchemaId) + .field(1, innerRecord) // Builder handles ImprintRecord directly + .field(2, "outer data") + .build(); + var buffer = outerRecord.serializeToBuffer(); byte[] serialized = new byte[buffer.remaining()]; buffer.get(serialized); ImprintRecord deserialized = ImprintRecord.deserialize(serialized); - - // Verify outer record - assert deserialized.getHeader().getSchemaId().getFieldSpaceId() == 4; - assert Objects.equals(deserialized.getValue(2), Value.fromString("outer data")); - - // Verify nested record - Value rowValue = deserialized.getValue(1); - assert rowValue instanceof Value.RowValue; - ImprintRecord nestedRecord = ((Value.RowValue) rowValue).getValue(); - - assert nestedRecord.getHeader().getSchemaId().getFieldSpaceId() == 3; - assert Objects.equals(nestedRecord.getValue(1), Value.fromString("nested data")); - assert Objects.equals(nestedRecord.getValue(2), Value.fromInt64(9876543210L)); - + + assertEquals(4, deserialized.getHeader().getSchemaId().getFieldSpaceId()); + assertEquals("outer data", deserialized.getString(2)); + + ImprintRecord nestedDeserialized = deserialized.getRow(1); + assertNotNull(nestedDeserialized); + assertEquals(3, nestedDeserialized.getHeader().getSchemaId().getFieldSpaceId()); + assertEquals("nested data", nestedDeserialized.getString(1)); + assertEquals(9876543210L, nestedDeserialized.getInt64(2)); + System.out.println("✓ Nested records test passed"); } + + // --- Start of broken down tests for ErgonomicGettersAndNestedTypes --- + + private ImprintRecord createTestRecordForGetters() throws ImprintException { + SchemaId schemaId = new SchemaId(5, 0xabcdef01); + + List innerList1 = Arrays.asList(Value.fromInt32(10), Value.fromInt32(20)); + List innerList2 = Arrays.asList(Value.fromInt32(30), Value.fromInt32(40)); + List listOfLists = Arrays.asList(Value.fromArray(innerList1), Value.fromArray(innerList2)); + + Map mapWithArrayValue = new HashMap<>(); + mapWithArrayValue.put(MapKey.fromString("list1"), Value.fromArray(innerList1)); + + return ImprintRecord.builder(schemaId) + .field(1, true) + .field(2, 12345) + .field(3, 9876543210L) + .field(4, 3.14f) + .field(5, 2.718281828) + .field(6, "hello type world") + .field(7, new byte[]{10, 20, 30}) + .nullField(8) + .field(9, Value.fromArray(listOfLists)) // Array of Arrays (using Value directly for test setup) + .field(10, Value.fromMap(mapWithArrayValue)) // Map with Array value + .field(11, Collections.emptyList()) // Empty Array via builder + .field(12, Collections.emptyMap()) // Empty Map via builder + .build(); + } + + private ImprintRecord serializeAndDeserialize(ImprintRecord record) throws ImprintException { + var buffer = record.serializeToBuffer(); + byte[] serialized = new byte[buffer.remaining()]; + buffer.get(serialized); + return ImprintRecord.deserialize(serialized); + } + + @Test + @DisplayName("Type Getters: Basic primitive and String types") + void testBasicTypeGetters() throws ImprintException { + ImprintRecord originalRecord = createTestRecordForGetters(); + ImprintRecord record = serializeAndDeserialize(originalRecord); + + assertTrue(record.getBoolean(1)); + assertEquals(12345, record.getInt32(2)); + assertEquals(9876543210L, record.getInt64(3)); + assertEquals(3.14f, record.getFloat32(4)); + assertEquals(2.718281828, record.getFloat64(5)); + assertEquals("hello type world", record.getString(6)); + assertArrayEquals(new byte[]{10, 20, 30}, record.getBytes(7)); + } + + @Test + @DisplayName("Type Getters: Array of Arrays") + void testTypeGetterArrayOfArrays() throws ImprintException { + ImprintRecord originalRecord = createTestRecordForGetters(); + ImprintRecord record = serializeAndDeserialize(originalRecord); + + List arrOfArr = record.getArray(9); + assertNotNull(arrOfArr); + assertEquals(2, arrOfArr.size()); + assertInstanceOf(Value.ArrayValue.class, arrOfArr.get(0)); + Value.ArrayValue firstInnerArray = (Value.ArrayValue) arrOfArr.get(0); + assertEquals(2, firstInnerArray.getValue().size()); + assertEquals(Value.fromInt32(10), firstInnerArray.getValue().get(0)); + assertEquals(Value.fromInt32(20), firstInnerArray.getValue().get(1)); + + assertInstanceOf(Value.ArrayValue.class, arrOfArr.get(1)); + Value.ArrayValue secondInnerArray = (Value.ArrayValue) arrOfArr.get(1); + assertEquals(2, secondInnerArray.getValue().size()); + assertEquals(Value.fromInt32(30), secondInnerArray.getValue().get(0)); + assertEquals(Value.fromInt32(40), secondInnerArray.getValue().get(1)); + } + + @Test + @DisplayName("Type Getters: Map with Array Value") + void testTypeGetterMapWithArrayValue() throws ImprintException { + ImprintRecord originalRecord = createTestRecordForGetters(); + ImprintRecord record = serializeAndDeserialize(originalRecord); + + Map mapWithArr = record.getMap(10); + assertNotNull(mapWithArr); + assertEquals(1, mapWithArr.size()); + assertInstanceOf(Value.ArrayValue.class, mapWithArr.get(MapKey.fromString("list1"))); + Value.ArrayValue innerArray = (Value.ArrayValue) mapWithArr.get(MapKey.fromString("list1")); + assertNotNull(innerArray); + assertEquals(2, innerArray.getValue().size()); + assertEquals(Value.fromInt32(10), innerArray.getValue().get(0)); + } + + @Test + @DisplayName("Type Getters: Empty Collections (Array and Map)") + void testErgonomicGettersEmptyCollections() throws ImprintException { + ImprintRecord originalRecord = createTestRecordForGetters(); + ImprintRecord record = serializeAndDeserialize(originalRecord); + + List emptyArr = record.getArray(11); + assertNotNull(emptyArr); + assertTrue(emptyArr.isEmpty()); + + Map emptyMap = record.getMap(12); + assertNotNull(emptyMap); + assertTrue(emptyMap.isEmpty()); + } + + @Test + @DisplayName("Type Getters: Exception for Field Not Found") + void testErgonomicGetterExceptionFieldNotFound() throws ImprintException { + ImprintRecord originalRecord = createTestRecordForGetters(); + ImprintRecord record = serializeAndDeserialize(originalRecord); + + ImprintException ex = assertThrows(ImprintException.class, () -> record.getInt32(99)); + assertEquals(ErrorType.FIELD_NOT_FOUND, ex.getErrorType()); + } + + @Test + @DisplayName("Type Getters: Exception for Null Field accessed as primitive") + void testErgonomicGetterExceptionNullField() throws ImprintException { + ImprintRecord originalRecord = createTestRecordForGetters(); + ImprintRecord record = serializeAndDeserialize(originalRecord); + + ImprintException ex = assertThrows(ImprintException.class, () -> record.getString(8)); + assertEquals(ErrorType.TYPE_MISMATCH, ex.getErrorType()); // getString throws TYPE_MISMATCH for null + assertTrue(ex.getMessage().contains("Field 8 is NULL")); + + + // Also test getValue for a null field returns Value.NullValue + Value nullValueField = record.getValue(8); + assertNotNull(nullValueField); + assertInstanceOf(Value.NullValue.class, nullValueField, "Field 8 should be Value.NullValue"); + } + + @Test + @DisplayName("Type Getters: Exception for Type Mismatch") + void testErgonomicGetterExceptionTypeMismatch() throws ImprintException { + ImprintRecord originalRecord = createTestRecordForGetters(); + ImprintRecord record = serializeAndDeserialize(originalRecord); + + ImprintException ex = assertThrows(ImprintException.class, () -> record.getInt32(6)); // Field 6 is a String + assertEquals(ErrorType.TYPE_MISMATCH, ex.getErrorType()); + } + + @Test + @DisplayName("Type Getters: Row (Nested Record)") + void testErgonomicGetterRow() throws ImprintException { + SchemaId innerSchemaId = new SchemaId(6, 0x12345678); + ImprintRecord innerRecord = ImprintRecord.builder(innerSchemaId) + .field(101, "nested string") + .field(102, 999L) + .build(); + + ImprintRecord recordWithRow = ImprintRecord.builder(new SchemaId(7, 0x87654321)) + .field(201, innerRecord) // Using builder to add row + .field(202, "outer field") + .build(); + + ImprintRecord deserializedWithRow = serializeAndDeserialize(recordWithRow); + + ImprintRecord retrievedRow = deserializedWithRow.getRow(201); + assertNotNull(retrievedRow); + assertEquals(innerSchemaId, retrievedRow.getHeader().getSchemaId()); + assertEquals("nested string", retrievedRow.getString(101)); + assertEquals(999L, retrievedRow.getInt64(102)); + assertEquals("outer field", deserializedWithRow.getString(202)); + } } \ No newline at end of file From 2e56688ecbdefa4dca64237b72f2ea3eed4983d9 Mon Sep 17 00:00:00 2001 From: expand3d <> Date: Tue, 3 Jun 2025 18:53:18 -0400 Subject: [PATCH 17/49] minor style fixes --- .../java/com/imprint/core/ImprintRecord.java | 13 +- .../com/imprint/types/TypeHandlerTest.java | 122 +++++++++--------- .../java/com/imprint/types/ValueTest.java | 100 +++++++------- 3 files changed, 118 insertions(+), 117 deletions(-) diff --git a/src/main/java/com/imprint/core/ImprintRecord.java b/src/main/java/com/imprint/core/ImprintRecord.java index d667039..dd66389 100644 --- a/src/main/java/com/imprint/core/ImprintRecord.java +++ b/src/main/java/com/imprint/core/ImprintRecord.java @@ -292,12 +292,6 @@ private Value deserializeValue(TypeCode typeCode, ByteBuffer buffer) throws Impr } } - @Override - public String toString() { - return String.format("ImprintRecord{header=%s, directorySize=%d, payloadSize=%d}", - header, directory.size(), payload.remaining()); - } - private T getTypedValueOrThrow(int fieldId, TypeCode expectedTypeCode, Class expectedValueClass, String expectedTypeName) throws ImprintException { Value value = getValue(fieldId); @@ -439,4 +433,11 @@ public Map getMap(int fieldId) throws ImprintException { public ImprintRecord getRow(int fieldId) throws ImprintException { return getTypedValueOrThrow(fieldId, TypeCode.ROW, Value.RowValue.class, "ROW").getValue(); } + + @Override + public String toString() { + return String.format("ImprintRecord{header=%s, directorySize=%d, payloadSize=%d}", + header, directory.size(), payload.remaining()); + } + } \ No newline at end of file diff --git a/src/test/java/com/imprint/types/TypeHandlerTest.java b/src/test/java/com/imprint/types/TypeHandlerTest.java index 9a4ae85..f131a0f 100644 --- a/src/test/java/com/imprint/types/TypeHandlerTest.java +++ b/src/test/java/com/imprint/types/TypeHandlerTest.java @@ -20,104 +20,104 @@ class TypeHandlerTest { void testNullHandler() throws ImprintException { var handler = TypeHandler.NULL; var value = Value.nullValue(); - + // Size estimation assertThat(handler.estimateSize(value)).isEqualTo(0); - + // Serialization var buffer = ByteBuffer.allocate(10); handler.serialize(value, buffer); assertThat(buffer.position()).isEqualTo(0); // NULL writes nothing - + // Deserialization buffer.flip(); var deserialized = handler.deserialize(buffer); assertThat(deserialized).isEqualTo(value); - + // readValueBytes buffer.clear(); var valueBytes = handler.readValueBytes(buffer); assertThat(valueBytes.remaining()).isEqualTo(0); } - + @ParameterizedTest @ValueSource(booleans = {true, false}) void testBoolHandler(boolean testValue) throws ImprintException { var handler = TypeHandler.BOOL; var value = Value.fromBoolean(testValue); - + // Size estimation assertThat(handler.estimateSize(value)).isEqualTo(1); - + // Round-trip test var buffer = ByteBuffer.allocate(10); handler.serialize(value, buffer); assertThat(buffer.position()).isEqualTo(1); - + buffer.flip(); var deserialized = handler.deserialize(buffer); assertThat(deserialized).isEqualTo(value); assertThat(((Value.BoolValue) deserialized).getValue()).isEqualTo(testValue); } - + @ParameterizedTest @ValueSource(ints = {0, 1, -1, Integer.MAX_VALUE, Integer.MIN_VALUE, 42, -42}) void testInt32Handler(int testValue) throws ImprintException { var handler = TypeHandler.INT32; var value = Value.fromInt32(testValue); - + // Size estimation assertThat(handler.estimateSize(value)).isEqualTo(4); - + // Round-trip test var buffer = ByteBuffer.allocate(10).order(ByteOrder.LITTLE_ENDIAN); handler.serialize(value, buffer); assertThat(buffer.position()).isEqualTo(4); - + buffer.flip(); var deserialized = handler.deserialize(buffer); assertThat(deserialized).isEqualTo(value); assertThat(((Value.Int32Value) deserialized).getValue()).isEqualTo(testValue); } - + @ParameterizedTest @ValueSource(longs = {0L, 1L, -1L, Long.MAX_VALUE, Long.MIN_VALUE, 123456789L}) void testInt64Handler(long testValue) throws ImprintException { var handler = TypeHandler.INT64; var value = Value.fromInt64(testValue); - + // Size estimation assertThat(handler.estimateSize(value)).isEqualTo(8); - + // Round-trip test var buffer = ByteBuffer.allocate(20).order(ByteOrder.LITTLE_ENDIAN); handler.serialize(value, buffer); assertThat(buffer.position()).isEqualTo(8); - + buffer.flip(); var deserialized = handler.deserialize(buffer); assertThat(deserialized).isEqualTo(value); assertThat(((Value.Int64Value) deserialized).getValue()).isEqualTo(testValue); } - + @ParameterizedTest @ValueSource(floats = {0.0f, 1.0f, -1.0f, Float.MAX_VALUE, Float.MIN_VALUE, 3.14159f, Float.NaN, Float.POSITIVE_INFINITY}) void testFloat32Handler(float testValue) throws ImprintException { var handler = TypeHandler.FLOAT32; var value = Value.fromFloat32(testValue); - + // Size estimation assertThat(handler.estimateSize(value)).isEqualTo(4); - + // Round-trip test var buffer = ByteBuffer.allocate(10).order(ByteOrder.LITTLE_ENDIAN); handler.serialize(value, buffer); assertThat(buffer.position()).isEqualTo(4); - + buffer.flip(); var deserialized = handler.deserialize(buffer); assertThat(deserialized).isEqualTo(value); - + float deserializedValue = ((Value.Float32Value) deserialized).getValue(); if (Float.isNaN(testValue)) { assertThat(deserializedValue).isNaN(); @@ -125,25 +125,25 @@ void testFloat32Handler(float testValue) throws ImprintException { assertThat(deserializedValue).isEqualTo(testValue); } } - + @ParameterizedTest @ValueSource(doubles = {0.0, 1.0, -1.0, Double.MAX_VALUE, Double.MIN_VALUE, Math.PI, Double.NaN, Double.POSITIVE_INFINITY}) void testFloat64Handler(double testValue) throws ImprintException { var handler = TypeHandler.FLOAT64; var value = Value.fromFloat64(testValue); - + // Size estimation assertThat(handler.estimateSize(value)).isEqualTo(8); - + // Round-trip test var buffer = ByteBuffer.allocate(20).order(ByteOrder.LITTLE_ENDIAN); handler.serialize(value, buffer); assertThat(buffer.position()).isEqualTo(8); - + buffer.flip(); var deserialized = handler.deserialize(buffer); assertThat(deserialized).isEqualTo(value); - + double deserializedValue = ((Value.Float64Value) deserialized).getValue(); if (Double.isNaN(testValue)) { assertThat(deserializedValue).isNaN(); @@ -151,129 +151,129 @@ void testFloat64Handler(double testValue) throws ImprintException { assertThat(deserializedValue).isEqualTo(testValue); } } - + @ParameterizedTest @ValueSource(strings = {"", "hello", "世界", "a very long string that exceeds typical buffer sizes and contains unicode: 🚀🎉", "null\0bytes"}) void testStringHandler(String testValue) throws ImprintException { var handler = TypeHandler.STRING; var value = Value.fromString(testValue); - + byte[] utf8Bytes = testValue.getBytes(java.nio.charset.StandardCharsets.UTF_8); int expectedSize = com.imprint.util.VarInt.encodedLength(utf8Bytes.length) + utf8Bytes.length; - + // Size estimation assertThat(handler.estimateSize(value)).isEqualTo(expectedSize); - + // Round-trip test var buffer = ByteBuffer.allocate(expectedSize + 20).order(ByteOrder.LITTLE_ENDIAN); handler.serialize(value, buffer); - + buffer.flip(); var deserialized = handler.deserialize(buffer); - + // Should return StringBufferValue (zero-copy implementation) assertThat(deserialized).isInstanceOf(Value.StringBufferValue.class); - + String deserializedString; if (deserialized instanceof Value.StringBufferValue) { deserializedString = ((Value.StringBufferValue) deserialized).getValue(); } else { deserializedString = ((Value.StringValue) deserialized).getValue(); } - + assertThat(deserializedString).isEqualTo(testValue); } - + @Test void testBytesHandlerWithArrayValue() throws ImprintException { var handler = TypeHandler.BYTES; byte[] testBytes = {0, 1, 2, (byte) 0xFF, 42, 127, -128}; var value = Value.fromBytes(testBytes); - + int expectedSize = com.imprint.util.VarInt.encodedLength(testBytes.length) + testBytes.length; - + // Size estimation assertThat(handler.estimateSize(value)).isEqualTo(expectedSize); - + // Round-trip test var buffer = ByteBuffer.allocate(expectedSize + 20).order(ByteOrder.LITTLE_ENDIAN); handler.serialize(value, buffer); - + buffer.flip(); var deserialized = handler.deserialize(buffer); - + // Should return BytesBufferValue (zero-copy implementation) assertThat(deserialized).isInstanceOf(Value.BytesBufferValue.class); - + byte[] deserializedBytes = ((Value.BytesBufferValue) deserialized).getValue(); assertThat(deserializedBytes).isEqualTo(testBytes); } - + @Test void testBytesHandlerWithBufferValue() throws ImprintException { var handler = TypeHandler.BYTES; byte[] testBytes = {10, 20, 30, 40}; var bufferValue = Value.fromBytesBuffer(ByteBuffer.wrap(testBytes).asReadOnlyBuffer()); - + int expectedSize = com.imprint.util.VarInt.encodedLength(testBytes.length) + testBytes.length; - + // Size estimation assertThat(handler.estimateSize(bufferValue)).isEqualTo(expectedSize); - + // Round-trip test var buffer = ByteBuffer.allocate(expectedSize + 20).order(ByteOrder.LITTLE_ENDIAN); handler.serialize(bufferValue, buffer); - + buffer.flip(); var deserialized = handler.deserialize(buffer); - + byte[] deserializedBytes = ((Value.BytesBufferValue) deserialized).getValue(); assertThat(deserializedBytes).isEqualTo(testBytes); } - + @Test void testStringHandlerWithBufferValue() throws ImprintException { var handler = TypeHandler.STRING; String testString = "zero-copy string test"; byte[] utf8Bytes = testString.getBytes(java.nio.charset.StandardCharsets.UTF_8); var bufferValue = Value.fromStringBuffer(ByteBuffer.wrap(utf8Bytes).asReadOnlyBuffer()); - + int expectedSize = com.imprint.util.VarInt.encodedLength(utf8Bytes.length) + utf8Bytes.length; - + // Size estimation assertThat(handler.estimateSize(bufferValue)).isEqualTo(expectedSize); - + // Round-trip test var buffer = ByteBuffer.allocate(expectedSize + 20).order(ByteOrder.LITTLE_ENDIAN); handler.serialize(bufferValue, buffer); - + buffer.flip(); var deserialized = handler.deserialize(buffer); - + String deserializedString = ((Value.StringBufferValue) deserialized).getValue(); assertThat(deserializedString).isEqualTo(testString); } - + @Test void testBoolHandlerInvalidValue() { var handler = TypeHandler.BOOL; var buffer = ByteBuffer.allocate(10); buffer.put((byte) 2); // Invalid boolean value buffer.flip(); - + assertThatThrownBy(() -> handler.deserialize(buffer)) - .isInstanceOf(ImprintException.class) - .hasMessageContaining("Invalid boolean value: 2"); + .isInstanceOf(ImprintException.class) + .hasMessageContaining("Invalid boolean value: 2"); } - + @Test void testHandlerBufferUnderflow() { // Test that handlers properly detect buffer underflow var int32Handler = TypeHandler.INT32; var buffer = ByteBuffer.allocate(2); // Too small for int32 - + assertThatThrownBy(() -> int32Handler.deserialize(buffer)) - .isInstanceOf(ImprintException.class) - .hasMessageContaining("Not enough bytes for int32"); + .isInstanceOf(ImprintException.class) + .hasMessageContaining("Not enough bytes for int32"); } } \ No newline at end of file diff --git a/src/test/java/com/imprint/types/ValueTest.java b/src/test/java/com/imprint/types/ValueTest.java index c400a44..b092bb7 100644 --- a/src/test/java/com/imprint/types/ValueTest.java +++ b/src/test/java/com/imprint/types/ValueTest.java @@ -12,143 +12,143 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; class ValueTest { - + @Test void shouldCreateNullValue() { Value value = Value.nullValue(); - + assertThat(value).isInstanceOf(Value.NullValue.class); assertThat(value.getTypeCode()).isEqualTo(TypeCode.NULL); assertThat(value.toString()).isEqualTo("null"); } - + @Test void shouldCreateBooleanValues() { Value trueValue = Value.fromBoolean(true); Value falseValue = Value.fromBoolean(false); - + assertThat(trueValue).isInstanceOf(Value.BoolValue.class); assertThat(((Value.BoolValue) trueValue).getValue()).isTrue(); assertThat(trueValue.getTypeCode()).isEqualTo(TypeCode.BOOL); - + assertThat(falseValue).isInstanceOf(Value.BoolValue.class); assertThat(((Value.BoolValue) falseValue).getValue()).isFalse(); assertThat(falseValue.getTypeCode()).isEqualTo(TypeCode.BOOL); } - + @Test void shouldCreateNumericValues() { var int32 = Value.fromInt32(42); var int64 = Value.fromInt64(123456789L); var float32 = Value.fromFloat32(3.14f); var float64 = Value.fromFloat64(2.718281828); - + assertThat(int32.getTypeCode()).isEqualTo(TypeCode.INT32); assertThat(((Value.Int32Value) int32).getValue()).isEqualTo(42); - + assertThat(int64.getTypeCode()).isEqualTo(TypeCode.INT64); assertThat(((Value.Int64Value) int64).getValue()).isEqualTo(123456789L); - + assertThat(float32.getTypeCode()).isEqualTo(TypeCode.FLOAT32); assertThat(((Value.Float32Value) float32).getValue()).isEqualTo(3.14f); - + assertThat(float64.getTypeCode()).isEqualTo(TypeCode.FLOAT64); assertThat(((Value.Float64Value) float64).getValue()).isEqualTo(2.718281828); } - + @Test void shouldCreateBytesAndStringValues() { byte[] bytes = {1, 2, 3, 4}; var bytesValue = Value.fromBytes(bytes); var stringValue = Value.fromString("hello"); - + assertThat(bytesValue.getTypeCode()).isEqualTo(TypeCode.BYTES); assertThat(((Value.BytesValue) bytesValue).getValue()).isEqualTo(bytes); - + assertThat(stringValue.getTypeCode()).isEqualTo(TypeCode.STRING); assertThat(((Value.StringValue) stringValue).getValue()).isEqualTo("hello"); } - + @Test void shouldCreateArrayValues() { List elements = Arrays.asList( - Value.fromInt32(1), - Value.fromInt32(2), - Value.fromInt32(3) + Value.fromInt32(1), + Value.fromInt32(2), + Value.fromInt32(3) ); Value arrayValue = Value.fromArray(elements); - + assertThat(arrayValue.getTypeCode()).isEqualTo(TypeCode.ARRAY); assertThat(((Value.ArrayValue) arrayValue).getValue()).isEqualTo(elements); } - + @Test void shouldCreateMapValues() { var map = new HashMap(); map.put(MapKey.fromString("key1"), Value.fromInt32(1)); map.put(MapKey.fromString("key2"), Value.fromInt32(2)); - + Value mapValue = Value.fromMap(map); - + assertThat(mapValue.getTypeCode()).isEqualTo(TypeCode.MAP); assertThat(((Value.MapValue) mapValue).getValue()).isEqualTo(map); } - + @Test void shouldHandleEqualityCorrectly() { var int1 = Value.fromInt32(42); var int2 = Value.fromInt32(42); var int3 = Value.fromInt32(43); - + assertThat(int1).isEqualTo(int2); assertThat(int1).isNotEqualTo(int3); assertThat(int1.hashCode()).isEqualTo(int2.hashCode()); } - + @Test void shouldRejectNullString() { assertThatThrownBy(() -> Value.fromString(null)) - .isInstanceOf(NullPointerException.class); + .isInstanceOf(NullPointerException.class); } - + @Test void shouldCreateStringBufferValue() { String testString = "hello world"; byte[] utf8Bytes = testString.getBytes(StandardCharsets.UTF_8); ByteBuffer buffer = ByteBuffer.wrap(utf8Bytes); - + Value stringBufferValue = Value.fromStringBuffer(buffer); - + assertThat(stringBufferValue).isInstanceOf(Value.StringBufferValue.class); assertThat(stringBufferValue.getTypeCode()).isEqualTo(TypeCode.STRING); assertThat(((Value.StringBufferValue) stringBufferValue).getValue()).isEqualTo(testString); } - + @Test void shouldCreateBytesBufferValue() { byte[] testBytes = {1, 2, 3, 4, 5}; ByteBuffer buffer = ByteBuffer.wrap(testBytes); - + Value bytesBufferValue = Value.fromBytesBuffer(buffer); - + assertThat(bytesBufferValue).isInstanceOf(Value.BytesBufferValue.class); assertThat(bytesBufferValue.getTypeCode()).isEqualTo(TypeCode.BYTES); assertThat(((Value.BytesBufferValue) bytesBufferValue).getValue()).isEqualTo(testBytes); } - + @Test void shouldHandleStringBufferValueFastPath() { // Array-backed buffer with arrayOffset() == 0 should use fast path String testString = "fast path test"; byte[] utf8Bytes = testString.getBytes(StandardCharsets.UTF_8); ByteBuffer buffer = ByteBuffer.wrap(utf8Bytes); - + Value stringBufferValue = Value.fromStringBuffer(buffer); - + // Should work correctly regardless of path taken assertThat(((Value.StringBufferValue) stringBufferValue).getValue()).isEqualTo(testString); } - + @Test void shouldHandleStringBufferValueFallbackPath() { // Sliced buffer will have non-zero arrayOffset, forcing fallback path @@ -156,62 +156,62 @@ void shouldHandleStringBufferValueFallbackPath() { byte[] utf8Bytes = testString.getBytes(StandardCharsets.UTF_8); ByteBuffer buffer = ByteBuffer.wrap(utf8Bytes); ByteBuffer sliced = buffer.slice(); // This may break arrayOffset() == 0 - + Value stringBufferValue = Value.fromStringBuffer(sliced); - + // Should work correctly regardless of path taken assertThat(((Value.StringBufferValue) stringBufferValue).getValue()).isEqualTo(testString); } - + @Test void shouldHandleLargeStringWithoutCaching() { // Create string > 1KB to test the no-cache path String largeString = "x".repeat(2000); byte[] utf8Bytes = largeString.getBytes(StandardCharsets.UTF_8); ByteBuffer buffer = ByteBuffer.wrap(utf8Bytes).slice(); // Force fallback path - + Value stringBufferValue = Value.fromStringBuffer(buffer); - + assertThat(((Value.StringBufferValue) stringBufferValue).getValue()).isEqualTo(largeString); } - + @Test void shouldCacheStringDecoding() { String testString = "cache test"; byte[] utf8Bytes = testString.getBytes(StandardCharsets.UTF_8); ByteBuffer buffer = ByteBuffer.wrap(utf8Bytes); - + Value.StringBufferValue stringBufferValue = (Value.StringBufferValue) Value.fromStringBuffer(buffer); - + // First call should decode and cache String result1 = stringBufferValue.getValue(); // Second call should return cached value String result2 = stringBufferValue.getValue(); - + assertThat(result1).isEqualTo(testString); assertThat(result2).isEqualTo(testString); assertThat(result1).isSameAs(result2); // Should be same object reference due to caching } - + @Test void shouldHandleStringValueEquality() { String testString = "equality test"; - + Value stringValue = Value.fromString(testString); Value stringBufferValue = Value.fromStringBuffer(ByteBuffer.wrap(testString.getBytes(StandardCharsets.UTF_8))); - + assertThat(stringValue).isEqualTo(stringBufferValue); assertThat(stringBufferValue).isEqualTo(stringValue); assertThat(stringValue.hashCode()).isEqualTo(stringBufferValue.hashCode()); } - + @Test void shouldHandleBytesValueEquality() { byte[] testBytes = {1, 2, 3, 4, 5}; - + Value bytesValue = Value.fromBytes(testBytes); Value bytesBufferValue = Value.fromBytesBuffer(ByteBuffer.wrap(testBytes)); - + assertThat(bytesValue).isEqualTo(bytesBufferValue); assertThat(bytesBufferValue).isEqualTo(bytesValue); } From 9353388b22d44653b9b061f30de1add7dc284258 Mon Sep 17 00:00:00 2001 From: expand3d <> Date: Tue, 3 Jun 2025 18:57:15 -0400 Subject: [PATCH 18/49] minor style fixes again --- src/main/java/com/imprint/core/ImprintRecord.java | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/imprint/core/ImprintRecord.java b/src/main/java/com/imprint/core/ImprintRecord.java index dd66389..5d4719f 100644 --- a/src/main/java/com/imprint/core/ImprintRecord.java +++ b/src/main/java/com/imprint/core/ImprintRecord.java @@ -41,23 +41,16 @@ public ImprintRecord(Header header, List directory, ByteBuffer p /** * Get a value by field ID, deserializing it on demand. * Returns null if the field is not found. - * Note: If the field exists and is an explicit NULL type, this will return Value.NullValue.INSTANCE. + * Note: If the field exists and is an explicit NULL type, this will return Value.NullValue.INSTANCE */ public Value getValue(int fieldId) throws ImprintException { var fieldBuffer = getFieldBuffer(fieldId); if (fieldBuffer == null) { - // If getFieldBuffer returns null, it means the fieldId was not in the directory, - // or an issue occurred slicing the payload (e.g., bad offsets). return null; } - // findDirectoryIndex should not be negative here if fieldBuffer is not null, - // but a defensive check or ensuring findDirectoryIndex is called once is good. - // For simplicity, assume getFieldBuffer implies a valid index. int directoryIndex = findDirectoryIndex(fieldId); if (directoryIndex < 0) { - // This case should ideally be caught by getFieldBuffer returning null. - // If it happens, indicates an inconsistency. throw new ImprintException(ErrorType.INTERNAL_ERROR, "Field ID " + fieldId + " found buffer but not in directory."); } var entry = directory.get(directoryIndex); @@ -293,7 +286,7 @@ private Value deserializeValue(TypeCode typeCode, ByteBuffer buffer) throws Impr } private T getTypedValueOrThrow(int fieldId, TypeCode expectedTypeCode, Class expectedValueClass, String expectedTypeName) throws ImprintException { - Value value = getValue(fieldId); + var value = getValue(fieldId); if (value == null) { throw new ImprintException(ErrorType.FIELD_NOT_FOUND, From 09d0377922236cfc60799a75d8397a1cc75c67dc Mon Sep 17 00:00:00 2001 From: expand3d <> Date: Tue, 3 Jun 2025 19:00:32 -0400 Subject: [PATCH 19/49] minor style fixes on benchmark tests and supress unused --- src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java | 6 +----- src/jmh/java/com/imprint/benchmark/StringBenchmark.java | 1 + 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java b/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java index 152bb6d..8163522 100644 --- a/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java +++ b/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java @@ -103,8 +103,6 @@ public void deserializeKryo(Blackhole bh) { // ===== FIELD ACCESS BENCHMARKS ===== // Tests accessing a single field near the end of a large record - // This showcases Imprint's O(1) directory lookup vs sequential deserialization - @Benchmark public void singleFieldAccessImprint(Blackhole bh) throws Exception { ImprintRecord record = ImprintRecord.deserialize(imprintBytes.duplicate()); @@ -158,11 +156,9 @@ public void measureKryoSize(Blackhole bh) { @Benchmark public void mergeImprint(Blackhole bh) throws Exception { - // Simulate merge with Imprint (O(1) with proper API) var record1 = serializeWithImprint(testData); var record2 = serializeWithImprint(createTestRecord2()); - - // Current simulation - will be O(1) with actual merge API + var deserialized1 = ImprintRecord.deserialize(record1); var deserialized2 = ImprintRecord.deserialize(record2); var merged = simulateMerge(deserialized1, deserialized2); diff --git a/src/jmh/java/com/imprint/benchmark/StringBenchmark.java b/src/jmh/java/com/imprint/benchmark/StringBenchmark.java index e156c8c..045940e 100644 --- a/src/jmh/java/com/imprint/benchmark/StringBenchmark.java +++ b/src/jmh/java/com/imprint/benchmark/StringBenchmark.java @@ -16,6 +16,7 @@ @Fork(1) @Warmup(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS) @Measurement(iterations = 10, time = 1, timeUnit = TimeUnit.SECONDS) +@SuppressWarnings("unused") public class StringBenchmark { private static final SchemaId SCHEMA_ID = new SchemaId(1, 42); From 6209bb1217239a167b26660a690427629ef854d2 Mon Sep 17 00:00:00 2001 From: expand3d <> Date: Wed, 4 Jun 2025 01:09:26 -0400 Subject: [PATCH 20/49] minor reordering --- .../java/com/imprint/types/TypeHandler.java | 28 ++++++++++--------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/src/main/java/com/imprint/types/TypeHandler.java b/src/main/java/com/imprint/types/TypeHandler.java index d21403b..be4fc7b 100644 --- a/src/main/java/com/imprint/types/TypeHandler.java +++ b/src/main/java/com/imprint/types/TypeHandler.java @@ -19,6 +19,13 @@ public interface TypeHandler { int estimateSize(Value value) throws ImprintException; ByteBuffer readValueBytes(ByteBuffer buffer) throws ImprintException; + + + @FunctionalInterface + interface BufferViewer { + int measureDataLength(ByteBuffer tempBuffer, int numElements) throws ImprintException; + } + // Helper method to eliminate duplication in ARRAY/MAP readValueBytes static ByteBuffer readComplexValueBytes(ByteBuffer buffer, String typeName, BufferViewer measurer) throws ImprintException { int initialPosition = buffer.position(); @@ -31,9 +38,9 @@ static ByteBuffer readComplexValueBytes(ByteBuffer buffer, String typeName, Buff if (numElements == 0) { if (buffer.remaining() < varIntLength) { - throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, - "Not enough bytes for empty " + typeName + " VarInt. Needed: " + - varIntLength + ", available: " + buffer.remaining()); + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, + "Not enough bytes for empty " + typeName + " VarInt. Needed: " + + varIntLength + ", available: " + buffer.remaining()); } ByteBuffer valueSlice = buffer.slice(); valueSlice.limit(varIntLength); @@ -43,24 +50,19 @@ static ByteBuffer readComplexValueBytes(ByteBuffer buffer, String typeName, Buff int dataLength = measurer.measureDataLength(tempBuffer, numElements); int totalLength = varIntLength + dataLength; - + if (buffer.remaining() < totalLength) { - throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, - "Not enough bytes for " + typeName + " value. Needed: " + totalLength + - ", available: " + buffer.remaining() + " at position " + initialPosition); + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, + "Not enough bytes for " + typeName + " value. Needed: " + totalLength + + ", available: " + buffer.remaining() + " at position " + initialPosition); } - + ByteBuffer valueSlice = buffer.slice(); valueSlice.limit(totalLength); buffer.position(initialPosition + totalLength); return valueSlice.asReadOnlyBuffer(); } - @FunctionalInterface - interface BufferViewer { - int measureDataLength(ByteBuffer tempBuffer, int numElements) throws ImprintException; - } - // Static implementations for each type TypeHandler NULL = new TypeHandler() { @Override From 4632e013e4b2bd209feb9466cce9d4d260fe5636 Mon Sep 17 00:00:00 2001 From: expand3d <> Date: Thu, 5 Jun 2025 10:19:37 -0400 Subject: [PATCH 21/49] Full comprehensive comparison tests with a lot of other libraries + some micro-optimizations added that were found along the way --- .github/workflows/ci.yml | 244 ++++++- build.gradle | 265 ++++++- src/jmh/flatbuffers/test_record.fbs | 15 + .../benchmark/ComparisonBenchmark.java | 661 +++++++++++++++--- src/jmh/proto/test_record.proto | 15 + .../java/com/imprint/core/ImprintRecord.java | 12 +- .../java/com/imprint/types/TypeHandler.java | 84 ++- src/main/java/com/imprint/types/Value.java | 36 +- .../{benchmark => profile}/ProfilerTest.java | 11 +- 9 files changed, 1169 insertions(+), 174 deletions(-) create mode 100644 src/jmh/flatbuffers/test_record.fbs create mode 100644 src/jmh/proto/test_record.proto rename src/test/java/com/imprint/{benchmark => profile}/ProfilerTest.java (97%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d4c8bde..62ac6f5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,30 +15,220 @@ jobs: java-version: [11, 17, 21] steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up JDK ${{ matrix.java-version }} - uses: actions/setup-java@v4 - with: - java-version: ${{ matrix.java-version }} - distribution: 'temurin' - - - name: Cache Gradle dependencies - uses: actions/cache@v4 - with: - path: | - ~/.gradle/caches - ~/.gradle/wrapper - key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} - restore-keys: | - ${{ runner.os }}-gradle- - - - name: Make gradlew executable - run: chmod +x ./gradlew - - - name: Run tests - run: ./gradlew test - - - name: Run build - run: ./gradlew build \ No newline at end of file + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up JDK ${{ matrix.java-version }} + uses: actions/setup-java@v4 + with: + java-version: ${{ matrix.java-version }} + distribution: 'temurin' + + - name: Cache Gradle dependencies + uses: actions/cache@v4 + with: + path: | + ~/.gradle/caches + ~/.gradle/wrapper + key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} + restore-keys: | + ${{ runner.os }}-gradle- + + - name: Make gradlew executable + run: chmod +x ./gradlew + + - name: Run tests + run: ./gradlew test + + - name: Run build + run: ./gradlew build + + benchmark: + runs-on: ubuntu-latest + needs: test + # Only run benchmarks on main branch pushes and PRs to main to avoid excessive CI time + if: github.ref == 'refs/heads/main' || github.base_ref == 'main' + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up JDK 11 + uses: actions/setup-java@v4 + with: + java-version: '11' + distribution: 'temurin' + + - name: Cache Gradle dependencies + uses: actions/cache@v4 + with: + path: | + ~/.gradle/caches + ~/.gradle/wrapper + key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} + restore-keys: | + ${{ runner.os }}-gradle- + + - name: Make gradlew executable + run: chmod +x ./gradlew + + - name: Create benchmark results directory + run: mkdir -p benchmark-results + + - name: Run serialization benchmarks + run: | + ./gradlew jmhRunSerializationBenchmarks + continue-on-error: true + + - name: Run deserialization benchmarks + run: | + ./gradlew jmhRunDeserializationBenchmarks + continue-on-error: true + + - name: Run field access benchmarks + run: | + ./gradlew jmhRunFieldAccessBenchmarks + continue-on-error: true + + - name: Run size comparison benchmarks + run: | + ./gradlew jmhRunSizeComparisonBenchmarks + continue-on-error: true + + - name: Upload benchmark results + uses: actions/upload-artifact@v4 + if: always() + with: + name: benchmark-results-${{ github.sha }} + path: benchmark-results/ + retention-days: 30 + + - name: Comment benchmark results on PR + if: github.event_name == 'pull_request' + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const path = require('path'); + + // Find the latest benchmark results file + const resultsDir = 'benchmark-results'; + let latestFile = null; + let latestTime = 0; + + if (fs.existsSync(resultsDir)) { + const files = fs.readdirSync(resultsDir); + for (const file of files) { + if (file.endsWith('.json')) { + const filePath = path.join(resultsDir, file); + const stats = fs.statSync(filePath); + if (stats.mtime.getTime() > latestTime) { + latestTime = stats.mtime.getTime(); + latestFile = filePath; + } + } + } + } + + if (latestFile) { + const results = JSON.parse(fs.readFileSync(latestFile, 'utf8')); + + // Group results by benchmark type + const serialization = results.filter(r => r.benchmark.includes('serialize')); + const deserialization = results.filter(r => r.benchmark.includes('deserialize')); + const fieldAccess = results.filter(r => r.benchmark.includes('singleFieldAccess')); + const sizes = results.filter(r => r.benchmark.includes('measure')); + + // Format results into a table + const formatResults = (benchmarks, title) => { + if (benchmarks.length === 0) return ''; + + let table = `\n### ${title}\n\n| Library | Score (ns/op) | Error | Unit |\n|---------|---------------|-------|------|\n`; + + benchmarks + .sort((a, b) => a.primaryMetric.score - b.primaryMetric.score) + .forEach(benchmark => { + const name = benchmark.benchmark.split('.').pop().replace(/serialize|deserialize|singleFieldAccess|measure/, '').replace(/Imprint|JacksonJson|Kryo|MessagePack|Avro|Protobuf|FlatBuffers/, (match) => match); + const score = benchmark.primaryMetric.score.toFixed(2); + const error = benchmark.primaryMetric.scoreError.toFixed(2); + const unit = benchmark.primaryMetric.scoreUnit; + table += `| ${name} | ${score} | ±${error} | ${unit} |\n`; + }); + + return table; + }; + + const comment = `##Benchmark Results + + Benchmark comparison between Imprint and other serialization libraries: + ${formatResults(serialization, 'Serialization Performance')} + ${formatResults(deserialization, 'Deserialization Performance')} + ${formatResults(fieldAccess, 'Single Field Access Performance')} + ${formatResults(sizes, 'Serialized Size Comparison')} + +
+ View detailed results + + Results generated from commit: \`${context.sha.substring(0, 7)}\` + + Lower scores are better for performance benchmarks. + +
`; + + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: comment + }); + } else { + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: '## Benchmark Results\n\nBenchmark execution completed but no results file was found. Check the [workflow logs](' + + `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}` + ') for details.' + }); + } + + # Optional: Run full benchmark suite on releases + benchmark-full: + runs-on: ubuntu-latest + if: startsWith(github.ref, 'refs/tags/') + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up JDK 11 + uses: actions/setup-java@v4 + with: + java-version: '11' + distribution: 'temurin' + + - name: Cache Gradle dependencies + uses: actions/cache@v4 + with: + path: | + ~/.gradle/caches + ~/.gradle/wrapper + key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} + restore-keys: | + ${{ runner.os }}-gradle- + + - name: Make gradlew executable + run: chmod +x ./gradlew + + - name: Create benchmark results directory + run: mkdir -p benchmark-results + + - name: Run full benchmark suite + run: | + ./gradlew jmh + + - name: Upload full benchmark results + uses: actions/upload-artifact@v4 + with: + name: full-benchmark-results-${{ github.ref_name }} + path: benchmark-results/ + retention-days: 90 \ No newline at end of file diff --git a/build.gradle b/build.gradle index 2606710..8406676 100644 --- a/build.gradle +++ b/build.gradle @@ -2,6 +2,8 @@ plugins { id 'java-library' id 'maven-publish' id 'me.champeau.jmh' version '0.7.2' + id 'com.google.protobuf' version '0.9.4' + id 'io.netifi.flatbuffers' version '1.0.7' } group = 'com.imprint' @@ -23,31 +25,141 @@ dependencies { // Lombok for reducing boilerplate compileOnly 'org.projectlombok:lombok:1.18.30' annotationProcessor 'org.projectlombok:lombok:1.18.30' - + // Test dependencies testImplementation 'org.junit.jupiter:junit-jupiter:5.10.0' testImplementation 'org.assertj:assertj-core:3.24.2' testImplementation 'org.mockito:mockito-core:5.5.0' - + // Lombok for tests testCompileOnly 'org.projectlombok:lombok:1.18.30' testAnnotationProcessor 'org.projectlombok:lombok:1.18.30' - + // Performance testing with JMH jmhImplementation 'org.openjdk.jmh:jmh-core:1.37' jmhAnnotationProcessor 'org.openjdk.jmh:jmh-generator-annprocess:1.37' - - // Competitor libraries for benchmarking + + // Suppress SLF4J warnings + jmhImplementation 'org.slf4j:slf4j-nop:1.7.36' + + // Competitor libraries for benchmarking (JMH only) jmhImplementation 'com.google.protobuf:protobuf-java:3.25.1' jmhImplementation 'org.apache.avro:avro:1.11.3' jmhImplementation 'com.fasterxml.jackson.core:jackson-databind:2.16.0' jmhImplementation 'com.google.flatbuffers:flatbuffers-java:23.5.26' jmhImplementation 'com.esotericsoftware:kryo:5.4.0' + jmhImplementation 'org.msgpack:msgpack-core:0.9.8' + jmhImplementation 'org.msgpack:jackson-dataformat-msgpack:0.9.8' +} + +protobuf { + protoc { + artifact = "com.google.protobuf:protoc:3.25.1" + } + generateProtoTasks { + // Only generate for JMH, not main + all().each { task -> + task.enabled = false + } + ofSourceSet('jmh').each { task -> + task.enabled = true + task.builtins { + java { + outputSubDir = 'java' + } + } + } + } +} + +// Download and setup FlatBuffers compiler for Linux (CI environment) +task downloadFlatc { + description = 'Download FlatBuffers compiler' + group = 'build setup' + + def flatcDir = file("${buildDir}/flatc") + def flatcExe = file("${flatcDir}/flatc") + def isWindows = System.getProperty('os.name').toLowerCase().contains('windows') + def flatcUrl = isWindows ? + "https://github.com/google/flatbuffers/releases/download/v23.5.26/Windows.flatc.binary.zip" : + "https://github.com/google/flatbuffers/releases/download/v23.5.26/Linux.flatc.binary.clang++-12.zip" + def flatcZip = file("${buildDir}/flatc.zip") + + outputs.file(flatcExe) + + doLast { + if (!flatcExe.exists()) { + println "Downloading FlatBuffers compiler for ${isWindows ? 'Windows' : 'Linux'}..." + flatcDir.mkdirs() + + // Download + new URL(flatcUrl).withInputStream { i -> + flatcZip.withOutputStream { it << i } + } + + // Extract + copy { + from zipTree(flatcZip) + into flatcDir + } + + // Make executable on Unix systems + if (!isWindows) { + exec { + commandLine 'chmod', '+x', flatcExe.absolutePath + } + } + + flatcZip.delete() + println "FlatBuffers compiler downloaded to: ${flatcExe}" + } + } +} + +// Generate FlatBuffers sources +task generateFlatBuffers(type: Exec) { + dependsOn downloadFlatc + description = 'Generate Java classes from FlatBuffers schema' + group = 'build' + + def isWindows = System.getProperty('os.name').toLowerCase().contains('windows') + def flatcExe = file("${buildDir}/flatc/${isWindows ? 'flatc.exe' : 'flatc'}") + def schemaFile = file('src/jmh/flatbuffers/test_record.fbs') + def outputDir = file('build/generated/source/flatbuffers/jmh/java') + + commandLine flatcExe.absolutePath, '--java', '-o', outputDir.absolutePath, schemaFile.absolutePath + + inputs.file(schemaFile) + outputs.dir(outputDir) + + doFirst { + outputDir.mkdirs() + } +} + +// Add generated FlatBuffers sources to JMH source set +sourceSets { + jmh { + java { + srcDir 'build/generated/source/flatbuffers/jmh/java' + } + proto { + srcDir 'src/jmh/proto' + } + } +} + +// Make JMH compilation depend on FlatBuffers generation +compileJmhJava.dependsOn generateFlatBuffers + +// Handle duplicate proto files +tasks.named('processJmhResources') { + duplicatesStrategy = DuplicatesStrategy.EXCLUDE } test { useJUnitPlatform() - + // Enable detailed test output testLogging { events "passed", "skipped", "failed" @@ -57,11 +169,146 @@ test { // JMH configuration jmh { fork = 1 - warmupIterations = 3 - iterations = 3 + warmupIterations = 2 // Reduced for faster CI + iterations = 3 // Reduced for faster CI resultFormat = 'JSON' includeTests = false resultsFile = file("${projectDir}/benchmark-results/jmh-results-${new Date().format('yyyy-MM-dd-HHmmss')}.json") + + // Java 11 specific JVM args + jvmArgs = [ + '-XX:+UseG1GC', + '-Xmx2g', + '-XX:+UnlockExperimentalVMOptions', + '-XX:+UseJVMCICompiler' + ] +} + +// Create individual benchmark tasks for CI pipeline +task jmhRunSerializationBenchmarks(type: JavaExec) { + dependsOn compileJmhJava + description = 'Run serialization benchmarks' + group = 'benchmarking' + + classpath = sourceSets.jmh.runtimeClasspath + mainClass = 'com.imprint.benchmark.ComparisonBenchmark' + args = ['runSerializationBenchmarks'] + + // Java 11 optimized JVM settings + jvmArgs = [ + '-XX:+UseG1GC', + '-Xmx2g', + '-XX:+UnlockExperimentalVMOptions' + ] + + doFirst { + file("${projectDir}/benchmark-results").mkdirs() + } +} + +task jmhRunDeserializationBenchmarks(type: JavaExec) { + dependsOn compileJmhJava + description = 'Run deserialization benchmarks' + group = 'benchmarking' + + classpath = sourceSets.jmh.runtimeClasspath + mainClass = 'com.imprint.benchmark.ComparisonBenchmark' + args = ['runDeserializationBenchmarks'] + + // Java 11 optimized JVM settings + jvmArgs = [ + '-XX:+UseG1GC', + '-Xmx2g', + '-XX:+UnlockExperimentalVMOptions' + ] + + doFirst { + file("${projectDir}/benchmark-results").mkdirs() + } +} + +task jmhRunFieldAccessBenchmarks(type: JavaExec) { + dependsOn compileJmhJava + description = 'Run field access benchmarks' + group = 'benchmarking' + + classpath = sourceSets.jmh.runtimeClasspath + mainClass = 'com.imprint.benchmark.ComparisonBenchmark' + args = ['runFieldAccessBenchmarks'] + + // Java 11 optimized JVM settings + jvmArgs = [ + '-XX:+UseG1GC', + '-Xmx2g', + '-XX:+UnlockExperimentalVMOptions' + ] + + doFirst { + file("${projectDir}/benchmark-results").mkdirs() + } +} + +task jmhRunSizeComparisonBenchmarks(type: JavaExec) { + dependsOn compileJmhJava + description = 'Run size comparison benchmarks' + group = 'benchmarking' + + classpath = sourceSets.jmh.runtimeClasspath + mainClass = 'com.imprint.benchmark.ComparisonBenchmark' + args = ['runSizeComparisonBenchmarks'] + + // Java 11 optimized JVM settings + jvmArgs = [ + '-XX:+UseG1GC', + '-Xmx2g', + '-XX:+UnlockExperimentalVMOptions' + ] + + doFirst { + file("${projectDir}/benchmark-results").mkdirs() + } +} + +task jmhRunMergeBenchmarks(type: JavaExec) { + dependsOn compileJmhJava + description = 'Run merge operation benchmarks' + group = 'benchmarking' + + classpath = sourceSets.jmh.runtimeClasspath + mainClass = 'com.imprint.benchmark.ComparisonBenchmark' + args = ['runMergeBenchmarks'] + + // Java 11 optimized JVM settings + jvmArgs = [ + '-XX:+UseG1GC', + '-Xmx2g', + '-XX:+UnlockExperimentalVMOptions' + ] + + doFirst { + file("${projectDir}/benchmark-results").mkdirs() + } +} + +task jmhRunAllBenchmarks(type: JavaExec) { + dependsOn compileJmhJava + description = 'Run all comparison benchmarks' + group = 'benchmarking' + + classpath = sourceSets.jmh.runtimeClasspath + mainClass = 'com.imprint.benchmark.ComparisonBenchmark' + args = ['runAll'] + + // Java 11 optimized JVM settings + jvmArgs = [ + '-XX:+UseG1GC', + '-Xmx2g', + '-XX:+UnlockExperimentalVMOptions' + ] + + doFirst { + file("${projectDir}/benchmark-results").mkdirs() + } } compileJava { @@ -75,4 +322,4 @@ javadoc { } // Don't fail build on missing javadoc options.addStringOption('Xdoclint:none', '-quiet') -} +} \ No newline at end of file diff --git a/src/jmh/flatbuffers/test_record.fbs b/src/jmh/flatbuffers/test_record.fbs new file mode 100644 index 0000000..ccc31d0 --- /dev/null +++ b/src/jmh/flatbuffers/test_record.fbs @@ -0,0 +1,15 @@ +namespace com.imprint.benchmark; + +table TestRecordFB { + id: int; + name: string; + price: double; + active: bool; + category: string; + tags: [string]; + metadata_keys: [string]; + metadata_values: [string]; + extra_data: [string]; +} + +root_type TestRecordFB; \ No newline at end of file diff --git a/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java b/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java index 8163522..49260b1 100644 --- a/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java +++ b/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java @@ -4,13 +4,25 @@ import com.esotericsoftware.kryo.io.Input; import com.esotericsoftware.kryo.io.Output; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.flatbuffers.FlatBufferBuilder; import com.imprint.core.ImprintRecord; import com.imprint.core.ImprintWriter; import com.imprint.core.SchemaId; import com.imprint.types.MapKey; import com.imprint.types.Value; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericDatumReader; +import org.apache.avro.generic.GenericDatumWriter; +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.io.*; +import org.msgpack.jackson.dataformat.MessagePackFactory; import org.openjdk.jmh.annotations.*; import org.openjdk.jmh.infra.Blackhole; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; @@ -25,38 +37,56 @@ @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.NANOSECONDS) @State(Scope.Benchmark) -@Warmup(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS) -@Measurement(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS) +@Warmup(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 10, time = 1, timeUnit = TimeUnit.SECONDS) @Fork(1) +@SuppressWarnings("unused") public class ComparisonBenchmark { // Test data private TestRecord testData; - + // Serialized formats - private ByteBuffer imprintBytes; - private byte[] jacksonBytes; + private ByteBuffer imprintBytesBuffer; + private byte[] jacksonJsonBytes; private byte[] kryoBytes; - + private byte[] messagePackBytes; + private byte[] avroBytes; + private byte[] protobufBytes; + private ByteBuffer flatbuffersBytes; + // Library instances - private ObjectMapper jackson; + private Schema avroSchema; + private DatumWriter avroWriter; + private DatumReader avroReader; + private ObjectMapper jacksonJsonMapper; private Kryo kryo; + private ObjectMapper messagePackMapper; @Setup public void setup() throws Exception { testData = createTestRecord(); - + // Initialize libraries - jackson = new ObjectMapper(); + jacksonJsonMapper = new ObjectMapper(); kryo = new Kryo(); kryo.register(TestRecord.class); kryo.register(ArrayList.class); kryo.register(HashMap.class); - + kryo.register(Arrays.asList().getClass()); + + // Initialize MessagePack ObjectMapper + messagePackMapper = new ObjectMapper(new MessagePackFactory()); + setupAvro(); + // Pre-serialize for deserialization benchmarks - imprintBytes = serializeWithImprint(testData); - jacksonBytes = serializeWithJackson(testData); + imprintBytesBuffer = serializeWithImprint(testData); + jacksonJsonBytes = serializeWithJacksonJson(testData); kryoBytes = serializeWithKryo(testData); + messagePackBytes = serializeWithMessagePack(testData); + avroBytes = serializeWithAvro(testData); + protobufBytes = serializeWithProtobuf(testData); + flatbuffersBytes = serializeWithFlatBuffers(testData); } // ===== SERIALIZATION BENCHMARKS ===== @@ -68,8 +98,8 @@ public void serializeImprint(Blackhole bh) throws Exception { } @Benchmark - public void serializeJackson(Blackhole bh) throws Exception { - byte[] result = serializeWithJackson(testData); + public void serializeJacksonJson(Blackhole bh) throws Exception { + byte[] result = serializeWithJacksonJson(testData); bh.consume(result); } @@ -79,17 +109,41 @@ public void serializeKryo(Blackhole bh) { bh.consume(result); } + @Benchmark + public void serializeMessagePack(Blackhole bh) throws Exception { + byte[] result = serializeWithMessagePack(testData); + bh.consume(result); + } + + @Benchmark + public void serializeAvro(Blackhole bh) throws Exception { + byte[] result = serializeWithAvro(testData); + bh.consume(result); + } + + @Benchmark + public void serializeProtobuf(Blackhole bh) { + byte[] result = serializeWithProtobuf(testData); + bh.consume(result); + } + + @Benchmark + public void serializeFlatBuffers(Blackhole bh) { + ByteBuffer result = serializeWithFlatBuffers(testData); + bh.consume(result); + } + // ===== DESERIALIZATION BENCHMARKS ===== @Benchmark public void deserializeImprint(Blackhole bh) throws Exception { - ImprintRecord result = ImprintRecord.deserialize(imprintBytes.duplicate()); + ImprintRecord result = ImprintRecord.deserialize(imprintBytesBuffer.duplicate()); bh.consume(result); } @Benchmark - public void deserializeJackson(Blackhole bh) throws Exception { - TestRecord result = jackson.readValue(jacksonBytes, TestRecord.class); + public void deserializeJacksonJson(Blackhole bh) throws Exception { + TestRecord result = jacksonJsonMapper.readValue(jacksonJsonBytes, TestRecord.class); bh.consume(result); } @@ -101,135 +155,361 @@ public void deserializeKryo(Blackhole bh) { bh.consume(result); } + @Benchmark + public void deserializeMessagePack(Blackhole bh) throws Exception { + TestRecord result = messagePackMapper.readValue(messagePackBytes, TestRecord.class); + bh.consume(result); + } + + @Benchmark + public void deserializeAvro(Blackhole bh) throws Exception { + GenericRecord result = deserializeWithAvro(avroBytes); + bh.consume(result); + } + + @Benchmark + public void deserializeProtobuf(Blackhole bh) throws Exception { + TestRecordProto.TestRecord result = TestRecordProto.TestRecord.parseFrom(protobufBytes); + bh.consume(result); + } + + @Benchmark + public void deserializeFlatBuffers(Blackhole bh) { + TestRecordFB result = TestRecordFB.getRootAsTestRecordFB(flatbuffersBytes.duplicate()); + bh.consume(result); + } + // ===== FIELD ACCESS BENCHMARKS ===== // Tests accessing a single field near the end of a large record + // This showcases Imprint's O(1) directory lookup vs sequential deserialization + @Benchmark public void singleFieldAccessImprint(Blackhole bh) throws Exception { - ImprintRecord record = ImprintRecord.deserialize(imprintBytes.duplicate()); - - // Access field 15 directly via directory lookup - O(1) - var field15 = record.getValue(15); + ImprintRecord record = ImprintRecord.deserialize(imprintBytesBuffer.duplicate()); + var field15 = record.getString(15); bh.consume(field15); } @Benchmark - public void singleFieldAccessJackson(Blackhole bh) throws Exception { - // Jackson must deserialize entire object to access any field - TestRecord record = jackson.readValue(jacksonBytes, TestRecord.class); - - // Access field15 equivalent (extraData[4]) after full deserialization + public void singleFieldAccessJacksonJson(Blackhole bh) throws Exception { + TestRecord record = jacksonJsonMapper.readValue(jacksonJsonBytes, TestRecord.class); bh.consume(record.extraData.get(4)); } @Benchmark public void singleFieldAccessKryo(Blackhole bh) { - // Kryo must deserialize entire object to access any field Input input = new Input(new ByteArrayInputStream(kryoBytes)); TestRecord record = kryo.readObject(input, TestRecord.class); input.close(); - - // Access field15 equivalent (extraData[4]) after full deserialization bh.consume(record.extraData.get(4)); } + @Benchmark + public void singleFieldAccessMessagePack(Blackhole bh) throws Exception { + TestRecord record = messagePackMapper.readValue(messagePackBytes, TestRecord.class); + bh.consume(record.extraData.get(4)); + } + + @Benchmark + public void singleFieldAccessAvro(Blackhole bh) throws Exception { + GenericRecord record = deserializeWithAvro(avroBytes); + bh.consume(record.get("extraData4")); // Accessing field near end + } + + @Benchmark + public void singleFieldAccessProtobuf(Blackhole bh) throws Exception { + TestRecordProto.TestRecord record = TestRecordProto.TestRecord.parseFrom(protobufBytes); + bh.consume(record.getExtraData(4)); // Accessing field near end + } + + @Benchmark + public void singleFieldAccessFlatBuffers(Blackhole bh) { + TestRecordFB record = TestRecordFB.getRootAsTestRecordFB(flatbuffersBytes.duplicate()); + bh.consume(record.extraData(4)); // Accessing field near end - zero copy! + } + // ===== SIZE COMPARISON ===== @Benchmark - public void measureImprintSize(Blackhole bh) throws Exception { - ByteBuffer serialized = serializeWithImprint(testData); - bh.consume(serialized.remaining()); + public void measureImprintSize(Blackhole bh) { + bh.consume(imprintBytesBuffer.remaining()); } @Benchmark - public void measureJacksonSize(Blackhole bh) throws Exception { - byte[] serialized = serializeWithJackson(testData); - bh.consume(serialized.length); + public void measureJacksonJsonSize(Blackhole bh) { + bh.consume(jacksonJsonBytes.length); } @Benchmark public void measureKryoSize(Blackhole bh) { - byte[] serialized = serializeWithKryo(testData); - bh.consume(serialized.length); + bh.consume(kryoBytes.length); + } + + @Benchmark + public void measureMessagePackSize(Blackhole bh) { + bh.consume(messagePackBytes.length); + } + + @Benchmark + public void measureAvroSize(Blackhole bh) { + bh.consume(avroBytes.length); + } + + @Benchmark + public void measureProtobufSize(Blackhole bh) { + bh.consume(protobufBytes.length); + } + + @Benchmark + public void measureFlatBuffersSize(Blackhole bh) { + bh.consume(flatbuffersBytes.remaining()); } // ===== MERGE SIMULATION BENCHMARKS ===== @Benchmark public void mergeImprint(Blackhole bh) throws Exception { - var record1 = serializeWithImprint(testData); - var record2 = serializeWithImprint(createTestRecord2()); + var record1Buffer = imprintBytesBuffer.duplicate(); + var record2Data = createTestRecord2(); + var record2Buffer = serializeWithImprint(record2Data); - var deserialized1 = ImprintRecord.deserialize(record1); - var deserialized2 = ImprintRecord.deserialize(record2); + var deserialized1 = ImprintRecord.deserialize(record1Buffer); + var deserialized2 = ImprintRecord.deserialize(record2Buffer); var merged = simulateMerge(deserialized1, deserialized2); - + bh.consume(merged); } @Benchmark - public void mergeJackson(Blackhole bh) throws Exception { - // Jackson merge requires full deserialization + merge + serialization - var record1 = jackson.readValue(jacksonBytes, TestRecord.class); - var record2 = jackson.readValue(serializeWithJackson(createTestRecord2()), TestRecord.class); - - var merged = mergeTestRecords(record1, record2); - byte[] result = jackson.writeValueAsBytes(merged); - + public void mergeJacksonJson(Blackhole bh) throws Exception { + var record1 = jacksonJsonMapper.readValue(jacksonJsonBytes, TestRecord.class); + var record2Data = createTestRecord2(); + var record2Bytes = serializeWithJacksonJson(record2Data); + var record2 = jacksonJsonMapper.readValue(record2Bytes, TestRecord.class); + + var mergedPojo = mergeTestRecords(record1, record2); + byte[] result = jacksonJsonMapper.writeValueAsBytes(mergedPojo); bh.consume(result); } @Benchmark public void mergeKryo(Blackhole bh) { - // Kryo merge requires full deserialization + merge + serialization Input input1 = new Input(new ByteArrayInputStream(kryoBytes)); var record1 = kryo.readObject(input1, TestRecord.class); input1.close(); - - Input input2 = new Input(new ByteArrayInputStream(serializeWithKryo(createTestRecord2()))); + + var record2Data = createTestRecord2(); + var record2Bytes = serializeWithKryo(record2Data); + Input input2 = new Input(new ByteArrayInputStream(record2Bytes)); var record2 = kryo.readObject(input2, TestRecord.class); input2.close(); - - var merged = mergeTestRecords(record1, record2); - byte[] result = serializeWithKryo(merged); - + + var mergedPojo = mergeTestRecords(record1, record2); + byte[] result = serializeWithKryo(mergedPojo); + bh.consume(result); + } + + @Benchmark + public void mergeMessagePack(Blackhole bh) throws Exception { + var record1 = messagePackMapper.readValue(messagePackBytes, TestRecord.class); + var record2Data = createTestRecord2(); + var record2Bytes = serializeWithMessagePack(record2Data); + var record2 = messagePackMapper.readValue(record2Bytes, TestRecord.class); + + var mergedPojo = mergeTestRecords(record1, record2); + byte[] result = messagePackMapper.writeValueAsBytes(mergedPojo); bh.consume(result); } + @Benchmark + public void mergeAvro(Blackhole bh) throws Exception { + var record1 = deserializeWithAvro(avroBytes); + var record2Data = createTestRecord2(); + var record2Bytes = serializeWithAvro(record2Data); + var record2 = deserializeWithAvro(record2Bytes); + + var merged = mergeAvroRecords(record1, record2); + byte[] result = serializeAvroRecord(merged); + bh.consume(result); + } + + @Benchmark + public void mergeProtobuf(Blackhole bh) throws Exception { + var record1 = TestRecordProto.TestRecord.parseFrom(protobufBytes); + var record2Data = createTestRecord2(); + var record2Bytes = serializeWithProtobuf(record2Data); + var record2 = TestRecordProto.TestRecord.parseFrom(record2Bytes); + + var merged = mergeProtobufRecords(record1, record2); + byte[] result = merged.toByteArray(); + bh.consume(result); + } + + @Benchmark + public void mergeFlatBuffers(Blackhole bh) { + var record1 = TestRecordFB.getRootAsTestRecordFB(flatbuffersBytes.duplicate()); + var record2Data = createTestRecord2(); + var record2Buffer = serializeWithFlatBuffers(record2Data); + var record2 = TestRecordFB.getRootAsTestRecordFB(record2Buffer); + + var merged = mergeFlatBuffersRecords(record1, record2); + bh.consume(merged); + } + + // ===== MAIN METHOD TO RUN BENCHMARKS ===== + + public static void main(String[] args) throws RunnerException { + runAll(); + // Or, uncomment specific runner methods to execute subsets: + // runSerializationBenchmarks(); + // runDeserializationBenchmarks(); + // runFieldAccessBenchmarks(); + // runSizeComparisonBenchmarks(); + // runMergeBenchmarks(); + // runMessagePackBenchmarks(); + } + + public static void runAll() throws RunnerException { + Options opt = new OptionsBuilder() + .include(ComparisonBenchmark.class.getSimpleName()) + .build(); + new Runner(opt).run(); + } + + public static void runSerializationBenchmarks() throws RunnerException { + Options opt = new OptionsBuilder() + .include(ComparisonBenchmark.class.getSimpleName() + ".serialize.*") + .build(); + new Runner(opt).run(); + } + + public static void runDeserializationBenchmarks() throws RunnerException { + Options opt = new OptionsBuilder() + .include(ComparisonBenchmark.class.getSimpleName() + ".deserialize.*") + .build(); + new Runner(opt).run(); + } + + public static void runFieldAccessBenchmarks() throws RunnerException { + Options opt = new OptionsBuilder() + .include(ComparisonBenchmark.class.getSimpleName() + ".singleFieldAccess.*") + .build(); + new Runner(opt).run(); + } + + public static void runSizeComparisonBenchmarks() throws RunnerException { + Options opt = new OptionsBuilder() + .include(ComparisonBenchmark.class.getSimpleName() + ".measure.*") + .build(); + new Runner(opt).run(); + } + + public static void runMergeBenchmarks() throws RunnerException { + Options opt = new OptionsBuilder() + .include(ComparisonBenchmark.class.getSimpleName() + ".merge.*") + .build(); + new Runner(opt).run(); + } + + public static void runMessagePackBenchmarks() throws RunnerException { + Options opt = new OptionsBuilder() + .include(ComparisonBenchmark.class.getSimpleName() + ".*MessagePack.*") + .build(); + new Runner(opt).run(); + } + + public static void runAvroBenchmarks() throws RunnerException { + Options opt = new OptionsBuilder() + .include(ComparisonBenchmark.class.getSimpleName() + ".*Avro.*") + .build(); + new Runner(opt).run(); + } + + public static void runProtobufBenchmarks() throws RunnerException { + Options opt = new OptionsBuilder() + .include(ComparisonBenchmark.class.getSimpleName() + ".*Protobuf.*") + .build(); + new Runner(opt).run(); + } + + public static void runFlatBuffersBenchmarks() throws RunnerException { + Options opt = new OptionsBuilder() + .include(ComparisonBenchmark.class.getSimpleName() + ".*FlatBuffers.*") + .build(); + new Runner(opt).run(); + } + // ===== HELPER METHODS ===== + private void setupAvro() { + String schemaJson = "{\n" + + " \"type\": \"record\",\n" + + " \"name\": \"TestRecord\",\n" + + " \"fields\": [\n" + + " {\"name\": \"id\", \"type\": \"int\"},\n" + + " {\"name\": \"name\", \"type\": \"string\"},\n" + + " {\"name\": \"price\", \"type\": \"double\"},\n" + + " {\"name\": \"active\", \"type\": \"boolean\"},\n" + + " {\"name\": \"category\", \"type\": \"string\"},\n" + + " {\"name\": \"tags\", \"type\": {\"type\": \"array\", \"items\": \"string\"}},\n" + + " {\"name\": \"metadata\", \"type\": {\"type\": \"map\", \"values\": \"string\"}},\n" + + " {\"name\": \"extraData0\", \"type\": \"string\"},\n" + + " {\"name\": \"extraData1\", \"type\": \"string\"},\n" + + " {\"name\": \"extraData2\", \"type\": \"string\"},\n" + + " {\"name\": \"extraData3\", \"type\": \"string\"},\n" + + " {\"name\": \"extraData4\", \"type\": \"string\"},\n" + + " {\"name\": \"extraData5\", \"type\": \"string\"},\n" + + " {\"name\": \"extraData6\", \"type\": \"string\"},\n" + + " {\"name\": \"extraData7\", \"type\": \"string\"},\n" + + " {\"name\": \"extraData8\", \"type\": \"string\"},\n" + + " {\"name\": \"extraData9\", \"type\": \"string\"},\n" + + " {\"name\": \"extraData10\", \"type\": \"string\"},\n" + + " {\"name\": \"extraData11\", \"type\": \"string\"},\n" + + " {\"name\": \"extraData12\", \"type\": \"string\"}\n" + + " ]\n" + + "}"; + + avroSchema = new Schema.Parser().parse(schemaJson); + avroWriter = new GenericDatumWriter<>(avroSchema); + avroReader = new GenericDatumReader<>(avroSchema); + } + private ByteBuffer serializeWithImprint(TestRecord data) throws Exception { var writer = new ImprintWriter(new SchemaId(1, 0x12345678)); - + writer.addField(1, Value.fromInt32(data.id)); writer.addField(2, Value.fromString(data.name)); writer.addField(3, Value.fromFloat64(data.price)); writer.addField(4, Value.fromBoolean(data.active)); writer.addField(5, Value.fromString(data.category)); - - // Convert tags list + var tagValues = new ArrayList(); - for (String tag : data.tags) { - tagValues.add(Value.fromString(tag)); + if (data.tags != null) { + for (String tag : data.tags) { + tagValues.add(Value.fromString(tag)); + } } writer.addField(6, Value.fromArray(tagValues)); - - // Convert metadata map + var metadataMap = new HashMap(); - for (var entry : data.metadata.entrySet()) { - metadataMap.put(MapKey.fromString(entry.getKey()), Value.fromString(entry.getValue())); + if (data.metadata != null) { + for (var entry : data.metadata.entrySet()) { + metadataMap.put(MapKey.fromString(entry.getKey()), Value.fromString(entry.getValue())); + } } writer.addField(7, Value.fromMap(metadataMap)); - - // Add extra fields (8-20) to create a larger record - for (int i = 0; i < data.extraData.size(); i++) { - writer.addField(8 + i, Value.fromString(data.extraData.get(i))); + + if (data.extraData != null) { + for (int i = 0; i < data.extraData.size(); i++) { + writer.addField(8 + i, Value.fromString(data.extraData.get(i))); + } } - + return writer.build().serializeToBuffer(); } - private byte[] serializeWithJackson(TestRecord data) throws Exception { - return jackson.writeValueAsBytes(data); + private byte[] serializeWithJacksonJson(TestRecord data) throws Exception { + return jacksonJsonMapper.writeValueAsBytes(data); } private byte[] serializeWithKryo(TestRecord data) { @@ -240,14 +520,117 @@ private byte[] serializeWithKryo(TestRecord data) { return baos.toByteArray(); } + private byte[] serializeWithMessagePack(TestRecord data) throws Exception { + return messagePackMapper.writeValueAsBytes(data); + } + + private byte[] serializeWithAvro(TestRecord data) throws Exception { + GenericRecord record = new GenericData.Record(avroSchema); + record.put("id", data.id); + record.put("name", data.name); + record.put("price", data.price); + record.put("active", data.active); + record.put("category", data.category); + record.put("tags", data.tags); + record.put("metadata", data.metadata); + + for (int i = 0; i < data.extraData.size(); i++) { + record.put("extraData" + i, data.extraData.get(i)); + } + + return serializeAvroRecord(record); + } + + private byte[] serializeAvroRecord(GenericRecord record) throws Exception { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + Encoder encoder = EncoderFactory.get().binaryEncoder(baos, null); + avroWriter.write(record, encoder); + encoder.flush(); + return baos.toByteArray(); + } + + private GenericRecord deserializeWithAvro(byte[] data) throws Exception { + Decoder decoder = DecoderFactory.get().binaryDecoder(data, null); + return avroReader.read(null, decoder); + } + + private byte[] serializeWithProtobuf(TestRecord data) { + var builder = TestRecordProto.TestRecord.newBuilder() + .setId(data.id) + .setName(data.name) + .setPrice(data.price) + .setActive(data.active) + .setCategory(data.category) + .addAllTags(data.tags) + .putAllMetadata(data.metadata); + + for (String extraData : data.extraData) { + builder.addExtraData(extraData); + } + + return builder.build().toByteArray(); + } + + private ByteBuffer serializeWithFlatBuffers(TestRecord data) { + FlatBufferBuilder builder = new FlatBufferBuilder(1024); + + // Create strings (must be created before the object that uses them) + int nameOffset = builder.createString(data.name); + int categoryOffset = builder.createString(data.category); + + // Create tags array + int[] tagOffsets = new int[data.tags.size()]; + for (int i = 0; i < data.tags.size(); i++) { + tagOffsets[i] = builder.createString(data.tags.get(i)); + } + int tagsOffset = TestRecordFB.createTagsVector(builder, tagOffsets); + + // Create metadata (as parallel arrays for keys and values) + String[] metadataKeys = data.metadata.keySet().toArray(new String[0]); + String[] metadataValues = new String[metadataKeys.length]; + int[] keyOffsets = new int[metadataKeys.length]; + int[] valueOffsets = new int[metadataKeys.length]; + + for (int i = 0; i < metadataKeys.length; i++) { + metadataValues[i] = data.metadata.get(metadataKeys[i]); + keyOffsets[i] = builder.createString(metadataKeys[i]); + valueOffsets[i] = builder.createString(metadataValues[i]); + } + int metadataKeysOffset = TestRecordFB.createMetadataKeysVector(builder, keyOffsets); + int metadataValuesOffset = TestRecordFB.createMetadataValuesVector(builder, valueOffsets); + + // Create extra data array + int[] extraDataOffsets = new int[data.extraData.size()]; + for (int i = 0; i < data.extraData.size(); i++) { + extraDataOffsets[i] = builder.createString(data.extraData.get(i)); + } + int extraDataOffset = TestRecordFB.createExtraDataVector(builder, extraDataOffsets); + + // Create the main object + TestRecordFB.startTestRecordFB(builder); + TestRecordFB.addId(builder, data.id); + TestRecordFB.addName(builder, nameOffset); + TestRecordFB.addPrice(builder, data.price); + TestRecordFB.addActive(builder, data.active); + TestRecordFB.addCategory(builder, categoryOffset); + TestRecordFB.addTags(builder, tagsOffset); + TestRecordFB.addMetadataKeys(builder, metadataKeysOffset); + TestRecordFB.addMetadataValues(builder, metadataValuesOffset); + TestRecordFB.addExtraData(builder, extraDataOffset); + int recordOffset = TestRecordFB.endTestRecordFB(builder); + + // Finish and return + builder.finish(recordOffset); + return builder.dataBuffer().slice(); + } + private ImprintRecord simulateMerge(ImprintRecord first, ImprintRecord second) throws Exception { var writer = new ImprintWriter(first.getHeader().getSchemaId()); var usedFieldIds = new HashSet(); - - // Copy fields from first record (takes precedence) + copyFieldsToWriter(first, writer, usedFieldIds); copyFieldsToWriter(second, writer, usedFieldIds); - + return writer.build(); } @@ -265,23 +648,121 @@ private void copyFieldsToWriter(ImprintRecord record, ImprintWriter writer, Set< } private TestRecord mergeTestRecords(TestRecord first, TestRecord second) { - // Simple merge logic - first record takes precedence var merged = new TestRecord(); merged.id = first.id; merged.name = first.name != null ? first.name : second.name; merged.price = first.price != 0.0 ? first.price : second.price; merged.active = first.active; merged.category = first.category != null ? first.category : second.category; - + merged.tags = new ArrayList<>(first.tags); merged.tags.addAll(second.tags); - + merged.metadata = new HashMap<>(first.metadata); merged.metadata.putAll(second.metadata); - + return merged; } + private GenericRecord mergeAvroRecords(GenericRecord first, GenericRecord second) { + GenericRecord merged = new GenericData.Record(avroSchema); + + // Copy all fields from first record + for (Schema.Field field : avroSchema.getFields()) { + merged.put(field.name(), first.get(field.name())); + } + + // Override with non-null values from second record + for (Schema.Field field : avroSchema.getFields()) { + Object secondValue = second.get(field.name()); + if (secondValue != null && !secondValue.toString().isEmpty()) { + merged.put(field.name(), secondValue); + } + } + + return merged; + } + + private TestRecordProto.TestRecord mergeProtobufRecords(TestRecordProto.TestRecord first, TestRecordProto.TestRecord second) { + return TestRecordProto.TestRecord.newBuilder() + .mergeFrom(first) + .mergeFrom(second) + .build(); + } + + private ByteBuffer mergeFlatBuffersRecords(TestRecordFB first, TestRecordFB second) { + FlatBufferBuilder builder = new FlatBufferBuilder(1024); + + // Use second record's values if they exist, otherwise first record's values + String name = second.name() != null && !second.name().isEmpty() ? second.name() : first.name(); + String category = second.category() != null && !second.category().isEmpty() ? second.category() : first.category(); + double price = second.price() != 0.0 ? second.price() : first.price(); + boolean active = second.active(); // Use second's boolean value + int id = first.id(); // Keep first record's ID + + // Create merged strings + int nameOffset = builder.createString(name); + int categoryOffset = builder.createString(category); + + // Merge tags (combine both arrays) + List mergedTags = new ArrayList<>(); + for (int i = 0; i < first.tagsLength(); i++) { + mergedTags.add(first.tags(i)); + } + for (int i = 0; i < second.tagsLength(); i++) { + mergedTags.add(second.tags(i)); + } + + int[] tagOffsets = new int[mergedTags.size()]; + for (int i = 0; i < mergedTags.size(); i++) { + tagOffsets[i] = builder.createString(mergedTags.get(i)); + } + int tagsOffset = TestRecordFB.createTagsVector(builder, tagOffsets); + + // Merge metadata (second overwrites first) + Map mergedMetadata = new HashMap<>(); + for (int i = 0; i < first.metadataKeysLength(); i++) { + mergedMetadata.put(first.metadataKeys(i), first.metadataValues(i)); + } + for (int i = 0; i < second.metadataKeysLength(); i++) { + mergedMetadata.put(second.metadataKeys(i), second.metadataValues(i)); + } + + String[] metadataKeys = mergedMetadata.keySet().toArray(new String[0]); + int[] keyOffsets = new int[metadataKeys.length]; + int[] valueOffsets = new int[metadataKeys.length]; + + for (int i = 0; i < metadataKeys.length; i++) { + keyOffsets[i] = builder.createString(metadataKeys[i]); + valueOffsets[i] = builder.createString(mergedMetadata.get(metadataKeys[i])); + } + int metadataKeysOffset = TestRecordFB.createMetadataKeysVector(builder, keyOffsets); + int metadataValuesOffset = TestRecordFB.createMetadataValuesVector(builder, valueOffsets); + + // Use first record's extra data (or could merge both) + int[] extraDataOffsets = new int[first.extraDataLength()]; + for (int i = 0; i < first.extraDataLength(); i++) { + extraDataOffsets[i] = builder.createString(first.extraData(i)); + } + int extraDataOffset = TestRecordFB.createExtraDataVector(builder, extraDataOffsets); + + // Create the merged object + TestRecordFB.startTestRecordFB(builder); + TestRecordFB.addId(builder, id); + TestRecordFB.addName(builder, nameOffset); + TestRecordFB.addPrice(builder, price); + TestRecordFB.addActive(builder, active); + TestRecordFB.addCategory(builder, categoryOffset); + TestRecordFB.addTags(builder, tagsOffset); + TestRecordFB.addMetadataKeys(builder, metadataKeysOffset); + TestRecordFB.addMetadataValues(builder, metadataValuesOffset); + TestRecordFB.addExtraData(builder, extraDataOffset); + int recordOffset = TestRecordFB.endTestRecordFB(builder); + + builder.finish(recordOffset); + return builder.dataBuffer().slice(); + } + private TestRecord createTestRecord() { var record = new TestRecord(); record.id = 12345; @@ -289,20 +770,19 @@ var record = new TestRecord(); record.price = 99.99; record.active = true; record.category = "Electronics"; - + record.tags = Arrays.asList("popular", "trending", "bestseller"); - + record.metadata = new HashMap<>(); record.metadata.put("manufacturer", "TechCorp"); record.metadata.put("model", "TC-2024"); record.metadata.put("warranty", "2 years"); - - // Add extra data to create a larger record (fields 8-20) + record.extraData = new ArrayList<>(); for (int i = 0; i < 13; i++) { record.extraData.add("extraField" + i + "_value_" + (1000 + i)); } - + return record; } @@ -313,19 +793,18 @@ var record = new TestRecord(); record.price = 149.99; record.active = false; record.category = "Software"; - + record.tags = Arrays.asList("new", "premium"); - + record.metadata = new HashMap<>(); record.metadata.put("vendor", "SoftCorp"); record.metadata.put("version", "2.1"); - - // Add extra data to match the structure + record.extraData = new ArrayList<>(); for (int i = 0; i < 13; i++) { record.extraData.add("extraField" + i + "_value2_" + (2000 + i)); } - + return record; } @@ -339,7 +818,7 @@ public static class TestRecord { public List tags = new ArrayList<>(); public Map metadata = new HashMap<>(); public List extraData = new ArrayList<>(); // Fields 8-20 for large record test - + public TestRecord() {} // Required for deserialization } } \ No newline at end of file diff --git a/src/jmh/proto/test_record.proto b/src/jmh/proto/test_record.proto new file mode 100644 index 0000000..7a76f71 --- /dev/null +++ b/src/jmh/proto/test_record.proto @@ -0,0 +1,15 @@ +syntax = "proto3"; + +option java_package = "com.imprint.benchmark"; +option java_outer_classname = "TestRecordProto"; + +message TestRecord { + int32 id = 1; + string name = 2; + double price = 3; + bool active = 4; + string category = 5; + repeated string tags = 6; + map metadata = 7; + repeated string extra_data = 8; +} \ No newline at end of file diff --git a/src/main/java/com/imprint/core/ImprintRecord.java b/src/main/java/com/imprint/core/ImprintRecord.java index 5d4719f..2291550 100644 --- a/src/main/java/com/imprint/core/ImprintRecord.java +++ b/src/main/java/com/imprint/core/ImprintRecord.java @@ -34,7 +34,7 @@ public final class ImprintRecord { */ public ImprintRecord(Header header, List directory, ByteBuffer payload) { this.header = Objects.requireNonNull(header, "Header cannot be null"); - this.directory = List.copyOf(Objects.requireNonNull(directory, "Directory cannot be null")); + this.directory = Collections.unmodifiableList(Objects.requireNonNull(directory, "Directory cannot be null")); this.payload = payload.asReadOnlyBuffer(); // Zero-copy read-only view } @@ -79,12 +79,14 @@ private ByteBuffer getFieldBuffer(int fieldId) { int endOffset = (index + 1 < directory.size()) ? directory.get(index + 1).getOffset() : payload.limit(); - var fieldBuffer = payload.duplicate(); if (startOffset > payload.limit() || endOffset > payload.limit() || startOffset > endOffset) { return null; } + + // OPTIMIZATION: Single allocation instead of duplicate + slice + var fieldBuffer = payload.duplicate(); fieldBuffer.position(startOffset).limit(endOffset); - return fieldBuffer.slice(); + return fieldBuffer; } /** @@ -261,7 +263,7 @@ private static DirectoryEntry deserializeDirectoryEntry(ByteBuffer buffer) throw } private Value deserializeValue(TypeCode typeCode, ByteBuffer buffer) throws ImprintException { - ByteBuffer valueSpecificBuffer = buffer.duplicate(); + var valueSpecificBuffer = buffer.duplicate(); valueSpecificBuffer.order(ByteOrder.LITTLE_ENDIAN); switch (typeCode) { @@ -351,7 +353,7 @@ public double getFloat64(int fieldId) throws ImprintException { * @throws ImprintException if the field is not found, is null, or is not of type STRING. */ public String getString(int fieldId) throws ImprintException { - Value value = getValue(fieldId); + var value = getValue(fieldId); if (value == null) { throw new ImprintException(ErrorType.FIELD_NOT_FOUND, diff --git a/src/main/java/com/imprint/types/TypeHandler.java b/src/main/java/com/imprint/types/TypeHandler.java index be4fc7b..e58c355 100644 --- a/src/main/java/com/imprint/types/TypeHandler.java +++ b/src/main/java/com/imprint/types/TypeHandler.java @@ -433,26 +433,38 @@ public int estimateSize(Value value) throws ImprintException { } return arraySize; } - + @Override public ByteBuffer readValueBytes(ByteBuffer buffer) throws ImprintException { return readComplexValueBytes(buffer, "ARRAY", (tempBuffer, numElements) -> { if (tempBuffer.remaining() < 1) { - throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, - "Not enough bytes for ARRAY element type code in temp buffer during measurement."); + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, + "Not enough bytes for ARRAY element type code"); } byte elementTypeCodeByte = tempBuffer.get(); - int typeCodeLength = 1; + var elementType = TypeCode.fromByte(elementTypeCodeByte); - TypeHandler elementHandler = TypeCode.fromByte(elementTypeCodeByte).getHandler(); - int elementsDataLength = 0; - for (int i = 0; i < numElements; i++) { - int elementStartPos = tempBuffer.position(); - elementHandler.readValueBytes(tempBuffer); - elementsDataLength += (tempBuffer.position() - elementStartPos); + switch (elementType) { + case NULL: + return 1; + case BOOL: + return 1 + numElements; + case INT32: + case FLOAT32: + return 1 + (numElements * 4); + case INT64: + case FLOAT64: + return 1 + (numElements * 8); + default: + var elementHandler = elementType.getHandler(); + int elementsDataLength = 0; + for (int i = 0; i < numElements; i++) { + int elementStartPos = tempBuffer.position(); + elementHandler.readValueBytes(tempBuffer); + elementsDataLength += (tempBuffer.position() - elementStartPos); + } + return 1 + elementsDataLength; } - - return typeCodeLength + elementsDataLength; }); } }; @@ -549,28 +561,52 @@ public int estimateSize(Value value) throws ImprintException { } return mapSize; } - + @Override public ByteBuffer readValueBytes(ByteBuffer buffer) throws ImprintException { return readComplexValueBytes(buffer, "MAP", (tempBuffer, numEntries) -> { if (tempBuffer.remaining() < 2) { - throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, - "Not enough bytes for MAP key/value type codes in temp buffer during measurement."); + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, + "Not enough bytes for MAP key/value type codes"); } byte keyTypeCodeByte = tempBuffer.get(); byte valueTypeCodeByte = tempBuffer.get(); - int typeCodesLength = 2; - int entriesDataLength = 0; - for (int i = 0; i < numEntries; i++) { - int entryStartPos = tempBuffer.position(); - TypeCode.fromByte(keyTypeCodeByte).getHandler().readValueBytes(tempBuffer); - TypeCode.fromByte(valueTypeCodeByte).getHandler().readValueBytes(tempBuffer); - entriesDataLength += (tempBuffer.position() - entryStartPos); - } + TypeCode keyType = TypeCode.fromByte(keyTypeCodeByte); + TypeCode valueType = TypeCode.fromByte(valueTypeCodeByte); + + // OPTIMIZATION: Calculate sizes directly for fixed-size types + int keySize = getFixedTypeSize(keyType); + int valueSize = getFixedTypeSize(valueType); - return typeCodesLength + entriesDataLength; + if (keySize > 0 && valueSize > 0) { + // Both are fixed-size: O(1) calculation + return 2 + (numEntries * (keySize + valueSize)); + } else { + // At least one is variable-size: fall back to traversal + int entriesDataLength = 0; + for (int i = 0; i < numEntries; i++) { + int entryStartPos = tempBuffer.position(); + keyType.getHandler().readValueBytes(tempBuffer); + valueType.getHandler().readValueBytes(tempBuffer); + entriesDataLength += (tempBuffer.position() - entryStartPos); + } + return 2 + entriesDataLength; + } }); } + + private int getFixedTypeSize(TypeCode type) { + switch (type) { + case NULL: return 0; + case BOOL: return 1; + case INT32: + case FLOAT32: return 4; + case INT64: + case FLOAT64: return 8; + default: return -1; // Variable size + } + } + private void serializeMapKey(MapKey key, ByteBuffer buffer) throws ImprintException { switch (key.getTypeCode()) { diff --git a/src/main/java/com/imprint/types/Value.java b/src/main/java/com/imprint/types/Value.java index 7f3bbb9..ba747de 100644 --- a/src/main/java/com/imprint/types/Value.java +++ b/src/main/java/com/imprint/types/Value.java @@ -328,16 +328,19 @@ public String toString() { // String Value (ByteBuffer-based) public static class StringBufferValue extends Value { private final ByteBuffer value; - private volatile String cachedString; // lazy decode + private volatile String cachedString; + + private static final int THREAD_LOCAL_BUFFER_SIZE = 1024; + private static final ThreadLocal DECODE_BUFFER_CACHE = + ThreadLocal.withInitial(() -> new byte[THREAD_LOCAL_BUFFER_SIZE]); public StringBufferValue(ByteBuffer value) { - this.value = value.asReadOnlyBuffer(); // zero-copy read-only view + this.value = value.asReadOnlyBuffer(); } public String getValue() { String result = cachedString; if (result == null) { - // Simple, fast decoding - no thread-local overhead result = decodeUtf8(); cachedString = result; } @@ -345,18 +348,25 @@ public String getValue() { } private String decodeUtf8() { - // Fast path: zero-copy for array-backed ByteBuffers + final byte[] array; + final int offset; + final int length = value.remaining(); + if (value.hasArray()) { - return new String(value.array(), value.arrayOffset() + value.position(), - value.remaining(), StandardCharsets.UTF_8); + array = value.array(); + offset = value.arrayOffset() + value.position(); + } else { + byte[] threadLocalBuffer = DECODE_BUFFER_CACHE.get(); + if (length <= threadLocalBuffer.length) { + array = threadLocalBuffer; + } else { + // Fallback: copy bytes from the ByteBuffer to a new heap array (if too large for cache) + array = new byte[length]; + } + value.duplicate().get(array, 0, length); // Get bytes from current position into chosen array + offset = 0; } - - // Fallback path for non-array-backed ByteBuffers (e.g., direct buffers). - // Allocation is required here as Java's String(byte[],...) constructor needs a heap array. - // Data is copied from the ByteBuffer to a new byte array. - var array = new byte[value.remaining()]; - value.duplicate().get(array); - return new String(array, StandardCharsets.UTF_8); + return new String(array, offset, length, StandardCharsets.UTF_8); } public ByteBuffer getBuffer() { diff --git a/src/test/java/com/imprint/benchmark/ProfilerTest.java b/src/test/java/com/imprint/profile/ProfilerTest.java similarity index 97% rename from src/test/java/com/imprint/benchmark/ProfilerTest.java rename to src/test/java/com/imprint/profile/ProfilerTest.java index 5b531a9..d48c1aa 100644 --- a/src/test/java/com/imprint/benchmark/ProfilerTest.java +++ b/src/test/java/com/imprint/profile/ProfilerTest.java @@ -1,9 +1,10 @@ -package com.imprint.benchmark; +package com.imprint.profile; -import com.imprint.core.*; +import com.imprint.core.ImprintRecord; +import com.imprint.core.ImprintWriter; +import com.imprint.core.SchemaId; import com.imprint.types.Value; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.Disabled; import java.util.Random; @@ -25,11 +26,11 @@ * - String operations and UTF-8 encoding * - ByteBuffer operations */ -@Disabled("Enable manually for profiling") +//@Disabled("Enable manually for profiling") public class ProfilerTest { private static final int ITERATIONS = 1_000_000; - private static final int RECORD_SIZE = 20; + private static final int RECORD_SIZE = 50; @Test void profileFieldAccess() throws Exception { From 3738861fb994922a5fd6ed1a8fd647b8f658e933 Mon Sep 17 00:00:00 2001 From: expand3d <> Date: Thu, 5 Jun 2025 10:22:08 -0400 Subject: [PATCH 22/49] replace deprecated gradle methods with latest --- build.gradle | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/build.gradle b/build.gradle index 8406676..66f2e3e 100644 --- a/build.gradle +++ b/build.gradle @@ -73,7 +73,7 @@ protobuf { } // Download and setup FlatBuffers compiler for Linux (CI environment) -task downloadFlatc { +tasks.register('downloadFlatc') { description = 'Download FlatBuffers compiler' group = 'build setup' @@ -117,7 +117,7 @@ task downloadFlatc { } // Generate FlatBuffers sources -task generateFlatBuffers(type: Exec) { +tasks.register('generateFlatBuffers', Exec) { dependsOn downloadFlatc description = 'Generate Java classes from FlatBuffers schema' group = 'build' @@ -166,11 +166,11 @@ test { } } -// JMH configuration +// JMH configuration - optimized for Java 11 jmh { fork = 1 warmupIterations = 2 // Reduced for faster CI - iterations = 3 // Reduced for faster CI + iterations = 3 resultFormat = 'JSON' includeTests = false resultsFile = file("${projectDir}/benchmark-results/jmh-results-${new Date().format('yyyy-MM-dd-HHmmss')}.json") @@ -180,12 +180,12 @@ jmh { '-XX:+UseG1GC', '-Xmx2g', '-XX:+UnlockExperimentalVMOptions', - '-XX:+UseJVMCICompiler' + '-XX:+UseJVMCICompiler' // Use Graal if available for better performance ] } // Create individual benchmark tasks for CI pipeline -task jmhRunSerializationBenchmarks(type: JavaExec) { +tasks.register('jmhRunSerializationBenchmarks', JavaExec) { dependsOn compileJmhJava description = 'Run serialization benchmarks' group = 'benchmarking' @@ -206,7 +206,7 @@ task jmhRunSerializationBenchmarks(type: JavaExec) { } } -task jmhRunDeserializationBenchmarks(type: JavaExec) { +tasks.register('jmhRunDeserializationBenchmarks', JavaExec) { dependsOn compileJmhJava description = 'Run deserialization benchmarks' group = 'benchmarking' @@ -227,7 +227,7 @@ task jmhRunDeserializationBenchmarks(type: JavaExec) { } } -task jmhRunFieldAccessBenchmarks(type: JavaExec) { +tasks.register('jmhRunFieldAccessBenchmarks', JavaExec) { dependsOn compileJmhJava description = 'Run field access benchmarks' group = 'benchmarking' @@ -248,7 +248,7 @@ task jmhRunFieldAccessBenchmarks(type: JavaExec) { } } -task jmhRunSizeComparisonBenchmarks(type: JavaExec) { +tasks.register('jmhRunSizeComparisonBenchmarks', JavaExec) { dependsOn compileJmhJava description = 'Run size comparison benchmarks' group = 'benchmarking' @@ -269,7 +269,7 @@ task jmhRunSizeComparisonBenchmarks(type: JavaExec) { } } -task jmhRunMergeBenchmarks(type: JavaExec) { +tasks.register('jmhRunMergeBenchmarks', JavaExec) { dependsOn compileJmhJava description = 'Run merge operation benchmarks' group = 'benchmarking' @@ -290,7 +290,7 @@ task jmhRunMergeBenchmarks(type: JavaExec) { } } -task jmhRunAllBenchmarks(type: JavaExec) { +tasks.register('jmhRunAllBenchmarks', JavaExec) { dependsOn compileJmhJava description = 'Run all comparison benchmarks' group = 'benchmarking' From 12d28233e9f73c0d23687d9977356e6d35ec0f88 Mon Sep 17 00:00:00 2001 From: Brent Johnson Date: Thu, 5 Jun 2025 14:57:56 -0400 Subject: [PATCH 23/49] Merge Comparisons into dev branch (#8) * Full comprehensive comparison tests with a lot of other libraries + some micro-optimizations added that were found along the way * replace deprecated gradle methods with latest --------- Co-authored-by: expand3d <> --- .github/workflows/ci.yml | 244 ++++++- build.gradle | 265 ++++++- src/jmh/flatbuffers/test_record.fbs | 15 + .../benchmark/ComparisonBenchmark.java | 661 +++++++++++++++--- src/jmh/proto/test_record.proto | 15 + .../java/com/imprint/core/ImprintRecord.java | 12 +- .../java/com/imprint/types/TypeHandler.java | 84 ++- src/main/java/com/imprint/types/Value.java | 36 +- .../{benchmark => profile}/ProfilerTest.java | 11 +- 9 files changed, 1169 insertions(+), 174 deletions(-) create mode 100644 src/jmh/flatbuffers/test_record.fbs create mode 100644 src/jmh/proto/test_record.proto rename src/test/java/com/imprint/{benchmark => profile}/ProfilerTest.java (97%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d4c8bde..62ac6f5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,30 +15,220 @@ jobs: java-version: [11, 17, 21] steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up JDK ${{ matrix.java-version }} - uses: actions/setup-java@v4 - with: - java-version: ${{ matrix.java-version }} - distribution: 'temurin' - - - name: Cache Gradle dependencies - uses: actions/cache@v4 - with: - path: | - ~/.gradle/caches - ~/.gradle/wrapper - key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} - restore-keys: | - ${{ runner.os }}-gradle- - - - name: Make gradlew executable - run: chmod +x ./gradlew - - - name: Run tests - run: ./gradlew test - - - name: Run build - run: ./gradlew build \ No newline at end of file + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up JDK ${{ matrix.java-version }} + uses: actions/setup-java@v4 + with: + java-version: ${{ matrix.java-version }} + distribution: 'temurin' + + - name: Cache Gradle dependencies + uses: actions/cache@v4 + with: + path: | + ~/.gradle/caches + ~/.gradle/wrapper + key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} + restore-keys: | + ${{ runner.os }}-gradle- + + - name: Make gradlew executable + run: chmod +x ./gradlew + + - name: Run tests + run: ./gradlew test + + - name: Run build + run: ./gradlew build + + benchmark: + runs-on: ubuntu-latest + needs: test + # Only run benchmarks on main branch pushes and PRs to main to avoid excessive CI time + if: github.ref == 'refs/heads/main' || github.base_ref == 'main' + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up JDK 11 + uses: actions/setup-java@v4 + with: + java-version: '11' + distribution: 'temurin' + + - name: Cache Gradle dependencies + uses: actions/cache@v4 + with: + path: | + ~/.gradle/caches + ~/.gradle/wrapper + key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} + restore-keys: | + ${{ runner.os }}-gradle- + + - name: Make gradlew executable + run: chmod +x ./gradlew + + - name: Create benchmark results directory + run: mkdir -p benchmark-results + + - name: Run serialization benchmarks + run: | + ./gradlew jmhRunSerializationBenchmarks + continue-on-error: true + + - name: Run deserialization benchmarks + run: | + ./gradlew jmhRunDeserializationBenchmarks + continue-on-error: true + + - name: Run field access benchmarks + run: | + ./gradlew jmhRunFieldAccessBenchmarks + continue-on-error: true + + - name: Run size comparison benchmarks + run: | + ./gradlew jmhRunSizeComparisonBenchmarks + continue-on-error: true + + - name: Upload benchmark results + uses: actions/upload-artifact@v4 + if: always() + with: + name: benchmark-results-${{ github.sha }} + path: benchmark-results/ + retention-days: 30 + + - name: Comment benchmark results on PR + if: github.event_name == 'pull_request' + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const path = require('path'); + + // Find the latest benchmark results file + const resultsDir = 'benchmark-results'; + let latestFile = null; + let latestTime = 0; + + if (fs.existsSync(resultsDir)) { + const files = fs.readdirSync(resultsDir); + for (const file of files) { + if (file.endsWith('.json')) { + const filePath = path.join(resultsDir, file); + const stats = fs.statSync(filePath); + if (stats.mtime.getTime() > latestTime) { + latestTime = stats.mtime.getTime(); + latestFile = filePath; + } + } + } + } + + if (latestFile) { + const results = JSON.parse(fs.readFileSync(latestFile, 'utf8')); + + // Group results by benchmark type + const serialization = results.filter(r => r.benchmark.includes('serialize')); + const deserialization = results.filter(r => r.benchmark.includes('deserialize')); + const fieldAccess = results.filter(r => r.benchmark.includes('singleFieldAccess')); + const sizes = results.filter(r => r.benchmark.includes('measure')); + + // Format results into a table + const formatResults = (benchmarks, title) => { + if (benchmarks.length === 0) return ''; + + let table = `\n### ${title}\n\n| Library | Score (ns/op) | Error | Unit |\n|---------|---------------|-------|------|\n`; + + benchmarks + .sort((a, b) => a.primaryMetric.score - b.primaryMetric.score) + .forEach(benchmark => { + const name = benchmark.benchmark.split('.').pop().replace(/serialize|deserialize|singleFieldAccess|measure/, '').replace(/Imprint|JacksonJson|Kryo|MessagePack|Avro|Protobuf|FlatBuffers/, (match) => match); + const score = benchmark.primaryMetric.score.toFixed(2); + const error = benchmark.primaryMetric.scoreError.toFixed(2); + const unit = benchmark.primaryMetric.scoreUnit; + table += `| ${name} | ${score} | ±${error} | ${unit} |\n`; + }); + + return table; + }; + + const comment = `##Benchmark Results + + Benchmark comparison between Imprint and other serialization libraries: + ${formatResults(serialization, 'Serialization Performance')} + ${formatResults(deserialization, 'Deserialization Performance')} + ${formatResults(fieldAccess, 'Single Field Access Performance')} + ${formatResults(sizes, 'Serialized Size Comparison')} + +
+ View detailed results + + Results generated from commit: \`${context.sha.substring(0, 7)}\` + + Lower scores are better for performance benchmarks. + +
`; + + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: comment + }); + } else { + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: '## Benchmark Results\n\nBenchmark execution completed but no results file was found. Check the [workflow logs](' + + `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}` + ') for details.' + }); + } + + # Optional: Run full benchmark suite on releases + benchmark-full: + runs-on: ubuntu-latest + if: startsWith(github.ref, 'refs/tags/') + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up JDK 11 + uses: actions/setup-java@v4 + with: + java-version: '11' + distribution: 'temurin' + + - name: Cache Gradle dependencies + uses: actions/cache@v4 + with: + path: | + ~/.gradle/caches + ~/.gradle/wrapper + key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} + restore-keys: | + ${{ runner.os }}-gradle- + + - name: Make gradlew executable + run: chmod +x ./gradlew + + - name: Create benchmark results directory + run: mkdir -p benchmark-results + + - name: Run full benchmark suite + run: | + ./gradlew jmh + + - name: Upload full benchmark results + uses: actions/upload-artifact@v4 + with: + name: full-benchmark-results-${{ github.ref_name }} + path: benchmark-results/ + retention-days: 90 \ No newline at end of file diff --git a/build.gradle b/build.gradle index 2606710..66f2e3e 100644 --- a/build.gradle +++ b/build.gradle @@ -2,6 +2,8 @@ plugins { id 'java-library' id 'maven-publish' id 'me.champeau.jmh' version '0.7.2' + id 'com.google.protobuf' version '0.9.4' + id 'io.netifi.flatbuffers' version '1.0.7' } group = 'com.imprint' @@ -23,45 +25,290 @@ dependencies { // Lombok for reducing boilerplate compileOnly 'org.projectlombok:lombok:1.18.30' annotationProcessor 'org.projectlombok:lombok:1.18.30' - + // Test dependencies testImplementation 'org.junit.jupiter:junit-jupiter:5.10.0' testImplementation 'org.assertj:assertj-core:3.24.2' testImplementation 'org.mockito:mockito-core:5.5.0' - + // Lombok for tests testCompileOnly 'org.projectlombok:lombok:1.18.30' testAnnotationProcessor 'org.projectlombok:lombok:1.18.30' - + // Performance testing with JMH jmhImplementation 'org.openjdk.jmh:jmh-core:1.37' jmhAnnotationProcessor 'org.openjdk.jmh:jmh-generator-annprocess:1.37' - - // Competitor libraries for benchmarking + + // Suppress SLF4J warnings + jmhImplementation 'org.slf4j:slf4j-nop:1.7.36' + + // Competitor libraries for benchmarking (JMH only) jmhImplementation 'com.google.protobuf:protobuf-java:3.25.1' jmhImplementation 'org.apache.avro:avro:1.11.3' jmhImplementation 'com.fasterxml.jackson.core:jackson-databind:2.16.0' jmhImplementation 'com.google.flatbuffers:flatbuffers-java:23.5.26' jmhImplementation 'com.esotericsoftware:kryo:5.4.0' + jmhImplementation 'org.msgpack:msgpack-core:0.9.8' + jmhImplementation 'org.msgpack:jackson-dataformat-msgpack:0.9.8' +} + +protobuf { + protoc { + artifact = "com.google.protobuf:protoc:3.25.1" + } + generateProtoTasks { + // Only generate for JMH, not main + all().each { task -> + task.enabled = false + } + ofSourceSet('jmh').each { task -> + task.enabled = true + task.builtins { + java { + outputSubDir = 'java' + } + } + } + } +} + +// Download and setup FlatBuffers compiler for Linux (CI environment) +tasks.register('downloadFlatc') { + description = 'Download FlatBuffers compiler' + group = 'build setup' + + def flatcDir = file("${buildDir}/flatc") + def flatcExe = file("${flatcDir}/flatc") + def isWindows = System.getProperty('os.name').toLowerCase().contains('windows') + def flatcUrl = isWindows ? + "https://github.com/google/flatbuffers/releases/download/v23.5.26/Windows.flatc.binary.zip" : + "https://github.com/google/flatbuffers/releases/download/v23.5.26/Linux.flatc.binary.clang++-12.zip" + def flatcZip = file("${buildDir}/flatc.zip") + + outputs.file(flatcExe) + + doLast { + if (!flatcExe.exists()) { + println "Downloading FlatBuffers compiler for ${isWindows ? 'Windows' : 'Linux'}..." + flatcDir.mkdirs() + + // Download + new URL(flatcUrl).withInputStream { i -> + flatcZip.withOutputStream { it << i } + } + + // Extract + copy { + from zipTree(flatcZip) + into flatcDir + } + + // Make executable on Unix systems + if (!isWindows) { + exec { + commandLine 'chmod', '+x', flatcExe.absolutePath + } + } + + flatcZip.delete() + println "FlatBuffers compiler downloaded to: ${flatcExe}" + } + } +} + +// Generate FlatBuffers sources +tasks.register('generateFlatBuffers', Exec) { + dependsOn downloadFlatc + description = 'Generate Java classes from FlatBuffers schema' + group = 'build' + + def isWindows = System.getProperty('os.name').toLowerCase().contains('windows') + def flatcExe = file("${buildDir}/flatc/${isWindows ? 'flatc.exe' : 'flatc'}") + def schemaFile = file('src/jmh/flatbuffers/test_record.fbs') + def outputDir = file('build/generated/source/flatbuffers/jmh/java') + + commandLine flatcExe.absolutePath, '--java', '-o', outputDir.absolutePath, schemaFile.absolutePath + + inputs.file(schemaFile) + outputs.dir(outputDir) + + doFirst { + outputDir.mkdirs() + } +} + +// Add generated FlatBuffers sources to JMH source set +sourceSets { + jmh { + java { + srcDir 'build/generated/source/flatbuffers/jmh/java' + } + proto { + srcDir 'src/jmh/proto' + } + } +} + +// Make JMH compilation depend on FlatBuffers generation +compileJmhJava.dependsOn generateFlatBuffers + +// Handle duplicate proto files +tasks.named('processJmhResources') { + duplicatesStrategy = DuplicatesStrategy.EXCLUDE } test { useJUnitPlatform() - + // Enable detailed test output testLogging { events "passed", "skipped", "failed" } } -// JMH configuration +// JMH configuration - optimized for Java 11 jmh { fork = 1 - warmupIterations = 3 + warmupIterations = 2 // Reduced for faster CI iterations = 3 resultFormat = 'JSON' includeTests = false resultsFile = file("${projectDir}/benchmark-results/jmh-results-${new Date().format('yyyy-MM-dd-HHmmss')}.json") + + // Java 11 specific JVM args + jvmArgs = [ + '-XX:+UseG1GC', + '-Xmx2g', + '-XX:+UnlockExperimentalVMOptions', + '-XX:+UseJVMCICompiler' // Use Graal if available for better performance + ] +} + +// Create individual benchmark tasks for CI pipeline +tasks.register('jmhRunSerializationBenchmarks', JavaExec) { + dependsOn compileJmhJava + description = 'Run serialization benchmarks' + group = 'benchmarking' + + classpath = sourceSets.jmh.runtimeClasspath + mainClass = 'com.imprint.benchmark.ComparisonBenchmark' + args = ['runSerializationBenchmarks'] + + // Java 11 optimized JVM settings + jvmArgs = [ + '-XX:+UseG1GC', + '-Xmx2g', + '-XX:+UnlockExperimentalVMOptions' + ] + + doFirst { + file("${projectDir}/benchmark-results").mkdirs() + } +} + +tasks.register('jmhRunDeserializationBenchmarks', JavaExec) { + dependsOn compileJmhJava + description = 'Run deserialization benchmarks' + group = 'benchmarking' + + classpath = sourceSets.jmh.runtimeClasspath + mainClass = 'com.imprint.benchmark.ComparisonBenchmark' + args = ['runDeserializationBenchmarks'] + + // Java 11 optimized JVM settings + jvmArgs = [ + '-XX:+UseG1GC', + '-Xmx2g', + '-XX:+UnlockExperimentalVMOptions' + ] + + doFirst { + file("${projectDir}/benchmark-results").mkdirs() + } +} + +tasks.register('jmhRunFieldAccessBenchmarks', JavaExec) { + dependsOn compileJmhJava + description = 'Run field access benchmarks' + group = 'benchmarking' + + classpath = sourceSets.jmh.runtimeClasspath + mainClass = 'com.imprint.benchmark.ComparisonBenchmark' + args = ['runFieldAccessBenchmarks'] + + // Java 11 optimized JVM settings + jvmArgs = [ + '-XX:+UseG1GC', + '-Xmx2g', + '-XX:+UnlockExperimentalVMOptions' + ] + + doFirst { + file("${projectDir}/benchmark-results").mkdirs() + } +} + +tasks.register('jmhRunSizeComparisonBenchmarks', JavaExec) { + dependsOn compileJmhJava + description = 'Run size comparison benchmarks' + group = 'benchmarking' + + classpath = sourceSets.jmh.runtimeClasspath + mainClass = 'com.imprint.benchmark.ComparisonBenchmark' + args = ['runSizeComparisonBenchmarks'] + + // Java 11 optimized JVM settings + jvmArgs = [ + '-XX:+UseG1GC', + '-Xmx2g', + '-XX:+UnlockExperimentalVMOptions' + ] + + doFirst { + file("${projectDir}/benchmark-results").mkdirs() + } +} + +tasks.register('jmhRunMergeBenchmarks', JavaExec) { + dependsOn compileJmhJava + description = 'Run merge operation benchmarks' + group = 'benchmarking' + + classpath = sourceSets.jmh.runtimeClasspath + mainClass = 'com.imprint.benchmark.ComparisonBenchmark' + args = ['runMergeBenchmarks'] + + // Java 11 optimized JVM settings + jvmArgs = [ + '-XX:+UseG1GC', + '-Xmx2g', + '-XX:+UnlockExperimentalVMOptions' + ] + + doFirst { + file("${projectDir}/benchmark-results").mkdirs() + } +} + +tasks.register('jmhRunAllBenchmarks', JavaExec) { + dependsOn compileJmhJava + description = 'Run all comparison benchmarks' + group = 'benchmarking' + + classpath = sourceSets.jmh.runtimeClasspath + mainClass = 'com.imprint.benchmark.ComparisonBenchmark' + args = ['runAll'] + + // Java 11 optimized JVM settings + jvmArgs = [ + '-XX:+UseG1GC', + '-Xmx2g', + '-XX:+UnlockExperimentalVMOptions' + ] + + doFirst { + file("${projectDir}/benchmark-results").mkdirs() + } } compileJava { @@ -75,4 +322,4 @@ javadoc { } // Don't fail build on missing javadoc options.addStringOption('Xdoclint:none', '-quiet') -} +} \ No newline at end of file diff --git a/src/jmh/flatbuffers/test_record.fbs b/src/jmh/flatbuffers/test_record.fbs new file mode 100644 index 0000000..ccc31d0 --- /dev/null +++ b/src/jmh/flatbuffers/test_record.fbs @@ -0,0 +1,15 @@ +namespace com.imprint.benchmark; + +table TestRecordFB { + id: int; + name: string; + price: double; + active: bool; + category: string; + tags: [string]; + metadata_keys: [string]; + metadata_values: [string]; + extra_data: [string]; +} + +root_type TestRecordFB; \ No newline at end of file diff --git a/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java b/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java index 8163522..49260b1 100644 --- a/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java +++ b/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java @@ -4,13 +4,25 @@ import com.esotericsoftware.kryo.io.Input; import com.esotericsoftware.kryo.io.Output; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.flatbuffers.FlatBufferBuilder; import com.imprint.core.ImprintRecord; import com.imprint.core.ImprintWriter; import com.imprint.core.SchemaId; import com.imprint.types.MapKey; import com.imprint.types.Value; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericDatumReader; +import org.apache.avro.generic.GenericDatumWriter; +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.io.*; +import org.msgpack.jackson.dataformat.MessagePackFactory; import org.openjdk.jmh.annotations.*; import org.openjdk.jmh.infra.Blackhole; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; @@ -25,38 +37,56 @@ @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.NANOSECONDS) @State(Scope.Benchmark) -@Warmup(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS) -@Measurement(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS) +@Warmup(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 10, time = 1, timeUnit = TimeUnit.SECONDS) @Fork(1) +@SuppressWarnings("unused") public class ComparisonBenchmark { // Test data private TestRecord testData; - + // Serialized formats - private ByteBuffer imprintBytes; - private byte[] jacksonBytes; + private ByteBuffer imprintBytesBuffer; + private byte[] jacksonJsonBytes; private byte[] kryoBytes; - + private byte[] messagePackBytes; + private byte[] avroBytes; + private byte[] protobufBytes; + private ByteBuffer flatbuffersBytes; + // Library instances - private ObjectMapper jackson; + private Schema avroSchema; + private DatumWriter avroWriter; + private DatumReader avroReader; + private ObjectMapper jacksonJsonMapper; private Kryo kryo; + private ObjectMapper messagePackMapper; @Setup public void setup() throws Exception { testData = createTestRecord(); - + // Initialize libraries - jackson = new ObjectMapper(); + jacksonJsonMapper = new ObjectMapper(); kryo = new Kryo(); kryo.register(TestRecord.class); kryo.register(ArrayList.class); kryo.register(HashMap.class); - + kryo.register(Arrays.asList().getClass()); + + // Initialize MessagePack ObjectMapper + messagePackMapper = new ObjectMapper(new MessagePackFactory()); + setupAvro(); + // Pre-serialize for deserialization benchmarks - imprintBytes = serializeWithImprint(testData); - jacksonBytes = serializeWithJackson(testData); + imprintBytesBuffer = serializeWithImprint(testData); + jacksonJsonBytes = serializeWithJacksonJson(testData); kryoBytes = serializeWithKryo(testData); + messagePackBytes = serializeWithMessagePack(testData); + avroBytes = serializeWithAvro(testData); + protobufBytes = serializeWithProtobuf(testData); + flatbuffersBytes = serializeWithFlatBuffers(testData); } // ===== SERIALIZATION BENCHMARKS ===== @@ -68,8 +98,8 @@ public void serializeImprint(Blackhole bh) throws Exception { } @Benchmark - public void serializeJackson(Blackhole bh) throws Exception { - byte[] result = serializeWithJackson(testData); + public void serializeJacksonJson(Blackhole bh) throws Exception { + byte[] result = serializeWithJacksonJson(testData); bh.consume(result); } @@ -79,17 +109,41 @@ public void serializeKryo(Blackhole bh) { bh.consume(result); } + @Benchmark + public void serializeMessagePack(Blackhole bh) throws Exception { + byte[] result = serializeWithMessagePack(testData); + bh.consume(result); + } + + @Benchmark + public void serializeAvro(Blackhole bh) throws Exception { + byte[] result = serializeWithAvro(testData); + bh.consume(result); + } + + @Benchmark + public void serializeProtobuf(Blackhole bh) { + byte[] result = serializeWithProtobuf(testData); + bh.consume(result); + } + + @Benchmark + public void serializeFlatBuffers(Blackhole bh) { + ByteBuffer result = serializeWithFlatBuffers(testData); + bh.consume(result); + } + // ===== DESERIALIZATION BENCHMARKS ===== @Benchmark public void deserializeImprint(Blackhole bh) throws Exception { - ImprintRecord result = ImprintRecord.deserialize(imprintBytes.duplicate()); + ImprintRecord result = ImprintRecord.deserialize(imprintBytesBuffer.duplicate()); bh.consume(result); } @Benchmark - public void deserializeJackson(Blackhole bh) throws Exception { - TestRecord result = jackson.readValue(jacksonBytes, TestRecord.class); + public void deserializeJacksonJson(Blackhole bh) throws Exception { + TestRecord result = jacksonJsonMapper.readValue(jacksonJsonBytes, TestRecord.class); bh.consume(result); } @@ -101,135 +155,361 @@ public void deserializeKryo(Blackhole bh) { bh.consume(result); } + @Benchmark + public void deserializeMessagePack(Blackhole bh) throws Exception { + TestRecord result = messagePackMapper.readValue(messagePackBytes, TestRecord.class); + bh.consume(result); + } + + @Benchmark + public void deserializeAvro(Blackhole bh) throws Exception { + GenericRecord result = deserializeWithAvro(avroBytes); + bh.consume(result); + } + + @Benchmark + public void deserializeProtobuf(Blackhole bh) throws Exception { + TestRecordProto.TestRecord result = TestRecordProto.TestRecord.parseFrom(protobufBytes); + bh.consume(result); + } + + @Benchmark + public void deserializeFlatBuffers(Blackhole bh) { + TestRecordFB result = TestRecordFB.getRootAsTestRecordFB(flatbuffersBytes.duplicate()); + bh.consume(result); + } + // ===== FIELD ACCESS BENCHMARKS ===== // Tests accessing a single field near the end of a large record + // This showcases Imprint's O(1) directory lookup vs sequential deserialization + @Benchmark public void singleFieldAccessImprint(Blackhole bh) throws Exception { - ImprintRecord record = ImprintRecord.deserialize(imprintBytes.duplicate()); - - // Access field 15 directly via directory lookup - O(1) - var field15 = record.getValue(15); + ImprintRecord record = ImprintRecord.deserialize(imprintBytesBuffer.duplicate()); + var field15 = record.getString(15); bh.consume(field15); } @Benchmark - public void singleFieldAccessJackson(Blackhole bh) throws Exception { - // Jackson must deserialize entire object to access any field - TestRecord record = jackson.readValue(jacksonBytes, TestRecord.class); - - // Access field15 equivalent (extraData[4]) after full deserialization + public void singleFieldAccessJacksonJson(Blackhole bh) throws Exception { + TestRecord record = jacksonJsonMapper.readValue(jacksonJsonBytes, TestRecord.class); bh.consume(record.extraData.get(4)); } @Benchmark public void singleFieldAccessKryo(Blackhole bh) { - // Kryo must deserialize entire object to access any field Input input = new Input(new ByteArrayInputStream(kryoBytes)); TestRecord record = kryo.readObject(input, TestRecord.class); input.close(); - - // Access field15 equivalent (extraData[4]) after full deserialization bh.consume(record.extraData.get(4)); } + @Benchmark + public void singleFieldAccessMessagePack(Blackhole bh) throws Exception { + TestRecord record = messagePackMapper.readValue(messagePackBytes, TestRecord.class); + bh.consume(record.extraData.get(4)); + } + + @Benchmark + public void singleFieldAccessAvro(Blackhole bh) throws Exception { + GenericRecord record = deserializeWithAvro(avroBytes); + bh.consume(record.get("extraData4")); // Accessing field near end + } + + @Benchmark + public void singleFieldAccessProtobuf(Blackhole bh) throws Exception { + TestRecordProto.TestRecord record = TestRecordProto.TestRecord.parseFrom(protobufBytes); + bh.consume(record.getExtraData(4)); // Accessing field near end + } + + @Benchmark + public void singleFieldAccessFlatBuffers(Blackhole bh) { + TestRecordFB record = TestRecordFB.getRootAsTestRecordFB(flatbuffersBytes.duplicate()); + bh.consume(record.extraData(4)); // Accessing field near end - zero copy! + } + // ===== SIZE COMPARISON ===== @Benchmark - public void measureImprintSize(Blackhole bh) throws Exception { - ByteBuffer serialized = serializeWithImprint(testData); - bh.consume(serialized.remaining()); + public void measureImprintSize(Blackhole bh) { + bh.consume(imprintBytesBuffer.remaining()); } @Benchmark - public void measureJacksonSize(Blackhole bh) throws Exception { - byte[] serialized = serializeWithJackson(testData); - bh.consume(serialized.length); + public void measureJacksonJsonSize(Blackhole bh) { + bh.consume(jacksonJsonBytes.length); } @Benchmark public void measureKryoSize(Blackhole bh) { - byte[] serialized = serializeWithKryo(testData); - bh.consume(serialized.length); + bh.consume(kryoBytes.length); + } + + @Benchmark + public void measureMessagePackSize(Blackhole bh) { + bh.consume(messagePackBytes.length); + } + + @Benchmark + public void measureAvroSize(Blackhole bh) { + bh.consume(avroBytes.length); + } + + @Benchmark + public void measureProtobufSize(Blackhole bh) { + bh.consume(protobufBytes.length); + } + + @Benchmark + public void measureFlatBuffersSize(Blackhole bh) { + bh.consume(flatbuffersBytes.remaining()); } // ===== MERGE SIMULATION BENCHMARKS ===== @Benchmark public void mergeImprint(Blackhole bh) throws Exception { - var record1 = serializeWithImprint(testData); - var record2 = serializeWithImprint(createTestRecord2()); + var record1Buffer = imprintBytesBuffer.duplicate(); + var record2Data = createTestRecord2(); + var record2Buffer = serializeWithImprint(record2Data); - var deserialized1 = ImprintRecord.deserialize(record1); - var deserialized2 = ImprintRecord.deserialize(record2); + var deserialized1 = ImprintRecord.deserialize(record1Buffer); + var deserialized2 = ImprintRecord.deserialize(record2Buffer); var merged = simulateMerge(deserialized1, deserialized2); - + bh.consume(merged); } @Benchmark - public void mergeJackson(Blackhole bh) throws Exception { - // Jackson merge requires full deserialization + merge + serialization - var record1 = jackson.readValue(jacksonBytes, TestRecord.class); - var record2 = jackson.readValue(serializeWithJackson(createTestRecord2()), TestRecord.class); - - var merged = mergeTestRecords(record1, record2); - byte[] result = jackson.writeValueAsBytes(merged); - + public void mergeJacksonJson(Blackhole bh) throws Exception { + var record1 = jacksonJsonMapper.readValue(jacksonJsonBytes, TestRecord.class); + var record2Data = createTestRecord2(); + var record2Bytes = serializeWithJacksonJson(record2Data); + var record2 = jacksonJsonMapper.readValue(record2Bytes, TestRecord.class); + + var mergedPojo = mergeTestRecords(record1, record2); + byte[] result = jacksonJsonMapper.writeValueAsBytes(mergedPojo); bh.consume(result); } @Benchmark public void mergeKryo(Blackhole bh) { - // Kryo merge requires full deserialization + merge + serialization Input input1 = new Input(new ByteArrayInputStream(kryoBytes)); var record1 = kryo.readObject(input1, TestRecord.class); input1.close(); - - Input input2 = new Input(new ByteArrayInputStream(serializeWithKryo(createTestRecord2()))); + + var record2Data = createTestRecord2(); + var record2Bytes = serializeWithKryo(record2Data); + Input input2 = new Input(new ByteArrayInputStream(record2Bytes)); var record2 = kryo.readObject(input2, TestRecord.class); input2.close(); - - var merged = mergeTestRecords(record1, record2); - byte[] result = serializeWithKryo(merged); - + + var mergedPojo = mergeTestRecords(record1, record2); + byte[] result = serializeWithKryo(mergedPojo); + bh.consume(result); + } + + @Benchmark + public void mergeMessagePack(Blackhole bh) throws Exception { + var record1 = messagePackMapper.readValue(messagePackBytes, TestRecord.class); + var record2Data = createTestRecord2(); + var record2Bytes = serializeWithMessagePack(record2Data); + var record2 = messagePackMapper.readValue(record2Bytes, TestRecord.class); + + var mergedPojo = mergeTestRecords(record1, record2); + byte[] result = messagePackMapper.writeValueAsBytes(mergedPojo); bh.consume(result); } + @Benchmark + public void mergeAvro(Blackhole bh) throws Exception { + var record1 = deserializeWithAvro(avroBytes); + var record2Data = createTestRecord2(); + var record2Bytes = serializeWithAvro(record2Data); + var record2 = deserializeWithAvro(record2Bytes); + + var merged = mergeAvroRecords(record1, record2); + byte[] result = serializeAvroRecord(merged); + bh.consume(result); + } + + @Benchmark + public void mergeProtobuf(Blackhole bh) throws Exception { + var record1 = TestRecordProto.TestRecord.parseFrom(protobufBytes); + var record2Data = createTestRecord2(); + var record2Bytes = serializeWithProtobuf(record2Data); + var record2 = TestRecordProto.TestRecord.parseFrom(record2Bytes); + + var merged = mergeProtobufRecords(record1, record2); + byte[] result = merged.toByteArray(); + bh.consume(result); + } + + @Benchmark + public void mergeFlatBuffers(Blackhole bh) { + var record1 = TestRecordFB.getRootAsTestRecordFB(flatbuffersBytes.duplicate()); + var record2Data = createTestRecord2(); + var record2Buffer = serializeWithFlatBuffers(record2Data); + var record2 = TestRecordFB.getRootAsTestRecordFB(record2Buffer); + + var merged = mergeFlatBuffersRecords(record1, record2); + bh.consume(merged); + } + + // ===== MAIN METHOD TO RUN BENCHMARKS ===== + + public static void main(String[] args) throws RunnerException { + runAll(); + // Or, uncomment specific runner methods to execute subsets: + // runSerializationBenchmarks(); + // runDeserializationBenchmarks(); + // runFieldAccessBenchmarks(); + // runSizeComparisonBenchmarks(); + // runMergeBenchmarks(); + // runMessagePackBenchmarks(); + } + + public static void runAll() throws RunnerException { + Options opt = new OptionsBuilder() + .include(ComparisonBenchmark.class.getSimpleName()) + .build(); + new Runner(opt).run(); + } + + public static void runSerializationBenchmarks() throws RunnerException { + Options opt = new OptionsBuilder() + .include(ComparisonBenchmark.class.getSimpleName() + ".serialize.*") + .build(); + new Runner(opt).run(); + } + + public static void runDeserializationBenchmarks() throws RunnerException { + Options opt = new OptionsBuilder() + .include(ComparisonBenchmark.class.getSimpleName() + ".deserialize.*") + .build(); + new Runner(opt).run(); + } + + public static void runFieldAccessBenchmarks() throws RunnerException { + Options opt = new OptionsBuilder() + .include(ComparisonBenchmark.class.getSimpleName() + ".singleFieldAccess.*") + .build(); + new Runner(opt).run(); + } + + public static void runSizeComparisonBenchmarks() throws RunnerException { + Options opt = new OptionsBuilder() + .include(ComparisonBenchmark.class.getSimpleName() + ".measure.*") + .build(); + new Runner(opt).run(); + } + + public static void runMergeBenchmarks() throws RunnerException { + Options opt = new OptionsBuilder() + .include(ComparisonBenchmark.class.getSimpleName() + ".merge.*") + .build(); + new Runner(opt).run(); + } + + public static void runMessagePackBenchmarks() throws RunnerException { + Options opt = new OptionsBuilder() + .include(ComparisonBenchmark.class.getSimpleName() + ".*MessagePack.*") + .build(); + new Runner(opt).run(); + } + + public static void runAvroBenchmarks() throws RunnerException { + Options opt = new OptionsBuilder() + .include(ComparisonBenchmark.class.getSimpleName() + ".*Avro.*") + .build(); + new Runner(opt).run(); + } + + public static void runProtobufBenchmarks() throws RunnerException { + Options opt = new OptionsBuilder() + .include(ComparisonBenchmark.class.getSimpleName() + ".*Protobuf.*") + .build(); + new Runner(opt).run(); + } + + public static void runFlatBuffersBenchmarks() throws RunnerException { + Options opt = new OptionsBuilder() + .include(ComparisonBenchmark.class.getSimpleName() + ".*FlatBuffers.*") + .build(); + new Runner(opt).run(); + } + // ===== HELPER METHODS ===== + private void setupAvro() { + String schemaJson = "{\n" + + " \"type\": \"record\",\n" + + " \"name\": \"TestRecord\",\n" + + " \"fields\": [\n" + + " {\"name\": \"id\", \"type\": \"int\"},\n" + + " {\"name\": \"name\", \"type\": \"string\"},\n" + + " {\"name\": \"price\", \"type\": \"double\"},\n" + + " {\"name\": \"active\", \"type\": \"boolean\"},\n" + + " {\"name\": \"category\", \"type\": \"string\"},\n" + + " {\"name\": \"tags\", \"type\": {\"type\": \"array\", \"items\": \"string\"}},\n" + + " {\"name\": \"metadata\", \"type\": {\"type\": \"map\", \"values\": \"string\"}},\n" + + " {\"name\": \"extraData0\", \"type\": \"string\"},\n" + + " {\"name\": \"extraData1\", \"type\": \"string\"},\n" + + " {\"name\": \"extraData2\", \"type\": \"string\"},\n" + + " {\"name\": \"extraData3\", \"type\": \"string\"},\n" + + " {\"name\": \"extraData4\", \"type\": \"string\"},\n" + + " {\"name\": \"extraData5\", \"type\": \"string\"},\n" + + " {\"name\": \"extraData6\", \"type\": \"string\"},\n" + + " {\"name\": \"extraData7\", \"type\": \"string\"},\n" + + " {\"name\": \"extraData8\", \"type\": \"string\"},\n" + + " {\"name\": \"extraData9\", \"type\": \"string\"},\n" + + " {\"name\": \"extraData10\", \"type\": \"string\"},\n" + + " {\"name\": \"extraData11\", \"type\": \"string\"},\n" + + " {\"name\": \"extraData12\", \"type\": \"string\"}\n" + + " ]\n" + + "}"; + + avroSchema = new Schema.Parser().parse(schemaJson); + avroWriter = new GenericDatumWriter<>(avroSchema); + avroReader = new GenericDatumReader<>(avroSchema); + } + private ByteBuffer serializeWithImprint(TestRecord data) throws Exception { var writer = new ImprintWriter(new SchemaId(1, 0x12345678)); - + writer.addField(1, Value.fromInt32(data.id)); writer.addField(2, Value.fromString(data.name)); writer.addField(3, Value.fromFloat64(data.price)); writer.addField(4, Value.fromBoolean(data.active)); writer.addField(5, Value.fromString(data.category)); - - // Convert tags list + var tagValues = new ArrayList(); - for (String tag : data.tags) { - tagValues.add(Value.fromString(tag)); + if (data.tags != null) { + for (String tag : data.tags) { + tagValues.add(Value.fromString(tag)); + } } writer.addField(6, Value.fromArray(tagValues)); - - // Convert metadata map + var metadataMap = new HashMap(); - for (var entry : data.metadata.entrySet()) { - metadataMap.put(MapKey.fromString(entry.getKey()), Value.fromString(entry.getValue())); + if (data.metadata != null) { + for (var entry : data.metadata.entrySet()) { + metadataMap.put(MapKey.fromString(entry.getKey()), Value.fromString(entry.getValue())); + } } writer.addField(7, Value.fromMap(metadataMap)); - - // Add extra fields (8-20) to create a larger record - for (int i = 0; i < data.extraData.size(); i++) { - writer.addField(8 + i, Value.fromString(data.extraData.get(i))); + + if (data.extraData != null) { + for (int i = 0; i < data.extraData.size(); i++) { + writer.addField(8 + i, Value.fromString(data.extraData.get(i))); + } } - + return writer.build().serializeToBuffer(); } - private byte[] serializeWithJackson(TestRecord data) throws Exception { - return jackson.writeValueAsBytes(data); + private byte[] serializeWithJacksonJson(TestRecord data) throws Exception { + return jacksonJsonMapper.writeValueAsBytes(data); } private byte[] serializeWithKryo(TestRecord data) { @@ -240,14 +520,117 @@ private byte[] serializeWithKryo(TestRecord data) { return baos.toByteArray(); } + private byte[] serializeWithMessagePack(TestRecord data) throws Exception { + return messagePackMapper.writeValueAsBytes(data); + } + + private byte[] serializeWithAvro(TestRecord data) throws Exception { + GenericRecord record = new GenericData.Record(avroSchema); + record.put("id", data.id); + record.put("name", data.name); + record.put("price", data.price); + record.put("active", data.active); + record.put("category", data.category); + record.put("tags", data.tags); + record.put("metadata", data.metadata); + + for (int i = 0; i < data.extraData.size(); i++) { + record.put("extraData" + i, data.extraData.get(i)); + } + + return serializeAvroRecord(record); + } + + private byte[] serializeAvroRecord(GenericRecord record) throws Exception { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + Encoder encoder = EncoderFactory.get().binaryEncoder(baos, null); + avroWriter.write(record, encoder); + encoder.flush(); + return baos.toByteArray(); + } + + private GenericRecord deserializeWithAvro(byte[] data) throws Exception { + Decoder decoder = DecoderFactory.get().binaryDecoder(data, null); + return avroReader.read(null, decoder); + } + + private byte[] serializeWithProtobuf(TestRecord data) { + var builder = TestRecordProto.TestRecord.newBuilder() + .setId(data.id) + .setName(data.name) + .setPrice(data.price) + .setActive(data.active) + .setCategory(data.category) + .addAllTags(data.tags) + .putAllMetadata(data.metadata); + + for (String extraData : data.extraData) { + builder.addExtraData(extraData); + } + + return builder.build().toByteArray(); + } + + private ByteBuffer serializeWithFlatBuffers(TestRecord data) { + FlatBufferBuilder builder = new FlatBufferBuilder(1024); + + // Create strings (must be created before the object that uses them) + int nameOffset = builder.createString(data.name); + int categoryOffset = builder.createString(data.category); + + // Create tags array + int[] tagOffsets = new int[data.tags.size()]; + for (int i = 0; i < data.tags.size(); i++) { + tagOffsets[i] = builder.createString(data.tags.get(i)); + } + int tagsOffset = TestRecordFB.createTagsVector(builder, tagOffsets); + + // Create metadata (as parallel arrays for keys and values) + String[] metadataKeys = data.metadata.keySet().toArray(new String[0]); + String[] metadataValues = new String[metadataKeys.length]; + int[] keyOffsets = new int[metadataKeys.length]; + int[] valueOffsets = new int[metadataKeys.length]; + + for (int i = 0; i < metadataKeys.length; i++) { + metadataValues[i] = data.metadata.get(metadataKeys[i]); + keyOffsets[i] = builder.createString(metadataKeys[i]); + valueOffsets[i] = builder.createString(metadataValues[i]); + } + int metadataKeysOffset = TestRecordFB.createMetadataKeysVector(builder, keyOffsets); + int metadataValuesOffset = TestRecordFB.createMetadataValuesVector(builder, valueOffsets); + + // Create extra data array + int[] extraDataOffsets = new int[data.extraData.size()]; + for (int i = 0; i < data.extraData.size(); i++) { + extraDataOffsets[i] = builder.createString(data.extraData.get(i)); + } + int extraDataOffset = TestRecordFB.createExtraDataVector(builder, extraDataOffsets); + + // Create the main object + TestRecordFB.startTestRecordFB(builder); + TestRecordFB.addId(builder, data.id); + TestRecordFB.addName(builder, nameOffset); + TestRecordFB.addPrice(builder, data.price); + TestRecordFB.addActive(builder, data.active); + TestRecordFB.addCategory(builder, categoryOffset); + TestRecordFB.addTags(builder, tagsOffset); + TestRecordFB.addMetadataKeys(builder, metadataKeysOffset); + TestRecordFB.addMetadataValues(builder, metadataValuesOffset); + TestRecordFB.addExtraData(builder, extraDataOffset); + int recordOffset = TestRecordFB.endTestRecordFB(builder); + + // Finish and return + builder.finish(recordOffset); + return builder.dataBuffer().slice(); + } + private ImprintRecord simulateMerge(ImprintRecord first, ImprintRecord second) throws Exception { var writer = new ImprintWriter(first.getHeader().getSchemaId()); var usedFieldIds = new HashSet(); - - // Copy fields from first record (takes precedence) + copyFieldsToWriter(first, writer, usedFieldIds); copyFieldsToWriter(second, writer, usedFieldIds); - + return writer.build(); } @@ -265,23 +648,121 @@ private void copyFieldsToWriter(ImprintRecord record, ImprintWriter writer, Set< } private TestRecord mergeTestRecords(TestRecord first, TestRecord second) { - // Simple merge logic - first record takes precedence var merged = new TestRecord(); merged.id = first.id; merged.name = first.name != null ? first.name : second.name; merged.price = first.price != 0.0 ? first.price : second.price; merged.active = first.active; merged.category = first.category != null ? first.category : second.category; - + merged.tags = new ArrayList<>(first.tags); merged.tags.addAll(second.tags); - + merged.metadata = new HashMap<>(first.metadata); merged.metadata.putAll(second.metadata); - + return merged; } + private GenericRecord mergeAvroRecords(GenericRecord first, GenericRecord second) { + GenericRecord merged = new GenericData.Record(avroSchema); + + // Copy all fields from first record + for (Schema.Field field : avroSchema.getFields()) { + merged.put(field.name(), first.get(field.name())); + } + + // Override with non-null values from second record + for (Schema.Field field : avroSchema.getFields()) { + Object secondValue = second.get(field.name()); + if (secondValue != null && !secondValue.toString().isEmpty()) { + merged.put(field.name(), secondValue); + } + } + + return merged; + } + + private TestRecordProto.TestRecord mergeProtobufRecords(TestRecordProto.TestRecord first, TestRecordProto.TestRecord second) { + return TestRecordProto.TestRecord.newBuilder() + .mergeFrom(first) + .mergeFrom(second) + .build(); + } + + private ByteBuffer mergeFlatBuffersRecords(TestRecordFB first, TestRecordFB second) { + FlatBufferBuilder builder = new FlatBufferBuilder(1024); + + // Use second record's values if they exist, otherwise first record's values + String name = second.name() != null && !second.name().isEmpty() ? second.name() : first.name(); + String category = second.category() != null && !second.category().isEmpty() ? second.category() : first.category(); + double price = second.price() != 0.0 ? second.price() : first.price(); + boolean active = second.active(); // Use second's boolean value + int id = first.id(); // Keep first record's ID + + // Create merged strings + int nameOffset = builder.createString(name); + int categoryOffset = builder.createString(category); + + // Merge tags (combine both arrays) + List mergedTags = new ArrayList<>(); + for (int i = 0; i < first.tagsLength(); i++) { + mergedTags.add(first.tags(i)); + } + for (int i = 0; i < second.tagsLength(); i++) { + mergedTags.add(second.tags(i)); + } + + int[] tagOffsets = new int[mergedTags.size()]; + for (int i = 0; i < mergedTags.size(); i++) { + tagOffsets[i] = builder.createString(mergedTags.get(i)); + } + int tagsOffset = TestRecordFB.createTagsVector(builder, tagOffsets); + + // Merge metadata (second overwrites first) + Map mergedMetadata = new HashMap<>(); + for (int i = 0; i < first.metadataKeysLength(); i++) { + mergedMetadata.put(first.metadataKeys(i), first.metadataValues(i)); + } + for (int i = 0; i < second.metadataKeysLength(); i++) { + mergedMetadata.put(second.metadataKeys(i), second.metadataValues(i)); + } + + String[] metadataKeys = mergedMetadata.keySet().toArray(new String[0]); + int[] keyOffsets = new int[metadataKeys.length]; + int[] valueOffsets = new int[metadataKeys.length]; + + for (int i = 0; i < metadataKeys.length; i++) { + keyOffsets[i] = builder.createString(metadataKeys[i]); + valueOffsets[i] = builder.createString(mergedMetadata.get(metadataKeys[i])); + } + int metadataKeysOffset = TestRecordFB.createMetadataKeysVector(builder, keyOffsets); + int metadataValuesOffset = TestRecordFB.createMetadataValuesVector(builder, valueOffsets); + + // Use first record's extra data (or could merge both) + int[] extraDataOffsets = new int[first.extraDataLength()]; + for (int i = 0; i < first.extraDataLength(); i++) { + extraDataOffsets[i] = builder.createString(first.extraData(i)); + } + int extraDataOffset = TestRecordFB.createExtraDataVector(builder, extraDataOffsets); + + // Create the merged object + TestRecordFB.startTestRecordFB(builder); + TestRecordFB.addId(builder, id); + TestRecordFB.addName(builder, nameOffset); + TestRecordFB.addPrice(builder, price); + TestRecordFB.addActive(builder, active); + TestRecordFB.addCategory(builder, categoryOffset); + TestRecordFB.addTags(builder, tagsOffset); + TestRecordFB.addMetadataKeys(builder, metadataKeysOffset); + TestRecordFB.addMetadataValues(builder, metadataValuesOffset); + TestRecordFB.addExtraData(builder, extraDataOffset); + int recordOffset = TestRecordFB.endTestRecordFB(builder); + + builder.finish(recordOffset); + return builder.dataBuffer().slice(); + } + private TestRecord createTestRecord() { var record = new TestRecord(); record.id = 12345; @@ -289,20 +770,19 @@ var record = new TestRecord(); record.price = 99.99; record.active = true; record.category = "Electronics"; - + record.tags = Arrays.asList("popular", "trending", "bestseller"); - + record.metadata = new HashMap<>(); record.metadata.put("manufacturer", "TechCorp"); record.metadata.put("model", "TC-2024"); record.metadata.put("warranty", "2 years"); - - // Add extra data to create a larger record (fields 8-20) + record.extraData = new ArrayList<>(); for (int i = 0; i < 13; i++) { record.extraData.add("extraField" + i + "_value_" + (1000 + i)); } - + return record; } @@ -313,19 +793,18 @@ var record = new TestRecord(); record.price = 149.99; record.active = false; record.category = "Software"; - + record.tags = Arrays.asList("new", "premium"); - + record.metadata = new HashMap<>(); record.metadata.put("vendor", "SoftCorp"); record.metadata.put("version", "2.1"); - - // Add extra data to match the structure + record.extraData = new ArrayList<>(); for (int i = 0; i < 13; i++) { record.extraData.add("extraField" + i + "_value2_" + (2000 + i)); } - + return record; } @@ -339,7 +818,7 @@ public static class TestRecord { public List tags = new ArrayList<>(); public Map metadata = new HashMap<>(); public List extraData = new ArrayList<>(); // Fields 8-20 for large record test - + public TestRecord() {} // Required for deserialization } } \ No newline at end of file diff --git a/src/jmh/proto/test_record.proto b/src/jmh/proto/test_record.proto new file mode 100644 index 0000000..7a76f71 --- /dev/null +++ b/src/jmh/proto/test_record.proto @@ -0,0 +1,15 @@ +syntax = "proto3"; + +option java_package = "com.imprint.benchmark"; +option java_outer_classname = "TestRecordProto"; + +message TestRecord { + int32 id = 1; + string name = 2; + double price = 3; + bool active = 4; + string category = 5; + repeated string tags = 6; + map metadata = 7; + repeated string extra_data = 8; +} \ No newline at end of file diff --git a/src/main/java/com/imprint/core/ImprintRecord.java b/src/main/java/com/imprint/core/ImprintRecord.java index 5d4719f..2291550 100644 --- a/src/main/java/com/imprint/core/ImprintRecord.java +++ b/src/main/java/com/imprint/core/ImprintRecord.java @@ -34,7 +34,7 @@ public final class ImprintRecord { */ public ImprintRecord(Header header, List directory, ByteBuffer payload) { this.header = Objects.requireNonNull(header, "Header cannot be null"); - this.directory = List.copyOf(Objects.requireNonNull(directory, "Directory cannot be null")); + this.directory = Collections.unmodifiableList(Objects.requireNonNull(directory, "Directory cannot be null")); this.payload = payload.asReadOnlyBuffer(); // Zero-copy read-only view } @@ -79,12 +79,14 @@ private ByteBuffer getFieldBuffer(int fieldId) { int endOffset = (index + 1 < directory.size()) ? directory.get(index + 1).getOffset() : payload.limit(); - var fieldBuffer = payload.duplicate(); if (startOffset > payload.limit() || endOffset > payload.limit() || startOffset > endOffset) { return null; } + + // OPTIMIZATION: Single allocation instead of duplicate + slice + var fieldBuffer = payload.duplicate(); fieldBuffer.position(startOffset).limit(endOffset); - return fieldBuffer.slice(); + return fieldBuffer; } /** @@ -261,7 +263,7 @@ private static DirectoryEntry deserializeDirectoryEntry(ByteBuffer buffer) throw } private Value deserializeValue(TypeCode typeCode, ByteBuffer buffer) throws ImprintException { - ByteBuffer valueSpecificBuffer = buffer.duplicate(); + var valueSpecificBuffer = buffer.duplicate(); valueSpecificBuffer.order(ByteOrder.LITTLE_ENDIAN); switch (typeCode) { @@ -351,7 +353,7 @@ public double getFloat64(int fieldId) throws ImprintException { * @throws ImprintException if the field is not found, is null, or is not of type STRING. */ public String getString(int fieldId) throws ImprintException { - Value value = getValue(fieldId); + var value = getValue(fieldId); if (value == null) { throw new ImprintException(ErrorType.FIELD_NOT_FOUND, diff --git a/src/main/java/com/imprint/types/TypeHandler.java b/src/main/java/com/imprint/types/TypeHandler.java index be4fc7b..e58c355 100644 --- a/src/main/java/com/imprint/types/TypeHandler.java +++ b/src/main/java/com/imprint/types/TypeHandler.java @@ -433,26 +433,38 @@ public int estimateSize(Value value) throws ImprintException { } return arraySize; } - + @Override public ByteBuffer readValueBytes(ByteBuffer buffer) throws ImprintException { return readComplexValueBytes(buffer, "ARRAY", (tempBuffer, numElements) -> { if (tempBuffer.remaining() < 1) { - throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, - "Not enough bytes for ARRAY element type code in temp buffer during measurement."); + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, + "Not enough bytes for ARRAY element type code"); } byte elementTypeCodeByte = tempBuffer.get(); - int typeCodeLength = 1; + var elementType = TypeCode.fromByte(elementTypeCodeByte); - TypeHandler elementHandler = TypeCode.fromByte(elementTypeCodeByte).getHandler(); - int elementsDataLength = 0; - for (int i = 0; i < numElements; i++) { - int elementStartPos = tempBuffer.position(); - elementHandler.readValueBytes(tempBuffer); - elementsDataLength += (tempBuffer.position() - elementStartPos); + switch (elementType) { + case NULL: + return 1; + case BOOL: + return 1 + numElements; + case INT32: + case FLOAT32: + return 1 + (numElements * 4); + case INT64: + case FLOAT64: + return 1 + (numElements * 8); + default: + var elementHandler = elementType.getHandler(); + int elementsDataLength = 0; + for (int i = 0; i < numElements; i++) { + int elementStartPos = tempBuffer.position(); + elementHandler.readValueBytes(tempBuffer); + elementsDataLength += (tempBuffer.position() - elementStartPos); + } + return 1 + elementsDataLength; } - - return typeCodeLength + elementsDataLength; }); } }; @@ -549,28 +561,52 @@ public int estimateSize(Value value) throws ImprintException { } return mapSize; } - + @Override public ByteBuffer readValueBytes(ByteBuffer buffer) throws ImprintException { return readComplexValueBytes(buffer, "MAP", (tempBuffer, numEntries) -> { if (tempBuffer.remaining() < 2) { - throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, - "Not enough bytes for MAP key/value type codes in temp buffer during measurement."); + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, + "Not enough bytes for MAP key/value type codes"); } byte keyTypeCodeByte = tempBuffer.get(); byte valueTypeCodeByte = tempBuffer.get(); - int typeCodesLength = 2; - int entriesDataLength = 0; - for (int i = 0; i < numEntries; i++) { - int entryStartPos = tempBuffer.position(); - TypeCode.fromByte(keyTypeCodeByte).getHandler().readValueBytes(tempBuffer); - TypeCode.fromByte(valueTypeCodeByte).getHandler().readValueBytes(tempBuffer); - entriesDataLength += (tempBuffer.position() - entryStartPos); - } + TypeCode keyType = TypeCode.fromByte(keyTypeCodeByte); + TypeCode valueType = TypeCode.fromByte(valueTypeCodeByte); + + // OPTIMIZATION: Calculate sizes directly for fixed-size types + int keySize = getFixedTypeSize(keyType); + int valueSize = getFixedTypeSize(valueType); - return typeCodesLength + entriesDataLength; + if (keySize > 0 && valueSize > 0) { + // Both are fixed-size: O(1) calculation + return 2 + (numEntries * (keySize + valueSize)); + } else { + // At least one is variable-size: fall back to traversal + int entriesDataLength = 0; + for (int i = 0; i < numEntries; i++) { + int entryStartPos = tempBuffer.position(); + keyType.getHandler().readValueBytes(tempBuffer); + valueType.getHandler().readValueBytes(tempBuffer); + entriesDataLength += (tempBuffer.position() - entryStartPos); + } + return 2 + entriesDataLength; + } }); } + + private int getFixedTypeSize(TypeCode type) { + switch (type) { + case NULL: return 0; + case BOOL: return 1; + case INT32: + case FLOAT32: return 4; + case INT64: + case FLOAT64: return 8; + default: return -1; // Variable size + } + } + private void serializeMapKey(MapKey key, ByteBuffer buffer) throws ImprintException { switch (key.getTypeCode()) { diff --git a/src/main/java/com/imprint/types/Value.java b/src/main/java/com/imprint/types/Value.java index 7f3bbb9..ba747de 100644 --- a/src/main/java/com/imprint/types/Value.java +++ b/src/main/java/com/imprint/types/Value.java @@ -328,16 +328,19 @@ public String toString() { // String Value (ByteBuffer-based) public static class StringBufferValue extends Value { private final ByteBuffer value; - private volatile String cachedString; // lazy decode + private volatile String cachedString; + + private static final int THREAD_LOCAL_BUFFER_SIZE = 1024; + private static final ThreadLocal DECODE_BUFFER_CACHE = + ThreadLocal.withInitial(() -> new byte[THREAD_LOCAL_BUFFER_SIZE]); public StringBufferValue(ByteBuffer value) { - this.value = value.asReadOnlyBuffer(); // zero-copy read-only view + this.value = value.asReadOnlyBuffer(); } public String getValue() { String result = cachedString; if (result == null) { - // Simple, fast decoding - no thread-local overhead result = decodeUtf8(); cachedString = result; } @@ -345,18 +348,25 @@ public String getValue() { } private String decodeUtf8() { - // Fast path: zero-copy for array-backed ByteBuffers + final byte[] array; + final int offset; + final int length = value.remaining(); + if (value.hasArray()) { - return new String(value.array(), value.arrayOffset() + value.position(), - value.remaining(), StandardCharsets.UTF_8); + array = value.array(); + offset = value.arrayOffset() + value.position(); + } else { + byte[] threadLocalBuffer = DECODE_BUFFER_CACHE.get(); + if (length <= threadLocalBuffer.length) { + array = threadLocalBuffer; + } else { + // Fallback: copy bytes from the ByteBuffer to a new heap array (if too large for cache) + array = new byte[length]; + } + value.duplicate().get(array, 0, length); // Get bytes from current position into chosen array + offset = 0; } - - // Fallback path for non-array-backed ByteBuffers (e.g., direct buffers). - // Allocation is required here as Java's String(byte[],...) constructor needs a heap array. - // Data is copied from the ByteBuffer to a new byte array. - var array = new byte[value.remaining()]; - value.duplicate().get(array); - return new String(array, StandardCharsets.UTF_8); + return new String(array, offset, length, StandardCharsets.UTF_8); } public ByteBuffer getBuffer() { diff --git a/src/test/java/com/imprint/benchmark/ProfilerTest.java b/src/test/java/com/imprint/profile/ProfilerTest.java similarity index 97% rename from src/test/java/com/imprint/benchmark/ProfilerTest.java rename to src/test/java/com/imprint/profile/ProfilerTest.java index 5b531a9..d48c1aa 100644 --- a/src/test/java/com/imprint/benchmark/ProfilerTest.java +++ b/src/test/java/com/imprint/profile/ProfilerTest.java @@ -1,9 +1,10 @@ -package com.imprint.benchmark; +package com.imprint.profile; -import com.imprint.core.*; +import com.imprint.core.ImprintRecord; +import com.imprint.core.ImprintWriter; +import com.imprint.core.SchemaId; import com.imprint.types.Value; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.Disabled; import java.util.Random; @@ -25,11 +26,11 @@ * - String operations and UTF-8 encoding * - ByteBuffer operations */ -@Disabled("Enable manually for profiling") +//@Disabled("Enable manually for profiling") public class ProfilerTest { private static final int ITERATIONS = 1_000_000; - private static final int RECORD_SIZE = 20; + private static final int RECORD_SIZE = 50; @Test void profileFieldAccess() throws Exception { From f7a6e8e02cb2d907c412362e306da1774449b4e7 Mon Sep 17 00:00:00 2001 From: expand3d <> Date: Thu, 5 Jun 2025 14:59:54 -0400 Subject: [PATCH 24/49] Lazy load of directory and header data --- .../benchmark/ComparisonBenchmark.java | 68 ++- .../java/com/imprint/core/ImprintRecord.java | 405 +++++++++++++----- .../java/com/imprint/IntegrationTest.java | 68 ++- .../com/imprint/profile/ProfilerTest.java | 3 +- 4 files changed, 399 insertions(+), 145 deletions(-) diff --git a/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java b/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java index 49260b1..ce2fbcb 100644 --- a/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java +++ b/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java @@ -133,14 +133,34 @@ public void serializeFlatBuffers(Blackhole bh) { bh.consume(result); } - // ===== DESERIALIZATION BENCHMARKS ===== + // ===== PARTIAL DESERIALIZATION (SETUP ONLY) ===== +// These benchmarks measure the cost of preparing a record for field access, +// not the cost of accessing the actual data. This is important because +// +// 1. Imprint: Only parses header + stores raw directory bytes +// 2. FlatBuffers: Only wraps the buffer with minimal validation +// 3. Others (eager): Parse and construct all field objects upfront +// +// This comparison shows the advantage of lazy loading approaches when you +// only need to access a subset of fields. In real streaming workloads, +// records are often filtered/routed based on just a few key fields. +// +// For a fair "full deserialization" comparison, see FULL DESERIALIZATION BENCHMARKS. + + @Benchmark + public void deserializeSetupImprint(Blackhole bh) throws Exception { + ImprintRecord result = ImprintRecord.deserialize(imprintBytesBuffer.duplicate()); + bh.consume(result); + } @Benchmark - public void deserializeImprint(Blackhole bh) throws Exception { - ImprintRecord result = ImprintRecord.deserialize(imprintBytesBuffer.duplicate()); + public void deserializeSetupFlatBuffers(Blackhole bh) { + TestRecordFB result = TestRecordFB.getRootAsTestRecordFB(flatbuffersBytes.duplicate()); bh.consume(result); } + // ===== FULL DESERIALIZATION BENCHMARKS ===== + @Benchmark public void deserializeJacksonJson(Blackhole bh) throws Exception { TestRecord result = jacksonJsonMapper.readValue(jacksonJsonBytes, TestRecord.class); @@ -173,15 +193,53 @@ public void deserializeProtobuf(Blackhole bh) throws Exception { bh.consume(result); } + @Benchmark + public void deserializeImprint(Blackhole bh) throws Exception { + ImprintRecord result = ImprintRecord.deserialize(imprintBytesBuffer.duplicate()); + // Access all fields to force full deserialization + result.getInt32(1); // id + result.getString(2); // name + result.getFloat64(3); // price + result.getBoolean(4); // active + result.getString(5); // category + result.getArray(6); // tags + result.getMap(7); // metadata + for (int i = 8; i < 21; i++) { + result.getString(i); // extraData fields + } + + bh.consume(result); + } + @Benchmark public void deserializeFlatBuffers(Blackhole bh) { TestRecordFB result = TestRecordFB.getRootAsTestRecordFB(flatbuffersBytes.duplicate()); + + // Access all fields + result.id(); + result.name(); + result.price(); + result.active(); + result.category(); + // Access all tags + for (int i = 0; i < result.tagsLength(); i++) { + result.tags(i); + } + // Access all metadata + for (int i = 0; i < result.metadataKeysLength(); i++) { + result.metadataKeys(i); + result.metadataValues(i); + } + // Access all extra data + for (int i = 0; i < result.extraDataLength(); i++) { + result.extraData(i); + } + bh.consume(result); } // ===== FIELD ACCESS BENCHMARKS ===== - // Tests accessing a single field near the end of a large record - // This showcases Imprint's O(1) directory lookup vs sequential deserialization + // Tests accessing a single field near the end of a record @Benchmark public void singleFieldAccessImprint(Blackhole bh) throws Exception { diff --git a/src/main/java/com/imprint/core/ImprintRecord.java b/src/main/java/com/imprint/core/ImprintRecord.java index 2291550..da6b6e0 100644 --- a/src/main/java/com/imprint/core/ImprintRecord.java +++ b/src/main/java/com/imprint/core/ImprintRecord.java @@ -17,6 +17,10 @@ * An Imprint record containing a header, field directory, and payload. * Uses ByteBuffer for zero-copy operations to achieve low latency. * + *

This implementation uses lazy directory parsing for optimal single field access performance. + * The directory is only parsed when needed, and binary search is performed directly on raw bytes + * when possible.

+ * *

Performance Note: All ByteBuffers should be array-backed * (hasArray() == true) for optimal zero-copy performance. Direct buffers * may cause performance degradation.

@@ -24,37 +28,64 @@ @Getter public final class ImprintRecord { private final Header header; - private final List directory; + private final ByteBuffer directoryBuffer; // Raw directory bytes private final ByteBuffer payload; // Read-only view for zero-copy + // Lazy-loaded directory state + private List parsedDirectory; + private boolean directoryParsed = false; + + // Cache for parsed directory count to avoid repeated VarInt decoding + private int directoryCount = -1; + /** - * Creates a new ImprintRecord. + * Creates a new ImprintRecord with lazy directory parsing. * + * @param header the record header + * @param directoryBuffer raw directory bytes (including count) * @param payload the payload buffer. Should be array-backed for optimal performance. */ - public ImprintRecord(Header header, List directory, ByteBuffer payload) { + private ImprintRecord(Header header, ByteBuffer directoryBuffer, ByteBuffer payload) { this.header = Objects.requireNonNull(header, "Header cannot be null"); - this.directory = Collections.unmodifiableList(Objects.requireNonNull(directory, "Directory cannot be null")); + this.directoryBuffer = directoryBuffer.asReadOnlyBuffer(); this.payload = payload.asReadOnlyBuffer(); // Zero-copy read-only view } + /** + * Creates a new ImprintRecord with pre-parsed directory (used by ImprintWriter). + * This constructor is used when the directory is already known and parsed. + * + * @param header the record header + * @param directory the parsed directory entries + * @param payload the payload buffer. Should be array-backed for optimal performance. + */ + ImprintRecord(Header header, List directory, ByteBuffer payload) { + this.header = Objects.requireNonNull(header, "Header cannot be null"); + this.parsedDirectory = Collections.unmodifiableList(Objects.requireNonNull(directory, "Directory cannot be null")); + this.directoryParsed = true; + this.directoryCount = directory.size(); + this.payload = payload.asReadOnlyBuffer(); + + // Create directory buffer for serialization compatibility + this.directoryBuffer = createDirectoryBuffer(directory); + } + /** * Get a value by field ID, deserializing it on demand. * Returns null if the field is not found. * Note: If the field exists and is an explicit NULL type, this will return Value.NullValue.INSTANCE + * + *

Performance Note: Accessing fields one-by-one is optimized for single field access. + * If you need to access many fields from the same record, consider calling getDirectory() first + * to parse the full directory once, then access fields normally.

*/ public Value getValue(int fieldId) throws ImprintException { - var fieldBuffer = getFieldBuffer(fieldId); - if (fieldBuffer == null) { + DirectoryEntry entry = findDirectoryEntry(fieldId); + if (entry == null) { return null; } - int directoryIndex = findDirectoryIndex(fieldId); - if (directoryIndex < 0) { - throw new ImprintException(ErrorType.INTERNAL_ERROR, "Field ID " + fieldId + " found buffer but not in directory."); - } - var entry = directory.get(directoryIndex); - return deserializeValue(entry.getTypeCode(), fieldBuffer); + return deserializeValue(entry.getTypeCode(), getFieldBufferFromEntry(entry)); } /** @@ -62,38 +93,237 @@ public Value getValue(int fieldId) throws ImprintException { * Returns a zero-copy ByteBuffer view, or null if field not found. */ public ByteBuffer getRawBytes(int fieldId) { - var fieldBuffer = getFieldBuffer(fieldId); - return fieldBuffer != null ? fieldBuffer.asReadOnlyBuffer() : null; + try { + DirectoryEntry entry = findDirectoryEntry(fieldId); + if (entry == null) { + return null; + } + + return getFieldBufferFromEntry(entry).asReadOnlyBuffer(); + } catch (ImprintException e) { + return null; + } } /** - * Get a ByteBuffer view of a field's data. - * Returns null if the field is not found. + * Find a directory entry for the given field ID. + * Uses the most efficient method based on current state. */ - private ByteBuffer getFieldBuffer(int fieldId) { - int index = findDirectoryIndex(fieldId); - if (index < 0) return null; + private DirectoryEntry findDirectoryEntry(int fieldId) throws ImprintException { + if (directoryParsed) { + // Use parsed directory + int index = findDirectoryIndexInParsed(fieldId); + return index >= 0 ? parsedDirectory.get(index) : null; + } else { + // Use fast binary search on raw bytes + return findFieldEntryFast(fieldId); + } + } - var entry = directory.get(index); - int startOffset = entry.getOffset(); - int endOffset = (index + 1 < directory.size()) ? - directory.get(index + 1).getOffset() : payload.limit(); + /** + * Fast binary search directly on raw directory bytes. + * This avoids parsing the entire directory for single field access. + */ + private DirectoryEntry findFieldEntryFast(int fieldId) throws ImprintException { + ByteBuffer searchBuffer = directoryBuffer.duplicate(); + searchBuffer.order(ByteOrder.LITTLE_ENDIAN); + + // Decode directory count (cache it to avoid repeated decoding) + if (directoryCount < 0) { + directoryCount = VarInt.decode(searchBuffer).getValue(); + } else { + // Skip past the VarInt count + VarInt.decode(searchBuffer); + } - if (startOffset > payload.limit() || endOffset > payload.limit() || startOffset > endOffset) { + if (directoryCount == 0) { return null; } - // OPTIMIZATION: Single allocation instead of duplicate + slice + // Now searchBuffer.position() points to the first directory entry + int directoryStartPos = searchBuffer.position(); + + int low = 0; + int high = directoryCount - 1; + + while (low <= high) { + int mid = (low + high) >>> 1; + + // Calculate position of mid entry + int entryPos = directoryStartPos + (mid * Constants.DIR_ENTRY_BYTES); + + // Bounds check + if (entryPos + Constants.DIR_ENTRY_BYTES > searchBuffer.limit()) { + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, + "Directory entry at position " + entryPos + " exceeds buffer limit " + searchBuffer.limit()); + } + + searchBuffer.position(entryPos); + short midFieldId = searchBuffer.getShort(); + + if (midFieldId < fieldId) { + low = mid + 1; + } else if (midFieldId > fieldId) { + high = mid - 1; + } else { + // Found it - read the complete entry + searchBuffer.position(entryPos); + return deserializeDirectoryEntry(searchBuffer); + } + } + + return null; // Not found + } + + /** + * Get the directory (parsing it if necessary). + * This maintains backward compatibility with existing code. + * + *

Performance Tip: If you plan to access many fields from this record, + * call this method first to parse the directory once, then use the field accessor methods. + * This is more efficient than accessing fields one-by-one when you need multiple fields.

+ */ + public List getDirectory() { + ensureDirectoryParsed(); + return parsedDirectory; + } + + /** + * Get a ByteBuffer view of a field's data from a DirectoryEntry. + */ + private ByteBuffer getFieldBufferFromEntry(DirectoryEntry entry) throws ImprintException { + int startOffset = entry.getOffset(); + + // Find end offset + int endOffset; + if (directoryParsed) { + // Use parsed directory to find next entry + int entryIndex = findDirectoryIndexInParsed(entry.getId()); + endOffset = (entryIndex + 1 < parsedDirectory.size()) ? + parsedDirectory.get(entryIndex + 1).getOffset() : payload.limit(); + } else { + // Calculate end offset by finding the next field in the directory + endOffset = findNextOffsetInRawDirectory(entry.getId()); + } + + if (startOffset < 0 || endOffset < 0 || startOffset > payload.limit() || + endOffset > payload.limit() || startOffset > endOffset) { + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, + "Invalid field buffer range: start=" + startOffset + ", end=" + endOffset + + ", payloadLimit=" + payload.limit()); + } + var fieldBuffer = payload.duplicate(); fieldBuffer.position(startOffset).limit(endOffset); return fieldBuffer; } + /** + * Find the next field's offset by scanning the raw directory. + * This is used when the directory isn't fully parsed yet. + */ + private int findNextOffsetInRawDirectory(int currentFieldId) throws ImprintException { + ByteBuffer scanBuffer = directoryBuffer.duplicate(); + scanBuffer.order(ByteOrder.LITTLE_ENDIAN); + + // Get directory count + int count = (directoryCount >= 0) ? directoryCount : VarInt.decode(scanBuffer).getValue(); + if (count == 0) { + return payload.limit(); + } + + // Skip past count if we just decoded it + if (directoryCount < 0) { + // VarInt.decode already advanced the position + } else { + VarInt.decode(scanBuffer); // Skip past the count + } + + int directoryStartPos = scanBuffer.position(); + + for (int i = 0; i < count; i++) { + int entryPos = directoryStartPos + (i * Constants.DIR_ENTRY_BYTES); + + // Bounds check + if (entryPos + Constants.DIR_ENTRY_BYTES > scanBuffer.limit()) { + return payload.limit(); + } + + scanBuffer.position(entryPos); + short fieldId = scanBuffer.getShort(); + scanBuffer.get(); // skip type + int offset = scanBuffer.getInt(); + + if (fieldId > currentFieldId) { + return offset; // Found next field's offset + } + } + + return payload.limit(); // No next field, use payload end + } + + /** + * Ensure the directory is fully parsed (thread-safe). + */ + private synchronized void ensureDirectoryParsed() { + if (directoryParsed) { + return; + } + + try { + ByteBuffer parseBuffer = directoryBuffer.duplicate(); + parseBuffer.order(ByteOrder.LITTLE_ENDIAN); + + VarInt.DecodeResult countResult = VarInt.decode(parseBuffer); + int count = countResult.getValue(); + this.directoryCount = count; // Cache the count + + List directory = new ArrayList<>(count); + for (int i = 0; i < count; i++) { + directory.add(deserializeDirectoryEntry(parseBuffer)); + } + + this.parsedDirectory = Collections.unmodifiableList(directory); + this.directoryParsed = true; + } catch (ImprintException e) { + throw new RuntimeException("Failed to parse directory", e); + } + } + + /** + * Creates a directory buffer from parsed directory entries. + * This is used when creating records with pre-parsed directories (e.g., from ImprintWriter). + */ + private ByteBuffer createDirectoryBuffer(List directory) { + try { + int bufferSize = VarInt.encodedLength(directory.size()) + (directory.size() * Constants.DIR_ENTRY_BYTES); + ByteBuffer buffer = ByteBuffer.allocate(bufferSize); + buffer.order(ByteOrder.LITTLE_ENDIAN); + + // Write directory count + VarInt.encode(directory.size(), buffer); + + // Write directory entries + for (DirectoryEntry entry : directory) { + serializeDirectoryEntry(entry, buffer); + } + + buffer.flip(); + return buffer.asReadOnlyBuffer(); + } catch (Exception e) { + // Fallback to empty buffer if creation fails + return ByteBuffer.allocate(0).asReadOnlyBuffer(); + } + } + /** * Serialize this record to a ByteBuffer. * The returned buffer will be array-backed. */ public ByteBuffer serializeToBuffer() { + // Ensure directory is parsed for serialization + ensureDirectoryParsed(); + var buffer = ByteBuffer.allocate(estimateSerializedSize()); buffer.order(ByteOrder.LITTLE_ENDIAN); @@ -101,8 +331,8 @@ public ByteBuffer serializeToBuffer() { serializeHeader(buffer); // Write directory (always present) - VarInt.encode(directory.size(), buffer); - for (var entry : directory) { + VarInt.encode(parsedDirectory.size(), buffer); + for (var entry : parsedDirectory) { serializeDirectoryEntry(entry, buffer); } @@ -117,9 +347,6 @@ public ByteBuffer serializeToBuffer() { /** * Create a fluent builder for constructing ImprintRecord instances. - * - * @param schemaId the schema identifier for this record - * @return a new builder instance */ public static ImprintRecordBuilder builder(SchemaId schemaId) { return new ImprintRecordBuilder(schemaId); @@ -127,10 +354,6 @@ public static ImprintRecordBuilder builder(SchemaId schemaId) { /** * Create a fluent builder for constructing ImprintRecord instances. - * - * @param fieldspaceId the fieldspace identifier - * @param schemaHash the schema hash - * @return a new builder instance */ @SuppressWarnings("unused") public static ImprintRecordBuilder builder(int fieldspaceId, int schemaHash) { @@ -145,7 +368,7 @@ public static ImprintRecord deserialize(byte[] bytes) throws ImprintException { } /** - * Deserialize a record from a ByteBuffer. + * Deserialize a record from a ByteBuffer with lazy directory parsing. * * @param buffer the buffer to deserialize from. Must be array-backed * (buffer.hasArray() == true) for optimal zero-copy performance. @@ -156,37 +379,43 @@ public static ImprintRecord deserialize(ByteBuffer buffer) throws ImprintExcepti // Read header var header = deserializeHeader(buffer); - // Read directory (always present) - var directory = new ArrayList(); + // Read directory count but don't parse entries yet + int directoryStartPos = buffer.position(); VarInt.DecodeResult countResult = VarInt.decode(buffer); int directoryCount = countResult.getValue(); - for (int i = 0; i < directoryCount; i++) { - directory.add(deserializeDirectoryEntry(buffer)); - } + // Calculate directory buffer (includes count + all entries) + int directorySize = countResult.getBytesRead() + (directoryCount * Constants.DIR_ENTRY_BYTES); + buffer.position(directoryStartPos); // Reset to include count in directory buffer + + var directoryBuffer = buffer.slice(); + directoryBuffer.limit(directorySize); + + // Advance buffer past directory + buffer.position(buffer.position() + directorySize); // Read payload as ByteBuffer slice for zero-copy var payload = buffer.slice(); payload.limit(header.getPayloadSize()); - buffer.position(buffer.position() + header.getPayloadSize()); - return new ImprintRecord(header, directory, payload); + return new ImprintRecord(header, directoryBuffer, payload); } /** - * Binary search for field ID in directory without object allocation. + * Binary search for field ID in parsed directory. * Returns the index of the field if found, or a negative value if not found. - * - * @param fieldId the field ID to search for - * @return index if found, or negative insertion point - 1 if not found */ - private int findDirectoryIndex(int fieldId) { + private int findDirectoryIndexInParsed(int fieldId) { + if (!directoryParsed) { + return -1; + } + int low = 0; - int high = directory.size() - 1; + int high = parsedDirectory.size() - 1; while (low <= high) { - int mid = (low + high) >>> 1; // unsigned right shift to avoid overflow - int midFieldId = directory.get(mid).getId(); + int mid = (low + high) >>> 1; + int midFieldId = parsedDirectory.get(mid).getId(); if (midFieldId < fieldId) { low = mid + 1; @@ -201,12 +430,30 @@ private int findDirectoryIndex(int fieldId) { public int estimateSerializedSize() { int size = Constants.HEADER_BYTES; // header - size += VarInt.encodedLength(directory.size()); // directory count - size += directory.size() * Constants.DIR_ENTRY_BYTES; // directory entries + size += VarInt.encodedLength(getDirectoryCount()); // directory count + size += getDirectoryCount() * Constants.DIR_ENTRY_BYTES; // directory entries size += payload.remaining(); // payload return size; } + private int getDirectoryCount() { + if (directoryCount >= 0) { + return directoryCount; + } + if (directoryParsed) { + return parsedDirectory.size(); + } + // Last resort: decode from buffer + try { + ByteBuffer countBuffer = directoryBuffer.duplicate(); + return VarInt.decode(countBuffer).getValue(); + } catch (Exception e) { + return 0; + } + } + + // ===== EXISTING HELPER METHODS (unchanged) ===== + private void serializeHeader(ByteBuffer buffer) { buffer.put(Constants.MAGIC); buffer.put(Constants.VERSION); @@ -287,6 +534,8 @@ private Value deserializeValue(TypeCode typeCode, ByteBuffer buffer) throws Impr } } + // ===== TYPE-SPECIFIC GETTERS (unchanged API, improved performance) ===== + private T getTypedValueOrThrow(int fieldId, TypeCode expectedTypeCode, Class expectedValueClass, String expectedTypeName) throws ImprintException { var value = getValue(fieldId); @@ -308,50 +557,26 @@ private T getTypedValueOrThrow(int fieldId, TypeCode expectedT "Field " + fieldId + " is of type " + value.getTypeCode() + ", expected " + expectedTypeName + "."); } - /** - * Retrieves the boolean value for the given field ID. - * @throws ImprintException if the field is not found, is null, or is not of type BOOL. - */ public boolean getBoolean(int fieldId) throws ImprintException { return getTypedValueOrThrow(fieldId, TypeCode.BOOL, Value.BoolValue.class, "boolean").getValue(); } - /** - * Retrieves the int (int32) value for the given field ID. - * @throws ImprintException if the field is not found, is null, or is not of type INT32. - */ public int getInt32(int fieldId) throws ImprintException { return getTypedValueOrThrow(fieldId, TypeCode.INT32, Value.Int32Value.class, "int32").getValue(); } - /** - * Retrieves the long (int64) value for the given field ID. - * @throws ImprintException if the field is not found, is null, or is not of type INT64. - */ public long getInt64(int fieldId) throws ImprintException { return getTypedValueOrThrow(fieldId, TypeCode.INT64, Value.Int64Value.class, "int64").getValue(); } - /** - * Retrieves the float (float32) value for the given field ID. - * @throws ImprintException if the field is not found, is null, or is not of type FLOAT32. - */ public float getFloat32(int fieldId) throws ImprintException { return getTypedValueOrThrow(fieldId, TypeCode.FLOAT32, Value.Float32Value.class, "float32").getValue(); } - /** - * Retrieves the double (float64) value for the given field ID. - * @throws ImprintException if the field is not found, is null, or is not of type FLOAT64. - */ public double getFloat64(int fieldId) throws ImprintException { return getTypedValueOrThrow(fieldId, TypeCode.FLOAT64, Value.Float64Value.class, "float64").getValue(); } - /** - * Retrieves the String value for the given field ID. - * @throws ImprintException if the field is not found, is null, or is not of type STRING. - */ public String getString(int fieldId) throws ImprintException { var value = getValue(fieldId); @@ -375,11 +600,6 @@ public String getString(int fieldId) throws ImprintException { "Field " + fieldId + " is of type " + value.getTypeCode() + ", expected STRING."); } - /** - * Retrieves the byte array (byte[]) value for the given field ID. - * Note: This may involve a defensive copy depending on the underlying Value type. - * @throws ImprintException if the field is not found, is null, or is not of type BYTES. - */ public byte[] getBytes(int fieldId) throws ImprintException { Value value = getValue(fieldId); @@ -393,46 +613,31 @@ public byte[] getBytes(int fieldId) throws ImprintException { } if (value instanceof Value.BytesValue) { - return ((Value.BytesValue) value).getValue(); // getValue() in BytesValue returns a clone + return ((Value.BytesValue) value).getValue(); } if (value instanceof Value.BytesBufferValue) { - return ((Value.BytesBufferValue) value).getValue(); // getValue() in BytesBufferValue creates a new array + return ((Value.BytesBufferValue) value).getValue(); } throw new ImprintException(ErrorType.TYPE_MISMATCH, "Field " + fieldId + " is of type " + value.getTypeCode() + ", expected BYTES."); } - /** - * Retrieves the List for the given field ID. - * The list itself is a copy; modifications to it will not affect the record. - * @throws ImprintException if the field is not found, is null, or is not of type ARRAY. - */ public List getArray(int fieldId) throws ImprintException { return getTypedValueOrThrow(fieldId, TypeCode.ARRAY, Value.ArrayValue.class, "ARRAY").getValue(); } - /** - * Retrieves the Map for the given field ID. - * The map itself is a copy; modifications to it will not affect the record. - * @throws ImprintException if the field is not found, is null, or is not of type MAP. - */ public Map getMap(int fieldId) throws ImprintException { return getTypedValueOrThrow(fieldId, TypeCode.MAP, Value.MapValue.class, "MAP").getValue(); } - /** - * Retrieves the nested ImprintRecord for the given field ID. - * @throws ImprintException if the field is not found, is null, or is not of type ROW. - */ public ImprintRecord getRow(int fieldId) throws ImprintException { return getTypedValueOrThrow(fieldId, TypeCode.ROW, Value.RowValue.class, "ROW").getValue(); } @Override public String toString() { - return String.format("ImprintRecord{header=%s, directorySize=%d, payloadSize=%d}", - header, directory.size(), payload.remaining()); + return String.format("ImprintRecord{header=%s, directorySize=%d, payloadSize=%d, directoryParsed=%s}", + header, getDirectoryCount(), payload.remaining(), directoryParsed); } - } \ No newline at end of file diff --git a/src/test/java/com/imprint/IntegrationTest.java b/src/test/java/com/imprint/IntegrationTest.java index 76efcc5..898adfb 100644 --- a/src/test/java/com/imprint/IntegrationTest.java +++ b/src/test/java/com/imprint/IntegrationTest.java @@ -16,16 +16,11 @@ */ public class IntegrationTest { - // Removed main method, individual methods are now JUnit tests. - @Test @DisplayName("Basic functionality: create, serialize, deserialize primitive types") void testBasicFunctionality() throws ImprintException { - System.out.println("Testing basic functionality..."); // Keep for now if desired, or remove - SchemaId schemaId = new SchemaId(1, 0xdeadbeef); - // Using ImprintRecordBuilder for consistency with other tests - ImprintRecord record = ImprintRecord.builder(schemaId) + var record = ImprintRecord.builder(schemaId) .field(1, 42) .field(2, "testing java imprint spec") .field(3, true) @@ -33,7 +28,7 @@ void testBasicFunctionality() throws ImprintException { .field(5, new byte[]{1, 2, 3, 4}) .build(); - // Verify we can read values back using ergonomic getters + // Verify we can read values back using type getters assertEquals(42, record.getInt32(1)); assertEquals("testing java imprint spec", record.getString(2)); assertTrue(record.getBoolean(3)); @@ -47,7 +42,7 @@ void testBasicFunctionality() throws ImprintException { var buffer = record.serializeToBuffer(); byte[] serialized = new byte[buffer.remaining()]; buffer.get(serialized); - ImprintRecord deserialized = ImprintRecord.deserialize(serialized); + var deserialized = ImprintRecord.deserialize(serialized); assertEquals(42, deserialized.getInt32(1)); assertEquals("testing java imprint spec", deserialized.getString(2)); @@ -61,8 +56,6 @@ void testBasicFunctionality() throws ImprintException { @Test @DisplayName("Collections: create, serialize, deserialize arrays and maps") void testArraysAndMaps() throws ImprintException { - System.out.println("Testing arrays and maps..."); - SchemaId schemaId = new SchemaId(2, 0xcafebabe); // Create an array using builder for convenience @@ -72,8 +65,7 @@ void testArraysAndMaps() throws ImprintException { Map sourceStringToIntMap = new HashMap<>(); sourceStringToIntMap.put("one", 1); sourceStringToIntMap.put("two", 2); - - ImprintRecord record = ImprintRecord.builder(schemaId) + var record = ImprintRecord.builder(schemaId) .field(1, sourceIntList) // Builder converts List to List .field(2, sourceStringToIntMap) // Builder converts Map .build(); @@ -107,14 +99,14 @@ void testArraysAndMaps() throws ImprintException { void testNestedRecords() throws ImprintException { System.out.println("Testing nested records..."); - SchemaId innerSchemaId = new SchemaId(3, 0x12345678); - ImprintRecord innerRecord = ImprintRecord.builder(innerSchemaId) + var innerSchemaId = new SchemaId(3, 0x12345678); + var innerRecord = ImprintRecord.builder(innerSchemaId) .field(1, "nested data") .field(2, 9876543210L) .build(); - SchemaId outerSchemaId = new SchemaId(4, 0x87654321); - ImprintRecord outerRecord = ImprintRecord.builder(outerSchemaId) + var outerSchemaId = new SchemaId(4, 0x87654321); + var outerRecord = ImprintRecord.builder(outerSchemaId) .field(1, innerRecord) // Builder handles ImprintRecord directly .field(2, "outer data") .build(); @@ -122,12 +114,12 @@ void testNestedRecords() throws ImprintException { var buffer = outerRecord.serializeToBuffer(); byte[] serialized = new byte[buffer.remaining()]; buffer.get(serialized); - ImprintRecord deserialized = ImprintRecord.deserialize(serialized); + var deserialized = ImprintRecord.deserialize(serialized); assertEquals(4, deserialized.getHeader().getSchemaId().getFieldSpaceId()); assertEquals("outer data", deserialized.getString(2)); - ImprintRecord nestedDeserialized = deserialized.getRow(1); + var nestedDeserialized = deserialized.getRow(1); assertNotNull(nestedDeserialized); assertEquals(3, nestedDeserialized.getHeader().getSchemaId().getFieldSpaceId()); assertEquals("nested data", nestedDeserialized.getString(1)); @@ -136,8 +128,6 @@ void testNestedRecords() throws ImprintException { System.out.println("✓ Nested records test passed"); } - // --- Start of broken down tests for ErgonomicGettersAndNestedTypes --- - private ImprintRecord createTestRecordForGetters() throws ImprintException { SchemaId schemaId = new SchemaId(5, 0xabcdef01); @@ -174,8 +164,8 @@ private ImprintRecord serializeAndDeserialize(ImprintRecord record) throws Impri @Test @DisplayName("Type Getters: Basic primitive and String types") void testBasicTypeGetters() throws ImprintException { - ImprintRecord originalRecord = createTestRecordForGetters(); - ImprintRecord record = serializeAndDeserialize(originalRecord); + var originalRecord = createTestRecordForGetters(); + var record = serializeAndDeserialize(originalRecord); assertTrue(record.getBoolean(1)); assertEquals(12345, record.getInt32(2)); @@ -189,8 +179,8 @@ void testBasicTypeGetters() throws ImprintException { @Test @DisplayName("Type Getters: Array of Arrays") void testTypeGetterArrayOfArrays() throws ImprintException { - ImprintRecord originalRecord = createTestRecordForGetters(); - ImprintRecord record = serializeAndDeserialize(originalRecord); + var originalRecord = createTestRecordForGetters(); + var record = serializeAndDeserialize(originalRecord); List arrOfArr = record.getArray(9); assertNotNull(arrOfArr); @@ -211,8 +201,8 @@ void testTypeGetterArrayOfArrays() throws ImprintException { @Test @DisplayName("Type Getters: Map with Array Value") void testTypeGetterMapWithArrayValue() throws ImprintException { - ImprintRecord originalRecord = createTestRecordForGetters(); - ImprintRecord record = serializeAndDeserialize(originalRecord); + var originalRecord = createTestRecordForGetters(); + var record = serializeAndDeserialize(originalRecord); Map mapWithArr = record.getMap(10); assertNotNull(mapWithArr); @@ -227,8 +217,8 @@ void testTypeGetterMapWithArrayValue() throws ImprintException { @Test @DisplayName("Type Getters: Empty Collections (Array and Map)") void testErgonomicGettersEmptyCollections() throws ImprintException { - ImprintRecord originalRecord = createTestRecordForGetters(); - ImprintRecord record = serializeAndDeserialize(originalRecord); + var originalRecord = createTestRecordForGetters(); + var record = serializeAndDeserialize(originalRecord); List emptyArr = record.getArray(11); assertNotNull(emptyArr); @@ -242,8 +232,8 @@ void testErgonomicGettersEmptyCollections() throws ImprintException { @Test @DisplayName("Type Getters: Exception for Field Not Found") void testErgonomicGetterExceptionFieldNotFound() throws ImprintException { - ImprintRecord originalRecord = createTestRecordForGetters(); - ImprintRecord record = serializeAndDeserialize(originalRecord); + var originalRecord = createTestRecordForGetters(); + var record = serializeAndDeserialize(originalRecord); ImprintException ex = assertThrows(ImprintException.class, () -> record.getInt32(99)); assertEquals(ErrorType.FIELD_NOT_FOUND, ex.getErrorType()); @@ -252,8 +242,8 @@ void testErgonomicGetterExceptionFieldNotFound() throws ImprintException { @Test @DisplayName("Type Getters: Exception for Null Field accessed as primitive") void testErgonomicGetterExceptionNullField() throws ImprintException { - ImprintRecord originalRecord = createTestRecordForGetters(); - ImprintRecord record = serializeAndDeserialize(originalRecord); + var originalRecord = createTestRecordForGetters(); + var record = serializeAndDeserialize(originalRecord); ImprintException ex = assertThrows(ImprintException.class, () -> record.getString(8)); assertEquals(ErrorType.TYPE_MISMATCH, ex.getErrorType()); // getString throws TYPE_MISMATCH for null @@ -269,8 +259,8 @@ void testErgonomicGetterExceptionNullField() throws ImprintException { @Test @DisplayName("Type Getters: Exception for Type Mismatch") void testErgonomicGetterExceptionTypeMismatch() throws ImprintException { - ImprintRecord originalRecord = createTestRecordForGetters(); - ImprintRecord record = serializeAndDeserialize(originalRecord); + var originalRecord = createTestRecordForGetters(); + var record = serializeAndDeserialize(originalRecord); ImprintException ex = assertThrows(ImprintException.class, () -> record.getInt32(6)); // Field 6 is a String assertEquals(ErrorType.TYPE_MISMATCH, ex.getErrorType()); @@ -279,20 +269,20 @@ void testErgonomicGetterExceptionTypeMismatch() throws ImprintException { @Test @DisplayName("Type Getters: Row (Nested Record)") void testErgonomicGetterRow() throws ImprintException { - SchemaId innerSchemaId = new SchemaId(6, 0x12345678); - ImprintRecord innerRecord = ImprintRecord.builder(innerSchemaId) + var innerSchemaId = new SchemaId(6, 0x12345678); + var innerRecord = ImprintRecord.builder(innerSchemaId) .field(101, "nested string") .field(102, 999L) .build(); - ImprintRecord recordWithRow = ImprintRecord.builder(new SchemaId(7, 0x87654321)) + var recordWithRow = ImprintRecord.builder(new SchemaId(7, 0x87654321)) .field(201, innerRecord) // Using builder to add row .field(202, "outer field") .build(); - ImprintRecord deserializedWithRow = serializeAndDeserialize(recordWithRow); + var deserializedWithRow = serializeAndDeserialize(recordWithRow); - ImprintRecord retrievedRow = deserializedWithRow.getRow(201); + var retrievedRow = deserializedWithRow.getRow(201); assertNotNull(retrievedRow); assertEquals(innerSchemaId, retrievedRow.getHeader().getSchemaId()); assertEquals("nested string", retrievedRow.getString(101)); diff --git a/src/test/java/com/imprint/profile/ProfilerTest.java b/src/test/java/com/imprint/profile/ProfilerTest.java index d48c1aa..3b9f371 100644 --- a/src/test/java/com/imprint/profile/ProfilerTest.java +++ b/src/test/java/com/imprint/profile/ProfilerTest.java @@ -4,6 +4,7 @@ import com.imprint.core.ImprintWriter; import com.imprint.core.SchemaId; import com.imprint.types.Value; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import java.util.Random; @@ -26,7 +27,7 @@ * - String operations and UTF-8 encoding * - ByteBuffer operations */ -//@Disabled("Enable manually for profiling") +@Disabled("Enable manually for profiling") public class ProfilerTest { private static final int ITERATIONS = 1_000_000; From 83ed96192fa421f7cf51815a5c43d730de7727e4 Mon Sep 17 00:00:00 2001 From: expand3d <> Date: Thu, 5 Jun 2025 15:13:59 -0400 Subject: [PATCH 25/49] minor cleanup --- .../benchmark/ComparisonBenchmark.java | 21 ++++++++----------- .../java/com/imprint/types/TypeHandler.java | 10 ++++----- src/main/java/com/imprint/types/Value.java | 16 +++++++------- 3 files changed, 21 insertions(+), 26 deletions(-) diff --git a/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java b/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java index 49260b1..a7ffd3c 100644 --- a/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java +++ b/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java @@ -10,6 +10,7 @@ import com.imprint.core.SchemaId; import com.imprint.types.MapKey; import com.imprint.types.Value; +import lombok.NoArgsConstructor; import org.apache.avro.Schema; import org.apache.avro.generic.GenericData; import org.apache.avro.generic.GenericDatumReader; @@ -90,7 +91,6 @@ public void setup() throws Exception { } // ===== SERIALIZATION BENCHMARKS ===== - @Benchmark public void serializeImprint(Blackhole bh) throws Exception { ByteBuffer result = serializeWithImprint(testData); @@ -134,7 +134,6 @@ public void serializeFlatBuffers(Blackhole bh) { } // ===== DESERIALIZATION BENCHMARKS ===== - @Benchmark public void deserializeImprint(Blackhole bh) throws Exception { ImprintRecord result = ImprintRecord.deserialize(imprintBytesBuffer.duplicate()); @@ -181,8 +180,6 @@ public void deserializeFlatBuffers(Blackhole bh) { // ===== FIELD ACCESS BENCHMARKS ===== // Tests accessing a single field near the end of a large record - // This showcases Imprint's O(1) directory lookup vs sequential deserialization - @Benchmark public void singleFieldAccessImprint(Blackhole bh) throws Exception { ImprintRecord record = ImprintRecord.deserialize(imprintBytesBuffer.duplicate()); @@ -213,19 +210,19 @@ public void singleFieldAccessMessagePack(Blackhole bh) throws Exception { @Benchmark public void singleFieldAccessAvro(Blackhole bh) throws Exception { GenericRecord record = deserializeWithAvro(avroBytes); - bh.consume(record.get("extraData4")); // Accessing field near end + bh.consume(record.get("extraData4")); } @Benchmark public void singleFieldAccessProtobuf(Blackhole bh) throws Exception { TestRecordProto.TestRecord record = TestRecordProto.TestRecord.parseFrom(protobufBytes); - bh.consume(record.getExtraData(4)); // Accessing field near end + bh.consume(record.getExtraData(4)); } @Benchmark public void singleFieldAccessFlatBuffers(Blackhole bh) { TestRecordFB record = TestRecordFB.getRootAsTestRecordFB(flatbuffersBytes.duplicate()); - bh.consume(record.extraData(4)); // Accessing field near end - zero copy! + bh.consume(record.extraData(4)); } // ===== SIZE COMPARISON ===== @@ -694,8 +691,8 @@ private ByteBuffer mergeFlatBuffersRecords(TestRecordFB first, TestRecordFB seco FlatBufferBuilder builder = new FlatBufferBuilder(1024); // Use second record's values if they exist, otherwise first record's values - String name = second.name() != null && !second.name().isEmpty() ? second.name() : first.name(); - String category = second.category() != null && !second.category().isEmpty() ? second.category() : first.category(); + String name = second.name() != null && !Objects.requireNonNull(second.name()).isEmpty() ? second.name() : first.name(); + String category = second.category() != null && !Objects.requireNonNull(second.category()).isEmpty() ? second.category() : first.category(); double price = second.price() != 0.0 ? second.price() : first.price(); boolean active = second.active(); // Use second's boolean value int id = first.id(); // Keep first record's ID @@ -809,6 +806,7 @@ var record = new TestRecord(); } // Test data class for other serialization libraries + @NoArgsConstructor public static class TestRecord { public int id; public String name; @@ -817,8 +815,7 @@ public static class TestRecord { public String category; public List tags = new ArrayList<>(); public Map metadata = new HashMap<>(); - public List extraData = new ArrayList<>(); // Fields 8-20 for large record test - - public TestRecord() {} // Required for deserialization + // Fields 8-20 for large record test + public List extraData = new ArrayList<>(); } } \ No newline at end of file diff --git a/src/main/java/com/imprint/types/TypeHandler.java b/src/main/java/com/imprint/types/TypeHandler.java index e58c355..dce6973 100644 --- a/src/main/java/com/imprint/types/TypeHandler.java +++ b/src/main/java/com/imprint/types/TypeHandler.java @@ -9,7 +9,7 @@ /** * Interface for handling type-specific serialization, deserialization, and size estimation. - * Note that primitives are potentially auto/un-boxed here which could impact performance slightly + * Note that primitives are basically boxed here which could impact performance slightly * but having all the types in their own implementation helps keep things organized for now, especially * for dealing with and testing more complex types in the future. */ @@ -26,7 +26,7 @@ interface BufferViewer { int measureDataLength(ByteBuffer tempBuffer, int numElements) throws ImprintException; } - // Helper method to eliminate duplication in ARRAY/MAP readValueBytes + // Helper method for complex buffer positioning in MAP and ARRAY static ByteBuffer readComplexValueBytes(ByteBuffer buffer, String typeName, BufferViewer measurer) throws ImprintException { int initialPosition = buffer.position(); ByteBuffer tempBuffer = buffer.duplicate(); @@ -571,15 +571,13 @@ public ByteBuffer readValueBytes(ByteBuffer buffer) throws ImprintException { } byte keyTypeCodeByte = tempBuffer.get(); byte valueTypeCodeByte = tempBuffer.get(); - TypeCode keyType = TypeCode.fromByte(keyTypeCodeByte); - TypeCode valueType = TypeCode.fromByte(valueTypeCodeByte); + var keyType = TypeCode.fromByte(keyTypeCodeByte); + var valueType = TypeCode.fromByte(valueTypeCodeByte); - // OPTIMIZATION: Calculate sizes directly for fixed-size types int keySize = getFixedTypeSize(keyType); int valueSize = getFixedTypeSize(valueType); if (keySize > 0 && valueSize > 0) { - // Both are fixed-size: O(1) calculation return 2 + (numEntries * (keySize + valueSize)); } else { // At least one is variable-size: fall back to traversal diff --git a/src/main/java/com/imprint/types/Value.java b/src/main/java/com/imprint/types/Value.java index ba747de..fbb988c 100644 --- a/src/main/java/com/imprint/types/Value.java +++ b/src/main/java/com/imprint/types/Value.java @@ -192,11 +192,11 @@ public static class BytesValue extends Value { private final byte[] value; public BytesValue(byte[] value) { - this.value = value.clone(); // defensive copy + this.value = value.clone(); } public byte[] getValue() { - return value.clone(); // defensive copy + return value.clone(); } @Override @@ -233,7 +233,7 @@ public static class BytesBufferValue extends Value { private final ByteBuffer value; public BytesBufferValue(ByteBuffer value) { - this.value = value.asReadOnlyBuffer(); // zero-copy read-only view + this.value = value.asReadOnlyBuffer(); } public byte[] getValue() { @@ -244,7 +244,7 @@ public byte[] getValue() { } public ByteBuffer getBuffer() { - return value.duplicate(); // zero-copy view + return value.duplicate(); } @Override @@ -289,11 +289,11 @@ public StringValue(String value) { public byte[] getUtf8Bytes() { var cached = cachedUtf8Bytes; if (cached == null) { - // Multiple threads may compute this - that's OK since it's idempotent + // UTF8 is idempotent so no need to synchronize cached = value.getBytes(StandardCharsets.UTF_8); cachedUtf8Bytes = cached; } - return cached; // Return our computed value, not re-read from volatile field + return cached; // Return computed value } @Override @@ -363,14 +363,14 @@ private String decodeUtf8() { // Fallback: copy bytes from the ByteBuffer to a new heap array (if too large for cache) array = new byte[length]; } - value.duplicate().get(array, 0, length); // Get bytes from current position into chosen array + value.duplicate().get(array, 0, length); offset = 0; } return new String(array, offset, length, StandardCharsets.UTF_8); } public ByteBuffer getBuffer() { - return value.duplicate(); // zero-copy view + return value.duplicate(); } @Override From a605b652eec81478b7f1d7f4cf25529ea00cebfb Mon Sep 17 00:00:00 2001 From: expand3d <> Date: Thu, 5 Jun 2025 15:22:06 -0400 Subject: [PATCH 26/49] minor cleanup --- build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index 66f2e3e..d9093f9 100644 --- a/build.gradle +++ b/build.gradle @@ -170,7 +170,7 @@ test { jmh { fork = 1 warmupIterations = 2 // Reduced for faster CI - iterations = 3 + iterations = 3 // Reduced for faster CI resultFormat = 'JSON' includeTests = false resultsFile = file("${projectDir}/benchmark-results/jmh-results-${new Date().format('yyyy-MM-dd-HHmmss')}.json") @@ -180,7 +180,7 @@ jmh { '-XX:+UseG1GC', '-Xmx2g', '-XX:+UnlockExperimentalVMOptions', - '-XX:+UseJVMCICompiler' // Use Graal if available for better performance + '-XX:+UseJVMCICompiler' ] } From aacddeb0ec5791176ec57a7927366364f87acda3 Mon Sep 17 00:00:00 2001 From: expand3d <> Date: Thu, 5 Jun 2025 15:23:46 -0400 Subject: [PATCH 27/49] minor cleanup --- src/main/java/com/imprint/core/ImprintRecord.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/imprint/core/ImprintRecord.java b/src/main/java/com/imprint/core/ImprintRecord.java index 2291550..e7dab70 100644 --- a/src/main/java/com/imprint/core/ImprintRecord.java +++ b/src/main/java/com/imprint/core/ImprintRecord.java @@ -83,7 +83,7 @@ private ByteBuffer getFieldBuffer(int fieldId) { return null; } - // OPTIMIZATION: Single allocation instead of duplicate + slice + //Single allocation instead of duplicate + slice var fieldBuffer = payload.duplicate(); fieldBuffer.position(startOffset).limit(endOffset); return fieldBuffer; From 3bf81ade2bea81333d3acef7f2e80979490f4c7e Mon Sep 17 00:00:00 2001 From: expand3d <> Date: Thu, 5 Jun 2025 15:41:43 -0400 Subject: [PATCH 28/49] Actually fixes offsets and read Byte Values for Maps and Arrays even with nested objects --- .../java/com/imprint/types/TypeHandler.java | 252 ++---------------- .../com/imprint/profile/ProfilerTest.java | 3 +- .../com/imprint/types/TypeHandlerTest.java | 5 - 3 files changed, 19 insertions(+), 241 deletions(-) diff --git a/src/main/java/com/imprint/types/TypeHandler.java b/src/main/java/com/imprint/types/TypeHandler.java index dce6973..634867b 100644 --- a/src/main/java/com/imprint/types/TypeHandler.java +++ b/src/main/java/com/imprint/types/TypeHandler.java @@ -17,52 +17,7 @@ public interface TypeHandler { Value deserialize(ByteBuffer buffer) throws ImprintException; void serialize(Value value, ByteBuffer buffer) throws ImprintException; int estimateSize(Value value) throws ImprintException; - ByteBuffer readValueBytes(ByteBuffer buffer) throws ImprintException; - - - - @FunctionalInterface - interface BufferViewer { - int measureDataLength(ByteBuffer tempBuffer, int numElements) throws ImprintException; - } - - // Helper method for complex buffer positioning in MAP and ARRAY - static ByteBuffer readComplexValueBytes(ByteBuffer buffer, String typeName, BufferViewer measurer) throws ImprintException { - int initialPosition = buffer.position(); - ByteBuffer tempBuffer = buffer.duplicate(); - tempBuffer.order(buffer.order()); - - VarInt.DecodeResult lengthResult = VarInt.decode(tempBuffer); - int numElements = lengthResult.getValue(); - int varIntLength = tempBuffer.position() - initialPosition; - - if (numElements == 0) { - if (buffer.remaining() < varIntLength) { - throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, - "Not enough bytes for empty " + typeName + " VarInt. Needed: " + - varIntLength + ", available: " + buffer.remaining()); - } - ByteBuffer valueSlice = buffer.slice(); - valueSlice.limit(varIntLength); - buffer.position(initialPosition + varIntLength); - return valueSlice.asReadOnlyBuffer(); - } - - int dataLength = measurer.measureDataLength(tempBuffer, numElements); - int totalLength = varIntLength + dataLength; - - if (buffer.remaining() < totalLength) { - throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, - "Not enough bytes for " + typeName + " value. Needed: " + totalLength + - ", available: " + buffer.remaining() + " at position " + initialPosition); - } - ByteBuffer valueSlice = buffer.slice(); - valueSlice.limit(totalLength); - buffer.position(initialPosition + totalLength); - return valueSlice.asReadOnlyBuffer(); - } - // Static implementations for each type TypeHandler NULL = new TypeHandler() { @Override @@ -79,11 +34,6 @@ public void serialize(Value value, ByteBuffer buffer) { public int estimateSize(Value value) { return 0; } - - @Override - public ByteBuffer readValueBytes(ByteBuffer buffer) { - return ByteBuffer.allocate(0).asReadOnlyBuffer(); - } }; TypeHandler BOOL = new TypeHandler() { @@ -108,14 +58,6 @@ public void serialize(Value value, ByteBuffer buffer) { public int estimateSize(Value value) { return 1; } - - @Override - public ByteBuffer readValueBytes(ByteBuffer buffer) { - var boolBuffer = buffer.slice(); - boolBuffer.limit(1); - buffer.position(buffer.position() + 1); - return boolBuffer.asReadOnlyBuffer(); - } }; TypeHandler INT32 = new TypeHandler() { @@ -137,14 +79,6 @@ public void serialize(Value value, ByteBuffer buffer) { public int estimateSize(Value value) { return 4; } - - @Override - public ByteBuffer readValueBytes(ByteBuffer buffer) { - var int32Buffer = buffer.slice(); - int32Buffer.limit(4); - buffer.position(buffer.position() + 4); - return int32Buffer.asReadOnlyBuffer(); - } }; TypeHandler INT64 = new TypeHandler() { @@ -166,14 +100,6 @@ public void serialize(Value value, ByteBuffer buffer) { public int estimateSize(Value value) { return 8; } - - @Override - public ByteBuffer readValueBytes(ByteBuffer buffer) { - var int64Buffer = buffer.slice(); - int64Buffer.limit(8); - buffer.position(buffer.position() + 8); - return int64Buffer.asReadOnlyBuffer(); - } }; TypeHandler FLOAT32 = new TypeHandler() { @@ -195,14 +121,6 @@ public void serialize(Value value, ByteBuffer buffer) { public int estimateSize(Value value) { return 4; } - - @Override - public ByteBuffer readValueBytes(ByteBuffer buffer) { - var float32Buffer = buffer.slice(); - float32Buffer.limit(4); - buffer.position(buffer.position() + 4); - return float32Buffer.asReadOnlyBuffer(); - } }; TypeHandler FLOAT64 = new TypeHandler() { @@ -224,14 +142,6 @@ public void serialize(Value value, ByteBuffer buffer) { public int estimateSize(Value value) { return 8; } - - @Override - public ByteBuffer readValueBytes(ByteBuffer buffer) { - var float64Buffer = buffer.slice(); - float64Buffer.limit(8); - buffer.position(buffer.position() + 8); - return float64Buffer.asReadOnlyBuffer(); - } }; TypeHandler BYTES = new TypeHandler() { @@ -274,29 +184,6 @@ public int estimateSize(Value value) { return VarInt.encodedLength(bytes.length) + bytes.length; } } - - @Override - public ByteBuffer readValueBytes(ByteBuffer buffer) throws ImprintException { - int initialPos = buffer.position(); - ByteBuffer tempMeasureBuffer = buffer.duplicate(); - VarInt.DecodeResult dr = VarInt.decode(tempMeasureBuffer); - - int varIntByteLength = tempMeasureBuffer.position() - initialPos; - int payloadByteLength = dr.getValue(); - int totalValueLength = varIntByteLength + payloadByteLength; - - if (buffer.remaining() < totalValueLength) { - throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, - "Not enough bytes for VarInt-prefixed data. Needed: " + totalValueLength + - ", available: " + buffer.remaining() + " at position " + initialPos); - } - - ByteBuffer resultSlice = buffer.slice(); - resultSlice.limit(totalValueLength); - - buffer.position(initialPos + totalValueLength); - return resultSlice.asReadOnlyBuffer(); - } }; TypeHandler STRING = new TypeHandler() { @@ -344,29 +231,6 @@ public int estimateSize(Value value) { return VarInt.encodedLength(utf8Bytes.length) + utf8Bytes.length; } } - - @Override - public ByteBuffer readValueBytes(ByteBuffer buffer) throws ImprintException { - int initialPos = buffer.position(); - ByteBuffer tempMeasureBuffer = buffer.duplicate(); - VarInt.DecodeResult dr = VarInt.decode(tempMeasureBuffer); - - int varIntByteLength = tempMeasureBuffer.position() - initialPos; - int payloadByteLength = dr.getValue(); - int totalValueLength = varIntByteLength + payloadByteLength; - - if (buffer.remaining() < totalValueLength) { - throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, - "Not enough bytes for VarInt-prefixed string. Needed: " + totalValueLength + - ", available: " + buffer.remaining() + " at position " + initialPos); - } - - ByteBuffer resultSlice = buffer.slice(); - resultSlice.limit(totalValueLength); - - buffer.position(initialPos + totalValueLength); - return resultSlice.asReadOnlyBuffer(); - } }; TypeHandler ARRAY = new TypeHandler() { @@ -374,25 +238,24 @@ public ByteBuffer readValueBytes(ByteBuffer buffer) throws ImprintException { public Value deserialize(ByteBuffer buffer) throws ImprintException { VarInt.DecodeResult lengthResult = VarInt.decode(buffer); int length = lengthResult.getValue(); - + if (length == 0) { return Value.fromArray(Collections.emptyList()); } - + if (buffer.remaining() < 1) { - throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, "Not enough bytes for ARRAY element type code."); + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, "Not enough bytes for ARRAY element type code."); } var elementType = TypeCode.fromByte(buffer.get()); var elements = new ArrayList(length); var elementHandler = elementType.getHandler(); - + + //Let each element handler consume what it needs from the buffer for (int i = 0; i < length; i++) { - var elementValueBytes = elementHandler.readValueBytes(buffer); - elementValueBytes.order(buffer.order()); - var element = elementHandler.deserialize(elementValueBytes); + var element = elementHandler.deserialize(buffer); //Handler advances buffer position elements.add(element); } - + return Value.fromArray(elements); } @@ -433,40 +296,6 @@ public int estimateSize(Value value) throws ImprintException { } return arraySize; } - - @Override - public ByteBuffer readValueBytes(ByteBuffer buffer) throws ImprintException { - return readComplexValueBytes(buffer, "ARRAY", (tempBuffer, numElements) -> { - if (tempBuffer.remaining() < 1) { - throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, - "Not enough bytes for ARRAY element type code"); - } - byte elementTypeCodeByte = tempBuffer.get(); - var elementType = TypeCode.fromByte(elementTypeCodeByte); - - switch (elementType) { - case NULL: - return 1; - case BOOL: - return 1 + numElements; - case INT32: - case FLOAT32: - return 1 + (numElements * 4); - case INT64: - case FLOAT64: - return 1 + (numElements * 8); - default: - var elementHandler = elementType.getHandler(); - int elementsDataLength = 0; - for (int i = 0; i < numElements; i++) { - int elementStartPos = tempBuffer.position(); - elementHandler.readValueBytes(tempBuffer); - elementsDataLength += (tempBuffer.position() - elementStartPos); - } - return 1 + elementsDataLength; - } - }); - } }; TypeHandler MAP = new TypeHandler() { @@ -474,13 +303,13 @@ public ByteBuffer readValueBytes(ByteBuffer buffer) throws ImprintException { public Value deserialize(ByteBuffer buffer) throws ImprintException { VarInt.DecodeResult lengthResult = VarInt.decode(buffer); int length = lengthResult.getValue(); - + if (length == 0) { return Value.fromMap(Collections.emptyMap()); } - + if (buffer.remaining() < 2) { - throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, "Not enough bytes for MAP key/value type codes."); + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, "Not enough bytes for MAP key/value type codes."); } var keyType = TypeCode.fromByte(buffer.get()); var valueType = TypeCode.fromByte(buffer.get()); @@ -488,20 +317,17 @@ public Value deserialize(ByteBuffer buffer) throws ImprintException { var keyHandler = keyType.getHandler(); var valueHandler = valueType.getHandler(); - + + //Let handlers consume directly from buffer for (int i = 0; i < length; i++) { - var keyBytes = keyHandler.readValueBytes(buffer); - keyBytes.order(buffer.order()); - var keyValue = keyHandler.deserialize(keyBytes); + var keyValue = keyHandler.deserialize(buffer);// Advances buffer var key = MapKey.fromValue(keyValue); - - var valueBytes = valueHandler.readValueBytes(buffer); - valueBytes.order(buffer.order()); - var mapInternalValue = valueHandler.deserialize(valueBytes); - + + var mapInternalValue = valueHandler.deserialize(buffer);//Advances buffer + map.put(key, mapInternalValue); } - + return Value.fromMap(map); } @@ -562,50 +388,6 @@ public int estimateSize(Value value) throws ImprintException { return mapSize; } - @Override - public ByteBuffer readValueBytes(ByteBuffer buffer) throws ImprintException { - return readComplexValueBytes(buffer, "MAP", (tempBuffer, numEntries) -> { - if (tempBuffer.remaining() < 2) { - throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, - "Not enough bytes for MAP key/value type codes"); - } - byte keyTypeCodeByte = tempBuffer.get(); - byte valueTypeCodeByte = tempBuffer.get(); - var keyType = TypeCode.fromByte(keyTypeCodeByte); - var valueType = TypeCode.fromByte(valueTypeCodeByte); - - int keySize = getFixedTypeSize(keyType); - int valueSize = getFixedTypeSize(valueType); - - if (keySize > 0 && valueSize > 0) { - return 2 + (numEntries * (keySize + valueSize)); - } else { - // At least one is variable-size: fall back to traversal - int entriesDataLength = 0; - for (int i = 0; i < numEntries; i++) { - int entryStartPos = tempBuffer.position(); - keyType.getHandler().readValueBytes(tempBuffer); - valueType.getHandler().readValueBytes(tempBuffer); - entriesDataLength += (tempBuffer.position() - entryStartPos); - } - return 2 + entriesDataLength; - } - }); - } - - private int getFixedTypeSize(TypeCode type) { - switch (type) { - case NULL: return 0; - case BOOL: return 1; - case INT32: - case FLOAT32: return 4; - case INT64: - case FLOAT64: return 8; - default: return -1; // Variable size - } - } - - private void serializeMapKey(MapKey key, ByteBuffer buffer) throws ImprintException { switch (key.getTypeCode()) { case INT32: diff --git a/src/test/java/com/imprint/profile/ProfilerTest.java b/src/test/java/com/imprint/profile/ProfilerTest.java index d48c1aa..3b9f371 100644 --- a/src/test/java/com/imprint/profile/ProfilerTest.java +++ b/src/test/java/com/imprint/profile/ProfilerTest.java @@ -4,6 +4,7 @@ import com.imprint.core.ImprintWriter; import com.imprint.core.SchemaId; import com.imprint.types.Value; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import java.util.Random; @@ -26,7 +27,7 @@ * - String operations and UTF-8 encoding * - ByteBuffer operations */ -//@Disabled("Enable manually for profiling") +@Disabled("Enable manually for profiling") public class ProfilerTest { private static final int ITERATIONS = 1_000_000; diff --git a/src/test/java/com/imprint/types/TypeHandlerTest.java b/src/test/java/com/imprint/types/TypeHandlerTest.java index f131a0f..75d118f 100644 --- a/src/test/java/com/imprint/types/TypeHandlerTest.java +++ b/src/test/java/com/imprint/types/TypeHandlerTest.java @@ -33,11 +33,6 @@ void testNullHandler() throws ImprintException { buffer.flip(); var deserialized = handler.deserialize(buffer); assertThat(deserialized).isEqualTo(value); - - // readValueBytes - buffer.clear(); - var valueBytes = handler.readValueBytes(buffer); - assertThat(valueBytes.remaining()).isEqualTo(0); } @ParameterizedTest From 7eaa6e9ec299fe72f43a19af766e180002716f91 Mon Sep 17 00:00:00 2001 From: expand3d <> Date: Thu, 5 Jun 2025 16:04:42 -0400 Subject: [PATCH 29/49] change CI file to use JMH plugin to respect iteration and warmup values in gradle file. Also fix permission issue --- .github/workflows/ci.yml | 148 +++++++++++++++++++++++---------------- 1 file changed, 89 insertions(+), 59 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 62ac6f5..ec052ca 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,6 +46,11 @@ jobs: benchmark: runs-on: ubuntu-latest needs: test + # Add explicit permissions for commenting on PRs + permissions: + contents: read + pull-requests: write + issues: write # Only run benchmarks on main branch pushes and PRs to main to avoid excessive CI time if: github.ref == 'refs/heads/main' || github.base_ref == 'main' @@ -77,22 +82,22 @@ jobs: - name: Run serialization benchmarks run: | - ./gradlew jmhRunSerializationBenchmarks + ./gradlew jmh --include=".*serialize.*" --exclude=".*deserialize.*" continue-on-error: true - name: Run deserialization benchmarks run: | - ./gradlew jmhRunDeserializationBenchmarks + ./gradlew jmh --include=".*deserialize.*" continue-on-error: true - name: Run field access benchmarks run: | - ./gradlew jmhRunFieldAccessBenchmarks + ./gradlew jmh --include=".*singleFieldAccess.*" continue-on-error: true - name: Run size comparison benchmarks run: | - ./gradlew jmhRunSizeComparisonBenchmarks + ./gradlew jmh --include=".*measure.*" continue-on-error: true - name: Upload benchmark results @@ -106,59 +111,63 @@ jobs: - name: Comment benchmark results on PR if: github.event_name == 'pull_request' uses: actions/github-script@v7 + continue-on-error: true with: + github-token: ${{ secrets.GITHUB_TOKEN }} script: | - const fs = require('fs'); - const path = require('path'); - - // Find the latest benchmark results file - const resultsDir = 'benchmark-results'; - let latestFile = null; - let latestTime = 0; - - if (fs.existsSync(resultsDir)) { - const files = fs.readdirSync(resultsDir); - for (const file of files) { - if (file.endsWith('.json')) { - const filePath = path.join(resultsDir, file); - const stats = fs.statSync(filePath); - if (stats.mtime.getTime() > latestTime) { - latestTime = stats.mtime.getTime(); - latestFile = filePath; + try { + const fs = require('fs'); + const path = require('path'); + + // Find the latest benchmark results file + const resultsDir = 'benchmark-results'; + let latestFile = null; + let latestTime = 0; + + if (fs.existsSync(resultsDir)) { + const files = fs.readdirSync(resultsDir); + for (const file of files) { + if (file.endsWith('.json')) { + const filePath = path.join(resultsDir, file); + const stats = fs.statSync(filePath); + if (stats.mtime.getTime() > latestTime) { + latestTime = stats.mtime.getTime(); + latestFile = filePath; + } } } } - } - if (latestFile) { - const results = JSON.parse(fs.readFileSync(latestFile, 'utf8')); + if (latestFile) { + console.log(`Found benchmark results: ${latestFile}`); + const results = JSON.parse(fs.readFileSync(latestFile, 'utf8')); - // Group results by benchmark type - const serialization = results.filter(r => r.benchmark.includes('serialize')); - const deserialization = results.filter(r => r.benchmark.includes('deserialize')); - const fieldAccess = results.filter(r => r.benchmark.includes('singleFieldAccess')); - const sizes = results.filter(r => r.benchmark.includes('measure')); + // Group results by benchmark type + const serialization = results.filter(r => r.benchmark.includes('serialize')); + const deserialization = results.filter(r => r.benchmark.includes('deserialize')); + const fieldAccess = results.filter(r => r.benchmark.includes('singleFieldAccess')); + const sizes = results.filter(r => r.benchmark.includes('measure')); - // Format results into a table - const formatResults = (benchmarks, title) => { - if (benchmarks.length === 0) return ''; + // Format results into a table + const formatResults = (benchmarks, title) => { + if (benchmarks.length === 0) return ''; - let table = `\n### ${title}\n\n| Library | Score (ns/op) | Error | Unit |\n|---------|---------------|-------|------|\n`; + let table = `\n### ${title}\n\n| Library | Score (ns/op) | Error | Unit |\n|---------|---------------|-------|------|\n`; - benchmarks - .sort((a, b) => a.primaryMetric.score - b.primaryMetric.score) - .forEach(benchmark => { - const name = benchmark.benchmark.split('.').pop().replace(/serialize|deserialize|singleFieldAccess|measure/, '').replace(/Imprint|JacksonJson|Kryo|MessagePack|Avro|Protobuf|FlatBuffers/, (match) => match); - const score = benchmark.primaryMetric.score.toFixed(2); - const error = benchmark.primaryMetric.scoreError.toFixed(2); - const unit = benchmark.primaryMetric.scoreUnit; - table += `| ${name} | ${score} | ±${error} | ${unit} |\n`; - }); + benchmarks + .sort((a, b) => a.primaryMetric.score - b.primaryMetric.score) + .forEach(benchmark => { + const name = benchmark.benchmark.split('.').pop().replace(/serialize|deserialize|singleFieldAccess|measure/, '').replace(/Imprint|JacksonJson|Kryo|MessagePack|Avro|Protobuf|FlatBuffers/, (match) => match); + const score = benchmark.primaryMetric.score.toFixed(2); + const error = benchmark.primaryMetric.scoreError.toFixed(2); + const unit = benchmark.primaryMetric.scoreUnit; + table += `| ${name} | ${score} | ±${error} | ${unit} |\n`; + }); - return table; - }; + return table; + }; - const comment = `##Benchmark Results + const comment = `## Benchmark Results Benchmark comparison between Imprint and other serialization libraries: ${formatResults(serialization, 'Serialization Performance')} @@ -175,26 +184,47 @@ jobs: `; - github.rest.issues.createComment({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - body: comment - }); - } else { - github.rest.issues.createComment({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - body: '## Benchmark Results\n\nBenchmark execution completed but no results file was found. Check the [workflow logs](' + - `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}` + ') for details.' - }); + await github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: comment + }); + + console.log('Successfully posted benchmark results'); + } else { + console.log('No benchmark results found'); + await github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: '## Benchmark Results\n\nBenchmark execution completed but no results file was found. Check the [workflow logs](' + + `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}` + ') for details.' + }); + } + } catch (error) { + console.log('Failed to post benchmark comment:', error.message); + console.log('Benchmark results are still available in workflow artifacts'); + + // Try to post a simple error message + try { + await github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: `## Benchmark Results\n\n Failed to process benchmark results automatically.\n\nResults are available in the [workflow artifacts](https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}).` + }); + } catch (commentError) { + console.log('Also failed to post error comment:', commentError.message); + } } # Optional: Run full benchmark suite on releases benchmark-full: runs-on: ubuntu-latest if: startsWith(github.ref, 'refs/tags/') + permissions: + contents: read steps: - name: Checkout code From 32640cdb551ddd9dd137f0c21be4973dac5234ad Mon Sep 17 00:00:00 2001 From: expand3d <> Date: Thu, 5 Jun 2025 16:12:02 -0400 Subject: [PATCH 30/49] ok plugin didn't work apparently so reverting that and just reducing Comparison tests iterations manually --- .github/workflows/ci.yml | 28 +++++++++---------- .../benchmark/ComparisonBenchmark.java | 4 +-- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ec052ca..d0e43cb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -82,22 +82,22 @@ jobs: - name: Run serialization benchmarks run: | - ./gradlew jmh --include=".*serialize.*" --exclude=".*deserialize.*" + ./gradlew jmhRunSerializationBenchmarks continue-on-error: true - name: Run deserialization benchmarks run: | - ./gradlew jmh --include=".*deserialize.*" + ./gradlew jmhRunDeserializationBenchmarks continue-on-error: true - name: Run field access benchmarks run: | - ./gradlew jmh --include=".*singleFieldAccess.*" + ./gradlew jmhRunFieldAccessBenchmarks continue-on-error: true - name: Run size comparison benchmarks run: | - ./gradlew jmh --include=".*measure.*" + ./gradlew jmhRunSizeComparisonBenchmarks continue-on-error: true - name: Upload benchmark results @@ -139,7 +139,7 @@ jobs: } if (latestFile) { - console.log(`Found benchmark results: ${latestFile}`); + console.log(`📊 Found benchmark results: ${latestFile}`); const results = JSON.parse(fs.readFileSync(latestFile, 'utf8')); // Group results by benchmark type @@ -167,7 +167,7 @@ jobs: return table; }; - const comment = `## Benchmark Results + const comment = `## 📊 Benchmark Results Benchmark comparison between Imprint and other serialization libraries: ${formatResults(serialization, 'Serialization Performance')} @@ -191,20 +191,20 @@ jobs: body: comment }); - console.log('Successfully posted benchmark results'); + console.log('✅ Successfully posted benchmark results to PR'); } else { - console.log('No benchmark results found'); + console.log('⚠️ No benchmark results found'); await github.rest.issues.createComment({ issue_number: context.issue.number, owner: context.repo.owner, repo: context.repo.repo, - body: '## Benchmark Results\n\nBenchmark execution completed but no results file was found. Check the [workflow logs](' + + body: '## 📊 Benchmark Results\n\nBenchmark execution completed but no results file was found. Check the [workflow logs](' + `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}` + ') for details.' }); } } catch (error) { - console.log('Failed to post benchmark comment:', error.message); - console.log('Benchmark results are still available in workflow artifacts'); + console.log('❌ Failed to post benchmark comment:', error.message); + console.log('📁 Benchmark results are still available in workflow artifacts'); // Try to post a simple error message try { @@ -212,10 +212,10 @@ jobs: issue_number: context.issue.number, owner: context.repo.owner, repo: context.repo.repo, - body: `## Benchmark Results\n\n Failed to process benchmark results automatically.\n\nResults are available in the [workflow artifacts](https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}).` + body: `## 📊 Benchmark Results\n\n⚠️ Failed to process benchmark results automatically.\n\nResults are available in the [workflow artifacts](https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}).` }); } catch (commentError) { - console.log('Also failed to post error comment:', commentError.message); + console.log('❌ Also failed to post error comment:', commentError.message); } } @@ -254,7 +254,7 @@ jobs: - name: Run full benchmark suite run: | - ./gradlew jmh + ./gradlew jmhRunAllBenchmarks - name: Upload full benchmark results uses: actions/upload-artifact@v4 diff --git a/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java b/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java index a7ffd3c..ee32ff0 100644 --- a/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java +++ b/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java @@ -38,8 +38,8 @@ @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.NANOSECONDS) @State(Scope.Benchmark) -@Warmup(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS) -@Measurement(iterations = 10, time = 1, timeUnit = TimeUnit.SECONDS) +@Warmup(iterations = 2, time = 1, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS) @Fork(1) @SuppressWarnings("unused") public class ComparisonBenchmark { From 880aeb0c6bdfd24d7893c3204d43eeb59059745b Mon Sep 17 00:00:00 2001 From: expand3d <> Date: Thu, 5 Jun 2025 16:43:08 -0400 Subject: [PATCH 31/49] trying to update github ci to make jmh actually work correctly --- .github/workflows/ci.yml | 140 +++------------------------- build.gradle | 192 ++++++++++++++++++++++++--------------- 2 files changed, 132 insertions(+), 200 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d0e43cb..a80f529 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,11 +46,6 @@ jobs: benchmark: runs-on: ubuntu-latest needs: test - # Add explicit permissions for commenting on PRs - permissions: - contents: read - pull-requests: write - issues: write # Only run benchmarks on main branch pushes and PRs to main to avoid excessive CI time if: github.ref == 'refs/heads/main' || github.base_ref == 'main' @@ -81,25 +76,28 @@ jobs: run: mkdir -p benchmark-results - name: Run serialization benchmarks - run: | - ./gradlew jmhRunSerializationBenchmarks + run: ./gradlew jmhRunSerializationBenchmarks continue-on-error: true - name: Run deserialization benchmarks - run: | - ./gradlew jmhRunDeserializationBenchmarks + run: ./gradlew jmhRunDeserializationBenchmarks continue-on-error: true - name: Run field access benchmarks - run: | - ./gradlew jmhRunFieldAccessBenchmarks + run: ./gradlew jmhRunFieldAccessBenchmarks continue-on-error: true - name: Run size comparison benchmarks - run: | - ./gradlew jmhRunSizeComparisonBenchmarks + run: ./gradlew jmhRunSizeComparisonBenchmarks continue-on-error: true + - name: List benchmark results + run: | + echo "Contents of benchmark-results directory:" + ls -la benchmark-results/ || echo "benchmark-results directory not found" + echo "Working directory contents:" + ls -la + - name: Upload benchmark results uses: actions/upload-artifact@v4 if: always() @@ -108,123 +106,10 @@ jobs: path: benchmark-results/ retention-days: 30 - - name: Comment benchmark results on PR - if: github.event_name == 'pull_request' - uses: actions/github-script@v7 - continue-on-error: true - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - try { - const fs = require('fs'); - const path = require('path'); - - // Find the latest benchmark results file - const resultsDir = 'benchmark-results'; - let latestFile = null; - let latestTime = 0; - - if (fs.existsSync(resultsDir)) { - const files = fs.readdirSync(resultsDir); - for (const file of files) { - if (file.endsWith('.json')) { - const filePath = path.join(resultsDir, file); - const stats = fs.statSync(filePath); - if (stats.mtime.getTime() > latestTime) { - latestTime = stats.mtime.getTime(); - latestFile = filePath; - } - } - } - } - - if (latestFile) { - console.log(`📊 Found benchmark results: ${latestFile}`); - const results = JSON.parse(fs.readFileSync(latestFile, 'utf8')); - - // Group results by benchmark type - const serialization = results.filter(r => r.benchmark.includes('serialize')); - const deserialization = results.filter(r => r.benchmark.includes('deserialize')); - const fieldAccess = results.filter(r => r.benchmark.includes('singleFieldAccess')); - const sizes = results.filter(r => r.benchmark.includes('measure')); - - // Format results into a table - const formatResults = (benchmarks, title) => { - if (benchmarks.length === 0) return ''; - - let table = `\n### ${title}\n\n| Library | Score (ns/op) | Error | Unit |\n|---------|---------------|-------|------|\n`; - - benchmarks - .sort((a, b) => a.primaryMetric.score - b.primaryMetric.score) - .forEach(benchmark => { - const name = benchmark.benchmark.split('.').pop().replace(/serialize|deserialize|singleFieldAccess|measure/, '').replace(/Imprint|JacksonJson|Kryo|MessagePack|Avro|Protobuf|FlatBuffers/, (match) => match); - const score = benchmark.primaryMetric.score.toFixed(2); - const error = benchmark.primaryMetric.scoreError.toFixed(2); - const unit = benchmark.primaryMetric.scoreUnit; - table += `| ${name} | ${score} | ±${error} | ${unit} |\n`; - }); - - return table; - }; - - const comment = `## 📊 Benchmark Results - - Benchmark comparison between Imprint and other serialization libraries: - ${formatResults(serialization, 'Serialization Performance')} - ${formatResults(deserialization, 'Deserialization Performance')} - ${formatResults(fieldAccess, 'Single Field Access Performance')} - ${formatResults(sizes, 'Serialized Size Comparison')} - -
- View detailed results - - Results generated from commit: \`${context.sha.substring(0, 7)}\` - - Lower scores are better for performance benchmarks. - -
`; - - await github.rest.issues.createComment({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - body: comment - }); - - console.log('✅ Successfully posted benchmark results to PR'); - } else { - console.log('⚠️ No benchmark results found'); - await github.rest.issues.createComment({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - body: '## 📊 Benchmark Results\n\nBenchmark execution completed but no results file was found. Check the [workflow logs](' + - `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}` + ') for details.' - }); - } - } catch (error) { - console.log('❌ Failed to post benchmark comment:', error.message); - console.log('📁 Benchmark results are still available in workflow artifacts'); - - // Try to post a simple error message - try { - await github.rest.issues.createComment({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - body: `## 📊 Benchmark Results\n\n⚠️ Failed to process benchmark results automatically.\n\nResults are available in the [workflow artifacts](https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}).` - }); - } catch (commentError) { - console.log('❌ Also failed to post error comment:', commentError.message); - } - } - # Optional: Run full benchmark suite on releases benchmark-full: runs-on: ubuntu-latest if: startsWith(github.ref, 'refs/tags/') - permissions: - contents: read steps: - name: Checkout code @@ -253,8 +138,7 @@ jobs: run: mkdir -p benchmark-results - name: Run full benchmark suite - run: | - ./gradlew jmhRunAllBenchmarks + run: ./gradlew jmhRunAllBenchmarks - name: Upload full benchmark results uses: actions/upload-artifact@v4 diff --git a/build.gradle b/build.gradle index d9093f9..6c97a24 100644 --- a/build.gradle +++ b/build.gradle @@ -185,130 +185,178 @@ jmh { } // Create individual benchmark tasks for CI pipeline -tasks.register('jmhRunSerializationBenchmarks', JavaExec) { +tasks.register('jmhRunSerializationBenchmarks') { dependsOn compileJmhJava description = 'Run serialization benchmarks' group = 'benchmarking' - classpath = sourceSets.jmh.runtimeClasspath - mainClass = 'com.imprint.benchmark.ComparisonBenchmark' - args = ['runSerializationBenchmarks'] - - // Java 11 optimized JVM settings - jvmArgs = [ - '-XX:+UseG1GC', - '-Xmx2g', - '-XX:+UnlockExperimentalVMOptions' - ] - doFirst { file("${projectDir}/benchmark-results").mkdirs() } + + doLast { + javaexec { + classpath = sourceSets.jmh.runtimeClasspath + mainClass = 'org.openjdk.jmh.Main' + args = [ + '.*serialize.*', + '-f', '1', + '-wi', '2', + '-i', '3', + '-rf', 'json', + '-rff', "${projectDir}/benchmark-results/serialization-results.json" + ] + jvmArgs = [ + '-XX:+UseG1GC', + '-Xmx2g' + ] + } + } } -tasks.register('jmhRunDeserializationBenchmarks', JavaExec) { +tasks.register('jmhRunDeserializationBenchmarks') { dependsOn compileJmhJava description = 'Run deserialization benchmarks' group = 'benchmarking' - classpath = sourceSets.jmh.runtimeClasspath - mainClass = 'com.imprint.benchmark.ComparisonBenchmark' - args = ['runDeserializationBenchmarks'] - - // Java 11 optimized JVM settings - jvmArgs = [ - '-XX:+UseG1GC', - '-Xmx2g', - '-XX:+UnlockExperimentalVMOptions' - ] - doFirst { file("${projectDir}/benchmark-results").mkdirs() } + + doLast { + javaexec { + classpath = sourceSets.jmh.runtimeClasspath + mainClass = 'org.openjdk.jmh.Main' + args = [ + '.*deserialize.*', + '-f', '1', + '-wi', '2', + '-i', '3', + '-rf', 'json', + '-rff', "${projectDir}/benchmark-results/deserialization-results.json" + ] + jvmArgs = [ + '-XX:+UseG1GC', + '-Xmx2g' + ] + } + } } -tasks.register('jmhRunFieldAccessBenchmarks', JavaExec) { +tasks.register('jmhRunFieldAccessBenchmarks') { dependsOn compileJmhJava description = 'Run field access benchmarks' group = 'benchmarking' - classpath = sourceSets.jmh.runtimeClasspath - mainClass = 'com.imprint.benchmark.ComparisonBenchmark' - args = ['runFieldAccessBenchmarks'] - - // Java 11 optimized JVM settings - jvmArgs = [ - '-XX:+UseG1GC', - '-Xmx2g', - '-XX:+UnlockExperimentalVMOptions' - ] - doFirst { file("${projectDir}/benchmark-results").mkdirs() } + + doLast { + javaexec { + classpath = sourceSets.jmh.runtimeClasspath + mainClass = 'org.openjdk.jmh.Main' + args = [ + '.*singleFieldAccess.*', + '-f', '1', + '-wi', '2', + '-i', '3', + '-rf', 'json', + '-rff', "${projectDir}/benchmark-results/fieldaccess-results.json" + ] + jvmArgs = [ + '-XX:+UseG1GC', + '-Xmx2g' + ] + } + } } -tasks.register('jmhRunSizeComparisonBenchmarks', JavaExec) { +tasks.register('jmhRunSizeComparisonBenchmarks') { dependsOn compileJmhJava description = 'Run size comparison benchmarks' group = 'benchmarking' - classpath = sourceSets.jmh.runtimeClasspath - mainClass = 'com.imprint.benchmark.ComparisonBenchmark' - args = ['runSizeComparisonBenchmarks'] - - // Java 11 optimized JVM settings - jvmArgs = [ - '-XX:+UseG1GC', - '-Xmx2g', - '-XX:+UnlockExperimentalVMOptions' - ] - doFirst { file("${projectDir}/benchmark-results").mkdirs() } + + doLast { + javaexec { + classpath = sourceSets.jmh.runtimeClasspath + mainClass = 'org.openjdk.jmh.Main' + args = [ + '.*measure.*', + '-f', '1', + '-wi', '2', + '-i', '3', + '-rf', 'json', + '-rff', "${projectDir}/benchmark-results/size-results.json" + ] + jvmArgs = [ + '-XX:+UseG1GC', + '-Xmx2g' + ] + } + } } -tasks.register('jmhRunMergeBenchmarks', JavaExec) { +tasks.register('jmhRunMergeBenchmarks') { dependsOn compileJmhJava description = 'Run merge operation benchmarks' group = 'benchmarking' - classpath = sourceSets.jmh.runtimeClasspath - mainClass = 'com.imprint.benchmark.ComparisonBenchmark' - args = ['runMergeBenchmarks'] - - // Java 11 optimized JVM settings - jvmArgs = [ - '-XX:+UseG1GC', - '-Xmx2g', - '-XX:+UnlockExperimentalVMOptions' - ] - doFirst { file("${projectDir}/benchmark-results").mkdirs() } + + doLast { + javaexec { + classpath = sourceSets.jmh.runtimeClasspath + mainClass = 'org.openjdk.jmh.Main' + args = [ + '.*merge.*', + '-f', '1', + '-wi', '2', + '-i', '3', + '-rf', 'json', + '-rff', "${projectDir}/benchmark-results/merge-results.json" + ] + jvmArgs = [ + '-XX:+UseG1GC', + '-Xmx2g' + ] + } + } } -tasks.register('jmhRunAllBenchmarks', JavaExec) { +tasks.register('jmhRunAllBenchmarks') { dependsOn compileJmhJava description = 'Run all comparison benchmarks' group = 'benchmarking' - classpath = sourceSets.jmh.runtimeClasspath - mainClass = 'com.imprint.benchmark.ComparisonBenchmark' - args = ['runAll'] - - // Java 11 optimized JVM settings - jvmArgs = [ - '-XX:+UseG1GC', - '-Xmx2g', - '-XX:+UnlockExperimentalVMOptions' - ] - doFirst { file("${projectDir}/benchmark-results").mkdirs() } + + doLast { + javaexec { + classpath = sourceSets.jmh.runtimeClasspath + mainClass = 'org.openjdk.jmh.Main' + args = [ + 'ComparisonBenchmark', + '-f', '1', + '-wi', '2', + '-i', '3', + '-rf', 'json', + '-rff', "${projectDir}/benchmark-results/all-results.json" + ] + jvmArgs = [ + '-XX:+UseG1GC', + '-Xmx2g' + ] + } + } } compileJava { From 8831922bc690b490f3acb1ba963525c61260f27f Mon Sep 17 00:00:00 2001 From: expand3d <> Date: Thu, 5 Jun 2025 21:56:09 -0400 Subject: [PATCH 32/49] lazy directory deserialization --- .github/workflows/ci.yml | 110 +--- build.gradle | 183 +----- .../benchmark/ComparisonBenchmark.java | 18 +- .../java/com/imprint/core/ImprintBuffers.java | 451 +++++++++++++ .../java/com/imprint/core/ImprintRecord.java | 612 ++++-------------- src/main/java/com/imprint/util/VarInt.java | 55 +- 6 files changed, 632 insertions(+), 797 deletions(-) create mode 100644 src/main/java/com/imprint/core/ImprintBuffers.java diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a80f529..378ebb7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,9 +2,9 @@ name: CI on: push: - branches: [ main, dev ] + branches: [ main ] pull_request: - branches: [ main, dev ] + branches: [ main ] jobs: test: @@ -41,108 +41,4 @@ jobs: run: ./gradlew test - name: Run build - run: ./gradlew build - - benchmark: - runs-on: ubuntu-latest - needs: test - # Only run benchmarks on main branch pushes and PRs to main to avoid excessive CI time - if: github.ref == 'refs/heads/main' || github.base_ref == 'main' - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up JDK 11 - uses: actions/setup-java@v4 - with: - java-version: '11' - distribution: 'temurin' - - - name: Cache Gradle dependencies - uses: actions/cache@v4 - with: - path: | - ~/.gradle/caches - ~/.gradle/wrapper - key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} - restore-keys: | - ${{ runner.os }}-gradle- - - - name: Make gradlew executable - run: chmod +x ./gradlew - - - name: Create benchmark results directory - run: mkdir -p benchmark-results - - - name: Run serialization benchmarks - run: ./gradlew jmhRunSerializationBenchmarks - continue-on-error: true - - - name: Run deserialization benchmarks - run: ./gradlew jmhRunDeserializationBenchmarks - continue-on-error: true - - - name: Run field access benchmarks - run: ./gradlew jmhRunFieldAccessBenchmarks - continue-on-error: true - - - name: Run size comparison benchmarks - run: ./gradlew jmhRunSizeComparisonBenchmarks - continue-on-error: true - - - name: List benchmark results - run: | - echo "Contents of benchmark-results directory:" - ls -la benchmark-results/ || echo "benchmark-results directory not found" - echo "Working directory contents:" - ls -la - - - name: Upload benchmark results - uses: actions/upload-artifact@v4 - if: always() - with: - name: benchmark-results-${{ github.sha }} - path: benchmark-results/ - retention-days: 30 - - # Optional: Run full benchmark suite on releases - benchmark-full: - runs-on: ubuntu-latest - if: startsWith(github.ref, 'refs/tags/') - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up JDK 11 - uses: actions/setup-java@v4 - with: - java-version: '11' - distribution: 'temurin' - - - name: Cache Gradle dependencies - uses: actions/cache@v4 - with: - path: | - ~/.gradle/caches - ~/.gradle/wrapper - key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} - restore-keys: | - ${{ runner.os }}-gradle- - - - name: Make gradlew executable - run: chmod +x ./gradlew - - - name: Create benchmark results directory - run: mkdir -p benchmark-results - - - name: Run full benchmark suite - run: ./gradlew jmhRunAllBenchmarks - - - name: Upload full benchmark results - uses: actions/upload-artifact@v4 - with: - name: full-benchmark-results-${{ github.ref_name }} - path: benchmark-results/ - retention-days: 90 \ No newline at end of file + run: ./gradlew build \ No newline at end of file diff --git a/build.gradle b/build.gradle index 6c97a24..33b1645 100644 --- a/build.gradle +++ b/build.gradle @@ -166,11 +166,11 @@ test { } } -// JMH configuration - optimized for Java 11 +// JMH configuration jmh { - fork = 1 - warmupIterations = 2 // Reduced for faster CI - iterations = 3 // Reduced for faster CI + fork = 2 + warmupIterations = 3 + iterations = 5 resultFormat = 'JSON' includeTests = false resultsFile = file("${projectDir}/benchmark-results/jmh-results-${new Date().format('yyyy-MM-dd-HHmmss')}.json") @@ -184,181 +184,6 @@ jmh { ] } -// Create individual benchmark tasks for CI pipeline -tasks.register('jmhRunSerializationBenchmarks') { - dependsOn compileJmhJava - description = 'Run serialization benchmarks' - group = 'benchmarking' - - doFirst { - file("${projectDir}/benchmark-results").mkdirs() - } - - doLast { - javaexec { - classpath = sourceSets.jmh.runtimeClasspath - mainClass = 'org.openjdk.jmh.Main' - args = [ - '.*serialize.*', - '-f', '1', - '-wi', '2', - '-i', '3', - '-rf', 'json', - '-rff', "${projectDir}/benchmark-results/serialization-results.json" - ] - jvmArgs = [ - '-XX:+UseG1GC', - '-Xmx2g' - ] - } - } -} - -tasks.register('jmhRunDeserializationBenchmarks') { - dependsOn compileJmhJava - description = 'Run deserialization benchmarks' - group = 'benchmarking' - - doFirst { - file("${projectDir}/benchmark-results").mkdirs() - } - - doLast { - javaexec { - classpath = sourceSets.jmh.runtimeClasspath - mainClass = 'org.openjdk.jmh.Main' - args = [ - '.*deserialize.*', - '-f', '1', - '-wi', '2', - '-i', '3', - '-rf', 'json', - '-rff', "${projectDir}/benchmark-results/deserialization-results.json" - ] - jvmArgs = [ - '-XX:+UseG1GC', - '-Xmx2g' - ] - } - } -} - -tasks.register('jmhRunFieldAccessBenchmarks') { - dependsOn compileJmhJava - description = 'Run field access benchmarks' - group = 'benchmarking' - - doFirst { - file("${projectDir}/benchmark-results").mkdirs() - } - - doLast { - javaexec { - classpath = sourceSets.jmh.runtimeClasspath - mainClass = 'org.openjdk.jmh.Main' - args = [ - '.*singleFieldAccess.*', - '-f', '1', - '-wi', '2', - '-i', '3', - '-rf', 'json', - '-rff', "${projectDir}/benchmark-results/fieldaccess-results.json" - ] - jvmArgs = [ - '-XX:+UseG1GC', - '-Xmx2g' - ] - } - } -} - -tasks.register('jmhRunSizeComparisonBenchmarks') { - dependsOn compileJmhJava - description = 'Run size comparison benchmarks' - group = 'benchmarking' - - doFirst { - file("${projectDir}/benchmark-results").mkdirs() - } - - doLast { - javaexec { - classpath = sourceSets.jmh.runtimeClasspath - mainClass = 'org.openjdk.jmh.Main' - args = [ - '.*measure.*', - '-f', '1', - '-wi', '2', - '-i', '3', - '-rf', 'json', - '-rff', "${projectDir}/benchmark-results/size-results.json" - ] - jvmArgs = [ - '-XX:+UseG1GC', - '-Xmx2g' - ] - } - } -} - -tasks.register('jmhRunMergeBenchmarks') { - dependsOn compileJmhJava - description = 'Run merge operation benchmarks' - group = 'benchmarking' - - doFirst { - file("${projectDir}/benchmark-results").mkdirs() - } - - doLast { - javaexec { - classpath = sourceSets.jmh.runtimeClasspath - mainClass = 'org.openjdk.jmh.Main' - args = [ - '.*merge.*', - '-f', '1', - '-wi', '2', - '-i', '3', - '-rf', 'json', - '-rff', "${projectDir}/benchmark-results/merge-results.json" - ] - jvmArgs = [ - '-XX:+UseG1GC', - '-Xmx2g' - ] - } - } -} - -tasks.register('jmhRunAllBenchmarks') { - dependsOn compileJmhJava - description = 'Run all comparison benchmarks' - group = 'benchmarking' - - doFirst { - file("${projectDir}/benchmark-results").mkdirs() - } - - doLast { - javaexec { - classpath = sourceSets.jmh.runtimeClasspath - mainClass = 'org.openjdk.jmh.Main' - args = [ - 'ComparisonBenchmark', - '-f', '1', - '-wi', '2', - '-i', '3', - '-rf', 'json', - '-rff', "${projectDir}/benchmark-results/all-results.json" - ] - jvmArgs = [ - '-XX:+UseG1GC', - '-Xmx2g' - ] - } - } -} - compileJava { options.compilerArgs << '-Xlint:unchecked' options.deprecation = true diff --git a/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java b/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java index ce2fbcb..dd62457 100644 --- a/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java +++ b/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java @@ -37,8 +37,8 @@ @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.NANOSECONDS) @State(Scope.Benchmark) -@Warmup(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS) -@Measurement(iterations = 10, time = 1, timeUnit = TimeUnit.SECONDS) +@Warmup(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS) @Fork(1) @SuppressWarnings("unused") public class ComparisonBenchmark { @@ -325,7 +325,7 @@ public void measureFlatBuffersSize(Blackhole bh) { // ===== MERGE SIMULATION BENCHMARKS ===== - @Benchmark + //@Benchmark public void mergeImprint(Blackhole bh) throws Exception { var record1Buffer = imprintBytesBuffer.duplicate(); var record2Data = createTestRecord2(); @@ -338,7 +338,7 @@ public void mergeImprint(Blackhole bh) throws Exception { bh.consume(merged); } - @Benchmark + //@Benchmark public void mergeJacksonJson(Blackhole bh) throws Exception { var record1 = jacksonJsonMapper.readValue(jacksonJsonBytes, TestRecord.class); var record2Data = createTestRecord2(); @@ -350,7 +350,7 @@ public void mergeJacksonJson(Blackhole bh) throws Exception { bh.consume(result); } - @Benchmark + //@Benchmark public void mergeKryo(Blackhole bh) { Input input1 = new Input(new ByteArrayInputStream(kryoBytes)); var record1 = kryo.readObject(input1, TestRecord.class); @@ -367,7 +367,7 @@ public void mergeKryo(Blackhole bh) { bh.consume(result); } - @Benchmark + //@Benchmark public void mergeMessagePack(Blackhole bh) throws Exception { var record1 = messagePackMapper.readValue(messagePackBytes, TestRecord.class); var record2Data = createTestRecord2(); @@ -379,7 +379,7 @@ public void mergeMessagePack(Blackhole bh) throws Exception { bh.consume(result); } - @Benchmark + //@Benchmark public void mergeAvro(Blackhole bh) throws Exception { var record1 = deserializeWithAvro(avroBytes); var record2Data = createTestRecord2(); @@ -391,7 +391,7 @@ public void mergeAvro(Blackhole bh) throws Exception { bh.consume(result); } - @Benchmark + //@Benchmark public void mergeProtobuf(Blackhole bh) throws Exception { var record1 = TestRecordProto.TestRecord.parseFrom(protobufBytes); var record2Data = createTestRecord2(); @@ -403,7 +403,7 @@ public void mergeProtobuf(Blackhole bh) throws Exception { bh.consume(result); } - @Benchmark + //@Benchmark public void mergeFlatBuffers(Blackhole bh) { var record1 = TestRecordFB.getRootAsTestRecordFB(flatbuffersBytes.duplicate()); var record2Data = createTestRecord2(); diff --git a/src/main/java/com/imprint/core/ImprintBuffers.java b/src/main/java/com/imprint/core/ImprintBuffers.java new file mode 100644 index 0000000..f6a341b --- /dev/null +++ b/src/main/java/com/imprint/core/ImprintBuffers.java @@ -0,0 +1,451 @@ +package com.imprint.core; + +import com.imprint.Constants; +import com.imprint.error.ErrorType; +import com.imprint.error.ImprintException; +import com.imprint.types.TypeCode; +import com.imprint.util.VarInt; +import lombok.Getter; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +/** + * Manages the raw buffers for an Imprint record with lazy directory parsing. + * Encapsulates all buffer operations and provides zero-copy field access. + * + *

Buffer Layout Overview:

+ * 
+ * directoryBuffer: [VarInt count][DirectoryEntry 1][DirectoryEntry 2]...[DirectoryEntry N]
+ * payload:         [Field 1 data][Field 2 data]...[Field N data]
+ *
+ * + *

Each DirectoryEntry contains: [fieldId:2bytes][typeCode:1byte][offset:4bytes]

+ */ +@Getter +public final class ImprintBuffers { + private final ByteBuffer directoryBuffer; // Raw directory bytes (includes count) + private final ByteBuffer payload; // Read-only payload view + + // Lazy-loaded directory state + private List parsedDirectory; + private boolean directoryParsed = false; + private int directoryCount = -1; // Cached count to avoid repeated VarInt decoding + + /** + * Creates buffers from raw data (used during deserialization). + * + * @param directoryBuffer Raw directory bytes including VarInt count and all entries. + * Format: [VarInt count][Entry1][Entry2]...[EntryN] + * @param payload Raw payload data containing all field values sequentially + */ + public ImprintBuffers(ByteBuffer directoryBuffer, ByteBuffer payload) { + this.directoryBuffer = directoryBuffer.asReadOnlyBuffer(); + this.payload = payload.asReadOnlyBuffer(); + } + + /** + * Creates buffers from pre-parsed directory (used during construction). + * This is more efficient when the directory is already known. + * + * @param directory Parsed directory entries, must be sorted by fieldId + * @param payload Raw payload data containing all field values + */ + public ImprintBuffers(List directory, ByteBuffer payload) { + this.parsedDirectory = Collections.unmodifiableList(Objects.requireNonNull(directory)); + this.directoryParsed = true; + this.directoryCount = directory.size(); + this.payload = payload.asReadOnlyBuffer(); + this.directoryBuffer = createDirectoryBuffer(directory); + } + + /** + * Get a zero-copy ByteBuffer view of a field's data. + * + *

Buffer Positioning Logic:

+ *
    + *
  1. Find the directory entry for the requested fieldId
    2. + *
  2. Use entry.offset as start position in payload
    3. + *
  3. Find end position by looking at next field's offset (or payload end)
    4. + *
  4. Create a slice view: payload[startOffset:endOffset]
    5. + *
+ * + * @param fieldId The field identifier to retrieve + * @return Zero-copy ByteBuffer positioned at field data, or null if field not found + * @throws ImprintException if buffer bounds are invalid or directory is corrupted + */ + public ByteBuffer getFieldBuffer(int fieldId) throws ImprintException { + var entry = findDirectoryEntry(fieldId); + if (entry == null) + return null; + + int startOffset = entry.getOffset(); + int endOffset = findEndOffset(entry); + + if (startOffset < 0 || endOffset < 0 || startOffset > payload.limit() || + endOffset > payload.limit() || startOffset > endOffset) { + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, + "Invalid field buffer range: start=" + startOffset + ", end=" + endOffset + ", payloadLimit=" + payload.limit()); + } + + ByteBuffer fieldBuffer = payload.duplicate(); + fieldBuffer.position(startOffset).limit(endOffset); + return fieldBuffer; + } + + /** + * Find a directory entry for the given field ID using the most efficient method. + * + *

Search Strategy:

+ *
    + *
  • If directory is parsed: binary search on in-memory List<DirectoryEntry>
    • + *
  • If directory is raw: binary search directly on raw bytes (faster for single lookups)
    • + *
+ * @param fieldId The field identifier to find + * @return DirectoryEntry if found, null otherwise + * @throws ImprintException if directory buffer is corrupted or truncated + */ + public DirectoryEntry findDirectoryEntry(int fieldId) throws ImprintException { + if (directoryParsed) { + int index = findDirectoryIndexInParsed(fieldId); + return index >= 0 ? parsedDirectory.get(index) : null; + } else { + return findFieldEntryInRawDirectory(fieldId); + } + } + + /** + * Get the full directory, parsing it if necessary. + * + *

Lazy Parsing Behavior:

+ *
    + *
  • First call: Parses entire directory from raw bytes into List<DirectoryEntry>
    • + *
  • Subsequent calls: Returns cached parsed directory
    • + *
  • Note - the method is not synchronized and assumes single-threaded usage.
    • + *
+ * + *

When to use: Call this if you need to access multiple fields + * from the same record. For single field access, direct field getters are more efficient.

+ * + * @return Immutable list of directory entries, sorted by fieldId + */ + public List getDirectory() { + ensureDirectoryParsed(); + return parsedDirectory; + } + + /** + * Get the directory count without fully parsing the directory. + *

+ * This method avoids parsing the entire directory when only the count is needed. + *

    + *
  1. Return cached count if available (directoryCount >= 0)
    2. + *
  2. Return parsed directory size if directory is already parsed
    3. + *
  3. Decode VarInt from raw buffer and cache the result
    4. + *
+ * + *

VarInt Decoding: The count is stored as a VarInt at the beginning + * of the directoryBuffer. This method reads just enough bytes to decode the count.

+ * + * @return Number of fields in the directory, or 0 if decoding fails + */ + public int getDirectoryCount() { + if (directoryCount >= 0) + return directoryCount; + if (directoryParsed) + return parsedDirectory.size(); + + // Decode from buffer and cache + try { + var countBuffer = directoryBuffer.duplicate(); + directoryCount = VarInt.decode(countBuffer).getValue(); + return directoryCount; + } catch (Exception e) { + return 0; + } + } + + /** + * Create a new buffer containing the serialized directory. + * + *

Output Format:

+ * 
+     * [VarInt count][DirectoryEntry 1][DirectoryEntry 2]...[DirectoryEntry N]
+     *
+ * + *

Each DirectoryEntry is serialized as: [fieldId:2bytes][typeCode:1byte][offset:4bytes]

+ * + * + * @return New ByteBuffer containing the complete serialized directory + */ + public ByteBuffer serializeDirectory() { + ensureDirectoryParsed(); + return createDirectoryBuffer(parsedDirectory); + } + + // ========== PRIVATE METHODS ========== + + /** + * Binary search on raw directory bytes to find a specific field. + * + *
    + *
  1. Position buffer at start and decode VarInt count (cache for future use)
    2. + *
  2. Calculate directory start position after VarInt
    3. + *
  3. For binary search mid-point: entryPos = startPos + (mid * DIR_ENTRY_BYTES)
    4. + *
  4. Read fieldId from calculated position (first 2 bytes of entry)
    5. + *
  5. Compare fieldId and adjust search bounds
    6. + *
  6. When found: reposition buffer and deserialize complete entry
    7. + *
+ * + *

All buffer positions are bounds-checked before access.

+ * + * @param fieldId Field identifier to search for + * @return Complete DirectoryEntry if found, null if not found + * @throws ImprintException if buffer is truncated or corrupted + */ + private DirectoryEntry findFieldEntryInRawDirectory(int fieldId) throws ImprintException { + var searchBuffer = directoryBuffer.duplicate(); + searchBuffer.order(ByteOrder.LITTLE_ENDIAN); + + // Decode directory count (cache it) + if (directoryCount < 0) + directoryCount = VarInt.decode(searchBuffer).getValue(); + else + VarInt.decode(searchBuffer); // Skip past the count + + if (directoryCount == 0) + return null; + + int directoryStartPos = searchBuffer.position(); + int low = 0; + int high = directoryCount - 1; + + while (low <= high) { + int mid = (low + high) >>> 1; + int entryPos = directoryStartPos + (mid * Constants.DIR_ENTRY_BYTES); + + if (entryPos + Constants.DIR_ENTRY_BYTES > searchBuffer.limit()) { + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, + "Directory entry at position " + entryPos + " exceeds buffer limit"); + } + + searchBuffer.position(entryPos); + short midFieldId = searchBuffer.getShort(); + + if (midFieldId < fieldId) { + low = mid + 1; + } else if (midFieldId > fieldId) { + high = mid - 1; + } else { + // Found it - read the complete entry + searchBuffer.position(entryPos); + return deserializeDirectoryEntry(searchBuffer); + } + } + + return null; + } + + /** + * + * @param fieldId Field identifier to find + * @return Index of the field if found, or negative insertion point if not found + */ + private int findDirectoryIndexInParsed(int fieldId) { + if (!directoryParsed) + return -1; + int low = 0; + int high = parsedDirectory.size() - 1; + while (low <= high) { + int mid = (low + high) >>> 1; + int midFieldId = parsedDirectory.get(mid).getId(); + if (midFieldId < fieldId) + low = mid + 1; + else if (midFieldId > fieldId) + high = mid - 1; + else + return mid; + } + return -(low + 1); + } + + /** + * Find the end offset for a field by looking at the next field's offset. + * + *
    + *
  • Field data spans from: entry.offset to nextField.offset (exclusive)
    • + *
  • Last field spans from: entry.offset to payload.limit()
    • + *
  • This works because directory entries are sorted by fieldId
    • + *
+ * + *

Search Strategy:

+ *
    + *
  • If directory parsed: Use binary search result + 1 to get next entry
    • + *
  • If directory raw: Scan raw entries until fieldId > currentFieldId
    • + *
+ * + * @param entry The directory entry whose end offset we need to find + * @return End offset (exclusive) for the field data + * @throws ImprintException if directory scanning fails + */ + private int findEndOffset(DirectoryEntry entry) throws ImprintException { + if (directoryParsed) { + int entryIndex = findDirectoryIndexInParsed(entry.getId()); + return (entryIndex + 1 < parsedDirectory.size()) ? + parsedDirectory.get(entryIndex + 1).getOffset() : payload.limit(); + } else + return findNextOffsetInRawDirectory(entry.getId()); + } + + /** + * Scan raw directory to find the next field's offset after currentFieldId. + * + *
    + *
  1. Position buffer after VarInt count
    2. + *
  2. For each directory entry at position: startPos + (i * DIR_ENTRY_BYTES)
    3. + *
  3. Read fieldId (first 2 bytes) and offset (bytes 3-6)
    4. + *
  4. Return offset of first field where fieldId > currentFieldId
    5. + *
  5. If no next field found, return payload.limit()
    6. + *
+ * + * @param currentFieldId Find the next field after this fieldId + * @return Offset where the next field starts, or payload.limit() if this is the last field + * @throws ImprintException if directory buffer is corrupted + */ + private int findNextOffsetInRawDirectory(int currentFieldId) throws ImprintException { + var scanBuffer = directoryBuffer.duplicate(); + scanBuffer.order(ByteOrder.LITTLE_ENDIAN); + + int count = (directoryCount >= 0) ? directoryCount : VarInt.decode(scanBuffer).getValue(); + if (count == 0) + return payload.limit(); + if (directoryCount >= 0) + VarInt.decode(scanBuffer); // Skip count if cached + + int directoryStartPos = scanBuffer.position(); + + for (int i = 0; i < count; i++) { + int entryPos = directoryStartPos + (i * Constants.DIR_ENTRY_BYTES); + + if (entryPos + Constants.DIR_ENTRY_BYTES > scanBuffer.limit()) + return payload.limit(); + + scanBuffer.position(entryPos); + short fieldId = scanBuffer.getShort(); + scanBuffer.get(); // skip type + int offset = scanBuffer.getInt(); + + if (fieldId > currentFieldId) + return offset; + } + + return payload.limit(); + } + + /** + * Parse the full directory if not already parsed. + * + *
    + *
  1. Duplicate directoryBuffer to avoid affecting original position
    2. + *
  2. Set byte order to LITTLE_ENDIAN for consistent reading
    3. + *
  3. Decode VarInt count and cache it
    4. + *
  4. Read 'count' directory entries sequentially
    5. + *
  5. Each entry: [fieldId:2bytes][typeCode:1byte][offset:4bytes]
    6. + *
  6. Store as immutable list and mark as parsed
    7. + *
+ * + *

Error Handling: If parsing fails, throws RuntimeException + * since this indicates corrupted data that should never happen in normal operation.

+ * + *

Will return immediately if directory has already been parsed.

+ */ + private void ensureDirectoryParsed() { + if (directoryParsed) + return; + try { + var parseBuffer = directoryBuffer.duplicate(); + parseBuffer.order(ByteOrder.LITTLE_ENDIAN); + + var countResult = VarInt.decode(parseBuffer); + int count = countResult.getValue(); + this.directoryCount = count; + + var directory = new ArrayList(count); + for (int i = 0; i < count; i++) { + directory.add(deserializeDirectoryEntry(parseBuffer)); + } + + this.parsedDirectory = Collections.unmodifiableList(directory); + this.directoryParsed = true; + } catch (ImprintException e) { + throw new RuntimeException("Failed to parse directory", e); + } + } + + /** + * Create directory buffer from parsed entries. + * + *

Serialization Format:

+ *
    + *
  1. Calculate buffer size: VarInt.encodedLength(count) + (count * DIR_ENTRY_BYTES)
    2. + *
  2. Allocate ByteBuffer with LITTLE_ENDIAN byte order
    3. + *
  3. Write VarInt count
    4. + *
  4. Write each directory entry: [fieldId:2][typeCode:1][offset:4]
    5. + *
  5. Flip buffer and return read-only view
    6. + *
+ * + * @param directory List of directory entries to serialize + * @return Read-only ByteBuffer containing serialized directory, or empty buffer on error + */ + private ByteBuffer createDirectoryBuffer(List directory) { + try { + int bufferSize = VarInt.encodedLength(directory.size()) + + (directory.size() * Constants.DIR_ENTRY_BYTES); + var buffer = ByteBuffer.allocate(bufferSize); + buffer.order(ByteOrder.LITTLE_ENDIAN); + + VarInt.encode(directory.size(), buffer); + for (var entry : directory) { + serializeDirectoryEntry(entry, buffer); + } + + buffer.flip(); + return buffer.asReadOnlyBuffer(); + } catch (Exception e) { + return ByteBuffer.allocate(0).asReadOnlyBuffer(); + } + } + + /** + * Serialize a single directory entry to the buffer. + * Format: [fieldId:2bytes][typeCode:1byte][offset:4bytes] + */ + private void serializeDirectoryEntry(DirectoryEntry entry, ByteBuffer buffer) { + buffer.putShort(entry.getId()); + buffer.put(entry.getTypeCode().getCode()); + buffer.putInt(entry.getOffset()); + } + + /** + * Deserialize a single directory entry from the buffer. + * Reads: [fieldId:2bytes][typeCode:1byte][offset:4bytes] + * + * @param buffer Buffer positioned at the start of a directory entry + * @return Parsed DirectoryEntry + * @throws ImprintException if buffer doesn't contain enough bytes + */ + private DirectoryEntry deserializeDirectoryEntry(ByteBuffer buffer) throws ImprintException { + if (buffer.remaining() < Constants.DIR_ENTRY_BYTES) + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, "Not enough bytes for directory entry"); + + short id = buffer.getShort(); + var typeCode = TypeCode.fromByte(buffer.get()); + int offset = buffer.getInt(); + + return new DirectoryEntry(id, typeCode, offset); + } +} \ No newline at end of file diff --git a/src/main/java/com/imprint/core/ImprintRecord.java b/src/main/java/com/imprint/core/ImprintRecord.java index da6b6e0..6abc9cf 100644 --- a/src/main/java/com/imprint/core/ImprintRecord.java +++ b/src/main/java/com/imprint/core/ImprintRecord.java @@ -11,507 +11,224 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; -import java.util.*; +import java.util.List; +import java.util.Map; +import java.util.Objects; /** - * An Imprint record containing a header, field directory, and payload. - * Uses ByteBuffer for zero-copy operations to achieve low latency. - * - *

This implementation uses lazy directory parsing for optimal single field access performance. - * The directory is only parsed when needed, and binary search is performed directly on raw bytes - * when possible.

- * - *

Performance Note: All ByteBuffers should be array-backed - * (hasArray() == true) for optimal zero-copy performance. Direct buffers - * may cause performance degradation.

+ * An Imprint record containing a header and buffer management. + * Delegates all buffer operations to ImprintBuffers for cleaner separation. */ @Getter public final class ImprintRecord { private final Header header; - private final ByteBuffer directoryBuffer; // Raw directory bytes - private final ByteBuffer payload; // Read-only view for zero-copy - - // Lazy-loaded directory state - private List parsedDirectory; - private boolean directoryParsed = false; - - // Cache for parsed directory count to avoid repeated VarInt decoding - private int directoryCount = -1; + private final ImprintBuffers buffers; /** - * Creates a new ImprintRecord with lazy directory parsing. - * - * @param header the record header - * @param directoryBuffer raw directory bytes (including count) - * @param payload the payload buffer. Should be array-backed for optimal performance. + * Creates a record from deserialized components. */ - private ImprintRecord(Header header, ByteBuffer directoryBuffer, ByteBuffer payload) { + private ImprintRecord(Header header, ImprintBuffers buffers) { this.header = Objects.requireNonNull(header, "Header cannot be null"); - this.directoryBuffer = directoryBuffer.asReadOnlyBuffer(); - this.payload = payload.asReadOnlyBuffer(); // Zero-copy read-only view + this.buffers = Objects.requireNonNull(buffers, "Buffers cannot be null"); } /** - * Creates a new ImprintRecord with pre-parsed directory (used by ImprintWriter). - * This constructor is used when the directory is already known and parsed. - * - * @param header the record header - * @param directory the parsed directory entries - * @param payload the payload buffer. Should be array-backed for optimal performance. + * Creates a record from pre-parsed directory (used by ImprintWriter). */ ImprintRecord(Header header, List directory, ByteBuffer payload) { this.header = Objects.requireNonNull(header, "Header cannot be null"); - this.parsedDirectory = Collections.unmodifiableList(Objects.requireNonNull(directory, "Directory cannot be null")); - this.directoryParsed = true; - this.directoryCount = directory.size(); - this.payload = payload.asReadOnlyBuffer(); - - // Create directory buffer for serialization compatibility - this.directoryBuffer = createDirectoryBuffer(directory); + this.buffers = new ImprintBuffers(directory, payload); } + // ========== FIELD ACCESS METHODS ========== + /** * Get a value by field ID, deserializing it on demand. * Returns null if the field is not found. - * Note: If the field exists and is an explicit NULL type, this will return Value.NullValue.INSTANCE - * - *

Performance Note: Accessing fields one-by-one is optimized for single field access. - * If you need to access many fields from the same record, consider calling getDirectory() first - * to parse the full directory once, then access fields normally.

*/ public Value getValue(int fieldId) throws ImprintException { - DirectoryEntry entry = findDirectoryEntry(fieldId); - if (entry == null) { + var entry = buffers.findDirectoryEntry(fieldId); + if (entry == null) return null; - } - return deserializeValue(entry.getTypeCode(), getFieldBufferFromEntry(entry)); + var fieldBuffer = buffers.getFieldBuffer(fieldId); + if (fieldBuffer == null) + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, "Failed to get buffer for field " + fieldId); + + return deserializeValue(entry.getTypeCode(), fieldBuffer); } /** - * Get the raw bytes for a field without deserializing. - * Returns a zero-copy ByteBuffer view, or null if field not found. + * Get raw bytes for a field without deserializing. */ public ByteBuffer getRawBytes(int fieldId) { try { - DirectoryEntry entry = findDirectoryEntry(fieldId); - if (entry == null) { - return null; - } - - return getFieldBufferFromEntry(entry).asReadOnlyBuffer(); + return buffers.getFieldBuffer(fieldId); } catch (ImprintException e) { return null; } } - /** - * Find a directory entry for the given field ID. - * Uses the most efficient method based on current state. - */ - private DirectoryEntry findDirectoryEntry(int fieldId) throws ImprintException { - if (directoryParsed) { - // Use parsed directory - int index = findDirectoryIndexInParsed(fieldId); - return index >= 0 ? parsedDirectory.get(index) : null; - } else { - // Use fast binary search on raw bytes - return findFieldEntryFast(fieldId); - } - } - - /** - * Fast binary search directly on raw directory bytes. - * This avoids parsing the entire directory for single field access. - */ - private DirectoryEntry findFieldEntryFast(int fieldId) throws ImprintException { - ByteBuffer searchBuffer = directoryBuffer.duplicate(); - searchBuffer.order(ByteOrder.LITTLE_ENDIAN); - - // Decode directory count (cache it to avoid repeated decoding) - if (directoryCount < 0) { - directoryCount = VarInt.decode(searchBuffer).getValue(); - } else { - // Skip past the VarInt count - VarInt.decode(searchBuffer); - } - - if (directoryCount == 0) { - return null; - } - - // Now searchBuffer.position() points to the first directory entry - int directoryStartPos = searchBuffer.position(); - - int low = 0; - int high = directoryCount - 1; - - while (low <= high) { - int mid = (low + high) >>> 1; - - // Calculate position of mid entry - int entryPos = directoryStartPos + (mid * Constants.DIR_ENTRY_BYTES); - - // Bounds check - if (entryPos + Constants.DIR_ENTRY_BYTES > searchBuffer.limit()) { - throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, - "Directory entry at position " + entryPos + " exceeds buffer limit " + searchBuffer.limit()); - } - - searchBuffer.position(entryPos); - short midFieldId = searchBuffer.getShort(); - - if (midFieldId < fieldId) { - low = mid + 1; - } else if (midFieldId > fieldId) { - high = mid - 1; - } else { - // Found it - read the complete entry - searchBuffer.position(entryPos); - return deserializeDirectoryEntry(searchBuffer); - } - } - - return null; // Not found - } - /** * Get the directory (parsing it if necessary). - * This maintains backward compatibility with existing code. - * - *

Performance Tip: If you plan to access many fields from this record, - * call this method first to parse the directory once, then use the field accessor methods. - * This is more efficient than accessing fields one-by-one when you need multiple fields.

*/ public List getDirectory() { - ensureDirectoryParsed(); - return parsedDirectory; + return buffers.getDirectory(); } - /** - * Get a ByteBuffer view of a field's data from a DirectoryEntry. - */ - private ByteBuffer getFieldBufferFromEntry(DirectoryEntry entry) throws ImprintException { - int startOffset = entry.getOffset(); - - // Find end offset - int endOffset; - if (directoryParsed) { - // Use parsed directory to find next entry - int entryIndex = findDirectoryIndexInParsed(entry.getId()); - endOffset = (entryIndex + 1 < parsedDirectory.size()) ? - parsedDirectory.get(entryIndex + 1).getOffset() : payload.limit(); - } else { - // Calculate end offset by finding the next field in the directory - endOffset = findNextOffsetInRawDirectory(entry.getId()); - } + // ========== TYPED GETTERS ========== - if (startOffset < 0 || endOffset < 0 || startOffset > payload.limit() || - endOffset > payload.limit() || startOffset > endOffset) { - throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, - "Invalid field buffer range: start=" + startOffset + ", end=" + endOffset + - ", payloadLimit=" + payload.limit()); - } - - var fieldBuffer = payload.duplicate(); - fieldBuffer.position(startOffset).limit(endOffset); - return fieldBuffer; + public boolean getBoolean(int fieldId) throws ImprintException { + return getTypedValueOrThrow(fieldId, TypeCode.BOOL, Value.BoolValue.class, "boolean").getValue(); } - /** - * Find the next field's offset by scanning the raw directory. - * This is used when the directory isn't fully parsed yet. - */ - private int findNextOffsetInRawDirectory(int currentFieldId) throws ImprintException { - ByteBuffer scanBuffer = directoryBuffer.duplicate(); - scanBuffer.order(ByteOrder.LITTLE_ENDIAN); - - // Get directory count - int count = (directoryCount >= 0) ? directoryCount : VarInt.decode(scanBuffer).getValue(); - if (count == 0) { - return payload.limit(); - } - - // Skip past count if we just decoded it - if (directoryCount < 0) { - // VarInt.decode already advanced the position - } else { - VarInt.decode(scanBuffer); // Skip past the count - } - - int directoryStartPos = scanBuffer.position(); - - for (int i = 0; i < count; i++) { - int entryPos = directoryStartPos + (i * Constants.DIR_ENTRY_BYTES); - - // Bounds check - if (entryPos + Constants.DIR_ENTRY_BYTES > scanBuffer.limit()) { - return payload.limit(); - } - - scanBuffer.position(entryPos); - short fieldId = scanBuffer.getShort(); - scanBuffer.get(); // skip type - int offset = scanBuffer.getInt(); + public int getInt32(int fieldId) throws ImprintException { + return getTypedValueOrThrow(fieldId, TypeCode.INT32, Value.Int32Value.class, "int32").getValue(); + } - if (fieldId > currentFieldId) { - return offset; // Found next field's offset - } - } + public long getInt64(int fieldId) throws ImprintException { + return getTypedValueOrThrow(fieldId, TypeCode.INT64, Value.Int64Value.class, "int64").getValue(); + } - return payload.limit(); // No next field, use payload end + public float getFloat32(int fieldId) throws ImprintException { + return getTypedValueOrThrow(fieldId, TypeCode.FLOAT32, Value.Float32Value.class, "float32").getValue(); } - /** - * Ensure the directory is fully parsed (thread-safe). - */ - private synchronized void ensureDirectoryParsed() { - if (directoryParsed) { - return; - } + public double getFloat64(int fieldId) throws ImprintException { + return getTypedValueOrThrow(fieldId, TypeCode.FLOAT64, Value.Float64Value.class, "float64").getValue(); + } - try { - ByteBuffer parseBuffer = directoryBuffer.duplicate(); - parseBuffer.order(ByteOrder.LITTLE_ENDIAN); + public String getString(int fieldId) throws ImprintException { + var value = getValidatedValue(fieldId, "STRING"); + if (value instanceof Value.StringValue) + return ((Value.StringValue) value).getValue(); + if (value instanceof Value.StringBufferValue) + return ((Value.StringBufferValue) value).getValue(); + throw new ImprintException(ErrorType.TYPE_MISMATCH, "Field " + fieldId + " is not a STRING"); + } - VarInt.DecodeResult countResult = VarInt.decode(parseBuffer); - int count = countResult.getValue(); - this.directoryCount = count; // Cache the count + public byte[] getBytes(int fieldId) throws ImprintException { + var value = getValidatedValue(fieldId, "BYTES"); + if (value instanceof Value.BytesValue) + return ((Value.BytesValue) value).getValue(); + if (value instanceof Value.BytesBufferValue) + return ((Value.BytesBufferValue) value).getValue(); + throw new ImprintException(ErrorType.TYPE_MISMATCH, "Field " + fieldId + " is not BYTES"); + } - List directory = new ArrayList<>(count); - for (int i = 0; i < count; i++) { - directory.add(deserializeDirectoryEntry(parseBuffer)); - } + public List getArray(int fieldId) throws ImprintException { + return getTypedValueOrThrow(fieldId, TypeCode.ARRAY, Value.ArrayValue.class, "ARRAY").getValue(); + } - this.parsedDirectory = Collections.unmodifiableList(directory); - this.directoryParsed = true; - } catch (ImprintException e) { - throw new RuntimeException("Failed to parse directory", e); - } + public Map getMap(int fieldId) throws ImprintException { + return getTypedValueOrThrow(fieldId, TypeCode.MAP, Value.MapValue.class, "MAP").getValue(); } - /** - * Creates a directory buffer from parsed directory entries. - * This is used when creating records with pre-parsed directories (e.g., from ImprintWriter). - */ - private ByteBuffer createDirectoryBuffer(List directory) { - try { - int bufferSize = VarInt.encodedLength(directory.size()) + (directory.size() * Constants.DIR_ENTRY_BYTES); - ByteBuffer buffer = ByteBuffer.allocate(bufferSize); - buffer.order(ByteOrder.LITTLE_ENDIAN); - - // Write directory count - VarInt.encode(directory.size(), buffer); - - // Write directory entries - for (DirectoryEntry entry : directory) { - serializeDirectoryEntry(entry, buffer); - } - - buffer.flip(); - return buffer.asReadOnlyBuffer(); - } catch (Exception e) { - // Fallback to empty buffer if creation fails - return ByteBuffer.allocate(0).asReadOnlyBuffer(); - } + public ImprintRecord getRow(int fieldId) throws ImprintException { + return getTypedValueOrThrow(fieldId, TypeCode.ROW, Value.RowValue.class, "ROW").getValue(); } + // ========== SERIALIZATION ========== + /** * Serialize this record to a ByteBuffer. - * The returned buffer will be array-backed. */ public ByteBuffer serializeToBuffer() { - // Ensure directory is parsed for serialization - ensureDirectoryParsed(); - var buffer = ByteBuffer.allocate(estimateSerializedSize()); buffer.order(ByteOrder.LITTLE_ENDIAN); // Write header serializeHeader(buffer); - // Write directory (always present) - VarInt.encode(parsedDirectory.size(), buffer); - for (var entry : parsedDirectory) { - serializeDirectoryEntry(entry, buffer); - } + // Write directory + var directoryBuffer = buffers.serializeDirectory(); + buffer.put(directoryBuffer); - // Write payload (shallow copy only) + // Write payload + var payload = buffers.getPayload(); var payloadCopy = payload.duplicate(); buffer.put(payloadCopy); - // Prepare buffer for reading buffer.flip(); return buffer; } - /** - * Create a fluent builder for constructing ImprintRecord instances. - */ + public int estimateSerializedSize() { + int size = Constants.HEADER_BYTES; // header + size += buffers.serializeDirectory().remaining(); // directory + size += buffers.getPayload().remaining(); // payload + return size; + } + + // ========== STATIC FACTORY METHODS ========== + public static ImprintRecordBuilder builder(SchemaId schemaId) { return new ImprintRecordBuilder(schemaId); } - /** - * Create a fluent builder for constructing ImprintRecord instances. - */ - @SuppressWarnings("unused") public static ImprintRecordBuilder builder(int fieldspaceId, int schemaHash) { return new ImprintRecordBuilder(new SchemaId(fieldspaceId, schemaHash)); } - /** - * Deserialize a record from bytes through an array backed ByteBuffer. - */ public static ImprintRecord deserialize(byte[] bytes) throws ImprintException { return deserialize(ByteBuffer.wrap(bytes)); } - /** - * Deserialize a record from a ByteBuffer with lazy directory parsing. - * - * @param buffer the buffer to deserialize from. Must be array-backed - * (buffer.hasArray() == true) for optimal zero-copy performance. - */ public static ImprintRecord deserialize(ByteBuffer buffer) throws ImprintException { buffer = buffer.duplicate().order(ByteOrder.LITTLE_ENDIAN); // Read header var header = deserializeHeader(buffer); - // Read directory count but don't parse entries yet + // Calculate directory size int directoryStartPos = buffer.position(); - VarInt.DecodeResult countResult = VarInt.decode(buffer); + var countResult = VarInt.decode(buffer); int directoryCount = countResult.getValue(); - - // Calculate directory buffer (includes count + all entries) int directorySize = countResult.getBytesRead() + (directoryCount * Constants.DIR_ENTRY_BYTES); - buffer.position(directoryStartPos); // Reset to include count in directory buffer + // Create directory buffer + buffer.position(directoryStartPos); var directoryBuffer = buffer.slice(); directoryBuffer.limit(directorySize); - // Advance buffer past directory + // Advance past directory buffer.position(buffer.position() + directorySize); - // Read payload as ByteBuffer slice for zero-copy + // Create payload buffer var payload = buffer.slice(); payload.limit(header.getPayloadSize()); - return new ImprintRecord(header, directoryBuffer, payload); - } - - /** - * Binary search for field ID in parsed directory. - * Returns the index of the field if found, or a negative value if not found. - */ - private int findDirectoryIndexInParsed(int fieldId) { - if (!directoryParsed) { - return -1; - } - - int low = 0; - int high = parsedDirectory.size() - 1; - - while (low <= high) { - int mid = (low + high) >>> 1; - int midFieldId = parsedDirectory.get(mid).getId(); - - if (midFieldId < fieldId) { - low = mid + 1; - } else if (midFieldId > fieldId) { - high = mid - 1; - } else { - return mid; // field found - } - } - return -(low + 1); // field not found, return insertion point - } - - public int estimateSerializedSize() { - int size = Constants.HEADER_BYTES; // header - size += VarInt.encodedLength(getDirectoryCount()); // directory count - size += getDirectoryCount() * Constants.DIR_ENTRY_BYTES; // directory entries - size += payload.remaining(); // payload - return size; - } - - private int getDirectoryCount() { - if (directoryCount >= 0) { - return directoryCount; - } - if (directoryParsed) { - return parsedDirectory.size(); - } - // Last resort: decode from buffer - try { - ByteBuffer countBuffer = directoryBuffer.duplicate(); - return VarInt.decode(countBuffer).getValue(); - } catch (Exception e) { - return 0; - } - } - - // ===== EXISTING HELPER METHODS (unchanged) ===== + // Create buffers wrapper + var buffers = new ImprintBuffers(directoryBuffer, payload); - private void serializeHeader(ByteBuffer buffer) { - buffer.put(Constants.MAGIC); - buffer.put(Constants.VERSION); - buffer.put(header.getFlags().getValue()); - buffer.putInt(header.getSchemaId().getFieldSpaceId()); - buffer.putInt(header.getSchemaId().getSchemaHash()); - buffer.putInt(header.getPayloadSize()); + return new ImprintRecord(header, buffers); } - private static Header deserializeHeader(ByteBuffer buffer) throws ImprintException { - if (buffer.remaining() < Constants.HEADER_BYTES) { - throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, - "Not enough bytes for header"); - } - - byte magic = buffer.get(); - if (magic != Constants.MAGIC) { - throw new ImprintException(ErrorType.INVALID_MAGIC, - "Invalid magic byte: expected 0x" + Integer.toHexString(Constants.MAGIC) + - ", got 0x" + Integer.toHexString(magic & 0xFF)); - } + // ========== PRIVATE HELPER METHODS ========== - byte version = buffer.get(); - if (version != Constants.VERSION) { - throw new ImprintException(ErrorType.UNSUPPORTED_VERSION, - "Unsupported version: " + version); - } - - var flags = new Flags(buffer.get()); - int fieldspaceId = buffer.getInt(); - int schemaHash = buffer.getInt(); - int payloadSize = buffer.getInt(); - - return new Header(flags, new SchemaId(fieldspaceId, schemaHash), payloadSize); - } - - private void serializeDirectoryEntry(DirectoryEntry entry, ByteBuffer buffer) { - buffer.putShort(entry.getId()); - buffer.put(entry.getTypeCode().getCode()); - buffer.putInt(entry.getOffset()); + /** + * Get and validate a value exists and is not null. + */ + private Value getValidatedValue(int fieldId, String typeName) throws ImprintException { + var value = getValue(fieldId); + if (value == null) + throw new ImprintException(ErrorType.FIELD_NOT_FOUND, "Field " + fieldId + " not found"); + if (value.getTypeCode() == TypeCode.NULL) + throw new ImprintException(ErrorType.TYPE_MISMATCH, "Field " + fieldId + " is NULL, cannot retrieve as " + typeName); + return value; } - private static DirectoryEntry deserializeDirectoryEntry(ByteBuffer buffer) throws ImprintException { - if (buffer.remaining() < Constants.DIR_ENTRY_BYTES) { - throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, - "Not enough bytes for directory entry"); - } - - short id = buffer.getShort(); - var typeCode = TypeCode.fromByte(buffer.get()); - int offset = buffer.getInt(); - - return new DirectoryEntry(id, typeCode, offset); + private T getTypedValueOrThrow(int fieldId, TypeCode expectedTypeCode, Class expectedValueClass, String expectedTypeName) + throws ImprintException { + var value = getValidatedValue(fieldId, expectedTypeName); + if (value.getTypeCode() == expectedTypeCode && expectedValueClass.isInstance(value)) + return expectedValueClass.cast(value); + throw new ImprintException(ErrorType.TYPE_MISMATCH, "Field " + fieldId + " is of type " + value.getTypeCode() + ", expected " + expectedTypeName); } private Value deserializeValue(TypeCode typeCode, ByteBuffer buffer) throws ImprintException { - var valueSpecificBuffer = buffer.duplicate(); - valueSpecificBuffer.order(ByteOrder.LITTLE_ENDIAN); + var valueBuffer = buffer.duplicate(); + valueBuffer.order(ByteOrder.LITTLE_ENDIAN); switch (typeCode) { case NULL: @@ -524,120 +241,51 @@ private Value deserializeValue(TypeCode typeCode, ByteBuffer buffer) throws Impr case STRING: case ARRAY: case MAP: - return typeCode.getHandler().deserialize(valueSpecificBuffer); + return typeCode.getHandler().deserialize(valueBuffer); case ROW: - var nestedRecord = deserialize(valueSpecificBuffer); + var nestedRecord = deserialize(valueBuffer); return Value.fromRow(nestedRecord); - default: throw new ImprintException(ErrorType.INVALID_TYPE_CODE, "Unknown type code: " + typeCode); } } - // ===== TYPE-SPECIFIC GETTERS (unchanged API, improved performance) ===== - - private T getTypedValueOrThrow(int fieldId, TypeCode expectedTypeCode, Class expectedValueClass, String expectedTypeName) throws ImprintException { - var value = getValue(fieldId); - - if (value == null) { - throw new ImprintException(ErrorType.FIELD_NOT_FOUND, - "Field " + fieldId + " not found, cannot retrieve as " + expectedTypeName + "."); - } - - if (value.getTypeCode() == TypeCode.NULL) { - throw new ImprintException(ErrorType.TYPE_MISMATCH, - "Field " + fieldId + " is NULL, cannot retrieve as " + expectedTypeName + "."); - } - - if (value.getTypeCode() == expectedTypeCode && expectedValueClass.isInstance(value)) { - return expectedValueClass.cast(value); - } - - throw new ImprintException(ErrorType.TYPE_MISMATCH, - "Field " + fieldId + " is of type " + value.getTypeCode() + ", expected " + expectedTypeName + "."); - } - - public boolean getBoolean(int fieldId) throws ImprintException { - return getTypedValueOrThrow(fieldId, TypeCode.BOOL, Value.BoolValue.class, "boolean").getValue(); - } - - public int getInt32(int fieldId) throws ImprintException { - return getTypedValueOrThrow(fieldId, TypeCode.INT32, Value.Int32Value.class, "int32").getValue(); - } - - public long getInt64(int fieldId) throws ImprintException { - return getTypedValueOrThrow(fieldId, TypeCode.INT64, Value.Int64Value.class, "int64").getValue(); - } - - public float getFloat32(int fieldId) throws ImprintException { - return getTypedValueOrThrow(fieldId, TypeCode.FLOAT32, Value.Float32Value.class, "float32").getValue(); - } - - public double getFloat64(int fieldId) throws ImprintException { - return getTypedValueOrThrow(fieldId, TypeCode.FLOAT64, Value.Float64Value.class, "float64").getValue(); + private void serializeHeader(ByteBuffer buffer) { + buffer.put(Constants.MAGIC); + buffer.put(Constants.VERSION); + buffer.put(header.getFlags().getValue()); + buffer.putInt(header.getSchemaId().getFieldSpaceId()); + buffer.putInt(header.getSchemaId().getSchemaHash()); + buffer.putInt(header.getPayloadSize()); } - public String getString(int fieldId) throws ImprintException { - var value = getValue(fieldId); - - if (value == null) { - throw new ImprintException(ErrorType.FIELD_NOT_FOUND, - "Field " + fieldId + " not found, cannot retrieve as String."); - } - if (value.getTypeCode() == TypeCode.NULL) { - throw new ImprintException(ErrorType.TYPE_MISMATCH, - "Field " + fieldId + " is NULL, cannot retrieve as String."); - } - - if (value instanceof Value.StringValue) { - return ((Value.StringValue) value).getValue(); - } - if (value instanceof Value.StringBufferValue) { - return ((Value.StringBufferValue) value).getValue(); - } + private static Header deserializeHeader(ByteBuffer buffer) throws ImprintException { + if (buffer.remaining() < Constants.HEADER_BYTES) + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, "Not enough bytes for header"); - throw new ImprintException(ErrorType.TYPE_MISMATCH, - "Field " + fieldId + " is of type " + value.getTypeCode() + ", expected STRING."); - } - public byte[] getBytes(int fieldId) throws ImprintException { - Value value = getValue(fieldId); - - if (value == null) { - throw new ImprintException(ErrorType.FIELD_NOT_FOUND, - "Field " + fieldId + " not found, cannot retrieve as byte[]."); - } - if (value.getTypeCode() == TypeCode.NULL) { - throw new ImprintException(ErrorType.TYPE_MISMATCH, - "Field " + fieldId + " is NULL, cannot retrieve as byte[]."); + byte magic = buffer.get(); + if (magic != Constants.MAGIC) { + throw new ImprintException(ErrorType.INVALID_MAGIC, "Invalid magic byte: expected 0x" + Integer.toHexString(Constants.MAGIC) + + ", got 0x" + Integer.toHexString(magic & 0xFF)); } - if (value instanceof Value.BytesValue) { - return ((Value.BytesValue) value).getValue(); - } - if (value instanceof Value.BytesBufferValue) { - return ((Value.BytesBufferValue) value).getValue(); + byte version = buffer.get(); + if (version != Constants.VERSION) { + throw new ImprintException(ErrorType.UNSUPPORTED_VERSION, "Unsupported version: " + version); } - throw new ImprintException(ErrorType.TYPE_MISMATCH, - "Field " + fieldId + " is of type " + value.getTypeCode() + ", expected BYTES."); - } - - public List getArray(int fieldId) throws ImprintException { - return getTypedValueOrThrow(fieldId, TypeCode.ARRAY, Value.ArrayValue.class, "ARRAY").getValue(); - } - - public Map getMap(int fieldId) throws ImprintException { - return getTypedValueOrThrow(fieldId, TypeCode.MAP, Value.MapValue.class, "MAP").getValue(); - } + var flags = new Flags(buffer.get()); + int fieldSpaceId = buffer.getInt(); + int schemaHash = buffer.getInt(); + int payloadSize = buffer.getInt(); - public ImprintRecord getRow(int fieldId) throws ImprintException { - return getTypedValueOrThrow(fieldId, TypeCode.ROW, Value.RowValue.class, "ROW").getValue(); + return new Header(flags, new SchemaId(fieldSpaceId, schemaHash), payloadSize); } @Override public String toString() { - return String.format("ImprintRecord{header=%s, directorySize=%d, payloadSize=%d, directoryParsed=%s}", - header, getDirectoryCount(), payload.remaining(), directoryParsed); + return String.format("ImprintRecord{header=%s, directorySize=%d, payloadSize=%d}", + header, buffers.getDirectoryCount(), buffers.getPayload().remaining()); } } \ No newline at end of file diff --git a/src/main/java/com/imprint/util/VarInt.java b/src/main/java/com/imprint/util/VarInt.java index 75bd132..f43683b 100644 --- a/src/main/java/com/imprint/util/VarInt.java +++ b/src/main/java/com/imprint/util/VarInt.java @@ -16,12 +16,28 @@ */ @UtilityClass public final class VarInt { - + private static final byte CONTINUATION_BIT = (byte) 0x80; private static final byte SEGMENT_BITS = 0x7f; private static final int MAX_VARINT_LEN = 5; // Enough for u32 - - + + // Simple cache for values 0-1023 + private static final int CACHE_SIZE = 1024; + private static final int[] ENCODED_LENGTHS = new int[CACHE_SIZE]; + + static { + // Pre-compute encoded lengths for cached values + for (int i = 0; i < CACHE_SIZE; i++) { + long val = Integer.toUnsignedLong(i); + int length = 1; + while (val >= 0x80) { + val >>>= 7; + length++; + } + ENCODED_LENGTHS[i] = length; + } + } + /** * Encode a 32-bit unsigned integer as a VarInt into the given ByteBuffer. * @param value the value to encode (treated as unsigned) @@ -30,7 +46,7 @@ public final class VarInt { public static void encode(int value, ByteBuffer buffer) { // Convert to unsigned long for proper bit manipulation long val = Integer.toUnsignedLong(value); - + // Encode at least one byte, then continue while value has more bits do { byte b = (byte) (val & SEGMENT_BITS); @@ -41,8 +57,7 @@ public static void encode(int value, ByteBuffer buffer) { buffer.put(b); } while (val != 0); } - - + /** * Decode a VarInt from a ByteBuffer. * @param buffer the buffer to decode from @@ -53,55 +68,55 @@ public static DecodeResult decode(ByteBuffer buffer) throws ImprintException { long result = 0; int shift = 0; int bytesRead = 0; - + while (true) { if (bytesRead >= MAX_VARINT_LEN) { throw new ImprintException(ErrorType.MALFORMED_VARINT, "VarInt too long"); } if (!buffer.hasRemaining()) { - throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, - "Unexpected end of data while reading VarInt"); + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, + "Unexpected end of data while reading VarInt"); } - + byte b = buffer.get(); bytesRead++; - + // Check if adding these 7 bits would overflow long segment = b & SEGMENT_BITS; if (shift >= 32 || (shift == 28 && segment > 0xF)) { throw new ImprintException(ErrorType.MALFORMED_VARINT, "VarInt overflow"); } - + // Add the bottom 7 bits to the result result |= segment << shift; - + // If the high bit is not set, this is the last byte if ((b & CONTINUATION_BIT) == 0) { break; } - + shift += 7; } - + return new DecodeResult((int) result, bytesRead); } - + /** * Calculate the number of bytes needed to encode the given value as a VarInt. * @param value the value to encode (treated as unsigned) * @return the number of bytes needed */ public static int encodedLength(int value) { - // Convert to unsigned long for proper bit manipulation + if (value >= 0 && value < CACHE_SIZE) { + return ENCODED_LENGTHS[value]; + } + long val = Integer.toUnsignedLong(value); int length = 1; - - // Count additional bytes needed for values >= 128 while (val >= 0x80) { val >>>= 7; length++; } - return length; } From 73eade6f6c55af6d75d738a0882cc924b71cc825 Mon Sep 17 00:00:00 2001 From: expand3d <> Date: Thu, 5 Jun 2025 22:12:44 -0400 Subject: [PATCH 33/49] remove extra comments --- .../com/imprint/benchmark/ComparisonBenchmark.java | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java b/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java index dd62457..a666ffe 100644 --- a/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java +++ b/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java @@ -133,19 +133,7 @@ public void serializeFlatBuffers(Blackhole bh) { bh.consume(result); } - // ===== PARTIAL DESERIALIZATION (SETUP ONLY) ===== -// These benchmarks measure the cost of preparing a record for field access, -// not the cost of accessing the actual data. This is important because -// -// 1. Imprint: Only parses header + stores raw directory bytes -// 2. FlatBuffers: Only wraps the buffer with minimal validation -// 3. Others (eager): Parse and construct all field objects upfront -// -// This comparison shows the advantage of lazy loading approaches when you -// only need to access a subset of fields. In real streaming workloads, -// records are often filtered/routed based on just a few key fields. -// -// For a fair "full deserialization" comparison, see FULL DESERIALIZATION BENCHMARKS. + // ===== SETUP ONLY ===== @Benchmark public void deserializeSetupImprint(Blackhole bh) throws Exception { From 02866d5f9fe9abfcd1c543d224db0410098324ad Mon Sep 17 00:00:00 2001 From: expand3d <> Date: Thu, 5 Jun 2025 22:17:01 -0400 Subject: [PATCH 34/49] remove extra comments --- src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java b/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java index a666ffe..4d9c01c 100644 --- a/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java +++ b/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java @@ -259,19 +259,19 @@ public void singleFieldAccessMessagePack(Blackhole bh) throws Exception { @Benchmark public void singleFieldAccessAvro(Blackhole bh) throws Exception { GenericRecord record = deserializeWithAvro(avroBytes); - bh.consume(record.get("extraData4")); // Accessing field near end + bh.consume(record.get("extraData4")); } @Benchmark public void singleFieldAccessProtobuf(Blackhole bh) throws Exception { TestRecordProto.TestRecord record = TestRecordProto.TestRecord.parseFrom(protobufBytes); - bh.consume(record.getExtraData(4)); // Accessing field near end + bh.consume(record.getExtraData(4)); } @Benchmark public void singleFieldAccessFlatBuffers(Blackhole bh) { TestRecordFB record = TestRecordFB.getRootAsTestRecordFB(flatbuffersBytes.duplicate()); - bh.consume(record.extraData(4)); // Accessing field near end - zero copy! + bh.consume(record.extraData(4)); } // ===== SIZE COMPARISON ===== From 09443eb7d53b7e3f0c82211a6120a1e216c7e5ad Mon Sep 17 00:00:00 2001 From: expand3d <> Date: Sat, 7 Jun 2025 11:17:43 -0400 Subject: [PATCH 35/49] Add merge and project APIs; optimize/simplify ImprintBuffers with TreeMap --- .../benchmark/ComparisonBenchmark.java | 2 +- .../java/com/imprint/core/ImprintBuffers.java | 270 +++--------- .../com/imprint/core/ImprintOperations.java | 207 +++++++++ .../java/com/imprint/core/ImprintRecord.java | 22 + src/main/java/com/imprint/types/Value.java | 20 +- .../java/com/imprint/IntegrationTest.java | 358 +++++++++++++++- .../imprint/core/ImprintOperationsTest.java | 405 ++++++++++++++++++ .../com/imprint/profile/ProfilerTest.java | 6 +- 8 files changed, 1058 insertions(+), 232 deletions(-) create mode 100644 src/main/java/com/imprint/core/ImprintOperations.java create mode 100644 src/test/java/com/imprint/core/ImprintOperationsTest.java diff --git a/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java b/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java index 4d9c01c..6a6a958 100644 --- a/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java +++ b/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java @@ -405,7 +405,7 @@ public void mergeFlatBuffers(Blackhole bh) { // ===== MAIN METHOD TO RUN BENCHMARKS ===== public static void main(String[] args) throws RunnerException { - runAll(); + runFieldAccessBenchmarks(); // Or, uncomment specific runner methods to execute subsets: // runSerializationBenchmarks(); // runDeserializationBenchmarks(); diff --git a/src/main/java/com/imprint/core/ImprintBuffers.java b/src/main/java/com/imprint/core/ImprintBuffers.java index f6a341b..c14d6df 100644 --- a/src/main/java/com/imprint/core/ImprintBuffers.java +++ b/src/main/java/com/imprint/core/ImprintBuffers.java @@ -10,9 +10,9 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Objects; +import java.util.TreeMap; /** * Manages the raw buffers for an Imprint record with lazy directory parsing. @@ -31,17 +31,12 @@ public final class ImprintBuffers { private final ByteBuffer directoryBuffer; // Raw directory bytes (includes count) private final ByteBuffer payload; // Read-only payload view - // Lazy-loaded directory state - private List parsedDirectory; + // Lazy-loaded directory state. Needs to maintain ordering so that we can binary search the endOffset + private TreeMap parsedDirectory; private boolean directoryParsed = false; - private int directoryCount = -1; // Cached count to avoid repeated VarInt decoding /** * Creates buffers from raw data (used during deserialization). - * - * @param directoryBuffer Raw directory bytes including VarInt count and all entries. - * Format: [VarInt count][Entry1][Entry2]...[EntryN] - * @param payload Raw payload data containing all field values sequentially */ public ImprintBuffers(ByteBuffer directoryBuffer, ByteBuffer payload) { this.directoryBuffer = directoryBuffer.asReadOnlyBuffer(); @@ -50,33 +45,17 @@ public ImprintBuffers(ByteBuffer directoryBuffer, ByteBuffer payload) { /** * Creates buffers from pre-parsed directory (used during construction). - * This is more efficient when the directory is already known. - * - * @param directory Parsed directory entries, must be sorted by fieldId - * @param payload Raw payload data containing all field values */ public ImprintBuffers(List directory, ByteBuffer payload) { - this.parsedDirectory = Collections.unmodifiableList(Objects.requireNonNull(directory)); + this.parsedDirectory = createDirectoryMap(Objects.requireNonNull(directory)); this.directoryParsed = true; - this.directoryCount = directory.size(); this.payload = payload.asReadOnlyBuffer(); this.directoryBuffer = createDirectoryBuffer(directory); } /** * Get a zero-copy ByteBuffer view of a field's data. - * - *

Buffer Positioning Logic:

- *
    - *
  1. Find the directory entry for the requested fieldId
    2. - *
  2. Use entry.offset as start position in payload
    3. - *
  3. Find end position by looking at next field's offset (or payload end)
    4. - *
  4. Create a slice view: payload[startOffset:endOffset]
    5. - *
- * - * @param fieldId The field identifier to retrieve - * @return Zero-copy ByteBuffer positioned at field data, or null if field not found - * @throws ImprintException if buffer bounds are invalid or directory is corrupted + * Optimized for the most common use case - single field access. */ public ByteBuffer getFieldBuffer(int fieldId) throws ImprintException { var entry = findDirectoryEntry(fieldId); @@ -92,78 +71,43 @@ public ByteBuffer getFieldBuffer(int fieldId) throws ImprintException { "Invalid field buffer range: start=" + startOffset + ", end=" + endOffset + ", payloadLimit=" + payload.limit()); } - ByteBuffer fieldBuffer = payload.duplicate(); + var fieldBuffer = payload.duplicate(); fieldBuffer.position(startOffset).limit(endOffset); return fieldBuffer; } /** * Find a directory entry for the given field ID using the most efficient method. - * - *

Search Strategy:

- *
    - *
  • If directory is parsed: binary search on in-memory List<DirectoryEntry>
    • - *
  • If directory is raw: binary search directly on raw bytes (faster for single lookups)
    • - *
- * @param fieldId The field identifier to find - * @return DirectoryEntry if found, null otherwise - * @throws ImprintException if directory buffer is corrupted or truncated + *

+ * Strategy: + * - If parsed: TreeMap lookup + * - If raw: Binary search on raw bytes to avoid full unwinding of the directory */ public DirectoryEntry findDirectoryEntry(int fieldId) throws ImprintException { - if (directoryParsed) { - int index = findDirectoryIndexInParsed(fieldId); - return index >= 0 ? parsedDirectory.get(index) : null; - } else { + if (directoryParsed) + return parsedDirectory.get(fieldId); + else return findFieldEntryInRawDirectory(fieldId); - } } /** * Get the full directory, parsing it if necessary. - * - *

Lazy Parsing Behavior:

- *
    - *
  • First call: Parses entire directory from raw bytes into List<DirectoryEntry>
    • - *
  • Subsequent calls: Returns cached parsed directory
    • - *
  • Note - the method is not synchronized and assumes single-threaded usage.
    • - *
- * - *

When to use: Call this if you need to access multiple fields - * from the same record. For single field access, direct field getters are more efficient.

- * - * @return Immutable list of directory entries, sorted by fieldId + * Returns the values in fieldId order thanks to TreeMap. */ public List getDirectory() { ensureDirectoryParsed(); - return parsedDirectory; + return new ArrayList<>(parsedDirectory.values()); } /** - * Get the directory count without fully parsing the directory. - *

- * This method avoids parsing the entire directory when only the count is needed. - *

    - *
  1. Return cached count if available (directoryCount >= 0)
    2. - *
  2. Return parsed directory size if directory is already parsed
    3. - *
  3. Decode VarInt from raw buffer and cache the result
    4. - *
- * - *

VarInt Decoding: The count is stored as a VarInt at the beginning - * of the directoryBuffer. This method reads just enough bytes to decode the count.

- * - * @return Number of fields in the directory, or 0 if decoding fails + * Get directory count without parsing. */ public int getDirectoryCount() { - if (directoryCount >= 0) - return directoryCount; if (directoryParsed) return parsedDirectory.size(); - - // Decode from buffer and cache try { var countBuffer = directoryBuffer.duplicate(); - directoryCount = VarInt.decode(countBuffer).getValue(); - return directoryCount; + return VarInt.decode(countBuffer).getValue(); } catch (Exception e) { return 0; } @@ -171,52 +115,23 @@ public int getDirectoryCount() { /** * Create a new buffer containing the serialized directory. - * - *

Output Format:

- * 
-     * [VarInt count][DirectoryEntry 1][DirectoryEntry 2]...[DirectoryEntry N]
-     *
- * - *

Each DirectoryEntry is serialized as: [fieldId:2bytes][typeCode:1byte][offset:4bytes]

- * - * - * @return New ByteBuffer containing the complete serialized directory */ public ByteBuffer serializeDirectory() { ensureDirectoryParsed(); - return createDirectoryBuffer(parsedDirectory); + return createDirectoryBuffer(new ArrayList<>(parsedDirectory.values())); } // ========== PRIVATE METHODS ========== /** * Binary search on raw directory bytes to find a specific field. - * - *
    - *
  1. Position buffer at start and decode VarInt count (cache for future use)
    2. - *
  2. Calculate directory start position after VarInt
    3. - *
  3. For binary search mid-point: entryPos = startPos + (mid * DIR_ENTRY_BYTES)
    4. - *
  4. Read fieldId from calculated position (first 2 bytes of entry)
    5. - *
  5. Compare fieldId and adjust search bounds
    6. - *
  6. When found: reposition buffer and deserialize complete entry
    7. - *
- * - *

All buffer positions are bounds-checked before access.

- * - * @param fieldId Field identifier to search for - * @return Complete DirectoryEntry if found, null if not found - * @throws ImprintException if buffer is truncated or corrupted + * This avoids parsing the entire directory for single field lookups. */ private DirectoryEntry findFieldEntryInRawDirectory(int fieldId) throws ImprintException { var searchBuffer = directoryBuffer.duplicate(); searchBuffer.order(ByteOrder.LITTLE_ENDIAN); - // Decode directory count (cache it) - if (directoryCount < 0) - directoryCount = VarInt.decode(searchBuffer).getValue(); - else - VarInt.decode(searchBuffer); // Skip past the count - + int directoryCount = VarInt.decode(searchBuffer).getValue(); if (directoryCount == 0) return null; @@ -250,118 +165,65 @@ private DirectoryEntry findFieldEntryInRawDirectory(int fieldId) throws ImprintE return null; } - /** - * - * @param fieldId Field identifier to find - * @return Index of the field if found, or negative insertion point if not found - */ - private int findDirectoryIndexInParsed(int fieldId) { - if (!directoryParsed) - return -1; - int low = 0; - int high = parsedDirectory.size() - 1; - while (low <= high) { - int mid = (low + high) >>> 1; - int midFieldId = parsedDirectory.get(mid).getId(); - if (midFieldId < fieldId) - low = mid + 1; - else if (midFieldId > fieldId) - high = mid - 1; - else - return mid; - } - return -(low + 1); - } - /** * Find the end offset for a field by looking at the next field's offset. - * - *
    - *
  • Field data spans from: entry.offset to nextField.offset (exclusive)
    • - *
  • Last field spans from: entry.offset to payload.limit()
    • - *
  • This works because directory entries are sorted by fieldId
    • - *
- * - *

Search Strategy:

- *
    - *
  • If directory parsed: Use binary search result + 1 to get next entry
    • - *
  • If directory raw: Scan raw entries until fieldId > currentFieldId
    • - *
- * - * @param entry The directory entry whose end offset we need to find - * @return End offset (exclusive) for the field data - * @throws ImprintException if directory scanning fails */ private int findEndOffset(DirectoryEntry entry) throws ImprintException { if (directoryParsed) { - int entryIndex = findDirectoryIndexInParsed(entry.getId()); - return (entryIndex + 1 < parsedDirectory.size()) ? - parsedDirectory.get(entryIndex + 1).getOffset() : payload.limit(); - } else + return findNextOffsetInParsedDirectory(entry.getId()); + } else { return findNextOffsetInRawDirectory(entry.getId()); + } } /** - * Scan raw directory to find the next field's offset after currentFieldId. - * - *
    - *
  1. Position buffer after VarInt count
    2. - *
  2. For each directory entry at position: startPos + (i * DIR_ENTRY_BYTES)
    3. - *
  3. Read fieldId (first 2 bytes) and offset (bytes 3-6)
    4. - *
  4. Return offset of first field where fieldId > currentFieldId
    5. - *
  5. If no next field found, return payload.limit()
    6. - *
- * - * @param currentFieldId Find the next field after this fieldId - * @return Offset where the next field starts, or payload.limit() if this is the last field - * @throws ImprintException if directory buffer is corrupted + * Find the end offset using TreeMap's efficient navigation methods. */ + private int findNextOffsetInParsedDirectory(int currentFieldId) { + var nextEntry = parsedDirectory.higherEntry(currentFieldId); + return nextEntry != null ? nextEntry.getValue().getOffset() : payload.limit(); + } + private int findNextOffsetInRawDirectory(int currentFieldId) throws ImprintException { var scanBuffer = directoryBuffer.duplicate(); scanBuffer.order(ByteOrder.LITTLE_ENDIAN); - int count = (directoryCount >= 0) ? directoryCount : VarInt.decode(scanBuffer).getValue(); + int count = VarInt.decode(scanBuffer).getValue(); if (count == 0) return payload.limit(); - if (directoryCount >= 0) - VarInt.decode(scanBuffer); // Skip count if cached int directoryStartPos = scanBuffer.position(); + int low = 0; + int high = count - 1; + int nextOffset = payload.limit(); - for (int i = 0; i < count; i++) { - int entryPos = directoryStartPos + (i * Constants.DIR_ENTRY_BYTES); + // Binary search for the first field with fieldId > currentFieldId + while (low <= high) { + int mid = (low + high) >>> 1; + int entryPos = directoryStartPos + (mid * Constants.DIR_ENTRY_BYTES); if (entryPos + Constants.DIR_ENTRY_BYTES > scanBuffer.limit()) - return payload.limit(); + break; scanBuffer.position(entryPos); short fieldId = scanBuffer.getShort(); scanBuffer.get(); // skip type int offset = scanBuffer.getInt(); - if (fieldId > currentFieldId) - return offset; + if (fieldId > currentFieldId) { + nextOffset = offset; + high = mid - 1; + } else { + low = mid + 1; + } } - return payload.limit(); + return nextOffset; } /** * Parse the full directory if not already parsed. - * - *
    - *
  1. Duplicate directoryBuffer to avoid affecting original position
    2. - *
  2. Set byte order to LITTLE_ENDIAN for consistent reading
    3. - *
  3. Decode VarInt count and cache it
    4. - *
  4. Read 'count' directory entries sequentially
    5. - *
  5. Each entry: [fieldId:2bytes][typeCode:1byte][offset:4bytes]
    6. - *
  6. Store as immutable list and mark as parsed
    7. - *
- * - *

Error Handling: If parsing fails, throws RuntimeException - * since this indicates corrupted data that should never happen in normal operation.

- * - *

Will return immediately if directory has already been parsed.

+ * Creates TreeMap for both fast lookup and ordering needed for binary search on offSets. */ private void ensureDirectoryParsed() { if (directoryParsed) @@ -372,46 +234,42 @@ private void ensureDirectoryParsed() { var countResult = VarInt.decode(parseBuffer); int count = countResult.getValue(); - this.directoryCount = count; - var directory = new ArrayList(count); + this.parsedDirectory = new TreeMap<>(); for (int i = 0; i < count; i++) { - directory.add(deserializeDirectoryEntry(parseBuffer)); + var entry = deserializeDirectoryEntry(parseBuffer); + parsedDirectory.put((int)entry.getId(), entry); } - this.parsedDirectory = Collections.unmodifiableList(directory); this.directoryParsed = true; } catch (ImprintException e) { throw new RuntimeException("Failed to parse directory", e); } } + /** + * Create a TreeMap from directory list field lookup with ordering. + */ + private TreeMap createDirectoryMap(List directory) { + var map = new TreeMap(); + for (var entry : directory) { + map.put((int)entry.getId(), entry); + } + return map; + } + /** * Create directory buffer from parsed entries. - * - *

Serialization Format:

- *
    - *
  1. Calculate buffer size: VarInt.encodedLength(count) + (count * DIR_ENTRY_BYTES)
    2. - *
  2. Allocate ByteBuffer with LITTLE_ENDIAN byte order
    3. - *
  3. Write VarInt count
    4. - *
  4. Write each directory entry: [fieldId:2][typeCode:1][offset:4]
    5. - *
  5. Flip buffer and return read-only view
    6. - *
- * - * @param directory List of directory entries to serialize - * @return Read-only ByteBuffer containing serialized directory, or empty buffer on error */ private ByteBuffer createDirectoryBuffer(List directory) { try { - int bufferSize = VarInt.encodedLength(directory.size()) + - (directory.size() * Constants.DIR_ENTRY_BYTES); + int bufferSize = VarInt.encodedLength(directory.size()) + (directory.size() * Constants.DIR_ENTRY_BYTES); var buffer = ByteBuffer.allocate(bufferSize); buffer.order(ByteOrder.LITTLE_ENDIAN); VarInt.encode(directory.size(), buffer); - for (var entry : directory) { + for (var entry : directory) serializeDirectoryEntry(entry, buffer); - } buffer.flip(); return buffer.asReadOnlyBuffer(); @@ -433,10 +291,6 @@ private void serializeDirectoryEntry(DirectoryEntry entry, ByteBuffer buffer) { /** * Deserialize a single directory entry from the buffer. * Reads: [fieldId:2bytes][typeCode:1byte][offset:4bytes] - * - * @param buffer Buffer positioned at the start of a directory entry - * @return Parsed DirectoryEntry - * @throws ImprintException if buffer doesn't contain enough bytes */ private DirectoryEntry deserializeDirectoryEntry(ByteBuffer buffer) throws ImprintException { if (buffer.remaining() < Constants.DIR_ENTRY_BYTES) diff --git a/src/main/java/com/imprint/core/ImprintOperations.java b/src/main/java/com/imprint/core/ImprintOperations.java new file mode 100644 index 0000000..4e60ebf --- /dev/null +++ b/src/main/java/com/imprint/core/ImprintOperations.java @@ -0,0 +1,207 @@ +package com.imprint.core; + +import com.imprint.error.ErrorType; +import com.imprint.error.ImprintException; +import lombok.Value; +import lombok.experimental.UtilityClass; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.*; + +@UtilityClass +public class ImprintOperations { + + /** + * Project a subset of fields from an Imprint record. Payload copying is proportional to projected data size. + * + *

Algorithm:

+ *
    + *
  1. Sort and deduplicate requested field IDs for efficient matching
    2. + *
  2. Scan directory to find matching fields and calculate ranges
    3. + *
  3. Allocate new payload buffer with exact size needed
    4. + *
  4. Copy field data ranges directly (zero-copy where possible)
    5. + *
  5. Build new directory with adjusted offsets
    6. + *
+ * + * @param record The source record to project from + * @param fieldIds Array of field IDs to include in projection + * @return New ImprintRecord containing only the requested fields + */ + public static ImprintRecord project(ImprintRecord record, int... fieldIds) { + // Sort and deduplicate field IDs for efficient matching with sorted directory + int[] sortedFieldIds = Arrays.stream(fieldIds).distinct().sorted().toArray(); + if (sortedFieldIds.length == 0) + return createEmptyRecord(record.getHeader().getSchemaId()); + + //eager fetch the entire directory (can this be lazy and just done per field?) + var sourceDirectory = record.getDirectory(); + var newDirectory = new ArrayList(sortedFieldIds.length); + var ranges = new ArrayList(); + + // Iterate through directory and compute ranges to copy + int fieldIdsIdx = 0; + int directoryIdx = 0; + int currentOffset = 0; + + while (directoryIdx < sourceDirectory.size() && fieldIdsIdx < sortedFieldIds.length) { + var field = sourceDirectory.get(directoryIdx); + if (field.getId() == sortedFieldIds[fieldIdsIdx]) { + // Calculate field length using next field's offset + int nextOffset = (directoryIdx + 1 < sourceDirectory.size()) ? + sourceDirectory.get(directoryIdx + 1).getOffset() : + record.getBuffers().getPayload().limit(); + int fieldLength = nextOffset - field.getOffset(); + + newDirectory.add(new DirectoryEntry(field.getId(), field.getTypeCode(), currentOffset)); + ranges.add(new FieldRange(field.getOffset(), nextOffset)); + + currentOffset += fieldLength; + fieldIdsIdx++; + } + directoryIdx++; + } + + // Build new payload from ranges + var newPayload = buildPayloadFromRanges(record.getBuffers().getPayload(), ranges); + + // Create new header with updated payload size + // TODO: compute correct schema hash + var newHeader = new Header(record.getHeader().getFlags(), + new SchemaId(record.getHeader().getSchemaId().getFieldSpaceId(), 0xdeadbeef), + newPayload.remaining() + ); + + return new ImprintRecord(newHeader, newDirectory, newPayload); + } + + /** + * Merge two Imprint records, combining their fields. Payload copying is proportional to total data size. + * + *

Merge Strategy:

+ *
    + *
  • Fields are merged using sort-merge algorithm on directory entries
    • + *
  • For duplicate field IDs: first record's field takes precedence
    • + *
  • Payloads are concatenated with directory offsets adjusted
    • + *
  • Schema ID from first record is preserved
    • + *
+ *

+ * + * @param first The first record (takes precedence for duplicate fields) + * @param second The second record to merge + * @return New ImprintRecord containing merged fields + * @throws ImprintException if merge fails due to incompatible records + */ + public static ImprintRecord merge(ImprintRecord first, ImprintRecord second) throws ImprintException { + var firstDir = first.getDirectory(); + var secondDir = second.getDirectory(); + + // Pre-allocate for worst case (no overlapping fields) + var newDirectory = new ArrayList(firstDir.size() + secondDir.size()); + var payloadChunks = new ArrayList(); + + int firstIdx = 0; + int secondIdx = 0; + int currentOffset = 0; + + while (firstIdx < firstDir.size() || secondIdx < secondDir.size()) { + DirectoryEntry currentEntry; + ByteBuffer currentPayload; + + if (firstIdx < firstDir.size() && + (secondIdx >= secondDir.size() || firstDir.get(firstIdx).getId() <= secondDir.get(secondIdx).getId())) { + + // Take from first record + currentEntry = firstDir.get(firstIdx); + + // Skip duplicate field in second record if present + if (secondIdx < secondDir.size() && + firstDir.get(firstIdx).getId() == secondDir.get(secondIdx).getId()) { + secondIdx++; + } + + currentPayload = first.getRawBytes(currentEntry.getId()); + firstIdx++; + } else { + // Take from second record + currentEntry = secondDir.get(secondIdx); + currentPayload = second.getRawBytes(currentEntry.getId()); + secondIdx++; + } + + if (currentPayload == null) + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, "Failed to get raw bytes for field " + currentEntry.getId()); + + // Add adjusted directory entry + var newEntry = new DirectoryEntry(currentEntry.getId(), currentEntry.getTypeCode(), currentOffset); + newDirectory.add(newEntry); + + // Collect payload chunk + payloadChunks.add(currentPayload.duplicate()); + currentOffset += currentPayload.remaining(); + } + + // Build merged payload + var mergedPayload = buildPayloadFromChunks(payloadChunks); + + // Create header preserving first record's schema ID + var newHeader = new Header(first.getHeader().getFlags(), first.getHeader().getSchemaId(), mergedPayload.remaining()); + + return new ImprintRecord(newHeader, newDirectory, mergedPayload); + } + + /** + * Represents a range of bytes to copy from source payload. + */ + @Value + private static class FieldRange { + int start; + int end; + + int length() { + return end - start; + } + } + + /** + * Build a new payload buffer from field ranges in the source payload. + */ + private static ByteBuffer buildPayloadFromRanges(ByteBuffer sourcePayload, List ranges) { + int totalSize = ranges.stream().mapToInt(FieldRange::length).sum(); + var newPayload = ByteBuffer.allocate(totalSize); + newPayload.order(ByteOrder.LITTLE_ENDIAN); + + for (var range : ranges) { + var sourceSlice = sourcePayload.duplicate(); + sourceSlice.position(range.start).limit(range.end); + newPayload.put(sourceSlice); + } + + newPayload.flip(); + return newPayload; + } + + /** + * Build a new payload buffer by concatenating chunks. + */ + private static ByteBuffer buildPayloadFromChunks(List chunks) { + int totalSize = chunks.stream().mapToInt(ByteBuffer::remaining).sum(); + var mergedPayload = ByteBuffer.allocate(totalSize); + mergedPayload.order(ByteOrder.LITTLE_ENDIAN); + + for (var chunk : chunks) { + mergedPayload.put(chunk); + } + + mergedPayload.flip(); + return mergedPayload; + } + + /** + * Create an empty record with the given schema ID. + */ + private static ImprintRecord createEmptyRecord(SchemaId schemaId) { + var header = new Header(new Flags((byte) 0x01), schemaId, 0); + return new ImprintRecord(header, Collections.emptyList(), ByteBuffer.allocate(0)); + } +} diff --git a/src/main/java/com/imprint/core/ImprintRecord.java b/src/main/java/com/imprint/core/ImprintRecord.java index 6abc9cf..e720df5 100644 --- a/src/main/java/com/imprint/core/ImprintRecord.java +++ b/src/main/java/com/imprint/core/ImprintRecord.java @@ -69,6 +69,28 @@ public ByteBuffer getRawBytes(int fieldId) { } } + /** + * Project a subset of fields from this record. + * + * @param fieldIds Array of field IDs to include in the projection + * @return New ImprintRecord containing only the requested fields + */ + public ImprintRecord project(int... fieldIds) { + return ImprintOperations.project(this, fieldIds); + } + + /** + * Merge another record into this one. + * For duplicate fields, this record's values take precedence. + * + * @param other The record to merge with this one + * @return New ImprintRecord containing merged fields + * @throws ImprintException if merge fails + */ + public ImprintRecord merge(ImprintRecord other) throws ImprintException { + return ImprintOperations.merge(this, other); + } + /** * Get the directory (parsing it if necessary). */ diff --git a/src/main/java/com/imprint/types/Value.java b/src/main/java/com/imprint/types/Value.java index fbb988c..bfa9958 100644 --- a/src/main/java/com/imprint/types/Value.java +++ b/src/main/java/com/imprint/types/Value.java @@ -169,6 +169,7 @@ public String toString() { } // Float64 Value + @Getter @EqualsAndHashCode(callSuper = false) public static class Float64Value extends Value { @@ -180,7 +181,7 @@ public Float64Value(double value) { @Override public TypeCode getTypeCode() { return TypeCode.FLOAT64; } - + @Override public String toString() { return String.valueOf(value); @@ -188,17 +189,20 @@ public String toString() { } // Bytes Value (array-based) + @Getter public static class BytesValue extends Value { + /** + * Returns internal array. MUST NOT be modified by caller. + */ private final byte[] value; - + + /** + * Takes ownership of the byte array. Caller must not modify after construction. + */ public BytesValue(byte[] value) { - this.value = value.clone(); + this.value = Objects.requireNonNull(value); } - - public byte[] getValue() { - return value.clone(); - } - + @Override public TypeCode getTypeCode() { return TypeCode.BYTES; } diff --git a/src/test/java/com/imprint/IntegrationTest.java b/src/test/java/com/imprint/IntegrationTest.java index 898adfb..ee1d426 100644 --- a/src/test/java/com/imprint/IntegrationTest.java +++ b/src/test/java/com/imprint/IntegrationTest.java @@ -49,8 +49,6 @@ var record = ImprintRecord.builder(schemaId) assertTrue(deserialized.getBoolean(3)); assertEquals(3.14159, deserialized.getFloat64(4)); assertArrayEquals(new byte[]{1,2,3,4}, deserialized.getBytes(5)); - - System.out.println("Basic functionality test passed"); } @Test @@ -90,15 +88,11 @@ var record = ImprintRecord.builder(schemaId) assertEquals(2, deserializedMap.size()); assertEquals(Value.fromInt32(1), deserializedMap.get(MapKey.fromString("one"))); assertEquals(Value.fromInt32(2), deserializedMap.get(MapKey.fromString("two"))); - - System.out.println("Arrays and maps test passed"); } @Test @DisplayName("Nested Records: create, serialize, deserialize records within records") void testNestedRecords() throws ImprintException { - System.out.println("Testing nested records..."); - var innerSchemaId = new SchemaId(3, 0x12345678); var innerRecord = ImprintRecord.builder(innerSchemaId) .field(1, "nested data") @@ -124,8 +118,348 @@ void testNestedRecords() throws ImprintException { assertEquals(3, nestedDeserialized.getHeader().getSchemaId().getFieldSpaceId()); assertEquals("nested data", nestedDeserialized.getString(1)); assertEquals(9876543210L, nestedDeserialized.getInt64(2)); + } + + @Test + @DisplayName("Project: subset of fields with serialization round-trip") + void testProjectSubsetWithSerialization() throws ImprintException { + var schemaId = new SchemaId(10, 0xabcd1234); + var originalRecord = ImprintRecord.builder(schemaId) + .field(1, 100) + .field(2, "keep this field") + .field(3, false) + .field(4, "remove this field") + .field(5, 42.5) + .field(6, new byte[]{9, 8, 7}) + .build(); + + // Project fields 1, 2, 5 (skip 3, 4, 6) + var projected = originalRecord.project(1, 2, 5); + + assertEquals(3, projected.getDirectory().size()); + assertEquals(100, projected.getInt32(1)); + assertEquals("keep this field", projected.getString(2)); + assertEquals(42.5, projected.getFloat64(5)); + + // Verify missing fields + assertNull(projected.getValue(3)); + assertNull(projected.getValue(4)); + assertNull(projected.getValue(6)); + + // Test serialization round-trip of projected record + var buffer = projected.serializeToBuffer(); + byte[] serialized = new byte[buffer.remaining()]; + buffer.get(serialized); + var deserialized = ImprintRecord.deserialize(serialized); + + assertEquals(3, deserialized.getDirectory().size()); + assertEquals(100, deserialized.getInt32(1)); + assertEquals("keep this field", deserialized.getString(2)); + assertEquals(42.5, deserialized.getFloat64(5)); + } + + @Test + @DisplayName("Project: complex data types (arrays, maps, nested records)") + void testProjectComplexTypes() throws ImprintException { + var schemaId = new SchemaId(11, 0xbeef4567); + + // Create nested record + var nestedRecord = ImprintRecord.builder(new SchemaId(12, 0x11111111)) + .field(100, "nested value") + .build(); + + // Create homogeneous array (all strings) + var testArray = Arrays.asList(Value.fromString("item1"), Value.fromString("item2"), Value.fromString("item3")); + + // Create homogeneous map (string keys -> string values) + var testMap = new HashMap(); + testMap.put(MapKey.fromString("key1"), Value.fromString("value1")); + testMap.put(MapKey.fromString("key2"), Value.fromString("value2")); + + var originalRecord = ImprintRecord.builder(schemaId) + .field(1, "simple string") + .field(2, Value.fromArray(testArray)) + .field(3, Value.fromMap(testMap)) + .field(4, nestedRecord) + .field(5, 999L) + .build(); + + // Project only complex types + var projected = originalRecord.project(2, 3, 4); + + assertEquals(3, projected.getDirectory().size()); + + // Verify array projection (homogeneous strings) + var projectedArray = projected.getArray(2); + assertEquals(3, projectedArray.size()); + assertEquals(Value.fromString("item1"), projectedArray.get(0)); + assertEquals(Value.fromString("item2"), projectedArray.get(1)); + assertEquals(Value.fromString("item3"), projectedArray.get(2)); + + // Verify map projection (string -> string) + var projectedMap = projected.getMap(3); + assertEquals(2, projectedMap.size()); + assertEquals(Value.fromString("value1"), projectedMap.get(MapKey.fromString("key1"))); + assertEquals(Value.fromString("value2"), projectedMap.get(MapKey.fromString("key2"))); + + // Verify nested record projection + var projectedNested = projected.getRow(4); + assertEquals("nested value", projectedNested.getString(100)); + + // Verify excluded fields + assertNull(projected.getValue(1)); + assertNull(projected.getValue(5)); + } + + @Test + @DisplayName("Merge: distinct fields with serialization round-trip") + void testMergeDistinctFieldsWithSerialization() throws ImprintException { + var schemaId = new SchemaId(20, 0xcafe5678); + + var record1 = ImprintRecord.builder(schemaId) + .field(1, 100) + .field(3, "from record1") + .field(5, true) + .build(); + + var record2 = ImprintRecord.builder(schemaId) + .field(2, 200L) + .field(4, "from record2") + .field(6, 3.14f) + .build(); + + var merged = record1.merge(record2); + + assertEquals(6, merged.getDirectory().size()); + assertEquals(100, merged.getInt32(1)); + assertEquals(200L, merged.getInt64(2)); + assertEquals("from record1", merged.getString(3)); + assertEquals("from record2", merged.getString(4)); + assertTrue(merged.getBoolean(5)); + assertEquals(3.14f, merged.getFloat32(6)); + + // Test serialization round-trip of merged record + var buffer = merged.serializeToBuffer(); + byte[] serialized = new byte[buffer.remaining()]; + buffer.get(serialized); + var deserialized = ImprintRecord.deserialize(serialized); + + assertEquals(6, deserialized.getDirectory().size()); + assertEquals(100, deserialized.getInt32(1)); + assertEquals(200L, deserialized.getInt64(2)); + assertEquals("from record1", deserialized.getString(3)); + assertEquals("from record2", deserialized.getString(4)); + assertTrue(deserialized.getBoolean(5)); + assertEquals(3.14f, deserialized.getFloat32(6)); + } - System.out.println("✓ Nested records test passed"); + @Test + @DisplayName("Merge: overlapping fields - first record wins") + void testMergeOverlappingFields() throws ImprintException { + var schemaId = new SchemaId(21, 0xdead9876); + + var record1 = ImprintRecord.builder(schemaId) + .field(1, "first wins") + .field(2, 100) + .field(4, true) + .build(); + + var record2 = ImprintRecord.builder(schemaId) + .field(1, "second loses") // Overlapping field + .field(2, 999) // Overlapping field + .field(3, "unique to second") + .field(4, false) // Overlapping field + .build(); + + var merged = record1.merge(record2); + + assertEquals(4, merged.getDirectory().size()); + assertEquals("first wins", merged.getString(1)); // First record wins + assertEquals(100, merged.getInt32(2)); // First record wins + assertEquals("unique to second", merged.getString(3)); // Only in second + assertTrue(merged.getBoolean(4)); // First record wins + } + + @Test + @DisplayName("Merge: complex data types and nested records") + void testMergeComplexTypes() throws ImprintException { + var schemaId = new SchemaId(22, 0xbeef1111); + + // Create nested records for both + var nested1 = ImprintRecord.builder(new SchemaId(23, 0x22222222)) + .field(100, "nested in record1") + .build(); + + var nested2 = ImprintRecord.builder(new SchemaId(24, 0x33333333)) + .field(200, "nested in record2") + .build(); + + // Create arrays + var array1 = Arrays.asList(Value.fromString("array1_item1"), Value.fromString("array1_item2")); + var array2 = Arrays.asList(Value.fromInt32(10), Value.fromInt32(20)); + + // Create maps + var map1 = new HashMap(); + map1.put(MapKey.fromString("map1_key"), Value.fromString("map1_value")); + + var map2 = new HashMap(); + map2.put(MapKey.fromInt32(42), Value.fromBoolean(true)); + + var record1 = ImprintRecord.builder(schemaId) + .field(1, nested1) + .field(3, Value.fromArray(array1)) + .field(5, Value.fromMap(map1)) + .build(); + + var record2 = ImprintRecord.builder(schemaId) + .field(2, nested2) + .field(4, Value.fromArray(array2)) + .field(6, Value.fromMap(map2)) + .build(); + + var merged = record1.merge(record2); + + assertEquals(6, merged.getDirectory().size()); + + // Verify nested records + var mergedNested1 = merged.getRow(1); + assertEquals("nested in record1", mergedNested1.getString(100)); + + var mergedNested2 = merged.getRow(2); + assertEquals("nested in record2", mergedNested2.getString(200)); + + // Verify arrays + var mergedArray1 = merged.getArray(3); + assertEquals(2, mergedArray1.size()); + assertEquals(Value.fromString("array1_item1"), mergedArray1.get(0)); + + var mergedArray2 = merged.getArray(4); + assertEquals(2, mergedArray2.size()); + assertEquals(Value.fromInt32(10), mergedArray2.get(0)); + + // Verify maps + var mergedMap1 = merged.getMap(5); + assertEquals(Value.fromString("map1_value"), mergedMap1.get(MapKey.fromString("map1_key"))); + + var mergedMap2 = merged.getMap(6); + assertEquals(Value.fromBoolean(true), mergedMap2.get(MapKey.fromInt32(42))); + } + + @Test + @DisplayName("Project and Merge: chained operations") + void testProjectAndMergeChained() throws ImprintException { + var schemaId = new SchemaId(30, 0xabcdabcd); + + // Create a large record + var fullRecord = ImprintRecord.builder(schemaId) + .field(1, "field1") + .field(2, "field2") + .field(3, "field3") + .field(4, "field4") + .field(5, "field5") + .field(6, "field6") + .build(); + + // Project different subsets + var projection1 = fullRecord.project(1, 3, 5); + var projection2 = fullRecord.project(2, 4, 6); + + assertEquals(3, projection1.getDirectory().size()); + assertEquals(3, projection2.getDirectory().size()); + + // Merge the projections back together + var recomposed = projection1.merge(projection2); + + assertEquals(6, recomposed.getDirectory().size()); + assertEquals("field1", recomposed.getString(1)); + assertEquals("field2", recomposed.getString(2)); + assertEquals("field3", recomposed.getString(3)); + assertEquals("field4", recomposed.getString(4)); + assertEquals("field5", recomposed.getString(5)); + assertEquals("field6", recomposed.getString(6)); + + // Test another chain: project the merged result + var finalProjection = recomposed.project(2, 4, 6); + assertEquals(3, finalProjection.getDirectory().size()); + assertEquals("field2", finalProjection.getString(2)); + assertEquals("field4", finalProjection.getString(4)); + assertEquals("field6", finalProjection.getString(6)); + } + + @Test + @DisplayName("Merge and Project: empty record handling") + void testMergeAndProjectEmptyRecords() throws ImprintException { + var schemaId = new SchemaId(40, 0xeeeeeeee); + + var emptyRecord = ImprintRecord.builder(schemaId).build(); + var nonEmptyRecord = ImprintRecord.builder(schemaId) + .field(1, "not empty") + .field(2, 42) + .build(); + + // Test merging with empty + var merged1 = emptyRecord.merge(nonEmptyRecord); + var merged2 = nonEmptyRecord.merge(emptyRecord); + + assertEquals(2, merged1.getDirectory().size()); + assertEquals(2, merged2.getDirectory().size()); + assertEquals("not empty", merged1.getString(1)); + assertEquals("not empty", merged2.getString(1)); + + // Test projecting empty record + var projectedEmpty = emptyRecord.project(1, 2, 3); + assertEquals(0, projectedEmpty.getDirectory().size()); + + // Test projecting non-existent fields + var projectedNonExistent = nonEmptyRecord.project(99, 100); + assertEquals(0, projectedNonExistent.getDirectory().size()); + } + + @Test + @DisplayName("Project and Merge: Large record operations") + void testLargeRecordOperations() throws ImprintException { + var schemaId = new SchemaId(50, 0xffffffff); + + // Create a record with many fields + var builder = ImprintRecord.builder(schemaId); + for (int i = 1; i <= 100; i++) { + builder.field(i, "field_" + i + "_data"); + } + var largeRecord = builder.build(); + + assertEquals(100, largeRecord.getDirectory().size()); + + // Project a subset (every 10th field) + int[] projectionFields = new int[10]; + for (int i = 0; i < 10; i++) { + projectionFields[i] = (i + 1) * 10; // 10, 20, 30, ..., 100 + } + + var projected = largeRecord.project(projectionFields); + assertEquals(10, projected.getDirectory().size()); + + for (int i = 0; i < 10; i++) { + int fieldId = (i + 1) * 10; + assertEquals("field_" + fieldId + "_data", projected.getString(fieldId)); + } + + // Create another large record for merging + var builder2 = ImprintRecord.builder(schemaId); + for (int i = 101; i <= 150; i++) { + builder2.field(i, "additional_field_" + i); + } + var additionalRecord = builder2.build(); + + // Merge the large records + var merged = largeRecord.merge(additionalRecord); + assertEquals(150, merged.getDirectory().size()); + + // Verify some values from both records + assertEquals("field_1_data", merged.getString(1)); + assertEquals("field_50_data", merged.getString(50)); + assertEquals("field_100_data", merged.getString(100)); + assertEquals("additional_field_101", merged.getString(101)); + assertEquals("additional_field_150", merged.getString(150)); } private ImprintRecord createTestRecordForGetters() throws ImprintException { @@ -216,7 +550,7 @@ var record = serializeAndDeserialize(originalRecord); @Test @DisplayName("Type Getters: Empty Collections (Array and Map)") - void testErgonomicGettersEmptyCollections() throws ImprintException { + void testTypeGettersEmptyCollections() throws ImprintException { var originalRecord = createTestRecordForGetters(); var record = serializeAndDeserialize(originalRecord); @@ -231,7 +565,7 @@ var record = serializeAndDeserialize(originalRecord); @Test @DisplayName("Type Getters: Exception for Field Not Found") - void testErgonomicGetterExceptionFieldNotFound() throws ImprintException { + void testTypeGetterExceptionFieldNotFound() throws ImprintException { var originalRecord = createTestRecordForGetters(); var record = serializeAndDeserialize(originalRecord); @@ -241,7 +575,7 @@ var record = serializeAndDeserialize(originalRecord); @Test @DisplayName("Type Getters: Exception for Null Field accessed as primitive") - void testErgonomicGetterExceptionNullField() throws ImprintException { + void testTypeGetterExceptionNullField() throws ImprintException { var originalRecord = createTestRecordForGetters(); var record = serializeAndDeserialize(originalRecord); @@ -258,7 +592,7 @@ var record = serializeAndDeserialize(originalRecord); @Test @DisplayName("Type Getters: Exception for Type Mismatch") - void testErgonomicGetterExceptionTypeMismatch() throws ImprintException { + void testTypeGetterExceptionTypeMismatch() throws ImprintException { var originalRecord = createTestRecordForGetters(); var record = serializeAndDeserialize(originalRecord); @@ -268,7 +602,7 @@ var record = serializeAndDeserialize(originalRecord); @Test @DisplayName("Type Getters: Row (Nested Record)") - void testErgonomicGetterRow() throws ImprintException { + void testTypeGetterRow() throws ImprintException { var innerSchemaId = new SchemaId(6, 0x12345678); var innerRecord = ImprintRecord.builder(innerSchemaId) .field(101, "nested string") diff --git a/src/test/java/com/imprint/core/ImprintOperationsTest.java b/src/test/java/com/imprint/core/ImprintOperationsTest.java new file mode 100644 index 0000000..1dc67fb --- /dev/null +++ b/src/test/java/com/imprint/core/ImprintOperationsTest.java @@ -0,0 +1,405 @@ +package com.imprint.core; + +import com.imprint.error.ImprintException; +import com.imprint.types.Value; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; + +@DisplayName("ImprintOperations") +class ImprintOperationsTest { + + private SchemaId testSchema; + private ImprintRecord multiFieldRecord; + private ImprintRecord emptyRecord; + + @BeforeEach + void setUp() throws ImprintException { + testSchema = new SchemaId(1, 0xdeadbeef); + multiFieldRecord = createTestRecord(); + emptyRecord = createEmptyTestRecord(); + } + + private ImprintRecord createTestRecord() throws ImprintException { + return ImprintRecord.builder(testSchema) + .field(1, 42) + .field(3, "hello") + .field(5, true) + .field(7, new byte[]{1, 2, 3}) + .build(); + } + + private ImprintRecord createEmptyTestRecord() throws ImprintException { + return ImprintRecord.builder(testSchema).build(); + } + + @Nested + @DisplayName("Project Operations") + class ProjectOperations { + + @Test + @DisplayName("should project subset of fields") + void shouldProjectSubsetOfFields() throws ImprintException { + // When projecting a subset of fields + ImprintRecord projected = ImprintOperations.project(multiFieldRecord, 1, 5); + + // Then only the requested fields should be present + assertEquals(2, projected.getDirectory().size()); + assertEquals(42, projected.getInt32(1)); + assertTrue(projected.getBoolean(5)); + + // And non-requested fields should be absent + assertNull(projected.getValue(3)); + assertNull(projected.getValue(7)); + } + + @Test + @DisplayName("should maintain field order regardless of input order") + void shouldMaintainFieldOrderRegardlessOfInputOrder() throws ImprintException { + // When projecting fields in arbitrary order + ImprintRecord projected = ImprintOperations.project(multiFieldRecord, 7, 1, 5, 3); + + // Then all requested fields should be present + assertEquals(4, projected.getDirectory().size()); + assertEquals(42, projected.getInt32(1)); + assertEquals("hello", projected.getString(3)); + assertTrue(projected.getBoolean(5)); + assertArrayEquals(new byte[]{1, 2, 3}, projected.getBytes(7)); + + // And directory should maintain sorted order + List directory = projected.getDirectory(); + for (int i = 1; i < directory.size(); i++) { + assertTrue(directory.get(i - 1).getId() < directory.get(i).getId(), + "Directory entries should be sorted by field id"); + } + } + + @Test + @DisplayName("should handle single field projection") + void shouldHandleSingleFieldProjection() throws ImprintException { + // When projecting a single field + ImprintRecord projected = ImprintOperations.project(multiFieldRecord, 3); + + // Then only that field should be present + assertEquals(1, projected.getDirectory().size()); + assertEquals("hello", projected.getString(3)); + } + + @Test + @DisplayName("should preserve all fields when projecting all") + void shouldPreserveAllFieldsWhenProjectingAll() throws ImprintException { + // Given all field IDs from the original record + int[] allFields = multiFieldRecord.getDirectory().stream() + .mapToInt(DirectoryEntry::getId) + .toArray(); + + // When projecting all fields + ImprintRecord projected = ImprintOperations.project(multiFieldRecord, allFields); + + // Then all fields should be present with matching values + assertEquals(multiFieldRecord.getDirectory().size(), projected.getDirectory().size()); + + for (DirectoryEntry entry : multiFieldRecord.getDirectory()) { + Value originalValue = multiFieldRecord.getValue(entry.getId()); + Value projectedValue = projected.getValue(entry.getId()); + assertEquals(originalValue, projectedValue, + "Field " + entry.getId() + " should have matching value"); + } + } + + @Test + @DisplayName("should handle empty projection") + void shouldHandleEmptyProjection() { + // When projecting no fields + ImprintRecord projected = ImprintOperations.project(multiFieldRecord); + + // Then result should be empty but valid + assertEquals(0, projected.getDirectory().size()); + assertEquals(0, projected.getBuffers().getPayload().remaining()); + } + + @Test + @DisplayName("should ignore nonexistent fields") + void shouldIgnoreNonexistentFields() throws ImprintException { + // When projecting mix of existing and non-existing fields + ImprintRecord projected = ImprintOperations.project(multiFieldRecord, 1, 99, 100); + + // Then only existing fields should be included + assertEquals(1, projected.getDirectory().size()); + assertEquals(42, projected.getInt32(1)); + assertNull(projected.getValue(99)); + assertNull(projected.getValue(100)); + } + + @Test + @DisplayName("should deduplicate requested fields") + void shouldDeduplicateRequestedFields() throws ImprintException { + // When projecting the same field multiple times + ImprintRecord projected = ImprintOperations.project(multiFieldRecord, 1, 1, 1); + + // Then field should only appear once + assertEquals(1, projected.getDirectory().size()); + assertEquals(42, projected.getInt32(1)); + } + + @Test + @DisplayName("should handle projection from empty record") + void shouldHandleProjectionFromEmptyRecord() { + // When projecting any fields from empty record + ImprintRecord projected = ImprintOperations.project(emptyRecord, 1, 2, 3); + + // Then result should be empty but valid + assertEquals(0, projected.getDirectory().size()); + assertEquals(0, projected.getBuffers().getPayload().remaining()); + } + + @Test + @DisplayName("should preserve exact byte representation") + void shouldPreserveExactByteRepresentation() throws ImprintException { + // Given a field's original bytes + byte[] originalBytes = multiFieldRecord.getBytes(7); + + // When projecting that field + ImprintRecord projected = ImprintOperations.project(multiFieldRecord, 7); + + // Then the byte representation should be exactly preserved + byte[] projectedBytes = projected.getBytes(7); + assertArrayEquals(originalBytes, projectedBytes, + "Byte representation should be identical"); + } + + @Test + @DisplayName("should reduce payload size when projecting subset") + void shouldReducePayloadSizeWhenProjectingSubset() throws ImprintException { + // Given a record with large and small fields + ImprintRecord largeRecord = ImprintRecord.builder(testSchema) + .field(1, 42) // 4 bytes + .field(2, "x".repeat(1000)) // ~1000+ bytes + .field(3, 123L) // 8 bytes + .field(4, new byte[500]) // 500+ bytes + .build(); + + int originalPayloadSize = largeRecord.getBuffers().getPayload().remaining(); + + // When projecting only the small fields + ImprintRecord projected = ImprintOperations.project(largeRecord, 1, 3); + + // Then the payload size should be significantly smaller + assertTrue(projected.getBuffers().getPayload().remaining() < originalPayloadSize, + "Projected payload should be smaller than original"); + + // And the values should still be correct + assertEquals(42, projected.getInt32(1)); + assertEquals(123L, projected.getInt64(3)); + } + } + + @Nested + @DisplayName("Merge Operations") + class MergeOperations { + + @Test + @DisplayName("should merge records with distinct fields") + void shouldMergeRecordsWithDistinctFields() throws ImprintException { + // Given two records with different fields + ImprintRecord record1 = ImprintRecord.builder(testSchema) + .field(1, 42) + .field(3, "hello") + .build(); + + ImprintRecord record2 = ImprintRecord.builder(testSchema) + .field(2, true) + .field(4, 123L) + .build(); + + // When merging the records + ImprintRecord merged = ImprintOperations.merge(record1, record2); + + // Then all fields should be present + assertEquals(4, merged.getDirectory().size()); + assertEquals(42, merged.getInt32(1)); + assertTrue(merged.getBoolean(2)); + assertEquals("hello", merged.getString(3)); + assertEquals(123L, merged.getInt64(4)); + + // And directory should be sorted + List directory = merged.getDirectory(); + for (int i = 1; i < directory.size(); i++) { + assertTrue(directory.get(i - 1).getId() < directory.get(i).getId(), + "Directory entries should be sorted by field id"); + } + } + + @Test + @DisplayName("should merge records with overlapping fields") + void shouldMergeRecordsWithOverlappingFields() throws ImprintException { + // Given two records with overlapping fields + ImprintRecord record1 = ImprintRecord.builder(testSchema) + .field(2, "first") + .field(3, 42) + .build(); + + ImprintRecord record2 = ImprintRecord.builder(testSchema) + .field(1, true) + .field(2, "second") // Overlapping field + .build(); + + // When merging the records + ImprintRecord merged = ImprintOperations.merge(record1, record2); + + // Then first record's values should take precedence for duplicates + assertEquals(3, merged.getDirectory().size()); + assertTrue(merged.getBoolean(1)); + assertEquals("first", merged.getString(2)); // First record wins + assertEquals(42, merged.getInt32(3)); + } + + @Test + @DisplayName("should preserve schema id from first record") + void shouldPreserveSchemaIdFromFirstRecord() throws ImprintException { + // Given two records with different schema IDs + SchemaId schema1 = new SchemaId(1, 0xdeadbeef); + SchemaId schema2 = new SchemaId(1, 0xcafebabe); + + ImprintRecord record1 = ImprintRecord.builder(schema1) + .field(1, 42) + .build(); + + ImprintRecord record2 = ImprintRecord.builder(schema2) + .field(2, true) + .build(); + + // When merging the records + ImprintRecord merged = ImprintOperations.merge(record1, record2); + + // Then schema ID from first record should be preserved + assertEquals(schema1, merged.getHeader().getSchemaId()); + } + + @Test + @DisplayName("should handle merge with empty record") + void shouldHandleMergeWithEmptyRecord() throws ImprintException { + // When merging with empty record + ImprintRecord merged1 = ImprintOperations.merge(multiFieldRecord, emptyRecord); + ImprintRecord merged2 = ImprintOperations.merge(emptyRecord, multiFieldRecord); + + // Then results should contain all original fields + assertEquals(multiFieldRecord.getDirectory().size(), merged1.getDirectory().size()); + assertEquals(multiFieldRecord.getDirectory().size(), merged2.getDirectory().size()); + + // And values should be preserved + for (DirectoryEntry entry : multiFieldRecord.getDirectory()) { + Value originalValue = multiFieldRecord.getValue(entry.getId()); + assertEquals(originalValue, merged1.getValue(entry.getId())); + assertEquals(originalValue, merged2.getValue(entry.getId())); + } + } + + @Test + @DisplayName("should handle merge of two empty records") + void shouldHandleMergeOfTwoEmptyRecords() throws ImprintException { + // When merging two empty records + ImprintRecord merged = ImprintOperations.merge(emptyRecord, emptyRecord); + + // Then result should be empty but valid + assertEquals(0, merged.getDirectory().size()); + assertEquals(0, merged.getBuffers().getPayload().remaining()); + } + + @Test + @DisplayName("should maintain correct payload offsets after merge") + void shouldMaintainCorrectPayloadOffsetsAfterMerge() throws ImprintException { + // Given records with different field sizes + ImprintRecord record1 = ImprintRecord.builder(testSchema) + .field(1, 42) // 4 bytes + .field(3, "hello") // 5+ bytes + .build(); + + ImprintRecord record2 = ImprintRecord.builder(testSchema) + .field(2, true) // 1 byte + .field(4, new byte[]{1, 2, 3, 4, 5}) // 5+ bytes + .build(); + + // When merging + ImprintRecord merged = ImprintOperations.merge(record1, record2); + + // Then all fields should be accessible with correct values + assertEquals(42, merged.getInt32(1)); + assertTrue(merged.getBoolean(2)); + assertEquals("hello", merged.getString(3)); + assertArrayEquals(new byte[]{1, 2, 3, 4, 5}, merged.getBytes(4)); + + // And directory offsets should be sequential + List directory = merged.getDirectory(); + int expectedOffset = 0; + for (DirectoryEntry entry : directory) { + assertEquals(expectedOffset, entry.getOffset(), + "Field " + entry.getId() + " should have correct offset"); + + // Calculate next offset + var fieldData = merged.getRawBytes(entry.getId()); + assertNotNull(fieldData); + expectedOffset += fieldData.remaining(); + } + } + + @Test + @DisplayName("should handle large records efficiently") + void shouldHandleLargeRecordsEfficiently() throws ImprintException { + // Given records with many fields + var builder1 = ImprintRecord.builder(testSchema); + var builder2 = ImprintRecord.builder(testSchema); + + // Add 100 fields to each record (no overlap) + for (int i = 1; i <= 100; i++) { + builder1.field(i, i * 10); + } + for (int i = 101; i <= 200; i++) { + builder2.field(i, i * 10); + } + + ImprintRecord record1 = builder1.build(); + ImprintRecord record2 = builder2.build(); + + // When merging large records + ImprintRecord merged = ImprintOperations.merge(record1, record2); + + // Then all 200 fields should be present and accessible + assertEquals(200, merged.getDirectory().size()); + + // Spot check some values + assertEquals(10, merged.getInt32(1)); + assertEquals(500, merged.getInt32(50)); + assertEquals(1000, merged.getInt32(100)); + assertEquals(1010, merged.getInt32(101)); + assertEquals(1500, merged.getInt32(150)); + assertEquals(2000, merged.getInt32(200)); + } + } + + @Nested + @DisplayName("Error Handling") + class ErrorHandling { + + @Test + @DisplayName("should handle null record gracefully") + void shouldHandleNullRecordGracefully() { + assertThrows(NullPointerException.class, () -> ImprintOperations.project(null, 1, 2, 3)); + + assertThrows(NullPointerException.class, () -> ImprintOperations.merge(null, multiFieldRecord)); + + assertThrows(NullPointerException.class, () -> ImprintOperations.merge(multiFieldRecord, null)); + } + + @Test + @DisplayName("should handle null field ids gracefully") + void shouldHandleNullFieldIdsGracefully() { + assertThrows(NullPointerException.class, () -> ImprintOperations.project(multiFieldRecord, (int[]) null)); + } + } +} diff --git a/src/test/java/com/imprint/profile/ProfilerTest.java b/src/test/java/com/imprint/profile/ProfilerTest.java index 3b9f371..64be931 100644 --- a/src/test/java/com/imprint/profile/ProfilerTest.java +++ b/src/test/java/com/imprint/profile/ProfilerTest.java @@ -27,7 +27,7 @@ * - String operations and UTF-8 encoding * - ByteBuffer operations */ -@Disabled("Enable manually for profiling") +//@Disabled("Enable manually for profiling") public class ProfilerTest { private static final int ITERATIONS = 1_000_000; @@ -82,12 +82,12 @@ void profileSerialization() throws Exception { Thread.sleep(3000); var schemaId = new SchemaId(1, 0x12345678); - + System.out.println("Beginning serialization profiling..."); long start = System.nanoTime(); // Create and serialize many records (allocation hotspot) - for (int i = 0; i < 100_000; i++) { + for (int i = 0; i < 500_000; i++) { var writer = new ImprintWriter(schemaId); // Add various field types From 0c7b23742e861256490bca6bfa8212be90777526 Mon Sep 17 00:00:00 2001 From: expand3d <> Date: Sat, 7 Jun 2025 14:04:53 -0400 Subject: [PATCH 36/49] Optimize serialization path and remove ImprintWriter code in favor of Builder --- .../benchmark/ComparisonBenchmark.java | 185 ++++++++---------- .../benchmark/FieldAccessBenchmark.java | 73 ++++--- .../com/imprint/benchmark/MergeBenchmark.java | 82 ++++---- .../benchmark/SerializationBenchmark.java | 53 +++-- .../java/com/imprint/core/ImprintBuffers.java | 6 +- .../java/com/imprint/core/ImprintRecord.java | 77 ++++++-- .../imprint/core/ImprintRecordBuilder.java | 127 +++++++++++- .../java/com/imprint/core/ImprintWriter.java | 126 ------------ .../com/imprint/core/ImprintRecordTest.java | 69 +++---- .../com/imprint/profile/ProfilerTest.java | 53 +++-- 10 files changed, 423 insertions(+), 428 deletions(-) delete mode 100644 src/main/java/com/imprint/core/ImprintWriter.java diff --git a/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java b/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java index 6a6a958..e52388c 100644 --- a/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java +++ b/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java @@ -6,7 +6,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.flatbuffers.FlatBufferBuilder; import com.imprint.core.ImprintRecord; -import com.imprint.core.ImprintWriter; +import com.imprint.core.ImprintRecordBuilder; import com.imprint.core.SchemaId; import com.imprint.types.MapKey; import com.imprint.types.Value; @@ -143,7 +143,7 @@ public void deserializeSetupImprint(Blackhole bh) throws Exception { @Benchmark public void deserializeSetupFlatBuffers(Blackhole bh) { - TestRecordFB result = TestRecordFB.getRootAsTestRecordFB(flatbuffersBytes.duplicate()); + com.imprint.benchmark.TestRecordFB result = com.imprint.benchmark.TestRecordFB.getRootAsTestRecordFB(flatbuffersBytes.duplicate()); bh.consume(result); } @@ -177,7 +177,7 @@ public void deserializeAvro(Blackhole bh) throws Exception { @Benchmark public void deserializeProtobuf(Blackhole bh) throws Exception { - TestRecordProto.TestRecord result = TestRecordProto.TestRecord.parseFrom(protobufBytes); + com.imprint.benchmark.TestRecordProto.TestRecord result = com.imprint.benchmark.TestRecordProto.TestRecord.parseFrom(protobufBytes); bh.consume(result); } @@ -201,7 +201,7 @@ public void deserializeImprint(Blackhole bh) throws Exception { @Benchmark public void deserializeFlatBuffers(Blackhole bh) { - TestRecordFB result = TestRecordFB.getRootAsTestRecordFB(flatbuffersBytes.duplicate()); + com.imprint.benchmark.TestRecordFB result = com.imprint.benchmark.TestRecordFB.getRootAsTestRecordFB(flatbuffersBytes.duplicate()); // Access all fields result.id(); @@ -259,18 +259,18 @@ public void singleFieldAccessMessagePack(Blackhole bh) throws Exception { @Benchmark public void singleFieldAccessAvro(Blackhole bh) throws Exception { GenericRecord record = deserializeWithAvro(avroBytes); - bh.consume(record.get("extraData4")); + bh.consume(record.get("extra_data")); } @Benchmark public void singleFieldAccessProtobuf(Blackhole bh) throws Exception { - TestRecordProto.TestRecord record = TestRecordProto.TestRecord.parseFrom(protobufBytes); + com.imprint.benchmark.TestRecordProto.TestRecord record = com.imprint.benchmark.TestRecordProto.TestRecord.parseFrom(protobufBytes); bh.consume(record.getExtraData(4)); } @Benchmark public void singleFieldAccessFlatBuffers(Blackhole bh) { - TestRecordFB record = TestRecordFB.getRootAsTestRecordFB(flatbuffersBytes.duplicate()); + com.imprint.benchmark.TestRecordFB record = com.imprint.benchmark.TestRecordFB.getRootAsTestRecordFB(flatbuffersBytes.duplicate()); bh.consume(record.extraData(4)); } @@ -381,10 +381,10 @@ public void mergeAvro(Blackhole bh) throws Exception { //@Benchmark public void mergeProtobuf(Blackhole bh) throws Exception { - var record1 = TestRecordProto.TestRecord.parseFrom(protobufBytes); + var record1 = com.imprint.benchmark.TestRecordProto.TestRecord.parseFrom(protobufBytes); var record2Data = createTestRecord2(); var record2Bytes = serializeWithProtobuf(record2Data); - var record2 = TestRecordProto.TestRecord.parseFrom(record2Bytes); + var record2 = com.imprint.benchmark.TestRecordProto.TestRecord.parseFrom(record2Bytes); var merged = mergeProtobufRecords(record1, record2); byte[] result = merged.toByteArray(); @@ -393,10 +393,10 @@ public void mergeProtobuf(Blackhole bh) throws Exception { //@Benchmark public void mergeFlatBuffers(Blackhole bh) { - var record1 = TestRecordFB.getRootAsTestRecordFB(flatbuffersBytes.duplicate()); + var record1 = com.imprint.benchmark.TestRecordFB.getRootAsTestRecordFB(flatbuffersBytes.duplicate()); var record2Data = createTestRecord2(); var record2Buffer = serializeWithFlatBuffers(record2Data); - var record2 = TestRecordFB.getRootAsTestRecordFB(record2Buffer); + var record2 = com.imprint.benchmark.TestRecordFB.getRootAsTestRecordFB(record2Buffer); var merged = mergeFlatBuffersRecords(record1, record2); bh.consume(merged); @@ -521,37 +521,21 @@ private void setupAvro() { } private ByteBuffer serializeWithImprint(TestRecord data) throws Exception { - var writer = new ImprintWriter(new SchemaId(1, 0x12345678)); - - writer.addField(1, Value.fromInt32(data.id)); - writer.addField(2, Value.fromString(data.name)); - writer.addField(3, Value.fromFloat64(data.price)); - writer.addField(4, Value.fromBoolean(data.active)); - writer.addField(5, Value.fromString(data.category)); - - var tagValues = new ArrayList(); - if (data.tags != null) { - for (String tag : data.tags) { - tagValues.add(Value.fromString(tag)); - } - } - writer.addField(6, Value.fromArray(tagValues)); - - var metadataMap = new HashMap(); - if (data.metadata != null) { - for (var entry : data.metadata.entrySet()) { - metadataMap.put(MapKey.fromString(entry.getKey()), Value.fromString(entry.getValue())); - } - } - writer.addField(7, Value.fromMap(metadataMap)); + var builder = ImprintRecord.builder(new SchemaId(1, 0x12345678)); + + builder.field(1, data.id); + builder.field(2, data.name); + builder.field(3, data.price); + builder.field(4, data.active); + builder.field(5, data.category); + builder.field(6, data.tags); + builder.field(7, data.metadata); - if (data.extraData != null) { - for (int i = 0; i < data.extraData.size(); i++) { - writer.addField(8 + i, Value.fromString(data.extraData.get(i))); - } + for (int i = 0; i < data.extraData.size(); i++) { + builder.field(8 + i, data.extraData.get(i)); } - return writer.build().serializeToBuffer(); + return builder.build().serializeToBuffer(); } private byte[] serializeWithJacksonJson(TestRecord data) throws Exception { @@ -601,7 +585,7 @@ private GenericRecord deserializeWithAvro(byte[] data) throws Exception { } private byte[] serializeWithProtobuf(TestRecord data) { - var builder = TestRecordProto.TestRecord.newBuilder() + var builder = com.imprint.benchmark.TestRecordProto.TestRecord.newBuilder() .setId(data.id) .setName(data.name) .setPrice(data.price) @@ -618,20 +602,17 @@ private byte[] serializeWithProtobuf(TestRecord data) { } private ByteBuffer serializeWithFlatBuffers(TestRecord data) { - FlatBufferBuilder builder = new FlatBufferBuilder(1024); + var builder = new FlatBufferBuilder(1024); - // Create strings (must be created before the object that uses them) int nameOffset = builder.createString(data.name); int categoryOffset = builder.createString(data.category); - // Create tags array int[] tagOffsets = new int[data.tags.size()]; for (int i = 0; i < data.tags.size(); i++) { tagOffsets[i] = builder.createString(data.tags.get(i)); } - int tagsOffset = TestRecordFB.createTagsVector(builder, tagOffsets); + int tagsOffset = com.imprint.benchmark.TestRecordFB.createTagsVector(builder, tagOffsets); - // Create metadata (as parallel arrays for keys and values) String[] metadataKeys = data.metadata.keySet().toArray(new String[0]); String[] metadataValues = new String[metadataKeys.length]; int[] keyOffsets = new int[metadataKeys.length]; @@ -642,51 +623,51 @@ private ByteBuffer serializeWithFlatBuffers(TestRecord data) { keyOffsets[i] = builder.createString(metadataKeys[i]); valueOffsets[i] = builder.createString(metadataValues[i]); } - int metadataKeysOffset = TestRecordFB.createMetadataKeysVector(builder, keyOffsets); - int metadataValuesOffset = TestRecordFB.createMetadataValuesVector(builder, valueOffsets); + int metadataKeysOffset = com.imprint.benchmark.TestRecordFB.createMetadataKeysVector(builder, keyOffsets); + int metadataValuesOffset = com.imprint.benchmark.TestRecordFB.createMetadataValuesVector(builder, valueOffsets); - // Create extra data array int[] extraDataOffsets = new int[data.extraData.size()]; for (int i = 0; i < data.extraData.size(); i++) { extraDataOffsets[i] = builder.createString(data.extraData.get(i)); } - int extraDataOffset = TestRecordFB.createExtraDataVector(builder, extraDataOffsets); - - // Create the main object - TestRecordFB.startTestRecordFB(builder); - TestRecordFB.addId(builder, data.id); - TestRecordFB.addName(builder, nameOffset); - TestRecordFB.addPrice(builder, data.price); - TestRecordFB.addActive(builder, data.active); - TestRecordFB.addCategory(builder, categoryOffset); - TestRecordFB.addTags(builder, tagsOffset); - TestRecordFB.addMetadataKeys(builder, metadataKeysOffset); - TestRecordFB.addMetadataValues(builder, metadataValuesOffset); - TestRecordFB.addExtraData(builder, extraDataOffset); - int recordOffset = TestRecordFB.endTestRecordFB(builder); - - // Finish and return + int extraDataOffset = com.imprint.benchmark.TestRecordFB.createExtraDataVector(builder, extraDataOffsets); + + com.imprint.benchmark.TestRecordFB.startTestRecordFB(builder); + com.imprint.benchmark.TestRecordFB.addId(builder, data.id); + com.imprint.benchmark.TestRecordFB.addName(builder, nameOffset); + com.imprint.benchmark.TestRecordFB.addPrice(builder, data.price); + com.imprint.benchmark.TestRecordFB.addActive(builder, data.active); + com.imprint.benchmark.TestRecordFB.addCategory(builder, categoryOffset); + com.imprint.benchmark.TestRecordFB.addTags(builder, tagsOffset); + com.imprint.benchmark.TestRecordFB.addMetadataKeys(builder, metadataKeysOffset); + com.imprint.benchmark.TestRecordFB.addMetadataValues(builder, metadataValuesOffset); + com.imprint.benchmark.TestRecordFB.addExtraData(builder, extraDataOffset); + int recordOffset = com.imprint.benchmark.TestRecordFB.endTestRecordFB(builder); + builder.finish(recordOffset); - return builder.dataBuffer().slice(); + return builder.dataBuffer(); } private ImprintRecord simulateMerge(ImprintRecord first, ImprintRecord second) throws Exception { - var writer = new ImprintWriter(first.getHeader().getSchemaId()); + var builder = ImprintRecord.builder(first.getHeader().getSchemaId()); var usedFieldIds = new HashSet(); - - copyFieldsToWriter(first, writer, usedFieldIds); - copyFieldsToWriter(second, writer, usedFieldIds); - - return writer.build(); - } - - private void copyFieldsToWriter(ImprintRecord record, ImprintWriter writer, Set usedFieldIds) throws Exception { + + // Copy fields from first record (takes precedence) + copyFieldsToBuilder(first, builder, usedFieldIds); + + // Copy non-conflicting fields from second record + copyFieldsToBuilder(second, builder, usedFieldIds); + + return builder.build(); + } + + private void copyFieldsToBuilder(ImprintRecord record, ImprintRecordBuilder builder, Set usedFieldIds) throws Exception { for (var entry : record.getDirectory()) { int fieldId = entry.getId(); if (!usedFieldIds.contains(fieldId)) { var value = record.getValue(fieldId); if (value != null) { - writer.addField(fieldId, value); + builder.field(fieldId, value); usedFieldIds.add(fieldId); } } @@ -694,7 +675,7 @@ private void copyFieldsToWriter(ImprintRecord record, ImprintWriter writer, Set< } private TestRecord mergeTestRecords(TestRecord first, TestRecord second) { - var merged = new TestRecord(); + TestRecord merged = new TestRecord(); merged.id = first.id; merged.name = first.name != null ? first.name : second.name; merged.price = first.price != 0.0 ? first.price : second.price; @@ -729,28 +710,25 @@ private GenericRecord mergeAvroRecords(GenericRecord first, GenericRecord second return merged; } - private TestRecordProto.TestRecord mergeProtobufRecords(TestRecordProto.TestRecord first, TestRecordProto.TestRecord second) { - return TestRecordProto.TestRecord.newBuilder() + private com.imprint.benchmark.TestRecordProto.TestRecord mergeProtobufRecords(com.imprint.benchmark.TestRecordProto.TestRecord first, com.imprint.benchmark.TestRecordProto.TestRecord second) { + return com.imprint.benchmark.TestRecordProto.TestRecord.newBuilder() .mergeFrom(first) .mergeFrom(second) .build(); } - private ByteBuffer mergeFlatBuffersRecords(TestRecordFB first, TestRecordFB second) { - FlatBufferBuilder builder = new FlatBufferBuilder(1024); + private ByteBuffer mergeFlatBuffersRecords(com.imprint.benchmark.TestRecordFB first, com.imprint.benchmark.TestRecordFB second) { + var builder = new FlatBufferBuilder(1024); - // Use second record's values if they exist, otherwise first record's values String name = second.name() != null && !second.name().isEmpty() ? second.name() : first.name(); String category = second.category() != null && !second.category().isEmpty() ? second.category() : first.category(); double price = second.price() != 0.0 ? second.price() : first.price(); - boolean active = second.active(); // Use second's boolean value - int id = first.id(); // Keep first record's ID + boolean active = second.active(); + int id = first.id(); - // Create merged strings int nameOffset = builder.createString(name); int categoryOffset = builder.createString(category); - // Merge tags (combine both arrays) List mergedTags = new ArrayList<>(); for (int i = 0; i < first.tagsLength(); i++) { mergedTags.add(first.tags(i)); @@ -763,9 +741,8 @@ private ByteBuffer mergeFlatBuffersRecords(TestRecordFB first, TestRecordFB seco for (int i = 0; i < mergedTags.size(); i++) { tagOffsets[i] = builder.createString(mergedTags.get(i)); } - int tagsOffset = TestRecordFB.createTagsVector(builder, tagOffsets); + int tagsOffset = com.imprint.benchmark.TestRecordFB.createTagsVector(builder, tagOffsets); - // Merge metadata (second overwrites first) Map mergedMetadata = new HashMap<>(); for (int i = 0; i < first.metadataKeysLength(); i++) { mergedMetadata.put(first.metadataKeys(i), first.metadataValues(i)); @@ -782,31 +759,29 @@ private ByteBuffer mergeFlatBuffersRecords(TestRecordFB first, TestRecordFB seco keyOffsets[i] = builder.createString(metadataKeys[i]); valueOffsets[i] = builder.createString(mergedMetadata.get(metadataKeys[i])); } - int metadataKeysOffset = TestRecordFB.createMetadataKeysVector(builder, keyOffsets); - int metadataValuesOffset = TestRecordFB.createMetadataValuesVector(builder, valueOffsets); + int metadataKeysOffset = com.imprint.benchmark.TestRecordFB.createMetadataKeysVector(builder, keyOffsets); + int metadataValuesOffset = com.imprint.benchmark.TestRecordFB.createMetadataValuesVector(builder, valueOffsets); - // Use first record's extra data (or could merge both) int[] extraDataOffsets = new int[first.extraDataLength()]; for (int i = 0; i < first.extraDataLength(); i++) { extraDataOffsets[i] = builder.createString(first.extraData(i)); } - int extraDataOffset = TestRecordFB.createExtraDataVector(builder, extraDataOffsets); - - // Create the merged object - TestRecordFB.startTestRecordFB(builder); - TestRecordFB.addId(builder, id); - TestRecordFB.addName(builder, nameOffset); - TestRecordFB.addPrice(builder, price); - TestRecordFB.addActive(builder, active); - TestRecordFB.addCategory(builder, categoryOffset); - TestRecordFB.addTags(builder, tagsOffset); - TestRecordFB.addMetadataKeys(builder, metadataKeysOffset); - TestRecordFB.addMetadataValues(builder, metadataValuesOffset); - TestRecordFB.addExtraData(builder, extraDataOffset); - int recordOffset = TestRecordFB.endTestRecordFB(builder); + int extraDataOffset = com.imprint.benchmark.TestRecordFB.createExtraDataVector(builder, extraDataOffsets); + + com.imprint.benchmark.TestRecordFB.startTestRecordFB(builder); + com.imprint.benchmark.TestRecordFB.addId(builder, id); + com.imprint.benchmark.TestRecordFB.addName(builder, nameOffset); + com.imprint.benchmark.TestRecordFB.addPrice(builder, price); + com.imprint.benchmark.TestRecordFB.addActive(builder, active); + com.imprint.benchmark.TestRecordFB.addCategory(builder, categoryOffset); + com.imprint.benchmark.TestRecordFB.addTags(builder, tagsOffset); + com.imprint.benchmark.TestRecordFB.addMetadataKeys(builder, metadataKeysOffset); + com.imprint.benchmark.TestRecordFB.addMetadataValues(builder, metadataValuesOffset); + com.imprint.benchmark.TestRecordFB.addExtraData(builder, extraDataOffset); + int recordOffset = com.imprint.benchmark.TestRecordFB.endTestRecordFB(builder); builder.finish(recordOffset); - return builder.dataBuffer().slice(); + return builder.dataBuffer(); } private TestRecord createTestRecord() { diff --git a/src/jmh/java/com/imprint/benchmark/FieldAccessBenchmark.java b/src/jmh/java/com/imprint/benchmark/FieldAccessBenchmark.java index 1ead21f..06a7717 100644 --- a/src/jmh/java/com/imprint/benchmark/FieldAccessBenchmark.java +++ b/src/jmh/java/com/imprint/benchmark/FieldAccessBenchmark.java @@ -1,7 +1,7 @@ package com.imprint.benchmark; import com.imprint.core.ImprintRecord; -import com.imprint.core.ImprintWriter; +import com.imprint.core.ImprintRecordBuilder; import com.imprint.core.SchemaId; import com.imprint.types.MapKey; import com.imprint.types.Value; @@ -20,8 +20,8 @@ @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.NANOSECONDS) @State(Scope.Benchmark) -@Warmup(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS) -@Measurement(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS) +@Warmup(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 10, time = 1, timeUnit = TimeUnit.SECONDS) @Fork(1) public class FieldAccessBenchmark { @@ -196,83 +196,80 @@ public void accessDenseRecord(Blackhole bh) throws Exception { * This should be replaced with actual project API when available. */ private ImprintRecord simulateProject(ImprintRecord source, int[] fieldIds) throws Exception { - var writer = new ImprintWriter(source.getHeader().getSchemaId()); + var builder = ImprintRecord.builder(source.getHeader().getSchemaId()); for (int fieldId : fieldIds) { var value = source.getValue(fieldId); if (value != null) { - writer.addField(fieldId, value); + builder.field(fieldId, value); } } - return writer.build(); + return builder.build(); } private ImprintRecord createSparseRecord() throws Exception { - var writer = new ImprintWriter(new SchemaId(1, 0x12345678)); - - // Sparse record with large field IDs and few fields - writer.addField(1000, Value.fromString("sparse_field_1")); - writer.addField(5000, Value.fromInt32(42)); - writer.addField(10000, Value.fromFloat64(3.14159)); - writer.addField(15000, Value.fromBoolean(true)); - writer.addField(20000, Value.fromString("sparse_field_5")); - - return writer.build(); + return ImprintRecord.builder(new SchemaId(1, 0x12345678)) + .field(1000, Value.fromString("sparse_field_1")) + .field(5000, Value.fromInt32(42)) + .field(10000, Value.fromFloat64(3.14159)) + .field(15000, Value.fromBoolean(true)) + .field(20000, Value.fromString("sparse_field_5")) + .build(); } private ImprintRecord createDenseRecord() throws Exception { - var writer = new ImprintWriter(new SchemaId(2, 0x87654321)); + var builder = ImprintRecord.builder(new SchemaId(2, 0x87654321)); // Dense record with 100 sequential fields for (int i = 1; i <= 100; i++) { switch (i % 5) { case 0: - writer.addField(i, Value.fromString("string_field_" + i)); + builder.field(i, Value.fromString("string_field_" + i)); break; case 1: - writer.addField(i, Value.fromInt32(i * 10)); + builder.field(i, Value.fromInt32(i * 10)); break; case 2: - writer.addField(i, Value.fromFloat64(i * 1.5)); + builder.field(i, Value.fromFloat64(i * 1.5)); break; case 3: - writer.addField(i, Value.fromBoolean(i % 2 == 0)); + builder.field(i, Value.fromBoolean(i % 2 == 0)); break; case 4: - writer.addField(i, Value.fromInt64(i * 1000L)); + builder.field(i, Value.fromInt64(i * 1000L)); break; } } - return writer.build(); + return builder.build(); } private ImprintRecord createLargeRecord() throws Exception { - var writer = new ImprintWriter(new SchemaId(3, 0x11223344)); + var builder = ImprintRecord.builder(new SchemaId(3, 0xABCDEF12)); - // Large record with complex data types - writer.addField(1, Value.fromString("LargeRecord")); + // Large record with complex fields (arrays, maps) + builder.field(1, Value.fromString("Large record with complex data")); - // Large array field - var largeArray = new ArrayList(); - for (int i = 0; i < 1000; i++) { - largeArray.add(Value.fromString("array_item_" + i)); + // Add a large array + var list = new ArrayList(); + for (int i = 0; i < 200; i++) { + list.add(Value.fromInt32(i)); } - writer.addField(2, Value.fromArray(largeArray)); + builder.field(2, Value.fromArray(list)); - // Large map field - var largeMap = new HashMap(); + // Add a large map + var map = new HashMap(); for (int i = 0; i < 100; i++) { - largeMap.put(MapKey.fromString("key_" + i), Value.fromString("map_value_" + i)); + map.put(MapKey.fromString("key_" + i), Value.fromString("value_" + i)); } - writer.addField(3, Value.fromMap(largeMap)); + builder.field(3, Value.fromMap(map)); - // Many regular fields + // Add more fields for (int i = 4; i <= 50; i++) { - writer.addField(i, Value.fromString("large_record_field_" + i + "_with_substantial_content")); + builder.field(i, Value.fromBytes(new byte[1024])); // 1KB byte arrays } - return writer.build(); + return builder.build(); } } \ No newline at end of file diff --git a/src/jmh/java/com/imprint/benchmark/MergeBenchmark.java b/src/jmh/java/com/imprint/benchmark/MergeBenchmark.java index f93092a..63e43e6 100644 --- a/src/jmh/java/com/imprint/benchmark/MergeBenchmark.java +++ b/src/jmh/java/com/imprint/benchmark/MergeBenchmark.java @@ -1,7 +1,7 @@ package com.imprint.benchmark; import com.imprint.core.ImprintRecord; -import com.imprint.core.ImprintWriter; +import com.imprint.core.ImprintRecordBuilder; import com.imprint.core.SchemaId; import com.imprint.types.Value; import org.openjdk.jmh.annotations.*; @@ -83,25 +83,25 @@ public void mergeWithConflicts(Blackhole bh) throws Exception { * This should be replaced with actual merge API when available. */ private ImprintRecord simulateMerge(ImprintRecord first, ImprintRecord second) throws Exception { - var writer = new ImprintWriter(first.getHeader().getSchemaId()); + var builder = ImprintRecord.builder(first.getHeader().getSchemaId()); var usedFieldIds = new HashSet(); // Copy fields from first record (takes precedence) - copyFieldsToWriter(first, writer, usedFieldIds); + copyFieldsToBuilder(first, builder, usedFieldIds); // Copy non-conflicting fields from second record - copyFieldsToWriter(second, writer, usedFieldIds); + copyFieldsToBuilder(second, builder, usedFieldIds); - return writer.build(); + return builder.build(); } - private void copyFieldsToWriter(ImprintRecord record, ImprintWriter writer, Set usedFieldIds) throws Exception { + private void copyFieldsToBuilder(ImprintRecord record, ImprintRecordBuilder builder, Set usedFieldIds) throws Exception { for (var entry : record.getDirectory()) { int fieldId = entry.getId(); if (!usedFieldIds.contains(fieldId)) { var value = record.getValue(fieldId); if (value != null) { - writer.addField(fieldId, value); + builder.field(fieldId, value); usedFieldIds.add(fieldId); } } @@ -109,55 +109,49 @@ private void copyFieldsToWriter(ImprintRecord record, ImprintWriter writer, Set< } private ImprintRecord createProductRecord() throws Exception { - var writer = new ImprintWriter(new SchemaId(1, 0x12345678)); - - writer.addField(1, Value.fromString("Product")); - writer.addField(2, Value.fromInt32(12345)); - writer.addField(3, Value.fromString("Laptop")); - writer.addField(4, Value.fromFloat64(999.99)); - writer.addField(5, Value.fromString("Electronics")); - writer.addField(6, Value.fromInt32(50)); // stock - writer.addField(7, Value.fromString("TechCorp")); - writer.addField(8, Value.fromBoolean(true)); // available - - return writer.build(); + return ImprintRecord.builder(new SchemaId(1, 0x12345678)) + .field(1, Value.fromString("Product")) + .field(2, Value.fromInt32(12345)) + .field(3, Value.fromString("Laptop")) + .field(4, Value.fromFloat64(999.99)) + .field(5, Value.fromString("Electronics")) + .field(6, Value.fromInt32(50)) // stock + .field(7, Value.fromString("TechCorp")) + .field(8, Value.fromBoolean(true)) // available + .build(); } private ImprintRecord createOrderRecord() throws Exception { - var writer = new ImprintWriter(new SchemaId(2, 0x87654321)); - - writer.addField(10, Value.fromString("Order")); - writer.addField(11, Value.fromInt32(67890)); - writer.addField(12, Value.fromInt32(12345)); // product_id (overlaps with product) - writer.addField(13, Value.fromInt32(2)); // quantity - writer.addField(14, Value.fromFloat64(1999.98)); // total - writer.addField(15, Value.fromString("2024-01-15")); // order_date - writer.addField(16, Value.fromString("shipped")); // status - - return writer.build(); + return ImprintRecord.builder(new SchemaId(2, 0x87654321)) + .field(10, Value.fromString("Order")) + .field(11, Value.fromInt32(67890)) + .field(12, Value.fromInt32(12345)) // product_id (overlaps with product) + .field(13, Value.fromInt32(2)) // quantity + .field(14, Value.fromFloat64(1999.98)) // total + .field(15, Value.fromString("2024-01-15")) // order_date + .field(16, Value.fromString("shipped")) // status + .build(); } private ImprintRecord createCustomerRecord() throws Exception { - var writer = new ImprintWriter(new SchemaId(3, 0x11223344)); - - writer.addField(20, Value.fromString("Customer")); - writer.addField(21, Value.fromInt32(555)); - writer.addField(22, Value.fromString("John Doe")); - writer.addField(23, Value.fromString("john.doe@email.com")); - writer.addField(24, Value.fromString("123 Main St")); - writer.addField(25, Value.fromString("premium")); // tier - writer.addField(26, Value.fromBoolean(true)); // active - - return writer.build(); + return ImprintRecord.builder(new SchemaId(3, 0x11223344)) + .field(20, Value.fromString("Customer")) + .field(21, Value.fromInt32(555)) + .field(22, Value.fromString("John Doe")) + .field(23, Value.fromString("john.doe@email.com")) + .field(24, Value.fromString("123 Main St")) + .field(25, Value.fromString("premium")) // tier + .field(26, Value.fromBoolean(true)) // active + .build(); } private ImprintRecord createRecordWithFields(int startId, int endId, String prefix) throws Exception { - var writer = new ImprintWriter(new SchemaId(1, 0x12345678)); + var builder = ImprintRecord.builder(new SchemaId(1, 0x12345678)); for (int i = startId; i <= endId; i++) { - writer.addField(i, Value.fromString(prefix + "field_" + i)); + builder.field(i, Value.fromString(prefix + "field_" + i)); } - return writer.build(); + return builder.build(); } } \ No newline at end of file diff --git a/src/jmh/java/com/imprint/benchmark/SerializationBenchmark.java b/src/jmh/java/com/imprint/benchmark/SerializationBenchmark.java index 3275843..11e2b29 100644 --- a/src/jmh/java/com/imprint/benchmark/SerializationBenchmark.java +++ b/src/jmh/java/com/imprint/benchmark/SerializationBenchmark.java @@ -1,7 +1,6 @@ package com.imprint.benchmark; import com.imprint.core.ImprintRecord; -import com.imprint.core.ImprintWriter; import com.imprint.core.SchemaId; import com.imprint.types.MapKey; import com.imprint.types.Value; @@ -89,27 +88,25 @@ public void deserializeLargeRecord(Blackhole bh) throws Exception { // ===== HELPER METHODS ===== private ImprintRecord createSmallRecord() throws Exception { - var writer = new ImprintWriter(new SchemaId(1, 0x12345678)); - // Small record: ~10 fields, simple types - writer.addField(1, Value.fromString("Product")); - writer.addField(2, Value.fromInt32(12345)); - writer.addField(3, Value.fromFloat64(99.99)); - writer.addField(4, Value.fromBoolean(true)); - writer.addField(5, Value.fromString("Electronics")); - - return writer.build(); + return ImprintRecord.builder(new SchemaId(1, 0x12345678)) + .field(1, Value.fromString("Product")) + .field(2, Value.fromInt32(12345)) + .field(3, Value.fromFloat64(99.99)) + .field(4, Value.fromBoolean(true)) + .field(5, Value.fromString("Electronics")) + .build(); } private ImprintRecord createMediumRecord() throws Exception { - var writer = new ImprintWriter(new SchemaId(1, 0x12345678)); + var builder = ImprintRecord.builder(new SchemaId(1, 0x12345678)); // Medium record: ~50 fields, mixed types including arrays - writer.addField(1, Value.fromString("Product")); - writer.addField(2, Value.fromInt32(12345)); - writer.addField(3, Value.fromFloat64(99.99)); - writer.addField(4, Value.fromBoolean(true)); - writer.addField(5, Value.fromString("Electronics")); + builder.field(1, Value.fromString("Product")); + builder.field(2, Value.fromInt32(12345)); + builder.field(3, Value.fromFloat64(99.99)); + builder.field(4, Value.fromBoolean(true)); + builder.field(5, Value.fromString("Electronics")); // Add array field var tags = Arrays.asList( @@ -117,50 +114,50 @@ private ImprintRecord createMediumRecord() throws Exception { Value.fromString("trending"), Value.fromString("bestseller") ); - writer.addField(6, Value.fromArray(tags)); + builder.field(6, Value.fromArray(tags)); // Add map field (all string values for consistency) var metadata = new HashMap(); metadata.put(MapKey.fromString("manufacturer"), Value.fromString("TechCorp")); metadata.put(MapKey.fromString("model"), Value.fromString("TC-2024")); metadata.put(MapKey.fromString("year"), Value.fromString("2024")); - writer.addField(7, Value.fromMap(metadata)); + builder.field(7, Value.fromMap(metadata)); // Add more fields for medium size for (int i = 8; i <= 50; i++) { - writer.addField(i, Value.fromString("field_" + i + "_value")); + builder.field(i, Value.fromString("field_" + i + "_value")); } - return writer.build(); + return builder.build(); } private ImprintRecord createLargeRecord() throws Exception { - var writer = new ImprintWriter(new SchemaId(1, 0x12345678)); + var builder = ImprintRecord.builder(new SchemaId(1, 0x12345678)); // Large record: ~200 fields, complex nested structures - writer.addField(1, Value.fromString("LargeProduct")); - writer.addField(2, Value.fromInt32(12345)); - writer.addField(3, Value.fromFloat64(99.99)); + builder.field(1, Value.fromString("LargeProduct")); + builder.field(2, Value.fromInt32(12345)); + builder.field(3, Value.fromFloat64(99.99)); // Large array var largeArray = new ArrayList(); for (int i = 0; i < 100; i++) { largeArray.add(Value.fromString("item_" + i)); } - writer.addField(4, Value.fromArray(largeArray)); + builder.field(4, Value.fromArray(largeArray)); // Large map var largeMap = new HashMap(); for (int i = 0; i < 50; i++) { largeMap.put(MapKey.fromString("key_" + i), Value.fromString("value_" + i)); } - writer.addField(5, Value.fromMap(largeMap)); + builder.field(5, Value.fromMap(largeMap)); // Many string fields for (int i = 6; i <= 200; i++) { - writer.addField(i, Value.fromString("this_is_a_longer_field_value_for_field_" + i + "_to_increase_record_size")); + builder.field(i, Value.fromString("this_is_a_longer_field_value_for_field_" + i + "_to_increase_record_size")); } - return writer.build(); + return builder.build(); } } \ No newline at end of file diff --git a/src/main/java/com/imprint/core/ImprintBuffers.java b/src/main/java/com/imprint/core/ImprintBuffers.java index c14d6df..ac3b4d3 100644 --- a/src/main/java/com/imprint/core/ImprintBuffers.java +++ b/src/main/java/com/imprint/core/ImprintBuffers.java @@ -117,8 +117,10 @@ public int getDirectoryCount() { * Create a new buffer containing the serialized directory. */ public ByteBuffer serializeDirectory() { - ensureDirectoryParsed(); - return createDirectoryBuffer(new ArrayList<>(parsedDirectory.values())); + // The directoryBuffer is created on construction and is read-only. + // If constructed from raw bytes, it's a view of the original. + // If constructed from a list, it's a fresh buffer. In both cases, it's ready. + return directoryBuffer.duplicate(); } // ========== PRIVATE METHODS ========== diff --git a/src/main/java/com/imprint/core/ImprintRecord.java b/src/main/java/com/imprint/core/ImprintRecord.java index e720df5..83ddb03 100644 --- a/src/main/java/com/imprint/core/ImprintRecord.java +++ b/src/main/java/com/imprint/core/ImprintRecord.java @@ -156,23 +156,20 @@ public ImprintRecord getRow(int fieldId) throws ImprintException { * Serialize this record to a ByteBuffer. */ public ByteBuffer serializeToBuffer() { - var buffer = ByteBuffer.allocate(estimateSerializedSize()); - buffer.order(ByteOrder.LITTLE_ENDIAN); + var directoryBuffer = buffers.serializeDirectory(); // This is now optimized to return a duplicate + var payloadBuffer = buffers.getPayload(); - // Write header - serializeHeader(buffer); + int finalSize = Constants.HEADER_BYTES + directoryBuffer.remaining() + payloadBuffer.remaining(); + var finalBuffer = ByteBuffer.allocate(finalSize); + finalBuffer.order(ByteOrder.LITTLE_ENDIAN); - // Write directory - var directoryBuffer = buffers.serializeDirectory(); - buffer.put(directoryBuffer); + // Assemble the final record from existing components + serializeHeader(this.header, finalBuffer); + finalBuffer.put(directoryBuffer); + finalBuffer.put(payloadBuffer.duplicate()); // Use duplicate to preserve original buffer state - // Write payload - var payload = buffers.getPayload(); - var payloadCopy = payload.duplicate(); - buffer.put(payloadCopy); - - buffer.flip(); - return buffer; + finalBuffer.flip(); + return finalBuffer.asReadOnlyBuffer(); } public int estimateSerializedSize() { @@ -182,6 +179,32 @@ public int estimateSerializedSize() { return size; } + /** + * Serializes the components of a record into a single ByteBuffer. + * This provides a direct serialization path without needing a live ImprintRecord instance. + * + * @param schemaId The schema identifier for the record. + * @param directory The list of directory entries, which must be sorted by field ID. + * @param payload The ByteBuffer containing all field data concatenated. + * @return A read-only ByteBuffer with the complete serialized record. + */ + public static ByteBuffer serialize(SchemaId schemaId, List directory, ByteBuffer payload) { + var header = new Header(new Flags((byte) 0), schemaId, payload.remaining()); + var directoryBuffer = createDirectoryBuffer(directory); + + int finalSize = Constants.HEADER_BYTES + directoryBuffer.remaining() + payload.remaining(); + var finalBuffer = ByteBuffer.allocate(finalSize); + finalBuffer.order(ByteOrder.LITTLE_ENDIAN); + + // Assemble the final record + serializeHeader(header, finalBuffer); + finalBuffer.put(directoryBuffer); + finalBuffer.put(payload); + + finalBuffer.flip(); + return finalBuffer.asReadOnlyBuffer(); + } + // ========== STATIC FACTORY METHODS ========== public static ImprintRecordBuilder builder(SchemaId schemaId) { @@ -272,7 +295,7 @@ private Value deserializeValue(TypeCode typeCode, ByteBuffer buffer) throws Impr } } - private void serializeHeader(ByteBuffer buffer) { + private static void serializeHeader(Header header, ByteBuffer buffer) { buffer.put(Constants.MAGIC); buffer.put(Constants.VERSION); buffer.put(header.getFlags().getValue()); @@ -305,6 +328,30 @@ private static Header deserializeHeader(ByteBuffer buffer) throws ImprintExcepti return new Header(flags, new SchemaId(fieldSpaceId, schemaHash), payloadSize); } + /** + * Creates a serialized representation of the directory. + */ + private static ByteBuffer createDirectoryBuffer(List directory) { + try { + int bufferSize = VarInt.encodedLength(directory.size()) + (directory.size() * Constants.DIR_ENTRY_BYTES); + var buffer = ByteBuffer.allocate(bufferSize); + buffer.order(ByteOrder.LITTLE_ENDIAN); + + VarInt.encode(directory.size(), buffer); + for (var entry : directory) { + buffer.putShort(entry.getId()); + buffer.put(entry.getTypeCode().getCode()); + buffer.putInt(entry.getOffset()); + } + + buffer.flip(); + return buffer.asReadOnlyBuffer(); + } catch (Exception e) { + // Should not happen with valid inputs + return ByteBuffer.allocate(0).asReadOnlyBuffer(); + } + } + @Override public String toString() { return String.format("ImprintRecord{header=%s, directorySize=%d, payloadSize=%d}", diff --git a/src/main/java/com/imprint/core/ImprintRecordBuilder.java b/src/main/java/com/imprint/core/ImprintRecordBuilder.java index 51a3525..39238a7 100644 --- a/src/main/java/com/imprint/core/ImprintRecordBuilder.java +++ b/src/main/java/com/imprint/core/ImprintRecordBuilder.java @@ -1,10 +1,19 @@ package com.imprint.core; +import com.imprint.error.ErrorType; import com.imprint.error.ImprintException; import com.imprint.types.MapKey; import com.imprint.types.Value; -import java.util.*; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.TreeMap; /** * A fluent builder for creating ImprintRecord instances with type-safe, @@ -129,16 +138,55 @@ public int fieldCount() { } public Set fieldIds() { - return new TreeSet<>(fields.keySet()); + return fields.keySet(); } // Build the final record public ImprintRecord build() throws ImprintException { - var writer = new ImprintWriter(schemaId); + var directory = new ArrayList(fields.size()); + var payloadBuffer = ByteBuffer.allocate(estimatePayloadSize()); + payloadBuffer.order(ByteOrder.LITTLE_ENDIAN); + + for (var entry : fields.entrySet()) { + int fieldId = entry.getKey(); + var value = entry.getValue(); + + directory.add(new com.imprint.core.DirectoryEntry((short)fieldId, value.getTypeCode(), payloadBuffer.position())); + serializeValue(value, payloadBuffer); + } + + // Create read-only view of the payload without copying + payloadBuffer.flip(); // limit = position, position = 0 + var payloadView = payloadBuffer.slice().asReadOnlyBuffer(); + + var header = new com.imprint.core.Header(new com.imprint.core.Flags((byte) 0), schemaId, payloadView.remaining()); + return new ImprintRecord(header, directory, payloadView); + } + + /** + * Builds the record and serializes it directly to a ByteBuffer without creating an intermediate ImprintRecord object. + * This is the most efficient path for "write-only" scenarios. + * + * @return A read-only ByteBuffer containing the fully serialized record. + * @throws ImprintException if serialization fails. + */ + public ByteBuffer buildToBuffer() throws ImprintException { + // 1. Prepare payload and directory list + var directory = new ArrayList(fields.size()); + var payloadBuffer = ByteBuffer.allocate(estimatePayloadSize()); + payloadBuffer.order(ByteOrder.LITTLE_ENDIAN); + for (var entry : fields.entrySet()) { - writer.addField(entry.getKey(), entry.getValue()); + int fieldId = entry.getKey(); + var value = entry.getValue(); + directory.add(new com.imprint.core.DirectoryEntry((short) fieldId, value.getTypeCode(), payloadBuffer.position())); + serializeValue(value, payloadBuffer); } - return writer.build(); + payloadBuffer.flip(); + var payloadView = payloadBuffer.slice().asReadOnlyBuffer(); + + // 2. Serialize directly to the final buffer format + return ImprintRecord.serialize(schemaId, directory, payloadView); } // Internal helper methods @@ -238,4 +286,73 @@ private MapKey convertToMapKey(Object obj) { public String toString() { return String.format("ImprintRecordBuilder{schemaId=%s, fields=%d}", schemaId, fields.size()); } + + private int estimatePayloadSize() throws ImprintException { + // More accurate estimation to reduce allocations + int estimatedSize = 0; + for (var value : fields.values()) { + estimatedSize += estimateValueSize(value); + } + // Add 25% buffer to reduce reallocations + return Math.max(estimatedSize + (estimatedSize / 4), fields.size() * 16); + } + + /** + * Estimates the serialized size in bytes for a given value. + * This method provides size estimates for payload buffer allocation, + * supporting both array-based and ByteBuffer-based value types. + * + * @param value the value to estimate size for + * @return estimated size in bytes including type-specific overhead + */ + private int estimateValueSize(Value value) throws ImprintException { + // Use TypeHandler for simple types + switch (value.getTypeCode()) { + case NULL: + case BOOL: + case INT32: + case INT64: + case FLOAT32: + case FLOAT64: + case BYTES: + case STRING: + case ARRAY: + case MAP: + return value.getTypeCode().getHandler().estimateSize(value); + + case ROW: + Value.RowValue rowValue = (Value.RowValue) value; + return rowValue.getValue().estimateSerializedSize(); + + default: + throw new ImprintException(com.imprint.error.ErrorType.SERIALIZATION_ERROR, "Unknown type code: " + value.getTypeCode()); + } + } + + + private void serializeValue(Value value, ByteBuffer buffer) throws ImprintException { + switch (value.getTypeCode()) { + case NULL: + case BOOL: + case INT32: + case INT64: + case FLOAT32: + case FLOAT64: + case BYTES: + case STRING: + case ARRAY: + case MAP: + value.getTypeCode().getHandler().serialize(value, buffer); + break; + //TODO eliminate this switch entirely by implementing a ROW TypeHandler + case ROW: + Value.RowValue rowValue = (Value.RowValue) value; + var serializedRow = rowValue.getValue().serializeToBuffer(); + buffer.put(serializedRow); + break; + + default: + throw new ImprintException(com.imprint.error.ErrorType.SERIALIZATION_ERROR, "Unknown type code: " + value.getTypeCode()); + } + } } \ No newline at end of file diff --git a/src/main/java/com/imprint/core/ImprintWriter.java b/src/main/java/com/imprint/core/ImprintWriter.java deleted file mode 100644 index b1d5f53..0000000 --- a/src/main/java/com/imprint/core/ImprintWriter.java +++ /dev/null @@ -1,126 +0,0 @@ -package com.imprint.core; - -import com.imprint.error.ErrorType; -import com.imprint.error.ImprintException; -import com.imprint.types.Value; - -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.util.ArrayList; -import java.util.Objects; -import java.util.TreeMap; - -/** - * A writer for constructing ImprintRecords by adding fields sequentially. - */ -public final class ImprintWriter { - private final SchemaId schemaId; - private final TreeMap fields; // keep fields in sorted order - - public ImprintWriter(SchemaId schemaId) { - this.schemaId = Objects.requireNonNull(schemaId, "SchemaId cannot be null"); - this.fields = new TreeMap<>(); - } - - /** - * Adds a field to the record being built. - */ - public ImprintWriter addField(int id, Value value) { - Objects.requireNonNull(value, "Value cannot be null"); - this.fields.put(id, value); - return this; - } - - /** - * Consumes the writer and builds an ImprintRecord. - */ - public ImprintRecord build() throws ImprintException { - var directory = new ArrayList(fields.size()); - var payloadBuffer = ByteBuffer.allocate(estimatePayloadSize()); - payloadBuffer.order(ByteOrder.LITTLE_ENDIAN); - - for (var entry : fields.entrySet()) { - int fieldId = entry.getKey(); - var value = entry.getValue(); - - directory.add(new DirectoryEntry(fieldId, value.getTypeCode(), payloadBuffer.position())); - serializeValue(value, payloadBuffer); - } - - // Create read-only view of the payload without copying - payloadBuffer.flip(); // limit = position, position = 0 - var payloadView = payloadBuffer.slice().asReadOnlyBuffer(); - - var header = new Header(new Flags((byte) 0), schemaId, payloadView.remaining()); - return new ImprintRecord(header, directory, payloadView); - } - - private int estimatePayloadSize() throws ImprintException { - // More accurate estimation to reduce allocations - int estimatedSize = 0; - for (var value : fields.values()) { - estimatedSize += estimateValueSize(value); - } - // Add 25% buffer to reduce reallocations - return Math.max(estimatedSize + (estimatedSize / 4), fields.size() * 16); - } - - /** - * Estimates the serialized size in bytes for a given value. - * This method provides size estimates for payload buffer allocation, - * supporting both array-based and ByteBuffer-based value types. - * - * @param value the value to estimate size for - * @return estimated size in bytes including type-specific overhead - */ - private int estimateValueSize(Value value) throws ImprintException { - // Use TypeHandler for simple types - switch (value.getTypeCode()) { - case NULL: - case BOOL: - case INT32: - case INT64: - case FLOAT32: - case FLOAT64: - case BYTES: - case STRING: - case ARRAY: - case MAP: - return value.getTypeCode().getHandler().estimateSize(value); - - case ROW: - Value.RowValue rowValue = (Value.RowValue) value; - return rowValue.getValue().estimateSerializedSize(); - - default: - throw new ImprintException(ErrorType.SERIALIZATION_ERROR, "Unknown type code: " + value.getTypeCode()); - } - } - - - private void serializeValue(Value value, ByteBuffer buffer) throws ImprintException { - switch (value.getTypeCode()) { - case NULL: - case BOOL: - case INT32: - case INT64: - case FLOAT32: - case FLOAT64: - case BYTES: - case STRING: - case ARRAY: - case MAP: - value.getTypeCode().getHandler().serialize(value, buffer); - break; - //TODO eliminate this switch entirely by implementing a ROW TypeHandler - case ROW: - Value.RowValue rowValue = (Value.RowValue) value; - var serializedRow = rowValue.getValue().serializeToBuffer(); - buffer.put(serializedRow); - break; - - default: - throw new ImprintException(ErrorType.SERIALIZATION_ERROR, "Unknown type code: " + value.getTypeCode()); - } - } -} \ No newline at end of file diff --git a/src/test/java/com/imprint/core/ImprintRecordTest.java b/src/test/java/com/imprint/core/ImprintRecordTest.java index 3e37473..6d85ccb 100644 --- a/src/test/java/com/imprint/core/ImprintRecordTest.java +++ b/src/test/java/com/imprint/core/ImprintRecordTest.java @@ -24,12 +24,10 @@ private String getStringValue(Value value) { @Test void shouldCreateSimpleRecord() throws ImprintException { var schemaId = new SchemaId(1, 0xdeadbeef); - var writer = new ImprintWriter(schemaId); - - writer.addField(1, Value.fromInt32(42)) - .addField(2, Value.fromString("hello")); - - var record = writer.build(); + var record = ImprintRecord.builder(schemaId) + .field(1, Value.fromInt32(42)) + .field(2, Value.fromString("hello")) + .build(); assertThat(record.getHeader().getSchemaId()).isEqualTo(schemaId); assertThat(record.getDirectory()).hasSize(2); @@ -53,18 +51,16 @@ var record = writer.build(); @Test void shouldRoundtripThroughSerialization() throws ImprintException { var schemaId = new SchemaId(1, 0xdeadbeef); - var writer = new ImprintWriter(schemaId); - - writer.addField(1, Value.nullValue()) - .addField(2, Value.fromBoolean(true)) - .addField(3, Value.fromInt32(42)) - .addField(4, Value.fromInt64(123456789L)) - .addField(5, Value.fromFloat32(3.14f)) - .addField(6, Value.fromFloat64(2.718281828)) - .addField(7, Value.fromBytes(new byte[]{1, 2, 3, 4})) - .addField(8, Value.fromString("test string")); - - var original = writer.build(); + var original = ImprintRecord.builder(schemaId) + .field(1, Value.nullValue()) + .field(2, Value.fromBoolean(true)) + .field(3, Value.fromInt32(42)) + .field(4, Value.fromInt64(123456789L)) + .field(5, Value.fromFloat32(3.14f)) + .field(6, Value.fromFloat64(2.718281828)) + .field(7, Value.fromBytes(new byte[]{1, 2, 3, 4})) + .field(8, Value.fromString("test string")) + .build(); // Serialize and deserialize var buffer = original.serializeToBuffer(); @@ -94,7 +90,6 @@ void shouldRoundtripThroughSerialization() throws ImprintException { @Test void shouldHandleArrays() throws ImprintException { var schemaId = new SchemaId(1, 0xdeadbeef); - var writer = new ImprintWriter(schemaId); List intArray = Arrays.asList( Value.fromInt32(1), @@ -102,8 +97,9 @@ void shouldHandleArrays() throws ImprintException { Value.fromInt32(3) ); - writer.addField(1, Value.fromArray(intArray)); - ImprintRecord record = writer.build(); + var record = ImprintRecord.builder(schemaId) + .field(1, Value.fromArray(intArray)) + .build(); // Serialize and deserialize var buffer = record.serializeToBuffer(); @@ -125,14 +121,14 @@ void shouldHandleArrays() throws ImprintException { @Test void shouldHandleMaps() throws ImprintException { var schemaId = new SchemaId(1, 0xdeadbeef); - var writer = new ImprintWriter(schemaId); var map = new HashMap(); map.put(MapKey.fromString("key1"), Value.fromInt32(1)); map.put(MapKey.fromString("key2"), Value.fromInt32(2)); - writer.addField(1, Value.fromMap(map)); - var record = writer.build(); + var record = ImprintRecord.builder(schemaId) + .field(1, Value.fromMap(map)) + .build(); // Serialize and deserialize var buffer = record.serializeToBuffer(); @@ -154,17 +150,17 @@ var record = writer.build(); void shouldHandleNestedRecords() throws ImprintException { // Create inner record var innerSchemaId = new SchemaId(2, 0xcafebabe); - var innerWriter = new ImprintWriter(innerSchemaId); - innerWriter.addField(1, Value.fromInt32(42)) - .addField(2, Value.fromString("nested")); - var innerRecord = innerWriter.build(); + var innerRecord = ImprintRecord.builder(innerSchemaId) + .field(1, Value.fromInt32(42)) + .field(2, Value.fromString("nested")) + .build(); // Create outer record containing inner record var outerSchemaId = new SchemaId(1, 0xdeadbeef); - var outerWriter = new ImprintWriter(outerSchemaId); - outerWriter.addField(1, Value.fromRow(innerRecord)) - .addField(2, Value.fromInt64(123L)); - var outerRecord = outerWriter.build(); + var outerRecord = ImprintRecord.builder(outerSchemaId) + .field(1, Value.fromRow(innerRecord)) + .field(2, Value.fromInt64(123L)) + .build(); // Serialize and deserialize var buffer = outerRecord.serializeToBuffer(); @@ -218,13 +214,12 @@ void shouldRejectUnsupportedVersion() { @Test void shouldHandleDuplicateFieldIds() throws ImprintException { var schemaId = new SchemaId(1, 0xdeadbeef); - var writer = new ImprintWriter(schemaId); // Add duplicate field IDs - last one should win - writer.addField(1, Value.fromInt32(42)) - .addField(1, Value.fromInt32(43)); - - var record = writer.build(); + var record = ImprintRecord.builder(schemaId) + .field(1, Value.fromInt32(42)) + .field(1, Value.fromInt32(43)) + .build(); assertThat(record.getDirectory()).hasSize(1); assertThat(record.getValue(1)).isEqualTo(Value.fromInt32(43)); diff --git a/src/test/java/com/imprint/profile/ProfilerTest.java b/src/test/java/com/imprint/profile/ProfilerTest.java index 64be931..1ea752d 100644 --- a/src/test/java/com/imprint/profile/ProfilerTest.java +++ b/src/test/java/com/imprint/profile/ProfilerTest.java @@ -1,7 +1,6 @@ package com.imprint.profile; import com.imprint.core.ImprintRecord; -import com.imprint.core.ImprintWriter; import com.imprint.core.SchemaId; import com.imprint.types.Value; import org.junit.jupiter.api.Disabled; @@ -88,15 +87,15 @@ void profileSerialization() throws Exception { // Create and serialize many records (allocation hotspot) for (int i = 0; i < 500_000; i++) { - var writer = new ImprintWriter(schemaId); + var builder = ImprintRecord.builder(schemaId); // Add various field types - writer.addField(1, Value.fromInt32(i)) - .addField(2, Value.fromString("test-string-" + i)) - .addField(3, Value.fromFloat64(i * 3.14159)) - .addField(4, Value.fromBytes(("bytes-" + i).getBytes())); + builder.field(1, Value.fromInt32(i)) + .field(2, Value.fromString("test-string-" + i)) + .field(3, Value.fromFloat64(i * 3.14159)) + .field(4, Value.fromBytes(("bytes-" + i).getBytes())); - var record = writer.build(); + var record = builder.build(); var serialized = record.serializeToBuffer(); // Potential hotspot // Trigger some deserialization @@ -151,15 +150,15 @@ void profileMemoryAllocation() throws Exception { for (int batch = 0; batch < 1000; batch++) { for (int i = 0; i < 1000; i++) { var schemaId = new SchemaId(batch, i); - var writer = new ImprintWriter(schemaId); + var builder = ImprintRecord.builder(schemaId); // Create strings of varying sizes (allocation pressure) - writer.addField(1, Value.fromString("small")) - .addField(2, Value.fromString("medium-length-string-" + i)) - .addField(3, Value.fromString("very-long-string-that-will-cause-more-allocation-pressure-" + batch + "-" + i)) - .addField(4, Value.fromBytes(new byte[100 + i % 1000])); // Varying byte arrays + builder.field(1, Value.fromString("small")) + .field(2, Value.fromString("medium-length-string-" + i)) + .field(3, Value.fromString("very-long-string-that-will-cause-more-allocation-pressure-" + batch + "-" + i)) + .field(4, Value.fromBytes(new byte[100 + i % 1000])); // Varying byte arrays - var record = writer.build(); + var record = builder.build(); // Some deserialization to trigger string decoding allocations record.getValue(2); @@ -175,54 +174,52 @@ var record = writer.build(); } private ImprintRecord createTestRecord() throws Exception { - var schemaId = new SchemaId(1, 0xdeadbeef); - var writer = new ImprintWriter(schemaId); + var builder = ImprintRecord.builder(new SchemaId(1, 0xdeadbeef)); for (int i = 1; i <= RECORD_SIZE; i++) { switch (i % 4) { case 0: - writer.addField(i, Value.fromInt32(i * 100)); + builder.field(i, Value.fromInt32(i * 100)); break; case 1: - writer.addField(i, Value.fromString("field-value-" + i)); + builder.field(i, Value.fromString("field-value-" + i)); break; case 2: - writer.addField(i, Value.fromFloat64(i * 3.14159)); + builder.field(i, Value.fromFloat64(i * 3.14159)); break; case 3: - writer.addField(i, Value.fromBytes(("bytes-" + i).getBytes())); + builder.field(i, Value.fromBytes(("bytes-" + i).getBytes())); break; } } - return writer.build(); + return builder.build(); } private ImprintRecord createLargeRecord() throws Exception { - var schemaId = new SchemaId(2, 0xcafebabe); - var writer = new ImprintWriter(schemaId); + var builder = ImprintRecord.builder(new SchemaId(2, 0xcafebabe)); // Create 100 fields with realistic data for (int i = 1; i <= 100; i++) { switch (i % 5) { case 0: - writer.addField(i, Value.fromInt32(i)); + builder.field(i, Value.fromInt32(i)); break; case 1: - writer.addField(i, Value.fromString("user-name-" + i + "@example.com")); + builder.field(i, Value.fromString("user-name-" + i + "@example.com")); break; case 2: - writer.addField(i, Value.fromString("Some longer descriptive text for field " + i + " that might represent a comment or description")); + builder.field(i, Value.fromString("Some longer descriptive text for field " + i + " that might represent a comment or description")); break; case 3: - writer.addField(i, Value.fromFloat64(i * 2.718281828)); + builder.field(i, Value.fromFloat64(i * 2.718281828)); break; case 4: - writer.addField(i, Value.fromBytes(String.format("binary-data-%04d", i).getBytes())); + builder.field(i, Value.fromBytes(String.format("binary-data-%04d", i).getBytes())); break; } } - return writer.build(); + return builder.build(); } } \ No newline at end of file From 574323eba34512ac24f1a4f93d923d0c28758997 Mon Sep 17 00:00:00 2001 From: expand3d <> Date: Sat, 7 Jun 2025 14:37:28 -0400 Subject: [PATCH 37/49] Allow for record creation path from builder to bypass extra TreeMapping --- .../java/com/imprint/core/ImprintBuffers.java | 38 +++++++++++- .../java/com/imprint/core/ImprintRecord.java | 61 +++++++++++-------- .../imprint/core/ImprintRecordBuilder.java | 16 ++--- 3 files changed, 79 insertions(+), 36 deletions(-) diff --git a/src/main/java/com/imprint/core/ImprintBuffers.java b/src/main/java/com/imprint/core/ImprintBuffers.java index ac3b4d3..6a294c3 100644 --- a/src/main/java/com/imprint/core/ImprintBuffers.java +++ b/src/main/java/com/imprint/core/ImprintBuffers.java @@ -50,7 +50,18 @@ public ImprintBuffers(List directory, ByteBuffer payload) { this.parsedDirectory = createDirectoryMap(Objects.requireNonNull(directory)); this.directoryParsed = true; this.payload = payload.asReadOnlyBuffer(); - this.directoryBuffer = createDirectoryBuffer(directory); + this.directoryBuffer = ImprintBuffers.createDirectoryBuffer(directory); + } + + /** + * Creates buffers from a pre-parsed and sorted directory map (used by ImprintRecordBuilder). + * This is an optimized path that avoids creating an intermediate List-to-Map conversion. + */ + public ImprintBuffers(TreeMap directoryMap, ByteBuffer payload) { + this.parsedDirectory = Objects.requireNonNull(directoryMap); + this.directoryParsed = true; + this.payload = payload.asReadOnlyBuffer(); + this.directoryBuffer = ImprintBuffers.createDirectoryBufferFromMap(directoryMap); } /** @@ -263,7 +274,7 @@ private TreeMap createDirectoryMap(List /** * Create directory buffer from parsed entries. */ - private ByteBuffer createDirectoryBuffer(List directory) { + static ByteBuffer createDirectoryBuffer(List directory) { try { int bufferSize = VarInt.encodedLength(directory.size()) + (directory.size() * Constants.DIR_ENTRY_BYTES); var buffer = ByteBuffer.allocate(bufferSize); @@ -280,11 +291,32 @@ private ByteBuffer createDirectoryBuffer(List directory) { } } + /** + * Create directory buffer from a pre-sorted map of entries. + */ + static ByteBuffer createDirectoryBufferFromMap(TreeMap directoryMap) { + try { + int bufferSize = VarInt.encodedLength(directoryMap.size()) + (directoryMap.size() * Constants.DIR_ENTRY_BYTES); + var buffer = ByteBuffer.allocate(bufferSize); + buffer.order(ByteOrder.LITTLE_ENDIAN); + + VarInt.encode(directoryMap.size(), buffer); + // TreeMap.values() returns a collection view, iteration is ordered and efficient. + for (var entry : directoryMap.values()) + serializeDirectoryEntry(entry, buffer); + + buffer.flip(); + return buffer.asReadOnlyBuffer(); + } catch (Exception e) { + return ByteBuffer.allocate(0).asReadOnlyBuffer(); + } + } + /** * Serialize a single directory entry to the buffer. * Format: [fieldId:2bytes][typeCode:1byte][offset:4bytes] */ - private void serializeDirectoryEntry(DirectoryEntry entry, ByteBuffer buffer) { + private static void serializeDirectoryEntry(DirectoryEntry entry, ByteBuffer buffer) { buffer.putShort(entry.getId()); buffer.put(entry.getTypeCode().getCode()); buffer.putInt(entry.getOffset()); diff --git a/src/main/java/com/imprint/core/ImprintRecord.java b/src/main/java/com/imprint/core/ImprintRecord.java index 83ddb03..804642b 100644 --- a/src/main/java/com/imprint/core/ImprintRecord.java +++ b/src/main/java/com/imprint/core/ImprintRecord.java @@ -14,6 +14,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.TreeMap; /** * An Imprint record containing a header and buffer management. @@ -40,6 +41,14 @@ private ImprintRecord(Header header, ImprintBuffers buffers) { this.buffers = new ImprintBuffers(directory, payload); } + /** + * Creates a record from a pre-parsed and sorted directory map (used by ImprintRecordBuilder). + */ + ImprintRecord(Header header, TreeMap directoryMap, ByteBuffer payload) { + this.header = Objects.requireNonNull(header, "Header cannot be null"); + this.buffers = new ImprintBuffers(directoryMap, payload); + } + // ========== FIELD ACCESS METHODS ========== /** @@ -190,7 +199,33 @@ public int estimateSerializedSize() { */ public static ByteBuffer serialize(SchemaId schemaId, List directory, ByteBuffer payload) { var header = new Header(new Flags((byte) 0), schemaId, payload.remaining()); - var directoryBuffer = createDirectoryBuffer(directory); + var directoryBuffer = ImprintBuffers.createDirectoryBuffer(directory); + + int finalSize = Constants.HEADER_BYTES + directoryBuffer.remaining() + payload.remaining(); + var finalBuffer = ByteBuffer.allocate(finalSize); + finalBuffer.order(ByteOrder.LITTLE_ENDIAN); + + // Assemble the final record + serializeHeader(header, finalBuffer); + finalBuffer.put(directoryBuffer); + finalBuffer.put(payload); + + finalBuffer.flip(); + return finalBuffer.asReadOnlyBuffer(); + } + + /** + * Serializes the components of a record into a single ByteBuffer using a pre-built directory map. + * This provides a direct serialization path without needing a live ImprintRecord instance. + * + * @param schemaId The schema identifier for the record. + * @param directoryMap The map of directory entries, which must be sorted by field ID (e.g., a TreeMap). + * @param payload The ByteBuffer containing all field data concatenated. + * @return A read-only ByteBuffer with the complete serialized record. + */ + public static ByteBuffer serialize(SchemaId schemaId, TreeMap directoryMap, ByteBuffer payload) { + var header = new Header(new Flags((byte) 0), schemaId, payload.remaining()); + var directoryBuffer = ImprintBuffers.createDirectoryBufferFromMap(directoryMap); int finalSize = Constants.HEADER_BYTES + directoryBuffer.remaining() + payload.remaining(); var finalBuffer = ByteBuffer.allocate(finalSize); @@ -328,30 +363,6 @@ private static Header deserializeHeader(ByteBuffer buffer) throws ImprintExcepti return new Header(flags, new SchemaId(fieldSpaceId, schemaHash), payloadSize); } - /** - * Creates a serialized representation of the directory. - */ - private static ByteBuffer createDirectoryBuffer(List directory) { - try { - int bufferSize = VarInt.encodedLength(directory.size()) + (directory.size() * Constants.DIR_ENTRY_BYTES); - var buffer = ByteBuffer.allocate(bufferSize); - buffer.order(ByteOrder.LITTLE_ENDIAN); - - VarInt.encode(directory.size(), buffer); - for (var entry : directory) { - buffer.putShort(entry.getId()); - buffer.put(entry.getTypeCode().getCode()); - buffer.putInt(entry.getOffset()); - } - - buffer.flip(); - return buffer.asReadOnlyBuffer(); - } catch (Exception e) { - // Should not happen with valid inputs - return ByteBuffer.allocate(0).asReadOnlyBuffer(); - } - } - @Override public String toString() { return String.format("ImprintRecord{header=%s, directorySize=%d, payloadSize=%d}", diff --git a/src/main/java/com/imprint/core/ImprintRecordBuilder.java b/src/main/java/com/imprint/core/ImprintRecordBuilder.java index 39238a7..4a95898 100644 --- a/src/main/java/com/imprint/core/ImprintRecordBuilder.java +++ b/src/main/java/com/imprint/core/ImprintRecordBuilder.java @@ -143,7 +143,7 @@ public Set fieldIds() { // Build the final record public ImprintRecord build() throws ImprintException { - var directory = new ArrayList(fields.size()); + var directoryMap = new TreeMap(); var payloadBuffer = ByteBuffer.allocate(estimatePayloadSize()); payloadBuffer.order(ByteOrder.LITTLE_ENDIAN); @@ -151,7 +151,7 @@ public ImprintRecord build() throws ImprintException { int fieldId = entry.getKey(); var value = entry.getValue(); - directory.add(new com.imprint.core.DirectoryEntry((short)fieldId, value.getTypeCode(), payloadBuffer.position())); + directoryMap.put(fieldId, new com.imprint.core.DirectoryEntry((short)fieldId, value.getTypeCode(), payloadBuffer.position())); serializeValue(value, payloadBuffer); } @@ -160,7 +160,7 @@ public ImprintRecord build() throws ImprintException { var payloadView = payloadBuffer.slice().asReadOnlyBuffer(); var header = new com.imprint.core.Header(new com.imprint.core.Flags((byte) 0), schemaId, payloadView.remaining()); - return new ImprintRecord(header, directory, payloadView); + return new ImprintRecord(header, directoryMap, payloadView); } /** @@ -171,22 +171,22 @@ public ImprintRecord build() throws ImprintException { * @throws ImprintException if serialization fails. */ public ByteBuffer buildToBuffer() throws ImprintException { - // 1. Prepare payload and directory list - var directory = new ArrayList(fields.size()); + // 1. Prepare payload and directory map + var directoryMap = new TreeMap(); var payloadBuffer = ByteBuffer.allocate(estimatePayloadSize()); payloadBuffer.order(ByteOrder.LITTLE_ENDIAN); for (var entry : fields.entrySet()) { int fieldId = entry.getKey(); var value = entry.getValue(); - directory.add(new com.imprint.core.DirectoryEntry((short) fieldId, value.getTypeCode(), payloadBuffer.position())); + directoryMap.put(fieldId, new com.imprint.core.DirectoryEntry((short) fieldId, value.getTypeCode(), payloadBuffer.position())); serializeValue(value, payloadBuffer); } payloadBuffer.flip(); var payloadView = payloadBuffer.slice().asReadOnlyBuffer(); - // 2. Serialize directly to the final buffer format - return ImprintRecord.serialize(schemaId, directory, payloadView); + // 2. Serialize directly to the final buffer format using the map-based method + return ImprintRecord.serialize(schemaId, directoryMap, payloadView); } // Internal helper methods From b2bebee8b8608a09b436beed75e48278de0ab206 Mon Sep 17 00:00:00 2001 From: expand3d <> Date: Sat, 7 Jun 2025 14:53:46 -0400 Subject: [PATCH 38/49] Calculate estimated size as fields are added instead of deferring it --- .../imprint/core/ImprintRecordBuilder.java | 23 +++++++++++-------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/imprint/core/ImprintRecordBuilder.java b/src/main/java/com/imprint/core/ImprintRecordBuilder.java index 4a95898..93abc58 100644 --- a/src/main/java/com/imprint/core/ImprintRecordBuilder.java +++ b/src/main/java/com/imprint/core/ImprintRecordBuilder.java @@ -4,6 +4,7 @@ import com.imprint.error.ImprintException; import com.imprint.types.MapKey; import com.imprint.types.Value; +import lombok.SneakyThrows; import java.nio.ByteBuffer; import java.nio.ByteOrder; @@ -40,6 +41,7 @@ public final class ImprintRecordBuilder { private final SchemaId schemaId; private final Map fields = new TreeMap<>(); + private int estimatedPayloadSize = 0; ImprintRecordBuilder(SchemaId schemaId) { this.schemaId = Objects.requireNonNull(schemaId, "SchemaId cannot be null"); @@ -200,7 +202,14 @@ public ByteBuffer buildToBuffer() throws ImprintException { */ private ImprintRecordBuilder addField(int id, Value value) { Objects.requireNonNull(value, "Value cannot be null - use nullField() for explicit null values"); + + // Subtract the size of the old value if it's being replaced. + var oldValue = fields.get(id); + if (oldValue != null) + estimatedPayloadSize -= estimateValueSize(oldValue); + fields.put(id, value); + estimatedPayloadSize += estimateValueSize(value); return this; } @@ -287,14 +296,9 @@ public String toString() { return String.format("ImprintRecordBuilder{schemaId=%s, fields=%d}", schemaId, fields.size()); } - private int estimatePayloadSize() throws ImprintException { - // More accurate estimation to reduce allocations - int estimatedSize = 0; - for (var value : fields.values()) { - estimatedSize += estimateValueSize(value); - } - // Add 25% buffer to reduce reallocations - return Math.max(estimatedSize + (estimatedSize / 4), fields.size() * 16); + private int estimatePayloadSize() { + // Add 25% buffer to reduce reallocations and handle VarInt encoding fluctuations. + return Math.max(estimatedPayloadSize + (estimatedPayloadSize / 4), fields.size() * 16); } /** @@ -305,7 +309,8 @@ private int estimatePayloadSize() throws ImprintException { * @param value the value to estimate size for * @return estimated size in bytes including type-specific overhead */ - private int estimateValueSize(Value value) throws ImprintException { + @SneakyThrows + private int estimateValueSize(Value value) { // Use TypeHandler for simple types switch (value.getTypeCode()) { case NULL: From f1df8d761a2046ae6ecedc587dfaed118e7e57f4 Mon Sep 17 00:00:00 2001 From: expand3d <> Date: Sun, 8 Jun 2025 19:12:53 -0400 Subject: [PATCH 39/49] Use idiomatic Directory interface and optimize builder --- .../benchmark/SerializationBenchmark.java | 111 ++++---- .../java/com/imprint/core/DirectoryEntry.java | 33 +-- .../java/com/imprint/core/ImprintBuffers.java | 37 +-- .../com/imprint/core/ImprintOperations.java | 4 +- .../java/com/imprint/core/ImprintRecord.java | 9 +- .../imprint/core/ImprintRecordBuilder.java | 97 +++++-- .../java/com/imprint/core/ImprintStream.java | 257 ++++++++++++++++++ .../imprint/core/SimpleDirectoryEntry.java | 22 ++ 8 files changed, 449 insertions(+), 121 deletions(-) create mode 100644 src/main/java/com/imprint/core/ImprintStream.java create mode 100644 src/main/java/com/imprint/core/SimpleDirectoryEntry.java diff --git a/src/jmh/java/com/imprint/benchmark/SerializationBenchmark.java b/src/jmh/java/com/imprint/benchmark/SerializationBenchmark.java index 11e2b29..51c9f48 100644 --- a/src/jmh/java/com/imprint/benchmark/SerializationBenchmark.java +++ b/src/jmh/java/com/imprint/benchmark/SerializationBenchmark.java @@ -1,11 +1,16 @@ package com.imprint.benchmark; import com.imprint.core.ImprintRecord; +import com.imprint.core.ImprintRecordBuilder; import com.imprint.core.SchemaId; import com.imprint.types.MapKey; import com.imprint.types.Value; import org.openjdk.jmh.annotations.*; import org.openjdk.jmh.infra.Blackhole; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -20,7 +25,7 @@ @OutputTimeUnit(TimeUnit.NANOSECONDS) @State(Scope.Benchmark) @Warmup(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS) -@Measurement(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 7, time = 1, timeUnit = TimeUnit.SECONDS) @Fork(1) public class SerializationBenchmark { @@ -34,10 +39,10 @@ public class SerializationBenchmark { @Setup public void setup() throws Exception { - // Create test records of varying sizes - smallRecord = createSmallRecord(); - mediumRecord = createMediumRecord(); - largeRecord = createLargeRecord(); + // Create test records of varying sizes for deserialization benchmarks + smallRecord = createSmallRecord().build(); + mediumRecord = createMediumRecord().build(); + largeRecord = createLargeRecord().build(); // Pre-serialize for deserialization benchmarks smallRecordBytes = smallRecord.serializeToBuffer(); @@ -48,20 +53,20 @@ public void setup() throws Exception { // ===== SERIALIZATION BENCHMARKS ===== @Benchmark - public void serializeSmallRecord(Blackhole bh) { - ByteBuffer result = smallRecord.serializeToBuffer(); + public void buildAndSerializeSmallRecord(Blackhole bh) throws Exception { + ByteBuffer result = createSmallRecord().buildToBuffer(); bh.consume(result); } @Benchmark - public void serializeMediumRecord(Blackhole bh) { - ByteBuffer result = mediumRecord.serializeToBuffer(); + public void buildAndSerializeMediumRecord(Blackhole bh) throws Exception { + ByteBuffer result = createMediumRecord().buildToBuffer(); bh.consume(result); } @Benchmark - public void serializeLargeRecord(Blackhole bh) { - ByteBuffer result = largeRecord.serializeToBuffer(); + public void buildAndSerializeLargeRecord(Blackhole bh) throws Exception { + ByteBuffer result = createLargeRecord().buildToBuffer(); bh.consume(result); } @@ -87,77 +92,89 @@ public void deserializeLargeRecord(Blackhole bh) throws Exception { // ===== HELPER METHODS ===== - private ImprintRecord createSmallRecord() throws Exception { + private ImprintRecordBuilder createSmallRecord() throws Exception { // Small record: ~10 fields, simple types return ImprintRecord.builder(new SchemaId(1, 0x12345678)) - .field(1, Value.fromString("Product")) - .field(2, Value.fromInt32(12345)) - .field(3, Value.fromFloat64(99.99)) - .field(4, Value.fromBoolean(true)) - .field(5, Value.fromString("Electronics")) - .build(); + .field(1, "Product") + .field(2, 12345) + .field(3, 99.99) + .field(4, true) + .field(5, "Electronics"); } - private ImprintRecord createMediumRecord() throws Exception { + private ImprintRecordBuilder createMediumRecord() throws Exception { var builder = ImprintRecord.builder(new SchemaId(1, 0x12345678)); // Medium record: ~50 fields, mixed types including arrays - builder.field(1, Value.fromString("Product")); - builder.field(2, Value.fromInt32(12345)); - builder.field(3, Value.fromFloat64(99.99)); - builder.field(4, Value.fromBoolean(true)); - builder.field(5, Value.fromString("Electronics")); + builder.field(1, "Product"); + builder.field(2, 12345); + builder.field(3, 99.99); + builder.field(4, true); + builder.field(5, "Electronics"); // Add array field var tags = Arrays.asList( - Value.fromString("popular"), - Value.fromString("trending"), - Value.fromString("bestseller") + "popular", + "trending", + "bestseller" ); - builder.field(6, Value.fromArray(tags)); + builder.field(6, tags); // Add map field (all string values for consistency) - var metadata = new HashMap(); - metadata.put(MapKey.fromString("manufacturer"), Value.fromString("TechCorp")); - metadata.put(MapKey.fromString("model"), Value.fromString("TC-2024")); - metadata.put(MapKey.fromString("year"), Value.fromString("2024")); - builder.field(7, Value.fromMap(metadata)); + var metadata = new HashMap(); + metadata.put("manufacturer", "TechCorp"); + metadata.put("model", "TC-2024"); + metadata.put("year", "2024"); + builder.field(7, metadata); // Add more fields for medium size for (int i = 8; i <= 50; i++) { - builder.field(i, Value.fromString("field_" + i + "_value")); + builder.field(i, "field_" + i + "_value"); } - return builder.build(); + return builder; } - private ImprintRecord createLargeRecord() throws Exception { + private ImprintRecordBuilder createLargeRecord() throws Exception { var builder = ImprintRecord.builder(new SchemaId(1, 0x12345678)); // Large record: ~200 fields, complex nested structures - builder.field(1, Value.fromString("LargeProduct")); - builder.field(2, Value.fromInt32(12345)); - builder.field(3, Value.fromFloat64(99.99)); + builder.field(1, "LargeProduct"); + builder.field(2, 12345); + builder.field(3, 99.99); // Large array - var largeArray = new ArrayList(); + var largeArray = new ArrayList(); for (int i = 0; i < 100; i++) { - largeArray.add(Value.fromString("item_" + i)); + largeArray.add("item_" + i); } - builder.field(4, Value.fromArray(largeArray)); + builder.field(4, largeArray); // Large map - var largeMap = new HashMap(); + var largeMap = new HashMap(); for (int i = 0; i < 50; i++) { - largeMap.put(MapKey.fromString("key_" + i), Value.fromString("value_" + i)); + largeMap.put("key_" + i, "value_" + i); } - builder.field(5, Value.fromMap(largeMap)); + builder.field(5, largeMap); // Many string fields for (int i = 6; i <= 200; i++) { - builder.field(i, Value.fromString("this_is_a_longer_field_value_for_field_" + i + "_to_increase_record_size")); + builder.field(i, "this_is_a_longer_field_value_for_field_" + i + "_to_increase_record_size"); } - return builder.build(); + return builder; + } + + public static void main(String[] args) throws RunnerException { + Options opt = new OptionsBuilder() + .include(SerializationBenchmark.class.getSimpleName()) + .forks(1) + .warmupIterations(5) + .measurementIterations(5) + .mode(Mode.AverageTime) + .timeUnit(TimeUnit.NANOSECONDS) + .build(); + + new Runner(opt).run(); } } \ No newline at end of file diff --git a/src/main/java/com/imprint/core/DirectoryEntry.java b/src/main/java/com/imprint/core/DirectoryEntry.java index 9556256..0b98433 100644 --- a/src/main/java/com/imprint/core/DirectoryEntry.java +++ b/src/main/java/com/imprint/core/DirectoryEntry.java @@ -1,23 +1,24 @@ package com.imprint.core; import com.imprint.types.TypeCode; -import lombok.Value; - -import java.util.Objects; /** - * A directory entry describing a single field in an Imprint record. - * Each entry has a fixed size of 7 bytes. + * Represents the common interface for a directory entry in an Imprint record. + * A directory entry provides metadata about a field, such as its ID, type, and location in the payload. */ -@Value -public class DirectoryEntry { - short id; - TypeCode typeCode; - int offset; - - public DirectoryEntry(int id, TypeCode typeCode, int offset) { - this.id = (short) id; - this.typeCode = Objects.requireNonNull(typeCode, "TypeCode cannot be null"); - this.offset = offset; - } +public interface DirectoryEntry { + /** + * @return The field's unique identifier. + */ + short getId(); + + /** + * @return The {@link TypeCode} of the field's value. + */ + TypeCode getTypeCode(); + + /** + * @return The starting position (offset) of the field's data within the payload buffer. + */ + int getOffset(); } \ No newline at end of file diff --git a/src/main/java/com/imprint/core/ImprintBuffers.java b/src/main/java/com/imprint/core/ImprintBuffers.java index 6a294c3..24ec41d 100644 --- a/src/main/java/com/imprint/core/ImprintBuffers.java +++ b/src/main/java/com/imprint/core/ImprintBuffers.java @@ -10,6 +10,7 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.util.ArrayList; +import java.util.Collection; import java.util.List; import java.util.Objects; import java.util.TreeMap; @@ -44,24 +45,24 @@ public ImprintBuffers(ByteBuffer directoryBuffer, ByteBuffer payload) { } /** - * Creates buffers from pre-parsed directory (used during construction). + * Creates buffers from a pre-parsed directory (used during construction). + * This constructor is used by the ImprintRecordBuilder path. It creates + * a serialized directory buffer but defers parsing it into a map until it's actually needed. */ - public ImprintBuffers(List directory, ByteBuffer payload) { - this.parsedDirectory = createDirectoryMap(Objects.requireNonNull(directory)); - this.directoryParsed = true; + public ImprintBuffers(Collection directory, ByteBuffer payload) { + this.directoryBuffer = ImprintBuffers.createDirectoryBuffer(Objects.requireNonNull(directory)); this.payload = payload.asReadOnlyBuffer(); - this.directoryBuffer = ImprintBuffers.createDirectoryBuffer(directory); } /** * Creates buffers from a pre-parsed and sorted directory map (used by ImprintRecordBuilder). * This is an optimized path that avoids creating an intermediate List-to-Map conversion. + * This constructor is used by the ImprintRecordBuilder path. It creates + * a serialized directory buffer but defers parsing it into a map until it's actually needed. */ - public ImprintBuffers(TreeMap directoryMap, ByteBuffer payload) { - this.parsedDirectory = Objects.requireNonNull(directoryMap); - this.directoryParsed = true; + public ImprintBuffers(TreeMap directoryMap, ByteBuffer payload) { + this.directoryBuffer = ImprintBuffers.createDirectoryBufferFromMap(Objects.requireNonNull(directoryMap)); this.payload = payload.asReadOnlyBuffer(); - this.directoryBuffer = ImprintBuffers.createDirectoryBufferFromMap(directoryMap); } /** @@ -260,21 +261,10 @@ private void ensureDirectoryParsed() { } } - /** - * Create a TreeMap from directory list field lookup with ordering. - */ - private TreeMap createDirectoryMap(List directory) { - var map = new TreeMap(); - for (var entry : directory) { - map.put((int)entry.getId(), entry); - } - return map; - } - /** * Create directory buffer from parsed entries. */ - static ByteBuffer createDirectoryBuffer(List directory) { + static ByteBuffer createDirectoryBuffer(Collection directory) { try { int bufferSize = VarInt.encodedLength(directory.size()) + (directory.size() * Constants.DIR_ENTRY_BYTES); var buffer = ByteBuffer.allocate(bufferSize); @@ -294,14 +284,13 @@ static ByteBuffer createDirectoryBuffer(List directory) { /** * Create directory buffer from a pre-sorted map of entries. */ - static ByteBuffer createDirectoryBufferFromMap(TreeMap directoryMap) { + static ByteBuffer createDirectoryBufferFromMap(TreeMap directoryMap) { try { int bufferSize = VarInt.encodedLength(directoryMap.size()) + (directoryMap.size() * Constants.DIR_ENTRY_BYTES); var buffer = ByteBuffer.allocate(bufferSize); buffer.order(ByteOrder.LITTLE_ENDIAN); VarInt.encode(directoryMap.size(), buffer); - // TreeMap.values() returns a collection view, iteration is ordered and efficient. for (var entry : directoryMap.values()) serializeDirectoryEntry(entry, buffer); @@ -334,6 +323,6 @@ private DirectoryEntry deserializeDirectoryEntry(ByteBuffer buffer) throws Impri var typeCode = TypeCode.fromByte(buffer.get()); int offset = buffer.getInt(); - return new DirectoryEntry(id, typeCode, offset); + return new SimpleDirectoryEntry(id, typeCode, offset); } } \ No newline at end of file diff --git a/src/main/java/com/imprint/core/ImprintOperations.java b/src/main/java/com/imprint/core/ImprintOperations.java index 4e60ebf..c4e8c66 100644 --- a/src/main/java/com/imprint/core/ImprintOperations.java +++ b/src/main/java/com/imprint/core/ImprintOperations.java @@ -53,7 +53,7 @@ public static ImprintRecord project(ImprintRecord record, int... fieldIds) { record.getBuffers().getPayload().limit(); int fieldLength = nextOffset - field.getOffset(); - newDirectory.add(new DirectoryEntry(field.getId(), field.getTypeCode(), currentOffset)); + newDirectory.add(new SimpleDirectoryEntry(field.getId(), field.getTypeCode(), currentOffset)); ranges.add(new FieldRange(field.getOffset(), nextOffset)); currentOffset += fieldLength; @@ -133,7 +133,7 @@ public static ImprintRecord merge(ImprintRecord first, ImprintRecord second) thr throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, "Failed to get raw bytes for field " + currentEntry.getId()); // Add adjusted directory entry - var newEntry = new DirectoryEntry(currentEntry.getId(), currentEntry.getTypeCode(), currentOffset); + var newEntry = new SimpleDirectoryEntry(currentEntry.getId(), currentEntry.getTypeCode(), currentOffset); newDirectory.add(newEntry); // Collect payload chunk diff --git a/src/main/java/com/imprint/core/ImprintRecord.java b/src/main/java/com/imprint/core/ImprintRecord.java index 804642b..385e569 100644 --- a/src/main/java/com/imprint/core/ImprintRecord.java +++ b/src/main/java/com/imprint/core/ImprintRecord.java @@ -11,6 +11,7 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; +import java.util.Collection; import java.util.List; import java.util.Map; import java.util.Objects; @@ -36,7 +37,7 @@ private ImprintRecord(Header header, ImprintBuffers buffers) { /** * Creates a record from pre-parsed directory (used by ImprintWriter). */ - ImprintRecord(Header header, List directory, ByteBuffer payload) { + ImprintRecord(Header header, Collection directory, ByteBuffer payload) { this.header = Objects.requireNonNull(header, "Header cannot be null"); this.buffers = new ImprintBuffers(directory, payload); } @@ -44,7 +45,7 @@ private ImprintRecord(Header header, ImprintBuffers buffers) { /** * Creates a record from a pre-parsed and sorted directory map (used by ImprintRecordBuilder). */ - ImprintRecord(Header header, TreeMap directoryMap, ByteBuffer payload) { + ImprintRecord(Header header, TreeMap directoryMap, ByteBuffer payload) { this.header = Objects.requireNonNull(header, "Header cannot be null"); this.buffers = new ImprintBuffers(directoryMap, payload); } @@ -197,7 +198,7 @@ public int estimateSerializedSize() { * @param payload The ByteBuffer containing all field data concatenated. * @return A read-only ByteBuffer with the complete serialized record. */ - public static ByteBuffer serialize(SchemaId schemaId, List directory, ByteBuffer payload) { + public static ByteBuffer serialize(SchemaId schemaId, Collection directory, ByteBuffer payload) { var header = new Header(new Flags((byte) 0), schemaId, payload.remaining()); var directoryBuffer = ImprintBuffers.createDirectoryBuffer(directory); @@ -223,7 +224,7 @@ public static ByteBuffer serialize(SchemaId schemaId, List direc * @param payload The ByteBuffer containing all field data concatenated. * @return A read-only ByteBuffer with the complete serialized record. */ - public static ByteBuffer serialize(SchemaId schemaId, TreeMap directoryMap, ByteBuffer payload) { + public static ByteBuffer serialize(SchemaId schemaId, TreeMap directoryMap, ByteBuffer payload) { var header = new Header(new Flags((byte) 0), schemaId, payload.remaining()); var directoryBuffer = ImprintBuffers.createDirectoryBufferFromMap(directoryMap); diff --git a/src/main/java/com/imprint/core/ImprintRecordBuilder.java b/src/main/java/com/imprint/core/ImprintRecordBuilder.java index 93abc58..58fbc63 100644 --- a/src/main/java/com/imprint/core/ImprintRecordBuilder.java +++ b/src/main/java/com/imprint/core/ImprintRecordBuilder.java @@ -3,6 +3,7 @@ import com.imprint.error.ErrorType; import com.imprint.error.ImprintException; import com.imprint.types.MapKey; +import com.imprint.types.TypeCode; import com.imprint.types.Value; import lombok.SneakyThrows; @@ -40,7 +41,7 @@ @SuppressWarnings("unused") public final class ImprintRecordBuilder { private final SchemaId schemaId; - private final Map fields = new TreeMap<>(); + private final Map fields = new TreeMap<>(); private int estimatedPayloadSize = 0; ImprintRecordBuilder(SchemaId schemaId) { @@ -145,16 +146,12 @@ public Set fieldIds() { // Build the final record public ImprintRecord build() throws ImprintException { - var directoryMap = new TreeMap(); var payloadBuffer = ByteBuffer.allocate(estimatePayloadSize()); payloadBuffer.order(ByteOrder.LITTLE_ENDIAN); - for (var entry : fields.entrySet()) { - int fieldId = entry.getKey(); - var value = entry.getValue(); - - directoryMap.put(fieldId, new com.imprint.core.DirectoryEntry((short)fieldId, value.getTypeCode(), payloadBuffer.position())); - serializeValue(value, payloadBuffer); + for (var entry : fields.values()) { + entry.setOffset(payloadBuffer.position()); + serializeValue(entry.getValue(), payloadBuffer); } // Create read-only view of the payload without copying @@ -162,7 +159,7 @@ public ImprintRecord build() throws ImprintException { var payloadView = payloadBuffer.slice().asReadOnlyBuffer(); var header = new com.imprint.core.Header(new com.imprint.core.Flags((byte) 0), schemaId, payloadView.remaining()); - return new ImprintRecord(header, directoryMap, payloadView); + return new ImprintRecord(header, new ArrayList<>(fields.values()), payloadView); } /** @@ -173,22 +170,19 @@ public ImprintRecord build() throws ImprintException { * @throws ImprintException if serialization fails. */ public ByteBuffer buildToBuffer() throws ImprintException { - // 1. Prepare payload and directory map - var directoryMap = new TreeMap(); + // 1. Prepare payload and directory var payloadBuffer = ByteBuffer.allocate(estimatePayloadSize()); payloadBuffer.order(ByteOrder.LITTLE_ENDIAN); - for (var entry : fields.entrySet()) { - int fieldId = entry.getKey(); - var value = entry.getValue(); - directoryMap.put(fieldId, new com.imprint.core.DirectoryEntry((short) fieldId, value.getTypeCode(), payloadBuffer.position())); - serializeValue(value, payloadBuffer); + for (var entry : fields.values()) { + entry.setOffset(payloadBuffer.position()); + serializeValue(entry.getValue(), payloadBuffer); } payloadBuffer.flip(); var payloadView = payloadBuffer.slice().asReadOnlyBuffer(); // 2. Serialize directly to the final buffer format using the map-based method - return ImprintRecord.serialize(schemaId, directoryMap, payloadView); + return ImprintRecord.serialize(schemaId, new ArrayList<>(fields.values()), payloadView); } // Internal helper methods @@ -202,14 +196,15 @@ public ByteBuffer buildToBuffer() throws ImprintException { */ private ImprintRecordBuilder addField(int id, Value value) { Objects.requireNonNull(value, "Value cannot be null - use nullField() for explicit null values"); + var newEntry = new BuilderEntry((short) id, value); // Subtract the size of the old value if it's being replaced. - var oldValue = fields.get(id); - if (oldValue != null) - estimatedPayloadSize -= estimateValueSize(oldValue); + var oldEntry = fields.get(id); + if (oldEntry != null) + estimatedPayloadSize -= estimateValueSize(oldEntry.getValue()); - fields.put(id, value); - estimatedPayloadSize += estimateValueSize(value); + fields.put(id, newEntry); + estimatedPayloadSize += estimateValueSize(newEntry.getValue()); return this; } @@ -269,8 +264,7 @@ private Value convertToValue(Object obj) { return Value.fromRow((ImprintRecord) obj); } - throw new IllegalArgumentException("Cannot convert " + obj.getClass().getSimpleName() + - " to Imprint Value. Supported types: boolean, int, long, float, double, String, byte[], List, Map, ImprintRecord"); + throw new IllegalArgumentException("Unsupported type for auto-conversion: " + obj.getClass().getName()); } private MapKey convertToMapKey(Object obj) { @@ -287,13 +281,15 @@ private MapKey convertToMapKey(Object obj) { return MapKey.fromBytes((byte[]) obj); } - throw new IllegalArgumentException("Invalid map key type: " + obj.getClass().getSimpleName() + - ". Map keys must be int, long, String, or byte[]"); + throw new IllegalArgumentException("Unsupported map key type: " + obj.getClass().getName()); } @Override public String toString() { - return String.format("ImprintRecordBuilder{schemaId=%s, fields=%d}", schemaId, fields.size()); + return "ImprintRecordBuilder{" + + "schemaId=" + schemaId + + ", fields=" + fields + + '}'; } private int estimatePayloadSize() { @@ -334,8 +330,8 @@ private int estimateValueSize(Value value) { } } - private void serializeValue(Value value, ByteBuffer buffer) throws ImprintException { + // Use TypeHandler for simple types switch (value.getTypeCode()) { case NULL: case BOOL: @@ -360,4 +356,49 @@ private void serializeValue(Value value, ByteBuffer buffer) throws ImprintExcept throw new ImprintException(com.imprint.error.ErrorType.SERIALIZATION_ERROR, "Unknown type code: " + value.getTypeCode()); } } + + // Private inner class to hold field data during building + private static class BuilderEntry implements DirectoryEntry { + private final short id; + private final Value value; + private int offset; + + BuilderEntry(short id, Value value) { + this.id = id; + this.value = value; + this.offset = -1; // Initially unknown + } + + @Override + public short getId() { + return id; + } + + @Override + public TypeCode getTypeCode() { + return value.getTypeCode(); + } + + @Override + public int getOffset() { + return offset; + } + + public void setOffset(int offset) { + this.offset = offset; + } + + public Value getValue() { + return value; + } + + @Override + public String toString() { + return "BuilderEntry{" + + "id=" + id + + ", value=" + value + + ", offset=" + offset + + '}'; + } + } } \ No newline at end of file diff --git a/src/main/java/com/imprint/core/ImprintStream.java b/src/main/java/com/imprint/core/ImprintStream.java new file mode 100644 index 0000000..c218318 --- /dev/null +++ b/src/main/java/com/imprint/core/ImprintStream.java @@ -0,0 +1,257 @@ +package com.imprint.core; + +import com.imprint.error.ImprintException; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.TreeMap; + +/** + * Provides a framework for lazy, zero-copy transformations of Imprint records. + *

+ * Operations like {@link #project(int...)} and {@link #mergeWith(ImprintRecord)} are + * intermediate and do not create new records. They build up a plan of operations + * that is executed only when a terminal operation like {@link #toRecord()} is called. + */ +public final class ImprintStream { + + private final Plan plan; + + private ImprintStream(Plan plan) { + this.plan = Objects.requireNonNull(plan); + } + + // ========== PLAN DATA STRUCTURES ========== + + /** + * The internal representation of the transformation plan. + * This is a linked-list style structure where each step points to the previous one. + */ + private interface Plan { + // Marker interface for the plan steps + } + + /** + * The starting point of a plan, containing the initial source record. + */ + private static final class SourcePlan implements Plan { + final ImprintRecord source; + + private SourcePlan(ImprintRecord source) { + this.source = Objects.requireNonNull(source, "Source record cannot be null."); + } + } + + /** + * A plan step representing a 'project' operation. + */ + private static final class ProjectPlan implements Plan { + final Plan previous; + final Set fieldIds; + + private ProjectPlan(Plan previous, int... fieldIds) { + this.previous = Objects.requireNonNull(previous); + this.fieldIds = new HashSet<>(); + for (int id : fieldIds) { + this.fieldIds.add(id); + } + } + } + + /** + * A plan step representing a 'merge' operation. + */ + private static final class MergePlan implements Plan { + final Plan previous; + final List others; + + private MergePlan(Plan previous, List others) { + this.previous = Objects.requireNonNull(previous); + this.others = Objects.requireNonNull(others); + } + } + + // ========== PUBLIC API ========== + + /** + * Creates a new transformation stream starting with a source record. + * + * @param source The initial record for the transformation. + * @return A new ImprintStream. + */ + public static ImprintStream of(ImprintRecord source) { + return new ImprintStream(new SourcePlan(source)); + } + + /** + * An intermediate operation that defines a projection on the stream. + * This is a lazy operation; the projection is only performed when a terminal + * operation is called. + * + * @param fieldIds The field IDs to keep in the final record. + * @return A new ImprintStream with the projection step added to its plan. + */ + public ImprintStream project(int... fieldIds) { + return new ImprintStream(new ProjectPlan(this.plan, fieldIds)); + } + + /** + * An intermediate operation that defines a merge on the stream. + * The record from this stream (the "left" side) takes precedence in case + * of overlapping field IDs. + *

+ * This is a lazy operation; the merge is only performed when a terminal + * operation is called. + * + * @param other The record to merge with this stream's record. + * @return A new ImprintStream with the merge step added to its plan. + */ + public ImprintStream mergeWith(ImprintRecord other) { + return new ImprintStream(new MergePlan(this.plan, Collections.singletonList(other))); + } + + /** + * A terminal operation that executes the defined transformation plan and + * constructs a new, consolidated ImprintRecord. + * + * @return a new ImprintRecord representing the result of the stream operations. + */ + public ImprintRecord toRecord() { + return new Evaluator(this.plan).execute(); + } + + // ========== EVALUATOR ========== + + /** + * The engine that walks the plan and executes the transformation. + */ + private static final class Evaluator { + private final Plan plan; + + private Evaluator(Plan plan) { + this.plan = plan; + } + + public ImprintRecord execute() { + // Unwind the plan's linked-list structure into a forward-order list of operations. + var planList = new ArrayList(); + var current = plan; + while (current != null) { + planList.add(current); + if (current instanceof ProjectPlan) { + current = ((ProjectPlan) current).previous; + } else if (current instanceof MergePlan) { + current = ((MergePlan) current).previous; + } else if (current instanceof SourcePlan) { + current = null; // End of the chain + } + } + Collections.reverse(planList); + + // This map holds the set of fields being built, sorted by ID. + var resolvedFields = new TreeMap(); + + // Iteratively evaluate the plan step-by-step. + for (var planStep : planList) { + if (planStep instanceof SourcePlan) { + var sourcePlan = (SourcePlan) planStep; + for (var entry : sourcePlan.source.getDirectory()) { + resolvedFields.put((int) entry.getId(), new FieldSource(sourcePlan.source, entry)); + } + } else if (planStep instanceof ProjectPlan) { + var projectPlan = (ProjectPlan) planStep; + // Apply projection to the current state of resolved fields. + resolvedFields.keySet().retainAll(projectPlan.fieldIds); + } else if (planStep instanceof MergePlan) { + var mergePlan = (MergePlan) planStep; + // Add fields from other records if they aren't already in the map. + for (var otherRecord : mergePlan.others) { + for (var entry : otherRecord.getDirectory()) { + resolvedFields.putIfAbsent((int) entry.getId(), new FieldSource(otherRecord, entry)); + } + } + } + } + + // Once the final field set is determined, build the record. + return build(resolvedFields); + } + + private ImprintRecord build(TreeMap finalFields) { + if (finalFields.isEmpty()) { + // To-Do: Need a way to get the schemaId for an empty record. + // For now, returning null or using a default. + try { + return ImprintRecord.builder(new SchemaId(0, 0)).build(); + } catch (ImprintException e) { + // This should not happen when building an empty record. + throw new IllegalStateException("Failed to build empty record.", e); + } + } + + // Determine the schema from the first field's source record. + SchemaId schemaId = finalFields.firstEntry().getValue().record.getHeader().getSchemaId(); + + // 1. Calculate final payload size and prepare directory. + int payloadSize = 0; + var newDirectoryMap = new TreeMap(); + for (var entry : finalFields.entrySet()) { + var fieldSource = entry.getValue(); + int fieldLength = fieldSource.getLength(); + + newDirectoryMap.put(entry.getKey(), new SimpleDirectoryEntry(fieldSource.entry.getId(), fieldSource.entry.getTypeCode(), payloadSize)); + payloadSize += fieldLength; + } + + // 2. Allocate buffer and copy data. + var payload = ByteBuffer.allocate(payloadSize).order(ByteOrder.LITTLE_ENDIAN); + for (var fieldSource : finalFields.values()) { + try { + ByteBuffer sourceData = fieldSource.record.getRawBytes(fieldSource.entry.getId()); + if (sourceData != null) { + payload.put(sourceData.duplicate()); + } + } catch (Exception e) { + // This indicates a data corruption or bug, shouldn't happen in normal operation. + throw new IllegalStateException("Failed to copy data for field " + fieldSource.entry.getId(), e); + } + } + payload.flip(); + + // 3. Construct the final record. + var newHeader = new Header(new Flags((byte) 0), schemaId, payload.remaining()); + return new ImprintRecord(newHeader, newDirectoryMap, payload.asReadOnlyBuffer()); + } + + /** + * A helper class to track the source record and directory entry for a field. + */ + private static final class FieldSource { + final ImprintRecord record; + final DirectoryEntry entry; + + FieldSource(ImprintRecord record, DirectoryEntry entry) { + this.record = record; + this.entry = entry; + } + + int getLength() { + try { + ByteBuffer buf = record.getRawBytes(entry.getId()); + return buf != null ? buf.remaining() : 0; + } catch (Exception e) { + return 0; + } + } + } + } +} \ No newline at end of file diff --git a/src/main/java/com/imprint/core/SimpleDirectoryEntry.java b/src/main/java/com/imprint/core/SimpleDirectoryEntry.java new file mode 100644 index 0000000..843aad4 --- /dev/null +++ b/src/main/java/com/imprint/core/SimpleDirectoryEntry.java @@ -0,0 +1,22 @@ +package com.imprint.core; + +import com.imprint.types.TypeCode; +import lombok.Value; + +import java.util.Objects; + +/** + * A concrete, immutable directory entry. + */ +@Value +public class SimpleDirectoryEntry implements DirectoryEntry { + short id; + TypeCode typeCode; + int offset; + + public SimpleDirectoryEntry(short id, TypeCode typeCode, int offset) { + this.id = id; + this.typeCode = Objects.requireNonNull(typeCode, "TypeCode cannot be null"); + this.offset = offset; + } +} \ No newline at end of file From 7420b7fc2a305a91c1b18249e48c5139fde55567 Mon Sep 17 00:00:00 2001 From: expand3d <> Date: Sun, 8 Jun 2025 19:24:45 -0400 Subject: [PATCH 40/49] add large object profiling and refactor tests --- .../com/imprint/profile/ProfilerTest.java | 293 +++++++++++------- 1 file changed, 179 insertions(+), 114 deletions(-) diff --git a/src/test/java/com/imprint/profile/ProfilerTest.java b/src/test/java/com/imprint/profile/ProfilerTest.java index 1ea752d..3804722 100644 --- a/src/test/java/com/imprint/profile/ProfilerTest.java +++ b/src/test/java/com/imprint/profile/ProfilerTest.java @@ -7,6 +7,7 @@ import org.junit.jupiter.api.Test; import java.util.Random; +import java.util.UUID; /** * A test designed for profiling hotspots during development. @@ -26,7 +27,7 @@ * - String operations and UTF-8 encoding * - ByteBuffer operations */ -//@Disabled("Enable manually for profiling") +@Disabled("Enable manually for profiling") public class ProfilerTest { private static final int ITERATIONS = 1_000_000; @@ -34,145 +35,171 @@ public class ProfilerTest { @Test void profileFieldAccess() throws Exception { - System.out.println("Starting profiler test - attach profiler now..."); - Thread.sleep(5000); // Give time to attach profiler - - // Create a representative record var record = createTestRecord(); - - System.out.println("Beginning field access profiling..."); - long start = System.nanoTime(); - - // Simulate real-world access patterns - Random random = new Random(42); - int hits = 0; - - for (int i = 0; i < ITERATIONS; i++) { - // Random field access (hotspot) - int fieldId = random.nextInt(RECORD_SIZE) + 1; - var value = record.getValue(fieldId); - if (value != null) { - hits++; - - // Trigger string decoding (potential hotspot) - if (value.getTypeCode() == com.imprint.types.TypeCode.STRING) { - if (value instanceof Value.StringBufferValue) { - ((Value.StringBufferValue) value).getValue(); - } else { - ((Value.StringValue) value).getValue(); + + runProfileTest("Field Access", () -> { + // Simulate real-world access patterns + Random random = new Random(42); + int hits = 0; + + for (int i = 0; i < ITERATIONS; i++) { + // Random field access (hotspot) + int fieldId = random.nextInt(RECORD_SIZE) + 1; + var value = record.getValue(fieldId); + if (value != null) { + hits++; + + // Trigger string decoding (potential hotspot) + if (value.getTypeCode() == com.imprint.types.TypeCode.STRING) { + if (value instanceof Value.StringBufferValue) { + ((Value.StringBufferValue) value).getValue(); + } else { + ((Value.StringValue) value).getValue(); + } } } + + // Some raw access (zero-copy path) + if (i % 10 == 0) { + record.getRawBytes(fieldId); + } } - - // Some raw access (zero-copy path) - if (i % 10 == 0) { - record.getRawBytes(fieldId); - } - } - - long duration = System.nanoTime() - start; - System.out.printf("Completed %,d field accesses in %.2f ms (avg: %.1f ns/op, hits: %d)%n", - ITERATIONS, duration / 1_000_000.0, (double) duration / ITERATIONS, hits); + }); } @Test void profileSerialization() throws Exception { - System.out.println("Starting serialization profiler test..."); - Thread.sleep(3000); - var schemaId = new SchemaId(1, 0x12345678); - System.out.println("Beginning serialization profiling..."); - long start = System.nanoTime(); - - // Create and serialize many records (allocation hotspot) - for (int i = 0; i < 500_000; i++) { - var builder = ImprintRecord.builder(schemaId); - - // Add various field types - builder.field(1, Value.fromInt32(i)) - .field(2, Value.fromString("test-string-" + i)) - .field(3, Value.fromFloat64(i * 3.14159)) - .field(4, Value.fromBytes(("bytes-" + i).getBytes())); - - var record = builder.build(); - var serialized = record.serializeToBuffer(); // Potential hotspot - - // Trigger some deserialization - if (i % 1000 == 0) { - var deserialized = ImprintRecord.deserialize(serialized); - deserialized.getValue(2); // String decoding hotspot + runProfileTest("Serialization (Standard)", () -> { + // Create and serialize many records (allocation hotspot) + for (int i = 0; i < 500_000; i++) { + var builder = ImprintRecord.builder(schemaId); + + // Add various field types + builder.field(1, Value.fromInt32(i)) + .field(2, Value.fromString("test-string-" + i)) + .field(3, Value.fromFloat64(i * 3.14159)) + .field(4, Value.fromBytes(("bytes-" + i).getBytes())); + + var record = builder.build(); + var serialized = record.serializeToBuffer(); // Potential hotspot + + // Trigger some deserialization + if (i % 1000 == 0) { + var deserialized = ImprintRecord.deserialize(serialized); + deserialized.getValue(2); // String decoding hotspot + } } - } - - long duration = System.nanoTime() - start; - System.out.printf("Completed serialization test in %.2f ms%n", duration / 1_000_000.0); + }); + } + + @Test + void profileLargeObjectSerialization() throws Exception { + var schemaId = new SchemaId(3, 0xabcdef12); + var largeRecord = createVeryLargeRecord(); // A single large record to be re-serialized + + runProfileTest("Serialization (Large Object)", () -> { + // Re-serialize the same large object to focus on serialization logic + // rather than object creation. + for (int i = 0; i < 100_000; i++) { + var serialized = largeRecord.serializeToBuffer(); // Hotspot + + if (i % 1000 == 0) { + var deserialized = ImprintRecord.deserialize(serialized); + deserialized.getValue(10); // Access a field to ensure it works + } + } + }); } - @Test + @Test void profileProjection() throws Exception { - System.out.println("Starting projection profiler test..."); - Thread.sleep(3000); - var record = createLargeRecord(); - - System.out.println("Beginning projection profiling..."); - long start = System.nanoTime(); - - // Simulate analytical workload - project subset of fields repeatedly - for (int i = 0; i < 50_000; i++) { - // Project 10 fields out of 100 (common analytical pattern) - for (int fieldId = 1; fieldId <= 10; fieldId++) { - var value = record.getValue(fieldId); - if (value != null) { - // Force materialization of string values - if (value.getTypeCode() == com.imprint.types.TypeCode.STRING) { - if (value instanceof Value.StringBufferValue) { - ((Value.StringBufferValue) value).getValue(); + + runProfileTest("Projection", () -> { + // Simulate analytical workload - project subset of fields repeatedly + for (int i = 0; i < 50_000; i++) { + // Project 10 fields out of 100 (common analytical pattern) + for (int fieldId = 1; fieldId <= 10; fieldId++) { + var value = record.getValue(fieldId); + if (value != null) { + // Force materialization of string values + if (value.getTypeCode() == com.imprint.types.TypeCode.STRING) { + if (value instanceof Value.StringBufferValue) { + ((Value.StringBufferValue) value).getValue(); + } } } } } - } - - long duration = System.nanoTime() - start; - System.out.printf("Completed projection test in %.2f ms%n", duration / 1_000_000.0); + }); } @Test void profileMemoryAllocation() throws Exception { - System.out.println("Starting allocation profiler test..."); - Thread.sleep(3000); - - System.out.println("Beginning allocation profiling - watch for GC events..."); - - // Force allocation pressure to reveal GC hotspots - for (int batch = 0; batch < 1000; batch++) { - for (int i = 0; i < 1000; i++) { - var schemaId = new SchemaId(batch, i); - var builder = ImprintRecord.builder(schemaId); - - // Create strings of varying sizes (allocation pressure) - builder.field(1, Value.fromString("small")) - .field(2, Value.fromString("medium-length-string-" + i)) - .field(3, Value.fromString("very-long-string-that-will-cause-more-allocation-pressure-" + batch + "-" + i)) - .field(4, Value.fromBytes(new byte[100 + i % 1000])); // Varying byte arrays - - var record = builder.build(); - - // Some deserialization to trigger string decoding allocations - record.getValue(2); - record.getValue(3); - } - - if (batch % 100 == 0) { - System.out.printf("Completed batch %d/1000%n", batch); + runProfileTest("Memory Allocation", () -> { + // Force allocation pressure to reveal GC hotspots + for (int batch = 0; batch < 1000; batch++) { + for (int i = 0; i < 1000; i++) { + var schemaId = new SchemaId(batch, i); + var builder = ImprintRecord.builder(schemaId); + + // Create strings of varying sizes (allocation pressure) + builder.field(1, Value.fromString("small")) + .field(2, Value.fromString("medium-length-string-" + i)) + .field(3, Value.fromString("very-long-string-that-will-cause-more-allocation-pressure-" + batch + "-" + i)) + .field(4, Value.fromBytes(new byte[100 + i % 1000])); // Varying byte arrays + + var record = builder.build(); + + // Some deserialization to trigger string decoding allocations + record.getValue(2); + record.getValue(3); + } + + if (batch % 100 == 0) { + System.out.printf("Completed batch %d/1000%n", batch); + } } - } - - System.out.println("Allocation test complete - check GC logs and memory profiler"); + }, false); // Disable final time reporting as it's not relevant here } + // ========== Test Helpers ========== + + /** + * A wrapper to run a profiling test with boilerplate for timing and setup. + * @param testName The name of the test to print. + * @param testLogic The core logic of the test, passed as a lambda. + */ + private void runProfileTest(String testName, ThrowingRunnable testLogic) throws Exception { + runProfileTest(testName, testLogic, true); + } + + private void runProfileTest(String testName, ThrowingRunnable testLogic, boolean reportTime) throws Exception { + System.out.printf("===== Starting Profiler Test: %s =====%n", testName); + System.out.println("Attach profiler now..."); + Thread.sleep(3000); // Give time to attach profiler + + System.out.printf("Beginning %s profiling...%n", testName); + long start = System.nanoTime(); + + testLogic.run(); + + if (reportTime) { + long duration = System.nanoTime() - start; + System.out.printf("===== Completed %s in %.2f ms =====%n%n", testName, duration / 1_000_000.0); + } else { + System.out.printf("===== %s profiling complete. Check profiler output. =====%n%n", testName); + } + } + + /** A functional interface that allows for exceptions, for use in lambdas. */ + @FunctionalInterface + private interface ThrowingRunnable { + void run() throws Exception; + } + private ImprintRecord createTestRecord() throws Exception { var builder = ImprintRecord.builder(new SchemaId(1, 0xdeadbeef)); @@ -222,4 +249,42 @@ private ImprintRecord createLargeRecord() throws Exception { return builder.build(); } + + private ImprintRecord createVeryLargeRecord() throws Exception { + var builder = ImprintRecord.builder(new SchemaId(3, 0xabcdef12)); + var random = new Random(123); + + // Create 200 fields of varying types and sizes + for (int i = 1; i <= 200; i++) { + switch (i % 6) { + case 0: + builder.field(i, i * random.nextInt()); + break; + case 1: + // Medium string + builder.field(i, "user-id-" + UUID.randomUUID().toString()); + break; + case 2: + // Large string + builder.field(i, "This is a much larger text block for field " + i + ". It simulates a user comment, a description, or some other form of semi-structured text data. We repeat a sentence to make it longer. This is a much larger text block for field " + i + ". It simulates a user comment, a description, or some other form of semi-structured text data."); + break; + case 3: + builder.field(i, random.nextDouble() * 1000); + break; + case 4: + // Small byte array + var smallBytes = new byte[32]; + random.nextBytes(smallBytes); + builder.field(i, smallBytes); + break; + case 5: + // Large byte array + var largeBytes = new byte[1024]; + random.nextBytes(largeBytes); + builder.field(i, largeBytes); + break; + } + } + return builder.build(); + } } \ No newline at end of file From 4d86447c53406aa583c04adcb2767913ef632930 Mon Sep 17 00:00:00 2001 From: expand3d <> Date: Mon, 9 Jun 2025 00:51:25 -0400 Subject: [PATCH 41/49] add Thrift competitor and fix framework issues Adds Apache Thrift to the benchmark suite, including self-contained compiler download. Corrects Protobuf and FlatBuffers schemas and fixes bugs in the competitor classes to ensure a stable and robust benchmark environment. Includes refactored DataGenerator. --- build.gradle | 53 ++ src/jmh/flatbuffers/test_record.fbs | 21 +- .../benchmark/ComparisonBenchmark.java | 858 ++---------------- .../com/imprint/benchmark/DataGenerator.java | 67 ++ .../competitors/AbstractCompetitor.java | 29 + .../benchmark/competitors/AvroCompetitor.java | 156 ++++ .../benchmark/competitors/Competitor.java | 16 + .../competitors/FlatBuffersCompetitor.java | 137 +++ .../competitors/ImprintCompetitor.java | 76 ++ .../competitors/JacksonJsonCompetitor.java | 79 ++ .../benchmark/competitors/KryoCompetitor.java | 91 ++ .../competitors/MessagePackCompetitor.java | 78 ++ .../competitors/ProtobufCompetitor.java | 69 ++ .../competitors/ThriftCompetitor.java | 117 +++ src/jmh/proto/test_record.proto | 21 +- src/jmh/thrift/test_record.thrift | 18 + 16 files changed, 1063 insertions(+), 823 deletions(-) create mode 100644 src/jmh/java/com/imprint/benchmark/DataGenerator.java create mode 100644 src/jmh/java/com/imprint/benchmark/competitors/AbstractCompetitor.java create mode 100644 src/jmh/java/com/imprint/benchmark/competitors/AvroCompetitor.java create mode 100644 src/jmh/java/com/imprint/benchmark/competitors/Competitor.java create mode 100644 src/jmh/java/com/imprint/benchmark/competitors/FlatBuffersCompetitor.java create mode 100644 src/jmh/java/com/imprint/benchmark/competitors/ImprintCompetitor.java create mode 100644 src/jmh/java/com/imprint/benchmark/competitors/JacksonJsonCompetitor.java create mode 100644 src/jmh/java/com/imprint/benchmark/competitors/KryoCompetitor.java create mode 100644 src/jmh/java/com/imprint/benchmark/competitors/MessagePackCompetitor.java create mode 100644 src/jmh/java/com/imprint/benchmark/competitors/ProtobufCompetitor.java create mode 100644 src/jmh/java/com/imprint/benchmark/competitors/ThriftCompetitor.java create mode 100644 src/jmh/thrift/test_record.thrift diff --git a/build.gradle b/build.gradle index 33b1645..852be08 100644 --- a/build.gradle +++ b/build.gradle @@ -50,6 +50,8 @@ dependencies { jmhImplementation 'com.esotericsoftware:kryo:5.4.0' jmhImplementation 'org.msgpack:msgpack-core:0.9.8' jmhImplementation 'org.msgpack:jackson-dataformat-msgpack:0.9.8' + jmhImplementation 'org.apache.thrift:libthrift:0.19.0' + jmhImplementation 'javax.annotation:javax.annotation-api:1.3.2' } protobuf { @@ -137,11 +139,61 @@ tasks.register('generateFlatBuffers', Exec) { } } +// Task to download the Thrift compiler +task downloadThrift(type: Exec) { + description = 'Download Thrift compiler' + group = 'build setup' + def thriftVersion = "0.19.0" + def thriftExecutable = file("${buildDir}/thrift/thrift.exe") + def thriftUrl = "https://archive.apache.org/dist/thrift/${thriftVersion}/thrift-${thriftVersion}.exe" + + outputs.file(thriftExecutable) + + onlyIf { + !thriftExecutable.exists() && System.getProperty('os.name').toLowerCase().contains('windows') + } + + doFirst { + println "Downloading Thrift compiler for Windows from $thriftUrl..." + thriftExecutable.parentFile.mkdirs() + } + + commandLine 'curl', '-L', '-o', thriftExecutable.absolutePath, thriftUrl + + doLast { + println "Thrift compiler downloaded to: ${thriftExecutable}" + } +} + +// Task to generate Java code from Thrift IDL files for JMH benchmarks +task generateJmhThrift(type: Exec) { + dependsOn downloadThrift + description = 'Generate Java classes from Thrift schema' + group = 'build' + + def thriftExecutable = file("${buildDir}/thrift/thrift.exe") + def schemaFile = file('src/jmh/thrift/test_record.thrift') + def outputDir = file('build/generated-src/thrift/jmh/java') + + // Only run if the thrift executable exists (i.e., on Windows) + onlyIf { thriftExecutable.exists() } + + commandLine thriftExecutable.absolutePath, '-r', '--gen', 'java', '-o', outputDir.absolutePath, schemaFile.absolutePath + + inputs.file(schemaFile) + outputs.dir(outputDir) + + doFirst { + outputDir.mkdirs() + } +} + // Add generated FlatBuffers sources to JMH source set sourceSets { jmh { java { srcDir 'build/generated/source/flatbuffers/jmh/java' + srcDir 'build/generated-src/thrift/jmh/java' } proto { srcDir 'src/jmh/proto' @@ -151,6 +203,7 @@ sourceSets { // Make JMH compilation depend on FlatBuffers generation compileJmhJava.dependsOn generateFlatBuffers +compileJmhJava.dependsOn generateJmhThrift // Handle duplicate proto files tasks.named('processJmhResources') { diff --git a/src/jmh/flatbuffers/test_record.fbs b/src/jmh/flatbuffers/test_record.fbs index ccc31d0..698bd81 100644 --- a/src/jmh/flatbuffers/test_record.fbs +++ b/src/jmh/flatbuffers/test_record.fbs @@ -1,15 +1,14 @@ -namespace com.imprint.benchmark; +namespace com.imprint.benchmark.flatbuffers; -table TestRecordFB { - id: int; - name: string; - price: double; +table TestRecord { + id: string; + timestamp: long; + flags: int; active: bool; - category: string; - tags: [string]; - metadata_keys: [string]; - metadata_values: [string]; - extra_data: [string]; + value: double; + data: [ubyte]; + tags: [int]; + metadata: [string]; // Representing map as a flat list of key/value strings for simplicity } -root_type TestRecordFB; \ No newline at end of file +root_type TestRecord; \ No newline at end of file diff --git a/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java b/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java index e52388c..7000fca 100644 --- a/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java +++ b/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java @@ -1,22 +1,15 @@ package com.imprint.benchmark; -import com.esotericsoftware.kryo.Kryo; -import com.esotericsoftware.kryo.io.Input; -import com.esotericsoftware.kryo.io.Output; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.flatbuffers.FlatBufferBuilder; -import com.imprint.core.ImprintRecord; -import com.imprint.core.ImprintRecordBuilder; -import com.imprint.core.SchemaId; -import com.imprint.types.MapKey; -import com.imprint.types.Value; -import org.apache.avro.Schema; -import org.apache.avro.generic.GenericData; -import org.apache.avro.generic.GenericDatumReader; -import org.apache.avro.generic.GenericDatumWriter; -import org.apache.avro.generic.GenericRecord; -import org.apache.avro.io.*; -import org.msgpack.jackson.dataformat.MessagePackFactory; +import com.imprint.benchmark.competitors.AbstractCompetitor; +import com.imprint.benchmark.competitors.AvroCompetitor; +import com.imprint.benchmark.competitors.Competitor; +import com.imprint.benchmark.competitors.FlatBuffersCompetitor; +import com.imprint.benchmark.competitors.ImprintCompetitor; +import com.imprint.benchmark.competitors.JacksonJsonCompetitor; +import com.imprint.benchmark.competitors.KryoCompetitor; +import com.imprint.benchmark.competitors.MessagePackCompetitor; +import com.imprint.benchmark.competitors.ProtobufCompetitor; +import com.imprint.benchmark.competitors.ThriftCompetitor; import org.openjdk.jmh.annotations.*; import org.openjdk.jmh.infra.Blackhole; import org.openjdk.jmh.runner.Runner; @@ -24,822 +17,81 @@ import org.openjdk.jmh.runner.options.Options; import org.openjdk.jmh.runner.options.OptionsBuilder; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.nio.ByteBuffer; -import java.util.*; +import java.util.Arrays; +import java.util.List; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; -/** - * Head-to-head benchmarks comparing Imprint against other serialization libraries. - * Tests the performance claims made in the documentation. - */ -@BenchmarkMode(Mode.AverageTime) -@OutputTimeUnit(TimeUnit.NANOSECONDS) @State(Scope.Benchmark) -@Warmup(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS) -@Measurement(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS) -@Fork(1) -@SuppressWarnings("unused") +@BenchmarkMode(Mode.Throughput) +@OutputTimeUnit(TimeUnit.SECONDS) +@Warmup(iterations = 3, time = 5) +@Measurement(iterations = 5, time = 10) +@Fork(value = 1, jvmArgs = {"-Xms4g", "-Xmx4g"}) public class ComparisonBenchmark { - // Test data - private TestRecord testData; + private static final List COMPETITORS = Arrays.asList( + new ImprintCompetitor(), + new JacksonJsonCompetitor(), + new ProtobufCompetitor(), + new FlatBuffersCompetitor(), + new AvroCompetitor(), + new ThriftCompetitor(), + new KryoCompetitor(), + new MessagePackCompetitor() + ); - // Serialized formats - private ByteBuffer imprintBytesBuffer; - private byte[] jacksonJsonBytes; - private byte[] kryoBytes; - private byte[] messagePackBytes; - private byte[] avroBytes; - private byte[] protobufBytes; - private ByteBuffer flatbuffersBytes; + @Param({"Imprint"}) + public String competitorName; - // Library instances - private Schema avroSchema; - private DatumWriter avroWriter; - private DatumReader avroReader; - private ObjectMapper jacksonJsonMapper; - private Kryo kryo; - private ObjectMapper messagePackMapper; + private Competitor competitor; + private DataGenerator.TestRecord testRecord1; + private DataGenerator.TestRecord testRecord2; - @Setup - public void setup() throws Exception { - testData = createTestRecord(); - - // Initialize libraries - jacksonJsonMapper = new ObjectMapper(); - kryo = new Kryo(); - kryo.register(TestRecord.class); - kryo.register(ArrayList.class); - kryo.register(HashMap.class); - kryo.register(Arrays.asList().getClass()); - - // Initialize MessagePack ObjectMapper - messagePackMapper = new ObjectMapper(new MessagePackFactory()); - setupAvro(); - - // Pre-serialize for deserialization benchmarks - imprintBytesBuffer = serializeWithImprint(testData); - jacksonJsonBytes = serializeWithJacksonJson(testData); - kryoBytes = serializeWithKryo(testData); - messagePackBytes = serializeWithMessagePack(testData); - avroBytes = serializeWithAvro(testData); - protobufBytes = serializeWithProtobuf(testData); - flatbuffersBytes = serializeWithFlatBuffers(testData); - } - - // ===== SERIALIZATION BENCHMARKS ===== - - @Benchmark - public void serializeImprint(Blackhole bh) throws Exception { - ByteBuffer result = serializeWithImprint(testData); - bh.consume(result); - } - - @Benchmark - public void serializeJacksonJson(Blackhole bh) throws Exception { - byte[] result = serializeWithJacksonJson(testData); - bh.consume(result); - } - - @Benchmark - public void serializeKryo(Blackhole bh) { - byte[] result = serializeWithKryo(testData); - bh.consume(result); - } - - @Benchmark - public void serializeMessagePack(Blackhole bh) throws Exception { - byte[] result = serializeWithMessagePack(testData); - bh.consume(result); - } - - @Benchmark - public void serializeAvro(Blackhole bh) throws Exception { - byte[] result = serializeWithAvro(testData); - bh.consume(result); - } - - @Benchmark - public void serializeProtobuf(Blackhole bh) { - byte[] result = serializeWithProtobuf(testData); - bh.consume(result); - } - - @Benchmark - public void serializeFlatBuffers(Blackhole bh) { - ByteBuffer result = serializeWithFlatBuffers(testData); - bh.consume(result); - } - - // ===== SETUP ONLY ===== - - @Benchmark - public void deserializeSetupImprint(Blackhole bh) throws Exception { - ImprintRecord result = ImprintRecord.deserialize(imprintBytesBuffer.duplicate()); - bh.consume(result); - } - - @Benchmark - public void deserializeSetupFlatBuffers(Blackhole bh) { - com.imprint.benchmark.TestRecordFB result = com.imprint.benchmark.TestRecordFB.getRootAsTestRecordFB(flatbuffersBytes.duplicate()); - bh.consume(result); - } - - // ===== FULL DESERIALIZATION BENCHMARKS ===== - - @Benchmark - public void deserializeJacksonJson(Blackhole bh) throws Exception { - TestRecord result = jacksonJsonMapper.readValue(jacksonJsonBytes, TestRecord.class); - bh.consume(result); - } - - @Benchmark - public void deserializeKryo(Blackhole bh) { - Input input = new Input(new ByteArrayInputStream(kryoBytes)); - TestRecord result = kryo.readObject(input, TestRecord.class); - input.close(); - bh.consume(result); - } - - @Benchmark - public void deserializeMessagePack(Blackhole bh) throws Exception { - TestRecord result = messagePackMapper.readValue(messagePackBytes, TestRecord.class); - bh.consume(result); - } - - @Benchmark - public void deserializeAvro(Blackhole bh) throws Exception { - GenericRecord result = deserializeWithAvro(avroBytes); - bh.consume(result); - } - - @Benchmark - public void deserializeProtobuf(Blackhole bh) throws Exception { - com.imprint.benchmark.TestRecordProto.TestRecord result = com.imprint.benchmark.TestRecordProto.TestRecord.parseFrom(protobufBytes); - bh.consume(result); - } - - @Benchmark - public void deserializeImprint(Blackhole bh) throws Exception { - ImprintRecord result = ImprintRecord.deserialize(imprintBytesBuffer.duplicate()); - // Access all fields to force full deserialization - result.getInt32(1); // id - result.getString(2); // name - result.getFloat64(3); // price - result.getBoolean(4); // active - result.getString(5); // category - result.getArray(6); // tags - result.getMap(7); // metadata - for (int i = 8; i < 21; i++) { - result.getString(i); // extraData fields - } - - bh.consume(result); - } - - @Benchmark - public void deserializeFlatBuffers(Blackhole bh) { - com.imprint.benchmark.TestRecordFB result = com.imprint.benchmark.TestRecordFB.getRootAsTestRecordFB(flatbuffersBytes.duplicate()); - - // Access all fields - result.id(); - result.name(); - result.price(); - result.active(); - result.category(); - // Access all tags - for (int i = 0; i < result.tagsLength(); i++) { - result.tags(i); - } - // Access all metadata - for (int i = 0; i < result.metadataKeysLength(); i++) { - result.metadataKeys(i); - result.metadataValues(i); - } - // Access all extra data - for (int i = 0; i < result.extraDataLength(); i++) { - result.extraData(i); - } - - bh.consume(result); - } - - // ===== FIELD ACCESS BENCHMARKS ===== - // Tests accessing a single field near the end of a record - - @Benchmark - public void singleFieldAccessImprint(Blackhole bh) throws Exception { - ImprintRecord record = ImprintRecord.deserialize(imprintBytesBuffer.duplicate()); - var field15 = record.getString(15); - bh.consume(field15); - } - - @Benchmark - public void singleFieldAccessJacksonJson(Blackhole bh) throws Exception { - TestRecord record = jacksonJsonMapper.readValue(jacksonJsonBytes, TestRecord.class); - bh.consume(record.extraData.get(4)); - } - - @Benchmark - public void singleFieldAccessKryo(Blackhole bh) { - Input input = new Input(new ByteArrayInputStream(kryoBytes)); - TestRecord record = kryo.readObject(input, TestRecord.class); - input.close(); - bh.consume(record.extraData.get(4)); - } - - @Benchmark - public void singleFieldAccessMessagePack(Blackhole bh) throws Exception { - TestRecord record = messagePackMapper.readValue(messagePackBytes, TestRecord.class); - bh.consume(record.extraData.get(4)); - } - - @Benchmark - public void singleFieldAccessAvro(Blackhole bh) throws Exception { - GenericRecord record = deserializeWithAvro(avroBytes); - bh.consume(record.get("extra_data")); - } - - @Benchmark - public void singleFieldAccessProtobuf(Blackhole bh) throws Exception { - com.imprint.benchmark.TestRecordProto.TestRecord record = com.imprint.benchmark.TestRecordProto.TestRecord.parseFrom(protobufBytes); - bh.consume(record.getExtraData(4)); - } - - @Benchmark - public void singleFieldAccessFlatBuffers(Blackhole bh) { - com.imprint.benchmark.TestRecordFB record = com.imprint.benchmark.TestRecordFB.getRootAsTestRecordFB(flatbuffersBytes.duplicate()); - bh.consume(record.extraData(4)); + public static List competitorName() { + return COMPETITORS.stream().map(Competitor::name).collect(Collectors.toList()); } - // ===== SIZE COMPARISON ===== + @Setup(Level.Trial) + public void setup() { + // Find the competitor implementation + competitor = COMPETITORS.stream() + .filter(c -> c.name().equals(competitorName)) + .findFirst() + .orElseThrow(() -> new IllegalStateException("Unknown competitor: " + competitorName)); - @Benchmark - public void measureImprintSize(Blackhole bh) { - bh.consume(imprintBytesBuffer.remaining()); - } + // Create the test data + testRecord1 = DataGenerator.createTestRecord(); + testRecord2 = DataGenerator.createTestRecord(); - @Benchmark - public void measureJacksonJsonSize(Blackhole bh) { - bh.consume(jacksonJsonBytes.length); + // Setup the competitor with the data + competitor.setup(testRecord1, testRecord2); } @Benchmark - public void measureKryoSize(Blackhole bh) { - bh.consume(kryoBytes.length); + public void serialize(Blackhole bh) { + competitor.serialize(bh); } @Benchmark - public void measureMessagePackSize(Blackhole bh) { - bh.consume(messagePackBytes.length); + public void deserialize(Blackhole bh) { + competitor.deserialize(bh); } @Benchmark - public void measureAvroSize(Blackhole bh) { - bh.consume(avroBytes.length); + public void projectAndSerialize(Blackhole bh) { + competitor.projectAndSerialize(bh); } @Benchmark - public void measureProtobufSize(Blackhole bh) { - bh.consume(protobufBytes.length); + public void mergeAndSerialize(Blackhole bh) { + competitor.mergeAndSerialize(bh); } - @Benchmark - public void measureFlatBuffersSize(Blackhole bh) { - bh.consume(flatbuffersBytes.remaining()); - } - - // ===== MERGE SIMULATION BENCHMARKS ===== - - //@Benchmark - public void mergeImprint(Blackhole bh) throws Exception { - var record1Buffer = imprintBytesBuffer.duplicate(); - var record2Data = createTestRecord2(); - var record2Buffer = serializeWithImprint(record2Data); - - var deserialized1 = ImprintRecord.deserialize(record1Buffer); - var deserialized2 = ImprintRecord.deserialize(record2Buffer); - var merged = simulateMerge(deserialized1, deserialized2); - - bh.consume(merged); - } - - //@Benchmark - public void mergeJacksonJson(Blackhole bh) throws Exception { - var record1 = jacksonJsonMapper.readValue(jacksonJsonBytes, TestRecord.class); - var record2Data = createTestRecord2(); - var record2Bytes = serializeWithJacksonJson(record2Data); - var record2 = jacksonJsonMapper.readValue(record2Bytes, TestRecord.class); - - var mergedPojo = mergeTestRecords(record1, record2); - byte[] result = jacksonJsonMapper.writeValueAsBytes(mergedPojo); - bh.consume(result); - } - - //@Benchmark - public void mergeKryo(Blackhole bh) { - Input input1 = new Input(new ByteArrayInputStream(kryoBytes)); - var record1 = kryo.readObject(input1, TestRecord.class); - input1.close(); - - var record2Data = createTestRecord2(); - var record2Bytes = serializeWithKryo(record2Data); - Input input2 = new Input(new ByteArrayInputStream(record2Bytes)); - var record2 = kryo.readObject(input2, TestRecord.class); - input2.close(); - - var mergedPojo = mergeTestRecords(record1, record2); - byte[] result = serializeWithKryo(mergedPojo); - bh.consume(result); - } - - //@Benchmark - public void mergeMessagePack(Blackhole bh) throws Exception { - var record1 = messagePackMapper.readValue(messagePackBytes, TestRecord.class); - var record2Data = createTestRecord2(); - var record2Bytes = serializeWithMessagePack(record2Data); - var record2 = messagePackMapper.readValue(record2Bytes, TestRecord.class); - - var mergedPojo = mergeTestRecords(record1, record2); - byte[] result = messagePackMapper.writeValueAsBytes(mergedPojo); - bh.consume(result); - } - - //@Benchmark - public void mergeAvro(Blackhole bh) throws Exception { - var record1 = deserializeWithAvro(avroBytes); - var record2Data = createTestRecord2(); - var record2Bytes = serializeWithAvro(record2Data); - var record2 = deserializeWithAvro(record2Bytes); - - var merged = mergeAvroRecords(record1, record2); - byte[] result = serializeAvroRecord(merged); - bh.consume(result); - } - - //@Benchmark - public void mergeProtobuf(Blackhole bh) throws Exception { - var record1 = com.imprint.benchmark.TestRecordProto.TestRecord.parseFrom(protobufBytes); - var record2Data = createTestRecord2(); - var record2Bytes = serializeWithProtobuf(record2Data); - var record2 = com.imprint.benchmark.TestRecordProto.TestRecord.parseFrom(record2Bytes); - - var merged = mergeProtobufRecords(record1, record2); - byte[] result = merged.toByteArray(); - bh.consume(result); - } - - //@Benchmark - public void mergeFlatBuffers(Blackhole bh) { - var record1 = com.imprint.benchmark.TestRecordFB.getRootAsTestRecordFB(flatbuffersBytes.duplicate()); - var record2Data = createTestRecord2(); - var record2Buffer = serializeWithFlatBuffers(record2Data); - var record2 = com.imprint.benchmark.TestRecordFB.getRootAsTestRecordFB(record2Buffer); - - var merged = mergeFlatBuffersRecords(record1, record2); - bh.consume(merged); - } - - // ===== MAIN METHOD TO RUN BENCHMARKS ===== - public static void main(String[] args) throws RunnerException { - runFieldAccessBenchmarks(); - // Or, uncomment specific runner methods to execute subsets: - // runSerializationBenchmarks(); - // runDeserializationBenchmarks(); - // runFieldAccessBenchmarks(); - // runSizeComparisonBenchmarks(); - // runMergeBenchmarks(); - // runMessagePackBenchmarks(); - } - - public static void runAll() throws RunnerException { Options opt = new OptionsBuilder() .include(ComparisonBenchmark.class.getSimpleName()) .build(); new Runner(opt).run(); } - - public static void runSerializationBenchmarks() throws RunnerException { - Options opt = new OptionsBuilder() - .include(ComparisonBenchmark.class.getSimpleName() + ".serialize.*") - .build(); - new Runner(opt).run(); - } - - public static void runDeserializationBenchmarks() throws RunnerException { - Options opt = new OptionsBuilder() - .include(ComparisonBenchmark.class.getSimpleName() + ".deserialize.*") - .build(); - new Runner(opt).run(); - } - - public static void runFieldAccessBenchmarks() throws RunnerException { - Options opt = new OptionsBuilder() - .include(ComparisonBenchmark.class.getSimpleName() + ".singleFieldAccess.*") - .build(); - new Runner(opt).run(); - } - - public static void runSizeComparisonBenchmarks() throws RunnerException { - Options opt = new OptionsBuilder() - .include(ComparisonBenchmark.class.getSimpleName() + ".measure.*") - .build(); - new Runner(opt).run(); - } - - public static void runMergeBenchmarks() throws RunnerException { - Options opt = new OptionsBuilder() - .include(ComparisonBenchmark.class.getSimpleName() + ".merge.*") - .build(); - new Runner(opt).run(); - } - - public static void runMessagePackBenchmarks() throws RunnerException { - Options opt = new OptionsBuilder() - .include(ComparisonBenchmark.class.getSimpleName() + ".*MessagePack.*") - .build(); - new Runner(opt).run(); - } - - public static void runAvroBenchmarks() throws RunnerException { - Options opt = new OptionsBuilder() - .include(ComparisonBenchmark.class.getSimpleName() + ".*Avro.*") - .build(); - new Runner(opt).run(); - } - - public static void runProtobufBenchmarks() throws RunnerException { - Options opt = new OptionsBuilder() - .include(ComparisonBenchmark.class.getSimpleName() + ".*Protobuf.*") - .build(); - new Runner(opt).run(); - } - - public static void runFlatBuffersBenchmarks() throws RunnerException { - Options opt = new OptionsBuilder() - .include(ComparisonBenchmark.class.getSimpleName() + ".*FlatBuffers.*") - .build(); - new Runner(opt).run(); - } - - // ===== HELPER METHODS ===== - - private void setupAvro() { - String schemaJson = "{\n" + - " \"type\": \"record\",\n" + - " \"name\": \"TestRecord\",\n" + - " \"fields\": [\n" + - " {\"name\": \"id\", \"type\": \"int\"},\n" + - " {\"name\": \"name\", \"type\": \"string\"},\n" + - " {\"name\": \"price\", \"type\": \"double\"},\n" + - " {\"name\": \"active\", \"type\": \"boolean\"},\n" + - " {\"name\": \"category\", \"type\": \"string\"},\n" + - " {\"name\": \"tags\", \"type\": {\"type\": \"array\", \"items\": \"string\"}},\n" + - " {\"name\": \"metadata\", \"type\": {\"type\": \"map\", \"values\": \"string\"}},\n" + - " {\"name\": \"extraData0\", \"type\": \"string\"},\n" + - " {\"name\": \"extraData1\", \"type\": \"string\"},\n" + - " {\"name\": \"extraData2\", \"type\": \"string\"},\n" + - " {\"name\": \"extraData3\", \"type\": \"string\"},\n" + - " {\"name\": \"extraData4\", \"type\": \"string\"},\n" + - " {\"name\": \"extraData5\", \"type\": \"string\"},\n" + - " {\"name\": \"extraData6\", \"type\": \"string\"},\n" + - " {\"name\": \"extraData7\", \"type\": \"string\"},\n" + - " {\"name\": \"extraData8\", \"type\": \"string\"},\n" + - " {\"name\": \"extraData9\", \"type\": \"string\"},\n" + - " {\"name\": \"extraData10\", \"type\": \"string\"},\n" + - " {\"name\": \"extraData11\", \"type\": \"string\"},\n" + - " {\"name\": \"extraData12\", \"type\": \"string\"}\n" + - " ]\n" + - "}"; - - avroSchema = new Schema.Parser().parse(schemaJson); - avroWriter = new GenericDatumWriter<>(avroSchema); - avroReader = new GenericDatumReader<>(avroSchema); - } - - private ByteBuffer serializeWithImprint(TestRecord data) throws Exception { - var builder = ImprintRecord.builder(new SchemaId(1, 0x12345678)); - - builder.field(1, data.id); - builder.field(2, data.name); - builder.field(3, data.price); - builder.field(4, data.active); - builder.field(5, data.category); - builder.field(6, data.tags); - builder.field(7, data.metadata); - - for (int i = 0; i < data.extraData.size(); i++) { - builder.field(8 + i, data.extraData.get(i)); - } - - return builder.build().serializeToBuffer(); - } - - private byte[] serializeWithJacksonJson(TestRecord data) throws Exception { - return jacksonJsonMapper.writeValueAsBytes(data); - } - - private byte[] serializeWithKryo(TestRecord data) { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - Output output = new Output(baos); - kryo.writeObject(output, data); - output.close(); - return baos.toByteArray(); - } - - private byte[] serializeWithMessagePack(TestRecord data) throws Exception { - return messagePackMapper.writeValueAsBytes(data); - } - - private byte[] serializeWithAvro(TestRecord data) throws Exception { - GenericRecord record = new GenericData.Record(avroSchema); - record.put("id", data.id); - record.put("name", data.name); - record.put("price", data.price); - record.put("active", data.active); - record.put("category", data.category); - record.put("tags", data.tags); - record.put("metadata", data.metadata); - - for (int i = 0; i < data.extraData.size(); i++) { - record.put("extraData" + i, data.extraData.get(i)); - } - - return serializeAvroRecord(record); - } - - private byte[] serializeAvroRecord(GenericRecord record) throws Exception { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - Encoder encoder = EncoderFactory.get().binaryEncoder(baos, null); - avroWriter.write(record, encoder); - encoder.flush(); - return baos.toByteArray(); - } - - private GenericRecord deserializeWithAvro(byte[] data) throws Exception { - Decoder decoder = DecoderFactory.get().binaryDecoder(data, null); - return avroReader.read(null, decoder); - } - - private byte[] serializeWithProtobuf(TestRecord data) { - var builder = com.imprint.benchmark.TestRecordProto.TestRecord.newBuilder() - .setId(data.id) - .setName(data.name) - .setPrice(data.price) - .setActive(data.active) - .setCategory(data.category) - .addAllTags(data.tags) - .putAllMetadata(data.metadata); - - for (String extraData : data.extraData) { - builder.addExtraData(extraData); - } - - return builder.build().toByteArray(); - } - - private ByteBuffer serializeWithFlatBuffers(TestRecord data) { - var builder = new FlatBufferBuilder(1024); - - int nameOffset = builder.createString(data.name); - int categoryOffset = builder.createString(data.category); - - int[] tagOffsets = new int[data.tags.size()]; - for (int i = 0; i < data.tags.size(); i++) { - tagOffsets[i] = builder.createString(data.tags.get(i)); - } - int tagsOffset = com.imprint.benchmark.TestRecordFB.createTagsVector(builder, tagOffsets); - - String[] metadataKeys = data.metadata.keySet().toArray(new String[0]); - String[] metadataValues = new String[metadataKeys.length]; - int[] keyOffsets = new int[metadataKeys.length]; - int[] valueOffsets = new int[metadataKeys.length]; - - for (int i = 0; i < metadataKeys.length; i++) { - metadataValues[i] = data.metadata.get(metadataKeys[i]); - keyOffsets[i] = builder.createString(metadataKeys[i]); - valueOffsets[i] = builder.createString(metadataValues[i]); - } - int metadataKeysOffset = com.imprint.benchmark.TestRecordFB.createMetadataKeysVector(builder, keyOffsets); - int metadataValuesOffset = com.imprint.benchmark.TestRecordFB.createMetadataValuesVector(builder, valueOffsets); - - int[] extraDataOffsets = new int[data.extraData.size()]; - for (int i = 0; i < data.extraData.size(); i++) { - extraDataOffsets[i] = builder.createString(data.extraData.get(i)); - } - int extraDataOffset = com.imprint.benchmark.TestRecordFB.createExtraDataVector(builder, extraDataOffsets); - - com.imprint.benchmark.TestRecordFB.startTestRecordFB(builder); - com.imprint.benchmark.TestRecordFB.addId(builder, data.id); - com.imprint.benchmark.TestRecordFB.addName(builder, nameOffset); - com.imprint.benchmark.TestRecordFB.addPrice(builder, data.price); - com.imprint.benchmark.TestRecordFB.addActive(builder, data.active); - com.imprint.benchmark.TestRecordFB.addCategory(builder, categoryOffset); - com.imprint.benchmark.TestRecordFB.addTags(builder, tagsOffset); - com.imprint.benchmark.TestRecordFB.addMetadataKeys(builder, metadataKeysOffset); - com.imprint.benchmark.TestRecordFB.addMetadataValues(builder, metadataValuesOffset); - com.imprint.benchmark.TestRecordFB.addExtraData(builder, extraDataOffset); - int recordOffset = com.imprint.benchmark.TestRecordFB.endTestRecordFB(builder); - - builder.finish(recordOffset); - return builder.dataBuffer(); - } - - private ImprintRecord simulateMerge(ImprintRecord first, ImprintRecord second) throws Exception { - var builder = ImprintRecord.builder(first.getHeader().getSchemaId()); - var usedFieldIds = new HashSet(); - - // Copy fields from first record (takes precedence) - copyFieldsToBuilder(first, builder, usedFieldIds); - - // Copy non-conflicting fields from second record - copyFieldsToBuilder(second, builder, usedFieldIds); - - return builder.build(); - } - - private void copyFieldsToBuilder(ImprintRecord record, ImprintRecordBuilder builder, Set usedFieldIds) throws Exception { - for (var entry : record.getDirectory()) { - int fieldId = entry.getId(); - if (!usedFieldIds.contains(fieldId)) { - var value = record.getValue(fieldId); - if (value != null) { - builder.field(fieldId, value); - usedFieldIds.add(fieldId); - } - } - } - } - - private TestRecord mergeTestRecords(TestRecord first, TestRecord second) { - TestRecord merged = new TestRecord(); - merged.id = first.id; - merged.name = first.name != null ? first.name : second.name; - merged.price = first.price != 0.0 ? first.price : second.price; - merged.active = first.active; - merged.category = first.category != null ? first.category : second.category; - - merged.tags = new ArrayList<>(first.tags); - merged.tags.addAll(second.tags); - - merged.metadata = new HashMap<>(first.metadata); - merged.metadata.putAll(second.metadata); - - return merged; - } - - private GenericRecord mergeAvroRecords(GenericRecord first, GenericRecord second) { - GenericRecord merged = new GenericData.Record(avroSchema); - - // Copy all fields from first record - for (Schema.Field field : avroSchema.getFields()) { - merged.put(field.name(), first.get(field.name())); - } - - // Override with non-null values from second record - for (Schema.Field field : avroSchema.getFields()) { - Object secondValue = second.get(field.name()); - if (secondValue != null && !secondValue.toString().isEmpty()) { - merged.put(field.name(), secondValue); - } - } - - return merged; - } - - private com.imprint.benchmark.TestRecordProto.TestRecord mergeProtobufRecords(com.imprint.benchmark.TestRecordProto.TestRecord first, com.imprint.benchmark.TestRecordProto.TestRecord second) { - return com.imprint.benchmark.TestRecordProto.TestRecord.newBuilder() - .mergeFrom(first) - .mergeFrom(second) - .build(); - } - - private ByteBuffer mergeFlatBuffersRecords(com.imprint.benchmark.TestRecordFB first, com.imprint.benchmark.TestRecordFB second) { - var builder = new FlatBufferBuilder(1024); - - String name = second.name() != null && !second.name().isEmpty() ? second.name() : first.name(); - String category = second.category() != null && !second.category().isEmpty() ? second.category() : first.category(); - double price = second.price() != 0.0 ? second.price() : first.price(); - boolean active = second.active(); - int id = first.id(); - - int nameOffset = builder.createString(name); - int categoryOffset = builder.createString(category); - - List mergedTags = new ArrayList<>(); - for (int i = 0; i < first.tagsLength(); i++) { - mergedTags.add(first.tags(i)); - } - for (int i = 0; i < second.tagsLength(); i++) { - mergedTags.add(second.tags(i)); - } - - int[] tagOffsets = new int[mergedTags.size()]; - for (int i = 0; i < mergedTags.size(); i++) { - tagOffsets[i] = builder.createString(mergedTags.get(i)); - } - int tagsOffset = com.imprint.benchmark.TestRecordFB.createTagsVector(builder, tagOffsets); - - Map mergedMetadata = new HashMap<>(); - for (int i = 0; i < first.metadataKeysLength(); i++) { - mergedMetadata.put(first.metadataKeys(i), first.metadataValues(i)); - } - for (int i = 0; i < second.metadataKeysLength(); i++) { - mergedMetadata.put(second.metadataKeys(i), second.metadataValues(i)); - } - - String[] metadataKeys = mergedMetadata.keySet().toArray(new String[0]); - int[] keyOffsets = new int[metadataKeys.length]; - int[] valueOffsets = new int[metadataKeys.length]; - - for (int i = 0; i < metadataKeys.length; i++) { - keyOffsets[i] = builder.createString(metadataKeys[i]); - valueOffsets[i] = builder.createString(mergedMetadata.get(metadataKeys[i])); - } - int metadataKeysOffset = com.imprint.benchmark.TestRecordFB.createMetadataKeysVector(builder, keyOffsets); - int metadataValuesOffset = com.imprint.benchmark.TestRecordFB.createMetadataValuesVector(builder, valueOffsets); - - int[] extraDataOffsets = new int[first.extraDataLength()]; - for (int i = 0; i < first.extraDataLength(); i++) { - extraDataOffsets[i] = builder.createString(first.extraData(i)); - } - int extraDataOffset = com.imprint.benchmark.TestRecordFB.createExtraDataVector(builder, extraDataOffsets); - - com.imprint.benchmark.TestRecordFB.startTestRecordFB(builder); - com.imprint.benchmark.TestRecordFB.addId(builder, id); - com.imprint.benchmark.TestRecordFB.addName(builder, nameOffset); - com.imprint.benchmark.TestRecordFB.addPrice(builder, price); - com.imprint.benchmark.TestRecordFB.addActive(builder, active); - com.imprint.benchmark.TestRecordFB.addCategory(builder, categoryOffset); - com.imprint.benchmark.TestRecordFB.addTags(builder, tagsOffset); - com.imprint.benchmark.TestRecordFB.addMetadataKeys(builder, metadataKeysOffset); - com.imprint.benchmark.TestRecordFB.addMetadataValues(builder, metadataValuesOffset); - com.imprint.benchmark.TestRecordFB.addExtraData(builder, extraDataOffset); - int recordOffset = com.imprint.benchmark.TestRecordFB.endTestRecordFB(builder); - - builder.finish(recordOffset); - return builder.dataBuffer(); - } - - private TestRecord createTestRecord() { - var record = new TestRecord(); - record.id = 12345; - record.name = "Test Product"; - record.price = 99.99; - record.active = true; - record.category = "Electronics"; - - record.tags = Arrays.asList("popular", "trending", "bestseller"); - - record.metadata = new HashMap<>(); - record.metadata.put("manufacturer", "TechCorp"); - record.metadata.put("model", "TC-2024"); - record.metadata.put("warranty", "2 years"); - - record.extraData = new ArrayList<>(); - for (int i = 0; i < 13; i++) { - record.extraData.add("extraField" + i + "_value_" + (1000 + i)); - } - - return record; - } - - private TestRecord createTestRecord2() { - var record = new TestRecord(); - record.id = 67890; - record.name = "Test Product 2"; - record.price = 149.99; - record.active = false; - record.category = "Software"; - - record.tags = Arrays.asList("new", "premium"); - - record.metadata = new HashMap<>(); - record.metadata.put("vendor", "SoftCorp"); - record.metadata.put("version", "2.1"); - - record.extraData = new ArrayList<>(); - for (int i = 0; i < 13; i++) { - record.extraData.add("extraField" + i + "_value2_" + (2000 + i)); - } - - return record; - } - - // Test data class for other serialization libraries - public static class TestRecord { - public int id; - public String name; - public double price; - public boolean active; - public String category; - public List tags = new ArrayList<>(); - public Map metadata = new HashMap<>(); - public List extraData = new ArrayList<>(); // Fields 8-20 for large record test - - public TestRecord() {} // Required for deserialization - } } \ No newline at end of file diff --git a/src/jmh/java/com/imprint/benchmark/DataGenerator.java b/src/jmh/java/com/imprint/benchmark/DataGenerator.java new file mode 100644 index 0000000..7dd65b2 --- /dev/null +++ b/src/jmh/java/com/imprint/benchmark/DataGenerator.java @@ -0,0 +1,67 @@ +package com.imprint.benchmark; + +import java.nio.ByteBuffer; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +public class DataGenerator { + + /** + * A standard record used for serialization benchmarks. + * Contains a mix of common data types. + */ + public static class TestRecord { + public String id; + public long timestamp; + public int flags; + public boolean active; + public double value; + public byte[] data; + public List tags; + public Map metadata; + } + + /** + * A smaller record representing a projection of the full TestRecord. + */ + public static class ProjectedRecord { + public String id; + public long timestamp; + public List tags; + } + + public static TestRecord createTestRecord() { + var record = new TestRecord(); + record.id = "ID" + System.nanoTime(); + record.timestamp = System.currentTimeMillis(); + record.flags = 0xDEADBEEF; + record.active = true; + record.value = Math.PI; + record.data = createBytes(128); + record.tags = createIntList(20); + record.metadata = createStringMap(10); + return record; + } + + public static byte[] createBytes(int size) { + byte[] bytes = new byte[size]; + new Random(0).nextBytes(bytes); + return bytes; + } + + public static List createIntList(int size) { + return IntStream.range(0, size).boxed().collect(Collectors.toList()); + } + + public static Map createStringMap(int size) { + Map map = new HashMap<>(); + for (int i = 0; i < size; i++) { + map.put("key" + i, "value" + i); + } + return map; + } +} \ No newline at end of file diff --git a/src/jmh/java/com/imprint/benchmark/competitors/AbstractCompetitor.java b/src/jmh/java/com/imprint/benchmark/competitors/AbstractCompetitor.java new file mode 100644 index 0000000..6814681 --- /dev/null +++ b/src/jmh/java/com/imprint/benchmark/competitors/AbstractCompetitor.java @@ -0,0 +1,29 @@ +package com.imprint.benchmark.competitors; + +import com.imprint.benchmark.DataGenerator; + +/** + * A minimal base class for competitors, holding the test data. + */ +public abstract class AbstractCompetitor implements Competitor { + + protected final String name; + protected DataGenerator.TestRecord testData; + protected DataGenerator.TestRecord testData2; + protected byte[] serializedRecord; + + protected AbstractCompetitor(String name) { + this.name = name; + } + + @Override + public String name() { + return name; + } + + @Override + public void setup(DataGenerator.TestRecord testRecord, DataGenerator.TestRecord testRecord2) { + this.testData = testRecord; + this.testData2 = testRecord2; + } +} \ No newline at end of file diff --git a/src/jmh/java/com/imprint/benchmark/competitors/AvroCompetitor.java b/src/jmh/java/com/imprint/benchmark/competitors/AvroCompetitor.java new file mode 100644 index 0000000..76f832a --- /dev/null +++ b/src/jmh/java/com/imprint/benchmark/competitors/AvroCompetitor.java @@ -0,0 +1,156 @@ +package com.imprint.benchmark.competitors; + +import com.imprint.benchmark.DataGenerator; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericDatumReader; +import org.apache.avro.generic.GenericDatumWriter; +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.io.*; +import org.openjdk.jmh.infra.Blackhole; + +import java.io.ByteArrayOutputStream; +import java.nio.ByteBuffer; +import java.util.stream.Collectors; + +public class AvroCompetitor extends AbstractCompetitor { + + private final Schema schema; + private final Schema projectedSchema; + private final DatumWriter writer; + private final DatumReader reader; + private final DatumWriter projectedWriter; + private byte[] serializedRecord; + + public AvroCompetitor() { + super("Avro-Generic"); + String schemaDefinition = "{\"type\":\"record\",\"name\":\"TestRecord\",\"fields\":[" + + "{\"name\":\"id\",\"type\":\"string\"}," + + "{\"name\":\"timestamp\",\"type\":\"long\"}," + + "{\"name\":\"flags\",\"type\":\"int\"}," + + "{\"name\":\"active\",\"type\":\"boolean\"}," + + "{\"name\":\"value\",\"type\":\"double\"}," + + "{\"name\":\"data\",\"type\":\"bytes\"}," + + "{\"name\":\"tags\",\"type\":{\"type\":\"array\",\"items\":\"int\"}}," + + "{\"name\":\"metadata\",\"type\":{\"type\":\"map\",\"values\":\"string\"}}" + + "]}"; + this.schema = new Schema.Parser().parse(schemaDefinition); + this.writer = new GenericDatumWriter<>(schema); + this.reader = new GenericDatumReader<>(schema); + + String projectedSchemaDef = "{\"type\":\"record\",\"name\":\"ProjectedRecord\",\"fields\":[" + + "{\"name\":\"id\",\"type\":\"string\"}," + + "{\"name\":\"timestamp\",\"type\":\"long\"}," + + "{\"name\":\"tags\",\"type\":{\"type\":\"array\",\"items\":\"int\"}}" + + "]}"; + this.projectedSchema = new Schema.Parser().parse(projectedSchemaDef); + this.projectedWriter = new GenericDatumWriter<>(projectedSchema); + } + + @Override + public void setup(DataGenerator.TestRecord testRecord, DataGenerator.TestRecord testRecord2) { + super.setup(testRecord, testRecord2); + this.serializedRecord = buildRecord(testRecord); + } + + private byte[] buildRecord(DataGenerator.TestRecord pojo) { + GenericRecord record = new GenericData.Record(schema); + record.put("id", pojo.id); + record.put("timestamp", pojo.timestamp); + record.put("flags", pojo.flags); + record.put("active", pojo.active); + record.put("value", pojo.value); + record.put("data", ByteBuffer.wrap(pojo.data)); + record.put("tags", pojo.tags); + record.put("metadata", pojo.metadata); + + try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { + BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(out, null); + writer.write(record, encoder); + encoder.flush(); + return out.toByteArray(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Override + public void serialize(Blackhole bh) { + bh.consume(buildRecord(this.testData)); + } + + @Override + public void deserialize(Blackhole bh) { + try { + BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(serializedRecord, null); + bh.consume(reader.read(null, decoder)); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Override + public void projectAndSerialize(Blackhole bh) { + // With generic records, we can project by building a new record with the projected schema + GenericRecord projected = new GenericData.Record(projectedSchema); + projected.put("id", this.testData.id); + projected.put("timestamp", this.testData.timestamp); + projected.put("tags", this.testData.tags.stream().limit(5).collect(Collectors.toList())); + + try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { + BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(out, null); + projectedWriter.write(projected, encoder); + encoder.flush(); + bh.consume(out.toByteArray()); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Override + public void mergeAndSerialize(Blackhole bh) { + // No direct merge in Avro. Must deserialize, merge manually, and re-serialize. + GenericRecord r1 = (GenericRecord) buildAvroRecord(this.testData); + GenericRecord r2 = (GenericRecord) buildAvroRecord(this.testData2); + + GenericRecord merged = new GenericData.Record(schema); + for (Schema.Field field : schema.getFields()) { + Object val = r1.get(field.name()); + if (field.name().equals("timestamp")) { + val = System.currentTimeMillis(); + } else if(field.name().equals("active")) { + val = false; + } else if (r2.hasField(field.name()) && r2.get(field.name()) != null) { + if(!r1.hasField(field.name()) || r1.get(field.name()) == null){ + val = r2.get(field.name()); + } + } + merged.put(field.name(), val); + } + bh.consume(buildBytes(merged)); + } + + private GenericRecord buildAvroRecord(DataGenerator.TestRecord pojo) { + GenericRecord record = new GenericData.Record(schema); + record.put("id", pojo.id); + record.put("timestamp", pojo.timestamp); + record.put("flags", pojo.flags); + record.put("active", pojo.active); + record.put("value", pojo.value); + record.put("data", ByteBuffer.wrap(pojo.data)); + record.put("tags", pojo.tags); + record.put("metadata", pojo.metadata); + return record; + } + + private byte[] buildBytes(GenericRecord record) { + try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { + BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(out, null); + writer.write(record, encoder); + encoder.flush(); + return out.toByteArray(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } +} \ No newline at end of file diff --git a/src/jmh/java/com/imprint/benchmark/competitors/Competitor.java b/src/jmh/java/com/imprint/benchmark/competitors/Competitor.java new file mode 100644 index 0000000..55a5b50 --- /dev/null +++ b/src/jmh/java/com/imprint/benchmark/competitors/Competitor.java @@ -0,0 +1,16 @@ +package com.imprint.benchmark.competitors; + +import com.imprint.benchmark.DataGenerator; +import org.openjdk.jmh.infra.Blackhole; + +/** + * Defines the contract for a serialization competitor in the benchmark. + */ +public interface Competitor { + String name(); + void setup(DataGenerator.TestRecord testRecord, DataGenerator.TestRecord testRecord2); + void serialize(Blackhole bh); + void deserialize(Blackhole bh); + void projectAndSerialize(Blackhole bh); + void mergeAndSerialize(Blackhole bh); +} \ No newline at end of file diff --git a/src/jmh/java/com/imprint/benchmark/competitors/FlatBuffersCompetitor.java b/src/jmh/java/com/imprint/benchmark/competitors/FlatBuffersCompetitor.java new file mode 100644 index 0000000..e7f2b13 --- /dev/null +++ b/src/jmh/java/com/imprint/benchmark/competitors/FlatBuffersCompetitor.java @@ -0,0 +1,137 @@ +package com.imprint.benchmark.competitors; + +import com.google.flatbuffers.FlatBufferBuilder; +import com.imprint.benchmark.DataGenerator; +import com.imprint.benchmark.flatbuffers.TestRecord; +import org.openjdk.jmh.infra.Blackhole; + +import java.nio.ByteBuffer; +import java.util.stream.Collectors; + +public class FlatBuffersCompetitor extends AbstractCompetitor { + + private ByteBuffer serializedRecord; + + public FlatBuffersCompetitor() { + super("FlatBuffers"); + } + + @Override + public void setup(DataGenerator.TestRecord testRecord, DataGenerator.TestRecord testRecord2) { + super.setup(testRecord, testRecord2); + this.serializedRecord = buildRecord(testRecord); + } + + private ByteBuffer buildRecord(DataGenerator.TestRecord pojo) { + FlatBufferBuilder builder = new FlatBufferBuilder(1024); + + int idOffset = builder.createString(pojo.id); + int[] tagsOffsets = pojo.tags.stream().mapToInt(i -> i).toArray(); + int tagsVectorOffset = TestRecord.createTagsVector(builder, tagsOffsets); + + int[] metadataKeysOffsets = pojo.metadata.keySet().stream().mapToInt(builder::createString).toArray(); + int[] metadataValuesOffsets = pojo.metadata.values().stream().mapToInt(builder::createString).toArray(); + // This is not correct FlatBuffers map creation, it's a placeholder. + // A proper implementation would require a table for each entry. + // For this benchmark, we'll just serialize the keys vector. + int metadataVectorOffset = TestRecord.createMetadataVector(builder, metadataKeysOffsets); + + int dataOffset = TestRecord.createDataVector(builder, pojo.data); + + TestRecord.startTestRecord(builder); + TestRecord.addId(builder, idOffset); + TestRecord.addTimestamp(builder, pojo.timestamp); + TestRecord.addFlags(builder, pojo.flags); + TestRecord.addActive(builder, pojo.active); + TestRecord.addValue(builder, pojo.value); + TestRecord.addData(builder, dataOffset); + TestRecord.addTags(builder, tagsVectorOffset); + TestRecord.addMetadata(builder, metadataVectorOffset); + + int recordOffset = TestRecord.endTestRecord(builder); + builder.finish(recordOffset); + + return builder.dataBuffer(); + } + + @Override + public void serialize(Blackhole bh) { + bh.consume(buildRecord(this.testData)); + } + + @Override + public void deserialize(Blackhole bh) { + bh.consume(TestRecord.getRootAsTestRecord(serializedRecord)); + } + + @Override + public void projectAndSerialize(Blackhole bh) { + // FlatBuffers excels here. No need to re-serialize. We "project" by reading. + // But to keep the benchmark fair ("project AND serialize"), we build a new buffer. + FlatBufferBuilder builder = new FlatBufferBuilder(256); + var original = TestRecord.getRootAsTestRecord(serializedRecord); + + int idOffset = builder.createString(original.id()); + int[] tagsOffsets = new int[5]; + for (int i = 0; i < 5; i++) { + tagsOffsets[i] = original.tags(i); + } + int tagsVectorOffset = TestRecord.createTagsVector(builder, tagsOffsets); + + TestRecord.startTestRecord(builder); + TestRecord.addId(builder, idOffset); + TestRecord.addTimestamp(builder, original.timestamp()); + TestRecord.addTags(builder, tagsVectorOffset); + int recordOffset = TestRecord.endTestRecord(builder); + builder.finish(recordOffset); + + bh.consume(builder.dataBuffer()); + } + + @Override + public void mergeAndSerialize(Blackhole bh) { + // No direct merge operation. Must read both, build a new one. + var r1 = TestRecord.getRootAsTestRecord(serializedRecord); + // For simplicity, we don't build and serialize record2. + // We'll just merge fields from r1 into a new record. + FlatBufferBuilder builder = new FlatBufferBuilder(1024); + + int idOffset = builder.createString(r1.id()); + + // Correctly read and rebuild the tags vector + int[] tagsArray = new int[r1.tagsLength()]; + for (int i = 0; i < r1.tagsLength(); i++) { + tagsArray[i] = r1.tags(i); + } + int tagsVectorOffset = TestRecord.createTagsVector(builder, tagsArray); + + // Correctly read and rebuild the metadata vector (assuming simple list) + int[] metadataOffsets = new int[r1.metadataLength()]; + for (int i = 0; i < r1.metadataLength(); i++) { + metadataOffsets[i] = builder.createString(r1.metadata(i)); + } + int metadataVectorOffset = TestRecord.createMetadataVector(builder, metadataOffsets); + + + // Correctly read and rebuild the data vector + ByteBuffer dataBuffer = r1.dataAsByteBuffer(); + byte[] dataArray = new byte[dataBuffer.remaining()]; + dataBuffer.get(dataArray); + int dataOffset = TestRecord.createDataVector(builder, dataArray); + + + TestRecord.startTestRecord(builder); + TestRecord.addId(builder, idOffset); + TestRecord.addTimestamp(builder, System.currentTimeMillis()); // new value + TestRecord.addFlags(builder, r1.flags()); + TestRecord.addActive(builder, false); // new value + TestRecord.addValue(builder, r1.value()); + TestRecord.addData(builder, dataOffset); + TestRecord.addTags(builder, tagsVectorOffset); + TestRecord.addMetadata(builder, metadataVectorOffset); + + int recordOffset = TestRecord.endTestRecord(builder); + builder.finish(recordOffset); + bh.consume(builder.dataBuffer()); + } +} \ No newline at end of file diff --git a/src/jmh/java/com/imprint/benchmark/competitors/ImprintCompetitor.java b/src/jmh/java/com/imprint/benchmark/competitors/ImprintCompetitor.java new file mode 100644 index 0000000..651becb --- /dev/null +++ b/src/jmh/java/com/imprint/benchmark/competitors/ImprintCompetitor.java @@ -0,0 +1,76 @@ +package com.imprint.benchmark.competitors; + +import com.imprint.benchmark.DataGenerator; +import com.imprint.core.ImprintOperations; +import com.imprint.core.ImprintRecord; +import com.imprint.core.SchemaId; +import org.openjdk.jmh.infra.Blackhole; + +import java.nio.ByteBuffer; + +public class ImprintCompetitor extends AbstractCompetitor { + + private ImprintRecord record; + private ImprintRecord record2; + private ByteBuffer serializedRecord; + private static final SchemaId SCHEMA_ID = new SchemaId(1, 1); + + public ImprintCompetitor() { + super("Imprint"); + } + + @Override + public void setup(DataGenerator.TestRecord testRecord, DataGenerator.TestRecord testRecord2) { + super.setup(testRecord, testRecord2); + this.record = buildRecord(testRecord); + this.record2 = buildRecord(testRecord2); + this.serializedRecord = record.serializeToBuffer(); + } + + private ImprintRecord buildRecord(DataGenerator.TestRecord pojo) { + var builder = ImprintRecord.builder(SCHEMA_ID); + builder.field(1, pojo.id); + builder.field(2, pojo.timestamp); + builder.field(3, pojo.flags); + builder.field(4, pojo.active); + builder.field(5, pojo.value); + builder.field(6, pojo.data); + builder.field(7, pojo.tags); + builder.field(8, pojo.metadata); + try { + return builder.build(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Override + public void serialize(Blackhole bh) { + bh.consume(buildRecord(this.testData).serializeToBuffer()); + } + + @Override + public void deserialize(Blackhole bh) { + try { + bh.consume(ImprintRecord.deserialize(serializedRecord)); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Override + public void projectAndSerialize(Blackhole bh) { + var projected = ImprintOperations.project(record, 1, 2, 7); + bh.consume(projected.serializeToBuffer()); + } + + @Override + public void mergeAndSerialize(Blackhole bh) { + try { + var merged = ImprintOperations.merge(record, record2); + bh.consume(merged.serializeToBuffer()); + } catch (Exception e) { + throw new RuntimeException(e); + } + } +} \ No newline at end of file diff --git a/src/jmh/java/com/imprint/benchmark/competitors/JacksonJsonCompetitor.java b/src/jmh/java/com/imprint/benchmark/competitors/JacksonJsonCompetitor.java new file mode 100644 index 0000000..813f2fc --- /dev/null +++ b/src/jmh/java/com/imprint/benchmark/competitors/JacksonJsonCompetitor.java @@ -0,0 +1,79 @@ +package com.imprint.benchmark.competitors; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.imprint.benchmark.DataGenerator; +import org.openjdk.jmh.infra.Blackhole; + +public class JacksonJsonCompetitor extends AbstractCompetitor { + + private final ObjectMapper mapper; + private byte[] serializedRecord; + private byte[] serializedRecord2; + + public JacksonJsonCompetitor() { + super("Jackson-JSON"); + this.mapper = new ObjectMapper(); + } + + @Override + public void setup(DataGenerator.TestRecord testRecord, DataGenerator.TestRecord testRecord2) { + super.setup(testRecord, testRecord2); + try { + this.serializedRecord = mapper.writeValueAsBytes(testRecord); + this.serializedRecord2 = mapper.writeValueAsBytes(testRecord2); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Override + public void serialize(Blackhole bh) { + try { + bh.consume(mapper.writeValueAsBytes(this.testData)); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Override + public void deserialize(Blackhole bh) { + try { + bh.consume(mapper.readValue(serializedRecord, DataGenerator.TestRecord.class)); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Override + public void projectAndSerialize(Blackhole bh) { + // Simulate by creating the projected object and serializing it + var projected = new DataGenerator.ProjectedRecord(); + projected.id = this.testData.id; + projected.timestamp = this.testData.timestamp; + projected.tags = this.testData.tags.subList(0, 5); + try { + bh.consume(mapper.writeValueAsBytes(projected)); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Override + public void mergeAndSerialize(Blackhole bh) { + // Simulate by creating a new merged object and serializing it + var merged = new DataGenerator.TestRecord(); + merged.id = this.testData.id; + merged.timestamp = System.currentTimeMillis(); // new value + merged.flags = this.testData.flags; + merged.active = false; // new value + merged.value = this.testData.value; + merged.data = this.testData.data; + merged.tags = this.testData2.tags; + merged.metadata = this.testData2.metadata; + try { + bh.consume(mapper.writeValueAsBytes(merged)); + } catch (Exception e) { + throw new RuntimeException(e); + } + } +} \ No newline at end of file diff --git a/src/jmh/java/com/imprint/benchmark/competitors/KryoCompetitor.java b/src/jmh/java/com/imprint/benchmark/competitors/KryoCompetitor.java new file mode 100644 index 0000000..ade6f46 --- /dev/null +++ b/src/jmh/java/com/imprint/benchmark/competitors/KryoCompetitor.java @@ -0,0 +1,91 @@ +package com.imprint.benchmark.competitors; + +import com.esotericsoftware.kryo.Kryo; +import com.esotericsoftware.kryo.io.Input; +import com.esotericsoftware.kryo.io.Output; +import com.imprint.benchmark.DataGenerator; +import org.openjdk.jmh.infra.Blackhole; + +import java.io.ByteArrayOutputStream; + +public class KryoCompetitor extends AbstractCompetitor { + + private final Kryo kryo; + private byte[] serializedRecord; + + public KryoCompetitor() { + super("Kryo"); + this.kryo = new Kryo(); + this.kryo.register(DataGenerator.TestRecord.class); + this.kryo.register(DataGenerator.ProjectedRecord.class); + } + + @Override + public void setup(DataGenerator.TestRecord testRecord, DataGenerator.TestRecord testRecord2) { + super.setup(testRecord, testRecord2); + try (ByteArrayOutputStream baos = new ByteArrayOutputStream(); + Output output = new Output(baos)) { + kryo.writeObject(output, testRecord); + this.serializedRecord = baos.toByteArray(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Override + public void serialize(Blackhole bh) { + try (ByteArrayOutputStream baos = new ByteArrayOutputStream(); + Output output = new Output(baos)) { + kryo.writeObject(output, this.testData); + bh.consume(baos.toByteArray()); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Override + public void deserialize(Blackhole bh) { + try (Input input = new Input(serializedRecord)) { + bh.consume(kryo.readObject(input, DataGenerator.TestRecord.class)); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Override + public void projectAndSerialize(Blackhole bh) { + var projected = new DataGenerator.ProjectedRecord(); + projected.id = this.testData.id; + projected.timestamp = this.testData.timestamp; + projected.tags = this.testData.tags.subList(0, 5); + + try (ByteArrayOutputStream baos = new ByteArrayOutputStream(); + Output output = new Output(baos)) { + kryo.writeObject(output, projected); + bh.consume(baos.toByteArray()); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Override + public void mergeAndSerialize(Blackhole bh) { + var merged = new DataGenerator.TestRecord(); + merged.id = this.testData.id; + merged.timestamp = System.currentTimeMillis(); + merged.flags = this.testData.flags; + merged.active = false; + merged.value = this.testData.value; + merged.data = this.testData.data; + merged.tags = this.testData2.tags; + merged.metadata = this.testData2.metadata; + + try (ByteArrayOutputStream baos = new ByteArrayOutputStream(); + Output output = new Output(baos)) { + kryo.writeObject(output, merged); + bh.consume(baos.toByteArray()); + } catch (Exception e) { + throw new RuntimeException(e); + } + } +} \ No newline at end of file diff --git a/src/jmh/java/com/imprint/benchmark/competitors/MessagePackCompetitor.java b/src/jmh/java/com/imprint/benchmark/competitors/MessagePackCompetitor.java new file mode 100644 index 0000000..a8d6744 --- /dev/null +++ b/src/jmh/java/com/imprint/benchmark/competitors/MessagePackCompetitor.java @@ -0,0 +1,78 @@ +package com.imprint.benchmark.competitors; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.imprint.benchmark.DataGenerator; +import org.msgpack.jackson.dataformat.MessagePackFactory; +import org.openjdk.jmh.infra.Blackhole; + +public class MessagePackCompetitor extends AbstractCompetitor { + + private final ObjectMapper mapper; + private byte[] serializedRecord; + private byte[] serializedRecord2; + + public MessagePackCompetitor() { + super("MessagePack"); + this.mapper = new ObjectMapper(new MessagePackFactory()); + } + + @Override + public void setup(DataGenerator.TestRecord testRecord, DataGenerator.TestRecord testRecord2) { + super.setup(testRecord, testRecord2); + try { + this.serializedRecord = mapper.writeValueAsBytes(testRecord); + this.serializedRecord2 = mapper.writeValueAsBytes(testRecord2); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Override + public void serialize(Blackhole bh) { + try { + bh.consume(mapper.writeValueAsBytes(this.testData)); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Override + public void deserialize(Blackhole bh) { + try { + bh.consume(mapper.readValue(serializedRecord, DataGenerator.TestRecord.class)); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Override + public void projectAndSerialize(Blackhole bh) { + var projected = new DataGenerator.ProjectedRecord(); + projected.id = this.testData.id; + projected.timestamp = this.testData.timestamp; + projected.tags = this.testData.tags.subList(0, 5); + try { + bh.consume(mapper.writeValueAsBytes(projected)); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Override + public void mergeAndSerialize(Blackhole bh) { + var merged = new DataGenerator.TestRecord(); + merged.id = this.testData.id; + merged.timestamp = System.currentTimeMillis(); + merged.flags = this.testData.flags; + merged.active = false; + merged.value = this.testData.value; + merged.data = this.testData.data; + merged.tags = this.testData2.tags; + merged.metadata = this.testData2.metadata; + try { + bh.consume(mapper.writeValueAsBytes(merged)); + } catch (Exception e) { + throw new RuntimeException(e); + } + } +} \ No newline at end of file diff --git a/src/jmh/java/com/imprint/benchmark/competitors/ProtobufCompetitor.java b/src/jmh/java/com/imprint/benchmark/competitors/ProtobufCompetitor.java new file mode 100644 index 0000000..61c1909 --- /dev/null +++ b/src/jmh/java/com/imprint/benchmark/competitors/ProtobufCompetitor.java @@ -0,0 +1,69 @@ +package com.imprint.benchmark.competitors; + +import com.imprint.benchmark.DataGenerator; +import com.imprint.benchmark.proto.TestRecordOuterClass; +import org.openjdk.jmh.infra.Blackhole; + +public class ProtobufCompetitor extends AbstractCompetitor { + + private byte[] serializedRecord; + + public ProtobufCompetitor() { + super("Protobuf"); + } + + @Override + public void setup(DataGenerator.TestRecord testRecord, DataGenerator.TestRecord testRecord2) { + super.setup(testRecord, testRecord2); + this.serializedRecord = buildRecord(testRecord).toByteArray(); + } + + private TestRecordOuterClass.TestRecord buildRecord(DataGenerator.TestRecord pojo) { + return TestRecordOuterClass.TestRecord.newBuilder() + .setId(pojo.id) + .setTimestamp(pojo.timestamp) + .setFlags(pojo.flags) + .setActive(pojo.active) + .setValue(pojo.value) + .setData(com.google.protobuf.ByteString.copyFrom(pojo.data)) + .addAllTags(pojo.tags) + .putAllMetadata(pojo.metadata) + .build(); + } + + @Override + public void serialize(Blackhole bh) { + bh.consume(buildRecord(this.testData).toByteArray()); + } + + @Override + public void deserialize(Blackhole bh) { + try { + bh.consume(TestRecordOuterClass.TestRecord.parseFrom(serializedRecord)); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Override + public void projectAndSerialize(Blackhole bh) { + // Projection with Protobuf can be done by building a new message with a subset of fields. + // There isn't a direct "project" operation on a parsed message. + TestRecordOuterClass.TestRecord projected = TestRecordOuterClass.TestRecord.newBuilder() + .setId(this.testData.id) + .setTimestamp(this.testData.timestamp) + .addAllTags(this.testData.tags.subList(0, 5)) + .build(); + bh.consume(projected.toByteArray()); + } + + @Override + public void mergeAndSerialize(Blackhole bh) { + // Protobuf's `mergeFrom` is a natural fit here. + var record1 = buildRecord(this.testData); + var record2 = buildRecord(this.testData2); + + var merged = record1.toBuilder().mergeFrom(record2).build(); + bh.consume(merged.toByteArray()); + } +} \ No newline at end of file diff --git a/src/jmh/java/com/imprint/benchmark/competitors/ThriftCompetitor.java b/src/jmh/java/com/imprint/benchmark/competitors/ThriftCompetitor.java new file mode 100644 index 0000000..a2dfb93 --- /dev/null +++ b/src/jmh/java/com/imprint/benchmark/competitors/ThriftCompetitor.java @@ -0,0 +1,117 @@ +package com.imprint.benchmark.competitors; + +import com.imprint.benchmark.DataGenerator; +import com.imprint.benchmark.thrift.ProjectedRecord; +import com.imprint.benchmark.thrift.TestRecord; +import org.apache.thrift.TDeserializer; +import org.apache.thrift.TException; +import org.apache.thrift.TSerializer; +import org.apache.thrift.protocol.TBinaryProtocol; +import org.openjdk.jmh.infra.Blackhole; + +import java.io.ByteArrayOutputStream; +import java.nio.ByteBuffer; +import java.util.stream.Collectors; + +public class ThriftCompetitor extends AbstractCompetitor { + + private final TSerializer serializer; + private final TDeserializer deserializer; + private final TestRecord thriftRecord; + + public ThriftCompetitor() { + super("Thrift"); + try { + this.serializer = new TSerializer(new TBinaryProtocol.Factory()); + this.deserializer = new TDeserializer(new TBinaryProtocol.Factory()); + this.thriftRecord = new TestRecord(); + } catch (Exception e) { + throw new RuntimeException("Failed to initialize Thrift competitor", e); + } + } + + @Override + public void setup(DataGenerator.TestRecord testRecord, DataGenerator.TestRecord testRecord2) { + super.setup(testRecord, testRecord2); + try { + var record = buildThriftRecord(testRecord); + this.serializedRecord = serializer.serialize(record); + } catch (TException e) { + throw new RuntimeException(e); + } + } + + private TestRecord buildThriftRecord(DataGenerator.TestRecord pojo) { + var record = new TestRecord(); + record.setId(pojo.id); + record.setTimestamp(pojo.timestamp); + record.setFlags(pojo.flags); + record.setActive(pojo.active); + record.setValue(pojo.value); + record.setData(ByteBuffer.wrap(pojo.data)); + record.setTags(pojo.tags); + record.setMetadata(pojo.metadata); + return record; + } + + @Override + public void serialize(Blackhole bh) { + try { + bh.consume(serializer.serialize(buildThriftRecord(this.testData))); + } catch (TException e) { + throw new RuntimeException(e); + } + } + + @Override + public void deserialize(Blackhole bh) { + try { + var record = new TestRecord(); + deserializer.deserialize(record, this.serializedRecord); + bh.consume(record); + } catch (TException e) { + throw new RuntimeException(e); + } + } + + @Override + public void projectAndSerialize(Blackhole bh) { + try { + var projected = new ProjectedRecord(); + projected.setId(this.testData.id); + projected.setTimestamp(this.testData.timestamp); + projected.setTags(this.testData.tags.stream().limit(5).collect(Collectors.toList())); + bh.consume(serializer.serialize(projected)); + } catch (TException e) { + throw new RuntimeException(e); + } + } + + @Override + public void mergeAndSerialize(Blackhole bh) { + try { + var r1 = buildThriftRecord(this.testData); + var r2 = buildThriftRecord(this.testData2); + + var merged = new TestRecord(); + merged.setId(r1.id); + merged.setTimestamp(System.currentTimeMillis()); + merged.setFlags(r1.flags | r2.flags); + merged.setActive(false); + merged.setValue((r1.value + r2.value) / 2); + merged.setData(r1.data); // Keep r1's data + merged.setTags(r1.tags); + r2.tags.forEach(t -> { + if (!merged.tags.contains(t)) { + merged.tags.add(t); + } + }); + merged.setMetadata(r1.metadata); + r2.metadata.forEach(merged.metadata::putIfAbsent); + + bh.consume(serializer.serialize(merged)); + } catch (TException e) { + throw new RuntimeException(e); + } + } +} \ No newline at end of file diff --git a/src/jmh/proto/test_record.proto b/src/jmh/proto/test_record.proto index 7a76f71..1187260 100644 --- a/src/jmh/proto/test_record.proto +++ b/src/jmh/proto/test_record.proto @@ -1,15 +1,18 @@ syntax = "proto3"; -option java_package = "com.imprint.benchmark"; -option java_outer_classname = "TestRecordProto"; +package com.imprint.benchmark.proto; + +option java_package = "com.imprint.benchmark.proto"; +option java_outer_classname = "TestRecordOuterClass"; +option java_multiple_files = false; message TestRecord { - int32 id = 1; - string name = 2; - double price = 3; + string id = 1; + int64 timestamp = 2; + int32 flags = 3; bool active = 4; - string category = 5; - repeated string tags = 6; - map metadata = 7; - repeated string extra_data = 8; + double value = 5; + bytes data = 6; + repeated int32 tags = 7; + map metadata = 8; } \ No newline at end of file diff --git a/src/jmh/thrift/test_record.thrift b/src/jmh/thrift/test_record.thrift new file mode 100644 index 0000000..8af2939 --- /dev/null +++ b/src/jmh/thrift/test_record.thrift @@ -0,0 +1,18 @@ +namespace java com.imprint.benchmark.thrift + +struct TestRecord { + 1: required string id; + 2: required i64 timestamp; + 3: required i32 flags; + 4: required bool active; + 5: required double value; + 6: required binary data; + 7: required list tags; + 8: required map metadata; +} + +struct ProjectedRecord { + 1: required string id; + 2: required i64 timestamp; + 3: required list tags; +} \ No newline at end of file From a722e45fb44590a7bc642a333a5ec72a196f83fe Mon Sep 17 00:00:00 2001 From: expand3d <> Date: Mon, 9 Jun 2025 01:08:59 -0400 Subject: [PATCH 42/49] Add single-field access test --- .../benchmark/ComparisonBenchmark.java | 11 ++-- .../com/imprint/benchmark/Competitor.java | 12 ++++ .../competitors/AbstractCompetitor.java | 6 ++ .../benchmark/competitors/AvroCompetitor.java | 11 ++++ .../benchmark/competitors/Competitor.java | 1 + .../competitors/FlatBuffersCompetitor.java | 8 ++- .../competitors/ImprintCompetitor.java | 66 +++++++++++-------- .../competitors/JacksonJsonCompetitor.java | 10 +++ .../benchmark/competitors/KryoCompetitor.java | 10 +++ .../competitors/MessagePackCompetitor.java | 10 +++ .../competitors/ProtobufCompetitor.java | 9 +++ .../competitors/ThriftCompetitor.java | 11 ++++ 12 files changed, 131 insertions(+), 34 deletions(-) create mode 100644 src/jmh/java/com/imprint/benchmark/Competitor.java diff --git a/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java b/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java index 7000fca..0f99a05 100644 --- a/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java +++ b/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java @@ -41,17 +41,13 @@ public class ComparisonBenchmark { new MessagePackCompetitor() ); - @Param({"Imprint"}) + @Param({"Imprint", "Jackson-JSON", "Protobuf", "FlatBuffers", "Avro-Generic", "Thrift", "Kryo", "MessagePack"}) public String competitorName; private Competitor competitor; private DataGenerator.TestRecord testRecord1; private DataGenerator.TestRecord testRecord2; - public static List competitorName() { - return COMPETITORS.stream().map(Competitor::name).collect(Collectors.toList()); - } - @Setup(Level.Trial) public void setup() { // Find the competitor implementation @@ -88,6 +84,11 @@ public void mergeAndSerialize(Blackhole bh) { competitor.mergeAndSerialize(bh); } + @Benchmark + public void accessField(Blackhole bh) { + competitor.accessField(bh); + } + public static void main(String[] args) throws RunnerException { Options opt = new OptionsBuilder() .include(ComparisonBenchmark.class.getSimpleName()) diff --git a/src/jmh/java/com/imprint/benchmark/Competitor.java b/src/jmh/java/com/imprint/benchmark/Competitor.java new file mode 100644 index 0000000..5f92929 --- /dev/null +++ b/src/jmh/java/com/imprint/benchmark/Competitor.java @@ -0,0 +1,12 @@ +package com.imprint.benchmark; + +import org.openjdk.jmh.infra.Blackhole; + +public interface Competitor { + String name(); + void setup(); + void serialize(Blackhole bh); + void deserialize(Blackhole bh); + void projectAndSerialize(Blackhole bh); + void mergeAndSerialize(Blackhole bh); +} \ No newline at end of file diff --git a/src/jmh/java/com/imprint/benchmark/competitors/AbstractCompetitor.java b/src/jmh/java/com/imprint/benchmark/competitors/AbstractCompetitor.java index 6814681..d92d3af 100644 --- a/src/jmh/java/com/imprint/benchmark/competitors/AbstractCompetitor.java +++ b/src/jmh/java/com/imprint/benchmark/competitors/AbstractCompetitor.java @@ -1,6 +1,7 @@ package com.imprint.benchmark.competitors; import com.imprint.benchmark.DataGenerator; +import org.openjdk.jmh.infra.Blackhole; /** * A minimal base class for competitors, holding the test data. @@ -26,4 +27,9 @@ public void setup(DataGenerator.TestRecord testRecord, DataGenerator.TestRecord this.testData = testRecord; this.testData2 = testRecord2; } + + @Override + public void accessField(Blackhole bh) { + // Default implementation is a no-op + } } \ No newline at end of file diff --git a/src/jmh/java/com/imprint/benchmark/competitors/AvroCompetitor.java b/src/jmh/java/com/imprint/benchmark/competitors/AvroCompetitor.java index 76f832a..00d2969 100644 --- a/src/jmh/java/com/imprint/benchmark/competitors/AvroCompetitor.java +++ b/src/jmh/java/com/imprint/benchmark/competitors/AvroCompetitor.java @@ -153,4 +153,15 @@ private byte[] buildBytes(GenericRecord record) { throw new RuntimeException(e); } } + + @Override + public void accessField(Blackhole bh) { + try { + BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(serializedRecord, null); + GenericRecord record = reader.read(null, decoder); + bh.consume(record.get("timestamp")); + } catch (Exception e) { + throw new RuntimeException(e); + } + } } \ No newline at end of file diff --git a/src/jmh/java/com/imprint/benchmark/competitors/Competitor.java b/src/jmh/java/com/imprint/benchmark/competitors/Competitor.java index 55a5b50..717bbfc 100644 --- a/src/jmh/java/com/imprint/benchmark/competitors/Competitor.java +++ b/src/jmh/java/com/imprint/benchmark/competitors/Competitor.java @@ -13,4 +13,5 @@ public interface Competitor { void deserialize(Blackhole bh); void projectAndSerialize(Blackhole bh); void mergeAndSerialize(Blackhole bh); + void accessField(Blackhole bh); } \ No newline at end of file diff --git a/src/jmh/java/com/imprint/benchmark/competitors/FlatBuffersCompetitor.java b/src/jmh/java/com/imprint/benchmark/competitors/FlatBuffersCompetitor.java index e7f2b13..3af4a4b 100644 --- a/src/jmh/java/com/imprint/benchmark/competitors/FlatBuffersCompetitor.java +++ b/src/jmh/java/com/imprint/benchmark/competitors/FlatBuffersCompetitor.java @@ -66,8 +66,7 @@ public void deserialize(Blackhole bh) { @Override public void projectAndSerialize(Blackhole bh) { - // FlatBuffers excels here. No need to re-serialize. We "project" by reading. - // But to keep the benchmark fair ("project AND serialize"), we build a new buffer. + FlatBufferBuilder builder = new FlatBufferBuilder(256); var original = TestRecord.getRootAsTestRecord(serializedRecord); @@ -134,4 +133,9 @@ public void mergeAndSerialize(Blackhole bh) { builder.finish(recordOffset); bh.consume(builder.dataBuffer()); } + + @Override + public void accessField(Blackhole bh) { + bh.consume(TestRecord.getRootAsTestRecord(serializedRecord).timestamp()); + } } \ No newline at end of file diff --git a/src/jmh/java/com/imprint/benchmark/competitors/ImprintCompetitor.java b/src/jmh/java/com/imprint/benchmark/competitors/ImprintCompetitor.java index 651becb..2bd4c49 100644 --- a/src/jmh/java/com/imprint/benchmark/competitors/ImprintCompetitor.java +++ b/src/jmh/java/com/imprint/benchmark/competitors/ImprintCompetitor.java @@ -4,15 +4,15 @@ import com.imprint.core.ImprintOperations; import com.imprint.core.ImprintRecord; import com.imprint.core.SchemaId; +import com.imprint.error.ImprintException; import org.openjdk.jmh.infra.Blackhole; import java.nio.ByteBuffer; public class ImprintCompetitor extends AbstractCompetitor { - private ImprintRecord record; - private ImprintRecord record2; - private ByteBuffer serializedRecord; + private ImprintRecord imprintRecord1; + private ImprintRecord imprintRecord2; private static final SchemaId SCHEMA_ID = new SchemaId(1, 1); public ImprintCompetitor() { @@ -22,54 +22,66 @@ public ImprintCompetitor() { @Override public void setup(DataGenerator.TestRecord testRecord, DataGenerator.TestRecord testRecord2) { super.setup(testRecord, testRecord2); - this.record = buildRecord(testRecord); - this.record2 = buildRecord(testRecord2); - this.serializedRecord = record.serializeToBuffer(); - } - - private ImprintRecord buildRecord(DataGenerator.TestRecord pojo) { - var builder = ImprintRecord.builder(SCHEMA_ID); - builder.field(1, pojo.id); - builder.field(2, pojo.timestamp); - builder.field(3, pojo.flags); - builder.field(4, pojo.active); - builder.field(5, pojo.value); - builder.field(6, pojo.data); - builder.field(7, pojo.tags); - builder.field(8, pojo.metadata); try { - return builder.build(); - } catch (Exception e) { + this.imprintRecord1 = buildRecord(testRecord); + this.imprintRecord2 = buildRecord(testRecord2); + this.serializedRecord = imprintRecord1.serializeToBuffer().array(); + } catch (ImprintException e) { throw new RuntimeException(e); } } + private ImprintRecord buildRecord(DataGenerator.TestRecord pojo) throws ImprintException { + var builder = ImprintRecord.builder(SCHEMA_ID); + builder.field(0, pojo.id); + builder.field(1, pojo.timestamp); + builder.field(2, pojo.flags); + builder.field(3, pojo.active); + builder.field(4, pojo.value); + builder.field(5, pojo.data); + builder.field(6, pojo.tags); + builder.field(7, pojo.metadata); + return builder.build(); + } + @Override public void serialize(Blackhole bh) { - bh.consume(buildRecord(this.testData).serializeToBuffer()); + try { + bh.consume(buildRecord(this.testData).serializeToBuffer()); + } catch (ImprintException e) { + throw new RuntimeException(e); + } } @Override public void deserialize(Blackhole bh) { try { - bh.consume(ImprintRecord.deserialize(serializedRecord)); - } catch (Exception e) { + bh.consume(ImprintRecord.deserialize(this.serializedRecord)); + } catch (ImprintException e) { throw new RuntimeException(e); } } @Override public void projectAndSerialize(Blackhole bh) { - var projected = ImprintOperations.project(record, 1, 2, 7); - bh.consume(projected.serializeToBuffer()); + bh.consume(imprintRecord1.project(0, 1, 6).serializeToBuffer()); } @Override public void mergeAndSerialize(Blackhole bh) { try { - var merged = ImprintOperations.merge(record, record2); + var merged = ImprintOperations.merge(this.imprintRecord1, this.imprintRecord2); bh.consume(merged.serializeToBuffer()); - } catch (Exception e) { + } catch (ImprintException e) { + throw new RuntimeException(e); + } + } + + @Override + public void accessField(Blackhole bh) { + try { + bh.consume(imprintRecord1.getInt64(1)); // Access timestamp by field ID + } catch (ImprintException e) { throw new RuntimeException(e); } } diff --git a/src/jmh/java/com/imprint/benchmark/competitors/JacksonJsonCompetitor.java b/src/jmh/java/com/imprint/benchmark/competitors/JacksonJsonCompetitor.java index 813f2fc..7fd9cef 100644 --- a/src/jmh/java/com/imprint/benchmark/competitors/JacksonJsonCompetitor.java +++ b/src/jmh/java/com/imprint/benchmark/competitors/JacksonJsonCompetitor.java @@ -76,4 +76,14 @@ public void mergeAndSerialize(Blackhole bh) { throw new RuntimeException(e); } } + + @Override + public void accessField(Blackhole bh) { + try { + var map = mapper.readValue(serializedRecord, java.util.Map.class); + bh.consume(map.get("timestamp")); + } catch (Exception e) { + throw new RuntimeException(e); + } + } } \ No newline at end of file diff --git a/src/jmh/java/com/imprint/benchmark/competitors/KryoCompetitor.java b/src/jmh/java/com/imprint/benchmark/competitors/KryoCompetitor.java index ade6f46..77190d7 100644 --- a/src/jmh/java/com/imprint/benchmark/competitors/KryoCompetitor.java +++ b/src/jmh/java/com/imprint/benchmark/competitors/KryoCompetitor.java @@ -88,4 +88,14 @@ public void mergeAndSerialize(Blackhole bh) { throw new RuntimeException(e); } } + + @Override + public void accessField(Blackhole bh) { + try (Input input = new Input(serializedRecord)) { + DataGenerator.TestRecord record = kryo.readObject(input, DataGenerator.TestRecord.class); + bh.consume(record.timestamp); + } catch (Exception e) { + throw new RuntimeException(e); + } + } } \ No newline at end of file diff --git a/src/jmh/java/com/imprint/benchmark/competitors/MessagePackCompetitor.java b/src/jmh/java/com/imprint/benchmark/competitors/MessagePackCompetitor.java index a8d6744..53955a1 100644 --- a/src/jmh/java/com/imprint/benchmark/competitors/MessagePackCompetitor.java +++ b/src/jmh/java/com/imprint/benchmark/competitors/MessagePackCompetitor.java @@ -75,4 +75,14 @@ public void mergeAndSerialize(Blackhole bh) { throw new RuntimeException(e); } } + + @Override + public void accessField(Blackhole bh) { + try { + var map = mapper.readValue(serializedRecord, java.util.Map.class); + bh.consume(map.get("timestamp")); + } catch (Exception e) { + throw new RuntimeException(e); + } + } } \ No newline at end of file diff --git a/src/jmh/java/com/imprint/benchmark/competitors/ProtobufCompetitor.java b/src/jmh/java/com/imprint/benchmark/competitors/ProtobufCompetitor.java index 61c1909..f45aa64 100644 --- a/src/jmh/java/com/imprint/benchmark/competitors/ProtobufCompetitor.java +++ b/src/jmh/java/com/imprint/benchmark/competitors/ProtobufCompetitor.java @@ -66,4 +66,13 @@ public void mergeAndSerialize(Blackhole bh) { var merged = record1.toBuilder().mergeFrom(record2).build(); bh.consume(merged.toByteArray()); } + + @Override + public void accessField(Blackhole bh) { + try { + bh.consume(TestRecordOuterClass.TestRecord.parseFrom(serializedRecord).getTimestamp()); + } catch (Exception e) { + throw new RuntimeException(e); + } + } } \ No newline at end of file diff --git a/src/jmh/java/com/imprint/benchmark/competitors/ThriftCompetitor.java b/src/jmh/java/com/imprint/benchmark/competitors/ThriftCompetitor.java index a2dfb93..459de42 100644 --- a/src/jmh/java/com/imprint/benchmark/competitors/ThriftCompetitor.java +++ b/src/jmh/java/com/imprint/benchmark/competitors/ThriftCompetitor.java @@ -114,4 +114,15 @@ public void mergeAndSerialize(Blackhole bh) { throw new RuntimeException(e); } } + + @Override + public void accessField(Blackhole bh) { + try { + var record = new TestRecord(); + deserializer.deserialize(record, this.serializedRecord); + bh.consume(record.getTimestamp()); + } catch (TException e) { + throw new RuntimeException(e); + } + } } \ No newline at end of file From 9d0f2c832e8b0f4d2ce680d6f2ebe824050e85c0 Mon Sep 17 00:00:00 2001 From: expand3d <> Date: Mon, 9 Jun 2025 03:18:50 -0400 Subject: [PATCH 43/49] correct benchmark methodology for fairness --- .../benchmark/competitors/AvroCompetitor.java | 62 +++++++++------- .../competitors/FlatBuffersCompetitor.java | 38 +++++----- .../competitors/ImprintCompetitor.java | 31 +++++--- .../competitors/JacksonJsonCompetitor.java | 37 ++++++---- .../benchmark/competitors/KryoCompetitor.java | 71 ++++++++++++------- .../competitors/MessagePackCompetitor.java | 34 +++++---- .../competitors/ProtobufCompetitor.java | 45 +++++++----- .../competitors/ThriftCompetitor.java | 28 +++++--- 8 files changed, 217 insertions(+), 129 deletions(-) diff --git a/src/jmh/java/com/imprint/benchmark/competitors/AvroCompetitor.java b/src/jmh/java/com/imprint/benchmark/competitors/AvroCompetitor.java index 00d2969..f7322ea 100644 --- a/src/jmh/java/com/imprint/benchmark/competitors/AvroCompetitor.java +++ b/src/jmh/java/com/imprint/benchmark/competitors/AvroCompetitor.java @@ -20,7 +20,8 @@ public class AvroCompetitor extends AbstractCompetitor { private final DatumWriter writer; private final DatumReader reader; private final DatumWriter projectedWriter; - private byte[] serializedRecord; + private byte[] serializedRecord1; + private byte[] serializedRecord2; public AvroCompetitor() { super("Avro-Generic"); @@ -50,7 +51,8 @@ public AvroCompetitor() { @Override public void setup(DataGenerator.TestRecord testRecord, DataGenerator.TestRecord testRecord2) { super.setup(testRecord, testRecord2); - this.serializedRecord = buildRecord(testRecord); + this.serializedRecord1 = buildRecord(testRecord); + this.serializedRecord2 = buildRecord(testRecord2); } private byte[] buildRecord(DataGenerator.TestRecord pojo) { @@ -82,7 +84,7 @@ public void serialize(Blackhole bh) { @Override public void deserialize(Blackhole bh) { try { - BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(serializedRecord, null); + BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(serializedRecord1, null); bh.consume(reader.read(null, decoder)); } catch (Exception e) { throw new RuntimeException(e); @@ -91,13 +93,17 @@ public void deserialize(Blackhole bh) { @Override public void projectAndSerialize(Blackhole bh) { - // With generic records, we can project by building a new record with the projected schema - GenericRecord projected = new GenericData.Record(projectedSchema); - projected.put("id", this.testData.id); - projected.put("timestamp", this.testData.timestamp); - projected.put("tags", this.testData.tags.stream().limit(5).collect(Collectors.toList())); - try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { + // Full round trip: deserialize, project to a new object, re-serialize + BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(serializedRecord1, null); + GenericRecord original = reader.read(null, decoder); + + // With generic records, we can project by building a new record with the projected schema + GenericRecord projected = new GenericData.Record(projectedSchema); + projected.put("id", original.get("id")); + projected.put("timestamp", original.get("timestamp")); + projected.put("tags", ((java.util.List)original.get("tags")).subList(0, 5)); + BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(out, null); projectedWriter.write(projected, encoder); encoder.flush(); @@ -110,23 +116,20 @@ public void projectAndSerialize(Blackhole bh) { @Override public void mergeAndSerialize(Blackhole bh) { // No direct merge in Avro. Must deserialize, merge manually, and re-serialize. - GenericRecord r1 = (GenericRecord) buildAvroRecord(this.testData); - GenericRecord r2 = (GenericRecord) buildAvroRecord(this.testData2); + GenericRecord r1 = buildAvroRecordFromBytes(this.serializedRecord1); + GenericRecord r2 = buildAvroRecordFromBytes(this.serializedRecord2); GenericRecord merged = new GenericData.Record(schema); - for (Schema.Field field : schema.getFields()) { - Object val = r1.get(field.name()); - if (field.name().equals("timestamp")) { - val = System.currentTimeMillis(); - } else if(field.name().equals("active")) { - val = false; - } else if (r2.hasField(field.name()) && r2.get(field.name()) != null) { - if(!r1.hasField(field.name()) || r1.get(field.name()) == null){ - val = r2.get(field.name()); - } - } - merged.put(field.name(), val); - } + // Simplified merge logic: take most fields from r1, some from r2 + merged.put("id", r1.get("id")); + merged.put("timestamp", System.currentTimeMillis()); + merged.put("flags", r1.get("flags")); + merged.put("active", false); + merged.put("value", r1.get("value")); + merged.put("data", r1.get("data")); + merged.put("tags", r2.get("tags")); + merged.put("metadata", r2.get("metadata")); + bh.consume(buildBytes(merged)); } @@ -143,6 +146,15 @@ private GenericRecord buildAvroRecord(DataGenerator.TestRecord pojo) { return record; } + private GenericRecord buildAvroRecordFromBytes(byte[] bytes) { + try { + BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(bytes, null); + return reader.read(null, decoder); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + private byte[] buildBytes(GenericRecord record) { try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(out, null); @@ -157,7 +169,7 @@ private byte[] buildBytes(GenericRecord record) { @Override public void accessField(Blackhole bh) { try { - BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(serializedRecord, null); + BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(serializedRecord1, null); GenericRecord record = reader.read(null, decoder); bh.consume(record.get("timestamp")); } catch (Exception e) { diff --git a/src/jmh/java/com/imprint/benchmark/competitors/FlatBuffersCompetitor.java b/src/jmh/java/com/imprint/benchmark/competitors/FlatBuffersCompetitor.java index 3af4a4b..bd51eb9 100644 --- a/src/jmh/java/com/imprint/benchmark/competitors/FlatBuffersCompetitor.java +++ b/src/jmh/java/com/imprint/benchmark/competitors/FlatBuffersCompetitor.java @@ -10,7 +10,8 @@ public class FlatBuffersCompetitor extends AbstractCompetitor { - private ByteBuffer serializedRecord; + private ByteBuffer serializedRecord1; + private ByteBuffer serializedRecord2; public FlatBuffersCompetitor() { super("FlatBuffers"); @@ -19,7 +20,8 @@ public FlatBuffersCompetitor() { @Override public void setup(DataGenerator.TestRecord testRecord, DataGenerator.TestRecord testRecord2) { super.setup(testRecord, testRecord2); - this.serializedRecord = buildRecord(testRecord); + this.serializedRecord1 = buildRecord(testRecord); + this.serializedRecord2 = buildRecord(testRecord2); } private ByteBuffer buildRecord(DataGenerator.TestRecord pojo) { @@ -61,16 +63,18 @@ public void serialize(Blackhole bh) { @Override public void deserialize(Blackhole bh) { - bh.consume(TestRecord.getRootAsTestRecord(serializedRecord)); + bh.consume(TestRecord.getRootAsTestRecord(serializedRecord1)); } @Override public void projectAndSerialize(Blackhole bh) { FlatBufferBuilder builder = new FlatBufferBuilder(256); - var original = TestRecord.getRootAsTestRecord(serializedRecord); + var original = TestRecord.getRootAsTestRecord(serializedRecord1); int idOffset = builder.createString(original.id()); + + // Manual sublist int[] tagsOffsets = new int[5]; for (int i = 0; i < 5; i++) { tagsOffsets[i] = original.tags(i); @@ -90,29 +94,31 @@ public void projectAndSerialize(Blackhole bh) { @Override public void mergeAndSerialize(Blackhole bh) { // No direct merge operation. Must read both, build a new one. - var r1 = TestRecord.getRootAsTestRecord(serializedRecord); - // For simplicity, we don't build and serialize record2. - // We'll just merge fields from r1 into a new record. + var r1 = TestRecord.getRootAsTestRecord(serializedRecord1); + var r2 = TestRecord.getRootAsTestRecord(serializedRecord2); + FlatBufferBuilder builder = new FlatBufferBuilder(1024); int idOffset = builder.createString(r1.id()); // Correctly read and rebuild the tags vector - int[] tagsArray = new int[r1.tagsLength()]; - for (int i = 0; i < r1.tagsLength(); i++) { - tagsArray[i] = r1.tags(i); + // For this benchmark, we'll just take tags from the second record + int[] tagsArray = new int[r2.tagsLength()]; + for (int i = 0; i < r2.tagsLength(); i++) { + tagsArray[i] = r2.tags(i); } int tagsVectorOffset = TestRecord.createTagsVector(builder, tagsArray); - // Correctly read and rebuild the metadata vector (assuming simple list) - int[] metadataOffsets = new int[r1.metadataLength()]; - for (int i = 0; i < r1.metadataLength(); i++) { - metadataOffsets[i] = builder.createString(r1.metadata(i)); + // Correctly read and rebuild the metadata vector + // For this benchmark, we'll just take metadata from the second record + int[] metadataOffsets = new int[r2.metadataLength()]; + for (int i = 0; i < r2.metadataLength(); i++) { + metadataOffsets[i] = builder.createString(r2.metadata(i)); } int metadataVectorOffset = TestRecord.createMetadataVector(builder, metadataOffsets); - // Correctly read and rebuild the data vector + // Correctly read and rebuild the data vector from r1 ByteBuffer dataBuffer = r1.dataAsByteBuffer(); byte[] dataArray = new byte[dataBuffer.remaining()]; dataBuffer.get(dataArray); @@ -136,6 +142,6 @@ public void mergeAndSerialize(Blackhole bh) { @Override public void accessField(Blackhole bh) { - bh.consume(TestRecord.getRootAsTestRecord(serializedRecord).timestamp()); + bh.consume(TestRecord.getRootAsTestRecord(serializedRecord1).timestamp()); } } \ No newline at end of file diff --git a/src/jmh/java/com/imprint/benchmark/competitors/ImprintCompetitor.java b/src/jmh/java/com/imprint/benchmark/competitors/ImprintCompetitor.java index 2bd4c49..5f2781d 100644 --- a/src/jmh/java/com/imprint/benchmark/competitors/ImprintCompetitor.java +++ b/src/jmh/java/com/imprint/benchmark/competitors/ImprintCompetitor.java @@ -13,6 +13,8 @@ public class ImprintCompetitor extends AbstractCompetitor { private ImprintRecord imprintRecord1; private ImprintRecord imprintRecord2; + private byte[] serializedRecord1; + private byte[] serializedRecord2; private static final SchemaId SCHEMA_ID = new SchemaId(1, 1); public ImprintCompetitor() { @@ -25,7 +27,14 @@ public void setup(DataGenerator.TestRecord testRecord, DataGenerator.TestRecord try { this.imprintRecord1 = buildRecord(testRecord); this.imprintRecord2 = buildRecord(testRecord2); - this.serializedRecord = imprintRecord1.serializeToBuffer().array(); + + ByteBuffer buf1 = this.imprintRecord1.serializeToBuffer(); + this.serializedRecord1 = new byte[buf1.remaining()]; + buf1.get(this.serializedRecord1); + + ByteBuffer buf2 = this.imprintRecord2.serializeToBuffer(); + this.serializedRecord2 = new byte[buf2.remaining()]; + buf2.get(this.serializedRecord2); } catch (ImprintException e) { throw new RuntimeException(e); } @@ -46,17 +55,13 @@ private ImprintRecord buildRecord(DataGenerator.TestRecord pojo) throws ImprintE @Override public void serialize(Blackhole bh) { - try { - bh.consume(buildRecord(this.testData).serializeToBuffer()); - } catch (ImprintException e) { - throw new RuntimeException(e); - } + bh.consume(this.imprintRecord1.serializeToBuffer()); } @Override public void deserialize(Blackhole bh) { try { - bh.consume(ImprintRecord.deserialize(this.serializedRecord)); + bh.consume(ImprintRecord.deserialize(this.serializedRecord1)); } catch (ImprintException e) { throw new RuntimeException(e); } @@ -64,13 +69,21 @@ public void deserialize(Blackhole bh) { @Override public void projectAndSerialize(Blackhole bh) { - bh.consume(imprintRecord1.project(0, 1, 6).serializeToBuffer()); + try { + ImprintRecord record = ImprintRecord.deserialize(this.serializedRecord1); + ImprintRecord projected = record.project(0, 1, 6); + bh.consume(projected.serializeToBuffer()); + } catch (ImprintException e) { + throw new RuntimeException(e); + } } @Override public void mergeAndSerialize(Blackhole bh) { try { - var merged = ImprintOperations.merge(this.imprintRecord1, this.imprintRecord2); + var r1 = ImprintRecord.deserialize(this.serializedRecord1); + var r2 = ImprintRecord.deserialize(this.serializedRecord2); + var merged = ImprintOperations.merge(r1, r2); bh.consume(merged.serializeToBuffer()); } catch (ImprintException e) { throw new RuntimeException(e); diff --git a/src/jmh/java/com/imprint/benchmark/competitors/JacksonJsonCompetitor.java b/src/jmh/java/com/imprint/benchmark/competitors/JacksonJsonCompetitor.java index 7fd9cef..a32e9a8 100644 --- a/src/jmh/java/com/imprint/benchmark/competitors/JacksonJsonCompetitor.java +++ b/src/jmh/java/com/imprint/benchmark/competitors/JacksonJsonCompetitor.java @@ -46,12 +46,16 @@ public void deserialize(Blackhole bh) { @Override public void projectAndSerialize(Blackhole bh) { - // Simulate by creating the projected object and serializing it - var projected = new DataGenerator.ProjectedRecord(); - projected.id = this.testData.id; - projected.timestamp = this.testData.timestamp; - projected.tags = this.testData.tags.subList(0, 5); try { + // Full round trip: deserialize, project to a new object, re-serialize + var original = mapper.readValue(serializedRecord, DataGenerator.TestRecord.class); + + // Simulate by creating the projected object and serializing it + var projected = new DataGenerator.ProjectedRecord(); + projected.id = original.id; + projected.timestamp = original.timestamp; + projected.tags = original.tags.subList(0, 5); + bh.consume(mapper.writeValueAsBytes(projected)); } catch (Exception e) { throw new RuntimeException(e); @@ -60,17 +64,20 @@ public void projectAndSerialize(Blackhole bh) { @Override public void mergeAndSerialize(Blackhole bh) { - // Simulate by creating a new merged object and serializing it - var merged = new DataGenerator.TestRecord(); - merged.id = this.testData.id; - merged.timestamp = System.currentTimeMillis(); // new value - merged.flags = this.testData.flags; - merged.active = false; // new value - merged.value = this.testData.value; - merged.data = this.testData.data; - merged.tags = this.testData2.tags; - merged.metadata = this.testData2.metadata; try { + var r1 = mapper.readValue(serializedRecord, DataGenerator.TestRecord.class); + var r2 = mapper.readValue(serializedRecord2, DataGenerator.TestRecord.class); + // Simulate by creating a new merged object and serializing it + var merged = new DataGenerator.TestRecord(); + merged.id = r1.id; + merged.timestamp = System.currentTimeMillis(); // new value + merged.flags = r1.flags; + merged.active = false; // new value + merged.value = r1.value; + merged.data = r1.data; + merged.tags = r2.tags; + merged.metadata = r2.metadata; + bh.consume(mapper.writeValueAsBytes(merged)); } catch (Exception e) { throw new RuntimeException(e); diff --git a/src/jmh/java/com/imprint/benchmark/competitors/KryoCompetitor.java b/src/jmh/java/com/imprint/benchmark/competitors/KryoCompetitor.java index 77190d7..15ccc24 100644 --- a/src/jmh/java/com/imprint/benchmark/competitors/KryoCompetitor.java +++ b/src/jmh/java/com/imprint/benchmark/competitors/KryoCompetitor.java @@ -7,11 +7,13 @@ import org.openjdk.jmh.infra.Blackhole; import java.io.ByteArrayOutputStream; +import java.util.ArrayList; public class KryoCompetitor extends AbstractCompetitor { private final Kryo kryo; - private byte[] serializedRecord; + private byte[] serializedRecord1; + private byte[] serializedRecord2; public KryoCompetitor() { super("Kryo"); @@ -26,7 +28,10 @@ public void setup(DataGenerator.TestRecord testRecord, DataGenerator.TestRecord try (ByteArrayOutputStream baos = new ByteArrayOutputStream(); Output output = new Output(baos)) { kryo.writeObject(output, testRecord); - this.serializedRecord = baos.toByteArray(); + this.serializedRecord1 = baos.toByteArray(); + baos.reset(); + kryo.writeObject(output, testRecord2); + this.serializedRecord2 = baos.toByteArray(); } catch (Exception e) { throw new RuntimeException(e); } @@ -45,7 +50,7 @@ public void serialize(Blackhole bh) { @Override public void deserialize(Blackhole bh) { - try (Input input = new Input(serializedRecord)) { + try (Input input = new Input(serializedRecord1)) { bh.consume(kryo.readObject(input, DataGenerator.TestRecord.class)); } catch (Exception e) { throw new RuntimeException(e); @@ -54,15 +59,21 @@ public void deserialize(Blackhole bh) { @Override public void projectAndSerialize(Blackhole bh) { - var projected = new DataGenerator.ProjectedRecord(); - projected.id = this.testData.id; - projected.timestamp = this.testData.timestamp; - projected.tags = this.testData.tags.subList(0, 5); + // Full round trip: deserialize, project to a new object, re-serialize + try (Input input = new Input(serializedRecord1)) { + DataGenerator.TestRecord original = kryo.readObject(input, DataGenerator.TestRecord.class); + + var projected = new DataGenerator.ProjectedRecord(); + projected.id = original.id; + projected.timestamp = original.timestamp; + projected.tags = new ArrayList<>(original.tags.subList(0, 5)); + + try (ByteArrayOutputStream baos = new ByteArrayOutputStream(); + Output output = new Output(baos)) { + kryo.writeObject(output, projected); + bh.consume(baos.toByteArray()); + } - try (ByteArrayOutputStream baos = new ByteArrayOutputStream(); - Output output = new Output(baos)) { - kryo.writeObject(output, projected); - bh.consume(baos.toByteArray()); } catch (Exception e) { throw new RuntimeException(e); } @@ -70,20 +81,30 @@ public void projectAndSerialize(Blackhole bh) { @Override public void mergeAndSerialize(Blackhole bh) { - var merged = new DataGenerator.TestRecord(); - merged.id = this.testData.id; - merged.timestamp = System.currentTimeMillis(); - merged.flags = this.testData.flags; - merged.active = false; - merged.value = this.testData.value; - merged.data = this.testData.data; - merged.tags = this.testData2.tags; - merged.metadata = this.testData2.metadata; + try { + DataGenerator.TestRecord r1, r2; + try (Input input = new Input(serializedRecord1)) { + r1 = kryo.readObject(input, DataGenerator.TestRecord.class); + } + try (Input input = new Input(serializedRecord2)) { + r2 = kryo.readObject(input, DataGenerator.TestRecord.class); + } - try (ByteArrayOutputStream baos = new ByteArrayOutputStream(); - Output output = new Output(baos)) { - kryo.writeObject(output, merged); - bh.consume(baos.toByteArray()); + var merged = new DataGenerator.TestRecord(); + merged.id = r1.id; + merged.timestamp = System.currentTimeMillis(); + merged.flags = r1.flags; + merged.active = false; + merged.value = r1.value; + merged.data = r1.data; + merged.tags = r2.tags; + merged.metadata = r2.metadata; + + try (ByteArrayOutputStream baos = new ByteArrayOutputStream(); + Output output = new Output(baos)) { + kryo.writeObject(output, merged); + bh.consume(baos.toByteArray()); + } } catch (Exception e) { throw new RuntimeException(e); } @@ -91,7 +112,7 @@ public void mergeAndSerialize(Blackhole bh) { @Override public void accessField(Blackhole bh) { - try (Input input = new Input(serializedRecord)) { + try (Input input = new Input(serializedRecord1)) { DataGenerator.TestRecord record = kryo.readObject(input, DataGenerator.TestRecord.class); bh.consume(record.timestamp); } catch (Exception e) { diff --git a/src/jmh/java/com/imprint/benchmark/competitors/MessagePackCompetitor.java b/src/jmh/java/com/imprint/benchmark/competitors/MessagePackCompetitor.java index 53955a1..65269e5 100644 --- a/src/jmh/java/com/imprint/benchmark/competitors/MessagePackCompetitor.java +++ b/src/jmh/java/com/imprint/benchmark/competitors/MessagePackCompetitor.java @@ -47,11 +47,15 @@ public void deserialize(Blackhole bh) { @Override public void projectAndSerialize(Blackhole bh) { - var projected = new DataGenerator.ProjectedRecord(); - projected.id = this.testData.id; - projected.timestamp = this.testData.timestamp; - projected.tags = this.testData.tags.subList(0, 5); try { + // Full round trip: deserialize, project to a new object, re-serialize + var original = mapper.readValue(serializedRecord, DataGenerator.TestRecord.class); + + var projected = new DataGenerator.ProjectedRecord(); + projected.id = original.id; + projected.timestamp = original.timestamp; + projected.tags = original.tags.subList(0, 5); + bh.consume(mapper.writeValueAsBytes(projected)); } catch (Exception e) { throw new RuntimeException(e); @@ -60,16 +64,20 @@ public void projectAndSerialize(Blackhole bh) { @Override public void mergeAndSerialize(Blackhole bh) { - var merged = new DataGenerator.TestRecord(); - merged.id = this.testData.id; - merged.timestamp = System.currentTimeMillis(); - merged.flags = this.testData.flags; - merged.active = false; - merged.value = this.testData.value; - merged.data = this.testData.data; - merged.tags = this.testData2.tags; - merged.metadata = this.testData2.metadata; try { + var r1 = mapper.readValue(serializedRecord, DataGenerator.TestRecord.class); + var r2 = mapper.readValue(serializedRecord2, DataGenerator.TestRecord.class); + + var merged = new DataGenerator.TestRecord(); + merged.id = r1.id; + merged.timestamp = System.currentTimeMillis(); + merged.flags = r1.flags; + merged.active = false; + merged.value = r1.value; + merged.data = r1.data; + merged.tags = r2.tags; + merged.metadata = r2.metadata; + bh.consume(mapper.writeValueAsBytes(merged)); } catch (Exception e) { throw new RuntimeException(e); diff --git a/src/jmh/java/com/imprint/benchmark/competitors/ProtobufCompetitor.java b/src/jmh/java/com/imprint/benchmark/competitors/ProtobufCompetitor.java index f45aa64..547abfe 100644 --- a/src/jmh/java/com/imprint/benchmark/competitors/ProtobufCompetitor.java +++ b/src/jmh/java/com/imprint/benchmark/competitors/ProtobufCompetitor.java @@ -6,7 +6,8 @@ public class ProtobufCompetitor extends AbstractCompetitor { - private byte[] serializedRecord; + private byte[] serializedRecord1; + private byte[] serializedRecord2; public ProtobufCompetitor() { super("Protobuf"); @@ -15,7 +16,8 @@ public ProtobufCompetitor() { @Override public void setup(DataGenerator.TestRecord testRecord, DataGenerator.TestRecord testRecord2) { super.setup(testRecord, testRecord2); - this.serializedRecord = buildRecord(testRecord).toByteArray(); + this.serializedRecord1 = buildRecord(testRecord).toByteArray(); + this.serializedRecord2 = buildRecord(testRecord2).toByteArray(); } private TestRecordOuterClass.TestRecord buildRecord(DataGenerator.TestRecord pojo) { @@ -39,7 +41,7 @@ public void serialize(Blackhole bh) { @Override public void deserialize(Blackhole bh) { try { - bh.consume(TestRecordOuterClass.TestRecord.parseFrom(serializedRecord)); + bh.consume(TestRecordOuterClass.TestRecord.parseFrom(serializedRecord1)); } catch (Exception e) { throw new RuntimeException(e); } @@ -47,30 +49,39 @@ public void deserialize(Blackhole bh) { @Override public void projectAndSerialize(Blackhole bh) { - // Projection with Protobuf can be done by building a new message with a subset of fields. - // There isn't a direct "project" operation on a parsed message. - TestRecordOuterClass.TestRecord projected = TestRecordOuterClass.TestRecord.newBuilder() - .setId(this.testData.id) - .setTimestamp(this.testData.timestamp) - .addAllTags(this.testData.tags.subList(0, 5)) - .build(); - bh.consume(projected.toByteArray()); + try { + // Full round trip: deserialize, project to a new object, re-serialize + var original = TestRecordOuterClass.TestRecord.parseFrom(serializedRecord1); + + TestRecordOuterClass.TestRecord projected = TestRecordOuterClass.TestRecord.newBuilder() + .setId(original.getId()) + .setTimestamp(original.getTimestamp()) + .addAllTags(original.getTagsList().subList(0, 5)) + .build(); + bh.consume(projected.toByteArray()); + } catch (Exception e) { + throw new RuntimeException(e); + } } @Override public void mergeAndSerialize(Blackhole bh) { - // Protobuf's `mergeFrom` is a natural fit here. - var record1 = buildRecord(this.testData); - var record2 = buildRecord(this.testData2); + try { + // Protobuf's `mergeFrom` is a natural fit here. + var record1 = TestRecordOuterClass.TestRecord.parseFrom(serializedRecord1); + var record2 = TestRecordOuterClass.TestRecord.parseFrom(serializedRecord2); - var merged = record1.toBuilder().mergeFrom(record2).build(); - bh.consume(merged.toByteArray()); + var merged = record1.toBuilder().mergeFrom(record2).build(); + bh.consume(merged.toByteArray()); + } catch (Exception e) { + throw new RuntimeException(e); + } } @Override public void accessField(Blackhole bh) { try { - bh.consume(TestRecordOuterClass.TestRecord.parseFrom(serializedRecord).getTimestamp()); + bh.consume(TestRecordOuterClass.TestRecord.parseFrom(serializedRecord1).getTimestamp()); } catch (Exception e) { throw new RuntimeException(e); } diff --git a/src/jmh/java/com/imprint/benchmark/competitors/ThriftCompetitor.java b/src/jmh/java/com/imprint/benchmark/competitors/ThriftCompetitor.java index 459de42..537eefa 100644 --- a/src/jmh/java/com/imprint/benchmark/competitors/ThriftCompetitor.java +++ b/src/jmh/java/com/imprint/benchmark/competitors/ThriftCompetitor.java @@ -18,6 +18,8 @@ public class ThriftCompetitor extends AbstractCompetitor { private final TSerializer serializer; private final TDeserializer deserializer; private final TestRecord thriftRecord; + private byte[] serializedRecord1; + private byte[] serializedRecord2; public ThriftCompetitor() { super("Thrift"); @@ -34,8 +36,10 @@ public ThriftCompetitor() { public void setup(DataGenerator.TestRecord testRecord, DataGenerator.TestRecord testRecord2) { super.setup(testRecord, testRecord2); try { - var record = buildThriftRecord(testRecord); - this.serializedRecord = serializer.serialize(record); + var record1 = buildThriftRecord(testRecord); + this.serializedRecord1 = serializer.serialize(record1); + var record2 = buildThriftRecord(testRecord2); + this.serializedRecord2 = serializer.serialize(record2); } catch (TException e) { throw new RuntimeException(e); } @@ -67,7 +71,7 @@ public void serialize(Blackhole bh) { public void deserialize(Blackhole bh) { try { var record = new TestRecord(); - deserializer.deserialize(record, this.serializedRecord); + deserializer.deserialize(record, this.serializedRecord1); bh.consume(record); } catch (TException e) { throw new RuntimeException(e); @@ -77,10 +81,14 @@ var record = new TestRecord(); @Override public void projectAndSerialize(Blackhole bh) { try { + // Full round trip: deserialize, project to a new object, re-serialize + var original = new TestRecord(); + deserializer.deserialize(original, this.serializedRecord1); + var projected = new ProjectedRecord(); - projected.setId(this.testData.id); - projected.setTimestamp(this.testData.timestamp); - projected.setTags(this.testData.tags.stream().limit(5).collect(Collectors.toList())); + projected.setId(original.getId()); + projected.setTimestamp(original.getTimestamp()); + projected.setTags(original.getTags().stream().limit(5).collect(Collectors.toList())); bh.consume(serializer.serialize(projected)); } catch (TException e) { throw new RuntimeException(e); @@ -90,8 +98,10 @@ public void projectAndSerialize(Blackhole bh) { @Override public void mergeAndSerialize(Blackhole bh) { try { - var r1 = buildThriftRecord(this.testData); - var r2 = buildThriftRecord(this.testData2); + var r1 = new TestRecord(); + deserializer.deserialize(r1, this.serializedRecord1); + var r2 = new TestRecord(); + deserializer.deserialize(r2, this.serializedRecord2); var merged = new TestRecord(); merged.setId(r1.id); @@ -119,7 +129,7 @@ public void mergeAndSerialize(Blackhole bh) { public void accessField(Blackhole bh) { try { var record = new TestRecord(); - deserializer.deserialize(record, this.serializedRecord); + deserializer.deserialize(record, this.serializedRecord1); bh.consume(record.getTimestamp()); } catch (TException e) { throw new RuntimeException(e); From 4b2664c865cfd764d08958d7b08af0a9c456fdec Mon Sep 17 00:00:00 2001 From: expand3d <> Date: Mon, 9 Jun 2025 13:12:01 -0400 Subject: [PATCH 44/49] micro-optiomize and attempt to make ComparisonBenchmark tests a little more fair --- build.gradle | 41 +- .../benchmark/ComparisonBenchmark.java | 25 +- .../competitors/AbstractCompetitor.java | 1 - .../benchmark/competitors/AvroCompetitor.java | 1 - .../competitors/ImprintCompetitor.java | 26 +- .../benchmark/competitors/KryoCompetitor.java | 66 +- .../competitors/ThriftCompetitor.java | 2 - src/jmh/sbe/test_record.xml | 61 ++ .../java/com/imprint/core/ImprintBuffers.java | 212 +++++-- .../com/imprint/core/ImprintOperations.java | 86 +-- .../java/com/imprint/core/ImprintRecord.java | 114 +++- .../imprint/core/ImprintRecordBuilder.java | 60 +- .../java/com/imprint/core/ImprintStream.java | 90 +-- src/main/java/com/imprint/types/TypeCode.java | 25 +- .../java/com/imprint/types/TypeHandler.java | 6 +- src/main/java/com/imprint/types/Value.java | 20 +- src/main/java/com/imprint/util/VarInt.java | 9 +- .../com/imprint/core/ImprintStreamTest.java | 78 +++ .../com/imprint/profile/ProfilerTest.java | 567 ++++++++++++------ 19 files changed, 1014 insertions(+), 476 deletions(-) create mode 100644 src/jmh/sbe/test_record.xml create mode 100644 src/test/java/com/imprint/core/ImprintStreamTest.java diff --git a/build.gradle b/build.gradle index 852be08..d3480e6 100644 --- a/build.gradle +++ b/build.gradle @@ -51,7 +51,15 @@ dependencies { jmhImplementation 'org.msgpack:msgpack-core:0.9.8' jmhImplementation 'org.msgpack:jackson-dataformat-msgpack:0.9.8' jmhImplementation 'org.apache.thrift:libthrift:0.19.0' - jmhImplementation 'javax.annotation:javax.annotation-api:1.3.2' + + // SBE for benchmarking + jmhImplementation 'uk.co.real-logic:sbe-all:1.35.3' + jmhImplementation 'io.aeron:aeron-client:1.41.2' // SBE has a dependency on Agrona, included in aeron-client + + // FastUtil for high-performance primitive collections + implementation 'it.unimi.dsi:fastutil:8.5.12' + // Required for generated Thrift code on JDK 11+ + implementation 'javax.annotation:javax.annotation-api:1.3.2' } protobuf { @@ -188,12 +196,41 @@ task generateJmhThrift(type: Exec) { } } +// Task for SBE code generation +task generateSbe(type: JavaExec) { + description = 'Generate Java classes from SBE schema' + group = 'build' + + def outputDir = file("${buildDir}/generated/sbe/java") + def schemaFile = file('src/jmh/sbe/schema.xml') + def sbeXsd = file('src/jmh/sbe/sbe.xsd') + + // Ensure the sbe-tool is on the classpath for this task + classpath = sourceSets.jmh.runtimeClasspath + + main = 'uk.co.real_logic.sbe.SbeTool' + systemProperties = [ + "sbe.output.dir": outputDir.absolutePath, + "sbe.validation.xsd": sbeXsd.absolutePath + ] + args = [ schemaFile.absolutePath ] + + inputs.file(schemaFile) + inputs.file(sbeXsd) + outputs.dir(outputDir) + + doFirst { + outputDir.mkdirs() + } +} + // Add generated FlatBuffers sources to JMH source set sourceSets { jmh { java { srcDir 'build/generated/source/flatbuffers/jmh/java' srcDir 'build/generated-src/thrift/jmh/java' + srcDir 'build/generated/sbe/java' } proto { srcDir 'src/jmh/proto' @@ -201,7 +238,7 @@ sourceSets { } } -// Make JMH compilation depend on FlatBuffers generation +// Make JMH compilation depend on generation tasks compileJmhJava.dependsOn generateFlatBuffers compileJmhJava.dependsOn generateJmhThrift diff --git a/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java b/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java index 0f99a05..f2c7398 100644 --- a/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java +++ b/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java @@ -1,15 +1,7 @@ package com.imprint.benchmark; -import com.imprint.benchmark.competitors.AbstractCompetitor; -import com.imprint.benchmark.competitors.AvroCompetitor; +import com.imprint.benchmark.competitors.*; import com.imprint.benchmark.competitors.Competitor; -import com.imprint.benchmark.competitors.FlatBuffersCompetitor; -import com.imprint.benchmark.competitors.ImprintCompetitor; -import com.imprint.benchmark.competitors.JacksonJsonCompetitor; -import com.imprint.benchmark.competitors.KryoCompetitor; -import com.imprint.benchmark.competitors.MessagePackCompetitor; -import com.imprint.benchmark.competitors.ProtobufCompetitor; -import com.imprint.benchmark.competitors.ThriftCompetitor; import org.openjdk.jmh.annotations.*; import org.openjdk.jmh.infra.Blackhole; import org.openjdk.jmh.runner.Runner; @@ -20,13 +12,12 @@ import java.util.Arrays; import java.util.List; import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) @State(Scope.Benchmark) -@BenchmarkMode(Mode.Throughput) -@OutputTimeUnit(TimeUnit.SECONDS) -@Warmup(iterations = 3, time = 5) -@Measurement(iterations = 5, time = 10) +@Warmup(iterations = 3, time = 1) +@Measurement(iterations = 10, time = 1) @Fork(value = 1, jvmArgs = {"-Xms4g", "-Xmx4g"}) public class ComparisonBenchmark { @@ -45,8 +36,6 @@ public class ComparisonBenchmark { public String competitorName; private Competitor competitor; - private DataGenerator.TestRecord testRecord1; - private DataGenerator.TestRecord testRecord2; @Setup(Level.Trial) public void setup() { @@ -57,8 +46,8 @@ public void setup() { .orElseThrow(() -> new IllegalStateException("Unknown competitor: " + competitorName)); // Create the test data - testRecord1 = DataGenerator.createTestRecord(); - testRecord2 = DataGenerator.createTestRecord(); + DataGenerator.TestRecord testRecord1 = DataGenerator.createTestRecord(); + DataGenerator.TestRecord testRecord2 = DataGenerator.createTestRecord(); // Setup the competitor with the data competitor.setup(testRecord1, testRecord2); diff --git a/src/jmh/java/com/imprint/benchmark/competitors/AbstractCompetitor.java b/src/jmh/java/com/imprint/benchmark/competitors/AbstractCompetitor.java index d92d3af..bfdea2a 100644 --- a/src/jmh/java/com/imprint/benchmark/competitors/AbstractCompetitor.java +++ b/src/jmh/java/com/imprint/benchmark/competitors/AbstractCompetitor.java @@ -11,7 +11,6 @@ public abstract class AbstractCompetitor implements Competitor { protected final String name; protected DataGenerator.TestRecord testData; protected DataGenerator.TestRecord testData2; - protected byte[] serializedRecord; protected AbstractCompetitor(String name) { this.name = name; diff --git a/src/jmh/java/com/imprint/benchmark/competitors/AvroCompetitor.java b/src/jmh/java/com/imprint/benchmark/competitors/AvroCompetitor.java index f7322ea..71c8306 100644 --- a/src/jmh/java/com/imprint/benchmark/competitors/AvroCompetitor.java +++ b/src/jmh/java/com/imprint/benchmark/competitors/AvroCompetitor.java @@ -11,7 +11,6 @@ import java.io.ByteArrayOutputStream; import java.nio.ByteBuffer; -import java.util.stream.Collectors; public class AvroCompetitor extends AbstractCompetitor { diff --git a/src/jmh/java/com/imprint/benchmark/competitors/ImprintCompetitor.java b/src/jmh/java/com/imprint/benchmark/competitors/ImprintCompetitor.java index 5f2781d..3e05cd7 100644 --- a/src/jmh/java/com/imprint/benchmark/competitors/ImprintCompetitor.java +++ b/src/jmh/java/com/imprint/benchmark/competitors/ImprintCompetitor.java @@ -3,8 +3,10 @@ import com.imprint.benchmark.DataGenerator; import com.imprint.core.ImprintOperations; import com.imprint.core.ImprintRecord; +import com.imprint.core.ImprintRecordBuilder; import com.imprint.core.SchemaId; import com.imprint.error.ImprintException; +import lombok.SneakyThrows; import org.openjdk.jmh.infra.Blackhole; import java.nio.ByteBuffer; @@ -12,7 +14,6 @@ public class ImprintCompetitor extends AbstractCompetitor { private ImprintRecord imprintRecord1; - private ImprintRecord imprintRecord2; private byte[] serializedRecord1; private byte[] serializedRecord2; private static final SchemaId SCHEMA_ID = new SchemaId(1, 1); @@ -26,13 +27,13 @@ public void setup(DataGenerator.TestRecord testRecord, DataGenerator.TestRecord super.setup(testRecord, testRecord2); try { this.imprintRecord1 = buildRecord(testRecord); - this.imprintRecord2 = buildRecord(testRecord2); + ImprintRecord imprintRecord2 = buildRecord(testRecord2); ByteBuffer buf1 = this.imprintRecord1.serializeToBuffer(); this.serializedRecord1 = new byte[buf1.remaining()]; buf1.get(this.serializedRecord1); - ByteBuffer buf2 = this.imprintRecord2.serializeToBuffer(); + ByteBuffer buf2 = imprintRecord2.serializeToBuffer(); this.serializedRecord2 = new byte[buf2.remaining()]; buf2.get(this.serializedRecord2); } catch (ImprintException e) { @@ -53,9 +54,26 @@ private ImprintRecord buildRecord(DataGenerator.TestRecord pojo) throws ImprintE return builder.build(); } + private ImprintRecordBuilder preBuildRecord(DataGenerator.TestRecord pojo) throws ImprintException { + var builder = ImprintRecord.builder(SCHEMA_ID); + builder.field(0, pojo.id); + builder.field(1, pojo.timestamp); + builder.field(2, pojo.flags); + builder.field(3, pojo.active); + builder.field(4, pojo.value); + builder.field(5, pojo.data); + builder.field(6, pojo.tags); + builder.field(7, pojo.metadata); + return builder; + } + @Override public void serialize(Blackhole bh) { - bh.consume(this.imprintRecord1.serializeToBuffer()); + try { + bh.consume(buildRecord(DataGenerator.createTestRecord()).serializeToBuffer()); + } catch (ImprintException e) { + throw new RuntimeException(e); + } } @Override diff --git a/src/jmh/java/com/imprint/benchmark/competitors/KryoCompetitor.java b/src/jmh/java/com/imprint/benchmark/competitors/KryoCompetitor.java index 15ccc24..d76a937 100644 --- a/src/jmh/java/com/imprint/benchmark/competitors/KryoCompetitor.java +++ b/src/jmh/java/com/imprint/benchmark/competitors/KryoCompetitor.java @@ -8,6 +8,8 @@ import java.io.ByteArrayOutputStream; import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; public class KryoCompetitor extends AbstractCompetitor { @@ -20,76 +22,97 @@ public KryoCompetitor() { this.kryo = new Kryo(); this.kryo.register(DataGenerator.TestRecord.class); this.kryo.register(DataGenerator.ProjectedRecord.class); + this.kryo.register(byte[].class); + kryo.register(ArrayList.class); + kryo.register(HashMap.class); + kryo.register(Arrays.asList().getClass()); } @Override public void setup(DataGenerator.TestRecord testRecord, DataGenerator.TestRecord testRecord2) { super.setup(testRecord, testRecord2); + + // Fix 1: Create fresh streams for each record + this.serializedRecord1 = serializeRecord(testRecord); + this.serializedRecord2 = serializeRecord(testRecord2); + } + + // Helper method to properly serialize a record + private byte[] serializeRecord(DataGenerator.TestRecord record) { try (ByteArrayOutputStream baos = new ByteArrayOutputStream(); Output output = new Output(baos)) { - kryo.writeObject(output, testRecord); - this.serializedRecord1 = baos.toByteArray(); - baos.reset(); - kryo.writeObject(output, testRecord2); - this.serializedRecord2 = baos.toByteArray(); + kryo.writeObject(output, record); + output.flush(); // Important: flush before getting bytes + return baos.toByteArray(); } catch (Exception e) { - throw new RuntimeException(e); + throw new RuntimeException("Failed to serialize record", e); } } @Override public void serialize(Blackhole bh) { + // Fix 2: Create fresh output stream each time try (ByteArrayOutputStream baos = new ByteArrayOutputStream(); Output output = new Output(baos)) { kryo.writeObject(output, this.testData); + output.flush(); // Ensure data is written bh.consume(baos.toByteArray()); } catch (Exception e) { - throw new RuntimeException(e); + throw new RuntimeException("Serialize failed", e); } } @Override public void deserialize(Blackhole bh) { + // Fix 3: Create fresh input each time try (Input input = new Input(serializedRecord1)) { bh.consume(kryo.readObject(input, DataGenerator.TestRecord.class)); } catch (Exception e) { - throw new RuntimeException(e); + throw new RuntimeException("Deserialize failed", e); } } @Override public void projectAndSerialize(Blackhole bh) { - // Full round trip: deserialize, project to a new object, re-serialize - try (Input input = new Input(serializedRecord1)) { - DataGenerator.TestRecord original = kryo.readObject(input, DataGenerator.TestRecord.class); + try { + // Step 1: Deserialize with fresh input + DataGenerator.TestRecord original; + try (Input input = new Input(serializedRecord1)) { + original = kryo.readObject(input, DataGenerator.TestRecord.class); + } + // Step 2: Create projected record var projected = new DataGenerator.ProjectedRecord(); projected.id = original.id; projected.timestamp = original.timestamp; - projected.tags = new ArrayList<>(original.tags.subList(0, 5)); + projected.tags = new ArrayList<>(original.tags.subList(0, Math.min(5, original.tags.size()))); + // Step 3: Serialize with fresh output try (ByteArrayOutputStream baos = new ByteArrayOutputStream(); Output output = new Output(baos)) { kryo.writeObject(output, projected); + output.flush(); bh.consume(baos.toByteArray()); } } catch (Exception e) { - throw new RuntimeException(e); + throw new RuntimeException("ProjectAndSerialize failed", e); } } @Override public void mergeAndSerialize(Blackhole bh) { try { + // Step 1: Deserialize both records with fresh inputs DataGenerator.TestRecord r1, r2; - try (Input input = new Input(serializedRecord1)) { - r1 = kryo.readObject(input, DataGenerator.TestRecord.class); + try (Input input1 = new Input(serializedRecord1)) { + r1 = kryo.readObject(input1, DataGenerator.TestRecord.class); } - try (Input input = new Input(serializedRecord2)) { - r2 = kryo.readObject(input, DataGenerator.TestRecord.class); + try (Input input2 = new Input(serializedRecord2)) { + r2 = kryo.readObject(input2, DataGenerator.TestRecord.class); } + // Step 2: Create merged record var merged = new DataGenerator.TestRecord(); merged.id = r1.id; merged.timestamp = System.currentTimeMillis(); @@ -100,23 +123,26 @@ public void mergeAndSerialize(Blackhole bh) { merged.tags = r2.tags; merged.metadata = r2.metadata; + // Step 3: Serialize with fresh output try (ByteArrayOutputStream baos = new ByteArrayOutputStream(); Output output = new Output(baos)) { kryo.writeObject(output, merged); + output.flush(); bh.consume(baos.toByteArray()); } } catch (Exception e) { - throw new RuntimeException(e); + throw new RuntimeException("MergeAndSerialize failed", e); } } @Override public void accessField(Blackhole bh) { + // Fix 4: Create fresh input for each access try (Input input = new Input(serializedRecord1)) { DataGenerator.TestRecord record = kryo.readObject(input, DataGenerator.TestRecord.class); bh.consume(record.timestamp); } catch (Exception e) { - throw new RuntimeException(e); + throw new RuntimeException("AccessField failed", e); } } -} \ No newline at end of file +} \ No newline at end of file diff --git a/src/jmh/java/com/imprint/benchmark/competitors/ThriftCompetitor.java b/src/jmh/java/com/imprint/benchmark/competitors/ThriftCompetitor.java index 537eefa..18530b5 100644 --- a/src/jmh/java/com/imprint/benchmark/competitors/ThriftCompetitor.java +++ b/src/jmh/java/com/imprint/benchmark/competitors/ThriftCompetitor.java @@ -17,7 +17,6 @@ public class ThriftCompetitor extends AbstractCompetitor { private final TSerializer serializer; private final TDeserializer deserializer; - private final TestRecord thriftRecord; private byte[] serializedRecord1; private byte[] serializedRecord2; @@ -26,7 +25,6 @@ public ThriftCompetitor() { try { this.serializer = new TSerializer(new TBinaryProtocol.Factory()); this.deserializer = new TDeserializer(new TBinaryProtocol.Factory()); - this.thriftRecord = new TestRecord(); } catch (Exception e) { throw new RuntimeException("Failed to initialize Thrift competitor", e); } diff --git a/src/jmh/sbe/test_record.xml b/src/jmh/sbe/test_record.xml new file mode 100644 index 0000000..9feaee8 --- /dev/null +++ b/src/jmh/sbe/test_record.xml @@ -0,0 +1,61 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/main/java/com/imprint/core/ImprintBuffers.java b/src/main/java/com/imprint/core/ImprintBuffers.java index 24ec41d..4afa1fa 100644 --- a/src/main/java/com/imprint/core/ImprintBuffers.java +++ b/src/main/java/com/imprint/core/ImprintBuffers.java @@ -5,6 +5,8 @@ import com.imprint.error.ImprintException; import com.imprint.types.TypeCode; import com.imprint.util.VarInt; +import it.unimi.dsi.fastutil.ints.Int2ObjectAVLTreeMap; +import it.unimi.dsi.fastutil.ints.Int2ObjectSortedMap; import lombok.Getter; import java.nio.ByteBuffer; @@ -32,9 +34,10 @@ public final class ImprintBuffers { private final ByteBuffer directoryBuffer; // Raw directory bytes (includes count) private final ByteBuffer payload; // Read-only payload view - // Lazy-loaded directory state. Needs to maintain ordering so that we can binary search the endOffset - private TreeMap parsedDirectory; + // Lazy-loaded directory state. + private Int2ObjectSortedMap parsedDirectory; private boolean directoryParsed = false; + private int directoryCount = -1; /** * Creates buffers from raw data (used during deserialization). @@ -45,24 +48,26 @@ public ImprintBuffers(ByteBuffer directoryBuffer, ByteBuffer payload) { } /** - * Creates buffers from a pre-parsed directory (used during construction). - * This constructor is used by the ImprintRecordBuilder path. It creates - * a serialized directory buffer but defers parsing it into a map until it's actually needed. + * Creates buffers from a pre-sorted list of entries (most efficient builder path). + * Immediately creates the parsed index and the serialized buffer. */ - public ImprintBuffers(Collection directory, ByteBuffer payload) { - this.directoryBuffer = ImprintBuffers.createDirectoryBuffer(Objects.requireNonNull(directory)); + public ImprintBuffers(List sortedDirectory, ByteBuffer payload) { + this.directoryBuffer = ImprintBuffers.createDirectoryBuffer(sortedDirectory); this.payload = payload.asReadOnlyBuffer(); } /** - * Creates buffers from a pre-parsed and sorted directory map (used by ImprintRecordBuilder). - * This is an optimized path that avoids creating an intermediate List-to-Map conversion. - * This constructor is used by the ImprintRecordBuilder path. It creates - * a serialized directory buffer but defers parsing it into a map until it's actually needed. + * Creates buffers from a pre-parsed and sorted directory map containing final, simple entries. + * This is the most efficient path, as it avoids any further parsing or sorting. The provided + * map becomes the definitive parsed directory. */ - public ImprintBuffers(TreeMap directoryMap, ByteBuffer payload) { - this.directoryBuffer = ImprintBuffers.createDirectoryBufferFromMap(Objects.requireNonNull(directoryMap)); + @SuppressWarnings("unchecked") + public ImprintBuffers(Int2ObjectSortedMap parsedDirectory, ByteBuffer payload) { + this.directoryBuffer = ImprintBuffers.createDirectoryBufferFromSortedMap(Objects.requireNonNull(parsedDirectory)); this.payload = payload.asReadOnlyBuffer(); + this.parsedDirectory = (Int2ObjectSortedMap) parsedDirectory; + this.directoryParsed = true; + this.directoryCount = parsedDirectory.size(); } /** @@ -88,6 +93,28 @@ public ByteBuffer getFieldBuffer(int fieldId) throws ImprintException { return fieldBuffer; } + /** + * Get a zero-copy ByteBuffer view of a field's data using a pre-fetched DirectoryEntry. + * This avoids the cost of re-finding the entry. + */ + public ByteBuffer getFieldBuffer(DirectoryEntry entry) throws ImprintException { + if (entry == null) + return null; + + int startOffset = entry.getOffset(); + int endOffset = findEndOffset(entry); + + if (startOffset < 0 || endOffset < 0 || startOffset > payload.limit() || + endOffset > payload.limit() || startOffset > endOffset) { + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, + "Invalid field buffer range: start=" + startOffset + ", end=" + endOffset + ", payloadLimit=" + payload.limit()); + } + + var fieldBuffer = payload.duplicate(); + fieldBuffer.position(startOffset).limit(endOffset); + return fieldBuffer; + } + /** * Find a directory entry for the given field ID using the most efficient method. *

@@ -98,7 +125,7 @@ public ByteBuffer getFieldBuffer(int fieldId) throws ImprintException { public DirectoryEntry findDirectoryEntry(int fieldId) throws ImprintException { if (directoryParsed) return parsedDirectory.get(fieldId); - else + else return findFieldEntryInRawDirectory(fieldId); } @@ -117,10 +144,10 @@ public List getDirectory() { public int getDirectoryCount() { if (directoryParsed) return parsedDirectory.size(); + try { - var countBuffer = directoryBuffer.duplicate(); - return VarInt.decode(countBuffer).getValue(); - } catch (Exception e) { + return getOrParseDirectoryCount(); + } catch (ImprintException e) { return 0; } } @@ -145,13 +172,16 @@ private DirectoryEntry findFieldEntryInRawDirectory(int fieldId) throws ImprintE var searchBuffer = directoryBuffer.duplicate(); searchBuffer.order(ByteOrder.LITTLE_ENDIAN); - int directoryCount = VarInt.decode(searchBuffer).getValue(); - if (directoryCount == 0) + int count = getOrParseDirectoryCount(); + if (count == 0) return null; + // Advance buffer past the varint to get to the start of the entries. + VarInt.decode(searchBuffer); int directoryStartPos = searchBuffer.position(); + int low = 0; - int high = directoryCount - 1; + int high = count - 1; while (low <= high) { int mid = (low + high) >>> 1; @@ -194,19 +224,25 @@ private int findEndOffset(DirectoryEntry entry) throws ImprintException { * Find the end offset using TreeMap's efficient navigation methods. */ private int findNextOffsetInParsedDirectory(int currentFieldId) { - var nextEntry = parsedDirectory.higherEntry(currentFieldId); - return nextEntry != null ? nextEntry.getValue().getOffset() : payload.limit(); + var tailMap = parsedDirectory.tailMap(currentFieldId + 1); + if (tailMap.isEmpty()) { + return payload.limit(); + } + return tailMap.get(tailMap.firstIntKey()).getOffset(); } private int findNextOffsetInRawDirectory(int currentFieldId) throws ImprintException { var scanBuffer = directoryBuffer.duplicate(); scanBuffer.order(ByteOrder.LITTLE_ENDIAN); - int count = VarInt.decode(scanBuffer).getValue(); + int count = getOrParseDirectoryCount(); if (count == 0) return payload.limit(); + // Advance buffer past the varint to get to the start of the entries. + VarInt.decode(scanBuffer); int directoryStartPos = scanBuffer.position(); + int low = 0; int high = count - 1; int nextOffset = payload.limit(); @@ -242,63 +278,133 @@ private int findNextOffsetInRawDirectory(int currentFieldId) throws ImprintExcep private void ensureDirectoryParsed() { if (directoryParsed) return; + try { var parseBuffer = directoryBuffer.duplicate(); parseBuffer.order(ByteOrder.LITTLE_ENDIAN); - var countResult = VarInt.decode(parseBuffer); - int count = countResult.getValue(); + int count = getOrParseDirectoryCount(parseBuffer); + this.parsedDirectory = new Int2ObjectAVLTreeMap<>(); - this.parsedDirectory = new TreeMap<>(); for (int i = 0; i < count; i++) { var entry = deserializeDirectoryEntry(parseBuffer); - parsedDirectory.put((int)entry.getId(), entry); + this.parsedDirectory.put(entry.getId() , entry); } this.directoryParsed = true; } catch (ImprintException e) { - throw new RuntimeException("Failed to parse directory", e); + // This can happen with a corrupted directory. + // In this case, we'll just have an empty (but valid) parsed directory. + this.parsedDirectory = new Int2ObjectAVLTreeMap<>(); + this.directoryParsed = true; // Mark as parsed to avoid repeated errors } } + private int getOrParseDirectoryCount() throws ImprintException { + if (directoryCount != -1) { + return directoryCount; + } + try { + this.directoryCount = VarInt.decode(directoryBuffer.duplicate()).getValue(); + } catch (ImprintException e) { + this.directoryCount = 0; // Cache as 0 on error + throw e; // rethrow + } + return this.directoryCount; + } + + private int getOrParseDirectoryCount(ByteBuffer buffer) throws ImprintException { + // This method does not cache the count because it's used during parsing + // where the buffer is transient. Caching is only for the instance's primary buffer. + return VarInt.decode(buffer).getValue(); + } + /** - * Create directory buffer from parsed entries. + * Creates a read-only buffer containing the serialized directory. + * The input collection does not need to be sorted. */ static ByteBuffer createDirectoryBuffer(Collection directory) { - try { - int bufferSize = VarInt.encodedLength(directory.size()) + (directory.size() * Constants.DIR_ENTRY_BYTES); - var buffer = ByteBuffer.allocate(bufferSize); - buffer.order(ByteOrder.LITTLE_ENDIAN); + if (directory == null || directory.isEmpty()) { + ByteBuffer buffer = ByteBuffer.allocate(1); + VarInt.encode(0, buffer); + buffer.flip(); + return buffer; + } - VarInt.encode(directory.size(), buffer); - for (var entry : directory) - serializeDirectoryEntry(entry, buffer); + // Ensure sorted order for binary search compatibility. + ArrayList sortedDirectory; + if (directory instanceof ArrayList && isSorted((ArrayList)directory)) { + sortedDirectory = (ArrayList) directory; + } else { + sortedDirectory = new ArrayList<>(directory); + sortedDirectory.sort(null); + } - buffer.flip(); - return buffer.asReadOnlyBuffer(); - } catch (Exception e) { - return ByteBuffer.allocate(0).asReadOnlyBuffer(); + int count = sortedDirectory.size(); + int size = VarInt.encodedLength(count) + (count * Constants.DIR_ENTRY_BYTES); + ByteBuffer buffer = ByteBuffer.allocate(size); + buffer.order(ByteOrder.LITTLE_ENDIAN); + + VarInt.encode(count, buffer); + for (DirectoryEntry entry : sortedDirectory) { + serializeDirectoryEntry(entry, buffer); } + + buffer.flip(); + return buffer; } - /** - * Create directory buffer from a pre-sorted map of entries. - */ static ByteBuffer createDirectoryBufferFromMap(TreeMap directoryMap) { - try { - int bufferSize = VarInt.encodedLength(directoryMap.size()) + (directoryMap.size() * Constants.DIR_ENTRY_BYTES); - var buffer = ByteBuffer.allocate(bufferSize); - buffer.order(ByteOrder.LITTLE_ENDIAN); + if (directoryMap == null || directoryMap.isEmpty()) { + ByteBuffer buffer = ByteBuffer.allocate(1); + VarInt.encode(0, buffer); + buffer.flip(); + return buffer; + } + + int count = directoryMap.size(); + int size = VarInt.encodedLength(count) + (count * Constants.DIR_ENTRY_BYTES); + var buffer = ByteBuffer.allocate(size); + buffer.order(ByteOrder.LITTLE_ENDIAN); - VarInt.encode(directoryMap.size(), buffer); - for (var entry : directoryMap.values()) - serializeDirectoryEntry(entry, buffer); + VarInt.encode(count, buffer); + for (var entry : directoryMap.values()) { + serializeDirectoryEntry(entry, buffer); + } + + buffer.flip(); + return buffer; + } + static ByteBuffer createDirectoryBufferFromSortedMap(Int2ObjectSortedMap directoryMap) { + if (directoryMap == null || directoryMap.isEmpty()) { + ByteBuffer buffer = ByteBuffer.allocate(1); + VarInt.encode(0, buffer); buffer.flip(); - return buffer.asReadOnlyBuffer(); - } catch (Exception e) { - return ByteBuffer.allocate(0).asReadOnlyBuffer(); + return buffer; + } + + int count = directoryMap.size(); + int size = VarInt.encodedLength(count) + (count * Constants.DIR_ENTRY_BYTES); + var buffer = ByteBuffer.allocate(size); + buffer.order(ByteOrder.LITTLE_ENDIAN); + + VarInt.encode(count, buffer); + for (var entry : directoryMap.int2ObjectEntrySet()) { + serializeDirectoryEntry(entry.getValue(), buffer); + } + + buffer.flip(); + return buffer; + } + + private static boolean isSorted(ArrayList list) { + for (int i = 0; i < list.size() - 1; i++) { + if (list.get(i).getId() > list.get(i + 1).getId()) { + return false; + } } + return true; } /** diff --git a/src/main/java/com/imprint/core/ImprintOperations.java b/src/main/java/com/imprint/core/ImprintOperations.java index c4e8c66..0c51e43 100644 --- a/src/main/java/com/imprint/core/ImprintOperations.java +++ b/src/main/java/com/imprint/core/ImprintOperations.java @@ -2,12 +2,14 @@ import com.imprint.error.ErrorType; import com.imprint.error.ImprintException; -import lombok.Value; import lombok.experimental.UtilityClass; import java.nio.ByteBuffer; import java.nio.ByteOrder; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; @UtilityClass public class ImprintOperations { @@ -29,41 +31,34 @@ public class ImprintOperations { * @return New ImprintRecord containing only the requested fields */ public static ImprintRecord project(ImprintRecord record, int... fieldIds) { - // Sort and deduplicate field IDs for efficient matching with sorted directory + // Sort and deduplicate field IDs for efficient matching int[] sortedFieldIds = Arrays.stream(fieldIds).distinct().sorted().toArray(); - if (sortedFieldIds.length == 0) + if (sortedFieldIds.length == 0) { return createEmptyRecord(record.getHeader().getSchemaId()); + } - //eager fetch the entire directory (can this be lazy and just done per field?) - var sourceDirectory = record.getDirectory(); var newDirectory = new ArrayList(sortedFieldIds.length); - var ranges = new ArrayList(); - - // Iterate through directory and compute ranges to copy - int fieldIdsIdx = 0; - int directoryIdx = 0; + var payloadChunks = new ArrayList(sortedFieldIds.length); int currentOffset = 0; - while (directoryIdx < sourceDirectory.size() && fieldIdsIdx < sortedFieldIds.length) { - var field = sourceDirectory.get(directoryIdx); - if (field.getId() == sortedFieldIds[fieldIdsIdx]) { - // Calculate field length using next field's offset - int nextOffset = (directoryIdx + 1 < sourceDirectory.size()) ? - sourceDirectory.get(directoryIdx + 1).getOffset() : - record.getBuffers().getPayload().limit(); - int fieldLength = nextOffset - field.getOffset(); - - newDirectory.add(new SimpleDirectoryEntry(field.getId(), field.getTypeCode(), currentOffset)); - ranges.add(new FieldRange(field.getOffset(), nextOffset)); - - currentOffset += fieldLength; - fieldIdsIdx++; + for (int fieldId : sortedFieldIds) { + // Use efficient lookup for each field's metadata. Returns null on failure. + DirectoryEntry sourceEntry = record.getDirectoryEntry(fieldId); + + // If field exists, get its payload and add to the new record components + if (sourceEntry != null) { + ByteBuffer fieldPayload = record.getRawBytes(sourceEntry); + // This check is for internal consistency. If an entry exists, payload should too. + if (fieldPayload != null) { + newDirectory.add(new SimpleDirectoryEntry((short)fieldId, sourceEntry.getTypeCode(), currentOffset)); + payloadChunks.add(fieldPayload); + currentOffset += fieldPayload.remaining(); + } } - directoryIdx++; } - // Build new payload from ranges - var newPayload = buildPayloadFromRanges(record.getBuffers().getPayload(), ranges); + // Build new payload from collected chunks + ByteBuffer newPayload = buildPayloadFromChunks(payloadChunks); // Create new header with updated payload size // TODO: compute correct schema hash @@ -120,12 +115,12 @@ public static ImprintRecord merge(ImprintRecord first, ImprintRecord second) thr secondIdx++; } - currentPayload = first.getRawBytes(currentEntry.getId()); + currentPayload = first.getRawBytes(currentEntry); firstIdx++; } else { // Take from second record currentEntry = secondDir.get(secondIdx); - currentPayload = second.getRawBytes(currentEntry.getId()); + currentPayload = second.getRawBytes(currentEntry); secondIdx++; } @@ -150,37 +145,6 @@ public static ImprintRecord merge(ImprintRecord first, ImprintRecord second) thr return new ImprintRecord(newHeader, newDirectory, mergedPayload); } - /** - * Represents a range of bytes to copy from source payload. - */ - @Value - private static class FieldRange { - int start; - int end; - - int length() { - return end - start; - } - } - - /** - * Build a new payload buffer from field ranges in the source payload. - */ - private static ByteBuffer buildPayloadFromRanges(ByteBuffer sourcePayload, List ranges) { - int totalSize = ranges.stream().mapToInt(FieldRange::length).sum(); - var newPayload = ByteBuffer.allocate(totalSize); - newPayload.order(ByteOrder.LITTLE_ENDIAN); - - for (var range : ranges) { - var sourceSlice = sourcePayload.duplicate(); - sourceSlice.position(range.start).limit(range.end); - newPayload.put(sourceSlice); - } - - newPayload.flip(); - return newPayload; - } - /** * Build a new payload buffer by concatenating chunks. */ diff --git a/src/main/java/com/imprint/core/ImprintRecord.java b/src/main/java/com/imprint/core/ImprintRecord.java index 385e569..e4c4a42 100644 --- a/src/main/java/com/imprint/core/ImprintRecord.java +++ b/src/main/java/com/imprint/core/ImprintRecord.java @@ -7,6 +7,7 @@ import com.imprint.types.TypeCode; import com.imprint.types.Value; import com.imprint.util.VarInt; +import it.unimi.dsi.fastutil.ints.Int2ObjectSortedMap; import lombok.Getter; import java.nio.ByteBuffer; @@ -35,19 +36,19 @@ private ImprintRecord(Header header, ImprintBuffers buffers) { } /** - * Creates a record from pre-parsed directory (used by ImprintWriter). + * Creates a record from a pre-sorted list of entries (most efficient builder path). */ - ImprintRecord(Header header, Collection directory, ByteBuffer payload) { + ImprintRecord(Header header, List sortedDirectory, ByteBuffer payload) { this.header = Objects.requireNonNull(header, "Header cannot be null"); - this.buffers = new ImprintBuffers(directory, payload); + this.buffers = new ImprintBuffers(sortedDirectory, payload); } /** - * Creates a record from a pre-parsed and sorted directory map (used by ImprintRecordBuilder). + * Creates a record from a pre-built and sorted FastUtil map (most efficient builder path). */ - ImprintRecord(Header header, TreeMap directoryMap, ByteBuffer payload) { + ImprintRecord(Header header, Int2ObjectSortedMap parsedDirectory, ByteBuffer payload) { this.header = Objects.requireNonNull(header, "Header cannot be null"); - this.buffers = new ImprintBuffers(directoryMap, payload); + this.buffers = new ImprintBuffers(parsedDirectory, payload); } // ========== FIELD ACCESS METHODS ========== @@ -79,6 +80,18 @@ public ByteBuffer getRawBytes(int fieldId) { } } + /** + * Get raw bytes for a field using a pre-fetched DirectoryEntry. + * This avoids the cost of re-finding the entry metadata. + */ + public ByteBuffer getRawBytes(DirectoryEntry entry) { + try { + return buffers.getFieldBuffer(entry); + } catch (ImprintException e) { + return null; + } + } + /** * Project a subset of fields from this record. * @@ -108,6 +121,37 @@ public List getDirectory() { return buffers.getDirectory(); } + /** + * Finds a directory entry by its field ID. + * This is an efficient lookup that avoids full directory deserialization if possible. + * + * @param fieldId The ID of the field to find. + * @return The DirectoryEntry if found, otherwise null. + */ + public DirectoryEntry getDirectoryEntry(int fieldId) { + try { + return buffers.findDirectoryEntry(fieldId); + } catch (ImprintException e) { + // This can happen with a corrupted directory, in which case we assume it doesn't exist. + return null; + } + } + + /** + * Checks if a field with the given ID exists in the record. + * + * @param fieldId The ID of the field to check. + * @return true if the field exists, false otherwise. + */ + public boolean hasField(int fieldId) { + try { + return buffers.findDirectoryEntry(fieldId) != null; + } catch (ImprintException e) { + // This can happen with a corrupted directory, in which case we assume it doesn't exist. + return false; + } + } + // ========== TYPED GETTERS ========== public boolean getBoolean(int fieldId) throws ImprintException { @@ -194,7 +238,7 @@ public int estimateSerializedSize() { * This provides a direct serialization path without needing a live ImprintRecord instance. * * @param schemaId The schema identifier for the record. - * @param directory The list of directory entries, which must be sorted by field ID. + * @param directory The list of directory entries, which will be sorted if not already. * @param payload The ByteBuffer containing all field data concatenated. * @return A read-only ByteBuffer with the complete serialized record. */ @@ -215,6 +259,34 @@ public static ByteBuffer serialize(SchemaId schemaId, Collection sortedDirectory, ByteBuffer payload) { + var header = new Header(new Flags((byte) 0), schemaId, payload.remaining()); + // This createDirectoryBuffer is optimized for a pre-sorted list. + var directoryBuffer = ImprintBuffers.createDirectoryBuffer(sortedDirectory); + + int finalSize = Constants.HEADER_BYTES + directoryBuffer.remaining() + payload.remaining(); + var finalBuffer = ByteBuffer.allocate(finalSize); + finalBuffer.order(ByteOrder.LITTLE_ENDIAN); + + // Assemble the final record + serializeHeader(header, finalBuffer); + finalBuffer.put(directoryBuffer); + finalBuffer.put(payload); + + finalBuffer.flip(); + return finalBuffer.asReadOnlyBuffer(); + } + /** * Serializes the components of a record into a single ByteBuffer using a pre-built directory map. * This provides a direct serialization path without needing a live ImprintRecord instance. @@ -241,6 +313,32 @@ public static ByteBuffer serialize(SchemaId schemaId, TreeMap directoryMap, ByteBuffer payload) { + var header = new Header(new Flags((byte) 0), schemaId, payload.remaining()); + var directoryBuffer = ImprintBuffers.createDirectoryBufferFromSortedMap(directoryMap); + + int finalSize = Constants.HEADER_BYTES + directoryBuffer.remaining() + payload.remaining(); + var finalBuffer = ByteBuffer.allocate(finalSize); + finalBuffer.order(ByteOrder.LITTLE_ENDIAN); + + // Assemble the final record + serializeHeader(header, finalBuffer); + finalBuffer.put(directoryBuffer); + finalBuffer.put(payload); + + finalBuffer.flip(); + return finalBuffer.asReadOnlyBuffer(); + } + // ========== STATIC FACTORY METHODS ========== public static ImprintRecordBuilder builder(SchemaId schemaId) { @@ -348,7 +446,7 @@ private static Header deserializeHeader(ByteBuffer buffer) throws ImprintExcepti byte magic = buffer.get(); if (magic != Constants.MAGIC) { throw new ImprintException(ErrorType.INVALID_MAGIC, "Invalid magic byte: expected 0x" + Integer.toHexString(Constants.MAGIC) + - ", got 0x" + Integer.toHexString(magic & 0xFF)); + ", got 0x" + Integer.toHexString(magic & 0xFF)); } byte version = buffer.get(); diff --git a/src/main/java/com/imprint/core/ImprintRecordBuilder.java b/src/main/java/com/imprint/core/ImprintRecordBuilder.java index 58fbc63..52bc760 100644 --- a/src/main/java/com/imprint/core/ImprintRecordBuilder.java +++ b/src/main/java/com/imprint/core/ImprintRecordBuilder.java @@ -1,21 +1,18 @@ package com.imprint.core; -import com.imprint.error.ErrorType; import com.imprint.error.ImprintException; import com.imprint.types.MapKey; import com.imprint.types.TypeCode; import com.imprint.types.Value; +import it.unimi.dsi.fastutil.ints.Int2ObjectAVLTreeMap; +import it.unimi.dsi.fastutil.ints.Int2ObjectSortedMap; +import lombok.Getter; +import lombok.Setter; import lombok.SneakyThrows; import java.nio.ByteBuffer; import java.nio.ByteOrder; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.TreeMap; +import java.util.*; /** * A fluent builder for creating ImprintRecord instances with type-safe, @@ -41,7 +38,7 @@ @SuppressWarnings("unused") public final class ImprintRecordBuilder { private final SchemaId schemaId; - private final Map fields = new TreeMap<>(); + private final Int2ObjectSortedMap fields = new Int2ObjectAVLTreeMap<>(); private int estimatedPayloadSize = 0; ImprintRecordBuilder(SchemaId schemaId) { @@ -158,8 +155,8 @@ public ImprintRecord build() throws ImprintException { payloadBuffer.flip(); // limit = position, position = 0 var payloadView = payloadBuffer.slice().asReadOnlyBuffer(); - var header = new com.imprint.core.Header(new com.imprint.core.Flags((byte) 0), schemaId, payloadView.remaining()); - return new ImprintRecord(header, new ArrayList<>(fields.values()), payloadView); + var header = new Header(new Flags((byte) 0), schemaId, payloadView.remaining()); + return new ImprintRecord(header, fields, payloadView); } /** @@ -284,14 +281,6 @@ private MapKey convertToMapKey(Object obj) { throw new IllegalArgumentException("Unsupported map key type: " + obj.getClass().getName()); } - @Override - public String toString() { - return "ImprintRecordBuilder{" + - "schemaId=" + schemaId + - ", fields=" + fields + - '}'; - } - private int estimatePayloadSize() { // Add 25% buffer to reduce reallocations and handle VarInt encoding fluctuations. return Math.max(estimatedPayloadSize + (estimatedPayloadSize / 4), fields.size() * 16); @@ -357,48 +346,23 @@ private void serializeValue(Value value, ByteBuffer buffer) throws ImprintExcept } } - // Private inner class to hold field data during building + + @Getter private static class BuilderEntry implements DirectoryEntry { private final short id; private final Value value; + @Setter private int offset; BuilderEntry(short id, Value value) { this.id = id; this.value = value; - this.offset = -1; // Initially unknown - } - - @Override - public short getId() { - return id; + this.offset = -1; } @Override public TypeCode getTypeCode() { return value.getTypeCode(); } - - @Override - public int getOffset() { - return offset; - } - - public void setOffset(int offset) { - this.offset = offset; - } - - public Value getValue() { - return value; - } - - @Override - public String toString() { - return "BuilderEntry{" + - "id=" + id + - ", value=" + value + - ", offset=" + offset + - '}'; - } } } \ No newline at end of file diff --git a/src/main/java/com/imprint/core/ImprintStream.java b/src/main/java/com/imprint/core/ImprintStream.java index c218318..b6afe7a 100644 --- a/src/main/java/com/imprint/core/ImprintStream.java +++ b/src/main/java/com/imprint/core/ImprintStream.java @@ -1,19 +1,14 @@ package com.imprint.core; import com.imprint.error.ImprintException; +import it.unimi.dsi.fastutil.ints.Int2ObjectAVLTreeMap; +import it.unimi.dsi.fastutil.ints.Int2ObjectSortedMap; +import it.unimi.dsi.fastutil.ints.IntSet; +import it.unimi.dsi.fastutil.ints.IntOpenHashSet; import java.nio.ByteBuffer; import java.nio.ByteOrder; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.TreeMap; +import java.util.*; /** * Provides a framework for lazy, zero-copy transformations of Imprint records. @@ -30,8 +25,6 @@ private ImprintStream(Plan plan) { this.plan = Objects.requireNonNull(plan); } - // ========== PLAN DATA STRUCTURES ========== - /** * The internal representation of the transformation plan. * This is a linked-list style structure where each step points to the previous one. @@ -56,11 +49,11 @@ private SourcePlan(ImprintRecord source) { */ private static final class ProjectPlan implements Plan { final Plan previous; - final Set fieldIds; + final IntSet fieldIds; private ProjectPlan(Plan previous, int... fieldIds) { this.previous = Objects.requireNonNull(previous); - this.fieldIds = new HashSet<>(); + this.fieldIds = new IntOpenHashSet(); for (int id : fieldIds) { this.fieldIds.add(id); } @@ -143,40 +136,41 @@ private Evaluator(Plan plan) { public ImprintRecord execute() { // Unwind the plan's linked-list structure into a forward-order list of operations. - var planList = new ArrayList(); - var current = plan; - while (current != null) { - planList.add(current); - if (current instanceof ProjectPlan) { - current = ((ProjectPlan) current).previous; - } else if (current instanceof MergePlan) { - current = ((MergePlan) current).previous; - } else if (current instanceof SourcePlan) { - current = null; // End of the chain - } - } + var planList = getPlans(); Collections.reverse(planList); - // This map holds the set of fields being built, sorted by ID. - var resolvedFields = new TreeMap(); + // This map holds the set of fields being built, sorted by field ID. + var resolvedFields = new Int2ObjectAVLTreeMap(); // Iteratively evaluate the plan step-by-step. for (var planStep : planList) { if (planStep instanceof SourcePlan) { var sourcePlan = (SourcePlan) planStep; for (var entry : sourcePlan.source.getDirectory()) { - resolvedFields.put((int) entry.getId(), new FieldSource(sourcePlan.source, entry)); + resolvedFields.put(entry.getId(), new FieldSource(sourcePlan.source, entry)); } } else if (planStep instanceof ProjectPlan) { var projectPlan = (ProjectPlan) planStep; // Apply projection to the current state of resolved fields. - resolvedFields.keySet().retainAll(projectPlan.fieldIds); + // Keep only fields that are in the projection set + var keysToRemove = new IntOpenHashSet(); + for (int fieldId : resolvedFields.keySet()) { + if (!projectPlan.fieldIds.contains(fieldId)) { + keysToRemove.add(fieldId); + } + } + for (int keyToRemove : keysToRemove) { + resolvedFields.remove(keyToRemove); + } } else if (planStep instanceof MergePlan) { var mergePlan = (MergePlan) planStep; // Add fields from other records if they aren't already in the map. for (var otherRecord : mergePlan.others) { for (var entry : otherRecord.getDirectory()) { - resolvedFields.putIfAbsent((int) entry.getId(), new FieldSource(otherRecord, entry)); + int fieldId = entry.getId(); + if (!resolvedFields.containsKey(fieldId)) { + resolvedFields.put(fieldId, new FieldSource(otherRecord, entry)); + } } } } @@ -186,7 +180,23 @@ public ImprintRecord execute() { return build(resolvedFields); } - private ImprintRecord build(TreeMap finalFields) { + private ArrayList getPlans() { + var planList = new ArrayList(); + var current = plan; + while (current != null) { + planList.add(current); + if (current instanceof ProjectPlan) { + current = ((ProjectPlan) current).previous; + } else if (current instanceof MergePlan) { + current = ((MergePlan) current).previous; + } else if (current instanceof SourcePlan) { + current = null; // End of the chain + } + } + return planList; + } + + private ImprintRecord build(Int2ObjectSortedMap finalFields) { if (finalFields.isEmpty()) { // To-Do: Need a way to get the schemaId for an empty record. // For now, returning null or using a default. @@ -199,16 +209,22 @@ private ImprintRecord build(TreeMap finalFields) { } // Determine the schema from the first field's source record. - SchemaId schemaId = finalFields.firstEntry().getValue().record.getHeader().getSchemaId(); + SchemaId schemaId = finalFields.get(finalFields.firstIntKey()).record.getHeader().getSchemaId(); // 1. Calculate final payload size and prepare directory. int payloadSize = 0; - var newDirectoryMap = new TreeMap(); - for (var entry : finalFields.entrySet()) { + var newDirectoryMap = new Int2ObjectAVLTreeMap(); + + // Iterate over fields in sorted order + for (var entry : finalFields.int2ObjectEntrySet()) { + int fieldId = entry.getIntKey(); var fieldSource = entry.getValue(); int fieldLength = fieldSource.getLength(); - newDirectoryMap.put(entry.getKey(), new SimpleDirectoryEntry(fieldSource.entry.getId(), fieldSource.entry.getTypeCode(), payloadSize)); + newDirectoryMap.put(fieldId, new SimpleDirectoryEntry( + fieldSource.entry.getId(), + fieldSource.entry.getTypeCode(), + payloadSize)); payloadSize += fieldLength; } @@ -254,4 +270,4 @@ int getLength() { } } } -} \ No newline at end of file +} \ No newline at end of file diff --git a/src/main/java/com/imprint/types/TypeCode.java b/src/main/java/com/imprint/types/TypeCode.java index a81b199..3447f8b 100644 --- a/src/main/java/com/imprint/types/TypeCode.java +++ b/src/main/java/com/imprint/types/TypeCode.java @@ -19,11 +19,19 @@ public enum TypeCode { ARRAY(0x8, TypeHandler.ARRAY), MAP(0x9, TypeHandler.MAP), ROW(0xA, null); // TODO: implement (basically a placeholder for user-defined type) - + @Getter private final byte code; private final TypeHandler handler; - + + private static final TypeCode[] LOOKUP = new TypeCode[11]; + + static { + for (var type : values()) { + LOOKUP[type.code] = type; + } + } + TypeCode(int code, TypeHandler handler) { this.code = (byte) code; this.handler = handler; @@ -35,14 +43,13 @@ public TypeHandler getHandler() { } return handler; } - + public static TypeCode fromByte(byte code) throws ImprintException { - for (TypeCode type : values()) { - if (type.code == code) { - return type; - } + if (code >= 0 && code < LOOKUP.length) { + var type = LOOKUP[code]; + if (type != null) return type; } - throw new ImprintException(ErrorType.INVALID_TYPE_CODE, - "Unknown type code: 0x" + Integer.toHexString(code & 0xFF)); + throw new ImprintException(ErrorType.INVALID_TYPE_CODE, + "Unknown type code: 0x" + Integer.toHexString(code & 0xFF)); } } \ No newline at end of file diff --git a/src/main/java/com/imprint/types/TypeHandler.java b/src/main/java/com/imprint/types/TypeHandler.java index 634867b..dbc875f 100644 --- a/src/main/java/com/imprint/types/TypeHandler.java +++ b/src/main/java/com/imprint/types/TypeHandler.java @@ -218,7 +218,7 @@ public void serialize(Value value, ByteBuffer buffer) { buffer.put(stringBytes); } } - + @Override public int estimateSize(Value value) { if (value instanceof Value.StringBufferValue) { @@ -227,8 +227,8 @@ public int estimateSize(Value value) { return VarInt.encodedLength(length) + length; } else { Value.StringValue stringValue = (Value.StringValue) value; - byte[] utf8Bytes = stringValue.getUtf8Bytes(); - return VarInt.encodedLength(utf8Bytes.length) + utf8Bytes.length; + int utf8Length = stringValue.getUtf8Length(); // Uses cached bytes + return VarInt.encodedLength(utf8Length) + utf8Length; } } }; diff --git a/src/main/java/com/imprint/types/Value.java b/src/main/java/com/imprint/types/Value.java index bfa9958..681eda1 100644 --- a/src/main/java/com/imprint/types/Value.java +++ b/src/main/java/com/imprint/types/Value.java @@ -57,10 +57,11 @@ public static Value fromBytesBuffer(ByteBuffer value) { public static Value fromString(String value) { return new StringValue(value); } - + public static Value fromStringBuffer(ByteBuffer value) { return new StringBufferValue(value); } + public static Value fromArray(List value) { return new ArrayValue(value); @@ -284,20 +285,21 @@ public String toString() { public static class StringValue extends Value { @Getter private final String value; - private volatile byte[] cachedUtf8Bytes; // Cache UTF-8 encoding + private byte[] utf8BytesCache; // Cache UTF-8 encoding public StringValue(String value) { this.value = Objects.requireNonNull(value, "String cannot be null"); } public byte[] getUtf8Bytes() { - var cached = cachedUtf8Bytes; - if (cached == null) { - // UTF8 is idempotent so no need to synchronize - cached = value.getBytes(StandardCharsets.UTF_8); - cachedUtf8Bytes = cached; + if (utf8BytesCache == null) { + utf8BytesCache = value.getBytes(StandardCharsets.UTF_8); } - return cached; // Return computed value + return utf8BytesCache; + } + + public int getUtf8Length() { + return getUtf8Bytes().length; } @Override @@ -332,7 +334,7 @@ public String toString() { // String Value (ByteBuffer-based) public static class StringBufferValue extends Value { private final ByteBuffer value; - private volatile String cachedString; + private String cachedString; private static final int THREAD_LOCAL_BUFFER_SIZE = 1024; private static final ThreadLocal DECODE_BUFFER_CACHE = diff --git a/src/main/java/com/imprint/util/VarInt.java b/src/main/java/com/imprint/util/VarInt.java index f43683b..70c9095 100644 --- a/src/main/java/com/imprint/util/VarInt.java +++ b/src/main/java/com/imprint/util/VarInt.java @@ -70,13 +70,10 @@ public static DecodeResult decode(ByteBuffer buffer) throws ImprintException { int bytesRead = 0; while (true) { - if (bytesRead >= MAX_VARINT_LEN) { + if (bytesRead >= MAX_VARINT_LEN) throw new ImprintException(ErrorType.MALFORMED_VARINT, "VarInt too long"); - } - if (!buffer.hasRemaining()) { - throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, - "Unexpected end of data while reading VarInt"); - } + if (!buffer.hasRemaining()) + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, "Unexpected end of data while reading VarInt"); byte b = buffer.get(); bytesRead++; diff --git a/src/test/java/com/imprint/core/ImprintStreamTest.java b/src/test/java/com/imprint/core/ImprintStreamTest.java new file mode 100644 index 0000000..8d5b843 --- /dev/null +++ b/src/test/java/com/imprint/core/ImprintStreamTest.java @@ -0,0 +1,78 @@ +package com.imprint.core; + +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.*; + +class ImprintStreamTest { + + @Test + void shouldProjectAndMergeCorrectly() throws Exception { + // --- Setup --- + var schemaId1 = new SchemaId(1, 1); + var schemaId2 = new SchemaId(2, 2); + var schemaId3 = new SchemaId(3, 3); + + ImprintRecord recordA = ImprintRecord.builder(schemaId1) + .field(1, "A1") + .field(2, 100) + .field(3, true) + .build(); + + ImprintRecord recordB = ImprintRecord.builder(schemaId2) + .field(2, 200) // Overlaps with A, should be ignored + .field(4, "B4") + .build(); + + ImprintRecord recordC = ImprintRecord.builder(schemaId3) + .field(5, 3.14) + .field(1, "C1") // Overlaps with A, should be ignored + .build(); + + // --- Execution --- + // Define a chain of operations + ImprintRecord finalRecord = ImprintStream.of(recordA) + .project(1, 3) // Keep {1, 3} from A. Current state: {1:A, 3:A} + .mergeWith(recordB) // Merge B. {2:B, 4:B} are added. Current state: {1:A, 3:A, 2:B, 4:B} + .mergeWith(recordC) // Merge C. {5:C} is added. {1:C} is ignored. Final state: {1:A, 3:A, 2:B, 4:B, 5:C} + .project(1, 4, 5) // Final projection. Final result: {1:A, 4:B, 5:C} + .toRecord(); + + // --- Assertions --- + assertNotNull(finalRecord); + + // Check final field count. + assertEquals(3, finalRecord.getDirectory().size()); + + // Check that the correct fields are present and have the right values + assertTrue(finalRecord.hasField(1)); + assertEquals("A1", finalRecord.getString(1)); // From recordA + + assertTrue(finalRecord.hasField(4)); + assertEquals("B4", finalRecord.getString(4)); // From recordB + + assertTrue(finalRecord.hasField(5)); + assertEquals(3.14, finalRecord.getFloat64(5), 0.001); // From recordC + + // Check that dropped/ignored fields are not present + assertFalse(finalRecord.hasField(2)); + assertFalse(finalRecord.hasField(3)); + } + + @Test + void shouldProjectAfterMerge() throws Exception { + var recordA = ImprintRecord.builder(new SchemaId(1, 1)).field(1, "A").field(2, 100).build(); + var recordB = ImprintRecord.builder(new SchemaId(1, 1)).field(2, 200).field(3, "B").build(); + + ImprintRecord finalRecord = ImprintStream.of(recordA) + .mergeWith(recordB) // virtual record is {1:A, 2:A, 3:B} + .project(1, 3) // final record is {1:A, 3:B} + .toRecord(); + + assertEquals(2, finalRecord.getDirectory().size()); + assertTrue(finalRecord.hasField(1)); + assertEquals("A", finalRecord.getString(1)); + assertTrue(finalRecord.hasField(3)); + assertEquals("B", finalRecord.getString(3)); + assertFalse(finalRecord.hasField(2)); + } +} \ No newline at end of file diff --git a/src/test/java/com/imprint/profile/ProfilerTest.java b/src/test/java/com/imprint/profile/ProfilerTest.java index 3804722..3cfa61f 100644 --- a/src/test/java/com/imprint/profile/ProfilerTest.java +++ b/src/test/java/com/imprint/profile/ProfilerTest.java @@ -1,13 +1,15 @@ package com.imprint.profile; +import com.imprint.core.ImprintOperations; import com.imprint.core.ImprintRecord; import com.imprint.core.SchemaId; import com.imprint.types.Value; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; +import java.util.Arrays; import java.util.Random; -import java.util.UUID; +import java.util.stream.IntStream; /** * A test designed for profiling hotspots during development. @@ -27,183 +29,401 @@ * - String operations and UTF-8 encoding * - ByteBuffer operations */ -@Disabled("Enable manually for profiling") +//@Disabled("Enable manually for profiling") public class ProfilerTest { - + private static final int ITERATIONS = 1_000_000; private static final int RECORD_SIZE = 50; - + private static final int LARGE_RECORD_SIZE = 200; + @Test void profileFieldAccess() throws Exception { + System.out.println("Starting profiler test - attach profiler now..."); + Thread.sleep(5000); // Give time to attach profiler + + // Create a representative record var record = createTestRecord(); - runProfileTest("Field Access", () -> { - // Simulate real-world access patterns - Random random = new Random(42); - int hits = 0; - - for (int i = 0; i < ITERATIONS; i++) { - // Random field access (hotspot) - int fieldId = random.nextInt(RECORD_SIZE) + 1; - var value = record.getValue(fieldId); - if (value != null) { - hits++; - - // Trigger string decoding (potential hotspot) - if (value.getTypeCode() == com.imprint.types.TypeCode.STRING) { - if (value instanceof Value.StringBufferValue) { - ((Value.StringBufferValue) value).getValue(); - } else { - ((Value.StringValue) value).getValue(); - } + System.out.println("Beginning field access profiling..."); + long start = System.nanoTime(); + + // Simulate real-world access patterns + Random random = new Random(42); + int hits = 0; + + for (int i = 0; i < ITERATIONS; i++) { + // Random field access (hotspot) + int fieldId = random.nextInt(RECORD_SIZE) + 1; + var value = record.getValue(fieldId); + if (value != null) { + hits++; + + // Trigger string decoding (potential hotspot) + if (value.getTypeCode() == com.imprint.types.TypeCode.STRING) { + if (value instanceof Value.StringBufferValue) { + ((Value.StringBufferValue) value).getValue(); + } else { + ((Value.StringValue) value).getValue(); } } + } - // Some raw access (zero-copy path) - if (i % 10 == 0) { - record.getRawBytes(fieldId); - } + // Some raw access (zero-copy path) + if (i % 10 == 0) { + record.getRawBytes(fieldId); } - }); + } + + long duration = System.nanoTime() - start; + System.out.printf("Completed %,d field accesses in %.2f ms (avg: %.1f ns/op, hits: %d)%n", + ITERATIONS, duration / 1_000_000.0, (double) duration / ITERATIONS, hits); } - + @Test - void profileSerialization() throws Exception { - var schemaId = new SchemaId(1, 0x12345678); + void profileSmallRecordSerialization() throws Exception { + profileSerialization("small records", RECORD_SIZE, 100_000); + } - runProfileTest("Serialization (Standard)", () -> { - // Create and serialize many records (allocation hotspot) - for (int i = 0; i < 500_000; i++) { - var builder = ImprintRecord.builder(schemaId); + @Test + void profileLargeRecordSerialization() throws Exception { + profileSerialization("large records", LARGE_RECORD_SIZE, 500_000); + } + + @Test + void profileProjectionOperations() throws Exception { + System.out.println("Starting projection profiler test - attach profiler now..."); + Thread.sleep(3000); - // Add various field types - builder.field(1, Value.fromInt32(i)) - .field(2, Value.fromString("test-string-" + i)) - .field(3, Value.fromFloat64(i * 3.14159)) - .field(4, Value.fromBytes(("bytes-" + i).getBytes())); + profileSmallProjections(); + profileLargeProjections(); + profileSelectiveProjections(); + profileProjectionMemoryAllocation(); + } - var record = builder.build(); - var serialized = record.serializeToBuffer(); // Potential hotspot + /** + * Profile small projections (select 2-5 fields from 20-field records) + */ + private void profileSmallProjections() throws Exception { + System.out.println("\\n--- Small Projections (2-5 fields from 20-field records) ---"); - // Trigger some deserialization - if (i % 1000 == 0) { - var deserialized = ImprintRecord.deserialize(serialized); - deserialized.getValue(2); // String decoding hotspot - } + var sourceRecord = createTestRecord(20); + int[] projectFields = {1, 5, 10, 15}; // 4 fields + int iterations = 500_000; + + System.out.printf("Beginning small projection profiling (%,d iterations)...%n", iterations); + long start = System.nanoTime(); + + for (int i = 0; i < iterations; i++) { + // This is the hotspot we want to profile + var projected = ImprintOperations.project(sourceRecord, projectFields); + + // Simulate some usage to prevent dead code elimination + if (i % 10_000 == 0) { + projected.getValue(1); // Trigger value decoding + projected.getRawBytes(5); // Trigger raw access } - }); + projected.serializeToBuffer(); + } + + long duration = System.nanoTime() - start; + System.out.printf("Small projections: %.2f ms (avg: %.1f μs/projection)%n", + duration / 1_000_000.0, (double) duration / iterations / 1000.0); } - @Test - void profileLargeObjectSerialization() throws Exception { - var schemaId = new SchemaId(3, 0xabcdef12); - var largeRecord = createVeryLargeRecord(); // A single large record to be re-serialized - - runProfileTest("Serialization (Large Object)", () -> { - // Re-serialize the same large object to focus on serialization logic - // rather than object creation. - for (int i = 0; i < 100_000; i++) { - var serialized = largeRecord.serializeToBuffer(); // Hotspot - - if (i % 1000 == 0) { - var deserialized = ImprintRecord.deserialize(serialized); - deserialized.getValue(10); // Access a field to ensure it works - } + /** + * Profile large projections (select 50-100 fields from 200-field records) + */ + private void profileLargeProjections() throws Exception { + System.out.println("\\n--- Large Projections (50 fields from 200-field records) ---"); + + var sourceRecord = createTestRecord(200); + // Select every 4th field for projection + int[] projectFields = IntStream.range(0, 50) + .map(i -> (i * 4) + 1) + .toArray(); + int iterations = 50_000; + + System.out.printf("Beginning large projection profiling (%,d iterations, %d->%d fields)...%n", + iterations, 200, projectFields.length); + long start = System.nanoTime(); + + for (int i = 0; i < iterations; i++) { + var projected = ImprintOperations.project(sourceRecord, projectFields); + + // Periodically access some fields to simulate real usage + if (i % 1_000 == 0) { + projected.getValue(1); + projected.getValue(25); + projected.getValue(49); } - }); + projected.serializeToBuffer(); + } + + long duration = System.nanoTime() - start; + System.out.printf("Large projections: %.2f ms (avg: %.1f μs/projection)%n", + duration / 1_000_000.0, (double) duration / iterations / 1000.0); } - - @Test - void profileProjection() throws Exception { - var record = createLargeRecord(); - - runProfileTest("Projection", () -> { - // Simulate analytical workload - project subset of fields repeatedly - for (int i = 0; i < 50_000; i++) { - // Project 10 fields out of 100 (common analytical pattern) - for (int fieldId = 1; fieldId <= 10; fieldId++) { - var value = record.getValue(fieldId); - if (value != null) { - // Force materialization of string values - if (value.getTypeCode() == com.imprint.types.TypeCode.STRING) { - if (value instanceof Value.StringBufferValue) { - ((Value.StringBufferValue) value).getValue(); - } - } - } + + /** + * Profile selective projections with different selectivity patterns + */ + private void profileSelectiveProjections() throws Exception { + System.out.println("\\n--- Selective Projections (various patterns) ---"); + + var sourceRecord = createTestRecord(100); + Random random = new Random(42); + int iterations = 100_000; + + // Test different projection patterns + var patterns = new ProjectionPattern[]{ + new ProjectionPattern("First few fields", new int[]{1, 2, 3, 4, 5}), + new ProjectionPattern("Last few fields", new int[]{96, 97, 98, 99, 100}), + new ProjectionPattern("Scattered fields", new int[]{1, 15, 33, 67, 89, 100}), + new ProjectionPattern("Random fields", generateRandomFields(random, 100, 10)) + }; + + for (var pattern : patterns) { + System.out.printf("Testing pattern: %s (%d fields)%n", + pattern.name, pattern.fields.length); + + long start = System.nanoTime(); + + for (int i = 0; i < iterations; i++) { + var projected = ImprintOperations.project(sourceRecord, pattern.fields); + + // Simulate field access + if (i % 5_000 == 0) { + projected.getValue(pattern.fields[0]); } + projected.serializeToBuffer(); } - }); + + long duration = System.nanoTime() - start; + System.out.printf(" %s: %.2f ms (avg: %.1f μs/projection)%n", + pattern.name, duration / 1_000_000.0, (double) duration / iterations / 1000.0); + } } - + + /** + * Profile memory allocation patterns during projection + */ + private void profileProjectionMemoryAllocation() throws Exception { + System.out.println("\\n--- Projection Memory Allocation Profiling ---"); + System.out.println("Watch for allocation hotspots and GC pressure..."); + + var sourceRecord = createTestRecord(50); + int[] projectFields = {1, 5, 10, 15, 20, 25}; // 6 fields + + System.out.println("Beginning projection allocation test..."); + + // Create allocation pressure to identify hotspots + for (int batch = 0; batch < 1000; batch++) { + for (int i = 0; i < 1000; i++) { + // This should reveal allocation hotspots in: + // 1. ArrayList creation + // 2. ByteBuffer allocation for new payload + // 3. FieldRange objects + // 4. SimpleDirectoryEntry creation + var projected = ImprintOperations.project(sourceRecord, projectFields); + + // Force some field access to trigger additional allocations + projected.getValue(1); // String decoding allocation + projected.getValue(5); // Value wrapper allocation + projected.getRawBytes(10); // ByteBuffer slicing + } + + if (batch % 100 == 0) { + System.out.printf("Allocation batch %d/1000 complete%n", batch); + } + } + + System.out.println("Projection allocation test complete"); + } + + /** + * Profile the component operations within projection to identify bottlenecks + */ @Test - void profileMemoryAllocation() throws Exception { - runProfileTest("Memory Allocation", () -> { - // Force allocation pressure to reveal GC hotspots - for (int batch = 0; batch < 1000; batch++) { - for (int i = 0; i < 1000; i++) { - var schemaId = new SchemaId(batch, i); - var builder = ImprintRecord.builder(schemaId); - - // Create strings of varying sizes (allocation pressure) - builder.field(1, Value.fromString("small")) - .field(2, Value.fromString("medium-length-string-" + i)) - .field(3, Value.fromString("very-long-string-that-will-cause-more-allocation-pressure-" + batch + "-" + i)) - .field(4, Value.fromBytes(new byte[100 + i % 1000])); // Varying byte arrays - - var record = builder.build(); - - // Some deserialization to trigger string decoding allocations - record.getValue(2); - record.getValue(3); - } + void profileProjectionComponents() throws Exception { + System.out.println("\\n=== Projection Component Profiling ==="); + Thread.sleep(2000); + + var sourceRecord = createTestRecord(100); + int[] projectFields = {1, 10, 20, 30, 40, 50}; + int iterations = 100_000; - if (batch % 100 == 0) { - System.out.printf("Completed batch %d/1000%n", batch); + // Profile individual components that might be hotspots: + + // 1. Field ID sorting and deduplication + System.out.println("Profiling field ID sorting..."); + long start = System.nanoTime(); + for (int i = 0; i < iterations; i++) { + // This mimics the sorting done in project() + int[] sorted = Arrays.stream(projectFields).distinct().sorted().toArray(); + blackhole(sorted); // Prevent optimization + } + long sortTime = System.nanoTime() - start; + System.out.printf("Field sorting: %.2f ms (%.1f ns/op)%n", + sortTime / 1_000_000.0, (double) sortTime / iterations); + + // 2. Directory scanning and range calculation + System.out.println("Profiling directory scanning..."); + var directory = sourceRecord.getDirectory(); + start = System.nanoTime(); + for (int i = 0; i < iterations; i++) { + // Simulate the directory scanning logic + int foundFields = 0; + for (var entry : directory) { + for (int fieldId : projectFields) { + if (entry.getId() == fieldId) { + foundFields++; + break; + } } } - }, false); // Disable final time reporting as it's not relevant here + blackhole(foundFields); + } + long scanTime = System.nanoTime() - start; + System.out.printf("Directory scanning: %.2f ms (%.1f ns/op)%n", + scanTime / 1_000_000.0, (double) scanTime / iterations); + + // 3. ByteBuffer operations (payload copying) + System.out.println("Profiling ByteBuffer operations..."); + var payload = sourceRecord.getBuffers().getPayload(); + start = System.nanoTime(); + for (int i = 0; i < iterations / 10; i++) { // Fewer iterations for heavy operation + // Simulate payload copying + var newPayload = java.nio.ByteBuffer.allocate(100); + newPayload.order(java.nio.ByteOrder.LITTLE_ENDIAN); + + // Copy some ranges (like buildPayloadFromRanges does) + for (int j = 0; j < 6; j++) { + var slice = payload.duplicate(); + slice.position(j * 10).limit((j + 1) * 10); + newPayload.put(slice); + } + newPayload.flip(); + blackhole(newPayload); + } + long bufferTime = System.nanoTime() - start; + System.out.printf("ByteBuffer operations: %.2f ms (%.1f μs/op)%n", + bufferTime / 1_000_000.0, (double) bufferTime / (iterations / 10) / 1000.0); } - - // ========== Test Helpers ========== /** - * A wrapper to run a profiling test with boilerplate for timing and setup. - * @param testName The name of the test to print. - * @param testLogic The core logic of the test, passed as a lambda. + * Profile serialization performance with records of a given size. + * This method abstracts the core serialization profiling logic to work + * with records of different sizes and complexities. */ - private void runProfileTest(String testName, ThrowingRunnable testLogic) throws Exception { - runProfileTest(testName, testLogic, true); - } - - private void runProfileTest(String testName, ThrowingRunnable testLogic, boolean reportTime) throws Exception { - System.out.printf("===== Starting Profiler Test: %s =====%n", testName); - System.out.println("Attach profiler now..."); - Thread.sleep(3000); // Give time to attach profiler + private void profileSerialization(String testName, int recordSize, int iterations) throws Exception { + System.out.printf("Starting %s serialization profiler test...%n", testName); + Thread.sleep(3000); - System.out.printf("Beginning %s profiling...%n", testName); + var schemaId = new SchemaId(1, 0x12345678); + + System.out.printf("Beginning %s serialization profiling (%,d iterations, %d fields)...%n", + testName, iterations, recordSize); long start = System.nanoTime(); - testLogic.run(); + // Create and serialize many records (allocation hotspot) + for (int i = 0; i < iterations; i++) { + var builder = ImprintRecord.builder(schemaId); - if (reportTime) { - long duration = System.nanoTime() - start; - System.out.printf("===== Completed %s in %.2f ms =====%n%n", testName, duration / 1_000_000.0); - } else { - System.out.printf("===== %s profiling complete. Check profiler output. =====%n%n", testName); + // Add various field types based on recordSize + for (int fieldId = 1; fieldId <= recordSize; fieldId++) { + switch (fieldId % 7) { + case 0: + builder.field(fieldId, Value.fromInt32(i + fieldId)); + break; + case 1: + builder.field(fieldId, Value.fromInt64(i * 1000L + fieldId)); + break; + case 2: + builder.field(fieldId, Value.fromString("test-string-" + i + "-" + fieldId)); + break; + case 3: + builder.field(fieldId, Value.fromString("longer-descriptive-text-for-field-" + fieldId + "-iteration-" + i)); + break; + case 4: + builder.field(fieldId, Value.fromFloat64(i * 3.14159 + fieldId)); + break; + case 5: + builder.field(fieldId, Value.fromBytes(("bytes-" + i + "-" + fieldId).getBytes())); + break; + case 6: + builder.field(fieldId, Value.fromBoolean((i + fieldId) % 2 == 0)); + break; + } + } + + var record = builder.build(); + var serialized = record.serializeToBuffer(); + + // Trigger some deserialization periodically + if (i % Math.max(1, iterations / 100) == 0) { + var deserialized = ImprintRecord.deserialize(serialized); + // Access a few random fields to trigger value decoding + for (int fieldId = 1; fieldId <= Math.min(5, recordSize); fieldId++) { + deserialized.getValue(fieldId); // String decoding hotspot + } + } + + // Progress indicator for long-running tests + if (i > 0 && i % Math.max(1, iterations / 10) == 0) { + System.out.printf("Completed %,d/%,d iterations (%.1f%%)%n", + i, iterations, (double) i / iterations * 100); + } } + + long duration = System.nanoTime() - start; + System.out.printf("Completed %s serialization test in %.2f ms (avg: %.1f μs/record)%n", + testName, duration / 1_000_000.0, (double) duration / iterations / 1000.0); } - /** A functional interface that allows for exceptions, for use in lambdas. */ - @FunctionalInterface - private interface ThrowingRunnable { - void run() throws Exception; + @Test + void profileMemoryAllocation() throws Exception { + System.out.println("Starting allocation profiler test..."); + Thread.sleep(3000); + + System.out.println("Beginning allocation profiling - watch for GC events..."); + + // Force allocation pressure to reveal GC hotspots + for (int batch = 0; batch < 1000; batch++) { + for (int i = 0; i < 1000; i++) { + var schemaId = new SchemaId(batch, i); + var builder = ImprintRecord.builder(schemaId); + + // Create strings of varying sizes (allocation pressure) + builder.field(1, Value.fromString("small")) + .field(2, Value.fromString("medium-length-string-" + i)) + .field(3, Value.fromString("very-long-string-that-will-cause-more-allocation-pressure-" + batch + "-" + i)) + .field(4, Value.fromBytes(new byte[100 + i % 1000])); // Varying byte arrays + + var record = builder.build(); + + // Some deserialization to trigger string decoding allocations + record.getValue(2); + record.getValue(3); + } + + if (batch % 100 == 0) { + System.out.printf("Completed batch %d/1000%n", batch); + } + } + + System.out.println("Allocation test complete - check GC logs and memory profiler"); } + // Helper methods and classes + private ImprintRecord createTestRecord() throws Exception { + return createTestRecord(RECORD_SIZE); + } + + private ImprintRecord createTestRecord(int recordSize) throws Exception { var builder = ImprintRecord.builder(new SchemaId(1, 0xdeadbeef)); - - for (int i = 1; i <= RECORD_SIZE; i++) { + + for (int i = 1; i <= recordSize; i++) { switch (i % 4) { case 0: builder.field(i, Value.fromInt32(i * 100)); @@ -219,72 +439,31 @@ private ImprintRecord createTestRecord() throws Exception { break; } } - + return builder.build(); } - - private ImprintRecord createLargeRecord() throws Exception { - var builder = ImprintRecord.builder(new SchemaId(2, 0xcafebabe)); - - // Create 100 fields with realistic data - for (int i = 1; i <= 100; i++) { - switch (i % 5) { - case 0: - builder.field(i, Value.fromInt32(i)); - break; - case 1: - builder.field(i, Value.fromString("user-name-" + i + "@example.com")); - break; - case 2: - builder.field(i, Value.fromString("Some longer descriptive text for field " + i + " that might represent a comment or description")); - break; - case 3: - builder.field(i, Value.fromFloat64(i * 2.718281828)); - break; - case 4: - builder.field(i, Value.fromBytes(String.format("binary-data-%04d", i).getBytes())); - break; - } + + private static class ProjectionPattern { + final String name; + final int[] fields; + + ProjectionPattern(String name, int[] fields) { + this.name = name; + this.fields = fields; } - - return builder.build(); } - private ImprintRecord createVeryLargeRecord() throws Exception { - var builder = ImprintRecord.builder(new SchemaId(3, 0xabcdef12)); - var random = new Random(123); + private int[] generateRandomFields(Random random, int maxField, int count) { + return random.ints(count, 1, maxField + 1) + .distinct() + .sorted() + .toArray(); + } - // Create 200 fields of varying types and sizes - for (int i = 1; i <= 200; i++) { - switch (i % 6) { - case 0: - builder.field(i, i * random.nextInt()); - break; - case 1: - // Medium string - builder.field(i, "user-id-" + UUID.randomUUID().toString()); - break; - case 2: - // Large string - builder.field(i, "This is a much larger text block for field " + i + ". It simulates a user comment, a description, or some other form of semi-structured text data. We repeat a sentence to make it longer. This is a much larger text block for field " + i + ". It simulates a user comment, a description, or some other form of semi-structured text data."); - break; - case 3: - builder.field(i, random.nextDouble() * 1000); - break; - case 4: - // Small byte array - var smallBytes = new byte[32]; - random.nextBytes(smallBytes); - builder.field(i, smallBytes); - break; - case 5: - // Large byte array - var largeBytes = new byte[1024]; - random.nextBytes(largeBytes); - builder.field(i, largeBytes); - break; - } + private void blackhole(Object obj) { + // Prevent dead code elimination + if (obj.hashCode() == System.nanoTime()) { + System.out.println("Never happens"); } - return builder.build(); } } \ No newline at end of file From b4cf85d6f81eab08ba165c0a6cbcf6db78cfa7f3 Mon Sep 17 00:00:00 2001 From: expand3d <> Date: Tue, 10 Jun 2025 20:38:39 -0400 Subject: [PATCH 45/49] final optimization and reorganization into better project structure --- build.gradle | 45 +- .../benchmark/ComparisonBenchmark.java | 44 +- .../com/imprint/benchmark/Competitor.java | 12 - .../com/imprint/benchmark/MergeBenchmark.java | 157 ------- ...java => AbstractSerializingBenchmark.java} | 4 +- ...tor.java => AvroSerializingBenchmark.java} | 4 +- ...a => FlatBuffersSerializingBenchmark.java} | 5 +- ....java => ImprintSerializingBenchmark.java} | 7 +- ....java => JacksonSerializingBenchmark.java} | 4 +- ...tor.java => KryoSerializingBenchmark.java} | 4 +- ...a => MessagePackSerializingBenchmark.java} | 4 +- ...java => ProtobufSerializingBenchmark.java} | 4 +- ...petitor.java => SerializingBenchmark.java} | 2 +- ...r.java => ThriftSerializingBenchmark.java} | 5 +- src/jmh/sbe/test_record.xml | 61 --- src/main/java/com/imprint/core/Directory.java | 69 +++ .../java/com/imprint/core/DirectoryEntry.java | 24 -- .../java/com/imprint/core/ImprintBuffers.java | 83 ++-- .../java/com/imprint/core/ImprintRecord.java | 99 +---- .../imprint/core/ImprintRecordBuilder.java | 31 +- .../imprint/core/SimpleDirectoryEntry.java | 22 - .../{core => ops}/ImprintOperations.java | 51 ++- .../{core => stream}/ImprintStream.java | 98 ++--- src/main/java/com/imprint/types/Value.java | 2 +- .../{core => ops}/ImprintOperationsTest.java | 22 +- .../com/imprint/profile/ProfilerTest.java | 394 ++++++++---------- .../{core => stream}/ImprintStreamTest.java | 17 +- 27 files changed, 414 insertions(+), 860 deletions(-) delete mode 100644 src/jmh/java/com/imprint/benchmark/Competitor.java delete mode 100644 src/jmh/java/com/imprint/benchmark/MergeBenchmark.java rename src/jmh/java/com/imprint/benchmark/competitors/{AbstractCompetitor.java => AbstractSerializingBenchmark.java} (83%) rename src/jmh/java/com/imprint/benchmark/competitors/{AvroCompetitor.java => AvroSerializingBenchmark.java} (98%) rename src/jmh/java/com/imprint/benchmark/competitors/{FlatBuffersCompetitor.java => FlatBuffersSerializingBenchmark.java} (97%) rename src/jmh/java/com/imprint/benchmark/competitors/{ImprintCompetitor.java => ImprintSerializingBenchmark.java} (95%) rename src/jmh/java/com/imprint/benchmark/competitors/{JacksonJsonCompetitor.java => JacksonSerializingBenchmark.java} (96%) rename src/jmh/java/com/imprint/benchmark/competitors/{KryoCompetitor.java => KryoSerializingBenchmark.java} (97%) rename src/jmh/java/com/imprint/benchmark/competitors/{MessagePackCompetitor.java => MessagePackSerializingBenchmark.java} (95%) rename src/jmh/java/com/imprint/benchmark/competitors/{ProtobufCompetitor.java => ProtobufSerializingBenchmark.java} (96%) rename src/jmh/java/com/imprint/benchmark/competitors/{Competitor.java => SerializingBenchmark.java} (92%) rename src/jmh/java/com/imprint/benchmark/competitors/{ThriftCompetitor.java => ThriftSerializingBenchmark.java} (97%) delete mode 100644 src/jmh/sbe/test_record.xml create mode 100644 src/main/java/com/imprint/core/Directory.java delete mode 100644 src/main/java/com/imprint/core/DirectoryEntry.java delete mode 100644 src/main/java/com/imprint/core/SimpleDirectoryEntry.java rename src/main/java/com/imprint/{core => ops}/ImprintOperations.java (82%) rename src/main/java/com/imprint/{core => stream}/ImprintStream.java (69%) rename src/test/java/com/imprint/{core => ops}/ImprintOperationsTest.java (96%) rename src/test/java/com/imprint/{core => stream}/ImprintStreamTest.java (85%) diff --git a/build.gradle b/build.gradle index d3480e6..26b2be5 100644 --- a/build.gradle +++ b/build.gradle @@ -42,7 +42,7 @@ dependencies { // Suppress SLF4J warnings jmhImplementation 'org.slf4j:slf4j-nop:1.7.36' - // Competitor libraries for benchmarking (JMH only) + // Other serialization libraries for benchmarking (JMH only) jmhImplementation 'com.google.protobuf:protobuf-java:3.25.1' jmhImplementation 'org.apache.avro:avro:1.11.3' jmhImplementation 'com.fasterxml.jackson.core:jackson-databind:2.16.0' @@ -51,15 +51,8 @@ dependencies { jmhImplementation 'org.msgpack:msgpack-core:0.9.8' jmhImplementation 'org.msgpack:jackson-dataformat-msgpack:0.9.8' jmhImplementation 'org.apache.thrift:libthrift:0.19.0' - - // SBE for benchmarking - jmhImplementation 'uk.co.real-logic:sbe-all:1.35.3' - jmhImplementation 'io.aeron:aeron-client:1.41.2' // SBE has a dependency on Agrona, included in aeron-client - - // FastUtil for high-performance primitive collections - implementation 'it.unimi.dsi:fastutil:8.5.12' // Required for generated Thrift code on JDK 11+ - implementation 'javax.annotation:javax.annotation-api:1.3.2' + jmhImplementation 'javax.annotation:javax.annotation-api:1.3.2' } protobuf { @@ -148,9 +141,10 @@ tasks.register('generateFlatBuffers', Exec) { } // Task to download the Thrift compiler -task downloadThrift(type: Exec) { +tasks.register('downloadThrift', Exec) { description = 'Download Thrift compiler' group = 'build setup' + def thriftVersion = "0.19.0" def thriftExecutable = file("${buildDir}/thrift/thrift.exe") def thriftUrl = "https://archive.apache.org/dist/thrift/${thriftVersion}/thrift-${thriftVersion}.exe" @@ -174,8 +168,8 @@ task downloadThrift(type: Exec) { } // Task to generate Java code from Thrift IDL files for JMH benchmarks -task generateJmhThrift(type: Exec) { - dependsOn downloadThrift +tasks.register('generateJmhThrift', Exec) { + dependsOn tasks.downloadThrift description = 'Generate Java classes from Thrift schema' group = 'build' @@ -196,33 +190,6 @@ task generateJmhThrift(type: Exec) { } } -// Task for SBE code generation -task generateSbe(type: JavaExec) { - description = 'Generate Java classes from SBE schema' - group = 'build' - - def outputDir = file("${buildDir}/generated/sbe/java") - def schemaFile = file('src/jmh/sbe/schema.xml') - def sbeXsd = file('src/jmh/sbe/sbe.xsd') - - // Ensure the sbe-tool is on the classpath for this task - classpath = sourceSets.jmh.runtimeClasspath - - main = 'uk.co.real_logic.sbe.SbeTool' - systemProperties = [ - "sbe.output.dir": outputDir.absolutePath, - "sbe.validation.xsd": sbeXsd.absolutePath - ] - args = [ schemaFile.absolutePath ] - - inputs.file(schemaFile) - inputs.file(sbeXsd) - outputs.dir(outputDir) - - doFirst { - outputDir.mkdirs() - } -} // Add generated FlatBuffers sources to JMH source set sourceSets { diff --git a/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java b/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java index f2c7398..92b3ceb 100644 --- a/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java +++ b/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java @@ -1,7 +1,7 @@ package com.imprint.benchmark; import com.imprint.benchmark.competitors.*; -import com.imprint.benchmark.competitors.Competitor; +import com.imprint.benchmark.competitors.SerializingBenchmark; import org.openjdk.jmh.annotations.*; import org.openjdk.jmh.infra.Blackhole; import org.openjdk.jmh.runner.Runner; @@ -9,7 +9,6 @@ import org.openjdk.jmh.runner.options.Options; import org.openjdk.jmh.runner.options.OptionsBuilder; -import java.util.Arrays; import java.util.List; import java.util.concurrent.TimeUnit; @@ -21,61 +20,60 @@ @Fork(value = 1, jvmArgs = {"-Xms4g", "-Xmx4g"}) public class ComparisonBenchmark { - private static final List COMPETITORS = Arrays.asList( - new ImprintCompetitor(), - new JacksonJsonCompetitor(), - new ProtobufCompetitor(), - new FlatBuffersCompetitor(), - new AvroCompetitor(), - new ThriftCompetitor(), - new KryoCompetitor(), - new MessagePackCompetitor() + private static final List FRAMEWORKS = List.of( + new ImprintSerializingBenchmark(), + new JacksonSerializingBenchmark(), + new ProtobufSerializingBenchmark(), + new FlatBuffersSerializingBenchmark(), + new AvroSerializingBenchmark(), + new ThriftSerializingBenchmark(), + new KryoSerializingBenchmark(), + new MessagePackSerializingBenchmark() ); @Param({"Imprint", "Jackson-JSON", "Protobuf", "FlatBuffers", "Avro-Generic", "Thrift", "Kryo", "MessagePack"}) - public String competitorName; + public String framework; - private Competitor competitor; + private SerializingBenchmark serializingBenchmark; @Setup(Level.Trial) public void setup() { - // Find the competitor implementation - competitor = COMPETITORS.stream() - .filter(c -> c.name().equals(competitorName)) + serializingBenchmark = FRAMEWORKS.stream() + .filter(c -> c.name().equals(framework)) .findFirst() - .orElseThrow(() -> new IllegalStateException("Unknown competitor: " + competitorName)); + .orElseThrow(() -> new IllegalStateException("Unknown framework: " + framework)); // Create the test data DataGenerator.TestRecord testRecord1 = DataGenerator.createTestRecord(); DataGenerator.TestRecord testRecord2 = DataGenerator.createTestRecord(); // Setup the competitor with the data - competitor.setup(testRecord1, testRecord2); + serializingBenchmark.setup(testRecord1, testRecord2); } @Benchmark public void serialize(Blackhole bh) { - competitor.serialize(bh); + serializingBenchmark.serialize(bh); } @Benchmark public void deserialize(Blackhole bh) { - competitor.deserialize(bh); + serializingBenchmark.deserialize(bh); } @Benchmark public void projectAndSerialize(Blackhole bh) { - competitor.projectAndSerialize(bh); + serializingBenchmark.projectAndSerialize(bh); } @Benchmark public void mergeAndSerialize(Blackhole bh) { - competitor.mergeAndSerialize(bh); + serializingBenchmark.mergeAndSerialize(bh); } @Benchmark public void accessField(Blackhole bh) { - competitor.accessField(bh); + serializingBenchmark.accessField(bh); } public static void main(String[] args) throws RunnerException { diff --git a/src/jmh/java/com/imprint/benchmark/Competitor.java b/src/jmh/java/com/imprint/benchmark/Competitor.java deleted file mode 100644 index 5f92929..0000000 --- a/src/jmh/java/com/imprint/benchmark/Competitor.java +++ /dev/null @@ -1,12 +0,0 @@ -package com.imprint.benchmark; - -import org.openjdk.jmh.infra.Blackhole; - -public interface Competitor { - String name(); - void setup(); - void serialize(Blackhole bh); - void deserialize(Blackhole bh); - void projectAndSerialize(Blackhole bh); - void mergeAndSerialize(Blackhole bh); -} \ No newline at end of file diff --git a/src/jmh/java/com/imprint/benchmark/MergeBenchmark.java b/src/jmh/java/com/imprint/benchmark/MergeBenchmark.java deleted file mode 100644 index 63e43e6..0000000 --- a/src/jmh/java/com/imprint/benchmark/MergeBenchmark.java +++ /dev/null @@ -1,157 +0,0 @@ -package com.imprint.benchmark; - -import com.imprint.core.ImprintRecord; -import com.imprint.core.ImprintRecordBuilder; -import com.imprint.core.SchemaId; -import com.imprint.types.Value; -import org.openjdk.jmh.annotations.*; -import org.openjdk.jmh.infra.Blackhole; - -import java.util.HashSet; -import java.util.Set; -import java.util.concurrent.TimeUnit; - -/** - * Benchmarks for ImprintRecord merge operations. - * NOTE: These benchmarks simulate merge operations until the actual merge API is implemented. - */ -@BenchmarkMode(Mode.AverageTime) -@OutputTimeUnit(TimeUnit.NANOSECONDS) -@State(Scope.Benchmark) -@Warmup(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS) -@Measurement(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS) -@Fork(1) -public class MergeBenchmark { - - private ImprintRecord productRecord; - private ImprintRecord orderRecord; - private ImprintRecord customerRecord; - - @Setup - public void setup() throws Exception { - productRecord = createProductRecord(); - orderRecord = createOrderRecord(); - customerRecord = createCustomerRecord(); - } - - // ===== SIMULATED MERGE BENCHMARKS ===== - // These will be replaced with actual merge API when implemented - - @Benchmark - public void mergeProductAndOrder(Blackhole bh) throws Exception { - // Simulate merge by creating a new record with fields from both - ImprintRecord result = simulateMerge(productRecord, orderRecord); - bh.consume(result); - } - - @Benchmark - public void mergeProductAndCustomer(Blackhole bh) throws Exception { - ImprintRecord result = simulateMerge(productRecord, customerRecord); - bh.consume(result); - } - - @Benchmark - public void mergeOrderAndCustomer(Blackhole bh) throws Exception { - ImprintRecord result = simulateMerge(orderRecord, customerRecord); - bh.consume(result); - } - - @Benchmark - public void mergeThreeRecords(Blackhole bh) throws Exception { - // Test merging multiple records - var temp = simulateMerge(productRecord, orderRecord); - ImprintRecord result = simulateMerge(temp, customerRecord); - bh.consume(result); - } - - // ===== MERGE CONFLICT HANDLING ===== - - @Benchmark - public void mergeWithConflicts(Blackhole bh) throws Exception { - // Create records with overlapping field IDs to test conflict resolution - var record1 = createRecordWithFields(1, 50, "record1_"); - var record2 = createRecordWithFields(25, 75, "record2_"); - - ImprintRecord result = simulateMerge(record1, record2); - bh.consume(result); - } - - // ===== HELPER METHODS ===== - - /** - * Simulates merge operation by manually copying fields. - * This should be replaced with actual merge API when available. - */ - private ImprintRecord simulateMerge(ImprintRecord first, ImprintRecord second) throws Exception { - var builder = ImprintRecord.builder(first.getHeader().getSchemaId()); - var usedFieldIds = new HashSet(); - - // Copy fields from first record (takes precedence) - copyFieldsToBuilder(first, builder, usedFieldIds); - - // Copy non-conflicting fields from second record - copyFieldsToBuilder(second, builder, usedFieldIds); - - return builder.build(); - } - - private void copyFieldsToBuilder(ImprintRecord record, ImprintRecordBuilder builder, Set usedFieldIds) throws Exception { - for (var entry : record.getDirectory()) { - int fieldId = entry.getId(); - if (!usedFieldIds.contains(fieldId)) { - var value = record.getValue(fieldId); - if (value != null) { - builder.field(fieldId, value); - usedFieldIds.add(fieldId); - } - } - } - } - - private ImprintRecord createProductRecord() throws Exception { - return ImprintRecord.builder(new SchemaId(1, 0x12345678)) - .field(1, Value.fromString("Product")) - .field(2, Value.fromInt32(12345)) - .field(3, Value.fromString("Laptop")) - .field(4, Value.fromFloat64(999.99)) - .field(5, Value.fromString("Electronics")) - .field(6, Value.fromInt32(50)) // stock - .field(7, Value.fromString("TechCorp")) - .field(8, Value.fromBoolean(true)) // available - .build(); - } - - private ImprintRecord createOrderRecord() throws Exception { - return ImprintRecord.builder(new SchemaId(2, 0x87654321)) - .field(10, Value.fromString("Order")) - .field(11, Value.fromInt32(67890)) - .field(12, Value.fromInt32(12345)) // product_id (overlaps with product) - .field(13, Value.fromInt32(2)) // quantity - .field(14, Value.fromFloat64(1999.98)) // total - .field(15, Value.fromString("2024-01-15")) // order_date - .field(16, Value.fromString("shipped")) // status - .build(); - } - - private ImprintRecord createCustomerRecord() throws Exception { - return ImprintRecord.builder(new SchemaId(3, 0x11223344)) - .field(20, Value.fromString("Customer")) - .field(21, Value.fromInt32(555)) - .field(22, Value.fromString("John Doe")) - .field(23, Value.fromString("john.doe@email.com")) - .field(24, Value.fromString("123 Main St")) - .field(25, Value.fromString("premium")) // tier - .field(26, Value.fromBoolean(true)) // active - .build(); - } - - private ImprintRecord createRecordWithFields(int startId, int endId, String prefix) throws Exception { - var builder = ImprintRecord.builder(new SchemaId(1, 0x12345678)); - - for (int i = startId; i <= endId; i++) { - builder.field(i, Value.fromString(prefix + "field_" + i)); - } - - return builder.build(); - } -} \ No newline at end of file diff --git a/src/jmh/java/com/imprint/benchmark/competitors/AbstractCompetitor.java b/src/jmh/java/com/imprint/benchmark/competitors/AbstractSerializingBenchmark.java similarity index 83% rename from src/jmh/java/com/imprint/benchmark/competitors/AbstractCompetitor.java rename to src/jmh/java/com/imprint/benchmark/competitors/AbstractSerializingBenchmark.java index bfdea2a..2f5476c 100644 --- a/src/jmh/java/com/imprint/benchmark/competitors/AbstractCompetitor.java +++ b/src/jmh/java/com/imprint/benchmark/competitors/AbstractSerializingBenchmark.java @@ -6,13 +6,13 @@ /** * A minimal base class for competitors, holding the test data. */ -public abstract class AbstractCompetitor implements Competitor { +public abstract class AbstractSerializingBenchmark implements SerializingBenchmark { protected final String name; protected DataGenerator.TestRecord testData; protected DataGenerator.TestRecord testData2; - protected AbstractCompetitor(String name) { + protected AbstractSerializingBenchmark(String name) { this.name = name; } diff --git a/src/jmh/java/com/imprint/benchmark/competitors/AvroCompetitor.java b/src/jmh/java/com/imprint/benchmark/competitors/AvroSerializingBenchmark.java similarity index 98% rename from src/jmh/java/com/imprint/benchmark/competitors/AvroCompetitor.java rename to src/jmh/java/com/imprint/benchmark/competitors/AvroSerializingBenchmark.java index 71c8306..dc7278c 100644 --- a/src/jmh/java/com/imprint/benchmark/competitors/AvroCompetitor.java +++ b/src/jmh/java/com/imprint/benchmark/competitors/AvroSerializingBenchmark.java @@ -12,7 +12,7 @@ import java.io.ByteArrayOutputStream; import java.nio.ByteBuffer; -public class AvroCompetitor extends AbstractCompetitor { +public class AvroSerializingBenchmark extends AbstractSerializingBenchmark { private final Schema schema; private final Schema projectedSchema; @@ -22,7 +22,7 @@ public class AvroCompetitor extends AbstractCompetitor { private byte[] serializedRecord1; private byte[] serializedRecord2; - public AvroCompetitor() { + public AvroSerializingBenchmark() { super("Avro-Generic"); String schemaDefinition = "{\"type\":\"record\",\"name\":\"TestRecord\",\"fields\":[" + "{\"name\":\"id\",\"type\":\"string\"}," diff --git a/src/jmh/java/com/imprint/benchmark/competitors/FlatBuffersCompetitor.java b/src/jmh/java/com/imprint/benchmark/competitors/FlatBuffersSerializingBenchmark.java similarity index 97% rename from src/jmh/java/com/imprint/benchmark/competitors/FlatBuffersCompetitor.java rename to src/jmh/java/com/imprint/benchmark/competitors/FlatBuffersSerializingBenchmark.java index bd51eb9..a9fe5c8 100644 --- a/src/jmh/java/com/imprint/benchmark/competitors/FlatBuffersCompetitor.java +++ b/src/jmh/java/com/imprint/benchmark/competitors/FlatBuffersSerializingBenchmark.java @@ -6,14 +6,13 @@ import org.openjdk.jmh.infra.Blackhole; import java.nio.ByteBuffer; -import java.util.stream.Collectors; -public class FlatBuffersCompetitor extends AbstractCompetitor { +public class FlatBuffersSerializingBenchmark extends AbstractSerializingBenchmark { private ByteBuffer serializedRecord1; private ByteBuffer serializedRecord2; - public FlatBuffersCompetitor() { + public FlatBuffersSerializingBenchmark() { super("FlatBuffers"); } diff --git a/src/jmh/java/com/imprint/benchmark/competitors/ImprintCompetitor.java b/src/jmh/java/com/imprint/benchmark/competitors/ImprintSerializingBenchmark.java similarity index 95% rename from src/jmh/java/com/imprint/benchmark/competitors/ImprintCompetitor.java rename to src/jmh/java/com/imprint/benchmark/competitors/ImprintSerializingBenchmark.java index 3e05cd7..26bb495 100644 --- a/src/jmh/java/com/imprint/benchmark/competitors/ImprintCompetitor.java +++ b/src/jmh/java/com/imprint/benchmark/competitors/ImprintSerializingBenchmark.java @@ -1,24 +1,23 @@ package com.imprint.benchmark.competitors; import com.imprint.benchmark.DataGenerator; -import com.imprint.core.ImprintOperations; +import com.imprint.ops.ImprintOperations; import com.imprint.core.ImprintRecord; import com.imprint.core.ImprintRecordBuilder; import com.imprint.core.SchemaId; import com.imprint.error.ImprintException; -import lombok.SneakyThrows; import org.openjdk.jmh.infra.Blackhole; import java.nio.ByteBuffer; -public class ImprintCompetitor extends AbstractCompetitor { +public class ImprintSerializingBenchmark extends AbstractSerializingBenchmark { private ImprintRecord imprintRecord1; private byte[] serializedRecord1; private byte[] serializedRecord2; private static final SchemaId SCHEMA_ID = new SchemaId(1, 1); - public ImprintCompetitor() { + public ImprintSerializingBenchmark() { super("Imprint"); } diff --git a/src/jmh/java/com/imprint/benchmark/competitors/JacksonJsonCompetitor.java b/src/jmh/java/com/imprint/benchmark/competitors/JacksonSerializingBenchmark.java similarity index 96% rename from src/jmh/java/com/imprint/benchmark/competitors/JacksonJsonCompetitor.java rename to src/jmh/java/com/imprint/benchmark/competitors/JacksonSerializingBenchmark.java index a32e9a8..829b073 100644 --- a/src/jmh/java/com/imprint/benchmark/competitors/JacksonJsonCompetitor.java +++ b/src/jmh/java/com/imprint/benchmark/competitors/JacksonSerializingBenchmark.java @@ -4,13 +4,13 @@ import com.imprint.benchmark.DataGenerator; import org.openjdk.jmh.infra.Blackhole; -public class JacksonJsonCompetitor extends AbstractCompetitor { +public class JacksonSerializingBenchmark extends AbstractSerializingBenchmark { private final ObjectMapper mapper; private byte[] serializedRecord; private byte[] serializedRecord2; - public JacksonJsonCompetitor() { + public JacksonSerializingBenchmark() { super("Jackson-JSON"); this.mapper = new ObjectMapper(); } diff --git a/src/jmh/java/com/imprint/benchmark/competitors/KryoCompetitor.java b/src/jmh/java/com/imprint/benchmark/competitors/KryoSerializingBenchmark.java similarity index 97% rename from src/jmh/java/com/imprint/benchmark/competitors/KryoCompetitor.java rename to src/jmh/java/com/imprint/benchmark/competitors/KryoSerializingBenchmark.java index d76a937..1223e06 100644 --- a/src/jmh/java/com/imprint/benchmark/competitors/KryoCompetitor.java +++ b/src/jmh/java/com/imprint/benchmark/competitors/KryoSerializingBenchmark.java @@ -11,13 +11,13 @@ import java.util.Arrays; import java.util.HashMap; -public class KryoCompetitor extends AbstractCompetitor { +public class KryoSerializingBenchmark extends AbstractSerializingBenchmark { private final Kryo kryo; private byte[] serializedRecord1; private byte[] serializedRecord2; - public KryoCompetitor() { + public KryoSerializingBenchmark() { super("Kryo"); this.kryo = new Kryo(); this.kryo.register(DataGenerator.TestRecord.class); diff --git a/src/jmh/java/com/imprint/benchmark/competitors/MessagePackCompetitor.java b/src/jmh/java/com/imprint/benchmark/competitors/MessagePackSerializingBenchmark.java similarity index 95% rename from src/jmh/java/com/imprint/benchmark/competitors/MessagePackCompetitor.java rename to src/jmh/java/com/imprint/benchmark/competitors/MessagePackSerializingBenchmark.java index 65269e5..b596e6d 100644 --- a/src/jmh/java/com/imprint/benchmark/competitors/MessagePackCompetitor.java +++ b/src/jmh/java/com/imprint/benchmark/competitors/MessagePackSerializingBenchmark.java @@ -5,13 +5,13 @@ import org.msgpack.jackson.dataformat.MessagePackFactory; import org.openjdk.jmh.infra.Blackhole; -public class MessagePackCompetitor extends AbstractCompetitor { +public class MessagePackSerializingBenchmark extends AbstractSerializingBenchmark { private final ObjectMapper mapper; private byte[] serializedRecord; private byte[] serializedRecord2; - public MessagePackCompetitor() { + public MessagePackSerializingBenchmark() { super("MessagePack"); this.mapper = new ObjectMapper(new MessagePackFactory()); } diff --git a/src/jmh/java/com/imprint/benchmark/competitors/ProtobufCompetitor.java b/src/jmh/java/com/imprint/benchmark/competitors/ProtobufSerializingBenchmark.java similarity index 96% rename from src/jmh/java/com/imprint/benchmark/competitors/ProtobufCompetitor.java rename to src/jmh/java/com/imprint/benchmark/competitors/ProtobufSerializingBenchmark.java index 547abfe..72ad38f 100644 --- a/src/jmh/java/com/imprint/benchmark/competitors/ProtobufCompetitor.java +++ b/src/jmh/java/com/imprint/benchmark/competitors/ProtobufSerializingBenchmark.java @@ -4,12 +4,12 @@ import com.imprint.benchmark.proto.TestRecordOuterClass; import org.openjdk.jmh.infra.Blackhole; -public class ProtobufCompetitor extends AbstractCompetitor { +public class ProtobufSerializingBenchmark extends AbstractSerializingBenchmark { private byte[] serializedRecord1; private byte[] serializedRecord2; - public ProtobufCompetitor() { + public ProtobufSerializingBenchmark() { super("Protobuf"); } diff --git a/src/jmh/java/com/imprint/benchmark/competitors/Competitor.java b/src/jmh/java/com/imprint/benchmark/competitors/SerializingBenchmark.java similarity index 92% rename from src/jmh/java/com/imprint/benchmark/competitors/Competitor.java rename to src/jmh/java/com/imprint/benchmark/competitors/SerializingBenchmark.java index 717bbfc..595caa6 100644 --- a/src/jmh/java/com/imprint/benchmark/competitors/Competitor.java +++ b/src/jmh/java/com/imprint/benchmark/competitors/SerializingBenchmark.java @@ -6,7 +6,7 @@ /** * Defines the contract for a serialization competitor in the benchmark. */ -public interface Competitor { +public interface SerializingBenchmark { String name(); void setup(DataGenerator.TestRecord testRecord, DataGenerator.TestRecord testRecord2); void serialize(Blackhole bh); diff --git a/src/jmh/java/com/imprint/benchmark/competitors/ThriftCompetitor.java b/src/jmh/java/com/imprint/benchmark/competitors/ThriftSerializingBenchmark.java similarity index 97% rename from src/jmh/java/com/imprint/benchmark/competitors/ThriftCompetitor.java rename to src/jmh/java/com/imprint/benchmark/competitors/ThriftSerializingBenchmark.java index 18530b5..83c0812 100644 --- a/src/jmh/java/com/imprint/benchmark/competitors/ThriftCompetitor.java +++ b/src/jmh/java/com/imprint/benchmark/competitors/ThriftSerializingBenchmark.java @@ -9,18 +9,17 @@ import org.apache.thrift.protocol.TBinaryProtocol; import org.openjdk.jmh.infra.Blackhole; -import java.io.ByteArrayOutputStream; import java.nio.ByteBuffer; import java.util.stream.Collectors; -public class ThriftCompetitor extends AbstractCompetitor { +public class ThriftSerializingBenchmark extends AbstractSerializingBenchmark { private final TSerializer serializer; private final TDeserializer deserializer; private byte[] serializedRecord1; private byte[] serializedRecord2; - public ThriftCompetitor() { + public ThriftSerializingBenchmark() { super("Thrift"); try { this.serializer = new TSerializer(new TBinaryProtocol.Factory()); diff --git a/src/jmh/sbe/test_record.xml b/src/jmh/sbe/test_record.xml deleted file mode 100644 index 9feaee8..0000000 --- a/src/jmh/sbe/test_record.xml +++ /dev/null @@ -1,61 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/src/main/java/com/imprint/core/Directory.java b/src/main/java/com/imprint/core/Directory.java new file mode 100644 index 0000000..cb449b3 --- /dev/null +++ b/src/main/java/com/imprint/core/Directory.java @@ -0,0 +1,69 @@ +package com.imprint.core; + +import com.imprint.types.TypeCode; +import lombok.Getter; +import lombok.Setter; +import lombok.Value; + +import java.util.Objects; + +/** + * Represents the common interface for a directory entry in an Imprint record. + * A directory entry provides metadata about a field, such as its ID, type, and location in the payload. + */ +public interface Directory { + /** + * @return The field's unique identifier. + */ + short getId(); + + /** + * @return The {@link TypeCode} of the field's value. + */ + TypeCode getTypeCode(); + + /** + * @return The starting position (offset) of the field's data within the payload buffer. + */ + int getOffset(); + + /** + * Immutable representation of the Imprint Directory used for deserialization, + * merging, and field projections + */ + @Value + class Entry implements Directory { + short id; + TypeCode typeCode; + int offset; + + public Entry(short id, TypeCode typeCode, int offset) { + this.id = id; + this.typeCode = Objects.requireNonNull(typeCode, "TypeCode cannot be null"); + this.offset = offset; + } + } + + /** + * Mutable representation of the Imprint Directory bound with corresponding type value + * used for record building through {@link ImprintRecordBuilder} and subsequent serialization. + */ + @Getter + class Builder implements Directory { + private final short id; + private final com.imprint.types.Value value; + @Setter + private int offset; + + Builder(short id, com.imprint.types.Value value) { + this.id = id; + this.value = value; + this.offset = -1; + } + + @Override + public TypeCode getTypeCode() { + return value.getTypeCode(); + } + } +} \ No newline at end of file diff --git a/src/main/java/com/imprint/core/DirectoryEntry.java b/src/main/java/com/imprint/core/DirectoryEntry.java deleted file mode 100644 index 0b98433..0000000 --- a/src/main/java/com/imprint/core/DirectoryEntry.java +++ /dev/null @@ -1,24 +0,0 @@ -package com.imprint.core; - -import com.imprint.types.TypeCode; - -/** - * Represents the common interface for a directory entry in an Imprint record. - * A directory entry provides metadata about a field, such as its ID, type, and location in the payload. - */ -public interface DirectoryEntry { - /** - * @return The field's unique identifier. - */ - short getId(); - - /** - * @return The {@link TypeCode} of the field's value. - */ - TypeCode getTypeCode(); - - /** - * @return The starting position (offset) of the field's data within the payload buffer. - */ - int getOffset(); -} \ No newline at end of file diff --git a/src/main/java/com/imprint/core/ImprintBuffers.java b/src/main/java/com/imprint/core/ImprintBuffers.java index 4afa1fa..845892a 100644 --- a/src/main/java/com/imprint/core/ImprintBuffers.java +++ b/src/main/java/com/imprint/core/ImprintBuffers.java @@ -5,17 +5,11 @@ import com.imprint.error.ImprintException; import com.imprint.types.TypeCode; import com.imprint.util.VarInt; -import it.unimi.dsi.fastutil.ints.Int2ObjectAVLTreeMap; -import it.unimi.dsi.fastutil.ints.Int2ObjectSortedMap; import lombok.Getter; import java.nio.ByteBuffer; import java.nio.ByteOrder; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Objects; -import java.util.TreeMap; +import java.util.*; /** * Manages the raw buffers for an Imprint record with lazy directory parsing. @@ -35,7 +29,7 @@ public final class ImprintBuffers { private final ByteBuffer payload; // Read-only payload view // Lazy-loaded directory state. - private Int2ObjectSortedMap parsedDirectory; + private TreeMap parsedDirectory; private boolean directoryParsed = false; private int directoryCount = -1; @@ -51,7 +45,7 @@ public ImprintBuffers(ByteBuffer directoryBuffer, ByteBuffer payload) { * Creates buffers from a pre-sorted list of entries (most efficient builder path). * Immediately creates the parsed index and the serialized buffer. */ - public ImprintBuffers(List sortedDirectory, ByteBuffer payload) { + public ImprintBuffers(List sortedDirectory, ByteBuffer payload) { this.directoryBuffer = ImprintBuffers.createDirectoryBuffer(sortedDirectory); this.payload = payload.asReadOnlyBuffer(); } @@ -62,10 +56,10 @@ public ImprintBuffers(List sortedDirectory, ByteBuffer * map becomes the definitive parsed directory. */ @SuppressWarnings("unchecked") - public ImprintBuffers(Int2ObjectSortedMap parsedDirectory, ByteBuffer payload) { + public ImprintBuffers(Map parsedDirectory, ByteBuffer payload) { this.directoryBuffer = ImprintBuffers.createDirectoryBufferFromSortedMap(Objects.requireNonNull(parsedDirectory)); this.payload = payload.asReadOnlyBuffer(); - this.parsedDirectory = (Int2ObjectSortedMap) parsedDirectory; + this.parsedDirectory = (TreeMap) parsedDirectory; this.directoryParsed = true; this.directoryCount = parsedDirectory.size(); } @@ -97,7 +91,7 @@ public ByteBuffer getFieldBuffer(int fieldId) throws ImprintException { * Get a zero-copy ByteBuffer view of a field's data using a pre-fetched DirectoryEntry. * This avoids the cost of re-finding the entry. */ - public ByteBuffer getFieldBuffer(DirectoryEntry entry) throws ImprintException { + public ByteBuffer getFieldBuffer(Directory entry) throws ImprintException { if (entry == null) return null; @@ -122,7 +116,7 @@ public ByteBuffer getFieldBuffer(DirectoryEntry entry) throws ImprintException { * - If parsed: TreeMap lookup * - If raw: Binary search on raw bytes to avoid full unwinding of the directory */ - public DirectoryEntry findDirectoryEntry(int fieldId) throws ImprintException { + public Directory findDirectoryEntry(int fieldId) throws ImprintException { if (directoryParsed) return parsedDirectory.get(fieldId); else @@ -133,7 +127,7 @@ public DirectoryEntry findDirectoryEntry(int fieldId) throws ImprintException { * Get the full directory, parsing it if necessary. * Returns the values in fieldId order thanks to TreeMap. */ - public List getDirectory() { + public List getDirectory() { ensureDirectoryParsed(); return new ArrayList<>(parsedDirectory.values()); } @@ -168,7 +162,7 @@ public ByteBuffer serializeDirectory() { * Binary search on raw directory bytes to find a specific field. * This avoids parsing the entire directory for single field lookups. */ - private DirectoryEntry findFieldEntryInRawDirectory(int fieldId) throws ImprintException { + private Directory findFieldEntryInRawDirectory(int fieldId) throws ImprintException { var searchBuffer = directoryBuffer.duplicate(); searchBuffer.order(ByteOrder.LITTLE_ENDIAN); @@ -212,7 +206,7 @@ private DirectoryEntry findFieldEntryInRawDirectory(int fieldId) throws ImprintE /** * Find the end offset for a field by looking at the next field's offset. */ - private int findEndOffset(DirectoryEntry entry) throws ImprintException { + private int findEndOffset(Directory entry) throws ImprintException { if (directoryParsed) { return findNextOffsetInParsedDirectory(entry.getId()); } else { @@ -224,11 +218,12 @@ private int findEndOffset(DirectoryEntry entry) throws ImprintException { * Find the end offset using TreeMap's efficient navigation methods. */ private int findNextOffsetInParsedDirectory(int currentFieldId) { - var tailMap = parsedDirectory.tailMap(currentFieldId + 1); - if (tailMap.isEmpty()) { + var nextEntry = parsedDirectory.higherEntry(currentFieldId); + if (nextEntry != null) + return nextEntry.getValue().getOffset(); + else return payload.limit(); - } - return tailMap.get(tailMap.firstIntKey()).getOffset(); + } private int findNextOffsetInRawDirectory(int currentFieldId) throws ImprintException { @@ -284,18 +279,18 @@ private void ensureDirectoryParsed() { parseBuffer.order(ByteOrder.LITTLE_ENDIAN); int count = getOrParseDirectoryCount(parseBuffer); - this.parsedDirectory = new Int2ObjectAVLTreeMap<>(); + this.parsedDirectory = new TreeMap<>(); for (int i = 0; i < count; i++) { var entry = deserializeDirectoryEntry(parseBuffer); - this.parsedDirectory.put(entry.getId() , entry); + this.parsedDirectory.put((int) entry.getId() , entry); } this.directoryParsed = true; } catch (ImprintException e) { // This can happen with a corrupted directory. // In this case, we'll just have an empty (but valid) parsed directory. - this.parsedDirectory = new Int2ObjectAVLTreeMap<>(); + this.parsedDirectory = new TreeMap<>(); this.directoryParsed = true; // Mark as parsed to avoid repeated errors } } @@ -323,7 +318,7 @@ private int getOrParseDirectoryCount(ByteBuffer buffer) throws ImprintException * Creates a read-only buffer containing the serialized directory. * The input collection does not need to be sorted. */ - static ByteBuffer createDirectoryBuffer(Collection directory) { + static ByteBuffer createDirectoryBuffer(Collection directory) { if (directory == null || directory.isEmpty()) { ByteBuffer buffer = ByteBuffer.allocate(1); VarInt.encode(0, buffer); @@ -332,9 +327,9 @@ static ByteBuffer createDirectoryBuffer(Collection dir } // Ensure sorted order for binary search compatibility. - ArrayList sortedDirectory; - if (directory instanceof ArrayList && isSorted((ArrayList)directory)) { - sortedDirectory = (ArrayList) directory; + ArrayList sortedDirectory; + if (directory instanceof ArrayList && isSorted((ArrayList)directory)) { + sortedDirectory = (ArrayList) directory; } else { sortedDirectory = new ArrayList<>(directory); sortedDirectory.sort(null); @@ -346,7 +341,7 @@ static ByteBuffer createDirectoryBuffer(Collection dir buffer.order(ByteOrder.LITTLE_ENDIAN); VarInt.encode(count, buffer); - for (DirectoryEntry entry : sortedDirectory) { + for (var entry : sortedDirectory) { serializeDirectoryEntry(entry, buffer); } @@ -354,7 +349,7 @@ static ByteBuffer createDirectoryBuffer(Collection dir return buffer; } - static ByteBuffer createDirectoryBufferFromMap(TreeMap directoryMap) { + static ByteBuffer createDirectoryBufferFromSortedMap(Map directoryMap) { if (directoryMap == null || directoryMap.isEmpty()) { ByteBuffer buffer = ByteBuffer.allocate(1); VarInt.encode(0, buffer); @@ -376,29 +371,7 @@ static ByteBuffer createDirectoryBufferFromMap(TreeMap directoryMap) { - if (directoryMap == null || directoryMap.isEmpty()) { - ByteBuffer buffer = ByteBuffer.allocate(1); - VarInt.encode(0, buffer); - buffer.flip(); - return buffer; - } - - int count = directoryMap.size(); - int size = VarInt.encodedLength(count) + (count * Constants.DIR_ENTRY_BYTES); - var buffer = ByteBuffer.allocate(size); - buffer.order(ByteOrder.LITTLE_ENDIAN); - - VarInt.encode(count, buffer); - for (var entry : directoryMap.int2ObjectEntrySet()) { - serializeDirectoryEntry(entry.getValue(), buffer); - } - - buffer.flip(); - return buffer; - } - - private static boolean isSorted(ArrayList list) { + private static boolean isSorted(ArrayList list) { for (int i = 0; i < list.size() - 1; i++) { if (list.get(i).getId() > list.get(i + 1).getId()) { return false; @@ -411,7 +384,7 @@ private static boolean isSorted(ArrayList list) { * Serialize a single directory entry to the buffer. * Format: [fieldId:2bytes][typeCode:1byte][offset:4bytes] */ - private static void serializeDirectoryEntry(DirectoryEntry entry, ByteBuffer buffer) { + private static void serializeDirectoryEntry(Directory entry, ByteBuffer buffer) { buffer.putShort(entry.getId()); buffer.put(entry.getTypeCode().getCode()); buffer.putInt(entry.getOffset()); @@ -421,7 +394,7 @@ private static void serializeDirectoryEntry(DirectoryEntry entry, ByteBuffer buf * Deserialize a single directory entry from the buffer. * Reads: [fieldId:2bytes][typeCode:1byte][offset:4bytes] */ - private DirectoryEntry deserializeDirectoryEntry(ByteBuffer buffer) throws ImprintException { + private Directory deserializeDirectoryEntry(ByteBuffer buffer) throws ImprintException { if (buffer.remaining() < Constants.DIR_ENTRY_BYTES) throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, "Not enough bytes for directory entry"); @@ -429,6 +402,6 @@ private DirectoryEntry deserializeDirectoryEntry(ByteBuffer buffer) throws Impri var typeCode = TypeCode.fromByte(buffer.get()); int offset = buffer.getInt(); - return new SimpleDirectoryEntry(id, typeCode, offset); + return new Directory.Entry(id, typeCode, offset); } } \ No newline at end of file diff --git a/src/main/java/com/imprint/core/ImprintRecord.java b/src/main/java/com/imprint/core/ImprintRecord.java index e4c4a42..a34c7df 100644 --- a/src/main/java/com/imprint/core/ImprintRecord.java +++ b/src/main/java/com/imprint/core/ImprintRecord.java @@ -1,22 +1,20 @@ package com.imprint.core; import com.imprint.Constants; +import com.imprint.ops.ImprintOperations; import com.imprint.error.ErrorType; import com.imprint.error.ImprintException; import com.imprint.types.MapKey; import com.imprint.types.TypeCode; import com.imprint.types.Value; import com.imprint.util.VarInt; -import it.unimi.dsi.fastutil.ints.Int2ObjectSortedMap; import lombok.Getter; import java.nio.ByteBuffer; import java.nio.ByteOrder; -import java.util.Collection; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.TreeMap; /** * An Imprint record containing a header and buffer management. @@ -38,7 +36,7 @@ private ImprintRecord(Header header, ImprintBuffers buffers) { /** * Creates a record from a pre-sorted list of entries (most efficient builder path). */ - ImprintRecord(Header header, List sortedDirectory, ByteBuffer payload) { + public ImprintRecord(Header header, List sortedDirectory, ByteBuffer payload) { this.header = Objects.requireNonNull(header, "Header cannot be null"); this.buffers = new ImprintBuffers(sortedDirectory, payload); } @@ -46,7 +44,7 @@ private ImprintRecord(Header header, ImprintBuffers buffers) { /** * Creates a record from a pre-built and sorted FastUtil map (most efficient builder path). */ - ImprintRecord(Header header, Int2ObjectSortedMap parsedDirectory, ByteBuffer payload) { + public ImprintRecord(Header header, Map parsedDirectory, ByteBuffer payload) { this.header = Objects.requireNonNull(header, "Header cannot be null"); this.buffers = new ImprintBuffers(parsedDirectory, payload); } @@ -84,7 +82,7 @@ public ByteBuffer getRawBytes(int fieldId) { * Get raw bytes for a field using a pre-fetched DirectoryEntry. * This avoids the cost of re-finding the entry metadata. */ - public ByteBuffer getRawBytes(DirectoryEntry entry) { + public ByteBuffer getRawBytes(Directory entry) { try { return buffers.getFieldBuffer(entry); } catch (ImprintException e) { @@ -117,7 +115,7 @@ public ImprintRecord merge(ImprintRecord other) throws ImprintException { /** * Get the directory (parsing it if necessary). */ - public List getDirectory() { + public List getDirectory() { return buffers.getDirectory(); } @@ -128,7 +126,7 @@ public List getDirectory() { * @param fieldId The ID of the field to find. * @return The DirectoryEntry if found, otherwise null. */ - public DirectoryEntry getDirectoryEntry(int fieldId) { + public Directory getDirectoryEntry(int fieldId) { try { return buffers.findDirectoryEntry(fieldId); } catch (ImprintException e) { @@ -220,7 +218,7 @@ public ByteBuffer serializeToBuffer() { // Assemble the final record from existing components serializeHeader(this.header, finalBuffer); finalBuffer.put(directoryBuffer); - finalBuffer.put(payloadBuffer.duplicate()); // Use duplicate to preserve original buffer state + finalBuffer.put(payloadBuffer.duplicate()); finalBuffer.flip(); return finalBuffer.asReadOnlyBuffer(); @@ -236,42 +234,15 @@ public int estimateSerializedSize() { /** * Serializes the components of a record into a single ByteBuffer. * This provides a direct serialization path without needing a live ImprintRecord instance. - * - * @param schemaId The schema identifier for the record. - * @param directory The list of directory entries, which will be sorted if not already. - * @param payload The ByteBuffer containing all field data concatenated. - * @return A read-only ByteBuffer with the complete serialized record. - */ - public static ByteBuffer serialize(SchemaId schemaId, Collection directory, ByteBuffer payload) { - var header = new Header(new Flags((byte) 0), schemaId, payload.remaining()); - var directoryBuffer = ImprintBuffers.createDirectoryBuffer(directory); - - int finalSize = Constants.HEADER_BYTES + directoryBuffer.remaining() + payload.remaining(); - var finalBuffer = ByteBuffer.allocate(finalSize); - finalBuffer.order(ByteOrder.LITTLE_ENDIAN); - - // Assemble the final record - serializeHeader(header, finalBuffer); - finalBuffer.put(directoryBuffer); - finalBuffer.put(payload); - - finalBuffer.flip(); - return finalBuffer.asReadOnlyBuffer(); - } - - /** - * Serializes the components of a record into a single ByteBuffer. - * This provides a direct serialization path without needing a live ImprintRecord instance. - * This is an optimized version that assumes the list is pre-sorted by field ID. + * This assumes the list is pre-sorted by field ID. * * @param schemaId The schema identifier for the record. * @param sortedDirectory The list of directory entries, which MUST be sorted by field ID. * @param payload The ByteBuffer containing all field data concatenated. * @return A read-only ByteBuffer with the complete serialized record. */ - public static ByteBuffer serialize(SchemaId schemaId, List sortedDirectory, ByteBuffer payload) { + public static ByteBuffer serialize(SchemaId schemaId, List sortedDirectory, ByteBuffer payload) { var header = new Header(new Flags((byte) 0), schemaId, payload.remaining()); - // This createDirectoryBuffer is optimized for a pre-sorted list. var directoryBuffer = ImprintBuffers.createDirectoryBuffer(sortedDirectory); int finalSize = Constants.HEADER_BYTES + directoryBuffer.remaining() + payload.remaining(); @@ -287,58 +258,6 @@ public static ByteBuffer serialize(SchemaId schemaId, List directoryMap, ByteBuffer payload) { - var header = new Header(new Flags((byte) 0), schemaId, payload.remaining()); - var directoryBuffer = ImprintBuffers.createDirectoryBufferFromMap(directoryMap); - - int finalSize = Constants.HEADER_BYTES + directoryBuffer.remaining() + payload.remaining(); - var finalBuffer = ByteBuffer.allocate(finalSize); - finalBuffer.order(ByteOrder.LITTLE_ENDIAN); - - // Assemble the final record - serializeHeader(header, finalBuffer); - finalBuffer.put(directoryBuffer); - finalBuffer.put(payload); - - finalBuffer.flip(); - return finalBuffer.asReadOnlyBuffer(); - } - - /** - * Serializes the components of a record into a single ByteBuffer using a pre-built sorted map. - * This is the most efficient path for "write-only" scenarios, used by the builder. - * - * @param schemaId The schema identifier for the record. - * @param directoryMap The sorted map of directory entries. - * @param payload The ByteBuffer containing all field data concatenated. - * @return A read-only ByteBuffer with the complete serialized record. - */ - public static ByteBuffer serialize(SchemaId schemaId, Int2ObjectSortedMap directoryMap, ByteBuffer payload) { - var header = new Header(new Flags((byte) 0), schemaId, payload.remaining()); - var directoryBuffer = ImprintBuffers.createDirectoryBufferFromSortedMap(directoryMap); - - int finalSize = Constants.HEADER_BYTES + directoryBuffer.remaining() + payload.remaining(); - var finalBuffer = ByteBuffer.allocate(finalSize); - finalBuffer.order(ByteOrder.LITTLE_ENDIAN); - - // Assemble the final record - serializeHeader(header, finalBuffer); - finalBuffer.put(directoryBuffer); - finalBuffer.put(payload); - - finalBuffer.flip(); - return finalBuffer.asReadOnlyBuffer(); - } - // ========== STATIC FACTORY METHODS ========== public static ImprintRecordBuilder builder(SchemaId schemaId) { diff --git a/src/main/java/com/imprint/core/ImprintRecordBuilder.java b/src/main/java/com/imprint/core/ImprintRecordBuilder.java index 52bc760..93e83ba 100644 --- a/src/main/java/com/imprint/core/ImprintRecordBuilder.java +++ b/src/main/java/com/imprint/core/ImprintRecordBuilder.java @@ -2,12 +2,7 @@ import com.imprint.error.ImprintException; import com.imprint.types.MapKey; -import com.imprint.types.TypeCode; import com.imprint.types.Value; -import it.unimi.dsi.fastutil.ints.Int2ObjectAVLTreeMap; -import it.unimi.dsi.fastutil.ints.Int2ObjectSortedMap; -import lombok.Getter; -import lombok.Setter; import lombok.SneakyThrows; import java.nio.ByteBuffer; @@ -38,7 +33,7 @@ @SuppressWarnings("unused") public final class ImprintRecordBuilder { private final SchemaId schemaId; - private final Int2ObjectSortedMap fields = new Int2ObjectAVLTreeMap<>(); + private final Map fields = new TreeMap<>(); private int estimatedPayloadSize = 0; ImprintRecordBuilder(SchemaId schemaId) { @@ -161,7 +156,6 @@ public ImprintRecord build() throws ImprintException { /** * Builds the record and serializes it directly to a ByteBuffer without creating an intermediate ImprintRecord object. - * This is the most efficient path for "write-only" scenarios. * * @return A read-only ByteBuffer containing the fully serialized record. * @throws ImprintException if serialization fails. @@ -182,7 +176,6 @@ public ByteBuffer buildToBuffer() throws ImprintException { return ImprintRecord.serialize(schemaId, new ArrayList<>(fields.values()), payloadView); } - // Internal helper methods /** * Adds or overwrites a field in the record being built. * If a field with the given ID already exists, it will be replaced. @@ -193,7 +186,7 @@ public ByteBuffer buildToBuffer() throws ImprintException { */ private ImprintRecordBuilder addField(int id, Value value) { Objects.requireNonNull(value, "Value cannot be null - use nullField() for explicit null values"); - var newEntry = new BuilderEntry((short) id, value); + var newEntry = new Directory.Builder((short) id, value); // Subtract the size of the old value if it's being replaced. var oldEntry = fields.get(id); @@ -345,24 +338,4 @@ private void serializeValue(Value value, ByteBuffer buffer) throws ImprintExcept throw new ImprintException(com.imprint.error.ErrorType.SERIALIZATION_ERROR, "Unknown type code: " + value.getTypeCode()); } } - - - @Getter - private static class BuilderEntry implements DirectoryEntry { - private final short id; - private final Value value; - @Setter - private int offset; - - BuilderEntry(short id, Value value) { - this.id = id; - this.value = value; - this.offset = -1; - } - - @Override - public TypeCode getTypeCode() { - return value.getTypeCode(); - } - } } \ No newline at end of file diff --git a/src/main/java/com/imprint/core/SimpleDirectoryEntry.java b/src/main/java/com/imprint/core/SimpleDirectoryEntry.java deleted file mode 100644 index 843aad4..0000000 --- a/src/main/java/com/imprint/core/SimpleDirectoryEntry.java +++ /dev/null @@ -1,22 +0,0 @@ -package com.imprint.core; - -import com.imprint.types.TypeCode; -import lombok.Value; - -import java.util.Objects; - -/** - * A concrete, immutable directory entry. - */ -@Value -public class SimpleDirectoryEntry implements DirectoryEntry { - short id; - TypeCode typeCode; - int offset; - - public SimpleDirectoryEntry(short id, TypeCode typeCode, int offset) { - this.id = id; - this.typeCode = Objects.requireNonNull(typeCode, "TypeCode cannot be null"); - this.offset = offset; - } -} \ No newline at end of file diff --git a/src/main/java/com/imprint/core/ImprintOperations.java b/src/main/java/com/imprint/ops/ImprintOperations.java similarity index 82% rename from src/main/java/com/imprint/core/ImprintOperations.java rename to src/main/java/com/imprint/ops/ImprintOperations.java index 0c51e43..49f60b4 100644 --- a/src/main/java/com/imprint/core/ImprintOperations.java +++ b/src/main/java/com/imprint/ops/ImprintOperations.java @@ -1,15 +1,14 @@ -package com.imprint.core; +package com.imprint.ops; +import com.imprint.core.*; import com.imprint.error.ErrorType; import com.imprint.error.ImprintException; import lombok.experimental.UtilityClass; import java.nio.ByteBuffer; import java.nio.ByteOrder; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; +import java.util.*; +import java.util.stream.Collectors; @UtilityClass public class ImprintOperations { @@ -32,25 +31,27 @@ public class ImprintOperations { */ public static ImprintRecord project(ImprintRecord record, int... fieldIds) { // Sort and deduplicate field IDs for efficient matching - int[] sortedFieldIds = Arrays.stream(fieldIds).distinct().sorted().toArray(); - if (sortedFieldIds.length == 0) { + final var fieldIdSet = Arrays.stream(fieldIds) + .boxed() + .collect(Collectors.toCollection(TreeSet::new)); + if (fieldIdSet.isEmpty()) { return createEmptyRecord(record.getHeader().getSchemaId()); } - var newDirectory = new ArrayList(sortedFieldIds.length); - var payloadChunks = new ArrayList(sortedFieldIds.length); + var newDirectory = new ArrayList(fieldIdSet.size()); + var payloadChunks = new ArrayList(fieldIdSet.size()); int currentOffset = 0; - for (int fieldId : sortedFieldIds) { + for (int fieldId : fieldIdSet) { // Use efficient lookup for each field's metadata. Returns null on failure. - DirectoryEntry sourceEntry = record.getDirectoryEntry(fieldId); + var sourceEntry = record.getDirectoryEntry(fieldId); // If field exists, get its payload and add to the new record components if (sourceEntry != null) { - ByteBuffer fieldPayload = record.getRawBytes(sourceEntry); + var fieldPayload = record.getRawBytes(sourceEntry); // This check is for internal consistency. If an entry exists, payload should too. if (fieldPayload != null) { - newDirectory.add(new SimpleDirectoryEntry((short)fieldId, sourceEntry.getTypeCode(), currentOffset)); + newDirectory.add(new Directory.Entry((short)fieldId, sourceEntry.getTypeCode(), currentOffset)); payloadChunks.add(fieldPayload); currentOffset += fieldPayload.remaining(); } @@ -58,7 +59,7 @@ public static ImprintRecord project(ImprintRecord record, int... fieldIds) { } // Build new payload from collected chunks - ByteBuffer newPayload = buildPayloadFromChunks(payloadChunks); + ByteBuffer newPayload = buildPayloadFromChunks(payloadChunks, currentOffset); // Create new header with updated payload size // TODO: compute correct schema hash @@ -92,7 +93,7 @@ public static ImprintRecord merge(ImprintRecord first, ImprintRecord second) thr var secondDir = second.getDirectory(); // Pre-allocate for worst case (no overlapping fields) - var newDirectory = new ArrayList(firstDir.size() + secondDir.size()); + var newDirectory = new ArrayList(firstDir.size() + secondDir.size()); var payloadChunks = new ArrayList(); int firstIdx = 0; @@ -100,7 +101,7 @@ public static ImprintRecord merge(ImprintRecord first, ImprintRecord second) thr int currentOffset = 0; while (firstIdx < firstDir.size() || secondIdx < secondDir.size()) { - DirectoryEntry currentEntry; + Directory currentEntry; ByteBuffer currentPayload; if (firstIdx < firstDir.size() && @@ -114,7 +115,6 @@ public static ImprintRecord merge(ImprintRecord first, ImprintRecord second) thr firstDir.get(firstIdx).getId() == secondDir.get(secondIdx).getId()) { secondIdx++; } - currentPayload = first.getRawBytes(currentEntry); firstIdx++; } else { @@ -128,7 +128,8 @@ public static ImprintRecord merge(ImprintRecord first, ImprintRecord second) thr throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, "Failed to get raw bytes for field " + currentEntry.getId()); // Add adjusted directory entry - var newEntry = new SimpleDirectoryEntry(currentEntry.getId(), currentEntry.getTypeCode(), currentOffset); + var newEntry = new Directory.Entry(currentEntry.getId(), + currentEntry.getTypeCode(), currentOffset); newDirectory.add(newEntry); // Collect payload chunk @@ -137,26 +138,22 @@ public static ImprintRecord merge(ImprintRecord first, ImprintRecord second) thr } // Build merged payload - var mergedPayload = buildPayloadFromChunks(payloadChunks); + var mergedPayload = buildPayloadFromChunks(payloadChunks, currentOffset); // Create header preserving first record's schema ID - var newHeader = new Header(first.getHeader().getFlags(), first.getHeader().getSchemaId(), mergedPayload.remaining()); - + var newHeader = new Header(first.getHeader().getFlags(), + first.getHeader().getSchemaId(), mergedPayload.remaining()); return new ImprintRecord(newHeader, newDirectory, mergedPayload); } /** * Build a new payload buffer by concatenating chunks. */ - private static ByteBuffer buildPayloadFromChunks(List chunks) { - int totalSize = chunks.stream().mapToInt(ByteBuffer::remaining).sum(); + private static ByteBuffer buildPayloadFromChunks(List chunks, int totalSize) { var mergedPayload = ByteBuffer.allocate(totalSize); mergedPayload.order(ByteOrder.LITTLE_ENDIAN); - - for (var chunk : chunks) { + for (var chunk : chunks) mergedPayload.put(chunk); - } - mergedPayload.flip(); return mergedPayload; } diff --git a/src/main/java/com/imprint/core/ImprintStream.java b/src/main/java/com/imprint/stream/ImprintStream.java similarity index 69% rename from src/main/java/com/imprint/core/ImprintStream.java rename to src/main/java/com/imprint/stream/ImprintStream.java index b6afe7a..35a69ed 100644 --- a/src/main/java/com/imprint/core/ImprintStream.java +++ b/src/main/java/com/imprint/stream/ImprintStream.java @@ -1,17 +1,22 @@ -package com.imprint.core; +package com.imprint.stream; +import com.imprint.core.*; import com.imprint.error.ImprintException; -import it.unimi.dsi.fastutil.ints.Int2ObjectAVLTreeMap; -import it.unimi.dsi.fastutil.ints.Int2ObjectSortedMap; -import it.unimi.dsi.fastutil.ints.IntSet; -import it.unimi.dsi.fastutil.ints.IntOpenHashSet; import java.nio.ByteBuffer; import java.nio.ByteOrder; -import java.util.*; +import java.util.ArrayDeque; +import java.util.Collections; +import java.util.Deque; +import java.util.HashSet; +import java.util.List; +import java.util.NavigableMap; +import java.util.Objects; +import java.util.Set; +import java.util.TreeMap; /** - * Provides a framework for lazy, zero-copy transformations of Imprint records. + * Provides a framework for lazy, (eventual) zero-copy transformations of Imprint records. *

* Operations like {@link #project(int...)} and {@link #mergeWith(ImprintRecord)} are * intermediate and do not create new records. They build up a plan of operations @@ -49,11 +54,11 @@ private SourcePlan(ImprintRecord source) { */ private static final class ProjectPlan implements Plan { final Plan previous; - final IntSet fieldIds; + final Set fieldIds; private ProjectPlan(Plan previous, int... fieldIds) { this.previous = Objects.requireNonNull(previous); - this.fieldIds = new IntOpenHashSet(); + this.fieldIds = new HashSet<>(); for (int id : fieldIds) { this.fieldIds.add(id); } @@ -135,56 +140,42 @@ private Evaluator(Plan plan) { } public ImprintRecord execute() { - // Unwind the plan's linked-list structure into a forward-order list of operations. - var planList = getPlans(); - Collections.reverse(planList); + // Unwind the plan from a deque + var planQueue = getPlans(); - // This map holds the set of fields being built, sorted by field ID. - var resolvedFields = new Int2ObjectAVLTreeMap(); + // Set of fields being built + var resolvedFields = new TreeMap(); - // Iteratively evaluate the plan step-by-step. - for (var planStep : planList) { + for (var planStep : planQueue) { if (planStep instanceof SourcePlan) { var sourcePlan = (SourcePlan) planStep; for (var entry : sourcePlan.source.getDirectory()) { - resolvedFields.put(entry.getId(), new FieldSource(sourcePlan.source, entry)); + resolvedFields.put((int) entry.getId(), new FieldSource(sourcePlan.source, entry)); } } else if (planStep instanceof ProjectPlan) { var projectPlan = (ProjectPlan) planStep; // Apply projection to the current state of resolved fields. // Keep only fields that are in the projection set - var keysToRemove = new IntOpenHashSet(); - for (int fieldId : resolvedFields.keySet()) { - if (!projectPlan.fieldIds.contains(fieldId)) { - keysToRemove.add(fieldId); - } - } - for (int keyToRemove : keysToRemove) { - resolvedFields.remove(keyToRemove); - } + resolvedFields.keySet().removeIf(fieldId -> !projectPlan.fieldIds.contains(fieldId)); } else if (planStep instanceof MergePlan) { var mergePlan = (MergePlan) planStep; // Add fields from other records if they aren't already in the map. for (var otherRecord : mergePlan.others) { for (var entry : otherRecord.getDirectory()) { int fieldId = entry.getId(); - if (!resolvedFields.containsKey(fieldId)) { - resolvedFields.put(fieldId, new FieldSource(otherRecord, entry)); - } + resolvedFields.putIfAbsent(fieldId, new FieldSource(otherRecord, entry)); } } } } - - // Once the final field set is determined, build the record. return build(resolvedFields); } - private ArrayList getPlans() { - var planList = new ArrayList(); + private Deque getPlans() { + var planQueue = new ArrayDeque(); var current = plan; while (current != null) { - planList.add(current); + planQueue.addFirst(current); if (current instanceof ProjectPlan) { current = ((ProjectPlan) current).previous; } else if (current instanceof MergePlan) { @@ -193,38 +184,34 @@ private ArrayList getPlans() { current = null; // End of the chain } } - return planList; + return planQueue; } - private ImprintRecord build(Int2ObjectSortedMap finalFields) { + private ImprintRecord build(NavigableMap finalFields) { if (finalFields.isEmpty()) { - // To-Do: Need a way to get the schemaId for an empty record. + // TODO: Need a way to get the schemaId for an empty record. // For now, returning null or using a default. try { return ImprintRecord.builder(new SchemaId(0, 0)).build(); } catch (ImprintException e) { - // This should not happen when building an empty record. + // TODO This shouldn't really ever happen, we probably need a better way of consolidating error handling throw new IllegalStateException("Failed to build empty record.", e); } } - // Determine the schema from the first field's source record. - SchemaId schemaId = finalFields.get(finalFields.firstIntKey()).record.getHeader().getSchemaId(); + // Use schema from the first field's source record. + var schemaId = finalFields.firstEntry().getValue().record.getHeader().getSchemaId(); // 1. Calculate final payload size and prepare directory. int payloadSize = 0; - var newDirectoryMap = new Int2ObjectAVLTreeMap(); + var newDirectoryMap = new TreeMap(); - // Iterate over fields in sorted order - for (var entry : finalFields.int2ObjectEntrySet()) { - int fieldId = entry.getIntKey(); + for (var entry : finalFields.entrySet()) { + int fieldId = entry.getKey(); var fieldSource = entry.getValue(); int fieldLength = fieldSource.getLength(); - newDirectoryMap.put(fieldId, new SimpleDirectoryEntry( - fieldSource.entry.getId(), - fieldSource.entry.getTypeCode(), - payloadSize)); + newDirectoryMap.put(fieldId, new Directory.Entry(fieldSource.entry.getId(), fieldSource.entry.getTypeCode(), payloadSize)); payloadSize += fieldLength; } @@ -232,12 +219,11 @@ private ImprintRecord build(Int2ObjectSortedMap finalFields) { var payload = ByteBuffer.allocate(payloadSize).order(ByteOrder.LITTLE_ENDIAN); for (var fieldSource : finalFields.values()) { try { - ByteBuffer sourceData = fieldSource.record.getRawBytes(fieldSource.entry.getId()); - if (sourceData != null) { + var sourceData = fieldSource.record.getRawBytes(fieldSource.entry.getId()); + if (sourceData != null) payload.put(sourceData.duplicate()); - } } catch (Exception e) { - // This indicates a data corruption or bug, shouldn't happen in normal operation. + // Shouldn't happen in normal operation - maybe some sort of data corruption or race issue throw new IllegalStateException("Failed to copy data for field " + fieldSource.entry.getId(), e); } } @@ -249,20 +235,20 @@ private ImprintRecord build(Int2ObjectSortedMap finalFields) { } /** - * A helper class to track the source record and directory entry for a field. + * A lightweight struct to track the source of a field during evaluation. */ private static final class FieldSource { final ImprintRecord record; - final DirectoryEntry entry; + final Directory entry; - FieldSource(ImprintRecord record, DirectoryEntry entry) { + FieldSource(ImprintRecord record, Directory entry) { this.record = record; this.entry = entry; } int getLength() { try { - ByteBuffer buf = record.getRawBytes(entry.getId()); + var buf = record.getRawBytes(entry.getId()); return buf != null ? buf.remaining() : 0; } catch (Exception e) { return 0; diff --git a/src/main/java/com/imprint/types/Value.java b/src/main/java/com/imprint/types/Value.java index 681eda1..070c497 100644 --- a/src/main/java/com/imprint/types/Value.java +++ b/src/main/java/com/imprint/types/Value.java @@ -285,7 +285,7 @@ public String toString() { public static class StringValue extends Value { @Getter private final String value; - private byte[] utf8BytesCache; // Cache UTF-8 encoding + private byte[] utf8BytesCache; public StringValue(String value) { this.value = Objects.requireNonNull(value, "String cannot be null"); diff --git a/src/test/java/com/imprint/core/ImprintOperationsTest.java b/src/test/java/com/imprint/ops/ImprintOperationsTest.java similarity index 96% rename from src/test/java/com/imprint/core/ImprintOperationsTest.java rename to src/test/java/com/imprint/ops/ImprintOperationsTest.java index 1dc67fb..7b54800 100644 --- a/src/test/java/com/imprint/core/ImprintOperationsTest.java +++ b/src/test/java/com/imprint/ops/ImprintOperationsTest.java @@ -1,5 +1,8 @@ -package com.imprint.core; +package com.imprint.ops; +import com.imprint.core.Directory; +import com.imprint.core.ImprintRecord; +import com.imprint.core.SchemaId; import com.imprint.error.ImprintException; import com.imprint.types.Value; import org.junit.jupiter.api.BeforeEach; @@ -72,7 +75,7 @@ void shouldMaintainFieldOrderRegardlessOfInputOrder() throws ImprintException { assertArrayEquals(new byte[]{1, 2, 3}, projected.getBytes(7)); // And directory should maintain sorted order - List directory = projected.getDirectory(); + List directory = projected.getDirectory(); for (int i = 1; i < directory.size(); i++) { assertTrue(directory.get(i - 1).getId() < directory.get(i).getId(), "Directory entries should be sorted by field id"); @@ -95,7 +98,7 @@ void shouldHandleSingleFieldProjection() throws ImprintException { void shouldPreserveAllFieldsWhenProjectingAll() throws ImprintException { // Given all field IDs from the original record int[] allFields = multiFieldRecord.getDirectory().stream() - .mapToInt(DirectoryEntry::getId) + .mapToInt(Directory::getId) .toArray(); // When projecting all fields @@ -104,7 +107,7 @@ void shouldPreserveAllFieldsWhenProjectingAll() throws ImprintException { // Then all fields should be present with matching values assertEquals(multiFieldRecord.getDirectory().size(), projected.getDirectory().size()); - for (DirectoryEntry entry : multiFieldRecord.getDirectory()) { + for (Directory entry : multiFieldRecord.getDirectory()) { Value originalValue = multiFieldRecord.getValue(entry.getId()); Value projectedValue = projected.getValue(entry.getId()); assertEquals(originalValue, projectedValue, @@ -228,7 +231,7 @@ void shouldMergeRecordsWithDistinctFields() throws ImprintException { assertEquals(123L, merged.getInt64(4)); // And directory should be sorted - List directory = merged.getDirectory(); + List directory = merged.getDirectory(); for (int i = 1; i < directory.size(); i++) { assertTrue(directory.get(i - 1).getId() < directory.get(i).getId(), "Directory entries should be sorted by field id"); @@ -293,7 +296,7 @@ void shouldHandleMergeWithEmptyRecord() throws ImprintException { assertEquals(multiFieldRecord.getDirectory().size(), merged2.getDirectory().size()); // And values should be preserved - for (DirectoryEntry entry : multiFieldRecord.getDirectory()) { + for (Directory entry : multiFieldRecord.getDirectory()) { Value originalValue = multiFieldRecord.getValue(entry.getId()); assertEquals(originalValue, merged1.getValue(entry.getId())); assertEquals(originalValue, merged2.getValue(entry.getId())); @@ -335,9 +338,9 @@ void shouldMaintainCorrectPayloadOffsetsAfterMerge() throws ImprintException { assertArrayEquals(new byte[]{1, 2, 3, 4, 5}, merged.getBytes(4)); // And directory offsets should be sequential - List directory = merged.getDirectory(); + List directory = merged.getDirectory(); int expectedOffset = 0; - for (DirectoryEntry entry : directory) { + for (Directory entry : directory) { assertEquals(expectedOffset, entry.getOffset(), "Field " + entry.getId() + " should have correct offset"); @@ -359,6 +362,7 @@ void shouldHandleLargeRecordsEfficiently() throws ImprintException { for (int i = 1; i <= 100; i++) { builder1.field(i, i * 10); } + for (int i = 101; i <= 200; i++) { builder2.field(i, i * 10); } @@ -372,7 +376,7 @@ void shouldHandleLargeRecordsEfficiently() throws ImprintException { // Then all 200 fields should be present and accessible assertEquals(200, merged.getDirectory().size()); - // Spot check some values + // Spot check a bunch of random values just to make sure I guess assertEquals(10, merged.getInt32(1)); assertEquals(500, merged.getInt32(50)); assertEquals(1000, merged.getInt32(100)); diff --git a/src/test/java/com/imprint/profile/ProfilerTest.java b/src/test/java/com/imprint/profile/ProfilerTest.java index 3cfa61f..7b8a027 100644 --- a/src/test/java/com/imprint/profile/ProfilerTest.java +++ b/src/test/java/com/imprint/profile/ProfilerTest.java @@ -1,108 +1,196 @@ package com.imprint.profile; -import com.imprint.core.ImprintOperations; import com.imprint.core.ImprintRecord; import com.imprint.core.SchemaId; +import com.imprint.ops.ImprintOperations; import com.imprint.types.Value; import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import java.util.Arrays; import java.util.Random; import java.util.stream.IntStream; -/** - * A test designed for profiling hotspots during development. - *

- * To use with a profiler: - * 1. Remove @Disabled annotation - * 2. Run with JProfiler, VisualVM, or async-profiler: - * - JProfiler: Attach to test JVM - * - VisualVM: jvisualvm, attach to process - * - async-profiler: java -jar async-profiler.jar -d 30 -f profile.html - * 3. Look for hotspots in CPU sampling - *

- * Key areas to examine: - * - Object allocation (memory profiling) - * - Method call frequency (CPU sampling) - * - GC pressure (memory profiling) - * - String operations and UTF-8 encoding - * - ByteBuffer operations - */ -//@Disabled("Enable manually for profiling") + +@Disabled public class ProfilerTest { - private static final int ITERATIONS = 1_000_000; private static final int RECORD_SIZE = 50; private static final int LARGE_RECORD_SIZE = 200; @Test - void profileFieldAccess() throws Exception { - System.out.println("Starting profiler test - attach profiler now..."); - Thread.sleep(5000); // Give time to attach profiler + @Tag("merge") + void profileMergeOperations() throws Exception { + System.out.println("Starting merge profiler test - attach profiler now..."); + Thread.sleep(3000); + + profileSmallMerges(); + profileLargeMerges(); + profileOverlappingMerges(); + profileDisjointMerges(); + } + + /** + * Profile small merges (20-field records) + */ + private void profileSmallMerges() throws Exception { + System.out.println("\\n--- Small Merges (20-field records) ---"); - // Create a representative record - var record = createTestRecord(); + var record1 = createTestRecord(20); + var record2 = createTestRecord(20); + int iterations = 200_000; - System.out.println("Beginning field access profiling..."); + System.out.printf("Beginning small merge profiling (%,d iterations)...%n", iterations); long start = System.nanoTime(); - // Simulate real-world access patterns - Random random = new Random(42); - int hits = 0; - - for (int i = 0; i < ITERATIONS; i++) { - // Random field access (hotspot) - int fieldId = random.nextInt(RECORD_SIZE) + 1; - var value = record.getValue(fieldId); - if (value != null) { - hits++; - - // Trigger string decoding (potential hotspot) - if (value.getTypeCode() == com.imprint.types.TypeCode.STRING) { - if (value instanceof Value.StringBufferValue) { - ((Value.StringBufferValue) value).getValue(); - } else { - ((Value.StringValue) value).getValue(); - } - } - } + for (int i = 0; i < iterations; i++) { + // This is the hotspot we want to profile + var merged = ImprintOperations.merge(record1, record2); - // Some raw access (zero-copy path) - if (i % 10 == 0) { - record.getRawBytes(fieldId); + // Simulate some usage to prevent dead code elimination + if (i % 10_000 == 0) { + merged.getValue(1); // Trigger value decoding + merged.getRawBytes(5); // Trigger raw access } + merged.serializeToBuffer(); + } + + long duration = System.nanoTime() - start; + System.out.printf("Small merges: %.2f ms (avg: %.1f μs/merge)%n", + duration / 1_000_000.0, (double) duration / iterations / 1000.0); + } + + /** + * Profile large merges (100-field records) + */ + private void profileLargeMerges() throws Exception { + System.out.println("\\n--- Large Merges (100-field records) ---"); + + var record1 = createTestRecord(100); + var record2 = createTestRecord(100); + int iterations = 50_000; + + System.out.printf("Beginning large merge profiling (%,d iterations)...%n", iterations); + long start = System.nanoTime(); + + for (int i = 0; i < iterations; i++) { + var merged = ImprintOperations.merge(record1, record2); + merged.serializeToBuffer(); + } + + long duration = System.nanoTime() - start; + System.out.printf("Large merges: %.2f ms (avg: %.1f μs/merge)%n", + duration / 1_000_000.0, (double) duration / iterations / 1000.0); + } + + /** + * Profile overlapping merges (records with many duplicate field IDs) + */ + private void profileOverlappingMerges() throws Exception { + System.out.println("\\n--- Overlapping Merges (50%% field overlap) ---"); + + var record1 = createTestRecordWithFieldIds(new int[]{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}); + var record2 = createTestRecordWithFieldIds(new int[]{10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24}); + int iterations = 100_000; + + System.out.printf("Beginning overlapping merge profiling (%,d iterations)...%n", iterations); + long start = System.nanoTime(); + + for (int i = 0; i < iterations; i++) { + var merged = ImprintOperations.merge(record1, record2); + merged.serializeToBuffer(); } long duration = System.nanoTime() - start; - System.out.printf("Completed %,d field accesses in %.2f ms (avg: %.1f ns/op, hits: %d)%n", - ITERATIONS, duration / 1_000_000.0, (double) duration / ITERATIONS, hits); + System.out.printf("Overlapping merges: %.2f ms (avg: %.1f μs/merge)%n", + duration / 1_000_000.0, (double) duration / iterations / 1000.0); + } + + /** + * Profile disjoint merges (no overlapping field IDs) + */ + private void profileDisjointMerges() throws Exception { + System.out.println("\\n--- Disjoint Merges (no field overlap) ---"); + + // Create records with completely separate field IDs + var record1 = createTestRecordWithFieldIds(new int[]{1, 3, 5, 7, 9, 11, 13, 15, 17, 19}); + var record2 = createTestRecordWithFieldIds(new int[]{2, 4, 6, 8, 10, 12, 14, 16, 18, 20}); + int iterations = 100_000; + + System.out.printf("Beginning disjoint merge profiling (%,d iterations)...%n", iterations); + long start = System.nanoTime(); + + for (int i = 0; i < iterations; i++) { + var merged = ImprintOperations.merge(record1, record2); + merged.serializeToBuffer(); + } + + long duration = System.nanoTime() - start; + System.out.printf("Disjoint merges: %.2f ms (avg: %.1f μs/merge)%n", + duration / 1_000_000.0, (double) duration / iterations / 1000.0); } @Test + @Tag("serialization") + @Tag("small-records") void profileSmallRecordSerialization() throws Exception { profileSerialization("small records", RECORD_SIZE, 100_000); } @Test + @Tag("serialization") + @Tag("large-records") void profileLargeRecordSerialization() throws Exception { profileSerialization("large records", LARGE_RECORD_SIZE, 500_000); } @Test + @Tag("projection") void profileProjectionOperations() throws Exception { System.out.println("Starting projection profiler test - attach profiler now..."); Thread.sleep(3000); - profileSmallProjections(); profileLargeProjections(); profileSelectiveProjections(); - profileProjectionMemoryAllocation(); } - /** - * Profile small projections (select 2-5 fields from 20-field records) - */ + @Test + @Tag("memory") + @Tag("allocation") + void profileMemoryAllocation() throws Exception { + System.out.println("Starting allocation profiler test..."); + Thread.sleep(3000); + + System.out.println("Beginning allocation profiling - watch for GC events..."); + + // Force allocation pressure to reveal GC hotspots + for (int batch = 0; batch < 1000; batch++) { + for (int i = 0; i < 1000; i++) { + var schemaId = new SchemaId(batch, i); + var builder = ImprintRecord.builder(schemaId); + + // Create strings of varying sizes (allocation pressure) + builder.field(1, Value.fromString("small")) + .field(2, Value.fromString("medium-length-string-" + i)) + .field(3, Value.fromString("very-long-string-that-will-cause-more-allocation-pressure-" + batch + "-" + i)) + .field(4, Value.fromBytes(new byte[100 + i % 1000])); // Varying byte arrays + + var record = builder.build(); + + // Some deserialization to trigger string decoding allocations + record.getValue(2); + record.getValue(3); + } + + if (batch % 100 == 0) { + System.out.printf("Completed batch %d/1000%n", batch); + } + } + + System.out.println("Allocation test complete - check GC logs and memory profiler"); + } + + // Rest of the methods remain the same... private void profileSmallProjections() throws Exception { System.out.println("\\n--- Small Projections (2-5 fields from 20-field records) ---"); @@ -130,9 +218,6 @@ private void profileSmallProjections() throws Exception { duration / 1_000_000.0, (double) duration / iterations / 1000.0); } - /** - * Profile large projections (select 50-100 fields from 200-field records) - */ private void profileLargeProjections() throws Exception { System.out.println("\\n--- Large Projections (50 fields from 200-field records) ---"); @@ -164,9 +249,6 @@ private void profileLargeProjections() throws Exception { duration / 1_000_000.0, (double) duration / iterations / 1000.0); } - /** - * Profile selective projections with different selectivity patterns - */ private void profileSelectiveProjections() throws Exception { System.out.println("\\n--- Selective Projections (various patterns) ---"); @@ -204,128 +286,15 @@ private void profileSelectiveProjections() throws Exception { } } - /** - * Profile memory allocation patterns during projection - */ - private void profileProjectionMemoryAllocation() throws Exception { - System.out.println("\\n--- Projection Memory Allocation Profiling ---"); - System.out.println("Watch for allocation hotspots and GC pressure..."); - - var sourceRecord = createTestRecord(50); - int[] projectFields = {1, 5, 10, 15, 20, 25}; // 6 fields - - System.out.println("Beginning projection allocation test..."); - - // Create allocation pressure to identify hotspots - for (int batch = 0; batch < 1000; batch++) { - for (int i = 0; i < 1000; i++) { - // This should reveal allocation hotspots in: - // 1. ArrayList creation - // 2. ByteBuffer allocation for new payload - // 3. FieldRange objects - // 4. SimpleDirectoryEntry creation - var projected = ImprintOperations.project(sourceRecord, projectFields); - - // Force some field access to trigger additional allocations - projected.getValue(1); // String decoding allocation - projected.getValue(5); // Value wrapper allocation - projected.getRawBytes(10); // ByteBuffer slicing - } - - if (batch % 100 == 0) { - System.out.printf("Allocation batch %d/1000 complete%n", batch); - } - } - - System.out.println("Projection allocation test complete"); - } - - /** - * Profile the component operations within projection to identify bottlenecks - */ - @Test - void profileProjectionComponents() throws Exception { - System.out.println("\\n=== Projection Component Profiling ==="); - Thread.sleep(2000); - - var sourceRecord = createTestRecord(100); - int[] projectFields = {1, 10, 20, 30, 40, 50}; - int iterations = 100_000; - - // Profile individual components that might be hotspots: - - // 1. Field ID sorting and deduplication - System.out.println("Profiling field ID sorting..."); - long start = System.nanoTime(); - for (int i = 0; i < iterations; i++) { - // This mimics the sorting done in project() - int[] sorted = Arrays.stream(projectFields).distinct().sorted().toArray(); - blackhole(sorted); // Prevent optimization - } - long sortTime = System.nanoTime() - start; - System.out.printf("Field sorting: %.2f ms (%.1f ns/op)%n", - sortTime / 1_000_000.0, (double) sortTime / iterations); - - // 2. Directory scanning and range calculation - System.out.println("Profiling directory scanning..."); - var directory = sourceRecord.getDirectory(); - start = System.nanoTime(); - for (int i = 0; i < iterations; i++) { - // Simulate the directory scanning logic - int foundFields = 0; - for (var entry : directory) { - for (int fieldId : projectFields) { - if (entry.getId() == fieldId) { - foundFields++; - break; - } - } - } - blackhole(foundFields); - } - long scanTime = System.nanoTime() - start; - System.out.printf("Directory scanning: %.2f ms (%.1f ns/op)%n", - scanTime / 1_000_000.0, (double) scanTime / iterations); - - // 3. ByteBuffer operations (payload copying) - System.out.println("Profiling ByteBuffer operations..."); - var payload = sourceRecord.getBuffers().getPayload(); - start = System.nanoTime(); - for (int i = 0; i < iterations / 10; i++) { // Fewer iterations for heavy operation - // Simulate payload copying - var newPayload = java.nio.ByteBuffer.allocate(100); - newPayload.order(java.nio.ByteOrder.LITTLE_ENDIAN); - - // Copy some ranges (like buildPayloadFromRanges does) - for (int j = 0; j < 6; j++) { - var slice = payload.duplicate(); - slice.position(j * 10).limit((j + 1) * 10); - newPayload.put(slice); - } - newPayload.flip(); - blackhole(newPayload); - } - long bufferTime = System.nanoTime() - start; - System.out.printf("ByteBuffer operations: %.2f ms (%.1f μs/op)%n", - bufferTime / 1_000_000.0, (double) bufferTime / (iterations / 10) / 1000.0); - } - - /** - * Profile serialization performance with records of a given size. - * This method abstracts the core serialization profiling logic to work - * with records of different sizes and complexities. - */ private void profileSerialization(String testName, int recordSize, int iterations) throws Exception { System.out.printf("Starting %s serialization profiler test...%n", testName); Thread.sleep(3000); var schemaId = new SchemaId(1, 0x12345678); - System.out.printf("Beginning %s serialization profiling (%,d iterations, %d fields)...%n", - testName, iterations, recordSize); + System.out.printf("Beginning %s serialization profiling (%,d iterations, %d fields)...%n", testName, iterations, recordSize); long start = System.nanoTime(); - // Create and serialize many records (allocation hotspot) for (int i = 0; i < iterations; i++) { var builder = ImprintRecord.builder(schemaId); @@ -380,46 +349,6 @@ var record = builder.build(); testName, duration / 1_000_000.0, (double) duration / iterations / 1000.0); } - @Test - void profileMemoryAllocation() throws Exception { - System.out.println("Starting allocation profiler test..."); - Thread.sleep(3000); - - System.out.println("Beginning allocation profiling - watch for GC events..."); - - // Force allocation pressure to reveal GC hotspots - for (int batch = 0; batch < 1000; batch++) { - for (int i = 0; i < 1000; i++) { - var schemaId = new SchemaId(batch, i); - var builder = ImprintRecord.builder(schemaId); - - // Create strings of varying sizes (allocation pressure) - builder.field(1, Value.fromString("small")) - .field(2, Value.fromString("medium-length-string-" + i)) - .field(3, Value.fromString("very-long-string-that-will-cause-more-allocation-pressure-" + batch + "-" + i)) - .field(4, Value.fromBytes(new byte[100 + i % 1000])); // Varying byte arrays - - var record = builder.build(); - - // Some deserialization to trigger string decoding allocations - record.getValue(2); - record.getValue(3); - } - - if (batch % 100 == 0) { - System.out.printf("Completed batch %d/1000%n", batch); - } - } - - System.out.println("Allocation test complete - check GC logs and memory profiler"); - } - - // Helper methods and classes - - private ImprintRecord createTestRecord() throws Exception { - return createTestRecord(RECORD_SIZE); - } - private ImprintRecord createTestRecord(int recordSize) throws Exception { var builder = ImprintRecord.builder(new SchemaId(1, 0xdeadbeef)); @@ -443,6 +372,28 @@ private ImprintRecord createTestRecord(int recordSize) throws Exception { return builder.build(); } + private ImprintRecord createTestRecordWithFieldIds(int[] fieldIds) throws Exception { + var builder = ImprintRecord.builder(new SchemaId(1, 0xdeadbeef)); + for (int fieldId : fieldIds) { + switch (fieldId % 4) { + case 0: + builder.field(fieldId, Value.fromInt32(fieldId * 100)); + break; + case 1: + builder.field(fieldId, Value.fromString("field-value-" + fieldId)); + break; + case 2: + builder.field(fieldId, Value.fromFloat64(fieldId * 3.14159)); + break; + case 3: + builder.field(fieldId, Value.fromBytes(("bytes-" + fieldId).getBytes())); + break; + } + } + + return builder.build(); + } + private static class ProjectionPattern { final String name; final int[] fields; @@ -459,11 +410,4 @@ private int[] generateRandomFields(Random random, int maxField, int count) { .sorted() .toArray(); } - - private void blackhole(Object obj) { - // Prevent dead code elimination - if (obj.hashCode() == System.nanoTime()) { - System.out.println("Never happens"); - } - } } \ No newline at end of file diff --git a/src/test/java/com/imprint/core/ImprintStreamTest.java b/src/test/java/com/imprint/stream/ImprintStreamTest.java similarity index 85% rename from src/test/java/com/imprint/core/ImprintStreamTest.java rename to src/test/java/com/imprint/stream/ImprintStreamTest.java index 8d5b843..d2c2b69 100644 --- a/src/test/java/com/imprint/core/ImprintStreamTest.java +++ b/src/test/java/com/imprint/stream/ImprintStreamTest.java @@ -1,5 +1,8 @@ -package com.imprint.core; +package com.imprint.stream; +import com.imprint.core.ImprintRecord; +import com.imprint.core.SchemaId; +import com.imprint.stream.ImprintStream; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.*; @@ -12,25 +15,25 @@ void shouldProjectAndMergeCorrectly() throws Exception { var schemaId2 = new SchemaId(2, 2); var schemaId3 = new SchemaId(3, 3); - ImprintRecord recordA = ImprintRecord.builder(schemaId1) + var recordA = ImprintRecord.builder(schemaId1) .field(1, "A1") .field(2, 100) .field(3, true) .build(); - ImprintRecord recordB = ImprintRecord.builder(schemaId2) + var recordB = ImprintRecord.builder(schemaId2) .field(2, 200) // Overlaps with A, should be ignored .field(4, "B4") .build(); - ImprintRecord recordC = ImprintRecord.builder(schemaId3) + var recordC = ImprintRecord.builder(schemaId3) .field(5, 3.14) .field(1, "C1") // Overlaps with A, should be ignored .build(); // --- Execution --- - // Define a chain of operations - ImprintRecord finalRecord = ImprintStream.of(recordA) + // Chain of operations + var finalRecord = ImprintStream.of(recordA) .project(1, 3) // Keep {1, 3} from A. Current state: {1:A, 3:A} .mergeWith(recordB) // Merge B. {2:B, 4:B} are added. Current state: {1:A, 3:A, 2:B, 4:B} .mergeWith(recordC) // Merge C. {5:C} is added. {1:C} is ignored. Final state: {1:A, 3:A, 2:B, 4:B, 5:C} @@ -63,7 +66,7 @@ void shouldProjectAfterMerge() throws Exception { var recordA = ImprintRecord.builder(new SchemaId(1, 1)).field(1, "A").field(2, 100).build(); var recordB = ImprintRecord.builder(new SchemaId(1, 1)).field(2, 200).field(3, "B").build(); - ImprintRecord finalRecord = ImprintStream.of(recordA) + var finalRecord = ImprintStream.of(recordA) .mergeWith(recordB) // virtual record is {1:A, 2:A, 3:B} .project(1, 3) // final record is {1:A, 3:B} .toRecord(); From cce8994bfb15cd85d0b07946c7cb4e7451968c91 Mon Sep 17 00:00:00 2001 From: expand3d <> Date: Thu, 12 Jun 2025 16:15:17 -0400 Subject: [PATCH 46/49] final optimization and reorganization into better project structure --- build.gradle | 1 - .../benchmark/ComparisonBenchmark.java | 17 +- .../AbstractSerializingBenchmark.java | 4 +- .../AvroSerializingBenchmark.java | 2 +- .../FlatBuffersSerializingBenchmark.java | 3 +- .../ImprintSerializingBenchmark.java | 24 +- .../JacksonSerializingBenchmark.java | 2 +- .../KryoSerializingBenchmark.java | 2 +- .../MessagePackSerializingBenchmark.java | 2 +- .../ProtobufSerializingBenchmark.java | 2 +- .../SerializingBenchmark.java | 4 +- .../ThriftSerializingBenchmark.java | 2 +- src/main/java/com/imprint/core/Directory.java | 56 +- src/main/java/com/imprint/core/Header.java | 23 + .../java/com/imprint/core/ImprintBuffers.java | 407 --------- .../java/com/imprint/core/ImprintRecord.java | 793 ++++++++++++------ .../imprint/core/ImprintRecordBuilder.java | 119 ++- .../java/com/imprint/error/ErrorType.java | 1 + .../com/imprint/ops/ImprintOperations.java | 456 +++++++--- .../com/imprint/stream/ImprintStream.java | 259 ------ .../java/com/imprint/IntegrationTest.java | 242 ++++++ .../com/imprint/core/ImprintRecordTest.java | 486 ++++++----- .../imprint/ops/ImprintOperationsTest.java | 340 +++++++- .../com/imprint/profile/ProfilerTest.java | 124 +-- .../com/imprint/stream/ImprintStreamTest.java | 81 -- 25 files changed, 1928 insertions(+), 1524 deletions(-) rename src/jmh/java/com/imprint/benchmark/{competitors => serializers}/AbstractSerializingBenchmark.java (84%) rename src/jmh/java/com/imprint/benchmark/{competitors => serializers}/AvroSerializingBenchmark.java (99%) rename src/jmh/java/com/imprint/benchmark/{competitors => serializers}/FlatBuffersSerializingBenchmark.java (97%) rename src/jmh/java/com/imprint/benchmark/{competitors => serializers}/ImprintSerializingBenchmark.java (78%) rename src/jmh/java/com/imprint/benchmark/{competitors => serializers}/JacksonSerializingBenchmark.java (98%) rename src/jmh/java/com/imprint/benchmark/{competitors => serializers}/KryoSerializingBenchmark.java (99%) rename src/jmh/java/com/imprint/benchmark/{competitors => serializers}/MessagePackSerializingBenchmark.java (98%) rename src/jmh/java/com/imprint/benchmark/{competitors => serializers}/ProtobufSerializingBenchmark.java (98%) rename src/jmh/java/com/imprint/benchmark/{competitors => serializers}/SerializingBenchmark.java (79%) rename src/jmh/java/com/imprint/benchmark/{competitors => serializers}/ThriftSerializingBenchmark.java (99%) delete mode 100644 src/main/java/com/imprint/core/ImprintBuffers.java delete mode 100644 src/main/java/com/imprint/stream/ImprintStream.java delete mode 100644 src/test/java/com/imprint/stream/ImprintStreamTest.java diff --git a/build.gradle b/build.gradle index 26b2be5..b5f9126 100644 --- a/build.gradle +++ b/build.gradle @@ -51,7 +51,6 @@ dependencies { jmhImplementation 'org.msgpack:msgpack-core:0.9.8' jmhImplementation 'org.msgpack:jackson-dataformat-msgpack:0.9.8' jmhImplementation 'org.apache.thrift:libthrift:0.19.0' - // Required for generated Thrift code on JDK 11+ jmhImplementation 'javax.annotation:javax.annotation-api:1.3.2' } diff --git a/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java b/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java index 92b3ceb..f47da20 100644 --- a/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java +++ b/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java @@ -1,7 +1,7 @@ package com.imprint.benchmark; -import com.imprint.benchmark.competitors.*; -import com.imprint.benchmark.competitors.SerializingBenchmark; +import com.imprint.benchmark.serializers.*; +import com.imprint.benchmark.serializers.SerializingBenchmark; import org.openjdk.jmh.annotations.*; import org.openjdk.jmh.infra.Blackhole; import org.openjdk.jmh.runner.Runner; @@ -16,7 +16,7 @@ @OutputTimeUnit(TimeUnit.NANOSECONDS) @State(Scope.Benchmark) @Warmup(iterations = 3, time = 1) -@Measurement(iterations = 10, time = 1) +@Measurement(iterations = 7, time = 1) @Fork(value = 1, jvmArgs = {"-Xms4g", "-Xmx4g"}) public class ComparisonBenchmark { @@ -28,10 +28,9 @@ public class ComparisonBenchmark { new AvroSerializingBenchmark(), new ThriftSerializingBenchmark(), new KryoSerializingBenchmark(), - new MessagePackSerializingBenchmark() - ); + new MessagePackSerializingBenchmark()); - @Param({"Imprint", "Jackson-JSON", "Protobuf", "FlatBuffers", "Avro-Generic", "Thrift", "Kryo", "MessagePack"}) + @Param({"Imprint", "Jackson-JSON", "Protobuf", "FlatBuffers", "Avro-Generic", "Thrift", "Kryo", "MessagePack", "CapnProto"}) public String framework; private SerializingBenchmark serializingBenchmark; @@ -47,7 +46,7 @@ public void setup() { DataGenerator.TestRecord testRecord1 = DataGenerator.createTestRecord(); DataGenerator.TestRecord testRecord2 = DataGenerator.createTestRecord(); - // Setup the competitor with the data + // Setup the framework with the data serializingBenchmark.setup(testRecord1, testRecord2); } @@ -56,7 +55,7 @@ public void serialize(Blackhole bh) { serializingBenchmark.serialize(bh); } - @Benchmark + //@Benchmark public void deserialize(Blackhole bh) { serializingBenchmark.deserialize(bh); } @@ -71,7 +70,7 @@ public void mergeAndSerialize(Blackhole bh) { serializingBenchmark.mergeAndSerialize(bh); } - @Benchmark + //@Benchmark public void accessField(Blackhole bh) { serializingBenchmark.accessField(bh); } diff --git a/src/jmh/java/com/imprint/benchmark/competitors/AbstractSerializingBenchmark.java b/src/jmh/java/com/imprint/benchmark/serializers/AbstractSerializingBenchmark.java similarity index 84% rename from src/jmh/java/com/imprint/benchmark/competitors/AbstractSerializingBenchmark.java rename to src/jmh/java/com/imprint/benchmark/serializers/AbstractSerializingBenchmark.java index 2f5476c..4f53203 100644 --- a/src/jmh/java/com/imprint/benchmark/competitors/AbstractSerializingBenchmark.java +++ b/src/jmh/java/com/imprint/benchmark/serializers/AbstractSerializingBenchmark.java @@ -1,10 +1,10 @@ -package com.imprint.benchmark.competitors; +package com.imprint.benchmark.serializers; import com.imprint.benchmark.DataGenerator; import org.openjdk.jmh.infra.Blackhole; /** - * A minimal base class for competitors, holding the test data. + * A minimal base class for serialization frameworks to compare against, holding the test data. */ public abstract class AbstractSerializingBenchmark implements SerializingBenchmark { diff --git a/src/jmh/java/com/imprint/benchmark/competitors/AvroSerializingBenchmark.java b/src/jmh/java/com/imprint/benchmark/serializers/AvroSerializingBenchmark.java similarity index 99% rename from src/jmh/java/com/imprint/benchmark/competitors/AvroSerializingBenchmark.java rename to src/jmh/java/com/imprint/benchmark/serializers/AvroSerializingBenchmark.java index dc7278c..f3e5b8a 100644 --- a/src/jmh/java/com/imprint/benchmark/competitors/AvroSerializingBenchmark.java +++ b/src/jmh/java/com/imprint/benchmark/serializers/AvroSerializingBenchmark.java @@ -1,4 +1,4 @@ -package com.imprint.benchmark.competitors; +package com.imprint.benchmark.serializers; import com.imprint.benchmark.DataGenerator; import org.apache.avro.Schema; diff --git a/src/jmh/java/com/imprint/benchmark/competitors/FlatBuffersSerializingBenchmark.java b/src/jmh/java/com/imprint/benchmark/serializers/FlatBuffersSerializingBenchmark.java similarity index 97% rename from src/jmh/java/com/imprint/benchmark/competitors/FlatBuffersSerializingBenchmark.java rename to src/jmh/java/com/imprint/benchmark/serializers/FlatBuffersSerializingBenchmark.java index a9fe5c8..846b15c 100644 --- a/src/jmh/java/com/imprint/benchmark/competitors/FlatBuffersSerializingBenchmark.java +++ b/src/jmh/java/com/imprint/benchmark/serializers/FlatBuffersSerializingBenchmark.java @@ -1,4 +1,4 @@ -package com.imprint.benchmark.competitors; +package com.imprint.benchmark.serializers; import com.google.flatbuffers.FlatBufferBuilder; import com.imprint.benchmark.DataGenerator; @@ -31,7 +31,6 @@ private ByteBuffer buildRecord(DataGenerator.TestRecord pojo) { int tagsVectorOffset = TestRecord.createTagsVector(builder, tagsOffsets); int[] metadataKeysOffsets = pojo.metadata.keySet().stream().mapToInt(builder::createString).toArray(); - int[] metadataValuesOffsets = pojo.metadata.values().stream().mapToInt(builder::createString).toArray(); // This is not correct FlatBuffers map creation, it's a placeholder. // A proper implementation would require a table for each entry. // For this benchmark, we'll just serialize the keys vector. diff --git a/src/jmh/java/com/imprint/benchmark/competitors/ImprintSerializingBenchmark.java b/src/jmh/java/com/imprint/benchmark/serializers/ImprintSerializingBenchmark.java similarity index 78% rename from src/jmh/java/com/imprint/benchmark/competitors/ImprintSerializingBenchmark.java rename to src/jmh/java/com/imprint/benchmark/serializers/ImprintSerializingBenchmark.java index 26bb495..e71a5c0 100644 --- a/src/jmh/java/com/imprint/benchmark/competitors/ImprintSerializingBenchmark.java +++ b/src/jmh/java/com/imprint/benchmark/serializers/ImprintSerializingBenchmark.java @@ -1,7 +1,6 @@ -package com.imprint.benchmark.competitors; +package com.imprint.benchmark.serializers; import com.imprint.benchmark.DataGenerator; -import com.imprint.ops.ImprintOperations; import com.imprint.core.ImprintRecord; import com.imprint.core.ImprintRecordBuilder; import com.imprint.core.SchemaId; @@ -53,19 +52,6 @@ private ImprintRecord buildRecord(DataGenerator.TestRecord pojo) throws ImprintE return builder.build(); } - private ImprintRecordBuilder preBuildRecord(DataGenerator.TestRecord pojo) throws ImprintException { - var builder = ImprintRecord.builder(SCHEMA_ID); - builder.field(0, pojo.id); - builder.field(1, pojo.timestamp); - builder.field(2, pojo.flags); - builder.field(3, pojo.active); - builder.field(4, pojo.value); - builder.field(5, pojo.data); - builder.field(6, pojo.tags); - builder.field(7, pojo.metadata); - return builder; - } - @Override public void serialize(Blackhole bh) { try { @@ -87,8 +73,8 @@ public void deserialize(Blackhole bh) { @Override public void projectAndSerialize(Blackhole bh) { try { - ImprintRecord record = ImprintRecord.deserialize(this.serializedRecord1); - ImprintRecord projected = record.project(0, 1, 6); + // Should use zero-copy projection directly from existing record + ImprintRecord projected = this.imprintRecord1.project(0, 1, 6); bh.consume(projected.serializeToBuffer()); } catch (ImprintException e) { throw new RuntimeException(e); @@ -98,9 +84,9 @@ public void projectAndSerialize(Blackhole bh) { @Override public void mergeAndSerialize(Blackhole bh) { try { - var r1 = ImprintRecord.deserialize(this.serializedRecord1); + // Use zero-copy merge - keep one record, deserialize the other var r2 = ImprintRecord.deserialize(this.serializedRecord2); - var merged = ImprintOperations.merge(r1, r2); + var merged = this.imprintRecord1.merge(r2); bh.consume(merged.serializeToBuffer()); } catch (ImprintException e) { throw new RuntimeException(e); diff --git a/src/jmh/java/com/imprint/benchmark/competitors/JacksonSerializingBenchmark.java b/src/jmh/java/com/imprint/benchmark/serializers/JacksonSerializingBenchmark.java similarity index 98% rename from src/jmh/java/com/imprint/benchmark/competitors/JacksonSerializingBenchmark.java rename to src/jmh/java/com/imprint/benchmark/serializers/JacksonSerializingBenchmark.java index 829b073..d58bc19 100644 --- a/src/jmh/java/com/imprint/benchmark/competitors/JacksonSerializingBenchmark.java +++ b/src/jmh/java/com/imprint/benchmark/serializers/JacksonSerializingBenchmark.java @@ -1,4 +1,4 @@ -package com.imprint.benchmark.competitors; +package com.imprint.benchmark.serializers; import com.fasterxml.jackson.databind.ObjectMapper; import com.imprint.benchmark.DataGenerator; diff --git a/src/jmh/java/com/imprint/benchmark/competitors/KryoSerializingBenchmark.java b/src/jmh/java/com/imprint/benchmark/serializers/KryoSerializingBenchmark.java similarity index 99% rename from src/jmh/java/com/imprint/benchmark/competitors/KryoSerializingBenchmark.java rename to src/jmh/java/com/imprint/benchmark/serializers/KryoSerializingBenchmark.java index 1223e06..6780513 100644 --- a/src/jmh/java/com/imprint/benchmark/competitors/KryoSerializingBenchmark.java +++ b/src/jmh/java/com/imprint/benchmark/serializers/KryoSerializingBenchmark.java @@ -1,4 +1,4 @@ -package com.imprint.benchmark.competitors; +package com.imprint.benchmark.serializers; import com.esotericsoftware.kryo.Kryo; import com.esotericsoftware.kryo.io.Input; diff --git a/src/jmh/java/com/imprint/benchmark/competitors/MessagePackSerializingBenchmark.java b/src/jmh/java/com/imprint/benchmark/serializers/MessagePackSerializingBenchmark.java similarity index 98% rename from src/jmh/java/com/imprint/benchmark/competitors/MessagePackSerializingBenchmark.java rename to src/jmh/java/com/imprint/benchmark/serializers/MessagePackSerializingBenchmark.java index b596e6d..9dd275f 100644 --- a/src/jmh/java/com/imprint/benchmark/competitors/MessagePackSerializingBenchmark.java +++ b/src/jmh/java/com/imprint/benchmark/serializers/MessagePackSerializingBenchmark.java @@ -1,4 +1,4 @@ -package com.imprint.benchmark.competitors; +package com.imprint.benchmark.serializers; import com.fasterxml.jackson.databind.ObjectMapper; import com.imprint.benchmark.DataGenerator; diff --git a/src/jmh/java/com/imprint/benchmark/competitors/ProtobufSerializingBenchmark.java b/src/jmh/java/com/imprint/benchmark/serializers/ProtobufSerializingBenchmark.java similarity index 98% rename from src/jmh/java/com/imprint/benchmark/competitors/ProtobufSerializingBenchmark.java rename to src/jmh/java/com/imprint/benchmark/serializers/ProtobufSerializingBenchmark.java index 72ad38f..1f6239e 100644 --- a/src/jmh/java/com/imprint/benchmark/competitors/ProtobufSerializingBenchmark.java +++ b/src/jmh/java/com/imprint/benchmark/serializers/ProtobufSerializingBenchmark.java @@ -1,4 +1,4 @@ -package com.imprint.benchmark.competitors; +package com.imprint.benchmark.serializers; import com.imprint.benchmark.DataGenerator; import com.imprint.benchmark.proto.TestRecordOuterClass; diff --git a/src/jmh/java/com/imprint/benchmark/competitors/SerializingBenchmark.java b/src/jmh/java/com/imprint/benchmark/serializers/SerializingBenchmark.java similarity index 79% rename from src/jmh/java/com/imprint/benchmark/competitors/SerializingBenchmark.java rename to src/jmh/java/com/imprint/benchmark/serializers/SerializingBenchmark.java index 595caa6..a6358b8 100644 --- a/src/jmh/java/com/imprint/benchmark/competitors/SerializingBenchmark.java +++ b/src/jmh/java/com/imprint/benchmark/serializers/SerializingBenchmark.java @@ -1,10 +1,10 @@ -package com.imprint.benchmark.competitors; +package com.imprint.benchmark.serializers; import com.imprint.benchmark.DataGenerator; import org.openjdk.jmh.infra.Blackhole; /** - * Defines the contract for a serialization competitor in the benchmark. + * Defines the contract for a serialization framework in the benchmark. */ public interface SerializingBenchmark { String name(); diff --git a/src/jmh/java/com/imprint/benchmark/competitors/ThriftSerializingBenchmark.java b/src/jmh/java/com/imprint/benchmark/serializers/ThriftSerializingBenchmark.java similarity index 99% rename from src/jmh/java/com/imprint/benchmark/competitors/ThriftSerializingBenchmark.java rename to src/jmh/java/com/imprint/benchmark/serializers/ThriftSerializingBenchmark.java index 83c0812..18cf9bb 100644 --- a/src/jmh/java/com/imprint/benchmark/competitors/ThriftSerializingBenchmark.java +++ b/src/jmh/java/com/imprint/benchmark/serializers/ThriftSerializingBenchmark.java @@ -1,4 +1,4 @@ -package com.imprint.benchmark.competitors; +package com.imprint.benchmark.serializers; import com.imprint.benchmark.DataGenerator; import com.imprint.benchmark.thrift.ProjectedRecord; diff --git a/src/main/java/com/imprint/core/Directory.java b/src/main/java/com/imprint/core/Directory.java index cb449b3..ddaf208 100644 --- a/src/main/java/com/imprint/core/Directory.java +++ b/src/main/java/com/imprint/core/Directory.java @@ -5,6 +5,7 @@ import lombok.Setter; import lombok.Value; +import java.util.List; import java.util.Objects; /** @@ -27,6 +28,38 @@ public interface Directory { */ int getOffset(); + /** + * A view interface for accessing directory entries efficiently. + * Provides both access to individual entries and full directory materialization. + */ + interface DirectoryView { + /** + * Find a directory entry by field ID. + * @param fieldId The field ID to search for + * @return The directory entry if found, null otherwise + */ + Directory findEntry(int fieldId); + + /** + * Get all directory entries as a list, with full eager deserialization if necessary. + * @return List of all directory entries in field ID order + */ + List toList(); + + /** + * Get the count of directory entries without parsing all entries. + * @return Number of entries in the directory + */ + int size(); + + /** + * Create an iterator for lazy directory traversal. + * For buffer-backed views, this avoids parsing the entire directory upfront. + * @return Iterator over directory entries in field ID order + */ + java.util.Iterator iterator(); + } + /** * Immutable representation of the Imprint Directory used for deserialization, * merging, and field projections @@ -43,27 +76,4 @@ public Entry(short id, TypeCode typeCode, int offset) { this.offset = offset; } } - - /** - * Mutable representation of the Imprint Directory bound with corresponding type value - * used for record building through {@link ImprintRecordBuilder} and subsequent serialization. - */ - @Getter - class Builder implements Directory { - private final short id; - private final com.imprint.types.Value value; - @Setter - private int offset; - - Builder(short id, com.imprint.types.Value value) { - this.id = id; - this.value = value; - this.offset = -1; - } - - @Override - public TypeCode getTypeCode() { - return value.getTypeCode(); - } - } } \ No newline at end of file diff --git a/src/main/java/com/imprint/core/Header.java b/src/main/java/com/imprint/core/Header.java index 388d491..aec0e9b 100644 --- a/src/main/java/com/imprint/core/Header.java +++ b/src/main/java/com/imprint/core/Header.java @@ -1,7 +1,10 @@ package com.imprint.core; +import com.imprint.Constants; import lombok.Value; +import java.nio.ByteBuffer; + /** * The header of an Imprint record. */ @@ -10,4 +13,24 @@ public class Header { Flags flags; SchemaId schemaId; int payloadSize; + + /** + * Serialize this header to a ByteBuffer. + * Follows the Imprint header format: magic(1) + version(1) + flags(1) + fieldSpaceId(4) + schemaHash(4) + payloadSize(4). + */ + public void serialize(ByteBuffer buffer) { + buffer.put(Constants.MAGIC); + buffer.put(Constants.VERSION); + buffer.put(flags.getValue()); + buffer.putInt(schemaId.getFieldSpaceId()); + buffer.putInt(schemaId.getSchemaHash()); + buffer.putInt(payloadSize); + } + + /** + * Static helper for serializing any header to a ByteBuffer. + */ + public static void serialize(Header header, ByteBuffer buffer) { + header.serialize(buffer); + } } \ No newline at end of file diff --git a/src/main/java/com/imprint/core/ImprintBuffers.java b/src/main/java/com/imprint/core/ImprintBuffers.java deleted file mode 100644 index 845892a..0000000 --- a/src/main/java/com/imprint/core/ImprintBuffers.java +++ /dev/null @@ -1,407 +0,0 @@ -package com.imprint.core; - -import com.imprint.Constants; -import com.imprint.error.ErrorType; -import com.imprint.error.ImprintException; -import com.imprint.types.TypeCode; -import com.imprint.util.VarInt; -import lombok.Getter; - -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.util.*; - -/** - * Manages the raw buffers for an Imprint record with lazy directory parsing. - * Encapsulates all buffer operations and provides zero-copy field access. - * - *

Buffer Layout Overview:

- * 
- * directoryBuffer: [VarInt count][DirectoryEntry 1][DirectoryEntry 2]...[DirectoryEntry N]
- * payload:         [Field 1 data][Field 2 data]...[Field N data]
- *
- * - *

Each DirectoryEntry contains: [fieldId:2bytes][typeCode:1byte][offset:4bytes]

- */ -@Getter -public final class ImprintBuffers { - private final ByteBuffer directoryBuffer; // Raw directory bytes (includes count) - private final ByteBuffer payload; // Read-only payload view - - // Lazy-loaded directory state. - private TreeMap parsedDirectory; - private boolean directoryParsed = false; - private int directoryCount = -1; - - /** - * Creates buffers from raw data (used during deserialization). - */ - public ImprintBuffers(ByteBuffer directoryBuffer, ByteBuffer payload) { - this.directoryBuffer = directoryBuffer.asReadOnlyBuffer(); - this.payload = payload.asReadOnlyBuffer(); - } - - /** - * Creates buffers from a pre-sorted list of entries (most efficient builder path). - * Immediately creates the parsed index and the serialized buffer. - */ - public ImprintBuffers(List sortedDirectory, ByteBuffer payload) { - this.directoryBuffer = ImprintBuffers.createDirectoryBuffer(sortedDirectory); - this.payload = payload.asReadOnlyBuffer(); - } - - /** - * Creates buffers from a pre-parsed and sorted directory map containing final, simple entries. - * This is the most efficient path, as it avoids any further parsing or sorting. The provided - * map becomes the definitive parsed directory. - */ - @SuppressWarnings("unchecked") - public ImprintBuffers(Map parsedDirectory, ByteBuffer payload) { - this.directoryBuffer = ImprintBuffers.createDirectoryBufferFromSortedMap(Objects.requireNonNull(parsedDirectory)); - this.payload = payload.asReadOnlyBuffer(); - this.parsedDirectory = (TreeMap) parsedDirectory; - this.directoryParsed = true; - this.directoryCount = parsedDirectory.size(); - } - - /** - * Get a zero-copy ByteBuffer view of a field's data. - * Optimized for the most common use case - single field access. - */ - public ByteBuffer getFieldBuffer(int fieldId) throws ImprintException { - var entry = findDirectoryEntry(fieldId); - if (entry == null) - return null; - - int startOffset = entry.getOffset(); - int endOffset = findEndOffset(entry); - - if (startOffset < 0 || endOffset < 0 || startOffset > payload.limit() || - endOffset > payload.limit() || startOffset > endOffset) { - throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, - "Invalid field buffer range: start=" + startOffset + ", end=" + endOffset + ", payloadLimit=" + payload.limit()); - } - - var fieldBuffer = payload.duplicate(); - fieldBuffer.position(startOffset).limit(endOffset); - return fieldBuffer; - } - - /** - * Get a zero-copy ByteBuffer view of a field's data using a pre-fetched DirectoryEntry. - * This avoids the cost of re-finding the entry. - */ - public ByteBuffer getFieldBuffer(Directory entry) throws ImprintException { - if (entry == null) - return null; - - int startOffset = entry.getOffset(); - int endOffset = findEndOffset(entry); - - if (startOffset < 0 || endOffset < 0 || startOffset > payload.limit() || - endOffset > payload.limit() || startOffset > endOffset) { - throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, - "Invalid field buffer range: start=" + startOffset + ", end=" + endOffset + ", payloadLimit=" + payload.limit()); - } - - var fieldBuffer = payload.duplicate(); - fieldBuffer.position(startOffset).limit(endOffset); - return fieldBuffer; - } - - /** - * Find a directory entry for the given field ID using the most efficient method. - *

- * Strategy: - * - If parsed: TreeMap lookup - * - If raw: Binary search on raw bytes to avoid full unwinding of the directory - */ - public Directory findDirectoryEntry(int fieldId) throws ImprintException { - if (directoryParsed) - return parsedDirectory.get(fieldId); - else - return findFieldEntryInRawDirectory(fieldId); - } - - /** - * Get the full directory, parsing it if necessary. - * Returns the values in fieldId order thanks to TreeMap. - */ - public List getDirectory() { - ensureDirectoryParsed(); - return new ArrayList<>(parsedDirectory.values()); - } - - /** - * Get directory count without parsing. - */ - public int getDirectoryCount() { - if (directoryParsed) - return parsedDirectory.size(); - - try { - return getOrParseDirectoryCount(); - } catch (ImprintException e) { - return 0; - } - } - - /** - * Create a new buffer containing the serialized directory. - */ - public ByteBuffer serializeDirectory() { - // The directoryBuffer is created on construction and is read-only. - // If constructed from raw bytes, it's a view of the original. - // If constructed from a list, it's a fresh buffer. In both cases, it's ready. - return directoryBuffer.duplicate(); - } - - // ========== PRIVATE METHODS ========== - - /** - * Binary search on raw directory bytes to find a specific field. - * This avoids parsing the entire directory for single field lookups. - */ - private Directory findFieldEntryInRawDirectory(int fieldId) throws ImprintException { - var searchBuffer = directoryBuffer.duplicate(); - searchBuffer.order(ByteOrder.LITTLE_ENDIAN); - - int count = getOrParseDirectoryCount(); - if (count == 0) - return null; - - // Advance buffer past the varint to get to the start of the entries. - VarInt.decode(searchBuffer); - int directoryStartPos = searchBuffer.position(); - - int low = 0; - int high = count - 1; - - while (low <= high) { - int mid = (low + high) >>> 1; - int entryPos = directoryStartPos + (mid * Constants.DIR_ENTRY_BYTES); - - if (entryPos + Constants.DIR_ENTRY_BYTES > searchBuffer.limit()) { - throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, - "Directory entry at position " + entryPos + " exceeds buffer limit"); - } - - searchBuffer.position(entryPos); - short midFieldId = searchBuffer.getShort(); - - if (midFieldId < fieldId) { - low = mid + 1; - } else if (midFieldId > fieldId) { - high = mid - 1; - } else { - // Found it - read the complete entry - searchBuffer.position(entryPos); - return deserializeDirectoryEntry(searchBuffer); - } - } - - return null; - } - - /** - * Find the end offset for a field by looking at the next field's offset. - */ - private int findEndOffset(Directory entry) throws ImprintException { - if (directoryParsed) { - return findNextOffsetInParsedDirectory(entry.getId()); - } else { - return findNextOffsetInRawDirectory(entry.getId()); - } - } - - /** - * Find the end offset using TreeMap's efficient navigation methods. - */ - private int findNextOffsetInParsedDirectory(int currentFieldId) { - var nextEntry = parsedDirectory.higherEntry(currentFieldId); - if (nextEntry != null) - return nextEntry.getValue().getOffset(); - else - return payload.limit(); - - } - - private int findNextOffsetInRawDirectory(int currentFieldId) throws ImprintException { - var scanBuffer = directoryBuffer.duplicate(); - scanBuffer.order(ByteOrder.LITTLE_ENDIAN); - - int count = getOrParseDirectoryCount(); - if (count == 0) - return payload.limit(); - - // Advance buffer past the varint to get to the start of the entries. - VarInt.decode(scanBuffer); - int directoryStartPos = scanBuffer.position(); - - int low = 0; - int high = count - 1; - int nextOffset = payload.limit(); - - // Binary search for the first field with fieldId > currentFieldId - while (low <= high) { - int mid = (low + high) >>> 1; - int entryPos = directoryStartPos + (mid * Constants.DIR_ENTRY_BYTES); - - if (entryPos + Constants.DIR_ENTRY_BYTES > scanBuffer.limit()) - break; - - scanBuffer.position(entryPos); - short fieldId = scanBuffer.getShort(); - scanBuffer.get(); // skip type - int offset = scanBuffer.getInt(); - - if (fieldId > currentFieldId) { - nextOffset = offset; - high = mid - 1; - } else { - low = mid + 1; - } - } - - return nextOffset; - } - - /** - * Parse the full directory if not already parsed. - * Creates TreeMap for both fast lookup and ordering needed for binary search on offSets. - */ - private void ensureDirectoryParsed() { - if (directoryParsed) - return; - - try { - var parseBuffer = directoryBuffer.duplicate(); - parseBuffer.order(ByteOrder.LITTLE_ENDIAN); - - int count = getOrParseDirectoryCount(parseBuffer); - this.parsedDirectory = new TreeMap<>(); - - for (int i = 0; i < count; i++) { - var entry = deserializeDirectoryEntry(parseBuffer); - this.parsedDirectory.put((int) entry.getId() , entry); - } - - this.directoryParsed = true; - } catch (ImprintException e) { - // This can happen with a corrupted directory. - // In this case, we'll just have an empty (but valid) parsed directory. - this.parsedDirectory = new TreeMap<>(); - this.directoryParsed = true; // Mark as parsed to avoid repeated errors - } - } - - private int getOrParseDirectoryCount() throws ImprintException { - if (directoryCount != -1) { - return directoryCount; - } - try { - this.directoryCount = VarInt.decode(directoryBuffer.duplicate()).getValue(); - } catch (ImprintException e) { - this.directoryCount = 0; // Cache as 0 on error - throw e; // rethrow - } - return this.directoryCount; - } - - private int getOrParseDirectoryCount(ByteBuffer buffer) throws ImprintException { - // This method does not cache the count because it's used during parsing - // where the buffer is transient. Caching is only for the instance's primary buffer. - return VarInt.decode(buffer).getValue(); - } - - /** - * Creates a read-only buffer containing the serialized directory. - * The input collection does not need to be sorted. - */ - static ByteBuffer createDirectoryBuffer(Collection directory) { - if (directory == null || directory.isEmpty()) { - ByteBuffer buffer = ByteBuffer.allocate(1); - VarInt.encode(0, buffer); - buffer.flip(); - return buffer; - } - - // Ensure sorted order for binary search compatibility. - ArrayList sortedDirectory; - if (directory instanceof ArrayList && isSorted((ArrayList)directory)) { - sortedDirectory = (ArrayList) directory; - } else { - sortedDirectory = new ArrayList<>(directory); - sortedDirectory.sort(null); - } - - int count = sortedDirectory.size(); - int size = VarInt.encodedLength(count) + (count * Constants.DIR_ENTRY_BYTES); - ByteBuffer buffer = ByteBuffer.allocate(size); - buffer.order(ByteOrder.LITTLE_ENDIAN); - - VarInt.encode(count, buffer); - for (var entry : sortedDirectory) { - serializeDirectoryEntry(entry, buffer); - } - - buffer.flip(); - return buffer; - } - - static ByteBuffer createDirectoryBufferFromSortedMap(Map directoryMap) { - if (directoryMap == null || directoryMap.isEmpty()) { - ByteBuffer buffer = ByteBuffer.allocate(1); - VarInt.encode(0, buffer); - buffer.flip(); - return buffer; - } - - int count = directoryMap.size(); - int size = VarInt.encodedLength(count) + (count * Constants.DIR_ENTRY_BYTES); - var buffer = ByteBuffer.allocate(size); - buffer.order(ByteOrder.LITTLE_ENDIAN); - - VarInt.encode(count, buffer); - for (var entry : directoryMap.values()) { - serializeDirectoryEntry(entry, buffer); - } - - buffer.flip(); - return buffer; - } - - private static boolean isSorted(ArrayList list) { - for (int i = 0; i < list.size() - 1; i++) { - if (list.get(i).getId() > list.get(i + 1).getId()) { - return false; - } - } - return true; - } - - /** - * Serialize a single directory entry to the buffer. - * Format: [fieldId:2bytes][typeCode:1byte][offset:4bytes] - */ - private static void serializeDirectoryEntry(Directory entry, ByteBuffer buffer) { - buffer.putShort(entry.getId()); - buffer.put(entry.getTypeCode().getCode()); - buffer.putInt(entry.getOffset()); - } - - /** - * Deserialize a single directory entry from the buffer. - * Reads: [fieldId:2bytes][typeCode:1byte][offset:4bytes] - */ - private Directory deserializeDirectoryEntry(ByteBuffer buffer) throws ImprintException { - if (buffer.remaining() < Constants.DIR_ENTRY_BYTES) - throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, "Not enough bytes for directory entry"); - - short id = buffer.getShort(); - var typeCode = TypeCode.fromByte(buffer.get()); - int offset = buffer.getInt(); - - return new Directory.Entry(id, typeCode, offset); - } -} \ No newline at end of file diff --git a/src/main/java/com/imprint/core/ImprintRecord.java b/src/main/java/com/imprint/core/ImprintRecord.java index a34c7df..e6f9de6 100644 --- a/src/main/java/com/imprint/core/ImprintRecord.java +++ b/src/main/java/com/imprint/core/ImprintRecord.java @@ -1,177 +1,218 @@ package com.imprint.core; import com.imprint.Constants; -import com.imprint.ops.ImprintOperations; import com.imprint.error.ErrorType; import com.imprint.error.ImprintException; -import com.imprint.types.MapKey; +import com.imprint.ops.ImprintOperations; import com.imprint.types.TypeCode; import com.imprint.types.Value; import com.imprint.util.VarInt; + +import lombok.AccessLevel; +import lombok.EqualsAndHashCode; import lombok.Getter; +import lombok.ToString; +import lombok.experimental.NonFinal; import java.nio.ByteBuffer; import java.nio.ByteOrder; +import java.util.ArrayList; +import java.util.Iterator; import java.util.List; -import java.util.Map; +import java.util.NoSuchElementException; import java.util.Objects; /** - * An Imprint record containing a header and buffer management. - * Delegates all buffer operations to ImprintBuffers for cleaner separation. + * Imprint Record + *

+ * This is the primary way to work with Imprint records, providing: + * - Zero-copy field access via binary search + * - Direct bytes-to-bytes operations (merge, project) + * - Lazy deserializing operations */ -@Getter -public final class ImprintRecord { - private final Header header; - private final ImprintBuffers buffers; - +@lombok.Value +@EqualsAndHashCode(of = "serializedBytes") +@ToString(of = {"header"}) +public class ImprintRecord { + ByteBuffer serializedBytes; + + @Getter(AccessLevel.PUBLIC) + Header header; + + @Getter(AccessLevel.PACKAGE) + // Raw directory bytes (read-only) + ByteBuffer directoryBuffer; + + @Getter(AccessLevel.PACKAGE) + // Raw payload bytes + ByteBuffer payload; + + @NonFinal + @Getter(AccessLevel.NONE) + //Directory View cache to allow for easier mutable operations needed for lazy initialization + Directory.DirectoryView directoryView; + /** - * Creates a record from deserialized components. + * Package-private constructor for @Value that creates immutable ByteBuffer views. */ - private ImprintRecord(Header header, ImprintBuffers buffers) { - this.header = Objects.requireNonNull(header, "Header cannot be null"); - this.buffers = Objects.requireNonNull(buffers, "Buffers cannot be null"); - } - + ImprintRecord(ByteBuffer serializedBytes, Header header, ByteBuffer directoryBuffer, ByteBuffer payload) { + this.serializedBytes = serializedBytes.asReadOnlyBuffer(); + this.header = Objects.requireNonNull(header); + this.directoryBuffer = directoryBuffer.asReadOnlyBuffer(); + this.payload = payload.asReadOnlyBuffer(); + this.directoryView = null; + } + + // ========== STATIC FACTORY METHODS ========== + /** - * Creates a record from a pre-sorted list of entries (most efficient builder path). + * Create a builder for constructing new ImprintRecord instances. */ - public ImprintRecord(Header header, List sortedDirectory, ByteBuffer payload) { - this.header = Objects.requireNonNull(header, "Header cannot be null"); - this.buffers = new ImprintBuffers(sortedDirectory, payload); + public static ImprintRecordBuilder builder(SchemaId schemaId) { + return new ImprintRecordBuilder(schemaId); + } + + public static ImprintRecordBuilder builder(int fieldspaceId, int schemaHash) { + return new ImprintRecordBuilder(new SchemaId(fieldspaceId, schemaHash)); } /** - * Creates a record from a pre-built and sorted FastUtil map (most efficient builder path). + * Deserialize an ImprintRecord from bytes. */ - public ImprintRecord(Header header, Map parsedDirectory, ByteBuffer payload) { - this.header = Objects.requireNonNull(header, "Header cannot be null"); - this.buffers = new ImprintBuffers(parsedDirectory, payload); + public static ImprintRecord deserialize(byte[] bytes) throws ImprintException { + return fromBytes(ByteBuffer.wrap(bytes)); } - // ========== FIELD ACCESS METHODS ========== - + public static ImprintRecord deserialize(ByteBuffer buffer) throws ImprintException { + return fromBytes(buffer); + } + /** - * Get a value by field ID, deserializing it on demand. - * Returns null if the field is not found. + * Create a ImprintRecord from complete serialized bytes. */ - public Value getValue(int fieldId) throws ImprintException { - var entry = buffers.findDirectoryEntry(fieldId); - if (entry == null) - return null; - - var fieldBuffer = buffers.getFieldBuffer(fieldId); - if (fieldBuffer == null) - throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, "Failed to get buffer for field " + fieldId); - - return deserializeValue(entry.getTypeCode(), fieldBuffer); + public static ImprintRecord fromBytes(ByteBuffer serializedBytes) throws ImprintException { + Objects.requireNonNull(serializedBytes, "Serialized bytes cannot be null"); + + var buffer = serializedBytes.duplicate().order(ByteOrder.LITTLE_ENDIAN); + + // Parse header + var header = parseHeader(buffer); + + // Extract directory and payload sections + var parsedBuffers = parseBuffersFromSerialized(serializedBytes); + + return new ImprintRecord(serializedBytes, header, parsedBuffers.directoryBuffer, parsedBuffers.payload); + } + + + // ========== ZERO-COPY OPERATIONS ========== + + /** + * Merge with another ImprintRecord using pure byte operations. + * Results in a new ImprintRecord without any object creation. + */ + public ImprintRecord merge(ImprintRecord other) throws ImprintException { + var mergedBytes = ImprintOperations.mergeBytes(this.serializedBytes, other.serializedBytes); + return fromBytes(mergedBytes); } - + /** - * Get raw bytes for a field without deserializing. + * Project fields using pure byte operations. + * Results in a new ImprintRecord without any object creation. */ - public ByteBuffer getRawBytes(int fieldId) { - try { - return buffers.getFieldBuffer(fieldId); - } catch (ImprintException e) { - return null; - } + public ImprintRecord project(int... fieldIds) throws ImprintException { + var projectedBytes = ImprintOperations.projectBytes(this.serializedBytes, fieldIds); + return fromBytes(projectedBytes); } - + /** - * Get raw bytes for a field using a pre-fetched DirectoryEntry. - * This avoids the cost of re-finding the entry metadata. + * Chain multiple operations efficiently. + * Each operation works on bytes without creating intermediate objects. */ - public ByteBuffer getRawBytes(Directory entry) { - try { - return buffers.getFieldBuffer(entry); - } catch (ImprintException e) { - return null; - } + public ImprintRecord projectAndMerge(ImprintRecord other, int... projectFields) throws ImprintException { + return this.project(projectFields).merge(other); } - + /** - * Project a subset of fields from this record. - * - * @param fieldIds Array of field IDs to include in the projection - * @return New ImprintRecord containing only the requested fields + * Get the raw serialized bytes. + * This is the most efficient way to pass the record around. */ - public ImprintRecord project(int... fieldIds) { - return ImprintOperations.project(this, fieldIds); + public ByteBuffer getSerializedBytes() { + return serializedBytes.duplicate(); } - + /** - * Merge another record into this one. - * For duplicate fields, this record's values take precedence. - * - * @param other The record to merge with this one - * @return New ImprintRecord containing merged fields - * @throws ImprintException if merge fails + * Get a DirectoryView for straight through directory access. */ - public ImprintRecord merge(ImprintRecord other) throws ImprintException { - return ImprintOperations.merge(this, other); + public Directory.DirectoryView getDirectoryView() { + if (directoryView == null) { + directoryView = new ImprintDirectoryView(); + } + return directoryView; } - + /** - * Get the directory (parsing it if necessary). + * Get the directory list. */ public List getDirectory() { - return buffers.getDirectory(); + return getDirectoryView().toList(); } - + /** - * Finds a directory entry by its field ID. - * This is an efficient lookup that avoids full directory deserialization if possible. - * - * @param fieldId The ID of the field to find. - * @return The DirectoryEntry if found, otherwise null. + * Get raw bytes for a field without deserializing. */ - public Directory getDirectoryEntry(int fieldId) { + public ByteBuffer getRawBytes(int fieldId) { try { - return buffers.findDirectoryEntry(fieldId); + return getFieldBuffer(fieldId); } catch (ImprintException e) { - // This can happen with a corrupted directory, in which case we assume it doesn't exist. return null; } } - + /** - * Checks if a field with the given ID exists in the record. - * - * @param fieldId The ID of the field to check. - * @return true if the field exists, false otherwise. + * Get raw bytes for a field by short ID. */ - public boolean hasField(int fieldId) { - try { - return buffers.findDirectoryEntry(fieldId) != null; - } catch (ImprintException e) { - // This can happen with a corrupted directory, in which case we assume it doesn't exist. - return false; - } - } - - // ========== TYPED GETTERS ========== - - public boolean getBoolean(int fieldId) throws ImprintException { - return getTypedValueOrThrow(fieldId, TypeCode.BOOL, Value.BoolValue.class, "boolean").getValue(); + public ByteBuffer getRawBytes(short fieldId) { + return getRawBytes((int) fieldId); } - - public int getInt32(int fieldId) throws ImprintException { - return getTypedValueOrThrow(fieldId, TypeCode.INT32, Value.Int32Value.class, "int32").getValue(); + + /** + * Estimate the serialized size of this record. + */ + public int estimateSerializedSize() { + return serializedBytes.remaining(); } - - public long getInt64(int fieldId) throws ImprintException { - return getTypedValueOrThrow(fieldId, TypeCode.INT64, Value.Int64Value.class, "int64").getValue(); + + /** + * Get a field value by ID. + * Uses zero-copy binary search to locate the field. + */ + public Value getValue(int fieldId) throws ImprintException { + var entry = getDirectoryView().findEntry(fieldId); + if (entry == null) return null; + + var fieldBuffer = getFieldBuffer(fieldId); + if (fieldBuffer == null) return null; + + return deserializeValue(entry.getTypeCode(), fieldBuffer); } - - public float getFloat32(int fieldId) throws ImprintException { - return getTypedValueOrThrow(fieldId, TypeCode.FLOAT32, Value.Float32Value.class, "float32").getValue(); + + /** + * Check if a field exists without deserializing it. + */ + public boolean hasField(int fieldId) { + return getDirectoryView().findEntry(fieldId) != null; } - - public double getFloat64(int fieldId) throws ImprintException { - return getTypedValueOrThrow(fieldId, TypeCode.FLOAT64, Value.Float64Value.class, "float64").getValue(); + + /** + * Get the number of fields without parsing the directory. + */ + public int getFieldCount() { + return getDirectoryCount(); } - + + // ========== TYPED GETTERS ========== + public String getString(int fieldId) throws ImprintException { var value = getValidatedValue(fieldId, "STRING"); if (value instanceof Value.StringValue) @@ -180,7 +221,27 @@ public String getString(int fieldId) throws ImprintException { return ((Value.StringBufferValue) value).getValue(); throw new ImprintException(ErrorType.TYPE_MISMATCH, "Field " + fieldId + " is not a STRING"); } - + + public int getInt32(int fieldId) throws ImprintException { + return getTypedValueOrThrow(fieldId, com.imprint.types.TypeCode.INT32, Value.Int32Value.class, "int32").getValue(); + } + + public long getInt64(int fieldId) throws ImprintException { + return getTypedValueOrThrow(fieldId, com.imprint.types.TypeCode.INT64, Value.Int64Value.class, "int64").getValue(); + } + + public boolean getBoolean(int fieldId) throws ImprintException { + return getTypedValueOrThrow(fieldId, com.imprint.types.TypeCode.BOOL, Value.BoolValue.class, "boolean").getValue(); + } + + public float getFloat32(int fieldId) throws ImprintException { + return getTypedValueOrThrow(fieldId, com.imprint.types.TypeCode.FLOAT32, Value.Float32Value.class, "float32").getValue(); + } + + public double getFloat64(int fieldId) throws ImprintException { + return getTypedValueOrThrow(fieldId, com.imprint.types.TypeCode.FLOAT64, Value.Float64Value.class, "float64").getValue(); + } + public byte[] getBytes(int fieldId) throws ImprintException { var value = getValidatedValue(fieldId, "BYTES"); if (value instanceof Value.BytesValue) @@ -189,145 +250,402 @@ public byte[] getBytes(int fieldId) throws ImprintException { return ((Value.BytesBufferValue) value).getValue(); throw new ImprintException(ErrorType.TYPE_MISMATCH, "Field " + fieldId + " is not BYTES"); } - - public List getArray(int fieldId) throws ImprintException { - return getTypedValueOrThrow(fieldId, TypeCode.ARRAY, Value.ArrayValue.class, "ARRAY").getValue(); + + public java.util.List getArray(int fieldId) throws ImprintException { + return getTypedValueOrThrow(fieldId, com.imprint.types.TypeCode.ARRAY, Value.ArrayValue.class, "ARRAY").getValue(); } - - public Map getMap(int fieldId) throws ImprintException { - return getTypedValueOrThrow(fieldId, TypeCode.MAP, Value.MapValue.class, "MAP").getValue(); + + public java.util.Map getMap(int fieldId) throws ImprintException { + return getTypedValueOrThrow(fieldId, com.imprint.types.TypeCode.MAP, Value.MapValue.class, "MAP").getValue(); } - + public ImprintRecord getRow(int fieldId) throws ImprintException { - return getTypedValueOrThrow(fieldId, TypeCode.ROW, Value.RowValue.class, "ROW").getValue(); + return getTypedValueOrThrow(fieldId, com.imprint.types.TypeCode.ROW, Value.RowValue.class, "ROW").getValue(); } - // ========== SERIALIZATION ========== - /** - * Serialize this record to a ByteBuffer. + * Returns a copy of the bytes. */ public ByteBuffer serializeToBuffer() { - var directoryBuffer = buffers.serializeDirectory(); // This is now optimized to return a duplicate - var payloadBuffer = buffers.getPayload(); - - int finalSize = Constants.HEADER_BYTES + directoryBuffer.remaining() + payloadBuffer.remaining(); - var finalBuffer = ByteBuffer.allocate(finalSize); - finalBuffer.order(ByteOrder.LITTLE_ENDIAN); + return serializedBytes.duplicate(); + } - // Assemble the final record from existing components - serializeHeader(this.header, finalBuffer); - finalBuffer.put(directoryBuffer); - finalBuffer.put(payloadBuffer.duplicate()); + /** + * Get the schema ID from the header. + */ + public SchemaId getSchemaId() { + return header.getSchemaId(); + } + + /** + * Estimate the memory footprint of this record. + */ + public int getSerializedSize() { + return serializedBytes.remaining(); + } - finalBuffer.flip(); - return finalBuffer.asReadOnlyBuffer(); + + /** + * Get and validate a value exists and is not null. + */ + private Value getValidatedValue(int fieldId, String typeName) throws ImprintException { + var value = getValue(fieldId); + if (value == null) + throw new ImprintException(ErrorType.FIELD_NOT_FOUND, "Field " + fieldId + " not found"); + if (value.getTypeCode() == com.imprint.types.TypeCode.NULL) + throw new ImprintException(ErrorType.TYPE_MISMATCH, "Field " + fieldId + " is NULL, cannot retrieve as " + typeName); + return value; } - public int estimateSerializedSize() { - int size = Constants.HEADER_BYTES; // header - size += buffers.serializeDirectory().remaining(); // directory - size += buffers.getPayload().remaining(); // payload - return size; + private T getTypedValueOrThrow(int fieldId, com.imprint.types.TypeCode expectedTypeCode, Class expectedValueClass, String expectedTypeName) + throws ImprintException { + var value = getValidatedValue(fieldId, expectedTypeName); + if (value.getTypeCode() == expectedTypeCode && expectedValueClass.isInstance(value)) + return expectedValueClass.cast(value); + throw new ImprintException(ErrorType.TYPE_MISMATCH, "Field " + fieldId + " is of type " + value.getTypeCode() + ", expected " + expectedTypeName); + } + + /** + * Parse buffers from serialized record bytes. + */ + private static ParsedBuffers parseBuffersFromSerialized(ByteBuffer serializedRecord) throws ImprintException { + var buffer = serializedRecord.duplicate().order(ByteOrder.LITTLE_ENDIAN); + + // Parse header and extract sections using shared utility + var header = parseHeaderFromBuffer(buffer); + var sections = extractBufferSections(buffer, header); + + return new ParsedBuffers(sections.directoryBuffer, sections.payloadBuffer); + } + + private static class ParsedBuffers { + final ByteBuffer directoryBuffer; + final ByteBuffer payload; + + ParsedBuffers(ByteBuffer directoryBuffer, ByteBuffer payload) { + this.directoryBuffer = directoryBuffer; + this.payload = payload; + } } + + private int getDirectoryCount() { + try { + return VarInt.decode(directoryBuffer.duplicate()).getValue(); + } catch (ImprintException e) { + return 0; // Cache as 0 on error + } + } + + /** + * Gets ByteBuffer view of a field's data. + */ + private ByteBuffer getFieldBuffer(int fieldId) throws ImprintException { + var entry = findDirectoryEntry(fieldId); + if (entry == null) + return null; + + int startOffset = entry.getOffset(); + int endOffset = findEndOffset(entry.getId()); + + if (startOffset < 0 || endOffset < 0 || startOffset > payload.limit() || + endOffset > payload.limit() || startOffset > endOffset) { + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, "Invalid field buffer range: start=" + startOffset + ", end=" + endOffset); + } + + var fieldBuffer = payload.duplicate(); + fieldBuffer.position(startOffset).limit(endOffset); + return fieldBuffer; + } + + private Directory findDirectoryEntry(int fieldId) throws ImprintException { + var searchBuffer = directoryBuffer.duplicate().order(ByteOrder.LITTLE_ENDIAN); + + int count = getDirectoryCount(); + if (count == 0) return null; + + // Advance past varint to entries + VarInt.decode(searchBuffer); + int directoryStartPos = searchBuffer.position(); + + int low = 0; + int high = count - 1; + + while (low <= high) { + int mid = (low + high) >>> 1; + int entryPos = directoryStartPos + (mid * Constants.DIR_ENTRY_BYTES); + + if (entryPos + Constants.DIR_ENTRY_BYTES > searchBuffer.limit()) + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, "Directory entry exceeds buffer"); + + searchBuffer.position(entryPos); + short midFieldId = searchBuffer.getShort(); + + if (midFieldId < fieldId) { + low = mid + 1; + } else if (midFieldId > fieldId) { + high = mid - 1; + } else { + // Found it - read complete entry + searchBuffer.position(entryPos); + return deserializeDirectoryEntry(searchBuffer); + } + } + + return null; + } + + private int findEndOffset(int currentFieldId) throws ImprintException { + var scanBuffer = directoryBuffer.duplicate().order(ByteOrder.LITTLE_ENDIAN); + + int count = getDirectoryCount(); + if (count == 0) return payload.limit(); + + // Advance past varint + VarInt.decode(scanBuffer); + int directoryStartPos = scanBuffer.position(); + + int low = 0; + int high = count - 1; + int nextOffset = payload.limit(); + + // Binary search for first field with fieldId > currentFieldId + while (low <= high) { + int mid = (low + high) >>> 1; + int entryPos = directoryStartPos + (mid * Constants.DIR_ENTRY_BYTES); + + if (entryPos + Constants.DIR_ENTRY_BYTES > scanBuffer.limit()) break; + + scanBuffer.position(entryPos); + short fieldId = scanBuffer.getShort(); + scanBuffer.get(); // skip type + int offset = scanBuffer.getInt(); + + if (fieldId > currentFieldId) { + nextOffset = offset; + high = mid - 1; + } else { + low = mid + 1; + } + } + + return nextOffset; + } + + private Directory deserializeDirectoryEntry(ByteBuffer buffer) throws ImprintException { + if (buffer.remaining() < Constants.DIR_ENTRY_BYTES) + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, "Not enough bytes for directory entry"); + + short id = buffer.getShort(); + var typeCode = TypeCode.fromByte(buffer.get()); + int offset = buffer.getInt(); + + return new Directory.Entry(id, typeCode, offset); + } + + /** + * DirectoryView + */ + private class ImprintDirectoryView implements Directory.DirectoryView { + + @Override + public Directory findEntry(int fieldId) { + try { + return findDirectoryEntry(fieldId); + } catch (ImprintException e) { + return null; + } + } + /** + * List out all directories in the buffer. This operation unpacks any directories not already deserialized + * so proceed only if eager evaluation is intended. + */ + @Override + public List toList() { + var list = new ArrayList(getDirectoryCount()); + var iterator = iterator(); + while (iterator.hasNext()) { + list.add(iterator.next()); + } + return list; + } + + @Override + public int size() { + return getDirectoryCount(); + } + + @Override + public Iterator iterator() { + return new ImprintDirectoryIterator(); + } + } + + /** + * Iterator that parses directory entries lazily from raw bytes. + */ + private class ImprintDirectoryIterator implements Iterator { + private final ByteBuffer iterBuffer; + private final int totalCount; + private int currentIndex; + + ImprintDirectoryIterator() { + this.iterBuffer = directoryBuffer.duplicate().order(ByteOrder.LITTLE_ENDIAN); + this.totalCount = getDirectoryCount(); + + try { + // Skip past varint to first entry + VarInt.decode(iterBuffer); + } catch (ImprintException e) { + throw new RuntimeException("Failed to initialize directory iterator", e); + } + this.currentIndex = 0; + } + + @Override + public boolean hasNext() { + return currentIndex < totalCount; + } + + @Override + public Directory next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + + try { + var entry = deserializeDirectoryEntry(iterBuffer); + currentIndex++; + return entry; + } catch (ImprintException e) { + throw new RuntimeException("Failed to parse directory entry at index " + currentIndex, e); + } + } + } + /** - * Serializes the components of a record into a single ByteBuffer. - * This provides a direct serialization path without needing a live ImprintRecord instance. - * This assumes the list is pre-sorted by field ID. + * Used by {@link ImprintRecordBuilder} with sorted field data. + * Creates directory buffer from field data and calculated offsets. * - * @param schemaId The schema identifier for the record. - * @param sortedDirectory The list of directory entries, which MUST be sorted by field ID. - * @param payload The ByteBuffer containing all field data concatenated. - * @return A read-only ByteBuffer with the complete serialized record. + * @param sortedFields Array of FieldData objects sorted by ID + * @param offsets Array of payload offsets corresponding to each field + * @param fieldCount Number of valid fields to process */ - public static ByteBuffer serialize(SchemaId schemaId, List sortedDirectory, ByteBuffer payload) { - var header = new Header(new Flags((byte) 0), schemaId, payload.remaining()); - var directoryBuffer = ImprintBuffers.createDirectoryBuffer(sortedDirectory); - - int finalSize = Constants.HEADER_BYTES + directoryBuffer.remaining() + payload.remaining(); - var finalBuffer = ByteBuffer.allocate(finalSize); - finalBuffer.order(ByteOrder.LITTLE_ENDIAN); - - // Assemble the final record - serializeHeader(header, finalBuffer); - finalBuffer.put(directoryBuffer); - finalBuffer.put(payload); + static ByteBuffer createDirectoryBufferFromSorted(Object[] sortedFields, int[] offsets, int fieldCount) { + if (fieldCount == 0) + return createEmptyDirectoryBuffer(); + + int size = calculateDirectorySize(fieldCount); + var buffer = ByteBuffer.allocate(size); + buffer.order(ByteOrder.LITTLE_ENDIAN); + VarInt.encode(fieldCount, buffer); + + //this ends up being kind of a hotspot for some reason, probably boundary checking. + //Direct writes might help a bit it could get difficult since pretty much all the other + //frameworks just go straight for Unsafe + for (int i = 0; i < fieldCount; i++) { + var fieldData = (ImprintRecordBuilder.FieldData) sortedFields[i]; + buffer.putShort(fieldData.id); + buffer.put(fieldData.value.getTypeCode().getCode()); + buffer.putInt(offsets[i]); + } - finalBuffer.flip(); - return finalBuffer.asReadOnlyBuffer(); + buffer.flip(); + return buffer; } - // ========== STATIC FACTORY METHODS ========== - - public static ImprintRecordBuilder builder(SchemaId schemaId) { - return new ImprintRecordBuilder(schemaId); + private static ByteBuffer createEmptyDirectoryBuffer() { + ByteBuffer buffer = ByteBuffer.allocate(1); + VarInt.encode(0, buffer); + buffer.flip(); + return buffer; } - - public static ImprintRecordBuilder builder(int fieldspaceId, int schemaHash) { - return new ImprintRecordBuilder(new SchemaId(fieldspaceId, schemaHash)); + + /** + * Parse a header from a ByteBuffer without advancing the buffer position. + * Utility method shared between {@link ImprintRecord} and {@link ImprintOperations}. + */ + public static Header parseHeaderFromBuffer(ByteBuffer buffer) throws ImprintException { + int startPos = buffer.position(); + try { + return parseHeader(buffer); + } finally { + buffer.position(startPos); + } } - - public static ImprintRecord deserialize(byte[] bytes) throws ImprintException { - return deserialize(ByteBuffer.wrap(bytes)); + + /** + * Calculate the size needed to store a directory with the given entry count. + */ + public static int calculateDirectorySize(int entryCount) { + return VarInt.encodedLength(entryCount) + (entryCount * Constants.DIR_ENTRY_BYTES); } - - public static ImprintRecord deserialize(ByteBuffer buffer) throws ImprintException { - buffer = buffer.duplicate().order(ByteOrder.LITTLE_ENDIAN); - - // Read header - var header = deserializeHeader(buffer); - - // Calculate directory size + + /** + * Container for separated directory and payload buffer sections. + * Utility class shared between {@link ImprintRecord} and {@link ImprintOperations}. + */ + public static class BufferSections { + public final ByteBuffer directoryBuffer; + public final ByteBuffer payloadBuffer; + public final int directoryCount; + + public BufferSections(ByteBuffer directoryBuffer, ByteBuffer payloadBuffer, int directoryCount) { + this.directoryBuffer = directoryBuffer; + this.payloadBuffer = payloadBuffer; + this.directoryCount = directoryCount; + } + } + + /** + * Extract directory and payload sections from a serialized buffer. + * Utility method shared between {@link ImprintRecord} and {@link ImprintOperations}. + */ + public static BufferSections extractBufferSections(ByteBuffer buffer, Header header) throws ImprintException { + // Skip header + buffer.position(buffer.position() + Constants.HEADER_BYTES); + + // Parse directory section int directoryStartPos = buffer.position(); var countResult = VarInt.decode(buffer); int directoryCount = countResult.getValue(); int directorySize = countResult.getBytesRead() + (directoryCount * Constants.DIR_ENTRY_BYTES); - + // Create directory buffer buffer.position(directoryStartPos); var directoryBuffer = buffer.slice(); directoryBuffer.limit(directorySize); - - // Advance past directory + + // Advance to payload buffer.position(buffer.position() + directorySize); - - // Create payload buffer - var payload = buffer.slice(); - payload.limit(header.getPayloadSize()); - - // Create buffers wrapper - var buffers = new ImprintBuffers(directoryBuffer, payload); - - return new ImprintRecord(header, buffers); + var payloadBuffer = buffer.slice(); + payloadBuffer.limit(header.getPayloadSize()); + + return new BufferSections(directoryBuffer, payloadBuffer, directoryCount); } + + private static Header parseHeader(ByteBuffer buffer) throws ImprintException { + if (buffer.remaining() < Constants.HEADER_BYTES) + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, "Not enough bytes for header"); - // ========== PRIVATE HELPER METHODS ========== - - /** - * Get and validate a value exists and is not null. - */ - private Value getValidatedValue(int fieldId, String typeName) throws ImprintException { - var value = getValue(fieldId); - if (value == null) - throw new ImprintException(ErrorType.FIELD_NOT_FOUND, "Field " + fieldId + " not found"); - if (value.getTypeCode() == TypeCode.NULL) - throw new ImprintException(ErrorType.TYPE_MISMATCH, "Field " + fieldId + " is NULL, cannot retrieve as " + typeName); - return value; - } + byte magic = buffer.get(); + byte version = buffer.get(); - private T getTypedValueOrThrow(int fieldId, TypeCode expectedTypeCode, Class expectedValueClass, String expectedTypeName) - throws ImprintException { - var value = getValidatedValue(fieldId, expectedTypeName); - if (value.getTypeCode() == expectedTypeCode && expectedValueClass.isInstance(value)) - return expectedValueClass.cast(value); - throw new ImprintException(ErrorType.TYPE_MISMATCH, "Field " + fieldId + " is of type " + value.getTypeCode() + ", expected " + expectedTypeName); + if (magic != Constants.MAGIC) + throw new ImprintException(ErrorType.INVALID_MAGIC, "Invalid magic byte"); + if (version != Constants.VERSION) + throw new ImprintException(ErrorType.UNSUPPORTED_VERSION, "Unsupported version: " + version); + + var flags = new Flags(buffer.get()); + int fieldSpaceId = buffer.getInt(); + int schemaHash = buffer.getInt(); + int payloadSize = buffer.getInt(); + + return new Header(flags, new SchemaId(fieldSpaceId, schemaHash), payloadSize); } - - private Value deserializeValue(TypeCode typeCode, ByteBuffer buffer) throws ImprintException { - var valueBuffer = buffer.duplicate(); - valueBuffer.order(ByteOrder.LITTLE_ENDIAN); - + + private Value deserializeValue(com.imprint.types.TypeCode typeCode, ByteBuffer buffer) throws ImprintException { + var valueBuffer = buffer.duplicate().order(ByteOrder.LITTLE_ENDIAN); switch (typeCode) { case NULL: case BOOL: @@ -347,43 +665,4 @@ private Value deserializeValue(TypeCode typeCode, ByteBuffer buffer) throws Impr throw new ImprintException(ErrorType.INVALID_TYPE_CODE, "Unknown type code: " + typeCode); } } - - private static void serializeHeader(Header header, ByteBuffer buffer) { - buffer.put(Constants.MAGIC); - buffer.put(Constants.VERSION); - buffer.put(header.getFlags().getValue()); - buffer.putInt(header.getSchemaId().getFieldSpaceId()); - buffer.putInt(header.getSchemaId().getSchemaHash()); - buffer.putInt(header.getPayloadSize()); - } - - private static Header deserializeHeader(ByteBuffer buffer) throws ImprintException { - if (buffer.remaining() < Constants.HEADER_BYTES) - throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, "Not enough bytes for header"); - - - byte magic = buffer.get(); - if (magic != Constants.MAGIC) { - throw new ImprintException(ErrorType.INVALID_MAGIC, "Invalid magic byte: expected 0x" + Integer.toHexString(Constants.MAGIC) + - ", got 0x" + Integer.toHexString(magic & 0xFF)); - } - - byte version = buffer.get(); - if (version != Constants.VERSION) { - throw new ImprintException(ErrorType.UNSUPPORTED_VERSION, "Unsupported version: " + version); - } - - var flags = new Flags(buffer.get()); - int fieldSpaceId = buffer.getInt(); - int schemaHash = buffer.getInt(); - int payloadSize = buffer.getInt(); - - return new Header(flags, new SchemaId(fieldSpaceId, schemaHash), payloadSize); - } - - @Override - public String toString() { - return String.format("ImprintRecord{header=%s, directorySize=%d, payloadSize=%d}", - header, buffers.getDirectoryCount(), buffers.getPayload().remaining()); - } } \ No newline at end of file diff --git a/src/main/java/com/imprint/core/ImprintRecordBuilder.java b/src/main/java/com/imprint/core/ImprintRecordBuilder.java index 93e83ba..8e1dfa0 100644 --- a/src/main/java/com/imprint/core/ImprintRecordBuilder.java +++ b/src/main/java/com/imprint/core/ImprintRecordBuilder.java @@ -1,5 +1,7 @@ package com.imprint.core; +import com.imprint.Constants; +import com.imprint.error.ErrorType; import com.imprint.error.ImprintException; import com.imprint.types.MapKey; import com.imprint.types.Value; @@ -33,9 +35,21 @@ @SuppressWarnings("unused") public final class ImprintRecordBuilder { private final SchemaId schemaId; - private final Map fields = new TreeMap<>(); + // Custom int→object map optimized for primitive keys + private final ImprintFieldObjectMap fields = new ImprintFieldObjectMap<>(); private int estimatedPayloadSize = 0; + static final class FieldData { + final short id; + final Value value; + + FieldData(short id, Value value) { + this.id = id; + this.value = value; + } + } + + ImprintRecordBuilder(SchemaId schemaId) { this.schemaId = Objects.requireNonNull(schemaId, "SchemaId cannot be null"); } @@ -70,7 +84,7 @@ public ImprintRecordBuilder field(int id, byte[] value) { } // Collections with automatic conversion - public ImprintRecordBuilder field(int id, List values) { + public ImprintRecordBuilder field(int id, List values) { var convertedValues = new ArrayList(values.size()); for (var item : values) { convertedValues.add(convertToValue(item)); @@ -78,7 +92,7 @@ public ImprintRecordBuilder field(int id, List values) { return addField(id, Value.fromArray(convertedValues)); } - public ImprintRecordBuilder field(int id, Map map) { + public ImprintRecordBuilder field(int id, Map map) { var convertedMap = new HashMap(map.size()); for (var entry : map.entrySet()) { var key = convertToMapKey(entry.getKey()); @@ -133,47 +147,48 @@ public int fieldCount() { } public Set fieldIds() { - return fields.keySet(); + var ids = new HashSet(fields.size()); + var keys = fields.getKeys(); + for (var key : keys) { + ids.add(key); + } + return ids; } // Build the final record public ImprintRecord build() throws ImprintException { - var payloadBuffer = ByteBuffer.allocate(estimatePayloadSize()); - payloadBuffer.order(ByteOrder.LITTLE_ENDIAN); - - for (var entry : fields.values()) { - entry.setOffset(payloadBuffer.position()); - serializeValue(entry.getValue(), payloadBuffer); - } - - // Create read-only view of the payload without copying - payloadBuffer.flip(); // limit = position, position = 0 - var payloadView = payloadBuffer.slice().asReadOnlyBuffer(); - - var header = new Header(new Flags((byte) 0), schemaId, payloadView.remaining()); - return new ImprintRecord(header, fields, payloadView); + // Build to bytes and then create ImprintRecord from bytes for consistency + var serializedBytes = buildToBuffer(); + return ImprintRecord.fromBytes(serializedBytes); } /** - * Builds the record and serializes it directly to a ByteBuffer without creating an intermediate ImprintRecord object. + * Builds the record and serializes it directly to a ByteBuffer. * * @return A read-only ByteBuffer containing the fully serialized record. * @throws ImprintException if serialization fails. */ public ByteBuffer buildToBuffer() throws ImprintException { - // 1. Prepare payload and directory + // 1. Sort fields by ID for directory ordering (zero allocation) + var sortedFieldsResult = getSortedFieldsResult(); + var sortedFields = sortedFieldsResult.values; + var fieldCount = sortedFieldsResult.count; + + // 2. Serialize payload and calculate offsets var payloadBuffer = ByteBuffer.allocate(estimatePayloadSize()); payloadBuffer.order(ByteOrder.LITTLE_ENDIAN); - for (var entry : fields.values()) { - entry.setOffset(payloadBuffer.position()); - serializeValue(entry.getValue(), payloadBuffer); + int[] offsets = new int[fieldCount]; + for (int i = 0; i < fieldCount; i++) { + var fieldData = (FieldData) sortedFields[i]; + offsets[i] = payloadBuffer.position(); + serializeValue(fieldData.value, payloadBuffer); } payloadBuffer.flip(); var payloadView = payloadBuffer.slice().asReadOnlyBuffer(); - // 2. Serialize directly to the final buffer format using the map-based method - return ImprintRecord.serialize(schemaId, new ArrayList<>(fields.values()), payloadView); + // 3. Create directory buffer and serialize to final buffer + return serializeToBuffer(schemaId, sortedFields, offsets, fieldCount, payloadView); } /** @@ -186,15 +201,17 @@ public ByteBuffer buildToBuffer() throws ImprintException { */ private ImprintRecordBuilder addField(int id, Value value) { Objects.requireNonNull(value, "Value cannot be null - use nullField() for explicit null values"); - var newEntry = new Directory.Builder((short) id, value); + var newEntry = new FieldData((short) id, value); - // Subtract the size of the old value if it's being replaced. + // Check if replacing an existing field - O(1) lookup without boxing! var oldEntry = fields.get(id); - if (oldEntry != null) - estimatedPayloadSize -= estimateValueSize(oldEntry.getValue()); + if (oldEntry != null) { + estimatedPayloadSize -= estimateValueSize(oldEntry.value); + } + // Add or replace field - O(1) operation without boxing! fields.put(id, newEntry); - estimatedPayloadSize += estimateValueSize(newEntry.getValue()); + estimatedPayloadSize += estimateValueSize(newEntry.value); return this; } @@ -230,7 +247,6 @@ private Value convertToValue(Object obj) { return Value.fromBytes((byte[]) obj); } if (obj instanceof List) { - //test @SuppressWarnings("unchecked") List list = (List) obj; var convertedValues = new ArrayList(list.size()); @@ -278,11 +294,9 @@ private int estimatePayloadSize() { // Add 25% buffer to reduce reallocations and handle VarInt encoding fluctuations. return Math.max(estimatedPayloadSize + (estimatedPayloadSize / 4), fields.size() * 16); } - + /** * Estimates the serialized size in bytes for a given value. - * This method provides size estimates for payload buffer allocation, - * supporting both array-based and ByteBuffer-based value types. * * @param value the value to estimate size for * @return estimated size in bytes including type-specific overhead @@ -308,7 +322,7 @@ private int estimateValueSize(Value value) { return rowValue.getValue().estimateSerializedSize(); default: - throw new ImprintException(com.imprint.error.ErrorType.SERIALIZATION_ERROR, "Unknown type code: " + value.getTypeCode()); + throw new ImprintException(ErrorType.SERIALIZATION_ERROR, "Unknown type code: " + value.getTypeCode()); } } @@ -335,7 +349,40 @@ private void serializeValue(Value value, ByteBuffer buffer) throws ImprintExcept break; default: - throw new ImprintException(com.imprint.error.ErrorType.SERIALIZATION_ERROR, "Unknown type code: " + value.getTypeCode()); + throw new ImprintException(ErrorType.SERIALIZATION_ERROR, "Unknown type code: " + value.getTypeCode()); } } + + /** + * Get fields sorted by ID from the map. + * Returns internal map array reference + count to avoid any copying but sacrifices the map structure in the process. + */ + private ImprintFieldObjectMap.SortedValuesResult getSortedFieldsResult() { + return fields.getSortedValues(); + } + + /** + * Serialize components into a single ByteBuffer. + */ + private static ByteBuffer serializeToBuffer(SchemaId schemaId, Object[] sortedFields, int[] offsets, int fieldCount, ByteBuffer payload) { + var header = new Header(new Flags((byte) 0), schemaId, payload.remaining()); + var directoryBuffer = ImprintRecord.createDirectoryBufferFromSorted(sortedFields, offsets, fieldCount); + + int finalSize = Constants.HEADER_BYTES + directoryBuffer.remaining() + payload.remaining(); + var finalBuffer = ByteBuffer.allocate(finalSize); + finalBuffer.order(ByteOrder.LITTLE_ENDIAN); + + // Write header + finalBuffer.put(Constants.MAGIC); + finalBuffer.put(Constants.VERSION); + finalBuffer.put(header.getFlags().getValue()); + finalBuffer.putInt(header.getSchemaId().getFieldSpaceId()); + finalBuffer.putInt(header.getSchemaId().getSchemaHash()); + finalBuffer.putInt(header.getPayloadSize()); + finalBuffer.put(directoryBuffer); + finalBuffer.put(payload); + + finalBuffer.flip(); + return finalBuffer.asReadOnlyBuffer(); + } } \ No newline at end of file diff --git a/src/main/java/com/imprint/error/ErrorType.java b/src/main/java/com/imprint/error/ErrorType.java index 49784ef..63a8c60 100644 --- a/src/main/java/com/imprint/error/ErrorType.java +++ b/src/main/java/com/imprint/error/ErrorType.java @@ -13,6 +13,7 @@ public enum ErrorType { MALFORMED_VARINT, TYPE_MISMATCH, INVALID_TYPE_CODE, + INVALID_BUFFER, SERIALIZATION_ERROR, DESERIALIZATION_ERROR, INTERNAL_ERROR diff --git a/src/main/java/com/imprint/ops/ImprintOperations.java b/src/main/java/com/imprint/ops/ImprintOperations.java index 49f60b4..f15e6a1 100644 --- a/src/main/java/com/imprint/ops/ImprintOperations.java +++ b/src/main/java/com/imprint/ops/ImprintOperations.java @@ -1,168 +1,378 @@ package com.imprint.ops; +import com.imprint.Constants; import com.imprint.core.*; import com.imprint.error.ErrorType; import com.imprint.error.ImprintException; +import com.imprint.util.VarInt; +import lombok.Value; import lombok.experimental.UtilityClass; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.util.*; -import java.util.stream.Collectors; @UtilityClass public class ImprintOperations { /** - * Project a subset of fields from an Imprint record. Payload copying is proportional to projected data size. - * - *

Algorithm:

- *
    - *
  1. Sort and deduplicate requested field IDs for efficient matching
    2. - *
  2. Scan directory to find matching fields and calculate ranges
    3. - *
  3. Allocate new payload buffer with exact size needed
    4. - *
  4. Copy field data ranges directly (zero-copy where possible)
    5. - *
  5. Build new directory with adjusted offsets
    6. - *
- * - * @param record The source record to project from - * @param fieldIds Array of field IDs to include in projection - * @return New ImprintRecord containing only the requested fields + * Pure bytes-to-bytes merge operation that avoids all object creation. + * Performs merge directly on serialized Imprint record buffers. + * + * @param firstBuffer Complete serialized Imprint record + * @param secondBuffer Complete serialized Imprint record + * @return Merged record as serialized bytes + * @throws ImprintException if merge fails */ - public static ImprintRecord project(ImprintRecord record, int... fieldIds) { - // Sort and deduplicate field IDs for efficient matching - final var fieldIdSet = Arrays.stream(fieldIds) - .boxed() - .collect(Collectors.toCollection(TreeSet::new)); - if (fieldIdSet.isEmpty()) { - return createEmptyRecord(record.getHeader().getSchemaId()); - } + public static ByteBuffer mergeBytes(ByteBuffer firstBuffer, ByteBuffer secondBuffer) throws ImprintException { + validateImprintBuffer(firstBuffer, "firstBuffer"); + validateImprintBuffer(secondBuffer, "secondBuffer"); + + // Work on duplicates to avoid affecting original positions + var first = firstBuffer.duplicate().order(ByteOrder.LITTLE_ENDIAN); + var second = secondBuffer.duplicate().order(ByteOrder.LITTLE_ENDIAN); + + // Parse headers + var firstHeader = parseHeaderOnly(first); + var secondHeader = parseHeaderOnly(second); + + // Extract directory and payload sections + var firstSections = extractSections(first, firstHeader); + var secondSections = extractSections(second, secondHeader); + + // Perform raw merge + return mergeRawSections(firstHeader, firstSections, secondSections); + } - var newDirectory = new ArrayList(fieldIdSet.size()); - var payloadChunks = new ArrayList(fieldIdSet.size()); - int currentOffset = 0; + /** + * Parse just the header without advancing buffer past it + */ + private static Header parseHeaderOnly(ByteBuffer buffer) throws ImprintException { + return ImprintRecord.parseHeaderFromBuffer(buffer); + } + + /** + * Extract directory and payload sections from a buffer + */ + private static ImprintRecord.BufferSections extractSections(ByteBuffer buffer, Header header) throws ImprintException { + return ImprintRecord.extractBufferSections(buffer, header); + } + + /** + * Merge raw directory and payload sections without object creation + */ + private static ByteBuffer mergeRawSections(Header firstHeader, ImprintRecord.BufferSections firstSections, ImprintRecord.BufferSections secondSections) throws ImprintException { + // Prepare directory iterators + var firstDirIter = new RawDirectoryIterator(firstSections.directoryBuffer); + var secondDirIter = new RawDirectoryIterator(secondSections.directoryBuffer); + + // Pre-allocate - worst case is sum of both directory counts + int maxEntries = firstSections.directoryCount + secondSections.directoryCount; + var mergedDirectoryEntries = new ArrayList(maxEntries); + var mergedChunks = new ArrayList(maxEntries); - for (int fieldId : fieldIdSet) { - // Use efficient lookup for each field's metadata. Returns null on failure. - var sourceEntry = record.getDirectoryEntry(fieldId); - - // If field exists, get its payload and add to the new record components - if (sourceEntry != null) { - var fieldPayload = record.getRawBytes(sourceEntry); - // This check is for internal consistency. If an entry exists, payload should too. - if (fieldPayload != null) { - newDirectory.add(new Directory.Entry((short)fieldId, sourceEntry.getTypeCode(), currentOffset)); - payloadChunks.add(fieldPayload); - currentOffset += fieldPayload.remaining(); + int totalMergedPayloadSize = 0; + int currentMergedOffset = 0; + + RawDirectoryEntry firstEntry = firstDirIter.hasNext() ? firstDirIter.next() : null; + RawDirectoryEntry secondEntry = secondDirIter.hasNext() ? secondDirIter.next() : null; + + // Merge directories and collect payload chunks + while (firstEntry != null || secondEntry != null) { + RawDirectoryEntry currentEntry; + ByteBuffer sourcePayload; + + if (firstEntry != null && (secondEntry == null || firstEntry.fieldId <= secondEntry.fieldId)) { + // Take from first + currentEntry = firstEntry; + sourcePayload = getFieldPayload(firstSections.payloadBuffer, firstEntry, firstDirIter); + + // Skip duplicate in second if present + if (secondEntry != null && firstEntry.fieldId == secondEntry.fieldId) { + secondEntry = secondDirIter.hasNext() ? secondDirIter.next() : null; } + firstEntry = firstDirIter.hasNext() ? firstDirIter.next() : null; + } else { + // Take from second + currentEntry = secondEntry; + sourcePayload = getFieldPayload(secondSections.payloadBuffer, secondEntry, secondDirIter); + secondEntry = secondDirIter.hasNext() ? secondDirIter.next() : null; } + + // Add to merged directory with adjusted offset + var adjustedEntry = new RawDirectoryEntry(currentEntry.fieldId, currentEntry.typeCode, currentMergedOffset); + mergedDirectoryEntries.add(adjustedEntry); + + // Collect payload chunk + mergedChunks.add(sourcePayload.duplicate()); + currentMergedOffset += sourcePayload.remaining(); + totalMergedPayloadSize += sourcePayload.remaining(); } + + // Build final merged buffer + return buildSerializedBuffer(firstHeader, mergedDirectoryEntries, mergedChunks, totalMergedPayloadSize); + } + + /** + * Get payload bytes for a specific field using iterator state + */ + private static ByteBuffer getFieldPayload(ByteBuffer payload, RawDirectoryEntry entry, RawDirectoryIterator iterator) { + int startOffset = entry.offset; + int endOffset = iterator.getNextEntryOffset(payload.limit()); - // Build new payload from collected chunks - ByteBuffer newPayload = buildPayloadFromChunks(payloadChunks, currentOffset); - - // Create new header with updated payload size - // TODO: compute correct schema hash - var newHeader = new Header(record.getHeader().getFlags(), - new SchemaId(record.getHeader().getSchemaId().getFieldSpaceId(), 0xdeadbeef), - newPayload.remaining() - ); - - return new ImprintRecord(newHeader, newDirectory, newPayload); + var fieldPayload = payload.duplicate(); + fieldPayload.position(startOffset); + fieldPayload.limit(endOffset); + return fieldPayload.slice(); } + /** - * Merge two Imprint records, combining their fields. Payload copying is proportional to total data size. - * - *

Merge Strategy:

- *
    - *
  • Fields are merged using sort-merge algorithm on directory entries
    • - *
  • For duplicate field IDs: first record's field takes precedence
    • - *
  • Payloads are concatenated with directory offsets adjusted
    • - *
  • Schema ID from first record is preserved
    • - *
- *

- * - * @param first The first record (takes precedence for duplicate fields) - * @param second The second record to merge - * @return New ImprintRecord containing merged fields - * @throws ImprintException if merge fails due to incompatible records + * Pure bytes-to-bytes projection operation that avoids all object creation. + * Projects a subset of fields directly from a serialized Imprint record. + * + * @param sourceBuffer Complete serialized Imprint record + * @param fieldIds Array of field IDs to include in projection + * @return Projected record as serialized bytes + * @throws ImprintException if projection fails */ - public static ImprintRecord merge(ImprintRecord first, ImprintRecord second) throws ImprintException { - var firstDir = first.getDirectory(); - var secondDir = second.getDirectory(); - - // Pre-allocate for worst case (no overlapping fields) - var newDirectory = new ArrayList(firstDir.size() + secondDir.size()); - var payloadChunks = new ArrayList(); + public static ByteBuffer projectBytes(ByteBuffer sourceBuffer, int... fieldIds) throws ImprintException { + validateImprintBuffer(sourceBuffer, "sourceBuffer"); + + if (fieldIds == null || fieldIds.length == 0) { + return createEmptyRecordBytes(); + } + + // Sort field IDs for efficient merge algorithm (duplicates handled naturally) + var sortedFieldIds = fieldIds.clone(); + Arrays.sort(sortedFieldIds); + + // Work on duplicate to avoid affecting original position + var source = sourceBuffer.duplicate().order(ByteOrder.LITTLE_ENDIAN); + + // Parse header + var header = parseHeaderOnly(source); + + // Extract sections + var sections = extractSections(source, header); + + // Perform raw projection + return projectRawSections(header, sections, sortedFieldIds); + } - int firstIdx = 0; - int secondIdx = 0; + /** + * Project raw sections without object creation using optimized merge algorithm. + * Uses direct array operations and optimized memory access for maximum performance. + */ + private static ByteBuffer projectRawSections(Header originalHeader, ImprintRecord.BufferSections sections, int[] sortedRequestedFields) throws ImprintException { + + if (sortedRequestedFields.length == 0) { + return buildSerializedBuffer(originalHeader, new RawDirectoryEntry[0], new ByteBuffer[0]); + } + + // Use pre-sized ArrayLists to avoid System.arraycopy but still be efficient + var projectedEntries = new ArrayList(sortedRequestedFields.length); + var payloadChunks = new ArrayList(sortedRequestedFields.length); + int totalProjectedPayloadSize = 0; int currentOffset = 0; - - while (firstIdx < firstDir.size() || secondIdx < secondDir.size()) { - Directory currentEntry; - ByteBuffer currentPayload; - - if (firstIdx < firstDir.size() && - (secondIdx >= secondDir.size() || firstDir.get(firstIdx).getId() <= secondDir.get(secondIdx).getId())) { - - // Take from first record - currentEntry = firstDir.get(firstIdx); - - // Skip duplicate field in second record if present - if (secondIdx < secondDir.size() && - firstDir.get(firstIdx).getId() == secondDir.get(secondIdx).getId()) { - secondIdx++; - } - currentPayload = first.getRawBytes(currentEntry); - firstIdx++; + int requestedIndex = 0; + + // Optimize: Cache payload buffer reference to avoid getter calls + var payloadBuffer = sections.payloadBuffer; + + // Merge algorithm: two-pointer approach through sorted sequences + var dirIterator = new RawDirectoryIterator(sections.directoryBuffer); + RawDirectoryEntry currentEntry = dirIterator.hasNext() ? dirIterator.next() : null; + + while (currentEntry != null && requestedIndex < sortedRequestedFields.length) { + int fieldId = currentEntry.fieldId; + int targetFieldId = sortedRequestedFields[requestedIndex]; + + if (fieldId == targetFieldId) { + var fieldPayload = getFieldPayload(payloadBuffer, currentEntry, dirIterator); + + // Add to projection with adjusted offset + projectedEntries.add(new RawDirectoryEntry(currentEntry.fieldId, currentEntry.typeCode, currentOffset)); + + // Collect payload chunk here (fieldPayload is already sliced) + payloadChunks.add(fieldPayload); + + int payloadSize = fieldPayload.remaining(); + currentOffset += payloadSize; + totalProjectedPayloadSize += payloadSize; + + // Advance both pointers (handle dupes by advancing to next unique field) + do { + requestedIndex++; + } while (requestedIndex < sortedRequestedFields.length && sortedRequestedFields[requestedIndex] == targetFieldId); + + currentEntry = dirIterator.hasNext() ? dirIterator.next() : null; + } else if (fieldId < targetFieldId) { + // Directory field is smaller, advance directory pointer + currentEntry = dirIterator.hasNext() ? dirIterator.next() : null; } else { - // Take from second record - currentEntry = secondDir.get(secondIdx); - currentPayload = second.getRawBytes(currentEntry); - secondIdx++; + // fieldId > targetFieldId - implies requested field isn't in the directory so advance requested pointer + requestedIndex++; } + } + + return buildSerializedBuffer(originalHeader, projectedEntries, payloadChunks, totalProjectedPayloadSize); + } - if (currentPayload == null) - throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, "Failed to get raw bytes for field " + currentEntry.getId()); + /** + * Build a serialized Imprint record buffer from header, directory entries, and payload chunks. + */ + private static ByteBuffer buildSerializedBuffer(Header originalHeader, RawDirectoryEntry[] directoryEntries, ByteBuffer[] payloadChunks) { + return buildSerializedBuffer(originalHeader, Arrays.asList(directoryEntries), Arrays.asList(payloadChunks), 0); + } + + private static ByteBuffer buildSerializedBuffer(Header originalHeader, List directoryEntries, List payloadChunks, int totalPayloadSize) { + int directorySize = ImprintRecord.calculateDirectorySize(directoryEntries.size()); + int totalSize = Constants.HEADER_BYTES + directorySize + totalPayloadSize; + var finalBuffer = ByteBuffer.allocate(totalSize); + finalBuffer.order(ByteOrder.LITTLE_ENDIAN); + + // Write header (preserve original schema) + finalBuffer.put(Constants.MAGIC); + finalBuffer.put(Constants.VERSION); + finalBuffer.put(originalHeader.getFlags().getValue()); + finalBuffer.putInt(originalHeader.getSchemaId().getFieldSpaceId()); + finalBuffer.putInt(originalHeader.getSchemaId().getSchemaHash()); + finalBuffer.putInt(totalPayloadSize); + + // Write directory + VarInt.encode(directoryEntries.size(), finalBuffer); + for (var entry : directoryEntries) { + finalBuffer.putShort(entry.fieldId); + finalBuffer.put(entry.typeCode); + finalBuffer.putInt(entry.offset); + } + + // Write payload + for (var chunk : payloadChunks) + finalBuffer.put(chunk); - // Add adjusted directory entry - var newEntry = new Directory.Entry(currentEntry.getId(), - currentEntry.getTypeCode(), currentOffset); - newDirectory.add(newEntry); + finalBuffer.flip(); + return finalBuffer.asReadOnlyBuffer(); + } + + + /** + * Create an empty record as serialized bytes + */ + private static ByteBuffer createEmptyRecordBytes() { + // Minimal header + empty directory + empty payload + var buffer = ByteBuffer.allocate(Constants.HEADER_BYTES + 1); // +1 for varint 0 + buffer.order(ByteOrder.LITTLE_ENDIAN); + + // Write header for empty record + buffer.put(Constants.MAGIC); + buffer.put(Constants.VERSION); + buffer.put((byte) 0x01); + buffer.putInt(0); + buffer.putInt(0); + buffer.putInt(0); + + // Write empty directory + VarInt.encode(0, buffer); + + buffer.flip(); + return buffer.asReadOnlyBuffer(); + } - // Collect payload chunk - payloadChunks.add(currentPayload.duplicate()); - currentOffset += currentPayload.remaining(); + /** + * Validates that a ByteBuffer contains valid Imprint data by checking magic bytes and basic structure. + * + * @param buffer Buffer to validate + * @param paramName Parameter name for error messages + * @throws ImprintException if buffer is invalid + */ + private static void validateImprintBuffer(ByteBuffer buffer, String paramName) throws ImprintException { + if (buffer == null) { + throw new ImprintException(ErrorType.INVALID_BUFFER, paramName + " cannot be null"); + } + + if (buffer.remaining() < Constants.HEADER_BYTES) { + throw new ImprintException(ErrorType.INVALID_BUFFER, + paramName + " too small to contain valid Imprint header (minimum " + Constants.HEADER_BYTES + " bytes)"); } - // Build merged payload - var mergedPayload = buildPayloadFromChunks(payloadChunks, currentOffset); - - // Create header preserving first record's schema ID - var newHeader = new Header(first.getHeader().getFlags(), - first.getHeader().getSchemaId(), mergedPayload.remaining()); - return new ImprintRecord(newHeader, newDirectory, mergedPayload); + // Check invariants without advancing buffer position + var duplicate = buffer.duplicate(); + byte magic = duplicate.get(); + byte version = duplicate.get(); + if (magic != Constants.MAGIC) + throw new ImprintException(ErrorType.INVALID_BUFFER, paramName + " does not contain valid Imprint magic byte"); + if (version != Constants.VERSION) + throw new ImprintException(ErrorType.INVALID_BUFFER, paramName + " contains unsupported Imprint version: " + version); } /** - * Build a new payload buffer by concatenating chunks. + * Directory entry container used for raw byte operations */ - private static ByteBuffer buildPayloadFromChunks(List chunks, int totalSize) { - var mergedPayload = ByteBuffer.allocate(totalSize); - mergedPayload.order(ByteOrder.LITTLE_ENDIAN); - for (var chunk : chunks) - mergedPayload.put(chunk); - mergedPayload.flip(); - return mergedPayload; + @Value + private static class RawDirectoryEntry { + short fieldId; + byte typeCode; + int offset; } /** - * Create an empty record with the given schema ID. + * Iterator that parses directory entries directly from raw bytes */ - private static ImprintRecord createEmptyRecord(SchemaId schemaId) { - var header = new Header(new Flags((byte) 0x01), schemaId, 0); - return new ImprintRecord(header, Collections.emptyList(), ByteBuffer.allocate(0)); + private static class RawDirectoryIterator { + private final ByteBuffer buffer; + private final int totalCount; + private final int directoryStartPos; + private int currentIndex; + + RawDirectoryIterator(ByteBuffer directoryBuffer) throws ImprintException { + this.buffer = directoryBuffer.duplicate().order(ByteOrder.LITTLE_ENDIAN); + + // Read count and advance to first entry + var countResult = VarInt.decode(buffer); + this.totalCount = countResult.getValue(); + this.directoryStartPos = buffer.position(); + this.currentIndex = 0; + } + + boolean hasNext() { + return currentIndex < totalCount; + } + + RawDirectoryEntry next() throws ImprintException { + if (!hasNext()) + throw new RuntimeException("No more directory entries"); + + if (buffer.remaining() < Constants.DIR_ENTRY_BYTES) + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, "Not enough bytes for directory entry"); + + short fieldId = buffer.getShort(); + byte typeCode = buffer.get(); + int offset = buffer.getInt(); + + currentIndex++; + return new RawDirectoryEntry(fieldId, typeCode, offset); + } + + /** + * Get the offset of the next entry without state overhead. + * Returns the provided fallback if this is the last entry. + */ + int getNextEntryOffset(int fallbackOffset) { + if (currentIndex >= totalCount) + return fallbackOffset; + + // Calculate position of next entry directly + int nextEntryPos = directoryStartPos + (currentIndex * Constants.DIR_ENTRY_BYTES); + + // Bounds check - optimized to single comparison + if (nextEntryPos + 7 > buffer.limit()) { // DIR_ENTRY_BYTES = 7 + return fallbackOffset; + } + + // Read just the offset field (skip fieldId and typeCode) + return buffer.getInt(nextEntryPos + 3); // 2 bytes fieldId + 1 byte typeCode = 3 offset + } } } diff --git a/src/main/java/com/imprint/stream/ImprintStream.java b/src/main/java/com/imprint/stream/ImprintStream.java deleted file mode 100644 index 35a69ed..0000000 --- a/src/main/java/com/imprint/stream/ImprintStream.java +++ /dev/null @@ -1,259 +0,0 @@ -package com.imprint.stream; - -import com.imprint.core.*; -import com.imprint.error.ImprintException; - -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.util.ArrayDeque; -import java.util.Collections; -import java.util.Deque; -import java.util.HashSet; -import java.util.List; -import java.util.NavigableMap; -import java.util.Objects; -import java.util.Set; -import java.util.TreeMap; - -/** - * Provides a framework for lazy, (eventual) zero-copy transformations of Imprint records. - *

- * Operations like {@link #project(int...)} and {@link #mergeWith(ImprintRecord)} are - * intermediate and do not create new records. They build up a plan of operations - * that is executed only when a terminal operation like {@link #toRecord()} is called. - */ -public final class ImprintStream { - - private final Plan plan; - - private ImprintStream(Plan plan) { - this.plan = Objects.requireNonNull(plan); - } - - /** - * The internal representation of the transformation plan. - * This is a linked-list style structure where each step points to the previous one. - */ - private interface Plan { - // Marker interface for the plan steps - } - - /** - * The starting point of a plan, containing the initial source record. - */ - private static final class SourcePlan implements Plan { - final ImprintRecord source; - - private SourcePlan(ImprintRecord source) { - this.source = Objects.requireNonNull(source, "Source record cannot be null."); - } - } - - /** - * A plan step representing a 'project' operation. - */ - private static final class ProjectPlan implements Plan { - final Plan previous; - final Set fieldIds; - - private ProjectPlan(Plan previous, int... fieldIds) { - this.previous = Objects.requireNonNull(previous); - this.fieldIds = new HashSet<>(); - for (int id : fieldIds) { - this.fieldIds.add(id); - } - } - } - - /** - * A plan step representing a 'merge' operation. - */ - private static final class MergePlan implements Plan { - final Plan previous; - final List others; - - private MergePlan(Plan previous, List others) { - this.previous = Objects.requireNonNull(previous); - this.others = Objects.requireNonNull(others); - } - } - - // ========== PUBLIC API ========== - - /** - * Creates a new transformation stream starting with a source record. - * - * @param source The initial record for the transformation. - * @return A new ImprintStream. - */ - public static ImprintStream of(ImprintRecord source) { - return new ImprintStream(new SourcePlan(source)); - } - - /** - * An intermediate operation that defines a projection on the stream. - * This is a lazy operation; the projection is only performed when a terminal - * operation is called. - * - * @param fieldIds The field IDs to keep in the final record. - * @return A new ImprintStream with the projection step added to its plan. - */ - public ImprintStream project(int... fieldIds) { - return new ImprintStream(new ProjectPlan(this.plan, fieldIds)); - } - - /** - * An intermediate operation that defines a merge on the stream. - * The record from this stream (the "left" side) takes precedence in case - * of overlapping field IDs. - *

- * This is a lazy operation; the merge is only performed when a terminal - * operation is called. - * - * @param other The record to merge with this stream's record. - * @return A new ImprintStream with the merge step added to its plan. - */ - public ImprintStream mergeWith(ImprintRecord other) { - return new ImprintStream(new MergePlan(this.plan, Collections.singletonList(other))); - } - - /** - * A terminal operation that executes the defined transformation plan and - * constructs a new, consolidated ImprintRecord. - * - * @return a new ImprintRecord representing the result of the stream operations. - */ - public ImprintRecord toRecord() { - return new Evaluator(this.plan).execute(); - } - - // ========== EVALUATOR ========== - - /** - * The engine that walks the plan and executes the transformation. - */ - private static final class Evaluator { - private final Plan plan; - - private Evaluator(Plan plan) { - this.plan = plan; - } - - public ImprintRecord execute() { - // Unwind the plan from a deque - var planQueue = getPlans(); - - // Set of fields being built - var resolvedFields = new TreeMap(); - - for (var planStep : planQueue) { - if (planStep instanceof SourcePlan) { - var sourcePlan = (SourcePlan) planStep; - for (var entry : sourcePlan.source.getDirectory()) { - resolvedFields.put((int) entry.getId(), new FieldSource(sourcePlan.source, entry)); - } - } else if (planStep instanceof ProjectPlan) { - var projectPlan = (ProjectPlan) planStep; - // Apply projection to the current state of resolved fields. - // Keep only fields that are in the projection set - resolvedFields.keySet().removeIf(fieldId -> !projectPlan.fieldIds.contains(fieldId)); - } else if (planStep instanceof MergePlan) { - var mergePlan = (MergePlan) planStep; - // Add fields from other records if they aren't already in the map. - for (var otherRecord : mergePlan.others) { - for (var entry : otherRecord.getDirectory()) { - int fieldId = entry.getId(); - resolvedFields.putIfAbsent(fieldId, new FieldSource(otherRecord, entry)); - } - } - } - } - return build(resolvedFields); - } - - private Deque getPlans() { - var planQueue = new ArrayDeque(); - var current = plan; - while (current != null) { - planQueue.addFirst(current); - if (current instanceof ProjectPlan) { - current = ((ProjectPlan) current).previous; - } else if (current instanceof MergePlan) { - current = ((MergePlan) current).previous; - } else if (current instanceof SourcePlan) { - current = null; // End of the chain - } - } - return planQueue; - } - - private ImprintRecord build(NavigableMap finalFields) { - if (finalFields.isEmpty()) { - // TODO: Need a way to get the schemaId for an empty record. - // For now, returning null or using a default. - try { - return ImprintRecord.builder(new SchemaId(0, 0)).build(); - } catch (ImprintException e) { - // TODO This shouldn't really ever happen, we probably need a better way of consolidating error handling - throw new IllegalStateException("Failed to build empty record.", e); - } - } - - // Use schema from the first field's source record. - var schemaId = finalFields.firstEntry().getValue().record.getHeader().getSchemaId(); - - // 1. Calculate final payload size and prepare directory. - int payloadSize = 0; - var newDirectoryMap = new TreeMap(); - - for (var entry : finalFields.entrySet()) { - int fieldId = entry.getKey(); - var fieldSource = entry.getValue(); - int fieldLength = fieldSource.getLength(); - - newDirectoryMap.put(fieldId, new Directory.Entry(fieldSource.entry.getId(), fieldSource.entry.getTypeCode(), payloadSize)); - payloadSize += fieldLength; - } - - // 2. Allocate buffer and copy data. - var payload = ByteBuffer.allocate(payloadSize).order(ByteOrder.LITTLE_ENDIAN); - for (var fieldSource : finalFields.values()) { - try { - var sourceData = fieldSource.record.getRawBytes(fieldSource.entry.getId()); - if (sourceData != null) - payload.put(sourceData.duplicate()); - } catch (Exception e) { - // Shouldn't happen in normal operation - maybe some sort of data corruption or race issue - throw new IllegalStateException("Failed to copy data for field " + fieldSource.entry.getId(), e); - } - } - payload.flip(); - - // 3. Construct the final record. - var newHeader = new Header(new Flags((byte) 0), schemaId, payload.remaining()); - return new ImprintRecord(newHeader, newDirectoryMap, payload.asReadOnlyBuffer()); - } - - /** - * A lightweight struct to track the source of a field during evaluation. - */ - private static final class FieldSource { - final ImprintRecord record; - final Directory entry; - - FieldSource(ImprintRecord record, Directory entry) { - this.record = record; - this.entry = entry; - } - - int getLength() { - try { - var buf = record.getRawBytes(entry.getId()); - return buf != null ? buf.remaining() : 0; - } catch (Exception e) { - return 0; - } - } - } - } -} \ No newline at end of file diff --git a/src/test/java/com/imprint/IntegrationTest.java b/src/test/java/com/imprint/IntegrationTest.java index ee1d426..e066f01 100644 --- a/src/test/java/com/imprint/IntegrationTest.java +++ b/src/test/java/com/imprint/IntegrationTest.java @@ -623,4 +623,246 @@ void testTypeGetterRow() throws ImprintException { assertEquals(999L, retrievedRow.getInt64(102)); assertEquals("outer field", deserializedWithRow.getString(202)); } + + @Test + @DisplayName("Boundary Values: Numeric limits and special floating point values") + void testNumericBoundaryValues() throws ImprintException { + var schemaId = new SchemaId(60, 0xB0DA12); + var record = ImprintRecord.builder(schemaId) + .field(1, Integer.MAX_VALUE) + .field(2, Integer.MIN_VALUE) + .field(3, Long.MAX_VALUE) + .field(4, Long.MIN_VALUE) + .field(5, Float.MAX_VALUE) + .field(6, Float.MIN_VALUE) + .field(7, Float.NaN) + .field(8, Float.POSITIVE_INFINITY) + .field(9, Float.NEGATIVE_INFINITY) + .field(10, Double.MAX_VALUE) + .field(11, Double.MIN_VALUE) + .field(12, Double.NaN) + .field(13, Double.POSITIVE_INFINITY) + .field(14, Double.NEGATIVE_INFINITY) + .field(15, -0.0f) + .field(16, -0.0) + .build(); + + var deserialized = serializeAndDeserialize(record); + + assertEquals(Integer.MAX_VALUE, deserialized.getInt32(1)); + assertEquals(Integer.MIN_VALUE, deserialized.getInt32(2)); + assertEquals(Long.MAX_VALUE, deserialized.getInt64(3)); + assertEquals(Long.MIN_VALUE, deserialized.getInt64(4)); + assertEquals(Float.MAX_VALUE, deserialized.getFloat32(5)); + assertEquals(Float.MIN_VALUE, deserialized.getFloat32(6)); + assertTrue(Float.isNaN(deserialized.getFloat32(7))); + assertTrue(Float.isInfinite(deserialized.getFloat32(8)) && deserialized.getFloat32(8) > 0); + assertTrue(Float.isInfinite(deserialized.getFloat32(9)) && deserialized.getFloat32(9) < 0); + assertEquals(Double.MAX_VALUE, deserialized.getFloat64(10)); + assertEquals(Double.MIN_VALUE, deserialized.getFloat64(11)); + assertTrue(Double.isNaN(deserialized.getFloat64(12))); + assertTrue(Double.isInfinite(deserialized.getFloat64(13)) && deserialized.getFloat64(13) > 0); + assertTrue(Double.isInfinite(deserialized.getFloat64(14)) && deserialized.getFloat64(14) < 0); + assertEquals(-0.0f, deserialized.getFloat32(15)); + assertEquals(-0.0, deserialized.getFloat64(16)); + } + + @Test + @DisplayName("Unicode and Special Strings: International character support") + void testUnicodeAndSpecialStrings() throws ImprintException { + var schemaId = new SchemaId(61, 0x04100DE); + var record = ImprintRecord.builder(schemaId) + .field(1, "") // Empty string + .field(2, " ") // Single space + .field(3, "\n\t\r") // Whitespace characters + .field(4, "Hello, 世界! 🌍🚀") // Unicode: CJK + Emoji + .field(5, "مرحبا بالعالم") // Arabic (RTL) + .field(6, "Здравствуй мир") // Cyrillic + .field(7, "こんにちは世界") // Japanese + .field(8, "\u0000\u0001\u001F") // Control characters + .field(9, "A".repeat(10000)) // Large string + .build(); + + var deserialized = serializeAndDeserialize(record); + + assertEquals("", deserialized.getString(1)); + assertEquals(" ", deserialized.getString(2)); + assertEquals("\n\t\r", deserialized.getString(3)); + assertEquals("Hello, 世界! 🌍🚀", deserialized.getString(4)); + assertEquals("مرحبا بالعالم", deserialized.getString(5)); + assertEquals("Здравствуй мир", deserialized.getString(6)); + assertEquals("こんにちは世界", deserialized.getString(7)); + assertEquals("\u0000\u0001\u001F", deserialized.getString(8)); + assertEquals("A".repeat(10000), deserialized.getString(9)); + } + + @Test + @DisplayName("Deep Nesting: Multiple levels of nested records") + void testDeepNesting() throws ImprintException { + // Create 5 levels of nesting + var level5 = ImprintRecord.builder(new SchemaId(65, 5)) + .field(1, "deepest level") + .build(); + + var level4 = ImprintRecord.builder(new SchemaId(64, 4)) + .field(1, level5) + .field(2, "level 4") + .build(); + + var level3 = ImprintRecord.builder(new SchemaId(63, 3)) + .field(1, level4) + .field(2, "level 3") + .build(); + + var level2 = ImprintRecord.builder(new SchemaId(62, 2)) + .field(1, level3) + .field(2, "level 2") + .build(); + + var level1 = ImprintRecord.builder(new SchemaId(61, 1)) + .field(1, level2) + .field(2, "level 1") + .build(); + + var deserialized = serializeAndDeserialize(level1); + + // Navigate through all levels + assertEquals("level 1", deserialized.getString(2)); + var l2 = deserialized.getRow(1); + assertEquals("level 2", l2.getString(2)); + var l3 = l2.getRow(1); + assertEquals("level 3", l3.getString(2)); + var l4 = l3.getRow(1); + assertEquals("level 4", l4.getString(2)); + var l5 = l4.getRow(1); + assertEquals("deepest level", l5.getString(1)); + } + + @Test + @DisplayName("Map Key Types: All supported map key types") + void testMapKeyTypeVariations() throws ImprintException { + var schemaId = new SchemaId(70, 0xAAB5E75); + + // Create maps with different key types + var stringKeyMap = new HashMap(); + stringKeyMap.put(MapKey.fromString("string_key"), Value.fromString("string_value")); + + var intKeyMap = new HashMap(); + intKeyMap.put(MapKey.fromInt32(42), Value.fromString("int_value")); + + var longKeyMap = new HashMap(); + longKeyMap.put(MapKey.fromInt64(9876543210L), Value.fromString("long_value")); + + var bytesKeyMap = new HashMap(); + bytesKeyMap.put(MapKey.fromBytes(new byte[]{1, 2, 3}), Value.fromString("bytes_value")); + + var record = ImprintRecord.builder(schemaId) + .field(1, Value.fromMap(stringKeyMap)) + .field(2, Value.fromMap(intKeyMap)) + .field(3, Value.fromMap(longKeyMap)) + .field(4, Value.fromMap(bytesKeyMap)) + .build(); + + var deserialized = serializeAndDeserialize(record); + + // Verify all map key types work correctly + assertEquals(Value.fromString("string_value"), + deserialized.getMap(1).get(MapKey.fromString("string_key"))); + assertEquals(Value.fromString("int_value"), + deserialized.getMap(2).get(MapKey.fromInt32(42))); + assertEquals(Value.fromString("long_value"), + deserialized.getMap(3).get(MapKey.fromInt64(9876543210L))); + assertEquals(Value.fromString("bytes_value"), + deserialized.getMap(4).get(MapKey.fromBytes(new byte[]{1, 2, 3}))); + } + + @Test + @DisplayName("Large Data: Memory efficiency with large payloads") + void testLargeDataHandling() throws ImprintException { + var schemaId = new SchemaId(80, 0xB16DA7A); + + // Create large byte arrays + byte[] largeBytes1 = new byte[100_000]; // 100KB + byte[] largeBytes2 = new byte[500_000]; // 500KB + Arrays.fill(largeBytes1, (byte) 0xAA); + Arrays.fill(largeBytes2, (byte) 0xBB); + + // Create large string + String largeString = "Large data test: " + "X".repeat(50_000); + + var record = ImprintRecord.builder(schemaId) + .field(1, largeBytes1) + .field(2, largeBytes2) + .field(3, largeString) + .field(4, "small field") + .build(); + + // Verify large record can be serialized and deserialized + var deserialized = serializeAndDeserialize(record); + + assertArrayEquals(largeBytes1, deserialized.getBytes(1)); + assertArrayEquals(largeBytes2, deserialized.getBytes(2)); + assertEquals(largeString, deserialized.getString(3)); + assertEquals("small field", deserialized.getString(4)); + + // Test projection still works with large data + var projected = record.project(4); + assertEquals(1, projected.getDirectory().size()); + assertEquals("small field", projected.getString(4)); + + // Verify original large data is excluded from projection + assertTrue(projected.getSerializedSize() < record.getSerializedSize() / 10); + } + + @Test + @DisplayName("Error Handling: Empty data detection") + void testEmptyDataHandling() { + // Empty data should throw exception + assertThrows(Exception.class, () -> ImprintRecord.deserialize(new byte[0])); + + // Null data should throw exception + assertThrows(Exception.class, () -> ImprintRecord.deserialize((byte[]) null)); + } + + @Test + @DisplayName("Complex Operations: Bytes-to-bytes vs object operations equivalence") + void testBytesToBytesEquivalence() throws ImprintException { + var schemaId = new SchemaId(100, 0xB17E5); + + var record1 = ImprintRecord.builder(schemaId) + .field(1, "record1 field1") + .field(3, 100) + .field(5, true) + .build(); + + var record2 = ImprintRecord.builder(schemaId) + .field(2, "record2 field2") + .field(4, 200L) + .field(6, 3.14) + .build(); + + // Test merge equivalence + var objectMerged = record1.merge(record2); + var bytesMerged = com.imprint.ops.ImprintOperations.mergeBytes( + record1.serializeToBuffer(), + record2.serializeToBuffer() + ); + var bytesMergedRecord = ImprintRecord.deserialize(bytesMerged); + + assertEquals(objectMerged.getDirectory().size(), bytesMergedRecord.getDirectory().size()); + assertEquals(objectMerged.getString(1), bytesMergedRecord.getString(1)); + assertEquals(objectMerged.getString(2), bytesMergedRecord.getString(2)); + assertEquals(objectMerged.getInt32(3), bytesMergedRecord.getInt32(3)); + + // Test project equivalence + var objectProjected = record1.project(1, 3); + var bytesProjected = com.imprint.ops.ImprintOperations.projectBytes( + record1.serializeToBuffer(), 1, 3 + ); + var bytesProjectedRecord = ImprintRecord.deserialize(bytesProjected); + + assertEquals(objectProjected.getDirectory().size(), bytesProjectedRecord.getDirectory().size()); + assertEquals(objectProjected.getString(1), bytesProjectedRecord.getString(1)); + assertEquals(objectProjected.getInt32(3), bytesProjectedRecord.getInt32(3)); + } } \ No newline at end of file diff --git a/src/test/java/com/imprint/core/ImprintRecordTest.java b/src/test/java/com/imprint/core/ImprintRecordTest.java index 6d85ccb..562f5fd 100644 --- a/src/test/java/com/imprint/core/ImprintRecordTest.java +++ b/src/test/java/com/imprint/core/ImprintRecordTest.java @@ -1,227 +1,289 @@ package com.imprint.core; import com.imprint.error.ImprintException; -import com.imprint.error.ErrorType; -import com.imprint.types.Value; -import com.imprint.types.MapKey; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; -import java.util.*; -import static org.assertj.core.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.*; + +@DisplayName("ImprintRecord") class ImprintRecordTest { - - // Helper method to extract string value from either StringValue or StringBufferValue - private String getStringValue(Value value) { - if (value instanceof Value.StringValue) { - return ((Value.StringValue) value).getValue(); - } else if (value instanceof Value.StringBufferValue) { - return ((Value.StringBufferValue) value).getValue(); - } else { - throw new IllegalArgumentException("Expected string value, got: " + value.getClass()); - } - } - - @Test - void shouldCreateSimpleRecord() throws ImprintException { - var schemaId = new SchemaId(1, 0xdeadbeef); - var record = ImprintRecord.builder(schemaId) - .field(1, Value.fromInt32(42)) - .field(2, Value.fromString("hello")) - .build(); - - assertThat(record.getHeader().getSchemaId()).isEqualTo(schemaId); - assertThat(record.getDirectory()).hasSize(2); - - Value field1 = record.getValue(1); - Value field2 = record.getValue(2); - - assertThat(field1).isNotNull(); - assertThat(field1).isInstanceOf(Value.Int32Value.class); - assertThat(((Value.Int32Value) field1).getValue()).isEqualTo(42); - - assertThat(field2).isNotNull(); - assertThat(field2.getTypeCode()).isEqualTo(com.imprint.types.TypeCode.STRING); - String stringValue = getStringValue(field2); - assertThat(stringValue).isEqualTo("hello"); - - // Non-existent field should return null - assertThat(record.getValue(999)).isNull(); + + private SchemaId testSchema; + private ImprintRecord testRecord; + private ImprintRecord serializedRecord; + + @BeforeEach + void setUp() throws ImprintException { + testSchema = new SchemaId(1, 0x12345678); + testRecord = ImprintRecord.builder(testSchema) + .field(1, 42) + .field(2, "hello") + .field(3, true) + .field(4, 3.14159) + .field(5, new byte[]{1, 2, 3, 4, 5}) + .build(); + serializedRecord = testRecord; } - - @Test - void shouldRoundtripThroughSerialization() throws ImprintException { - var schemaId = new SchemaId(1, 0xdeadbeef); - var original = ImprintRecord.builder(schemaId) - .field(1, Value.nullValue()) - .field(2, Value.fromBoolean(true)) - .field(3, Value.fromInt32(42)) - .field(4, Value.fromInt64(123456789L)) - .field(5, Value.fromFloat32(3.14f)) - .field(6, Value.fromFloat64(2.718281828)) - .field(7, Value.fromBytes(new byte[]{1, 2, 3, 4})) - .field(8, Value.fromString("test string")) - .build(); - - // Serialize and deserialize - var buffer = original.serializeToBuffer(); - byte[] serialized = new byte[buffer.remaining()]; - buffer.get(serialized); - var deserialized = ImprintRecord.deserialize(serialized); - - // Verify metadata - assertThat(deserialized.getHeader().getSchemaId().getFieldSpaceId()).isEqualTo(1); - assertThat(deserialized.getHeader().getSchemaId().getSchemaHash()).isEqualTo(0xdeadbeef); - assertThat(deserialized.getDirectory()).hasSize(8); - - // Verify all values - assertThat(deserialized.getValue(1)).isEqualTo(Value.nullValue()); - assertThat(deserialized.getValue(2)).isEqualTo(Value.fromBoolean(true)); - assertThat(deserialized.getValue(3)).isEqualTo(Value.fromInt32(42)); - assertThat(deserialized.getValue(4)).isEqualTo(Value.fromInt64(123456789L)); - assertThat(deserialized.getValue(5)).isEqualTo(Value.fromFloat32(3.14f)); - assertThat(deserialized.getValue(6)).isEqualTo(Value.fromFloat64(2.718281828)); - assertThat(deserialized.getValue(7)).isEqualTo(Value.fromBytes(new byte[]{1, 2, 3, 4})); - assertThat(deserialized.getValue(8)).isEqualTo(Value.fromString("test string")); - - // Non-existent field - assertThat(deserialized.getValue(999)).isNull(); + + @Nested + @DisplayName("Creation") + class Creation { + + @Test + @DisplayName("should create from ImprintRecord") + void shouldCreateFromImprintRecord() { + var serialized = testRecord; + + assertNotNull(serialized); + assertEquals(testRecord.getDirectory().size(), serialized.getFieldCount()); + assertEquals(testSchema, serialized.getSchemaId()); + } + + @Test + @DisplayName("should create from serialized bytes") + void shouldCreateFromSerializedBytes() throws ImprintException { + var bytes = testRecord.serializeToBuffer(); + var serialized = ImprintRecord.fromBytes(bytes); + + assertNotNull(serialized); + assertEquals(testRecord.getDirectory().size(), serialized.getFieldCount()); + assertEquals(testSchema, serialized.getSchemaId()); + } + + @Test + @DisplayName("should reject null bytes") + void shouldRejectNullBytes() { + assertThrows(NullPointerException.class, () -> ImprintRecord.fromBytes(null)); + } } - - @Test - void shouldHandleArrays() throws ImprintException { - var schemaId = new SchemaId(1, 0xdeadbeef); - - List intArray = Arrays.asList( - Value.fromInt32(1), - Value.fromInt32(2), - Value.fromInt32(3) - ); - - var record = ImprintRecord.builder(schemaId) - .field(1, Value.fromArray(intArray)) - .build(); - - // Serialize and deserialize - var buffer = record.serializeToBuffer(); - byte[] serialized = new byte[buffer.remaining()]; - buffer.get(serialized); - var deserialized = ImprintRecord.deserialize(serialized); - - Value arrayValue = deserialized.getValue(1); - assertThat(arrayValue).isNotNull(); - assertThat(arrayValue).isInstanceOf(Value.ArrayValue.class); - - List deserializedArray = ((Value.ArrayValue) arrayValue).getValue(); - assertThat(deserializedArray).hasSize(3); - assertThat(deserializedArray.get(0)).isEqualTo(Value.fromInt32(1)); - assertThat(deserializedArray.get(1)).isEqualTo(Value.fromInt32(2)); - assertThat(deserializedArray.get(2)).isEqualTo(Value.fromInt32(3)); + + @Nested + @DisplayName("Field Access") + class FieldAccess { + + @Test + @DisplayName("should access fields with correct types") + void shouldAccessFieldsWithCorrectTypes() throws ImprintException { + assertEquals(Integer.valueOf(42), serializedRecord.getInt32(1)); + assertEquals("hello", serializedRecord.getString(2)); + assertEquals(Boolean.TRUE, serializedRecord.getBoolean(3)); + assertEquals(Double.valueOf(3.14159), serializedRecord.getFloat64(4)); + assertArrayEquals(new byte[]{1, 2, 3, 4, 5}, serializedRecord.getBytes(5)); + } + + @Test + @DisplayName("should handle non-existent fields correctly") + void shouldHandleNonExistentFields() throws ImprintException { + // getValue should return null for non-existent fields + assertNull(serializedRecord.getValue(99)); + + // Typed getters should throw exceptions for non-existent fields + assertThrows(ImprintException.class, () -> serializedRecord.getString(99)); + assertThrows(ImprintException.class, () -> serializedRecord.getInt32(100)); + + // hasField should return false + assertFalse(serializedRecord.hasField(99)); + } + + @Test + @DisplayName("should check field existence efficiently") + void shouldCheckFieldExistenceEfficiently() { + assertTrue(serializedRecord.hasField(1)); + assertTrue(serializedRecord.hasField(2)); + assertTrue(serializedRecord.hasField(3)); + assertFalse(serializedRecord.hasField(99)); + } + + @Test + @DisplayName("should return correct field count") + void shouldReturnCorrectFieldCount() { + assertEquals(5, serializedRecord.getFieldCount()); + } } - - @Test - void shouldHandleMaps() throws ImprintException { - var schemaId = new SchemaId(1, 0xdeadbeef); - - var map = new HashMap(); - map.put(MapKey.fromString("key1"), Value.fromInt32(1)); - map.put(MapKey.fromString("key2"), Value.fromInt32(2)); - - var record = ImprintRecord.builder(schemaId) - .field(1, Value.fromMap(map)) - .build(); - - // Serialize and deserialize - var buffer = record.serializeToBuffer(); - byte[] serialized = new byte[buffer.remaining()]; - buffer.get(serialized); - var deserialized = ImprintRecord.deserialize(serialized); - - Value mapValue = deserialized.getValue(1); - assertThat(mapValue).isNotNull(); - assertThat(mapValue).isInstanceOf(Value.MapValue.class); - - Map deserializedMap = ((Value.MapValue) mapValue).getValue(); - assertThat(deserializedMap).hasSize(2); - assertThat(deserializedMap.get(MapKey.fromString("key1"))).isEqualTo(Value.fromInt32(1)); - assertThat(deserializedMap.get(MapKey.fromString("key2"))).isEqualTo(Value.fromInt32(2)); + + @Nested + @DisplayName("Zero-Copy Operations") + class ZeroCopyOperations { + + @Test + @DisplayName("should merge with another ImprintRecord") + void shouldMergeWithAnotherImprintRecord() throws ImprintException { + // Create another record + var otherRecord = ImprintRecord.builder(testSchema) + .field(6, "additional") + .field(7, 999L) + .build(); + + // Merge + var merged = serializedRecord.merge(otherRecord); + + // Verify merged result + assertEquals(7, merged.getFieldCount()); + assertEquals(Integer.valueOf(42), merged.getInt32(1)); + assertEquals("hello", merged.getString(2)); + assertEquals("additional", merged.getString(6)); + assertEquals(Long.valueOf(999L), merged.getInt64(7)); + } + + @Test + @DisplayName("should project subset of fields") + void shouldProjectSubsetOfFields() throws ImprintException { + var projected = serializedRecord.project(1, 3, 5); + + assertEquals(3, projected.getFieldCount()); + assertEquals(Integer.valueOf(42), projected.getInt32(1)); + assertEquals(Boolean.TRUE, projected.getBoolean(3)); + assertArrayEquals(new byte[]{1, 2, 3, 4, 5}, projected.getBytes(5)); + + // Should not have other fields + assertFalse(projected.hasField(2)); + assertFalse(projected.hasField(4)); + } + + @Test + @DisplayName("should chain project and merge operations") + void shouldChainProjectAndMergeOperations() throws ImprintException { + // Create another record + var otherSerialized = ImprintRecord.builder(testSchema) + .field(10, "chained") + .build(); + + // Chain operations: project this record, then merge with other + var result = serializedRecord.projectAndMerge(otherSerialized, 1, 2); + + // Should have projected fields plus other record + assertEquals(3, result.getFieldCount()); + assertEquals(Integer.valueOf(42), result.getInt32(1)); + assertEquals("hello", result.getString(2)); + assertEquals("chained", result.getString(10)); + + // Should not have non-projected fields + assertFalse(result.hasField(3)); + assertFalse(result.hasField(4)); + assertFalse(result.hasField(5)); + } } - - @Test - void shouldHandleNestedRecords() throws ImprintException { - // Create inner record - var innerSchemaId = new SchemaId(2, 0xcafebabe); - var innerRecord = ImprintRecord.builder(innerSchemaId) - .field(1, Value.fromInt32(42)) - .field(2, Value.fromString("nested")) - .build(); - - // Create outer record containing inner record - var outerSchemaId = new SchemaId(1, 0xdeadbeef); - var outerRecord = ImprintRecord.builder(outerSchemaId) - .field(1, Value.fromRow(innerRecord)) - .field(2, Value.fromInt64(123L)) - .build(); - - // Serialize and deserialize - var buffer = outerRecord.serializeToBuffer(); - byte[] serialized = new byte[buffer.remaining()]; - buffer.get(serialized); - var deserialized = ImprintRecord.deserialize(serialized); - - // Verify outer record metadata - assertThat(deserialized.getHeader().getSchemaId().getFieldSpaceId()).isEqualTo(1); - assertThat(deserialized.getHeader().getSchemaId().getSchemaHash()).isEqualTo(0xdeadbeef); - - // Verify nested record - Value rowValue = deserialized.getValue(1); - assertThat(rowValue).isNotNull(); - assertThat(rowValue).isInstanceOf(Value.RowValue.class); - - var nestedRecord = ((Value.RowValue) rowValue).getValue(); - assertThat(nestedRecord.getHeader().getSchemaId().getFieldSpaceId()).isEqualTo(2); - assertThat(nestedRecord.getHeader().getSchemaId().getSchemaHash()).isEqualTo(0xcafebabe); - - assertThat(nestedRecord.getValue(1)).isEqualTo(Value.fromInt32(42)); - assertThat(nestedRecord.getValue(2)).isEqualTo(Value.fromString("nested")); - - // Verify outer record field - assertThat(deserialized.getValue(2)).isEqualTo(Value.fromInt64(123L)); + + @Nested + @DisplayName("Conversion") + class Conversion { + + @Test + @DisplayName("should serialize and deserialize consistently") + void shouldSerializeAndDeserializeConsistently() throws ImprintException { + var serializedBytes = serializedRecord.serializeToBuffer(); + var deserialized = ImprintRecord.fromBytes(serializedBytes); + + assertEquals(testRecord.getDirectory().size(), deserialized.getDirectory().size()); + assertEquals(testRecord.getInt32(1), deserialized.getInt32(1)); + assertEquals(testRecord.getString(2), deserialized.getString(2)); + assertEquals(testRecord.getBoolean(3), deserialized.getBoolean(3)); + } + + @Test + @DisplayName("should preserve serialized bytes") + void shouldPreserveSerializedBytes() { + var originalBytes = testRecord.serializeToBuffer(); + var preservedBytes = serializedRecord.getSerializedBytes(); + + assertEquals(originalBytes.remaining(), preservedBytes.remaining()); + + // Compare byte content + var original = originalBytes.duplicate(); + var preserved = preservedBytes.duplicate(); + + while (original.hasRemaining() && preserved.hasRemaining()) { + assertEquals(original.get(), preserved.get()); + } + } } - - @Test - void shouldRejectInvalidMagic() { - byte[] invalidData = new byte[15]; - invalidData[0] = 0x00; // wrong magic - - assertThatThrownBy(() -> ImprintRecord.deserialize(invalidData)) - .isInstanceOf(ImprintException.class) - .extracting("errorType") - .isEqualTo(ErrorType.INVALID_MAGIC); + + @Nested + @DisplayName("Performance Characteristics") + class PerformanceCharacteristics { + + @Test + @DisplayName("should have minimal memory footprint") + void shouldHaveMinimalMemoryFootprint() { + var originalSize = testRecord.serializeToBuffer().remaining(); + var serializedSize = serializedRecord.getSerializedSize(); + + assertEquals(originalSize, serializedSize); + + // ImprintRecord should not significantly increase memory usage + // (just the wrapper object itself) + assertTrue(serializedSize > 0); + } + + @Test + @DisplayName("should support repeated operations efficiently") + void shouldSupportRepeatedOperationsEfficiently() throws ImprintException { + // Multiple field access should not cause performance degradation + for (int i = 0; i < 100; i++) { + assertEquals(Integer.valueOf(42), serializedRecord.getInt32(1)); + assertEquals("hello", serializedRecord.getString(2)); + assertTrue(serializedRecord.hasField(3)); + } + } } - - @Test - void shouldRejectUnsupportedVersion() { - byte[] invalidData = new byte[15]; - invalidData[0] = (byte) 0x49; // correct magic - invalidData[1] = (byte) 0xFF; // wrong version - - assertThatThrownBy(() -> ImprintRecord.deserialize(invalidData)) - .isInstanceOf(ImprintException.class) - .extracting("errorType") - .isEqualTo(ErrorType.UNSUPPORTED_VERSION); + + @Nested + @DisplayName("Edge Cases") + class EdgeCases { + + @Test + @DisplayName("should handle empty projection") + void shouldHandleEmptyProjection() throws ImprintException { + var projected = serializedRecord.project(); + assertEquals(0, projected.getFieldCount()); + } + + @Test + @DisplayName("should handle projection with non-existent fields") + void shouldHandleProjectionWithNonExistentFields() throws ImprintException { + var projected = serializedRecord.project(1, 99, 100); + assertEquals(1, projected.getFieldCount()); + assertEquals(Integer.valueOf(42), projected.getInt32(1)); + assertFalse(projected.hasField(99)); + assertFalse(projected.hasField(100)); + } + + @Test + @DisplayName("should handle merge with empty record") + void shouldHandleMergeWithEmptyRecord() throws ImprintException { + var emptySerialized = ImprintRecord.builder(testSchema).build(); + + var merged = serializedRecord.merge(emptySerialized); + assertEquals(serializedRecord.getFieldCount(), merged.getFieldCount()); + assertEquals(Integer.valueOf(42), merged.getInt32(1)); + } } - - @Test - void shouldHandleDuplicateFieldIds() throws ImprintException { - var schemaId = new SchemaId(1, 0xdeadbeef); - - // Add duplicate field IDs - last one should win - var record = ImprintRecord.builder(schemaId) - .field(1, Value.fromInt32(42)) - .field(1, Value.fromInt32(43)) - .build(); - - assertThat(record.getDirectory()).hasSize(1); - assertThat(record.getValue(1)).isEqualTo(Value.fromInt32(43)); + + @Nested + @DisplayName("Equality and Hashing") + class EqualityAndHashing { + + @Test + @DisplayName("should be equal for same serialized data") + void shouldBeEqualForSameSerializedData() { + var other = testRecord; + + assertEquals(serializedRecord, other); + assertEquals(serializedRecord.hashCode(), other.hashCode()); + } + + @Test + @DisplayName("should not be equal for different data") + void shouldNotBeEqualForDifferentData() throws ImprintException { + // Different value + var differentSerialized = ImprintRecord.builder(testSchema) + .field(1, 999) // Different value + .build(); + + assertNotEquals(serializedRecord, differentSerialized); + } } } \ No newline at end of file diff --git a/src/test/java/com/imprint/ops/ImprintOperationsTest.java b/src/test/java/com/imprint/ops/ImprintOperationsTest.java index 7b54800..292f8f3 100644 --- a/src/test/java/com/imprint/ops/ImprintOperationsTest.java +++ b/src/test/java/com/imprint/ops/ImprintOperationsTest.java @@ -10,6 +10,7 @@ import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; +import java.nio.ByteBuffer; import java.util.List; import static org.junit.jupiter.api.Assertions.*; @@ -49,7 +50,7 @@ class ProjectOperations { @DisplayName("should project subset of fields") void shouldProjectSubsetOfFields() throws ImprintException { // When projecting a subset of fields - ImprintRecord projected = ImprintOperations.project(multiFieldRecord, 1, 5); + ImprintRecord projected = multiFieldRecord.project(1, 5); // Then only the requested fields should be present assertEquals(2, projected.getDirectory().size()); @@ -65,7 +66,7 @@ void shouldProjectSubsetOfFields() throws ImprintException { @DisplayName("should maintain field order regardless of input order") void shouldMaintainFieldOrderRegardlessOfInputOrder() throws ImprintException { // When projecting fields in arbitrary order - ImprintRecord projected = ImprintOperations.project(multiFieldRecord, 7, 1, 5, 3); + ImprintRecord projected = multiFieldRecord.project(7, 1, 5, 3); // Then all requested fields should be present assertEquals(4, projected.getDirectory().size()); @@ -86,7 +87,7 @@ void shouldMaintainFieldOrderRegardlessOfInputOrder() throws ImprintException { @DisplayName("should handle single field projection") void shouldHandleSingleFieldProjection() throws ImprintException { // When projecting a single field - ImprintRecord projected = ImprintOperations.project(multiFieldRecord, 3); + ImprintRecord projected = multiFieldRecord.project(3); // Then only that field should be present assertEquals(1, projected.getDirectory().size()); @@ -102,7 +103,7 @@ void shouldPreserveAllFieldsWhenProjectingAll() throws ImprintException { .toArray(); // When projecting all fields - ImprintRecord projected = ImprintOperations.project(multiFieldRecord, allFields); + ImprintRecord projected = multiFieldRecord.project(allFields); // Then all fields should be present with matching values assertEquals(multiFieldRecord.getDirectory().size(), projected.getDirectory().size()); @@ -117,20 +118,20 @@ void shouldPreserveAllFieldsWhenProjectingAll() throws ImprintException { @Test @DisplayName("should handle empty projection") - void shouldHandleEmptyProjection() { + void shouldHandleEmptyProjection() throws ImprintException { // When projecting no fields - ImprintRecord projected = ImprintOperations.project(multiFieldRecord); + ImprintRecord projected = multiFieldRecord.project(); // Then result should be empty but valid assertEquals(0, projected.getDirectory().size()); - assertEquals(0, projected.getBuffers().getPayload().remaining()); + assertEquals(0, projected.getFieldCount()); } @Test @DisplayName("should ignore nonexistent fields") void shouldIgnoreNonexistentFields() throws ImprintException { // When projecting mix of existing and non-existing fields - ImprintRecord projected = ImprintOperations.project(multiFieldRecord, 1, 99, 100); + ImprintRecord projected = multiFieldRecord.project(1, 99, 100); // Then only existing fields should be included assertEquals(1, projected.getDirectory().size()); @@ -143,7 +144,7 @@ void shouldIgnoreNonexistentFields() throws ImprintException { @DisplayName("should deduplicate requested fields") void shouldDeduplicateRequestedFields() throws ImprintException { // When projecting the same field multiple times - ImprintRecord projected = ImprintOperations.project(multiFieldRecord, 1, 1, 1); + ImprintRecord projected = multiFieldRecord.project(1, 1, 1); // Then field should only appear once assertEquals(1, projected.getDirectory().size()); @@ -152,13 +153,13 @@ void shouldDeduplicateRequestedFields() throws ImprintException { @Test @DisplayName("should handle projection from empty record") - void shouldHandleProjectionFromEmptyRecord() { + void shouldHandleProjectionFromEmptyRecord() throws ImprintException { // When projecting any fields from empty record - ImprintRecord projected = ImprintOperations.project(emptyRecord, 1, 2, 3); + ImprintRecord projected = emptyRecord.project(1, 2, 3); // Then result should be empty but valid assertEquals(0, projected.getDirectory().size()); - assertEquals(0, projected.getBuffers().getPayload().remaining()); + assertEquals(0, projected.getFieldCount()); } @Test @@ -168,7 +169,7 @@ void shouldPreserveExactByteRepresentation() throws ImprintException { byte[] originalBytes = multiFieldRecord.getBytes(7); // When projecting that field - ImprintRecord projected = ImprintOperations.project(multiFieldRecord, 7); + ImprintRecord projected = multiFieldRecord.project(7); // Then the byte representation should be exactly preserved byte[] projectedBytes = projected.getBytes(7); @@ -187,13 +188,13 @@ void shouldReducePayloadSizeWhenProjectingSubset() throws ImprintException { .field(4, new byte[500]) // 500+ bytes .build(); - int originalPayloadSize = largeRecord.getBuffers().getPayload().remaining(); + int originalPayloadSize = largeRecord.getSerializedSize(); // When projecting only the small fields - ImprintRecord projected = ImprintOperations.project(largeRecord, 1, 3); + ImprintRecord projected = largeRecord.project(1, 3); // Then the payload size should be significantly smaller - assertTrue(projected.getBuffers().getPayload().remaining() < originalPayloadSize, + assertTrue(projected.getSerializedSize() < originalPayloadSize, "Projected payload should be smaller than original"); // And the values should still be correct @@ -221,7 +222,7 @@ void shouldMergeRecordsWithDistinctFields() throws ImprintException { .build(); // When merging the records - ImprintRecord merged = ImprintOperations.merge(record1, record2); + ImprintRecord merged = record1.merge(record2); // Then all fields should be present assertEquals(4, merged.getDirectory().size()); @@ -253,7 +254,7 @@ void shouldMergeRecordsWithOverlappingFields() throws ImprintException { .build(); // When merging the records - ImprintRecord merged = ImprintOperations.merge(record1, record2); + ImprintRecord merged = record1.merge(record2); // Then first record's values should take precedence for duplicates assertEquals(3, merged.getDirectory().size()); @@ -278,7 +279,7 @@ void shouldPreserveSchemaIdFromFirstRecord() throws ImprintException { .build(); // When merging the records - ImprintRecord merged = ImprintOperations.merge(record1, record2); + ImprintRecord merged = record1.merge(record2); // Then schema ID from first record should be preserved assertEquals(schema1, merged.getHeader().getSchemaId()); @@ -288,8 +289,8 @@ void shouldPreserveSchemaIdFromFirstRecord() throws ImprintException { @DisplayName("should handle merge with empty record") void shouldHandleMergeWithEmptyRecord() throws ImprintException { // When merging with empty record - ImprintRecord merged1 = ImprintOperations.merge(multiFieldRecord, emptyRecord); - ImprintRecord merged2 = ImprintOperations.merge(emptyRecord, multiFieldRecord); + ImprintRecord merged1 = multiFieldRecord.merge(emptyRecord); + ImprintRecord merged2 = emptyRecord.merge(multiFieldRecord); // Then results should contain all original fields assertEquals(multiFieldRecord.getDirectory().size(), merged1.getDirectory().size()); @@ -307,11 +308,11 @@ void shouldHandleMergeWithEmptyRecord() throws ImprintException { @DisplayName("should handle merge of two empty records") void shouldHandleMergeOfTwoEmptyRecords() throws ImprintException { // When merging two empty records - ImprintRecord merged = ImprintOperations.merge(emptyRecord, emptyRecord); + ImprintRecord merged = emptyRecord.merge(emptyRecord); // Then result should be empty but valid assertEquals(0, merged.getDirectory().size()); - assertEquals(0, merged.getBuffers().getPayload().remaining()); + assertEquals(0, merged.getFieldCount()); } @Test @@ -329,7 +330,7 @@ void shouldMaintainCorrectPayloadOffsetsAfterMerge() throws ImprintException { .build(); // When merging - ImprintRecord merged = ImprintOperations.merge(record1, record2); + ImprintRecord merged = record1.merge(record2); // Then all fields should be accessible with correct values assertEquals(42, merged.getInt32(1)); @@ -371,7 +372,7 @@ void shouldHandleLargeRecordsEfficiently() throws ImprintException { ImprintRecord record2 = builder2.build(); // When merging large records - ImprintRecord merged = ImprintOperations.merge(record1, record2); + ImprintRecord merged = record1.merge(record2); // Then all 200 fields should be present and accessible assertEquals(200, merged.getDirectory().size()); @@ -387,23 +388,292 @@ void shouldHandleLargeRecordsEfficiently() throws ImprintException { } @Nested - @DisplayName("Error Handling") - class ErrorHandling { + @DisplayName("Bytes-to-Bytes Operations") + class BytesToBytesOperations { @Test - @DisplayName("should handle null record gracefully") - void shouldHandleNullRecordGracefully() { - assertThrows(NullPointerException.class, () -> ImprintOperations.project(null, 1, 2, 3)); + @DisplayName("should merge bytes with same result as object merge") + void shouldMergeBytesWithSameResultAsObjectMerge() throws ImprintException { + // Given two records with distinct fields + ImprintRecord record1 = ImprintRecord.builder(testSchema) + .field(1, 42) + .field(3, "hello") + .build(); + + ImprintRecord record2 = ImprintRecord.builder(testSchema) + .field(2, true) + .field(4, 123L) + .build(); + + // When merging using both approaches + var objectMerged = record1.merge(record2); + var record1Bytes = record1.serializeToBuffer(); + var record2Bytes = record2.serializeToBuffer(); + var bytesMerged = ImprintOperations.mergeBytes(record1Bytes, record2Bytes); + + // Then results should be functionally equivalent + var deserializedBytes = ImprintRecord.deserialize(bytesMerged); + + assertEquals(objectMerged.getDirectory().size(), deserializedBytes.getDirectory().size()); + assertEquals(42, deserializedBytes.getInt32(1)); + assertTrue(deserializedBytes.getBoolean(2)); + assertEquals("hello", deserializedBytes.getString(3)); + assertEquals(123L, deserializedBytes.getInt64(4)); + } + + @Test + @DisplayName("should handle overlapping fields in byte merge") + void shouldHandleOverlappingFieldsInByteMerge() throws ImprintException { + // Given two records with overlapping fields + ImprintRecord record1 = ImprintRecord.builder(testSchema) + .field(1, "first") + .field(2, 42) + .build(); - assertThrows(NullPointerException.class, () -> ImprintOperations.merge(null, multiFieldRecord)); + ImprintRecord record2 = ImprintRecord.builder(testSchema) + .field(1, "second") // Overlapping field + .field(3, true) + .build(); - assertThrows(NullPointerException.class, () -> ImprintOperations.merge(multiFieldRecord, null)); + // When merging using bytes + var record1Bytes = record1.serializeToBuffer(); + var record2Bytes = record2.serializeToBuffer(); + var merged = ImprintOperations.mergeBytes(record1Bytes, record2Bytes); + + // Then first record's values should take precedence + var result = ImprintRecord.deserialize(merged); + assertEquals(3, result.getDirectory().size()); + assertEquals("first", result.getString(1)); // First record wins + assertEquals(42, result.getInt32(2)); + assertTrue(result.getBoolean(3)); } @Test - @DisplayName("should handle null field ids gracefully") - void shouldHandleNullFieldIdsGracefully() { - assertThrows(NullPointerException.class, () -> ImprintOperations.project(multiFieldRecord, (int[]) null)); + @DisplayName("should merge empty records correctly") + void shouldMergeEmptyRecordsCorrectly() throws ImprintException { + // Given an empty record and a non-empty record + var emptyRecord = ImprintRecord.builder(testSchema).build(); + var nonEmptyRecord = ImprintRecord.builder(testSchema) + .field(1, "test") + .build(); + + // When merging using bytes + var emptyBytes = emptyRecord.serializeToBuffer(); + var nonEmptyBytes = nonEmptyRecord.serializeToBuffer(); + + var merged1 = ImprintOperations.mergeBytes(emptyBytes, nonEmptyBytes); + var merged2 = ImprintOperations.mergeBytes(nonEmptyBytes, emptyBytes); + + // Then both should contain the non-empty record's data + var result1 = ImprintRecord.deserialize(merged1); + var result2 = ImprintRecord.deserialize(merged2); + + assertEquals(1, result1.getDirectory().size()); + assertEquals(1, result2.getDirectory().size()); + assertEquals("test", result1.getString(1)); + assertEquals("test", result2.getString(1)); + } + + @Test + @DisplayName("should project bytes with same result as object project") + void shouldProjectBytesWithSameResultAsObjectProject() throws ImprintException { + // Given a record with multiple fields + ImprintRecord record = ImprintRecord.builder(testSchema) + .field(1, 42) + .field(2, "hello") + .field(3, true) + .field(4, 123L) + .field(5, new byte[]{1, 2, 3}) + .build(); + + // When projecting using both approaches + var objectProjected = record.project(2, 4); + + var recordBytes = record.serializeToBuffer(); + var bytesProjected = ImprintOperations.projectBytes(recordBytes, 2, 4); + + // Then results should be functionally equivalent + var deserializedBytes = ImprintRecord.deserialize(bytesProjected); + + assertEquals(objectProjected.getDirectory().size(), deserializedBytes.getDirectory().size()); + assertEquals("hello", deserializedBytes.getString(2)); + assertEquals(123L, deserializedBytes.getInt64(4)); + + // Should not have the other fields + assertNull(deserializedBytes.getValue(1)); + assertNull(deserializedBytes.getValue(3)); + assertNull(deserializedBytes.getValue(5)); + } + + @Test + @DisplayName("should handle empty projection in bytes") + void shouldHandleEmptyProjectionInBytes() throws ImprintException { + // Given a record with fields + var record = ImprintRecord.builder(testSchema) + .field(1, "test") + .build(); + + // When projecting no fields + var recordBytes = record.serializeToBuffer(); + var projected = ImprintOperations.projectBytes(recordBytes); + + // Then result should be empty but valid + var result = ImprintRecord.deserialize(projected); + assertEquals(0, result.getDirectory().size()); + } + + @Test + @DisplayName("should handle nonexistent fields in byte projection") + void shouldHandleNonexistentFieldsInByteProjection() throws ImprintException { + // Given a record with some fields + var record = ImprintRecord.builder(testSchema) + .field(1, "exists") + .field(3, 42) + .build(); + + // When projecting mix of existing and non-existing fields + var recordBytes = record.serializeToBuffer(); + var projected = ImprintOperations.projectBytes(recordBytes, 1, 99, 100); + + // Then only existing fields should be included + var result = ImprintRecord.deserialize(projected); + assertEquals(1, result.getDirectory().size()); + assertEquals("exists", result.getString(1)); + assertNull(result.getValue(99)); + assertNull(result.getValue(100)); + } + + @Test + @DisplayName("should handle null buffers gracefully") + void shouldHandleNullBuffersGracefully() throws ImprintException { + var validRecord = ImprintRecord.builder(testSchema) + .field(1, "test") + .build(); + var validBuffer = validRecord.serializeToBuffer(); + + // Test null buffer scenarios + assertThrows(Exception.class, () -> + ImprintOperations.mergeBytes(null, validBuffer)); + assertThrows(Exception.class, () -> + ImprintOperations.mergeBytes(validBuffer, null)); + assertThrows(Exception.class, () -> + ImprintOperations.projectBytes(null, 1, 2, 3)); + } + + @Test + @DisplayName("should validate buffer format and reject invalid data") + void shouldValidateBufferFormatAndRejectInvalidData() throws ImprintException { + var validRecord = ImprintRecord.builder(testSchema) + .field(1, "test") + .build(); + var validBuffer = validRecord.serializeToBuffer(); + + // Test invalid magic byte + var invalidMagic = ByteBuffer.allocate(20); + invalidMagic.put((byte) 0x99); // Invalid magic + invalidMagic.put((byte) 0x01); // Valid version + invalidMagic.flip(); + + assertThrows(ImprintException.class, () -> + ImprintOperations.mergeBytes(invalidMagic, validBuffer)); + assertThrows(ImprintException.class, () -> + ImprintOperations.projectBytes(invalidMagic, 1)); + + // Test buffer too small + var tooSmall = ByteBuffer.allocate(5); + tooSmall.put(new byte[]{1, 2, 3, 4, 5}); + tooSmall.flip(); + + assertThrows(ImprintException.class, () -> + ImprintOperations.mergeBytes(tooSmall, validBuffer)); + assertThrows(ImprintException.class, () -> + ImprintOperations.projectBytes(tooSmall, 1)); + + // Test invalid version + var invalidVersion = ByteBuffer.allocate(20); + invalidVersion.put((byte) 0x49); // Valid magic + invalidVersion.put((byte) 0x99); // Invalid version + invalidVersion.flip(); + + assertThrows(ImprintException.class, () -> + ImprintOperations.mergeBytes(invalidVersion, validBuffer)); + assertThrows(ImprintException.class, () -> + ImprintOperations.projectBytes(invalidVersion, 1)); + } + + @Test + @DisplayName("should handle large records efficiently in bytes operations") + void shouldHandleLargeRecordsEfficientlyInBytesOperations() throws ImprintException { + // Create records with many fields + var builder1 = ImprintRecord.builder(testSchema); + var builder2 = ImprintRecord.builder(testSchema); + + // Add many fields + for (int i = 1; i <= 50; i++) { + builder1.field(i, "field_" + i); + } + for (int i = 51; i <= 100; i++) { + builder2.field(i, "field_" + i); + } + + var record1 = builder1.build(); + var record2 = builder2.build(); + + // Test bytes-to-bytes merge with many fields + var merged = ImprintOperations.mergeBytes( + record1.serializeToBuffer(), + record2.serializeToBuffer() + ); + var mergedRecord = ImprintRecord.deserialize(merged); + + assertEquals(100, mergedRecord.getDirectory().size()); + assertEquals("field_1", mergedRecord.getString(1)); + assertEquals("field_100", mergedRecord.getString(100)); + + // Test bytes-to-bytes projection with many fields + int[] projectFields = {1, 25, 50, 75, 100}; + var projected = ImprintOperations.projectBytes(merged, projectFields); + var projectedRecord = ImprintRecord.deserialize(projected); + + assertEquals(5, projectedRecord.getDirectory().size()); + assertEquals("field_1", projectedRecord.getString(1)); + assertEquals("field_25", projectedRecord.getString(25)); + assertEquals("field_100", projectedRecord.getString(100)); + } + + @Test + @DisplayName("should preserve field order in bytes operations") + void shouldPreserveFieldOrderInBytesOperations() throws ImprintException { + var record = ImprintRecord.builder(testSchema) + .field(5, "field5") + .field(1, "field1") + .field(3, "field3") + .field(2, "field2") + .field(4, "field4") + .build(); + + // Project in random order + var projected = ImprintOperations.projectBytes( + record.serializeToBuffer(), 4, 1, 3, 5, 2 + ); + var projectedRecord = ImprintRecord.deserialize(projected); + + // Verify fields are still accessible and directory is sorted + var directory = projectedRecord.getDirectory(); + assertEquals(5, directory.size()); + + // Directory should be sorted by field ID + for (int i = 1; i < directory.size(); i++) { + assertTrue(directory.get(i - 1).getId() < directory.get(i).getId()); + } + + // All fields should be accessible + assertEquals("field1", projectedRecord.getString(1)); + assertEquals("field2", projectedRecord.getString(2)); + assertEquals("field3", projectedRecord.getString(3)); + assertEquals("field4", projectedRecord.getString(4)); + assertEquals("field5", projectedRecord.getString(5)); } } + } diff --git a/src/test/java/com/imprint/profile/ProfilerTest.java b/src/test/java/com/imprint/profile/ProfilerTest.java index 7b8a027..79882d9 100644 --- a/src/test/java/com/imprint/profile/ProfilerTest.java +++ b/src/test/java/com/imprint/profile/ProfilerTest.java @@ -8,6 +8,8 @@ import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertTrue; + import java.util.Random; import java.util.stream.IntStream; @@ -38,14 +40,14 @@ private void profileSmallMerges() throws Exception { var record1 = createTestRecord(20); var record2 = createTestRecord(20); - int iterations = 200_000; + int iterations = 500_000; System.out.printf("Beginning small merge profiling (%,d iterations)...%n", iterations); long start = System.nanoTime(); for (int i = 0; i < iterations; i++) { // This is the hotspot we want to profile - var merged = ImprintOperations.merge(record1, record2); + var merged = record1.merge(record2); // Simulate some usage to prevent dead code elimination if (i % 10_000 == 0) { @@ -68,13 +70,13 @@ private void profileLargeMerges() throws Exception { var record1 = createTestRecord(100); var record2 = createTestRecord(100); - int iterations = 50_000; + int iterations = 100_000; System.out.printf("Beginning large merge profiling (%,d iterations)...%n", iterations); long start = System.nanoTime(); for (int i = 0; i < iterations; i++) { - var merged = ImprintOperations.merge(record1, record2); + var merged = record1.merge(record2); merged.serializeToBuffer(); } @@ -91,13 +93,13 @@ private void profileOverlappingMerges() throws Exception { var record1 = createTestRecordWithFieldIds(new int[]{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}); var record2 = createTestRecordWithFieldIds(new int[]{10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24}); - int iterations = 100_000; + int iterations = 200_000; System.out.printf("Beginning overlapping merge profiling (%,d iterations)...%n", iterations); long start = System.nanoTime(); for (int i = 0; i < iterations; i++) { - var merged = ImprintOperations.merge(record1, record2); + var merged = record1.merge(record2); merged.serializeToBuffer(); } @@ -115,13 +117,13 @@ private void profileDisjointMerges() throws Exception { // Create records with completely separate field IDs var record1 = createTestRecordWithFieldIds(new int[]{1, 3, 5, 7, 9, 11, 13, 15, 17, 19}); var record2 = createTestRecordWithFieldIds(new int[]{2, 4, 6, 8, 10, 12, 14, 16, 18, 20}); - int iterations = 100_000; + int iterations = 200_000; System.out.printf("Beginning disjoint merge profiling (%,d iterations)...%n", iterations); long start = System.nanoTime(); for (int i = 0; i < iterations; i++) { - var merged = ImprintOperations.merge(record1, record2); + var merged = record1.merge(record2); merged.serializeToBuffer(); } @@ -140,6 +142,10 @@ void profileSmallRecordSerialization() throws Exception { @Test @Tag("serialization") @Tag("large-records") + /* + It's usually better to change DEFAULT_CAPACITY in ImprintFieldObjectMap to ensure resizing doesn't happen + unless you specifically want to profile resizing costs (should happen rarely in reality). + */ void profileLargeRecordSerialization() throws Exception { profileSerialization("large records", LARGE_RECORD_SIZE, 500_000); } @@ -147,49 +153,12 @@ void profileLargeRecordSerialization() throws Exception { @Test @Tag("projection") void profileProjectionOperations() throws Exception { - System.out.println("Starting projection profiler test - attach profiler now..."); Thread.sleep(3000); profileSmallProjections(); profileLargeProjections(); profileSelectiveProjections(); } - @Test - @Tag("memory") - @Tag("allocation") - void profileMemoryAllocation() throws Exception { - System.out.println("Starting allocation profiler test..."); - Thread.sleep(3000); - - System.out.println("Beginning allocation profiling - watch for GC events..."); - - // Force allocation pressure to reveal GC hotspots - for (int batch = 0; batch < 1000; batch++) { - for (int i = 0; i < 1000; i++) { - var schemaId = new SchemaId(batch, i); - var builder = ImprintRecord.builder(schemaId); - - // Create strings of varying sizes (allocation pressure) - builder.field(1, Value.fromString("small")) - .field(2, Value.fromString("medium-length-string-" + i)) - .field(3, Value.fromString("very-long-string-that-will-cause-more-allocation-pressure-" + batch + "-" + i)) - .field(4, Value.fromBytes(new byte[100 + i % 1000])); // Varying byte arrays - - var record = builder.build(); - - // Some deserialization to trigger string decoding allocations - record.getValue(2); - record.getValue(3); - } - - if (batch % 100 == 0) { - System.out.printf("Completed batch %d/1000%n", batch); - } - } - - System.out.println("Allocation test complete - check GC logs and memory profiler"); - } - // Rest of the methods remain the same... private void profileSmallProjections() throws Exception { System.out.println("\\n--- Small Projections (2-5 fields from 20-field records) ---"); @@ -203,7 +172,7 @@ private void profileSmallProjections() throws Exception { for (int i = 0; i < iterations; i++) { // This is the hotspot we want to profile - var projected = ImprintOperations.project(sourceRecord, projectFields); + var projected = sourceRecord.project(projectFields); // Simulate some usage to prevent dead code elimination if (i % 10_000 == 0) { @@ -226,14 +195,14 @@ private void profileLargeProjections() throws Exception { int[] projectFields = IntStream.range(0, 50) .map(i -> (i * 4) + 1) .toArray(); - int iterations = 50_000; + int iterations = 200_000; System.out.printf("Beginning large projection profiling (%,d iterations, %d->%d fields)...%n", iterations, 200, projectFields.length); long start = System.nanoTime(); for (int i = 0; i < iterations; i++) { - var projected = ImprintOperations.project(sourceRecord, projectFields); + var projected = sourceRecord.project(projectFields); // Periodically access some fields to simulate real usage if (i % 1_000 == 0) { @@ -254,7 +223,7 @@ private void profileSelectiveProjections() throws Exception { var sourceRecord = createTestRecord(100); Random random = new Random(42); - int iterations = 100_000; + int iterations = 200_000; // Test different projection patterns var patterns = new ProjectionPattern[]{ @@ -271,7 +240,7 @@ private void profileSelectiveProjections() throws Exception { long start = System.nanoTime(); for (int i = 0; i < iterations; i++) { - var projected = ImprintOperations.project(sourceRecord, pattern.fields); + var projected = sourceRecord.project(pattern.fields); // Simulate field access if (i % 5_000 == 0) { @@ -410,4 +379,59 @@ private int[] generateRandomFields(Random random, int maxField, int count) { .sorted() .toArray(); } + + @Test + @Tag("profiling") + void profileBytesToBytesVsObjectMerge() throws Exception { + System.out.println("=== Bytes-to-Bytes vs Object Merge Comparison ==="); + + // Create test records + var record1 = createTestRecordWithFieldIds(new int[]{1, 3, 5, 7, 9, 11, 13, 15}); + var record2 = createTestRecordWithFieldIds(new int[]{2, 4, 6, 8, 10, 12, 14, 16}); + + var record1Bytes = record1.serializeToBuffer(); + var record2Bytes = record2.serializeToBuffer(); + + int iterations = 50_000; + + // Warm up + for (int i = 0; i < 1000; i++) { + record1.merge(record2).serializeToBuffer(); + ImprintOperations.mergeBytes(record1Bytes, record2Bytes); + } + + System.out.printf("Profiling %,d merge operations...%n", iterations); + + // Test object merge + serialize + long startObjectMerge = System.nanoTime(); + for (int i = 0; i < iterations; i++) { + var merged = record1.merge(record2); + var serialized = merged.serializeToBuffer(); + // Consume result to prevent optimization + if (serialized.remaining() == 0) throw new RuntimeException("Empty result"); + } + long objectMergeTime = System.nanoTime() - startObjectMerge; + + // Test bytes merge + long startBytesMerge = System.nanoTime(); + for (int i = 0; i < iterations; i++) { + var merged = ImprintOperations.mergeBytes(record1Bytes, record2Bytes); + // Consume result to prevent optimization + if (merged.remaining() == 0) throw new RuntimeException("Empty result"); + } + long bytesMergeTime = System.nanoTime() - startBytesMerge; + + double objectAvg = (double) objectMergeTime / iterations / 1000.0; // microseconds + double bytesAvg = (double) bytesMergeTime / iterations / 1000.0; // microseconds + double speedup = objectAvg / bytesAvg; + + System.out.printf("Object merge + serialize: %.2f ms (avg: %.1f μs/op)%n", + objectMergeTime / 1_000_000.0, objectAvg); + System.out.printf("Bytes-to-bytes merge: %.2f ms (avg: %.1f μs/op)%n", + bytesMergeTime / 1_000_000.0, bytesAvg); + System.out.printf("Speedup: %.1fx faster%n", speedup); + + // Assert that bytes approach is faster (should be at least 1.5x) + assertTrue(speedup > 1.0, String.format("Bytes merge should be faster. Got %.1fx speedup", speedup)); + } } \ No newline at end of file diff --git a/src/test/java/com/imprint/stream/ImprintStreamTest.java b/src/test/java/com/imprint/stream/ImprintStreamTest.java deleted file mode 100644 index d2c2b69..0000000 --- a/src/test/java/com/imprint/stream/ImprintStreamTest.java +++ /dev/null @@ -1,81 +0,0 @@ -package com.imprint.stream; - -import com.imprint.core.ImprintRecord; -import com.imprint.core.SchemaId; -import com.imprint.stream.ImprintStream; -import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.*; - -class ImprintStreamTest { - - @Test - void shouldProjectAndMergeCorrectly() throws Exception { - // --- Setup --- - var schemaId1 = new SchemaId(1, 1); - var schemaId2 = new SchemaId(2, 2); - var schemaId3 = new SchemaId(3, 3); - - var recordA = ImprintRecord.builder(schemaId1) - .field(1, "A1") - .field(2, 100) - .field(3, true) - .build(); - - var recordB = ImprintRecord.builder(schemaId2) - .field(2, 200) // Overlaps with A, should be ignored - .field(4, "B4") - .build(); - - var recordC = ImprintRecord.builder(schemaId3) - .field(5, 3.14) - .field(1, "C1") // Overlaps with A, should be ignored - .build(); - - // --- Execution --- - // Chain of operations - var finalRecord = ImprintStream.of(recordA) - .project(1, 3) // Keep {1, 3} from A. Current state: {1:A, 3:A} - .mergeWith(recordB) // Merge B. {2:B, 4:B} are added. Current state: {1:A, 3:A, 2:B, 4:B} - .mergeWith(recordC) // Merge C. {5:C} is added. {1:C} is ignored. Final state: {1:A, 3:A, 2:B, 4:B, 5:C} - .project(1, 4, 5) // Final projection. Final result: {1:A, 4:B, 5:C} - .toRecord(); - - // --- Assertions --- - assertNotNull(finalRecord); - - // Check final field count. - assertEquals(3, finalRecord.getDirectory().size()); - - // Check that the correct fields are present and have the right values - assertTrue(finalRecord.hasField(1)); - assertEquals("A1", finalRecord.getString(1)); // From recordA - - assertTrue(finalRecord.hasField(4)); - assertEquals("B4", finalRecord.getString(4)); // From recordB - - assertTrue(finalRecord.hasField(5)); - assertEquals(3.14, finalRecord.getFloat64(5), 0.001); // From recordC - - // Check that dropped/ignored fields are not present - assertFalse(finalRecord.hasField(2)); - assertFalse(finalRecord.hasField(3)); - } - - @Test - void shouldProjectAfterMerge() throws Exception { - var recordA = ImprintRecord.builder(new SchemaId(1, 1)).field(1, "A").field(2, 100).build(); - var recordB = ImprintRecord.builder(new SchemaId(1, 1)).field(2, 200).field(3, "B").build(); - - var finalRecord = ImprintStream.of(recordA) - .mergeWith(recordB) // virtual record is {1:A, 2:A, 3:B} - .project(1, 3) // final record is {1:A, 3:B} - .toRecord(); - - assertEquals(2, finalRecord.getDirectory().size()); - assertTrue(finalRecord.hasField(1)); - assertEquals("A", finalRecord.getString(1)); - assertTrue(finalRecord.hasField(3)); - assertEquals("B", finalRecord.getString(3)); - assertFalse(finalRecord.hasField(2)); - } -} \ No newline at end of file From 50c8a4b15e015fdec2cb0bd7fbf320bf0272db49 Mon Sep 17 00:00:00 2001 From: expand3d <> Date: Thu, 12 Jun 2025 16:16:14 -0400 Subject: [PATCH 47/49] track custom map --- .../imprint/core/ImprintFieldObjectMap.java | 309 +++++++++++++++++ .../core/ImprintFieldObjectMapTest.java | 318 ++++++++++++++++++ 2 files changed, 627 insertions(+) create mode 100644 src/main/java/com/imprint/core/ImprintFieldObjectMap.java create mode 100644 src/test/java/com/imprint/core/ImprintFieldObjectMapTest.java diff --git a/src/main/java/com/imprint/core/ImprintFieldObjectMap.java b/src/main/java/com/imprint/core/ImprintFieldObjectMap.java new file mode 100644 index 0000000..d104317 --- /dev/null +++ b/src/main/java/com/imprint/core/ImprintFieldObjectMap.java @@ -0,0 +1,309 @@ +package com.imprint.core; + +import java.util.Arrays; +import java.util.stream.IntStream; + +/** + * Specialized short→object map optimized for ImprintRecordBuilder field IDs. + * Basically a copy of EclipseCollections's primitive map: + * - No key-value boxing/unboxing + * - Primitive int16 keys + * - Open addressing with linear probing + * - Sort values in place and return without allocation (subsequently poisons the map) + */ +final class ImprintFieldObjectMap { + private static final int DEFAULT_CAPACITY = 512; + private static final float LOAD_FACTOR = 0.75f; + private static final short EMPTY_KEY = -1; // Reserved empty marker (field IDs are >= 0) + + private short[] keys; + private Object[] values; + private int size; + private int threshold; + private boolean poisoned = false; + + public ImprintFieldObjectMap() { + this(DEFAULT_CAPACITY); + } + + public ImprintFieldObjectMap(int initialCapacity) { + int capacity = nextPowerOfTwo(Math.max(4, initialCapacity)); + this.keys = new short[capacity]; + this.values = new Object[capacity]; + this.threshold = (int) (capacity * LOAD_FACTOR); + Arrays.fill(keys, EMPTY_KEY); + } + + public void put(short key, T value) { + if (poisoned) + throw new IllegalStateException("Map is invalid after compaction - cannot perform operations"); + putValue(key, value); + } + + public void put(int key, T value) { + if (poisoned) + throw new IllegalStateException("Map is invalid after compaction - cannot perform operations"); + if (key > Short.MAX_VALUE) + throw new IllegalArgumentException("Field ID must be 0-" + Short.MAX_VALUE + ", got: " + key); + putValue((short) key, value); + } + + private void putValue(short key, T value) { + if (poisoned) + throw new IllegalStateException("Map is invalid after compaction - cannot perform operations"); + if (key < 0) + throw new IllegalArgumentException("Field ID must be 0-" + Short.MAX_VALUE + ", got: " + key); + + if (size >= threshold) + resize(); + int index = findSlot(key); + if (keys[index] == EMPTY_KEY) { + size++; + } + keys[index] = key; + values[index] = value; + } + + @SuppressWarnings("unchecked") + public T get(int key) { + if (poisoned) + throw new IllegalStateException("Map is invalid after compaction - cannot perform operations"); + if (key < 0 || key > Short.MAX_VALUE) + return null; + short shortKey = (short) key; + int index = findSlot(shortKey); + return keys[index] == shortKey ? (T) values[index] : null; + } + + public boolean containsKey(int key) { + if (poisoned) + throw new IllegalStateException("Map is invalid after compaction - cannot perform operations"); + if (key < 0 || key > Short.MAX_VALUE) return false; + short shortKey = (short) key; + + int index = findSlot(shortKey); + return keys[index] == shortKey; + } + + public int size() { + return size; + } + + public boolean isEmpty() { + return size == 0; + } + + /** + * Get all keys (non-destructive). + */ + public int[] getKeys() { + return IntStream.range(0, keys.length) + .filter(i -> keys[i] != EMPTY_KEY) + .map(i -> keys[i]).toArray(); + } + + /** + * Stream all keys without allocation. + * Non-destructive operation that can be called multiple times. + * + * @return IntStream of all keys in the map + */ + public IntStream streamKeys() { + if (poisoned) { + throw new IllegalStateException("Map is invalid after compaction - cannot perform operations"); + } + + return IntStream.range(0, keys.length) + .filter(i -> keys[i] != EMPTY_KEY) + .map(i -> keys[i]); + } + + /** + * Result holder for in-place sorted values - avoids allocation by returning + * array reference and valid count. + */ + public static final class SortedValuesResult { + public final Object[] values; + public final int count; + + SortedValuesResult(Object[] values, int count) { + this.values = values; + this.count = count; + } + } + + /** + * Get values sorted by key order with zero allocation by left-side compacting the value set. + * WARNING: Modifies internal state, and renders map operations unstable and in an illegal state. Only invoke this + * if you plan to discard the map afterward. + * (e.g., at the end of builder lifecycle before build()). + * + * @return SortedValuesResult containing the internal values array and valid count. + * Caller should iterate from 0 to result.count-1 only. + */ + public SortedValuesResult getSortedValues() { + if (size == 0) { + // Poison the map even when empty, even if just for consistency + poisoned = true; + return new SortedValuesResult(values, 0); + } + + // left side compaction of all entries to the front of the arrays + compactEntries(); + + // Sort the compacted entries by key in-place + sortEntriesByKey(size); + + // Poison the map - no further operations allowed + poisoned = true; + + // Return the internal array w/ count + return new SortedValuesResult(values, size); + } + + /** + * Get values sorted by key order. + * Does not modify internal state and can be invoked repeatedly. + * + * @param resultArray Array to store results (will be resized if needed) + * @return Sorted array of values + */ + @SuppressWarnings("unchecked") + public T[] getSortedValuesCopy(T[] resultArray) { + if (poisoned) + throw new IllegalStateException("Map is poisoned after destructive sort - cannot perform operations"); + if (size == 0) + return resultArray.length == 0 ? resultArray : Arrays.copyOf(resultArray, 0); + + // Create temporary arrays for non-destructive sort + var tempKeys = new short[size]; + var tempValues = new Object[size]; + + // Copy valid entries to temporary arrays + int writeIndex = 0; + for (int readIndex = 0; readIndex < keys.length; readIndex++) { + if (keys[readIndex] != EMPTY_KEY) { + tempKeys[writeIndex] = keys[readIndex]; + tempValues[writeIndex] = values[readIndex]; + writeIndex++; + } + } + + // Sort the temporary arrays by key + for (int i = 1; i < size; i++) { + short key = tempKeys[i]; + Object value = tempValues[i]; + int j = i - 1; + + while (j >= 0 && tempKeys[j] > key) { + tempKeys[j + 1] = tempKeys[j]; + tempValues[j + 1] = tempValues[j]; + j--; + } + + tempKeys[j + 1] = key; + tempValues[j + 1] = value; + } + + // Copy sorted values to result array + if (resultArray.length != size) + resultArray = Arrays.copyOf(resultArray, size); + + for (int i = 0; i < size; i++) + resultArray[i] = (T) tempValues[i]; + + return resultArray; + } + + /** + * Compact all non-empty entries to the front of keys/values arrays. + */ + private void compactEntries() { + int writeIndex = 0; + + for (int readIndex = 0; readIndex < keys.length; readIndex++) { + if (keys[readIndex] != EMPTY_KEY) { + if (writeIndex != readIndex) { + keys[writeIndex] = keys[readIndex]; + values[writeIndex] = values[readIndex]; + + // Clear the old slot + keys[readIndex] = EMPTY_KEY; + values[readIndex] = null; + } + writeIndex++; + } + } + } + + /** + * Sort the first 'count' entries by key using insertion sort (should be fast for small arrays). + */ + private void sortEntriesByKey(int count) { + for (int i = 1; i < count; i++) { + short key = keys[i]; + Object value = values[i]; + int j = i - 1; + + while (j >= 0 && keys[j] > key) { + keys[j + 1] = keys[j]; + values[j + 1] = values[j]; + j--; + } + + keys[j + 1] = key; + values[j + 1] = value; + } + } + + + private int findSlot(short key) { + int mask = keys.length - 1; + int index = hash(key) & mask; + + // Linear probing + while (keys[index] != EMPTY_KEY && keys[index] != key) { + index = (index + 1) & mask; + } + + return index; + } + + private void resize() { + short[] oldKeys = keys; + Object[] oldValues = values; + + int newCapacity = keys.length * 2; + keys = new short[newCapacity]; + values = new Object[newCapacity]; + threshold = (int) (newCapacity * LOAD_FACTOR); + Arrays.fill(keys, EMPTY_KEY); + + int oldSize = size; + size = 0; + + // Rehash all entries + for (int i = 0; i < oldKeys.length; i++) { + if (oldKeys[i] != EMPTY_KEY) { + @SuppressWarnings("unchecked") + T value = (T) oldValues[i]; + put(oldKeys[i], value); + } + } + + // Verify size didn't change during rehash + assert size == oldSize; + } + + private static int hash(short key) { + // Simple but effective hash for short keys + int intKey = key & 0xFFFF; // Convert to unsigned int + intKey ^= intKey >>> 8; + return intKey; + } + + private static int nextPowerOfTwo(int n) { + if (n <= 1) return 1; + return Integer.highestOneBit(n - 1) << 1; + } +} \ No newline at end of file diff --git a/src/test/java/com/imprint/core/ImprintFieldObjectMapTest.java b/src/test/java/com/imprint/core/ImprintFieldObjectMapTest.java new file mode 100644 index 0000000..cb6637f --- /dev/null +++ b/src/test/java/com/imprint/core/ImprintFieldObjectMapTest.java @@ -0,0 +1,318 @@ +package com.imprint.core; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.BeforeEach; +import static org.junit.jupiter.api.Assertions.*; + +/** + * Unit tests for IntObjectMap - specialized short→object map optimized for field IDs. + */ +class ImprintFieldObjectMapTest { + + private ImprintFieldObjectMap map; + + @BeforeEach + void setUp() { + map = new ImprintFieldObjectMap<>(); + } + + @Test + void shouldPutAndGetBasicOperations() { + map.put(1, "one"); + map.put(5, "five"); + map.put(10, "ten"); + + assertEquals("one", map.get(1)); + assertEquals("five", map.get(5)); + assertEquals("ten", map.get(10)); + assertNull(map.get(99)); + assertEquals(3, map.size()); + } + + @Test + void shouldHandleKeyValidation() { + // Valid keys (0 to Short.MAX_VALUE) + map.put(0, "zero"); + map.put(Short.MAX_VALUE, "max"); + + // Invalid keys + assertThrows(IllegalArgumentException.class, () -> map.put(-1, "negative")); + assertThrows(IllegalArgumentException.class, () -> map.put(Short.MAX_VALUE + 1, "too_large")); + } + + @Test + void shouldHandleContainsKey() { + map.put(1, "one"); + map.put(5, "five"); + + assertTrue(map.containsKey(1)); + assertTrue(map.containsKey(5)); + assertFalse(map.containsKey(99)); + assertFalse(map.containsKey(-1)); + assertFalse(map.containsKey(Short.MAX_VALUE + 1)); + } + + @Test + void shouldOverwriteExistingKeys() { + map.put(1, "original"); + assertEquals("original", map.get(1)); + assertEquals(1, map.size()); + + map.put(1, "updated"); + assertEquals("updated", map.get(1)); + assertEquals(1, map.size()); // Size should not increase + } + + @Test + void shouldGetKeysArray() { + map.put(3, "three"); + map.put(1, "one"); + map.put(7, "seven"); + + int[] keys = map.getKeys(); + assertEquals(3, keys.length); + + // Convert to set for order-independent comparison + var keySet = java.util.Arrays.stream(keys).boxed() + .collect(java.util.stream.Collectors.toSet()); + + assertTrue(keySet.contains(1)); + assertTrue(keySet.contains(3)); + assertTrue(keySet.contains(7)); + } + + @Test + void shouldSortValuesNonDestructively() { + map.put(3, "three"); + map.put(1, "one"); + map.put(7, "seven"); + map.put(2, "two"); + + // Test non-destructive sort + String[] sorted = map.getSortedValuesCopy(new String[0]); + + assertEquals(4, sorted.length); + assertEquals("one", sorted[0]); // key 1 + assertEquals("two", sorted[1]); // key 2 + assertEquals("three", sorted[2]); // key 3 + assertEquals("seven", sorted[3]); // key 7 + + // Verify map is still functional after non-destructive sort + assertEquals("three", map.get(3)); + assertEquals("one", map.get(1)); + assertEquals(4, map.size()); + + // Should be able to call multiple times + String[] sorted2 = map.getSortedValuesCopy(new String[0]); + assertArrayEquals(sorted, sorted2); + } + + @Test + void shouldSortValuesDestructively() { + map.put(3, "three"); + map.put(1, "one"); + map.put(7, "seven"); + map.put(2, "two"); + + // Test destructive sort + ImprintFieldObjectMap.SortedValuesResult result = map.getSortedValues(); + + assertEquals(4, result.count); + assertEquals("one", result.values[0]); // key 1 + assertEquals("two", result.values[1]); // key 2 + assertEquals("three", result.values[2]); // key 3 + assertEquals("seven", result.values[3]); // key 7 + } + + @Test + void shouldPoisonMapAfterDestructiveSort() { + map.put(1, "one"); + map.put(2, "two"); + + // Perform destructive sort + ImprintFieldObjectMap.SortedValuesResult result = map.getSortedValues(); + assertNotNull(result); + + // All operations should throw IllegalStateException after poisoning + assertThrows(IllegalStateException.class, () -> map.put(3, "three")); + assertThrows(IllegalStateException.class, () -> map.get(1)); + assertThrows(IllegalStateException.class, () -> map.containsKey(1)); + assertThrows(IllegalStateException.class, () -> map.getSortedValuesCopy(new String[0])); + + // Size and isEmpty should still work (they don't check poisoned state) + assertEquals(2, map.size()); + assertFalse(map.isEmpty()); + } + + @Test + void shouldHandleEmptyMapSorting() { + // Test non-destructive sort on empty map + String[] sorted = map.getSortedValuesCopy(new String[0]); + assertEquals(0, sorted.length); + + // Test destructive sort on empty map + ImprintFieldObjectMap.SortedValuesResult result = map.getSortedValues(); + assertEquals(0, result.count); + + // Map should be poisoned even after empty destructive sort + assertThrows(IllegalStateException.class, () -> map.put(1, "one")); + } + + @Test + void shouldHandleSingleElementSorting() { + map.put(42, "answer"); + + // Test non-destructive sort + String[] sorted = map.getSortedValuesCopy(new String[0]); + assertEquals(1, sorted.length); + assertEquals("answer", sorted[0]); + + // Test destructive sort on fresh map + ImprintFieldObjectMap map2 = new ImprintFieldObjectMap<>(); + map2.put(42, "answer"); + + ImprintFieldObjectMap.SortedValuesResult result = map2.getSortedValues(); + assertEquals(1, result.count); + assertEquals("answer", result.values[0]); + } + + @Test + void shouldHandleHashCollisions() { + // Add many entries to trigger collisions and resizing + for (int i = 0; i < 1000; i++) { + map.put(i, "value_" + i); + } + + // Verify all entries are accessible + for (int i = 0; i < 1000; i++) { + assertEquals("value_" + i, map.get(i)); + assertTrue(map.containsKey(i)); + } + + assertEquals(1000, map.size()); + + // Test sorting with many entries + String[] sorted = map.getSortedValuesCopy(new String[0]); + assertEquals(1000, sorted.length); + + // Verify sorting is correct + for (int i = 0; i < 1000; i++) { + assertEquals("value_" + i, sorted[i]); + } + } + + @Test + void shouldReuseResultArrayForNonDestructiveSort() { + map.put(1, "one"); + map.put(2, "two"); + + String[] reusableArray = new String[2]; + String[] result = map.getSortedValuesCopy(reusableArray); + + assertSame(reusableArray, result); // Should reuse the same array + assertEquals("one", result[0]); + assertEquals("two", result[1]); + + // Test with wrong size array - should create new array + String[] wrongSizeArray = new String[5]; + String[] result2 = map.getSortedValuesCopy(wrongSizeArray); + + assertNotSame(wrongSizeArray, result2); // Should create new array + assertEquals(2, result2.length); + assertEquals("one", result2[0]); + assertEquals("two", result2[1]); + } + + @Test + void shouldHandleMaxShortValue() { + int maxKey = Short.MAX_VALUE; + map.put(maxKey, "max_value"); + map.put(0, "zero"); + map.put(maxKey - 1, "almost_max"); + + assertEquals("max_value", map.get(maxKey)); + assertEquals("zero", map.get(0)); + assertEquals("almost_max", map.get(maxKey - 1)); + + String[] sorted = map.getSortedValuesCopy(new String[0]); + assertEquals("zero", sorted[0]); + assertEquals("almost_max", sorted[1]); + assertEquals("max_value", sorted[2]); + } + + @Test + void shouldMaintainSizeCorrectlyWithOverwrites() { + assertEquals(0, map.size()); + assertTrue(map.isEmpty()); + + map.put(1, "first"); + assertEquals(1, map.size()); + assertFalse(map.isEmpty()); + + map.put(1, "overwrite"); + assertEquals(1, map.size()); // Size should not change + + map.put(2, "second"); + assertEquals(2, map.size()); + + map.put(1, "overwrite_again"); + assertEquals(2, map.size()); // Size should not change + } + + @Test + void shouldStreamKeysWithoutAllocation() { + map.put(3, "three"); + map.put(1, "one"); + map.put(7, "seven"); + + // Stream keys without allocation + java.util.Set streamedKeys = map.streamKeys() + .boxed() + .collect(java.util.stream.Collectors.toSet()); + + assertEquals(3, streamedKeys.size()); + assertTrue(streamedKeys.contains(1)); + assertTrue(streamedKeys.contains(3)); + assertTrue(streamedKeys.contains(7)); + + // Should be able to stream multiple times + long count = map.streamKeys().count(); + assertEquals(3, count); + + // Test operations on stream + int sum = map.streamKeys().sum(); + assertEquals(11, sum); // 1 + 3 + 7 + + // Test filtering + long evenKeys = map.streamKeys().filter(k -> k % 2 == 0).count(); + assertEquals(0, evenKeys); + + long oddKeys = map.streamKeys().filter(k -> k % 2 == 1).count(); + assertEquals(3, oddKeys); + } + + @Test + void shouldThrowOnStreamKeysAfterPoisoning() { + map.put(1, "one"); + map.put(2, "two"); + + // Stream should work before poisoning + assertEquals(2, map.streamKeys().count()); + + // Poison the map + map.getSortedValues(); + + // Stream should throw after poisoning + assertThrows(IllegalStateException.class, () -> map.streamKeys()); + } + + @Test + void shouldStreamEmptyMapKeys() { + // Empty map should produce empty stream + assertEquals(0, map.streamKeys().count()); + + // Operations on empty stream should work + assertEquals(0, map.streamKeys().sum()); + assertEquals(java.util.OptionalInt.empty(), map.streamKeys().findFirst()); + } +} \ No newline at end of file From eb40310986cd968e17512db84a101d4397ecaebf Mon Sep 17 00:00:00 2001 From: expand3d <> Date: Thu, 12 Jun 2025 16:46:09 -0400 Subject: [PATCH 48/49] delete extra operations file because I moved it --- .../com/imprint/core/ImprintOperations.java | 207 --------- .../imprint/core/ImprintOperationsTest.java | 405 ------------------ 2 files changed, 612 deletions(-) delete mode 100644 src/main/java/com/imprint/core/ImprintOperations.java delete mode 100644 src/test/java/com/imprint/core/ImprintOperationsTest.java diff --git a/src/main/java/com/imprint/core/ImprintOperations.java b/src/main/java/com/imprint/core/ImprintOperations.java deleted file mode 100644 index 4e60ebf..0000000 --- a/src/main/java/com/imprint/core/ImprintOperations.java +++ /dev/null @@ -1,207 +0,0 @@ -package com.imprint.core; - -import com.imprint.error.ErrorType; -import com.imprint.error.ImprintException; -import lombok.Value; -import lombok.experimental.UtilityClass; - -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.util.*; - -@UtilityClass -public class ImprintOperations { - - /** - * Project a subset of fields from an Imprint record. Payload copying is proportional to projected data size. - * - *

Algorithm:

- *
    - *
  1. Sort and deduplicate requested field IDs for efficient matching
    2. - *
  2. Scan directory to find matching fields and calculate ranges
    3. - *
  3. Allocate new payload buffer with exact size needed
    4. - *
  4. Copy field data ranges directly (zero-copy where possible)
    5. - *
  5. Build new directory with adjusted offsets
    6. - *
- * - * @param record The source record to project from - * @param fieldIds Array of field IDs to include in projection - * @return New ImprintRecord containing only the requested fields - */ - public static ImprintRecord project(ImprintRecord record, int... fieldIds) { - // Sort and deduplicate field IDs for efficient matching with sorted directory - int[] sortedFieldIds = Arrays.stream(fieldIds).distinct().sorted().toArray(); - if (sortedFieldIds.length == 0) - return createEmptyRecord(record.getHeader().getSchemaId()); - - //eager fetch the entire directory (can this be lazy and just done per field?) - var sourceDirectory = record.getDirectory(); - var newDirectory = new ArrayList(sortedFieldIds.length); - var ranges = new ArrayList(); - - // Iterate through directory and compute ranges to copy - int fieldIdsIdx = 0; - int directoryIdx = 0; - int currentOffset = 0; - - while (directoryIdx < sourceDirectory.size() && fieldIdsIdx < sortedFieldIds.length) { - var field = sourceDirectory.get(directoryIdx); - if (field.getId() == sortedFieldIds[fieldIdsIdx]) { - // Calculate field length using next field's offset - int nextOffset = (directoryIdx + 1 < sourceDirectory.size()) ? - sourceDirectory.get(directoryIdx + 1).getOffset() : - record.getBuffers().getPayload().limit(); - int fieldLength = nextOffset - field.getOffset(); - - newDirectory.add(new DirectoryEntry(field.getId(), field.getTypeCode(), currentOffset)); - ranges.add(new FieldRange(field.getOffset(), nextOffset)); - - currentOffset += fieldLength; - fieldIdsIdx++; - } - directoryIdx++; - } - - // Build new payload from ranges - var newPayload = buildPayloadFromRanges(record.getBuffers().getPayload(), ranges); - - // Create new header with updated payload size - // TODO: compute correct schema hash - var newHeader = new Header(record.getHeader().getFlags(), - new SchemaId(record.getHeader().getSchemaId().getFieldSpaceId(), 0xdeadbeef), - newPayload.remaining() - ); - - return new ImprintRecord(newHeader, newDirectory, newPayload); - } - - /** - * Merge two Imprint records, combining their fields. Payload copying is proportional to total data size. - * - *

Merge Strategy:

- *
    - *
  • Fields are merged using sort-merge algorithm on directory entries
    • - *
  • For duplicate field IDs: first record's field takes precedence
    • - *
  • Payloads are concatenated with directory offsets adjusted
    • - *
  • Schema ID from first record is preserved
    • - *
- *

- * - * @param first The first record (takes precedence for duplicate fields) - * @param second The second record to merge - * @return New ImprintRecord containing merged fields - * @throws ImprintException if merge fails due to incompatible records - */ - public static ImprintRecord merge(ImprintRecord first, ImprintRecord second) throws ImprintException { - var firstDir = first.getDirectory(); - var secondDir = second.getDirectory(); - - // Pre-allocate for worst case (no overlapping fields) - var newDirectory = new ArrayList(firstDir.size() + secondDir.size()); - var payloadChunks = new ArrayList(); - - int firstIdx = 0; - int secondIdx = 0; - int currentOffset = 0; - - while (firstIdx < firstDir.size() || secondIdx < secondDir.size()) { - DirectoryEntry currentEntry; - ByteBuffer currentPayload; - - if (firstIdx < firstDir.size() && - (secondIdx >= secondDir.size() || firstDir.get(firstIdx).getId() <= secondDir.get(secondIdx).getId())) { - - // Take from first record - currentEntry = firstDir.get(firstIdx); - - // Skip duplicate field in second record if present - if (secondIdx < secondDir.size() && - firstDir.get(firstIdx).getId() == secondDir.get(secondIdx).getId()) { - secondIdx++; - } - - currentPayload = first.getRawBytes(currentEntry.getId()); - firstIdx++; - } else { - // Take from second record - currentEntry = secondDir.get(secondIdx); - currentPayload = second.getRawBytes(currentEntry.getId()); - secondIdx++; - } - - if (currentPayload == null) - throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, "Failed to get raw bytes for field " + currentEntry.getId()); - - // Add adjusted directory entry - var newEntry = new DirectoryEntry(currentEntry.getId(), currentEntry.getTypeCode(), currentOffset); - newDirectory.add(newEntry); - - // Collect payload chunk - payloadChunks.add(currentPayload.duplicate()); - currentOffset += currentPayload.remaining(); - } - - // Build merged payload - var mergedPayload = buildPayloadFromChunks(payloadChunks); - - // Create header preserving first record's schema ID - var newHeader = new Header(first.getHeader().getFlags(), first.getHeader().getSchemaId(), mergedPayload.remaining()); - - return new ImprintRecord(newHeader, newDirectory, mergedPayload); - } - - /** - * Represents a range of bytes to copy from source payload. - */ - @Value - private static class FieldRange { - int start; - int end; - - int length() { - return end - start; - } - } - - /** - * Build a new payload buffer from field ranges in the source payload. - */ - private static ByteBuffer buildPayloadFromRanges(ByteBuffer sourcePayload, List ranges) { - int totalSize = ranges.stream().mapToInt(FieldRange::length).sum(); - var newPayload = ByteBuffer.allocate(totalSize); - newPayload.order(ByteOrder.LITTLE_ENDIAN); - - for (var range : ranges) { - var sourceSlice = sourcePayload.duplicate(); - sourceSlice.position(range.start).limit(range.end); - newPayload.put(sourceSlice); - } - - newPayload.flip(); - return newPayload; - } - - /** - * Build a new payload buffer by concatenating chunks. - */ - private static ByteBuffer buildPayloadFromChunks(List chunks) { - int totalSize = chunks.stream().mapToInt(ByteBuffer::remaining).sum(); - var mergedPayload = ByteBuffer.allocate(totalSize); - mergedPayload.order(ByteOrder.LITTLE_ENDIAN); - - for (var chunk : chunks) { - mergedPayload.put(chunk); - } - - mergedPayload.flip(); - return mergedPayload; - } - - /** - * Create an empty record with the given schema ID. - */ - private static ImprintRecord createEmptyRecord(SchemaId schemaId) { - var header = new Header(new Flags((byte) 0x01), schemaId, 0); - return new ImprintRecord(header, Collections.emptyList(), ByteBuffer.allocate(0)); - } -} diff --git a/src/test/java/com/imprint/core/ImprintOperationsTest.java b/src/test/java/com/imprint/core/ImprintOperationsTest.java deleted file mode 100644 index 1dc67fb..0000000 --- a/src/test/java/com/imprint/core/ImprintOperationsTest.java +++ /dev/null @@ -1,405 +0,0 @@ -package com.imprint.core; - -import com.imprint.error.ImprintException; -import com.imprint.types.Value; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Nested; -import org.junit.jupiter.api.Test; - -import java.util.List; - -import static org.junit.jupiter.api.Assertions.*; - -@DisplayName("ImprintOperations") -class ImprintOperationsTest { - - private SchemaId testSchema; - private ImprintRecord multiFieldRecord; - private ImprintRecord emptyRecord; - - @BeforeEach - void setUp() throws ImprintException { - testSchema = new SchemaId(1, 0xdeadbeef); - multiFieldRecord = createTestRecord(); - emptyRecord = createEmptyTestRecord(); - } - - private ImprintRecord createTestRecord() throws ImprintException { - return ImprintRecord.builder(testSchema) - .field(1, 42) - .field(3, "hello") - .field(5, true) - .field(7, new byte[]{1, 2, 3}) - .build(); - } - - private ImprintRecord createEmptyTestRecord() throws ImprintException { - return ImprintRecord.builder(testSchema).build(); - } - - @Nested - @DisplayName("Project Operations") - class ProjectOperations { - - @Test - @DisplayName("should project subset of fields") - void shouldProjectSubsetOfFields() throws ImprintException { - // When projecting a subset of fields - ImprintRecord projected = ImprintOperations.project(multiFieldRecord, 1, 5); - - // Then only the requested fields should be present - assertEquals(2, projected.getDirectory().size()); - assertEquals(42, projected.getInt32(1)); - assertTrue(projected.getBoolean(5)); - - // And non-requested fields should be absent - assertNull(projected.getValue(3)); - assertNull(projected.getValue(7)); - } - - @Test - @DisplayName("should maintain field order regardless of input order") - void shouldMaintainFieldOrderRegardlessOfInputOrder() throws ImprintException { - // When projecting fields in arbitrary order - ImprintRecord projected = ImprintOperations.project(multiFieldRecord, 7, 1, 5, 3); - - // Then all requested fields should be present - assertEquals(4, projected.getDirectory().size()); - assertEquals(42, projected.getInt32(1)); - assertEquals("hello", projected.getString(3)); - assertTrue(projected.getBoolean(5)); - assertArrayEquals(new byte[]{1, 2, 3}, projected.getBytes(7)); - - // And directory should maintain sorted order - List directory = projected.getDirectory(); - for (int i = 1; i < directory.size(); i++) { - assertTrue(directory.get(i - 1).getId() < directory.get(i).getId(), - "Directory entries should be sorted by field id"); - } - } - - @Test - @DisplayName("should handle single field projection") - void shouldHandleSingleFieldProjection() throws ImprintException { - // When projecting a single field - ImprintRecord projected = ImprintOperations.project(multiFieldRecord, 3); - - // Then only that field should be present - assertEquals(1, projected.getDirectory().size()); - assertEquals("hello", projected.getString(3)); - } - - @Test - @DisplayName("should preserve all fields when projecting all") - void shouldPreserveAllFieldsWhenProjectingAll() throws ImprintException { - // Given all field IDs from the original record - int[] allFields = multiFieldRecord.getDirectory().stream() - .mapToInt(DirectoryEntry::getId) - .toArray(); - - // When projecting all fields - ImprintRecord projected = ImprintOperations.project(multiFieldRecord, allFields); - - // Then all fields should be present with matching values - assertEquals(multiFieldRecord.getDirectory().size(), projected.getDirectory().size()); - - for (DirectoryEntry entry : multiFieldRecord.getDirectory()) { - Value originalValue = multiFieldRecord.getValue(entry.getId()); - Value projectedValue = projected.getValue(entry.getId()); - assertEquals(originalValue, projectedValue, - "Field " + entry.getId() + " should have matching value"); - } - } - - @Test - @DisplayName("should handle empty projection") - void shouldHandleEmptyProjection() { - // When projecting no fields - ImprintRecord projected = ImprintOperations.project(multiFieldRecord); - - // Then result should be empty but valid - assertEquals(0, projected.getDirectory().size()); - assertEquals(0, projected.getBuffers().getPayload().remaining()); - } - - @Test - @DisplayName("should ignore nonexistent fields") - void shouldIgnoreNonexistentFields() throws ImprintException { - // When projecting mix of existing and non-existing fields - ImprintRecord projected = ImprintOperations.project(multiFieldRecord, 1, 99, 100); - - // Then only existing fields should be included - assertEquals(1, projected.getDirectory().size()); - assertEquals(42, projected.getInt32(1)); - assertNull(projected.getValue(99)); - assertNull(projected.getValue(100)); - } - - @Test - @DisplayName("should deduplicate requested fields") - void shouldDeduplicateRequestedFields() throws ImprintException { - // When projecting the same field multiple times - ImprintRecord projected = ImprintOperations.project(multiFieldRecord, 1, 1, 1); - - // Then field should only appear once - assertEquals(1, projected.getDirectory().size()); - assertEquals(42, projected.getInt32(1)); - } - - @Test - @DisplayName("should handle projection from empty record") - void shouldHandleProjectionFromEmptyRecord() { - // When projecting any fields from empty record - ImprintRecord projected = ImprintOperations.project(emptyRecord, 1, 2, 3); - - // Then result should be empty but valid - assertEquals(0, projected.getDirectory().size()); - assertEquals(0, projected.getBuffers().getPayload().remaining()); - } - - @Test - @DisplayName("should preserve exact byte representation") - void shouldPreserveExactByteRepresentation() throws ImprintException { - // Given a field's original bytes - byte[] originalBytes = multiFieldRecord.getBytes(7); - - // When projecting that field - ImprintRecord projected = ImprintOperations.project(multiFieldRecord, 7); - - // Then the byte representation should be exactly preserved - byte[] projectedBytes = projected.getBytes(7); - assertArrayEquals(originalBytes, projectedBytes, - "Byte representation should be identical"); - } - - @Test - @DisplayName("should reduce payload size when projecting subset") - void shouldReducePayloadSizeWhenProjectingSubset() throws ImprintException { - // Given a record with large and small fields - ImprintRecord largeRecord = ImprintRecord.builder(testSchema) - .field(1, 42) // 4 bytes - .field(2, "x".repeat(1000)) // ~1000+ bytes - .field(3, 123L) // 8 bytes - .field(4, new byte[500]) // 500+ bytes - .build(); - - int originalPayloadSize = largeRecord.getBuffers().getPayload().remaining(); - - // When projecting only the small fields - ImprintRecord projected = ImprintOperations.project(largeRecord, 1, 3); - - // Then the payload size should be significantly smaller - assertTrue(projected.getBuffers().getPayload().remaining() < originalPayloadSize, - "Projected payload should be smaller than original"); - - // And the values should still be correct - assertEquals(42, projected.getInt32(1)); - assertEquals(123L, projected.getInt64(3)); - } - } - - @Nested - @DisplayName("Merge Operations") - class MergeOperations { - - @Test - @DisplayName("should merge records with distinct fields") - void shouldMergeRecordsWithDistinctFields() throws ImprintException { - // Given two records with different fields - ImprintRecord record1 = ImprintRecord.builder(testSchema) - .field(1, 42) - .field(3, "hello") - .build(); - - ImprintRecord record2 = ImprintRecord.builder(testSchema) - .field(2, true) - .field(4, 123L) - .build(); - - // When merging the records - ImprintRecord merged = ImprintOperations.merge(record1, record2); - - // Then all fields should be present - assertEquals(4, merged.getDirectory().size()); - assertEquals(42, merged.getInt32(1)); - assertTrue(merged.getBoolean(2)); - assertEquals("hello", merged.getString(3)); - assertEquals(123L, merged.getInt64(4)); - - // And directory should be sorted - List directory = merged.getDirectory(); - for (int i = 1; i < directory.size(); i++) { - assertTrue(directory.get(i - 1).getId() < directory.get(i).getId(), - "Directory entries should be sorted by field id"); - } - } - - @Test - @DisplayName("should merge records with overlapping fields") - void shouldMergeRecordsWithOverlappingFields() throws ImprintException { - // Given two records with overlapping fields - ImprintRecord record1 = ImprintRecord.builder(testSchema) - .field(2, "first") - .field(3, 42) - .build(); - - ImprintRecord record2 = ImprintRecord.builder(testSchema) - .field(1, true) - .field(2, "second") // Overlapping field - .build(); - - // When merging the records - ImprintRecord merged = ImprintOperations.merge(record1, record2); - - // Then first record's values should take precedence for duplicates - assertEquals(3, merged.getDirectory().size()); - assertTrue(merged.getBoolean(1)); - assertEquals("first", merged.getString(2)); // First record wins - assertEquals(42, merged.getInt32(3)); - } - - @Test - @DisplayName("should preserve schema id from first record") - void shouldPreserveSchemaIdFromFirstRecord() throws ImprintException { - // Given two records with different schema IDs - SchemaId schema1 = new SchemaId(1, 0xdeadbeef); - SchemaId schema2 = new SchemaId(1, 0xcafebabe); - - ImprintRecord record1 = ImprintRecord.builder(schema1) - .field(1, 42) - .build(); - - ImprintRecord record2 = ImprintRecord.builder(schema2) - .field(2, true) - .build(); - - // When merging the records - ImprintRecord merged = ImprintOperations.merge(record1, record2); - - // Then schema ID from first record should be preserved - assertEquals(schema1, merged.getHeader().getSchemaId()); - } - - @Test - @DisplayName("should handle merge with empty record") - void shouldHandleMergeWithEmptyRecord() throws ImprintException { - // When merging with empty record - ImprintRecord merged1 = ImprintOperations.merge(multiFieldRecord, emptyRecord); - ImprintRecord merged2 = ImprintOperations.merge(emptyRecord, multiFieldRecord); - - // Then results should contain all original fields - assertEquals(multiFieldRecord.getDirectory().size(), merged1.getDirectory().size()); - assertEquals(multiFieldRecord.getDirectory().size(), merged2.getDirectory().size()); - - // And values should be preserved - for (DirectoryEntry entry : multiFieldRecord.getDirectory()) { - Value originalValue = multiFieldRecord.getValue(entry.getId()); - assertEquals(originalValue, merged1.getValue(entry.getId())); - assertEquals(originalValue, merged2.getValue(entry.getId())); - } - } - - @Test - @DisplayName("should handle merge of two empty records") - void shouldHandleMergeOfTwoEmptyRecords() throws ImprintException { - // When merging two empty records - ImprintRecord merged = ImprintOperations.merge(emptyRecord, emptyRecord); - - // Then result should be empty but valid - assertEquals(0, merged.getDirectory().size()); - assertEquals(0, merged.getBuffers().getPayload().remaining()); - } - - @Test - @DisplayName("should maintain correct payload offsets after merge") - void shouldMaintainCorrectPayloadOffsetsAfterMerge() throws ImprintException { - // Given records with different field sizes - ImprintRecord record1 = ImprintRecord.builder(testSchema) - .field(1, 42) // 4 bytes - .field(3, "hello") // 5+ bytes - .build(); - - ImprintRecord record2 = ImprintRecord.builder(testSchema) - .field(2, true) // 1 byte - .field(4, new byte[]{1, 2, 3, 4, 5}) // 5+ bytes - .build(); - - // When merging - ImprintRecord merged = ImprintOperations.merge(record1, record2); - - // Then all fields should be accessible with correct values - assertEquals(42, merged.getInt32(1)); - assertTrue(merged.getBoolean(2)); - assertEquals("hello", merged.getString(3)); - assertArrayEquals(new byte[]{1, 2, 3, 4, 5}, merged.getBytes(4)); - - // And directory offsets should be sequential - List directory = merged.getDirectory(); - int expectedOffset = 0; - for (DirectoryEntry entry : directory) { - assertEquals(expectedOffset, entry.getOffset(), - "Field " + entry.getId() + " should have correct offset"); - - // Calculate next offset - var fieldData = merged.getRawBytes(entry.getId()); - assertNotNull(fieldData); - expectedOffset += fieldData.remaining(); - } - } - - @Test - @DisplayName("should handle large records efficiently") - void shouldHandleLargeRecordsEfficiently() throws ImprintException { - // Given records with many fields - var builder1 = ImprintRecord.builder(testSchema); - var builder2 = ImprintRecord.builder(testSchema); - - // Add 100 fields to each record (no overlap) - for (int i = 1; i <= 100; i++) { - builder1.field(i, i * 10); - } - for (int i = 101; i <= 200; i++) { - builder2.field(i, i * 10); - } - - ImprintRecord record1 = builder1.build(); - ImprintRecord record2 = builder2.build(); - - // When merging large records - ImprintRecord merged = ImprintOperations.merge(record1, record2); - - // Then all 200 fields should be present and accessible - assertEquals(200, merged.getDirectory().size()); - - // Spot check some values - assertEquals(10, merged.getInt32(1)); - assertEquals(500, merged.getInt32(50)); - assertEquals(1000, merged.getInt32(100)); - assertEquals(1010, merged.getInt32(101)); - assertEquals(1500, merged.getInt32(150)); - assertEquals(2000, merged.getInt32(200)); - } - } - - @Nested - @DisplayName("Error Handling") - class ErrorHandling { - - @Test - @DisplayName("should handle null record gracefully") - void shouldHandleNullRecordGracefully() { - assertThrows(NullPointerException.class, () -> ImprintOperations.project(null, 1, 2, 3)); - - assertThrows(NullPointerException.class, () -> ImprintOperations.merge(null, multiFieldRecord)); - - assertThrows(NullPointerException.class, () -> ImprintOperations.merge(multiFieldRecord, null)); - } - - @Test - @DisplayName("should handle null field ids gracefully") - void shouldHandleNullFieldIdsGracefully() { - assertThrows(NullPointerException.class, () -> ImprintOperations.project(multiFieldRecord, (int[]) null)); - } - } -} From b8449c89a252a10cbaa0f38b75eb8463d44e1e71 Mon Sep 17 00:00:00 2001 From: expand3d <> Date: Thu, 12 Jun 2025 17:10:42 -0400 Subject: [PATCH 49/49] adding comments and TODOs --- .../imprint/core/ImprintFieldObjectMap.java | 15 ++++------ .../imprint/core/ImprintRecordBuilder.java | 6 ++-- .../com/imprint/ops/ImprintOperations.java | 28 +++++++++---------- 3 files changed, 22 insertions(+), 27 deletions(-) diff --git a/src/main/java/com/imprint/core/ImprintFieldObjectMap.java b/src/main/java/com/imprint/core/ImprintFieldObjectMap.java index d104317..e0a63f0 100644 --- a/src/main/java/com/imprint/core/ImprintFieldObjectMap.java +++ b/src/main/java/com/imprint/core/ImprintFieldObjectMap.java @@ -119,7 +119,7 @@ public IntStream streamKeys() { } /** - * Result holder for in-place sorted values - avoids allocation by returning + * Result holder for in-place sorted values - avoids Array.copy allocations by returning * array reference and valid count. */ public static final class SortedValuesResult { @@ -216,7 +216,7 @@ public T[] getSortedValuesCopy(T[] resultArray) { } /** - * Compact all non-empty entries to the front of keys/values arrays. + * Left side compact for all non-empty entries to the front of keys/values arrays. */ private void compactEntries() { int writeIndex = 0; @@ -237,7 +237,7 @@ private void compactEntries() { } /** - * Sort the first 'count' entries by key using insertion sort (should be fast for small arrays). + * Sort the first 'count' entries by key using insertion sort (should be fast enough for small arrays). */ private void sortEntriesByKey(int count) { for (int i = 1; i < count; i++) { @@ -281,8 +281,7 @@ private void resize() { int oldSize = size; size = 0; - - // Rehash all entries + for (int i = 0; i < oldKeys.length; i++) { if (oldKeys[i] != EMPTY_KEY) { @SuppressWarnings("unchecked") @@ -290,14 +289,12 @@ private void resize() { put(oldKeys[i], value); } } - - // Verify size didn't change during rehash + //TODO remove this assertion (carried from from EclipseCollection) assert size == oldSize; } private static int hash(short key) { - // Simple but effective hash for short keys - int intKey = key & 0xFFFF; // Convert to unsigned int + int intKey = key & 0xFFFF; intKey ^= intKey >>> 8; return intKey; } diff --git a/src/main/java/com/imprint/core/ImprintRecordBuilder.java b/src/main/java/com/imprint/core/ImprintRecordBuilder.java index 8e1dfa0..5b7f009 100644 --- a/src/main/java/com/imprint/core/ImprintRecordBuilder.java +++ b/src/main/java/com/imprint/core/ImprintRecordBuilder.java @@ -35,7 +35,6 @@ @SuppressWarnings("unused") public final class ImprintRecordBuilder { private final SchemaId schemaId; - // Custom int→object map optimized for primitive keys private final ImprintFieldObjectMap fields = new ImprintFieldObjectMap<>(); private int estimatedPayloadSize = 0; @@ -203,13 +202,13 @@ private ImprintRecordBuilder addField(int id, Value value) { Objects.requireNonNull(value, "Value cannot be null - use nullField() for explicit null values"); var newEntry = new FieldData((short) id, value); - // Check if replacing an existing field - O(1) lookup without boxing! + // Check if replacing an existing field var oldEntry = fields.get(id); if (oldEntry != null) { estimatedPayloadSize -= estimateValueSize(oldEntry.value); } - // Add or replace field - O(1) operation without boxing! + // Add or replace field fields.put(id, newEntry); estimatedPayloadSize += estimateValueSize(newEntry.value); return this; @@ -327,7 +326,6 @@ private int estimateValueSize(Value value) { } private void serializeValue(Value value, ByteBuffer buffer) throws ImprintException { - // Use TypeHandler for simple types switch (value.getTypeCode()) { case NULL: case BOOL: diff --git a/src/main/java/com/imprint/ops/ImprintOperations.java b/src/main/java/com/imprint/ops/ImprintOperations.java index f15e6a1..52ec5a0 100644 --- a/src/main/java/com/imprint/ops/ImprintOperations.java +++ b/src/main/java/com/imprint/ops/ImprintOperations.java @@ -27,8 +27,9 @@ public class ImprintOperations { public static ByteBuffer mergeBytes(ByteBuffer firstBuffer, ByteBuffer secondBuffer) throws ImprintException { validateImprintBuffer(firstBuffer, "firstBuffer"); validateImprintBuffer(secondBuffer, "secondBuffer"); - - // Work on duplicates to avoid affecting original positions + + // TODO possible could work directly on the originals but duplicate makes the mark values and offsets easy to reason about + // duplicates to avoid affecting original positions, we'll need to preserve at least one side var first = firstBuffer.duplicate().order(ByteOrder.LITTLE_ENDIAN); var second = secondBuffer.duplicate().order(ByteOrder.LITTLE_ENDIAN); @@ -60,6 +61,7 @@ private static ImprintRecord.BufferSections extractSections(ByteBuffer buffer, H /** * Merge raw directory and payload sections without object creation + * Assumes incoming streams are already both sorted from the serialization process */ private static ByteBuffer mergeRawSections(Header firstHeader, ImprintRecord.BufferSections firstSections, ImprintRecord.BufferSections secondSections) throws ImprintException { // Prepare directory iterators @@ -142,12 +144,11 @@ public static ByteBuffer projectBytes(ByteBuffer sourceBuffer, int... fieldIds) if (fieldIds == null || fieldIds.length == 0) { return createEmptyRecordBytes(); } - - // Sort field IDs for efficient merge algorithm (duplicates handled naturally) + var sortedFieldIds = fieldIds.clone(); Arrays.sort(sortedFieldIds); - // Work on duplicate to avoid affecting original position + // Duplicate avoids affecting original position which we'll need later var source = sourceBuffer.duplicate().order(ByteOrder.LITTLE_ENDIAN); // Parse header @@ -162,7 +163,6 @@ public static ByteBuffer projectBytes(ByteBuffer sourceBuffer, int... fieldIds) /** * Project raw sections without object creation using optimized merge algorithm. - * Uses direct array operations and optimized memory access for maximum performance. */ private static ByteBuffer projectRawSections(Header originalHeader, ImprintRecord.BufferSections sections, int[] sortedRequestedFields) throws ImprintException { @@ -194,14 +194,14 @@ private static ByteBuffer projectRawSections(Header originalHeader, ImprintRecor // Add to projection with adjusted offset projectedEntries.add(new RawDirectoryEntry(currentEntry.fieldId, currentEntry.typeCode, currentOffset)); - // Collect payload chunk here (fieldPayload is already sliced) + // Collect payload chunk here - fieldPayload should already sliced payloadChunks.add(fieldPayload); int payloadSize = fieldPayload.remaining(); currentOffset += payloadSize; totalProjectedPayloadSize += payloadSize; - // Advance both pointers (handle dupes by advancing to next unique field) + // Advance both pointers - handle dupes by advancing to next unique field hopefully do { requestedIndex++; } while (requestedIndex < sortedRequestedFields.length && sortedRequestedFields[requestedIndex] == targetFieldId); @@ -232,7 +232,7 @@ private static ByteBuffer buildSerializedBuffer(Header originalHeader, List