From 5cb33c9e9d9a2967a1e1cc753e3c308e9ef9e739 Mon Sep 17 00:00:00 2001 From: Brent Johnson Date: Sun, 1 Jun 2025 13:23:15 -0400 Subject: [PATCH 01/34] Update .gitignore --- .gitignore | 76 +++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 58 insertions(+), 18 deletions(-) diff --git a/.gitignore b/.gitignore index 524f096..09ea567 100644 --- a/.gitignore +++ b/.gitignore @@ -1,24 +1,64 @@ -# Compiled class file -*.class +# Gradle +.gradle/ +build/ +gradle-app.setting +!gradle-wrapper.jar +!gradle-wrapper.properties -# Log file +# IntelliJ IDEA +.idea/ +*.iml +*.ipr +*.iws + +# Eclipse +.project +.classpath +.settings/ +bin/ + +# NetBeans +nbproject/private/ +build/ +nbbuild/ +dist/ +nbdist/ +.nb-gradle/ + +# VS Code +.vscode/ + +# macOS +.DS_Store + +# Windows +Thumbs.db +ehthumbs.db + +# JMH +jmh-result.json + +# Logs *.log -# BlueJ files -*.ctxt +# Temporary files +*.tmp +*.temp -# Mobile Tools for Java (J2ME) -.mtj.tmp/ +# JVM Crash logs +hs_err_pid* -# Package Files # +# Maven (in case someone uses Maven) +target/ +pom.xml.tag +pom.xml.releaseBackup +pom.xml.versionsBackup +pom.xml.next +release.properties +dependency-reduced-pom.xml +buildNumber.properties +.mvn/timing.properties + +# JAR files (unless they're dependencies) *.jar -*.war -*.nar -*.ear -*.zip -*.tar.gz -*.rar - -# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml -hs_err_pid* -replay_pid* +!lombok.jar From 63313e4d81404b40ec649369286b47856574a002 Mon Sep 17 00:00:00 2001 From: expand3d <> Date: Sun, 1 Jun 2025 14:02:51 -0400 Subject: [PATCH 02/34] initial commit for imprint-java --- PROFILING.md | 106 +++++ gradle/wrapper/gradle-wrapper.properties | 7 + gradlew | 251 +++++++++++ gradlew.bat | 94 ++++ profile.sh | 98 +++++ settings.gradle | 1 + .../benchmark/ComparisonBenchmark.java | 344 +++++++++++++++ .../benchmark/FieldAccessBenchmark.java | 276 ++++++++++++ .../com/imprint/benchmark/MergeBenchmark.java | 163 +++++++ .../benchmark/SerializationBenchmark.java | 166 ++++++++ src/main/java/com/imprint/Constants.java | 10 + .../java/com/imprint/core/DirectoryEntry.java | 23 + src/main/java/com/imprint/core/Flags.java | 12 + src/main/java/com/imprint/core/Header.java | 13 + .../java/com/imprint/core/ImprintRecord.java | 365 ++++++++++++++++ .../imprint/core/ImprintRecordBuilder.java | 233 ++++++++++ .../java/com/imprint/core/ImprintWriter.java | 257 +++++++++++ src/main/java/com/imprint/core/SchemaId.java | 12 + .../java/com/imprint/error/ErrorType.java | 18 + .../com/imprint/error/ImprintException.java | 26 ++ src/main/java/com/imprint/types/MapKey.java | 163 +++++++ src/main/java/com/imprint/types/TypeCode.java | 48 +++ .../java/com/imprint/types/TypeHandler.java | 304 +++++++++++++ src/main/java/com/imprint/types/Value.java | 403 ++++++++++++++++++ src/main/java/com/imprint/util/VarInt.java | 118 +++++ .../imprint/ByteBufferIntegrationTest.java | 87 ++++ .../java/com/imprint/ComprehensiveTest.java | 208 +++++++++ .../java/com/imprint/IntegrationTest.java | 144 +++++++ .../com/imprint/benchmark/ProfilerTest.java | 226 ++++++++++ .../core/ImprintRecordBuilderTest.java | 234 ++++++++++ .../com/imprint/core/ImprintRecordTest.java | 232 ++++++++++ .../java/com/imprint/types/MapKeyTest.java | 91 ++++ .../com/imprint/types/TypeHandlerTest.java | 279 ++++++++++++ .../java/com/imprint/types/ValueTest.java | 123 ++++++ .../java/com/imprint/util/VarIntTest.java | 115 +++++ 35 files changed, 5250 insertions(+) create mode 100644 PROFILING.md create mode 100644 gradle/wrapper/gradle-wrapper.properties create mode 100644 gradlew create mode 100644 gradlew.bat create mode 100644 profile.sh create mode 100644 settings.gradle create mode 100644 src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java create mode 100644 src/jmh/java/com/imprint/benchmark/FieldAccessBenchmark.java create mode 100644 src/jmh/java/com/imprint/benchmark/MergeBenchmark.java create mode 100644 src/jmh/java/com/imprint/benchmark/SerializationBenchmark.java create mode 100644 src/main/java/com/imprint/Constants.java create mode 100644 src/main/java/com/imprint/core/DirectoryEntry.java create mode 100644 src/main/java/com/imprint/core/Flags.java create mode 100644 src/main/java/com/imprint/core/Header.java create mode 100644 src/main/java/com/imprint/core/ImprintRecord.java create mode 100644 src/main/java/com/imprint/core/ImprintRecordBuilder.java create mode 100644 src/main/java/com/imprint/core/ImprintWriter.java create mode 100644 src/main/java/com/imprint/core/SchemaId.java create mode 100644 src/main/java/com/imprint/error/ErrorType.java create mode 100644 src/main/java/com/imprint/error/ImprintException.java create mode 100644 src/main/java/com/imprint/types/MapKey.java create mode 100644 src/main/java/com/imprint/types/TypeCode.java create mode 100644 src/main/java/com/imprint/types/TypeHandler.java create mode 100644 src/main/java/com/imprint/types/Value.java create mode 100644 src/main/java/com/imprint/util/VarInt.java create mode 100644 src/test/java/com/imprint/ByteBufferIntegrationTest.java create mode 100644 src/test/java/com/imprint/ComprehensiveTest.java create mode 100644 src/test/java/com/imprint/IntegrationTest.java create mode 100644 src/test/java/com/imprint/benchmark/ProfilerTest.java create mode 100644 src/test/java/com/imprint/core/ImprintRecordBuilderTest.java create mode 100644 src/test/java/com/imprint/core/ImprintRecordTest.java create mode 100644 src/test/java/com/imprint/types/MapKeyTest.java create mode 100644 src/test/java/com/imprint/types/TypeHandlerTest.java create mode 100644 src/test/java/com/imprint/types/ValueTest.java create mode 100644 src/test/java/com/imprint/util/VarIntTest.java diff --git a/PROFILING.md b/PROFILING.md new file mode 100644 index 0000000..43f7b4e --- /dev/null +++ b/PROFILING.md @@ -0,0 +1,106 @@ +# Performance Profiling Guide + +This guide helps developers identify performance hotspots in the Imprint Java implementation. + +## Quick Start + +```bash +# Run field access profiling with async-profiler +./profile.sh profileFieldAccess asyncprofiler + +# Run memory allocation profiling with JFR +./profile.sh profileMemoryAllocation jfr +``` + +## Available Tests + +1. **`profileFieldAccess`** - Measures random field access patterns + - Focus: Binary search, TypeHandler dispatch, string decoding + - Good for: Optimizing read-heavy workloads + +2. **`profileSerialization`** - Tests record creation and serialization + - Focus: Object allocation, ByteBuffer operations, encoding + - Good for: Optimizing write-heavy workloads + +3. **`profileProjection`** - Simulates analytical field projection + - Focus: Bulk field access, string materialization + - Good for: Optimizing analytical workloads + +4. **`profileMemoryAllocation`** - Stress tests allocation patterns + - Focus: GC pressure, object lifecycle, string allocations + - Good for: Reducing memory footprint + +## Profiler Options + +### async-profiler (Recommended) +- **Setup**: Download from [async-profiler releases](https://github.com/jvm-profiling-tools/async-profiler/releases) +- **Output**: HTML flame graphs in `profiler-results/` +- **Best for**: CPU profiling, finding hot methods + +### Java Flight Recorder (JFR) +- **Setup**: Built into OpenJDK 11+ +- **Output**: `.jfr` files for Java Mission Control +- **Best for**: Memory profiling, GC analysis + +### VisualVM +- **Setup**: `jvisualvm` (usually pre-installed) +- **Output**: Real-time profiling UI +- **Best for**: Interactive profiling, heap dumps + +## Expected Hotspots + +Based on our optimizations, watch for: + +### CPU Hotspots +1. **Binary search** in `findDirectoryIndex()` - should be fast +2. **String decoding** in `StringBufferValue.getValue()` - lazy evaluation +3. **TypeHandler dispatch** - interface calls vs switch statements +4. **VarInt encoding/decoding** - variable-length integers +5. **ByteBuffer operations** - slicing and positioning + +### Memory Hotspots +1. **String allocations** during UTF-8 conversion +2. **Temporary objects** in binary search (should be eliminated) +3. **ByteBuffer slicing** (should be zero-copy) +4. **Array allocations** for BYTES values + +## Analyzing Results + +### async-profiler Flame Graphs +- **Wide bars** = high CPU usage (hotspots) +- **Deep stacks** = call overhead +- **Look for**: Red bars in `deserializeValue`, `findDirectoryIndex`, string operations + +### JFR Analysis +1. Open `.jfr` file in Java Mission Control +2. Check "Memory" tab for allocation hotspots +3. Check "Method Profiling" for CPU usage +4. Look at GC events for memory pressure + +### Memory Profiler Tips +- **Object allocation rate** should be low for zero-copy operations +- **String allocations** should be rare (lazy evaluation) +- **GC frequency** indicates allocation pressure + +## Performance Targets + +Based on our benchmarks: +- **Single field access**: < 50ns +- **Zero-copy operations**: < 30ns +- **String decoding**: Should be lazy, not in hot path +- **Binary search**: O(log n), ~10ns per comparison + +## Common Issues + +1. **High string allocation** → Enable lazy string decoding +2. **Object allocations in binary search** → Check DirectoryEntry creation +3. **ByteBuffer copying** → Ensure zero-copy slicing +4. **Switch statement overhead** → TypeHandler dispatch working? + +## Profiling Best Practices + +1. **Warm up JVM** - Run tests multiple times +2. **Use realistic data** - Match production patterns +3. **Profile different scenarios** - Read vs write heavy +4. **Check allocations** - Memory profiling reveals hidden costs +5. **Compare before/after** - Measure optimization impact \ No newline at end of file diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 0000000..37f853b --- /dev/null +++ b/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,7 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-8.13-bin.zip +networkTimeout=10000 +validateDistributionUrl=true +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/gradlew b/gradlew new file mode 100644 index 0000000..faf9300 --- /dev/null +++ b/gradlew @@ -0,0 +1,251 @@ +#!/bin/sh + +# +# Copyright © 2015-2021 the original authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# + +############################################################################## +# +# Gradle start up script for POSIX generated by Gradle. +# +# Important for running: +# +# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is +# noncompliant, but you have some other compliant shell such as ksh or +# bash, then to run this script, type that shell name before the whole +# command line, like: +# +# ksh Gradle +# +# Busybox and similar reduced shells will NOT work, because this script +# requires all of these POSIX shell features: +# * functions; +# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», +# «${var#prefix}», «${var%suffix}», and «$( cmd )»; +# * compound commands having a testable exit status, especially «case»; +# * various built-in commands including «command», «set», and «ulimit». +# +# Important for patching: +# +# (2) This script targets any POSIX shell, so it avoids extensions provided +# by Bash, Ksh, etc; in particular arrays are avoided. +# +# The "traditional" practice of packing multiple parameters into a +# space-separated string is a well documented source of bugs and security +# problems, so this is (mostly) avoided, by progressively accumulating +# options in "$@", and eventually passing that to Java. +# +# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, +# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; +# see the in-line comments for details. +# +# There are tweaks for specific operating systems such as AIX, CygWin, +# Darwin, MinGW, and NonStop. +# +# (3) This script is generated from the Groovy template +# https://github.com/gradle/gradle/blob/HEAD/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# within the Gradle project. +# +# You can find Gradle at https://github.com/gradle/gradle/. +# +############################################################################## + +# Attempt to set APP_HOME + +# Resolve links: $0 may be a link +app_path=$0 + +# Need this for daisy-chained symlinks. +while + APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path + [ -h "$app_path" ] +do + ls=$( ls -ld "$app_path" ) + link=${ls#*' -> '} + case $link in #( + /*) app_path=$link ;; #( + *) app_path=$APP_HOME$link ;; + esac +done + +# This is normally unused +# shellcheck disable=SC2034 +APP_BASE_NAME=${0##*/} +# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) +APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s\n' "$PWD" ) || exit + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD=maximum + +warn () { + echo "$*" +} >&2 + +die () { + echo + echo "$*" + echo + exit 1 +} >&2 + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "$( uname )" in #( + CYGWIN* ) cygwin=true ;; #( + Darwin* ) darwin=true ;; #( + MSYS* | MINGW* ) msys=true ;; #( + NONSTOP* ) nonstop=true ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD=$JAVA_HOME/jre/sh/java + else + JAVACMD=$JAVA_HOME/bin/java + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD=java + if ! command -v java >/dev/null 2>&1 + then + die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +fi + +# Increase the maximum file descriptors if we can. +if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then + case $MAX_FD in #( + max*) + # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + MAX_FD=$( ulimit -H -n ) || + warn "Could not query maximum file descriptor limit" + esac + case $MAX_FD in #( + '' | soft) :;; #( + *) + # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + ulimit -n "$MAX_FD" || + warn "Could not set maximum file descriptor limit to $MAX_FD" + esac +fi + +# Collect all arguments for the java command, stacking in reverse order: +# * args from the command line +# * the main class name +# * -classpath +# * -D...appname settings +# * --module-path (only if needed) +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. + +# For Cygwin or MSYS, switch paths to Windows format before running java +if "$cygwin" || "$msys" ; then + APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) + CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) + + JAVACMD=$( cygpath --unix "$JAVACMD" ) + + # Now convert the arguments - kludge to limit ourselves to /bin/sh + for arg do + if + case $arg in #( + -*) false ;; # don't mess with options #( + /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath + [ -e "$t" ] ;; #( + *) false ;; + esac + then + arg=$( cygpath --path --ignore --mixed "$arg" ) + fi + # Roll the args list around exactly as many times as the number of + # args, so each arg winds up back in the position where it started, but + # possibly modified. + # + # NB: a `for` loop captures its iteration list before it begins, so + # changing the positional parameters here affects neither the number of + # iterations, nor the values presented in `arg`. + shift # remove old arg + set -- "$@" "$arg" # push replacement arg + done +fi + + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Collect all arguments for the java command: +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, +# and any embedded shellness will be escaped. +# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be +# treated as '${Hostname}' itself on the command line. + +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -classpath "$CLASSPATH" \ + org.gradle.wrapper.GradleWrapperMain \ + "$@" + +# Stop when "xargs" is not available. +if ! command -v xargs >/dev/null 2>&1 +then + die "xargs is not available" +fi + +# Use "xargs" to parse quoted args. +# +# With -n1 it outputs one arg per line, with the quotes and backslashes removed. +# +# In Bash we could simply go: +# +# readarray ARGS < <( xargs -n1 <<<"$var" ) && +# set -- "${ARGS[@]}" "$@" +# +# but POSIX shell has neither arrays nor command substitution, so instead we +# post-process each arg (as a line of input to sed) to backslash-escape any +# character that might be a shell metacharacter, then use eval to reverse +# that process (while maintaining the separation between arguments), and wrap +# the whole thing up as a single "set" statement. +# +# This will of course break if any of these variables contains a newline or +# an unmatched quote. +# + +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | + tr '\n' ' ' + )" '"$@"' + +exec "$JAVACMD" "$@" diff --git a/gradlew.bat b/gradlew.bat new file mode 100644 index 0000000..9d21a21 --- /dev/null +++ b/gradlew.bat @@ -0,0 +1,94 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem +@rem SPDX-License-Identifier: Apache-2.0 +@rem + +@if "%DEBUG%"=="" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%"=="" set DIRNAME=. +@rem This is normally unused +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if %ERRORLEVEL% equ 0 goto execute + +echo. 1>&2 +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. 1>&2 +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 + +goto fail + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* + +:end +@rem End local scope for the variables with windows NT shell +if %ERRORLEVEL% equ 0 goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +set EXIT_CODE=%ERRORLEVEL% +if %EXIT_CODE% equ 0 set EXIT_CODE=1 +if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% +exit /b %EXIT_CODE% + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/profile.sh b/profile.sh new file mode 100644 index 0000000..1e618af --- /dev/null +++ b/profile.sh @@ -0,0 +1,98 @@ +#!/bin/bash + +# Profiling helper script for Imprint Java implementation +# +# Usage: +# ./profile.sh [test_method] [profiler] +# +# test_method: profileFieldAccess, profileSerialization, profileProjection, profileMemoryAllocation +# profiler: asyncprofiler, jfr, visualvm +# +# Examples: +# ./profile.sh profileFieldAccess asyncprofiler +# ./profile.sh profileSerialization jfr +# ./profile.sh profileMemoryAllocation + +set -e + +TEST_METHOD=${1:-profileFieldAccess} +PROFILER=${2:-asyncprofiler} + +echo "🔬 Starting profiling session for $TEST_METHOD using $PROFILER" + +# Enable the profiler test by removing @Disabled +sed -i 's/@Disabled.*/@Test/' src/test/java/com/imprint/benchmark/ProfilerTest.java + +case $PROFILER in + "asyncprofiler") + echo "📊 Using async-profiler (download from https://github.com/jvm-profiling-tools/async-profiler)" + echo " Will generate CPU profile in profiler-results/" + mkdir -p profiler-results + + # Run test in background and profile it + ./gradlew test --tests "*ProfilerTest.$TEST_METHOD" \ + -Dorg.gradle.jvmargs="-XX:+UnlockDiagnosticVMOptions -XX:+DebugNonSafepoints" & + + TEST_PID=$! + sleep 2 + + # Find the actual Java process (Gradle daemon) + JAVA_PID=$(pgrep -f "ProfilerTest.$TEST_METHOD" | head -1) + + if [ -n "$JAVA_PID" ]; then + echo " Profiling Java process $JAVA_PID" + if command -v async-profiler.jar >/dev/null 2>&1; then + java -jar async-profiler.jar -d 30 -f profiler-results/profile-$TEST_METHOD.html $JAVA_PID + else + echo " ⚠️ async-profiler.jar not found in PATH" + echo " 📥 Download from: https://github.com/jvm-profiling-tools/async-profiler/releases" + fi + fi + + wait $TEST_PID + ;; + + "jfr") + echo "📊 Using Java Flight Recorder" + mkdir -p profiler-results + + ./gradlew test --tests "*ProfilerTest.$TEST_METHOD" \ + -Dorg.gradle.jvmargs="-XX:+FlightRecorder -XX:StartFlightRecording=duration=60s,filename=profiler-results/profile-$TEST_METHOD.jfr,settings=profile" + + echo " 📂 JFR file saved to: profiler-results/profile-$TEST_METHOD.jfr" + echo " 🔍 Open with: jmc profiler-results/profile-$TEST_METHOD.jfr" + ;; + + "visualvm") + echo "📊 Using VisualVM" + echo " 1. Start VisualVM: jvisualvm" + echo " 2. Enable the ProfilerTest manually" + echo " 3. Run: ./gradlew test --tests '*ProfilerTest.$TEST_METHOD' --debug-jvm" + echo " 4. Attach VisualVM to the Gradle daemon process" + echo " 5. Start CPU/Memory profiling" + + read -p "Press Enter when VisualVM is ready..." + ./gradlew test --tests "*ProfilerTest.$TEST_METHOD" --debug-jvm + ;; + + *) + echo "❌ Unknown profiler: $PROFILER" + echo " Supported: asyncprofiler, jfr, visualvm" + exit 1 + ;; +esac + +# Restore @Disabled annotation +sed -i 's/@Test/@Disabled("Enable manually for profiling")/' src/test/java/com/imprint/benchmark/ProfilerTest.java + +echo "✅ Profiling complete!" +echo "" +echo "🔍 Key areas to examine:" +echo " • Object allocation hotspots (new, arrays, strings)" +echo " • ByteBuffer operations and slicing" +echo " • String UTF-8 encoding/decoding" +echo " • Binary search in directory lookup" +echo " • TypeHandler method dispatch" +echo " • VarInt encoding/decoding" +echo "" +echo "📊 Profile results in: profiler-results/" \ No newline at end of file diff --git a/settings.gradle b/settings.gradle new file mode 100644 index 0000000..0568c52 --- /dev/null +++ b/settings.gradle @@ -0,0 +1 @@ +rootProject.name = 'imprint-java' diff --git a/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java b/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java new file mode 100644 index 0000000..1293478 --- /dev/null +++ b/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java @@ -0,0 +1,344 @@ +package com.imprint.benchmark; + +import com.esotericsoftware.kryo.Kryo; +import com.esotericsoftware.kryo.io.Input; +import com.esotericsoftware.kryo.io.Output; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.imprint.core.ImprintRecord; +import com.imprint.core.ImprintWriter; +import com.imprint.core.SchemaId; +import com.imprint.types.MapKey; +import com.imprint.types.Value; +import org.openjdk.jmh.annotations.*; +import org.openjdk.jmh.infra.Blackhole; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.nio.ByteBuffer; +import java.util.*; +import java.util.concurrent.TimeUnit; + +/** + * Head-to-head benchmarks comparing Imprint against other serialization libraries. + * Tests the performance claims made in the documentation. + */ +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@State(Scope.Benchmark) +@Warmup(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS) +@Fork(1) +public class ComparisonBenchmark { + + // Test data + private TestRecord testData; + + // Serialized formats + private ByteBuffer imprintBytes; + private byte[] jacksonBytes; + private byte[] kryoBytes; + + // Library instances + private ObjectMapper jackson; + private Kryo kryo; + + @Setup + public void setup() throws Exception { + testData = createTestRecord(); + + // Initialize libraries + jackson = new ObjectMapper(); + kryo = new Kryo(); + kryo.register(TestRecord.class); + kryo.register(ArrayList.class); + kryo.register(HashMap.class); + + // Pre-serialize for deserialization benchmarks + imprintBytes = serializeWithImprint(testData); + jacksonBytes = serializeWithJackson(testData); + kryoBytes = serializeWithKryo(testData); + } + + // ===== SERIALIZATION BENCHMARKS ===== + + @Benchmark + public void serializeImprint(Blackhole bh) throws Exception { + ByteBuffer result = serializeWithImprint(testData); + bh.consume(result); + } + + @Benchmark + public void serializeJackson(Blackhole bh) throws Exception { + byte[] result = serializeWithJackson(testData); + bh.consume(result); + } + + @Benchmark + public void serializeKryo(Blackhole bh) throws Exception { + byte[] result = serializeWithKryo(testData); + bh.consume(result); + } + + // ===== DESERIALIZATION BENCHMARKS ===== + + @Benchmark + public void deserializeImprint(Blackhole bh) throws Exception { + ImprintRecord result = ImprintRecord.deserialize(imprintBytes.duplicate()); + bh.consume(result); + } + + @Benchmark + public void deserializeJackson(Blackhole bh) throws Exception { + TestRecord result = jackson.readValue(jacksonBytes, TestRecord.class); + bh.consume(result); + } + + @Benchmark + public void deserializeKryo(Blackhole bh) { + Input input = new Input(new ByteArrayInputStream(kryoBytes)); + TestRecord result = kryo.readObject(input, TestRecord.class); + input.close(); + bh.consume(result); + } + + // ===== FIELD ACCESS BENCHMARKS ===== + + @Benchmark + public void fieldAccessImprint(Blackhole bh) throws Exception { + ImprintRecord record = ImprintRecord.deserialize(imprintBytes.duplicate()); + + // Access multiple fields without full deserialization + var id = record.getValue(1); + var name = record.getValue(2); + var price = record.getValue(3); + var active = record.getValue(4); + var category = record.getValue(5); + + bh.consume(id); + bh.consume(name); + bh.consume(price); + bh.consume(active); + bh.consume(category); + } + + @Benchmark + public void fieldAccessJackson(Blackhole bh) throws Exception { + // Jackson requires full deserialization to access fields + TestRecord record = jackson.readValue(jacksonBytes, TestRecord.class); + + bh.consume(record.id); + bh.consume(record.name); + bh.consume(record.price); + bh.consume(record.active); + bh.consume(record.category); + } + + @Benchmark + public void fieldAccessKryo(Blackhole bh) { + // Kryo requires full deserialization to access fields + Input input = new Input(new ByteArrayInputStream(kryoBytes)); + TestRecord record = kryo.readObject(input, TestRecord.class); + input.close(); + + bh.consume(record.id); + bh.consume(record.name); + bh.consume(record.price); + bh.consume(record.active); + bh.consume(record.category); + } + + // ===== SIZE COMPARISON ===== + + @Benchmark + public void measureImprintSize(Blackhole bh) throws Exception { + ByteBuffer serialized = serializeWithImprint(testData); + bh.consume(serialized.remaining()); + } + + @Benchmark + public void measureJacksonSize(Blackhole bh) throws Exception { + byte[] serialized = serializeWithJackson(testData); + bh.consume(serialized.length); + } + + @Benchmark + public void measureKryoSize(Blackhole bh) throws Exception { + byte[] serialized = serializeWithKryo(testData); + bh.consume(serialized.length); + } + + // ===== MERGE SIMULATION BENCHMARKS ===== + + @Benchmark + public void mergeImprint(Blackhole bh) throws Exception { + // Simulate merge with Imprint (O(1) with proper API) + var record1 = serializeWithImprint(testData); + var record2 = serializeWithImprint(createTestRecord2()); + + // Current simulation - will be O(1) with actual merge API + var deserialized1 = ImprintRecord.deserialize(record1); + var deserialized2 = ImprintRecord.deserialize(record2); + var merged = simulateMerge(deserialized1, deserialized2); + + bh.consume(merged); + } + + @Benchmark + public void mergeJackson(Blackhole bh) throws Exception { + // Jackson merge requires full deserialization + merge + serialization + var record1 = jackson.readValue(jacksonBytes, TestRecord.class); + var record2 = jackson.readValue(serializeWithJackson(createTestRecord2()), TestRecord.class); + + var merged = mergeTestRecords(record1, record2); + byte[] result = jackson.writeValueAsBytes(merged); + + bh.consume(result); + } + + @Benchmark + public void mergeKryo(Blackhole bh) throws Exception { + // Kryo merge requires full deserialization + merge + serialization + Input input1 = new Input(new ByteArrayInputStream(kryoBytes)); + var record1 = kryo.readObject(input1, TestRecord.class); + input1.close(); + + Input input2 = new Input(new ByteArrayInputStream(serializeWithKryo(createTestRecord2()))); + var record2 = kryo.readObject(input2, TestRecord.class); + input2.close(); + + var merged = mergeTestRecords(record1, record2); + byte[] result = serializeWithKryo(merged); + + bh.consume(result); + } + + // ===== HELPER METHODS ===== + + private ByteBuffer serializeWithImprint(TestRecord data) throws Exception { + var writer = new ImprintWriter(new SchemaId(1, 0x12345678)); + + writer.addField(1, Value.fromInt32(data.id)); + writer.addField(2, Value.fromString(data.name)); + writer.addField(3, Value.fromFloat64(data.price)); + writer.addField(4, Value.fromBoolean(data.active)); + writer.addField(5, Value.fromString(data.category)); + + // Convert tags list + var tagValues = new ArrayList(); + for (String tag : data.tags) { + tagValues.add(Value.fromString(tag)); + } + writer.addField(6, Value.fromArray(tagValues)); + + // Convert metadata map + var metadataMap = new HashMap(); + for (var entry : data.metadata.entrySet()) { + metadataMap.put(MapKey.fromString(entry.getKey()), Value.fromString(entry.getValue())); + } + writer.addField(7, Value.fromMap(metadataMap)); + + return writer.build().serializeToBuffer(); + } + + private byte[] serializeWithJackson(TestRecord data) throws Exception { + return jackson.writeValueAsBytes(data); + } + + private byte[] serializeWithKryo(TestRecord data) { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + Output output = new Output(baos); + kryo.writeObject(output, data); + output.close(); + return baos.toByteArray(); + } + + private ImprintRecord simulateMerge(ImprintRecord first, ImprintRecord second) throws Exception { + var writer = new ImprintWriter(first.getHeader().getSchemaId()); + var usedFieldIds = new HashSet(); + + // Copy fields from first record (takes precedence) + copyFieldsToWriter(first, writer, usedFieldIds); + copyFieldsToWriter(second, writer, usedFieldIds); + + return writer.build(); + } + + private void copyFieldsToWriter(ImprintRecord record, ImprintWriter writer, Set usedFieldIds) throws Exception { + for (var entry : record.getDirectory()) { + int fieldId = entry.getId(); + if (!usedFieldIds.contains(fieldId)) { + var value = record.getValue(fieldId); + if (value.isPresent()) { + writer.addField(fieldId, value.get()); + usedFieldIds.add(fieldId); + } + } + } + } + + private TestRecord mergeTestRecords(TestRecord first, TestRecord second) { + // Simple merge logic - first record takes precedence + var merged = new TestRecord(); + merged.id = first.id; + merged.name = first.name != null ? first.name : second.name; + merged.price = first.price != 0.0 ? first.price : second.price; + merged.active = first.active; + merged.category = first.category != null ? first.category : second.category; + + merged.tags = new ArrayList<>(first.tags); + merged.tags.addAll(second.tags); + + merged.metadata = new HashMap<>(first.metadata); + merged.metadata.putAll(second.metadata); + + return merged; + } + + private TestRecord createTestRecord() { + var record = new TestRecord(); + record.id = 12345; + record.name = "Test Product"; + record.price = 99.99; + record.active = true; + record.category = "Electronics"; + + record.tags = Arrays.asList("popular", "trending", "bestseller"); + + record.metadata = new HashMap<>(); + record.metadata.put("manufacturer", "TechCorp"); + record.metadata.put("model", "TC-2024"); + record.metadata.put("warranty", "2 years"); + + return record; + } + + private TestRecord createTestRecord2() { + var record = new TestRecord(); + record.id = 67890; + record.name = "Test Product 2"; + record.price = 149.99; + record.active = false; + record.category = "Software"; + + record.tags = Arrays.asList("new", "premium"); + + record.metadata = new HashMap<>(); + record.metadata.put("vendor", "SoftCorp"); + record.metadata.put("version", "2.1"); + + return record; + } + + // Test data class for other serialization libraries + public static class TestRecord { + public int id; + public String name; + public double price; + public boolean active; + public String category; + public List tags = new ArrayList<>(); + public Map metadata = new HashMap<>(); + + public TestRecord() {} // Required for deserialization + } +} \ No newline at end of file diff --git a/src/jmh/java/com/imprint/benchmark/FieldAccessBenchmark.java b/src/jmh/java/com/imprint/benchmark/FieldAccessBenchmark.java new file mode 100644 index 0000000..f3abb7e --- /dev/null +++ b/src/jmh/java/com/imprint/benchmark/FieldAccessBenchmark.java @@ -0,0 +1,276 @@ +package com.imprint.benchmark; + +import com.imprint.core.ImprintRecord; +import com.imprint.core.ImprintWriter; +import com.imprint.core.SchemaId; +import com.imprint.types.MapKey; +import com.imprint.types.Value; +import org.openjdk.jmh.annotations.*; +import org.openjdk.jmh.infra.Blackhole; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.concurrent.TimeUnit; + +/** + * Benchmarks for ImprintRecord field access and projection operations. + * Tests the zero-copy field access performance claims. + */ +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@State(Scope.Benchmark) +@Warmup(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS) +@Fork(1) +public class FieldAccessBenchmark { + + private ImprintRecord sparseRecord; + private ImprintRecord denseRecord; + private ImprintRecord largeRecord; + + // Field IDs for testing different access patterns + private int[] firstFields; + private int[] middleFields; + private int[] lastFields; + private int[] randomFields; + private int[] allFields; + + @Setup + public void setup() throws Exception { + sparseRecord = createSparseRecord(); // Few fields, large field IDs + denseRecord = createDenseRecord(); // Many sequential fields + largeRecord = createLargeRecord(); // Large record with complex data + + // Setup field access patterns + firstFields = new int[]{1, 2, 3, 4, 5}; + middleFields = new int[]{45, 46, 47, 48, 49}; + lastFields = new int[]{95, 96, 97, 98, 99}; + randomFields = new int[]{7, 23, 41, 67, 89}; + allFields = new int[100]; + for (int i = 0; i < 100; i++) { + allFields[i] = i + 1; + } + } + + // ===== SINGLE FIELD ACCESS BENCHMARKS ===== + + @Benchmark + public void accessFirstField(Blackhole bh) throws Exception { + var value = denseRecord.getValue(1); + bh.consume(value); + } + + @Benchmark + public void accessMiddleField(Blackhole bh) throws Exception { + var value = denseRecord.getValue(50); + bh.consume(value); + } + + @Benchmark + public void accessLastField(Blackhole bh) throws Exception { + var value = denseRecord.getValue(100); + bh.consume(value); + } + + @Benchmark + public void accessNonExistentField(Blackhole bh) throws Exception { + var value = denseRecord.getValue(999); + bh.consume(value); + } + + // ===== MULTIPLE FIELD ACCESS PATTERNS ===== + + @Benchmark + public void accessFirstFields(Blackhole bh) throws Exception { + for (int fieldId : firstFields) { + var value = denseRecord.getValue(fieldId); + bh.consume(value); + } + } + + @Benchmark + public void accessMiddleFields(Blackhole bh) throws Exception { + for (int fieldId : middleFields) { + var value = denseRecord.getValue(fieldId); + bh.consume(value); + } + } + + @Benchmark + public void accessLastFields(Blackhole bh) throws Exception { + for (int fieldId : lastFields) { + var value = denseRecord.getValue(fieldId); + bh.consume(value); + } + } + + @Benchmark + public void accessRandomFields(Blackhole bh) throws Exception { + for (int fieldId : randomFields) { + var value = denseRecord.getValue(fieldId); + bh.consume(value); + } + } + + // ===== FIELD PROJECTION BENCHMARKS ===== + + @Benchmark + public void projectSmallSubset(Blackhole bh) throws Exception { + // Project 5 fields from a 100-field record + var projection = simulateProject(denseRecord, firstFields); + bh.consume(projection); + } + + @Benchmark + public void projectMediumSubset(Blackhole bh) throws Exception { + // Project 25 fields from a 100-field record + int[] fields = Arrays.copyOf(allFields, 25); + var projection = simulateProject(denseRecord, fields); + bh.consume(projection); + } + + @Benchmark + public void projectLargeSubset(Blackhole bh) throws Exception { + // Project 75 fields from a 100-field record + int[] fields = Arrays.copyOf(allFields, 75); + var projection = simulateProject(denseRecord, fields); + bh.consume(projection); + } + + @Benchmark + public void projectAllFields(Blackhole bh) throws Exception { + // Project all fields (should be nearly equivalent to full record) + var projection = simulateProject(denseRecord, allFields); + bh.consume(projection); + } + + // ===== RAW BYTES ACCESS BENCHMARKS ===== + + @Benchmark + public void getRawBytesFirstField(Blackhole bh) { + var rawBytes = denseRecord.getRawBytes(1); + bh.consume(rawBytes); + } + + @Benchmark + public void getRawBytesMiddleField(Blackhole bh) { + var rawBytes = denseRecord.getRawBytes(50); + bh.consume(rawBytes); + } + + @Benchmark + public void getRawBytesLastField(Blackhole bh) { + var rawBytes = denseRecord.getRawBytes(100); + bh.consume(rawBytes); + } + + // ===== SPARSE VS DENSE ACCESS PATTERNS ===== + + @Benchmark + public void accessSparseRecord(Blackhole bh) throws Exception { + // Access fields in sparse record (large field IDs, few fields) + var value1 = sparseRecord.getValue(1000); + var value2 = sparseRecord.getValue(5000); + var value3 = sparseRecord.getValue(10000); + bh.consume(value1); + bh.consume(value2); + bh.consume(value3); + } + + @Benchmark + public void accessDenseRecord(Blackhole bh) throws Exception { + // Access fields in dense record (sequential field IDs) + var value1 = denseRecord.getValue(1); + var value2 = denseRecord.getValue(2); + var value3 = denseRecord.getValue(3); + bh.consume(value1); + bh.consume(value2); + bh.consume(value3); + } + + // ===== HELPER METHODS ===== + + /** + * Simulates field projection by creating a new record with only specified fields. + * This should be replaced with actual project API when available. + */ + private ImprintRecord simulateProject(ImprintRecord source, int[] fieldIds) throws Exception { + var writer = new ImprintWriter(source.getHeader().getSchemaId()); + + for (int fieldId : fieldIds) { + var value = source.getValue(fieldId); + value.ifPresent(value1 -> writer.addField(fieldId, value1)); + } + + return writer.build(); + } + + private ImprintRecord createSparseRecord() throws Exception { + var writer = new ImprintWriter(new SchemaId(1, 0x12345678)); + + // Sparse record with large field IDs and few fields + writer.addField(1000, Value.fromString("sparse_field_1")); + writer.addField(5000, Value.fromInt32(42)); + writer.addField(10000, Value.fromFloat64(3.14159)); + writer.addField(15000, Value.fromBoolean(true)); + writer.addField(20000, Value.fromString("sparse_field_5")); + + return writer.build(); + } + + private ImprintRecord createDenseRecord() throws Exception { + var writer = new ImprintWriter(new SchemaId(2, 0x87654321)); + + // Dense record with 100 sequential fields + for (int i = 1; i <= 100; i++) { + switch (i % 5) { + case 0: + writer.addField(i, Value.fromString("string_field_" + i)); + break; + case 1: + writer.addField(i, Value.fromInt32(i * 10)); + break; + case 2: + writer.addField(i, Value.fromFloat64(i * 1.5)); + break; + case 3: + writer.addField(i, Value.fromBoolean(i % 2 == 0)); + break; + case 4: + writer.addField(i, Value.fromInt64(i * 1000L)); + break; + } + } + + return writer.build(); + } + + private ImprintRecord createLargeRecord() throws Exception { + var writer = new ImprintWriter(new SchemaId(3, 0x11223344)); + + // Large record with complex data types + writer.addField(1, Value.fromString("LargeRecord")); + + // Large array field + var largeArray = new ArrayList(); + for (int i = 0; i < 1000; i++) { + largeArray.add(Value.fromString("array_item_" + i)); + } + writer.addField(2, Value.fromArray(largeArray)); + + // Large map field + var largeMap = new HashMap(); + for (int i = 0; i < 100; i++) { + largeMap.put(MapKey.fromString("key_" + i), Value.fromString("map_value_" + i)); + } + writer.addField(3, Value.fromMap(largeMap)); + + // Many regular fields + for (int i = 4; i <= 50; i++) { + writer.addField(i, Value.fromString("large_record_field_" + i + "_with_substantial_content")); + } + + return writer.build(); + } +} \ No newline at end of file diff --git a/src/jmh/java/com/imprint/benchmark/MergeBenchmark.java b/src/jmh/java/com/imprint/benchmark/MergeBenchmark.java new file mode 100644 index 0000000..5c52908 --- /dev/null +++ b/src/jmh/java/com/imprint/benchmark/MergeBenchmark.java @@ -0,0 +1,163 @@ +package com.imprint.benchmark; + +import com.imprint.core.ImprintRecord; +import com.imprint.core.ImprintWriter; +import com.imprint.core.SchemaId; +import com.imprint.types.Value; +import org.openjdk.jmh.annotations.*; +import org.openjdk.jmh.infra.Blackhole; + +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +/** + * Benchmarks for ImprintRecord merge operations. + * NOTE: These benchmarks simulate merge operations until the actual merge API is implemented. + */ +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@State(Scope.Benchmark) +@Warmup(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS) +@Fork(1) +public class MergeBenchmark { + + private ImprintRecord productRecord; + private ImprintRecord orderRecord; + private ImprintRecord customerRecord; + + @Setup + public void setup() throws Exception { + productRecord = createProductRecord(); + orderRecord = createOrderRecord(); + customerRecord = createCustomerRecord(); + } + + // ===== SIMULATED MERGE BENCHMARKS ===== + // These will be replaced with actual merge API when implemented + + @Benchmark + public void mergeProductAndOrder(Blackhole bh) throws Exception { + // Simulate merge by creating a new record with fields from both + ImprintRecord result = simulateMerge(productRecord, orderRecord); + bh.consume(result); + } + + @Benchmark + public void mergeProductAndCustomer(Blackhole bh) throws Exception { + ImprintRecord result = simulateMerge(productRecord, customerRecord); + bh.consume(result); + } + + @Benchmark + public void mergeOrderAndCustomer(Blackhole bh) throws Exception { + ImprintRecord result = simulateMerge(orderRecord, customerRecord); + bh.consume(result); + } + + @Benchmark + public void mergeThreeRecords(Blackhole bh) throws Exception { + // Test merging multiple records + var temp = simulateMerge(productRecord, orderRecord); + ImprintRecord result = simulateMerge(temp, customerRecord); + bh.consume(result); + } + + // ===== MERGE CONFLICT HANDLING ===== + + @Benchmark + public void mergeWithConflicts(Blackhole bh) throws Exception { + // Create records with overlapping field IDs to test conflict resolution + var record1 = createRecordWithFields(1, 50, "record1_"); + var record2 = createRecordWithFields(25, 75, "record2_"); + + ImprintRecord result = simulateMerge(record1, record2); + bh.consume(result); + } + + // ===== HELPER METHODS ===== + + /** + * Simulates merge operation by manually copying fields. + * This should be replaced with actual merge API when available. + */ + private ImprintRecord simulateMerge(ImprintRecord first, ImprintRecord second) throws Exception { + var writer = new ImprintWriter(first.getHeader().getSchemaId()); + var usedFieldIds = new HashSet(); + + // Copy fields from first record (takes precedence) + copyFieldsToWriter(first, writer, usedFieldIds); + + // Copy non-conflicting fields from second record + copyFieldsToWriter(second, writer, usedFieldIds); + + return writer.build(); + } + + private void copyFieldsToWriter(ImprintRecord record, ImprintWriter writer, Set usedFieldIds) throws Exception { + for (var entry : record.getDirectory()) { + int fieldId = entry.getId(); + if (!usedFieldIds.contains(fieldId)) { + var value = record.getValue(fieldId); + if (value.isPresent()) { + writer.addField(fieldId, value.get()); + usedFieldIds.add(fieldId); + } + } + } + } + + private ImprintRecord createProductRecord() throws Exception { + var writer = new ImprintWriter(new SchemaId(1, 0x12345678)); + + writer.addField(1, Value.fromString("Product")); + writer.addField(2, Value.fromInt32(12345)); + writer.addField(3, Value.fromString("Laptop")); + writer.addField(4, Value.fromFloat64(999.99)); + writer.addField(5, Value.fromString("Electronics")); + writer.addField(6, Value.fromInt32(50)); // stock + writer.addField(7, Value.fromString("TechCorp")); + writer.addField(8, Value.fromBoolean(true)); // available + + return writer.build(); + } + + private ImprintRecord createOrderRecord() throws Exception { + var writer = new ImprintWriter(new SchemaId(2, 0x87654321)); + + writer.addField(10, Value.fromString("Order")); + writer.addField(11, Value.fromInt32(67890)); + writer.addField(12, Value.fromInt32(12345)); // product_id (overlaps with product) + writer.addField(13, Value.fromInt32(2)); // quantity + writer.addField(14, Value.fromFloat64(1999.98)); // total + writer.addField(15, Value.fromString("2024-01-15")); // order_date + writer.addField(16, Value.fromString("shipped")); // status + + return writer.build(); + } + + private ImprintRecord createCustomerRecord() throws Exception { + var writer = new ImprintWriter(new SchemaId(3, 0x11223344)); + + writer.addField(20, Value.fromString("Customer")); + writer.addField(21, Value.fromInt32(555)); + writer.addField(22, Value.fromString("John Doe")); + writer.addField(23, Value.fromString("john.doe@email.com")); + writer.addField(24, Value.fromString("123 Main St")); + writer.addField(25, Value.fromString("premium")); // tier + writer.addField(26, Value.fromBoolean(true)); // active + + return writer.build(); + } + + private ImprintRecord createRecordWithFields(int startId, int endId, String prefix) throws Exception { + var writer = new ImprintWriter(new SchemaId(1, 0x12345678)); + + for (int i = startId; i <= endId; i++) { + writer.addField(i, Value.fromString(prefix + "field_" + i)); + } + + return writer.build(); + } +} \ No newline at end of file diff --git a/src/jmh/java/com/imprint/benchmark/SerializationBenchmark.java b/src/jmh/java/com/imprint/benchmark/SerializationBenchmark.java new file mode 100644 index 0000000..2544b88 --- /dev/null +++ b/src/jmh/java/com/imprint/benchmark/SerializationBenchmark.java @@ -0,0 +1,166 @@ +package com.imprint.benchmark; + +import com.imprint.core.ImprintRecord; +import com.imprint.core.ImprintWriter; +import com.imprint.core.SchemaId; +import com.imprint.types.MapKey; +import com.imprint.types.Value; +import org.openjdk.jmh.annotations.*; +import org.openjdk.jmh.infra.Blackhole; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.concurrent.TimeUnit; + +/** + * Benchmarks for ImprintRecord serialization and deserialization operations. + */ +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@State(Scope.Benchmark) +@Warmup(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS) +@Fork(1) +public class SerializationBenchmark { + + private ImprintRecord smallRecord; + private ImprintRecord mediumRecord; + private ImprintRecord largeRecord; + + private ByteBuffer smallRecordBytes; + private ByteBuffer mediumRecordBytes; + private ByteBuffer largeRecordBytes; + + @Setup + public void setup() throws Exception { + // Create test records of varying sizes + smallRecord = createSmallRecord(); + mediumRecord = createMediumRecord(); + largeRecord = createLargeRecord(); + + // Pre-serialize for deserialization benchmarks + smallRecordBytes = smallRecord.serializeToBuffer(); + mediumRecordBytes = mediumRecord.serializeToBuffer(); + largeRecordBytes = largeRecord.serializeToBuffer(); + } + + // ===== SERIALIZATION BENCHMARKS ===== + + @Benchmark + public void serializeSmallRecord(Blackhole bh) { + ByteBuffer result = smallRecord.serializeToBuffer(); + bh.consume(result); + } + + @Benchmark + public void serializeMediumRecord(Blackhole bh) { + ByteBuffer result = mediumRecord.serializeToBuffer(); + bh.consume(result); + } + + @Benchmark + public void serializeLargeRecord(Blackhole bh) { + ByteBuffer result = largeRecord.serializeToBuffer(); + bh.consume(result); + } + + // ===== DESERIALIZATION BENCHMARKS ===== + + @Benchmark + public void deserializeSmallRecord(Blackhole bh) throws Exception { + ImprintRecord result = ImprintRecord.deserialize(smallRecordBytes.duplicate()); + bh.consume(result); + } + + @Benchmark + public void deserializeMediumRecord(Blackhole bh) throws Exception { + ImprintRecord result = ImprintRecord.deserialize(mediumRecordBytes.duplicate()); + bh.consume(result); + } + + @Benchmark + public void deserializeLargeRecord(Blackhole bh) throws Exception { + ImprintRecord result = ImprintRecord.deserialize(largeRecordBytes.duplicate()); + bh.consume(result); + } + + // ===== HELPER METHODS ===== + + private ImprintRecord createSmallRecord() throws Exception { + var writer = new ImprintWriter(new SchemaId(1, 0x12345678)); + + // Small record: ~10 fields, simple types + writer.addField(1, Value.fromString("Product")); + writer.addField(2, Value.fromInt32(12345)); + writer.addField(3, Value.fromFloat64(99.99)); + writer.addField(4, Value.fromBoolean(true)); + writer.addField(5, Value.fromString("Electronics")); + + return writer.build(); + } + + private ImprintRecord createMediumRecord() throws Exception { + var writer = new ImprintWriter(new SchemaId(1, 0x12345678)); + + // Medium record: ~50 fields, mixed types including arrays + writer.addField(1, Value.fromString("Product")); + writer.addField(2, Value.fromInt32(12345)); + writer.addField(3, Value.fromFloat64(99.99)); + writer.addField(4, Value.fromBoolean(true)); + writer.addField(5, Value.fromString("Electronics")); + + // Add array field + var tags = Arrays.asList( + Value.fromString("popular"), + Value.fromString("trending"), + Value.fromString("bestseller") + ); + writer.addField(6, Value.fromArray(tags)); + + // Add map field + var metadata = new HashMap(); + metadata.put(MapKey.fromString("manufacturer"), Value.fromString("TechCorp")); + metadata.put(MapKey.fromString("model"), Value.fromString("TC-2024")); + metadata.put(MapKey.fromString("year"), Value.fromInt32(2024)); + writer.addField(7, Value.fromMap(metadata)); + + // Add more fields for medium size + for (int i = 8; i <= 50; i++) { + writer.addField(i, Value.fromString("field_" + i + "_value")); + } + + return writer.build(); + } + + private ImprintRecord createLargeRecord() throws Exception { + var writer = new ImprintWriter(new SchemaId(1, 0x12345678)); + + // Large record: ~200 fields, complex nested structures + writer.addField(1, Value.fromString("LargeProduct")); + writer.addField(2, Value.fromInt32(12345)); + writer.addField(3, Value.fromFloat64(99.99)); + + // Large array + var largeArray = new ArrayList(); + for (int i = 0; i < 100; i++) { + largeArray.add(Value.fromString("item_" + i)); + } + writer.addField(4, Value.fromArray(largeArray)); + + // Large map + var largeMap = new HashMap(); + for (int i = 0; i < 50; i++) { + largeMap.put(MapKey.fromString("key_" + i), Value.fromString("value_" + i)); + } + writer.addField(5, Value.fromMap(largeMap)); + + // Many string fields + for (int i = 6; i <= 200; i++) { + writer.addField(i, Value.fromString("this_is_a_longer_field_value_for_field_" + i + "_to_increase_record_size")); + } + + return writer.build(); + } +} \ No newline at end of file diff --git a/src/main/java/com/imprint/Constants.java b/src/main/java/com/imprint/Constants.java new file mode 100644 index 0000000..78b91a0 --- /dev/null +++ b/src/main/java/com/imprint/Constants.java @@ -0,0 +1,10 @@ +package com.imprint; + +public final class Constants { + public static final byte MAGIC = 0x49; + public static final byte VERSION = 0x02; + public static final int HEADER_BYTES = 15; + public static final int DIR_ENTRY_BYTES = 7; + + private Constants() {} +} diff --git a/src/main/java/com/imprint/core/DirectoryEntry.java b/src/main/java/com/imprint/core/DirectoryEntry.java new file mode 100644 index 0000000..9556256 --- /dev/null +++ b/src/main/java/com/imprint/core/DirectoryEntry.java @@ -0,0 +1,23 @@ +package com.imprint.core; + +import com.imprint.types.TypeCode; +import lombok.Value; + +import java.util.Objects; + +/** + * A directory entry describing a single field in an Imprint record. + * Each entry has a fixed size of 7 bytes. + */ +@Value +public class DirectoryEntry { + short id; + TypeCode typeCode; + int offset; + + public DirectoryEntry(int id, TypeCode typeCode, int offset) { + this.id = (short) id; + this.typeCode = Objects.requireNonNull(typeCode, "TypeCode cannot be null"); + this.offset = offset; + } +} \ No newline at end of file diff --git a/src/main/java/com/imprint/core/Flags.java b/src/main/java/com/imprint/core/Flags.java new file mode 100644 index 0000000..ee6ffa7 --- /dev/null +++ b/src/main/java/com/imprint/core/Flags.java @@ -0,0 +1,12 @@ +package com.imprint.core; + +import lombok.Value; + +/** + * Bit flags for Imprint record header. + * Currently reserved for future use - field directory is always present. + */ +@Value +public class Flags { + byte value; +} \ No newline at end of file diff --git a/src/main/java/com/imprint/core/Header.java b/src/main/java/com/imprint/core/Header.java new file mode 100644 index 0000000..388d491 --- /dev/null +++ b/src/main/java/com/imprint/core/Header.java @@ -0,0 +1,13 @@ +package com.imprint.core; + +import lombok.Value; + +/** + * The header of an Imprint record. + */ +@Value +public class Header { + Flags flags; + SchemaId schemaId; + int payloadSize; +} \ No newline at end of file diff --git a/src/main/java/com/imprint/core/ImprintRecord.java b/src/main/java/com/imprint/core/ImprintRecord.java new file mode 100644 index 0000000..b7ed224 --- /dev/null +++ b/src/main/java/com/imprint/core/ImprintRecord.java @@ -0,0 +1,365 @@ + +package com.imprint.core; + +import com.imprint.Constants; +import com.imprint.error.ErrorType; +import com.imprint.error.ImprintException; +import com.imprint.types.MapKey; +import com.imprint.types.TypeCode; +import com.imprint.types.Value; +import com.imprint.util.VarInt; +import lombok.Getter; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.*; + +/** + * An Imprint record containing a header, optional field directory, and payload. + * Uses ByteBuffer for zero-copy operations to achieve low latency. + */ +@Getter +public final class ImprintRecord { + private final Header header; + private final List directory; + private final ByteBuffer payload; // Read-only view for zero-copy + + public ImprintRecord(Header header, List directory, ByteBuffer payload) { + this.header = Objects.requireNonNull(header, "Header cannot be null"); + this.directory = List.copyOf(Objects.requireNonNull(directory, "Directory cannot be null")); + this.payload = payload.asReadOnlyBuffer(); // Zero-copy read-only view + } + + /** + * Get a value by field ID, deserializing it on demand. + */ + public Optional getValue(int fieldId) throws ImprintException { + // Binary search for the field ID without allocation + int index = findDirectoryIndex(fieldId); + if (index < 0) return Optional.empty(); + + var entry = directory.get(index); + int startOffset = entry.getOffset(); + int endOffset = (index + 1 < directory.size()) ? + directory.get(index + 1).getOffset() : payload.remaining(); + + var valueBytes = payload.duplicate(); + valueBytes.position(startOffset).limit(endOffset); + var value = deserializeValue(entry.getTypeCode(), valueBytes.slice()); + return Optional.of(value); + } + + /** + * Get the raw bytes for a field without deserializing. + * Returns a zero-copy ByteBuffer view. + */ + public Optional getRawBytes(int fieldId) { + int index = findDirectoryIndex(fieldId); + if (index < 0) return Optional.empty(); + + var entry = directory.get(index); + int startOffset = entry.getOffset(); + int endOffset = (index + 1 < directory.size()) ? + directory.get(index + 1).getOffset() : payload.remaining(); + + var fieldBuffer = payload.duplicate(); + fieldBuffer.position(startOffset).limit(endOffset); + return Optional.of(fieldBuffer.slice().asReadOnlyBuffer()); + } + + /** + * Serialize this record to a ByteBuffer (zero-copy when possible). + */ + public ByteBuffer serializeToBuffer() { + var buffer = ByteBuffer.allocate(estimateSerializedSize()); + buffer.order(ByteOrder.LITTLE_ENDIAN); + + // Write header + serializeHeader(buffer); + + // Write directory (always present) + VarInt.encode(directory.size(), buffer); + for (var entry : directory) { + serializeDirectoryEntry(entry, buffer); + } + + // Write payload (shallow copy only) + var payloadCopy = payload.duplicate(); + buffer.put(payloadCopy); + + // Return read-only view of used portion + buffer.flip(); + return buffer.asReadOnlyBuffer(); + } + + /** + * Create a fluent builder for constructing ImprintRecord instances. + * + * @param schemaId the schema identifier for this record + * @return a new builder instance + */ + public static ImprintRecordBuilder builder(SchemaId schemaId) { + return new ImprintRecordBuilder(schemaId); + } + + /** + * Create a fluent builder for constructing ImprintRecord instances. + * + * @param fieldspaceId the fieldspace identifier + * @param schemaHash the schema hash + * @return a new builder instance + */ + public static ImprintRecordBuilder builder(int fieldspaceId, int schemaHash) { + return new ImprintRecordBuilder(new SchemaId(fieldspaceId, schemaHash)); + } + + /** + * Deserialize a record from bytes. + */ + public static ImprintRecord deserialize(byte[] bytes) throws ImprintException { + return deserialize(ByteBuffer.wrap(bytes)); + } + + /** + * Deserialize a record from a ByteBuffer (zero-copy when possible). + */ + public static ImprintRecord deserialize(ByteBuffer buffer) throws ImprintException { + buffer = buffer.duplicate().order(ByteOrder.LITTLE_ENDIAN); + + // Read header + var header = deserializeHeader(buffer); + + // Read directory (always present) + var directory = new ArrayList(); + VarInt.DecodeResult countResult = VarInt.decode(buffer); + int directoryCount = countResult.getValue(); + + for (int i = 0; i < directoryCount; i++) { + directory.add(deserializeDirectoryEntry(buffer)); + } + + // Read payload as ByteBuffer slice for zero-copy + var payload = buffer.slice(); + payload.limit(header.getPayloadSize()); + buffer.position(buffer.position() + header.getPayloadSize()); + + return new ImprintRecord(header, directory, payload); + } + + /** + * Binary search for field ID in directory without object allocation. + * Returns the index of the field if found, or a negative value if not found. + * + * @param fieldId the field ID to search for + * @return index if found, or negative insertion point - 1 if not found + */ + private int findDirectoryIndex(int fieldId) { + int low = 0; + int high = directory.size() - 1; + + while (low <= high) { + int mid = (low + high) >>> 1; // unsigned right shift to avoid overflow + int midFieldId = directory.get(mid).getId(); + + if (midFieldId < fieldId) { + low = mid + 1; + } else if (midFieldId > fieldId) { + high = mid - 1; + } else { + return mid; // field found + } + } + return -(low + 1); // field not found, return insertion point + } + + private int estimateSerializedSize() { + int size = Constants.HEADER_BYTES; // header + size += VarInt.encodedLength(directory.size()); // directory count + size += directory.size() * Constants.DIR_ENTRY_BYTES; // directory entries + size += payload.remaining(); // payload + return size; + } + + private void serializeHeader(ByteBuffer buffer) { + buffer.put(Constants.MAGIC); + buffer.put(Constants.VERSION); + buffer.put(header.getFlags().getValue()); + buffer.putInt(header.getSchemaId().getFieldspaceId()); + buffer.putInt(header.getSchemaId().getSchemaHash()); + buffer.putInt(header.getPayloadSize()); + } + + private static Header deserializeHeader(ByteBuffer buffer) throws ImprintException { + if (buffer.remaining() < Constants.HEADER_BYTES) { + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, + "Not enough bytes for header"); + } + + byte magic = buffer.get(); + if (magic != Constants.MAGIC) { + throw new ImprintException(ErrorType.INVALID_MAGIC, + "Invalid magic byte: expected 0x" + Integer.toHexString(Constants.MAGIC) + + ", got 0x" + Integer.toHexString(magic & 0xFF)); + } + + byte version = buffer.get(); + if (version != Constants.VERSION) { + throw new ImprintException(ErrorType.UNSUPPORTED_VERSION, + "Unsupported version: " + version); + } + + var flags = new Flags(buffer.get()); + int fieldspaceId = buffer.getInt(); + int schemaHash = buffer.getInt(); + int payloadSize = buffer.getInt(); + + return new Header(flags, new SchemaId(fieldspaceId, schemaHash), payloadSize); + } + + private void serializeDirectoryEntry(DirectoryEntry entry, ByteBuffer buffer) { + buffer.putShort(entry.getId()); + buffer.put(entry.getTypeCode().getCode()); + buffer.putInt(entry.getOffset()); + } + + private static DirectoryEntry deserializeDirectoryEntry(ByteBuffer buffer) throws ImprintException { + if (buffer.remaining() < Constants.DIR_ENTRY_BYTES) { + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, + "Not enough bytes for directory entry"); + } + + short id = buffer.getShort(); + var typeCode = TypeCode.fromByte(buffer.get()); + int offset = buffer.getInt(); + + return new DirectoryEntry(id, typeCode, offset); + } + + private Value deserializeValue(TypeCode typeCode, ByteBuffer buffer) throws ImprintException { + // Buffer is already positioned and limited correctly + buffer = buffer.duplicate().order(ByteOrder.LITTLE_ENDIAN); + + // Use TypeHandler for simple types + switch (typeCode) { + case NULL: + case BOOL: + case INT32: + case INT64: + case FLOAT32: + case FLOAT64: + case BYTES: + case STRING: + return typeCode.getHandler().deserialize(buffer); + + case ARRAY: + return deserializeArray(buffer); + + case MAP: + return deserializeMap(buffer); + + case ROW: + var remainingBuffer = buffer.slice(); + var nestedRecord = deserialize(remainingBuffer); + return Value.fromRow(nestedRecord); + + default: + throw new ImprintException(ErrorType.INVALID_TYPE_CODE, "Unknown type code: " + typeCode); + } + } + + private Value deserializeArray(ByteBuffer buffer) throws ImprintException { + VarInt.DecodeResult lengthResult = VarInt.decode(buffer); + int length = lengthResult.getValue(); + + if (length == 0) { + return Value.fromArray(Collections.emptyList()); + } + + var elementType = TypeCode.fromByte(buffer.get()); + var elements = new ArrayList(length); + + for (int i = 0; i < length; i++) { + var elementBytes = readValueBytes(elementType, buffer); + var element = deserializeValue(elementType, elementBytes); + elements.add(element); + } + + return Value.fromArray(elements); + } + + private Value deserializeMap(ByteBuffer buffer) throws ImprintException { + VarInt.DecodeResult lengthResult = VarInt.decode(buffer); + int length = lengthResult.getValue(); + + if (length == 0) { + return Value.fromMap(Collections.emptyMap()); + } + + var keyType = TypeCode.fromByte(buffer.get()); + var valueType = TypeCode.fromByte(buffer.get()); + var map = new HashMap(length); + + for (int i = 0; i < length; i++) { + // Read key + var keyBytes = readValueBytes(keyType, buffer); + var keyValue = deserializeValue(keyType, keyBytes); + var key = MapKey.fromValue(keyValue); + + // Read value + var valueBytes = readValueBytes(valueType, buffer); + var value = deserializeValue(valueType, valueBytes); + + map.put(key, value); + } + + return Value.fromMap(map); + } + + private ByteBuffer readValueBytes(TypeCode typeCode, ByteBuffer buffer) throws ImprintException { + // Use TypeHandler for simple types + switch (typeCode) { + case NULL: + case BOOL: + case INT32: + case INT64: + case FLOAT32: + case FLOAT64: + case BYTES: + case STRING: + return typeCode.getHandler().readValueBytes(buffer); + + case ARRAY: + case MAP: + case ROW: + // For complex types, return the entire remaining buffer for now + // The specific deserializer will handle parsing in the future + var remainingBuffer = buffer.slice(); + buffer.position(buffer.limit()); + return remainingBuffer.asReadOnlyBuffer(); + + default: + throw new ImprintException(ErrorType.INVALID_TYPE_CODE, "Unknown type code: " + typeCode); + } + } + + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (obj == null || getClass() != obj.getClass()) return false; + var that = (ImprintRecord) obj; + return header.equals(that.header) && + directory.equals(that.directory) && + payload.equals(that.payload); + } + + @Override + public int hashCode() { + return Objects.hash(header, directory, payload); + } + + @Override + public String toString() { + return String.format("ImprintRecord{header=%s, directorySize=%d, payloadSize=%d}", + header, directory.size(), payload.remaining()); + } +} \ No newline at end of file diff --git a/src/main/java/com/imprint/core/ImprintRecordBuilder.java b/src/main/java/com/imprint/core/ImprintRecordBuilder.java new file mode 100644 index 0000000..48b0998 --- /dev/null +++ b/src/main/java/com/imprint/core/ImprintRecordBuilder.java @@ -0,0 +1,233 @@ +package com.imprint.core; + +import com.imprint.error.ImprintException; +import com.imprint.types.MapKey; +import com.imprint.types.Value; + +import java.util.*; + +/** + * A fluent builder for creating ImprintRecord instances with type-safe, + * developer-friendly API that eliminates boilerplate Value.fromX() calls. + *

+ * Usage: + * var record = ImprintRecord.builder(schemaId) + * .field(1, 42) // int -> Int32Value + * .field(2, "hello") // String -> StringValue + * .field(3, 3.14) // double -> Float64Value + * .field(4, bytes) // byte[] -> BytesValue + * .field(5, true) // boolean -> BoolValue + * .nullField(6) // -> NullValue + * .build(); + */ +public final class ImprintRecordBuilder { + private final SchemaId schemaId; + private final Map fields = new TreeMap<>(); + + ImprintRecordBuilder(SchemaId schemaId) { + this.schemaId = Objects.requireNonNull(schemaId, "SchemaId cannot be null"); + } + + // Primitive types with automatic Value wrapping + public ImprintRecordBuilder field(int id, boolean value) { + return addField(id, Value.fromBoolean(value)); + } + + public ImprintRecordBuilder field(int id, int value) { + return addField(id, Value.fromInt32(value)); + } + + public ImprintRecordBuilder field(int id, long value) { + return addField(id, Value.fromInt64(value)); + } + + public ImprintRecordBuilder field(int id, float value) { + return addField(id, Value.fromFloat32(value)); + } + + public ImprintRecordBuilder field(int id, double value) { + return addField(id, Value.fromFloat64(value)); + } + + public ImprintRecordBuilder field(int id, String value) { + return addField(id, Value.fromString(value)); + } + + public ImprintRecordBuilder field(int id, byte[] value) { + return addField(id, Value.fromBytes(value)); + } + + // Collections with automatic conversion + public ImprintRecordBuilder field(int id, List values) { + var convertedValues = new ArrayList(values.size()); + for (var item : values) { + convertedValues.add(convertToValue(item)); + } + return addField(id, Value.fromArray(convertedValues)); + } + + public ImprintRecordBuilder field(int id, Map map) { + var convertedMap = new HashMap(map.size()); + for (var entry : map.entrySet()) { + var key = convertToMapKey(entry.getKey()); + var value = convertToValue(entry.getValue()); + convertedMap.put(key, value); + } + return addField(id, Value.fromMap(convertedMap)); + } + + // Nested records + public ImprintRecordBuilder field(int id, ImprintRecord nestedRecord) { + return addField(id, Value.fromRow(nestedRecord)); + } + + // Explicit null field + public ImprintRecordBuilder nullField(int id) { + return addField(id, Value.nullValue()); + } + + // Direct Value API (escape hatch for advanced usage) + public ImprintRecordBuilder field(int id, Value value) { + return addField(id, value); + } + + // Conditional field addition + public ImprintRecordBuilder fieldIf(boolean condition, int id, Object value) { + if (condition) { + return field(id, convertToValue(value)); + } + return this; + } + + public ImprintRecordBuilder fieldIfNotNull(int id, Object value) { + return fieldIf(value != null, id, value); + } + + // Bulk operations + public ImprintRecordBuilder fields(Map fieldsMap) { + for (var entry : fieldsMap.entrySet()) { + field(entry.getKey(), convertToValue(entry.getValue())); + } + return this; + } + + // Builder utilities + public boolean hasField(int id) { + return fields.containsKey(id); + } + + public int fieldCount() { + return fields.size(); + } + + public Set fieldIds() { + return new TreeSet<>(fields.keySet()); + } + + // Build the final record + public ImprintRecord build() throws ImprintException { + if (fields.isEmpty()) { + throw new ImprintException(com.imprint.error.ErrorType.SCHEMA_ERROR, + "Cannot build empty record - add at least one field"); + } + + var writer = new ImprintWriter(schemaId); + for (var entry : fields.entrySet()) { + writer.addField(entry.getKey(), entry.getValue()); + } + return writer.build(); + } + + // Internal helper methods + private ImprintRecordBuilder addField(int id, Value value) { + Objects.requireNonNull(value, "Value cannot be null - use nullField() for explicit null values"); + if (fields.containsKey(id)) { + throw new IllegalArgumentException("Field ID " + id + " already exists - field IDs must be unique"); + } + fields.put(id, value); + return this; + } + + private Value convertToValue(Object obj) { + if (obj == null) { + return Value.nullValue(); + } + + if (obj instanceof Value) { + return (Value) obj; + } + + // Auto-boxing conversion + if (obj instanceof Boolean) { + return Value.fromBoolean((Boolean) obj); + } + if (obj instanceof Integer) { + return Value.fromInt32((Integer) obj); + } + if (obj instanceof Long) { + return Value.fromInt64((Long) obj); + } + if (obj instanceof Float) { + return Value.fromFloat32((Float) obj); + } + if (obj instanceof Double) { + return Value.fromFloat64((Double) obj); + } + if (obj instanceof String) { + return Value.fromString((String) obj); + } + if (obj instanceof byte[]) { + return Value.fromBytes((byte[]) obj); + } + if (obj instanceof List) { + //test + @SuppressWarnings("unchecked") + List list = (List) obj; + var convertedValues = new ArrayList(list.size()); + for (var item : list) { + convertedValues.add(convertToValue(item)); + } + return Value.fromArray(convertedValues); + } + if (obj instanceof Map) { + @SuppressWarnings("unchecked") + Map map = (Map) obj; + var convertedMap = new HashMap(map.size()); + for (var entry : map.entrySet()) { + var key = convertToMapKey(entry.getKey()); + var value = convertToValue(entry.getValue()); + convertedMap.put(key, value); + } + return Value.fromMap(convertedMap); + } + if (obj instanceof ImprintRecord) { + return Value.fromRow((ImprintRecord) obj); + } + + throw new IllegalArgumentException("Cannot convert " + obj.getClass().getSimpleName() + + " to Imprint Value. Supported types: boolean, int, long, float, double, String, byte[], List, Map, ImprintRecord"); + } + + private MapKey convertToMapKey(Object obj) { + if (obj instanceof Integer) { + return MapKey.fromInt32((Integer) obj); + } + if (obj instanceof Long) { + return MapKey.fromInt64((Long) obj); + } + if (obj instanceof String) { + return MapKey.fromString((String) obj); + } + if (obj instanceof byte[]) { + return MapKey.fromBytes((byte[]) obj); + } + + throw new IllegalArgumentException("Invalid map key type: " + obj.getClass().getSimpleName() + + ". Map keys must be int, long, String, or byte[]"); + } + + @Override + public String toString() { + return String.format("ImprintRecordBuilder{schemaId=%s, fields=%d}", schemaId, fields.size()); + } +} \ No newline at end of file diff --git a/src/main/java/com/imprint/core/ImprintWriter.java b/src/main/java/com/imprint/core/ImprintWriter.java new file mode 100644 index 0000000..39ad9ea --- /dev/null +++ b/src/main/java/com/imprint/core/ImprintWriter.java @@ -0,0 +1,257 @@ +package com.imprint.core; + +import com.imprint.error.ErrorType; +import com.imprint.error.ImprintException; +import com.imprint.types.MapKey; +import com.imprint.types.Value; +import com.imprint.util.VarInt; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.util.*; + +/** + * A writer for constructing ImprintRecords by adding fields sequentially. + */ +public final class ImprintWriter { + private final SchemaId schemaId; + private final TreeMap fields; // keep fields in sorted order + + public ImprintWriter(SchemaId schemaId) { + this.schemaId = Objects.requireNonNull(schemaId, "SchemaId cannot be null"); + this.fields = new TreeMap<>(); + } + + /** + * Adds a field to the record being built. + */ + public ImprintWriter addField(int id, Value value) { + Objects.requireNonNull(value, "Value cannot be null"); + this.fields.put(id, value); + return this; + } + + /** + * Consumes the writer and builds an ImprintRecord. + */ + public ImprintRecord build() throws ImprintException { + var directory = new ArrayList(fields.size()); + var payloadBuffer = ByteBuffer.allocate(estimatePayloadSize()); + payloadBuffer.order(ByteOrder.LITTLE_ENDIAN); + + for (var entry : fields.entrySet()) { + int fieldId = entry.getKey(); + var value = entry.getValue(); + + directory.add(new DirectoryEntry(fieldId, value.getTypeCode(), payloadBuffer.position())); + serializeValue(value, payloadBuffer); + } + + // Create read-only view of the payload without copying + payloadBuffer.flip(); // limit = position, position = 0 + var payloadView = payloadBuffer.slice().asReadOnlyBuffer(); + + var header = new Header(new Flags((byte) 0), schemaId, payloadView.remaining()); + return new ImprintRecord(header, directory, payloadView); + } + + private int estimatePayloadSize() { + // More accurate estimation to reduce allocations + int estimatedSize = 0; + for (var value : fields.values()) { + estimatedSize += estimateValueSize(value); + } + // Add 25% buffer to reduce reallocations + return Math.max(estimatedSize + (estimatedSize / 4), fields.size() * 16); + } + + /** + * Estimates the serialized size in bytes for a given value. + * This method provides size estimates for payload buffer allocation, + * supporting both array-based and ByteBuffer-based value types. + * + * @param value the value to estimate size for + * @return estimated size in bytes including type-specific overhead + */ + private int estimateValueSize(Value value) { + // Use TypeHandler for simple types + switch (value.getTypeCode()) { + case NULL: + case BOOL: + case INT32: + case INT64: + case FLOAT32: + case FLOAT64: + case BYTES: + case STRING: + return value.getTypeCode().getHandler().estimateSize(value); + + case ARRAY: + List array = ((Value.ArrayValue) value).getValue(); + int arraySize = VarInt.encodedLength(array.size()) + 1; // length + type code + for (Value element : array) { + arraySize += estimateValueSize(element); + } + return arraySize; + + case MAP: + Map map = ((Value.MapValue) value).getValue(); + int mapSize = VarInt.encodedLength(map.size()) + 2; // length + 2 type codes + for (Map.Entry entry : map.entrySet()) { + mapSize += estimateMapKeySize(entry.getKey()); + mapSize += estimateValueSize(entry.getValue()); + } + return mapSize; + + case ROW: + // Estimate nested record size (rough approximation) + return 100; // Conservative estimate + + default: + return 32; // Default fallback + } + } + + private int estimateMapKeySize(MapKey key) { + switch (key.getTypeCode()) { + case INT32: return 4; + case INT64: return 8; + case BYTES: + byte[] bytes = ((MapKey.BytesKey) key).getValue(); + return VarInt.encodedLength(bytes.length) + bytes.length; + + case STRING: + var str = ((MapKey.StringKey) key).getValue(); + int utf8Length = str.getBytes(StandardCharsets.UTF_8).length; + return VarInt.encodedLength(utf8Length) + utf8Length; + + default: + return 16; // Default fallback + } + } + + private void serializeValue(Value value, ByteBuffer buffer) throws ImprintException { + switch (value.getTypeCode()) { + case NULL: + case BOOL: + case INT32: + case INT64: + case FLOAT32: + case FLOAT64: + case BYTES: + case STRING: + value.getTypeCode().getHandler().serialize(value, buffer); + break; + + case ARRAY: + serializeArray((Value.ArrayValue) value, buffer); + break; + + case MAP: + serializeMap((Value.MapValue) value, buffer); + break; + + case ROW: + Value.RowValue rowValue = (Value.RowValue) value; + var serializedRow = rowValue.getValue().serializeToBuffer(); + buffer.put(serializedRow); + break; + + default: + throw new ImprintException(ErrorType.SERIALIZATION_ERROR, + "Unknown type code: " + value.getTypeCode()); + } + } + + private void serializeArray(Value.ArrayValue arrayValue, ByteBuffer buffer) throws ImprintException { + var elements = arrayValue.getValue(); + VarInt.encode(elements.size(), buffer); + + if (elements.isEmpty()) return; + + // All elements must have the same type + var elementType = elements.get(0).getTypeCode(); + buffer.put(elementType.getCode()); + for (var element : elements) { + if (element.getTypeCode() != elementType) { + throw new ImprintException(ErrorType.SCHEMA_ERROR, + "Array elements must have same type code: " + + element.getTypeCode() + " != " + elementType); + } + serializeValue(element, buffer); + } + } + + private void serializeMap(Value.MapValue mapValue, ByteBuffer buffer) throws ImprintException { + var map = mapValue.getValue(); + VarInt.encode(map.size(), buffer); + + if (map.isEmpty()) { + return; + } + + // All keys and values must have consistent types + var iterator = map.entrySet().iterator(); + var first = iterator.next(); + var keyType = first.getKey().getTypeCode(); + var valueType = first.getValue().getTypeCode(); + + buffer.put(keyType.getCode()); + buffer.put(valueType.getCode()); + + // Serialize the first entry + serializeMapKey(first.getKey(), buffer); + serializeValue(first.getValue(), buffer); + + // Serialize remaining entries + while (iterator.hasNext()) { + var entry = iterator.next(); + if (entry.getKey().getTypeCode() != keyType) { + throw new ImprintException(ErrorType.SCHEMA_ERROR, + "Map keys must have same type code: " + + entry.getKey().getTypeCode() + " != " + keyType); + } + if (entry.getValue().getTypeCode() != valueType) { + throw new ImprintException(ErrorType.SCHEMA_ERROR, + "Map values must have same type code: " + + entry.getValue().getTypeCode() + " != " + valueType); + } + + serializeMapKey(entry.getKey(), buffer); + serializeValue(entry.getValue(), buffer); + } + } + + private void serializeMapKey(MapKey key, ByteBuffer buffer) throws ImprintException { + switch (key.getTypeCode()) { + case INT32: + MapKey.Int32Key int32Key = (MapKey.Int32Key) key; + buffer.putInt(int32Key.getValue()); + break; + + case INT64: + MapKey.Int64Key int64Key = (MapKey.Int64Key) key; + buffer.putLong(int64Key.getValue()); + break; + + case BYTES: + MapKey.BytesKey bytesKey = (MapKey.BytesKey) key; + byte[] bytes = bytesKey.getValue(); + VarInt.encode(bytes.length, buffer); + buffer.put(bytes); + break; + + case STRING: + MapKey.StringKey stringKey = (MapKey.StringKey) key; + byte[] stringBytes = stringKey.getValue().getBytes(StandardCharsets.UTF_8); + VarInt.encode(stringBytes.length, buffer); + buffer.put(stringBytes); + break; + + default: + throw new ImprintException(ErrorType.SERIALIZATION_ERROR, + "Invalid map key type: " + key.getTypeCode()); + } + } +} \ No newline at end of file diff --git a/src/main/java/com/imprint/core/SchemaId.java b/src/main/java/com/imprint/core/SchemaId.java new file mode 100644 index 0000000..cb03c1c --- /dev/null +++ b/src/main/java/com/imprint/core/SchemaId.java @@ -0,0 +1,12 @@ +package com.imprint.core; + +import lombok.Value; + +/** + * Schema identifier containing field-space ID and schema hash. + */ +@Value +public class SchemaId { + int fieldspaceId; + int schemaHash; +} \ No newline at end of file diff --git a/src/main/java/com/imprint/error/ErrorType.java b/src/main/java/com/imprint/error/ErrorType.java new file mode 100644 index 0000000..97b9772 --- /dev/null +++ b/src/main/java/com/imprint/error/ErrorType.java @@ -0,0 +1,18 @@ +package com.imprint.error; + +/** + * Types of errors that can occur in Imprint operations. + */ +public enum ErrorType { + INVALID_MAGIC, + UNSUPPORTED_VERSION, + BUFFER_UNDERFLOW, + FIELD_NOT_FOUND, + SCHEMA_ERROR, + INVALID_UTF8_STRING, + MALFORMED_VARINT, + TYPE_MISMATCH, + INVALID_TYPE_CODE, + SERIALIZATION_ERROR, + DESERIALIZATION_ERROR +} diff --git a/src/main/java/com/imprint/error/ImprintException.java b/src/main/java/com/imprint/error/ImprintException.java new file mode 100644 index 0000000..1b5a5d4 --- /dev/null +++ b/src/main/java/com/imprint/error/ImprintException.java @@ -0,0 +1,26 @@ +package com.imprint.error; + +import lombok.Getter; + +/** + * Exception thrown by Imprint operations. + */ +@Getter +public class ImprintException extends Exception { + private final ErrorType errorType; + + public ImprintException(ErrorType errorType, String message) { + super(message); + this.errorType = errorType; + } + + public ImprintException(ErrorType errorType, String message, Throwable cause) { + super(message, cause); + this.errorType = errorType; + } + + @Override + public String toString() { + return String.format("ImprintException{type=%s, message='%s'}", errorType, getMessage()); + } +} diff --git a/src/main/java/com/imprint/types/MapKey.java b/src/main/java/com/imprint/types/MapKey.java new file mode 100644 index 0000000..c0e0747 --- /dev/null +++ b/src/main/java/com/imprint/types/MapKey.java @@ -0,0 +1,163 @@ +package com.imprint.types; + +import com.imprint.error.ErrorType; +import com.imprint.error.ImprintException; +import lombok.EqualsAndHashCode; +import lombok.Getter; + +import java.util.Arrays; +import java.util.Objects; + +/** + * A subset of Value that's valid as a map key. + * Only Int32, Int64, Bytes, and String are valid map keys. + */ +public abstract class MapKey { + + public abstract TypeCode getTypeCode(); + public abstract boolean equals(Object obj); + public abstract int hashCode(); + public abstract String toString(); + + public static MapKey fromInt32(int value) { + return new Int32Key(value); + } + + public static MapKey fromInt64(long value) { + return new Int64Key(value); + } + + public static MapKey fromBytes(byte[] value) { + return new BytesKey(value); + } + + public static MapKey fromString(String value) { + return new StringKey(value); + } + + public static MapKey fromValue(Value value) throws ImprintException { + switch (value.getTypeCode()) { + case INT32: + return fromInt32(((Value.Int32Value) value).getValue()); + case INT64: + return fromInt64(((Value.Int64Value) value).getValue()); + case BYTES: + if (value instanceof Value.BytesBufferValue) { + return fromBytes(((Value.BytesBufferValue) value).getValue()); + } else { + return fromBytes(((Value.BytesValue) value).getValue()); + } + case STRING: + if (value instanceof Value.StringBufferValue) { + return fromString(((Value.StringBufferValue) value).getValue()); + } else { + return fromString(((Value.StringValue) value).getValue()); + } + default: + throw new ImprintException(ErrorType.TYPE_MISMATCH, + "Cannot convert " + value.getTypeCode() + " to MapKey"); + } + } + + public Value toValue() { + switch (getTypeCode()) { + case INT32: + return Value.fromInt32(((Int32Key) this).getValue()); + case INT64: + return Value.fromInt64(((Int64Key) this).getValue()); + case BYTES: + return Value.fromBytes(((BytesKey) this).getValue()); + case STRING: + return Value.fromString(((StringKey) this).getValue()); + default: + throw new IllegalStateException("Unknown MapKey type: " + getTypeCode()); + } + } + + @Getter + @EqualsAndHashCode(callSuper = false) + public static class Int32Key extends MapKey { + private final int value; + + public Int32Key(int value) { + this.value = value; + } + + @Override + public TypeCode getTypeCode() { return TypeCode.INT32; } + + @Override + public String toString() { + return String.valueOf(value); + } + } + + @Getter + @EqualsAndHashCode(callSuper = false) + public static class Int64Key extends MapKey { + private final long value; + + public Int64Key(long value) { + this.value = value; + } + + @Override + public TypeCode getTypeCode() { return TypeCode.INT64; } + + @Override + public String toString() { + return String.valueOf(value); + } + } + + public static class BytesKey extends MapKey { + private final byte[] value; + + public BytesKey(byte[] value) { + this.value = value.clone(); // defensive copy + } + + public byte[] getValue() { + return value.clone(); // defensive copy + } + + @Override + public TypeCode getTypeCode() { return TypeCode.BYTES; } + + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (obj == null || getClass() != obj.getClass()) return false; + BytesKey that = (BytesKey) obj; + return Arrays.equals(value, that.value); + } + + @Override + public int hashCode() { + return Arrays.hashCode(value); + } + + @Override + public String toString() { + return "bytes[" + value.length + "]"; + } + } + + @Getter + @EqualsAndHashCode(callSuper = false) + public static class StringKey extends MapKey { + private final String value; + + public StringKey(String value) { + this.value = Objects.requireNonNull(value, "String cannot be null"); + } + + @Override + public TypeCode getTypeCode() { return TypeCode.STRING; } + + @Override + public String toString() { + return "\"" + value + "\""; + } + } +} \ No newline at end of file diff --git a/src/main/java/com/imprint/types/TypeCode.java b/src/main/java/com/imprint/types/TypeCode.java new file mode 100644 index 0000000..6bf450d --- /dev/null +++ b/src/main/java/com/imprint/types/TypeCode.java @@ -0,0 +1,48 @@ +package com.imprint.types; + +import com.imprint.error.ImprintException; +import com.imprint.error.ErrorType; +import lombok.Getter; + +/** + * Type codes for Imprint values. + */ +public enum TypeCode { + NULL(0x0, TypeHandler.NULL), + BOOL(0x1, TypeHandler.BOOL), + INT32(0x2, TypeHandler.INT32), + INT64(0x3, TypeHandler.INT64), + FLOAT32(0x4, TypeHandler.FLOAT32), + FLOAT64(0x5, TypeHandler.FLOAT64), + BYTES(0x6, TypeHandler.BYTES), + STRING(0x7, TypeHandler.STRING), + ARRAY(0x8, null), // TODO: implement + MAP(0x9, null), // TODO: implement + ROW(0xA, null); // TODO: implement (basically a placeholder for user-defined type) + + @Getter + private final byte code; + private final TypeHandler handler; + + TypeCode(int code, TypeHandler handler) { + this.code = (byte) code; + this.handler = handler; + } + + public TypeHandler getHandler() { + if (handler == null) { + throw new UnsupportedOperationException("Handler not implemented for " + this); + } + return handler; + } + + public static TypeCode fromByte(byte code) throws ImprintException { + for (TypeCode type : values()) { + if (type.code == code) { + return type; + } + } + throw new ImprintException(ErrorType.INVALID_TYPE_CODE, + "Unknown type code: 0x" + Integer.toHexString(code & 0xFF)); + } +} \ No newline at end of file diff --git a/src/main/java/com/imprint/types/TypeHandler.java b/src/main/java/com/imprint/types/TypeHandler.java new file mode 100644 index 0000000..4b5830a --- /dev/null +++ b/src/main/java/com/imprint/types/TypeHandler.java @@ -0,0 +1,304 @@ +package com.imprint.types; + +import com.imprint.error.ImprintException; +import com.imprint.util.VarInt; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; + +/** + * Interface for handling type-specific serialization, deserialization, and size estimation. + * Note that primitives are potentially auto/un-boxed here which could impact performance slightly + * but having all the types in their own implementation helps keep things organized for now, especially + * for dealing with and testing more complex types in the future. + */ +public interface TypeHandler { + Value deserialize(ByteBuffer buffer) throws ImprintException; + void serialize(Value value, ByteBuffer buffer) throws ImprintException; + int estimateSize(Value value); + ByteBuffer readValueBytes(ByteBuffer buffer) throws ImprintException; + + // Static implementations for each type + TypeHandler NULL = new TypeHandler() { + @Override + public Value deserialize(ByteBuffer buffer) { + return Value.nullValue(); + } + + @Override + public void serialize(Value value, ByteBuffer buffer) { + // NULL values have no payload + } + + @Override + public int estimateSize(Value value) { + return 0; + } + + @Override + public ByteBuffer readValueBytes(ByteBuffer buffer) { + return ByteBuffer.allocate(0).asReadOnlyBuffer(); + } + }; + + TypeHandler BOOL = new TypeHandler() { + @Override + public Value deserialize(ByteBuffer buffer) throws ImprintException { + if (buffer.remaining() < 1) { + throw new ImprintException(com.imprint.error.ErrorType.BUFFER_UNDERFLOW, "Not enough bytes for bool"); + } + byte boolByte = buffer.get(); + if (boolByte == 0) return Value.fromBoolean(false); + if (boolByte == 1) return Value.fromBoolean(true); + throw new ImprintException(com.imprint.error.ErrorType.SCHEMA_ERROR, "Invalid boolean value: " + boolByte); + } + + @Override + public void serialize(Value value, ByteBuffer buffer) { + Value.BoolValue boolValue = (Value.BoolValue) value; + buffer.put((byte) (boolValue.getValue() ? 1 : 0)); + } + + @Override + public int estimateSize(Value value) { + return 1; + } + + @Override + public ByteBuffer readValueBytes(ByteBuffer buffer) { + var boolBuffer = buffer.slice(); + boolBuffer.limit(1); + buffer.position(buffer.position() + 1); + return boolBuffer.asReadOnlyBuffer(); + } + }; + + TypeHandler INT32 = new TypeHandler() { + @Override + public Value deserialize(ByteBuffer buffer) throws ImprintException { + if (buffer.remaining() < 4) { + throw new ImprintException(com.imprint.error.ErrorType.BUFFER_UNDERFLOW, "Not enough bytes for int32"); + } + return Value.fromInt32(buffer.getInt()); + } + + @Override + public void serialize(Value value, ByteBuffer buffer) { + Value.Int32Value int32Value = (Value.Int32Value) value; + buffer.putInt(int32Value.getValue()); + } + + @Override + public int estimateSize(Value value) { + return 4; + } + + @Override + public ByteBuffer readValueBytes(ByteBuffer buffer) { + var int32Buffer = buffer.slice(); + int32Buffer.limit(4); + buffer.position(buffer.position() + 4); + return int32Buffer.asReadOnlyBuffer(); + } + }; + + TypeHandler INT64 = new TypeHandler() { + @Override + public Value deserialize(ByteBuffer buffer) throws ImprintException { + if (buffer.remaining() < 8) { + throw new ImprintException(com.imprint.error.ErrorType.BUFFER_UNDERFLOW, "Not enough bytes for int64"); + } + return Value.fromInt64(buffer.getLong()); + } + + @Override + public void serialize(Value value, ByteBuffer buffer) { + Value.Int64Value int64Value = (Value.Int64Value) value; + buffer.putLong(int64Value.getValue()); + } + + @Override + public int estimateSize(Value value) { + return 8; + } + + @Override + public ByteBuffer readValueBytes(ByteBuffer buffer) { + var int64Buffer = buffer.slice(); + int64Buffer.limit(8); + buffer.position(buffer.position() + 8); + return int64Buffer.asReadOnlyBuffer(); + } + }; + + TypeHandler FLOAT32 = new TypeHandler() { + @Override + public Value deserialize(ByteBuffer buffer) throws ImprintException { + if (buffer.remaining() < 4) { + throw new ImprintException(com.imprint.error.ErrorType.BUFFER_UNDERFLOW, "Not enough bytes for float32"); + } + return Value.fromFloat32(buffer.getFloat()); + } + + @Override + public void serialize(Value value, ByteBuffer buffer) { + Value.Float32Value float32Value = (Value.Float32Value) value; + buffer.putFloat(float32Value.getValue()); + } + + @Override + public int estimateSize(Value value) { + return 4; + } + + @Override + public ByteBuffer readValueBytes(ByteBuffer buffer) { + var float32Buffer = buffer.slice(); + float32Buffer.limit(4); + buffer.position(buffer.position() + 4); + return float32Buffer.asReadOnlyBuffer(); + } + }; + + TypeHandler FLOAT64 = new TypeHandler() { + @Override + public Value deserialize(ByteBuffer buffer) throws ImprintException { + if (buffer.remaining() < 8) { + throw new ImprintException(com.imprint.error.ErrorType.BUFFER_UNDERFLOW, "Not enough bytes for float64"); + } + return Value.fromFloat64(buffer.getDouble()); + } + + @Override + public void serialize(Value value, ByteBuffer buffer) { + Value.Float64Value float64Value = (Value.Float64Value) value; + buffer.putDouble(float64Value.getValue()); + } + + @Override + public int estimateSize(Value value) { + return 8; + } + + @Override + public ByteBuffer readValueBytes(ByteBuffer buffer) { + var float64Buffer = buffer.slice(); + float64Buffer.limit(8); + buffer.position(buffer.position() + 8); + return float64Buffer.asReadOnlyBuffer(); + } + }; + + TypeHandler BYTES = new TypeHandler() { + @Override + public Value deserialize(ByteBuffer buffer) throws ImprintException { + VarInt.DecodeResult lengthResult = VarInt.decode(buffer); + int length = lengthResult.getValue(); + if (buffer.remaining() < length) { + throw new ImprintException(com.imprint.error.ErrorType.BUFFER_UNDERFLOW, "Not enough bytes for bytes value"); + } + var bytesView = buffer.slice(); + bytesView.limit(length); + buffer.position(buffer.position() + length); + return Value.fromBytesBuffer(bytesView.asReadOnlyBuffer()); + } + + @Override + public void serialize(Value value, ByteBuffer buffer) { + if (value instanceof Value.BytesBufferValue) { + Value.BytesBufferValue bufferValue = (Value.BytesBufferValue) value; + var bytesBuffer = bufferValue.getBuffer(); + VarInt.encode(bytesBuffer.remaining(), buffer); + buffer.put(bytesBuffer); + } else { + Value.BytesValue bytesValue = (Value.BytesValue) value; + byte[] bytes = bytesValue.getValue(); + VarInt.encode(bytes.length, buffer); + buffer.put(bytes); + } + } + + @Override + public int estimateSize(Value value) { + if (value instanceof Value.BytesBufferValue) { + Value.BytesBufferValue bufferValue = (Value.BytesBufferValue) value; + int length = bufferValue.getBuffer().remaining(); + return VarInt.encodedLength(length) + length; + } else { + byte[] bytes = ((Value.BytesValue) value).getValue(); + return VarInt.encodedLength(bytes.length) + bytes.length; + } + } + + @Override + public ByteBuffer readValueBytes(ByteBuffer buffer) throws ImprintException { + int originalPosition = buffer.position(); + VarInt.DecodeResult lengthResult = VarInt.decode(buffer); + int totalLength = lengthResult.getBytesRead() + lengthResult.getValue(); + buffer.position(originalPosition); + var valueBuffer = buffer.slice(); + valueBuffer.limit(totalLength); + buffer.position(buffer.position() + totalLength); + return valueBuffer.asReadOnlyBuffer(); + } + }; + + TypeHandler STRING = new TypeHandler() { + @Override + public Value deserialize(ByteBuffer buffer) throws ImprintException { + VarInt.DecodeResult strLengthResult = VarInt.decode(buffer); + int strLength = strLengthResult.getValue(); + if (buffer.remaining() < strLength) { + throw new ImprintException(com.imprint.error.ErrorType.BUFFER_UNDERFLOW, "Not enough bytes for string value"); + } + var stringBytesView = buffer.slice(); + stringBytesView.limit(strLength); + buffer.position(buffer.position() + strLength); + try { + return Value.fromStringBuffer(stringBytesView.asReadOnlyBuffer()); + } catch (Exception e) { + throw new ImprintException(com.imprint.error.ErrorType.INVALID_UTF8_STRING, "Invalid UTF-8 string"); + } + } + + @Override + public void serialize(Value value, ByteBuffer buffer) { + if (value instanceof Value.StringBufferValue) { + Value.StringBufferValue bufferValue = (Value.StringBufferValue) value; + var stringBuffer = bufferValue.getBuffer(); + VarInt.encode(stringBuffer.remaining(), buffer); + buffer.put(stringBuffer); + } else { + Value.StringValue stringValue = (Value.StringValue) value; + byte[] stringBytes = stringValue.getValue().getBytes(StandardCharsets.UTF_8); + VarInt.encode(stringBytes.length, buffer); + buffer.put(stringBytes); + } + } + + @Override + public int estimateSize(Value value) { + if (value instanceof Value.StringBufferValue) { + Value.StringBufferValue bufferValue = (Value.StringBufferValue) value; + int length = bufferValue.getBuffer().remaining(); + return VarInt.encodedLength(length) + length; + } else { + String str = ((Value.StringValue) value).getValue(); + int utf8Length = str.getBytes(StandardCharsets.UTF_8).length; + return VarInt.encodedLength(utf8Length) + utf8Length; + } + } + + @Override + public ByteBuffer readValueBytes(ByteBuffer buffer) throws ImprintException { + int originalPosition = buffer.position(); + VarInt.DecodeResult lengthResult = VarInt.decode(buffer); + int totalLength = lengthResult.getBytesRead() + lengthResult.getValue(); + buffer.position(originalPosition); + var valueBuffer = buffer.slice(); + valueBuffer.limit(totalLength); + buffer.position(buffer.position() + totalLength); + return valueBuffer.asReadOnlyBuffer(); + } + }; +} \ No newline at end of file diff --git a/src/main/java/com/imprint/types/Value.java b/src/main/java/com/imprint/types/Value.java new file mode 100644 index 0000000..4710ec5 --- /dev/null +++ b/src/main/java/com/imprint/types/Value.java @@ -0,0 +1,403 @@ +package com.imprint.types; + +import com.imprint.core.ImprintRecord; +import lombok.EqualsAndHashCode; +import lombok.Getter; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * A value that can be stored in an Imprint record. + */ +public abstract class Value { + + public abstract TypeCode getTypeCode(); + public abstract boolean equals(Object obj); + public abstract int hashCode(); + public abstract String toString(); + + // Factory methods + public static Value nullValue() { + return NullValue.INSTANCE; + } + + public static Value fromBoolean(boolean value) { + return new BoolValue(value); + } + + public static Value fromInt32(int value) { + return new Int32Value(value); + } + + public static Value fromInt64(long value) { + return new Int64Value(value); + } + + public static Value fromFloat32(float value) { + return new Float32Value(value); + } + + public static Value fromFloat64(double value) { + return new Float64Value(value); + } + + public static Value fromBytes(byte[] value) { + return new BytesValue(value); + } + + public static Value fromBytesBuffer(ByteBuffer value) { + return new BytesBufferValue(value); + } + + public static Value fromString(String value) { + return new StringValue(value); + } + + public static Value fromStringBuffer(ByteBuffer value) { + return new StringBufferValue(value); + } + + public static Value fromArray(List value) { + return new ArrayValue(value); + } + + public static Value fromMap(Map value) { + return new MapValue(value); + } + + public static Value fromRow(ImprintRecord value) { + return new RowValue(value); + } + + // Null Value + @EqualsAndHashCode(callSuper = false) + public static class NullValue extends Value { + public static final NullValue INSTANCE = new NullValue(); + + private NullValue() {} + + @Override + public TypeCode getTypeCode() { return TypeCode.NULL; } + + @Override + public String toString() { + return "null"; + } + } + + // Boolean Value + @Getter + @EqualsAndHashCode(callSuper = false) + public static class BoolValue extends Value { + private final boolean value; + + public BoolValue(boolean value) { + this.value = value; + } + + public boolean getValue() { return value; } + + @Override + public TypeCode getTypeCode() { return TypeCode.BOOL; } + + @Override + public String toString() { + return String.valueOf(value); + } + } + + // Int32 Value + @Getter + @EqualsAndHashCode(callSuper = false) + public static class Int32Value extends Value { + private final int value; + + public Int32Value(int value) { + this.value = value; + } + + @Override + public TypeCode getTypeCode() { return TypeCode.INT32; } + + @Override + public String toString() { + return String.valueOf(value); + } + } + + // Int64 Value + @Getter + @EqualsAndHashCode(callSuper = false) + public static class Int64Value extends Value { + private final long value; + + public Int64Value(long value) { + this.value = value; + } + + @Override + public TypeCode getTypeCode() { return TypeCode.INT64; } + + @Override + public String toString() { + return String.valueOf(value); + } + } + + // Float32 Value + @Getter + @EqualsAndHashCode(callSuper = false) + public static class Float32Value extends Value { + private final float value; + + public Float32Value(float value) { + this.value = value; + } + + @Override + public TypeCode getTypeCode() { return TypeCode.FLOAT32; } + + @Override + public String toString() { + return String.valueOf(value); + } + } + + // Float64 Value + @Getter + @EqualsAndHashCode(callSuper = false) + public static class Float64Value extends Value { + private final double value; + + public Float64Value(double value) { + this.value = value; + } + + @Override + public TypeCode getTypeCode() { return TypeCode.FLOAT64; } + + @Override + public String toString() { + return String.valueOf(value); + } + } + + // Bytes Value (array-based) + public static class BytesValue extends Value { + private final byte[] value; + + public BytesValue(byte[] value) { + this.value = value.clone(); // defensive copy + } + + public byte[] getValue() { + return value.clone(); // defensive copy + } + + @Override + public TypeCode getTypeCode() { return TypeCode.BYTES; } + + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (obj == null || getClass() != obj.getClass()) return false; + BytesValue that = (BytesValue) obj; + return Arrays.equals(value, that.value); + } + + @Override + public int hashCode() { + return Arrays.hashCode(value); + } + + @Override + public String toString() { + return "bytes[" + value.length + "]"; + } + } + + // Bytes Value (ByteBuffer-based, zero-copy) + public static class BytesBufferValue extends Value { + private final ByteBuffer value; + + public BytesBufferValue(ByteBuffer value) { + this.value = value.asReadOnlyBuffer(); // zero-copy read-only view + } + + public byte[] getValue() { + // Fallback to array when needed + byte[] array = new byte[value.remaining()]; + value.duplicate().get(array); + return array; + } + + public ByteBuffer getBuffer() { + return value.duplicate(); // zero-copy view + } + + @Override + public TypeCode getTypeCode() { return TypeCode.BYTES; } + + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (obj == null) return false; + if (obj instanceof BytesBufferValue) { + BytesBufferValue that = (BytesBufferValue) obj; + return value.equals(that.value); + } + if (obj instanceof BytesValue) { + BytesValue that = (BytesValue) obj; + return Arrays.equals(getValue(), that.getValue()); + } + return false; + } + + @Override + public int hashCode() { + return value.hashCode(); + } + + @Override + public String toString() { + return "bytes[" + value.remaining() + "]"; + } + } + + // String Value (String-based) + @Getter + @EqualsAndHashCode(callSuper = false) + public static class StringValue extends Value { + private final String value; + + public StringValue(String value) { + this.value = Objects.requireNonNull(value, "String cannot be null"); + } + + @Override + public TypeCode getTypeCode() { return TypeCode.STRING; } + + @Override + public String toString() { + return "\"" + value + "\""; + } + } + + // String Value (ByteBuffer-based, zero-copy) + public static class StringBufferValue extends Value { + private final ByteBuffer value; + private volatile String cachedString; // lazy decode + + public StringBufferValue(ByteBuffer value) { + this.value = value.asReadOnlyBuffer(); // zero-copy read-only view + } + + public String getValue() { + if (cachedString == null) { + synchronized (this) { + if (cachedString == null) { + var array = new byte[value.remaining()]; + value.duplicate().get(array); + cachedString = new String(array, StandardCharsets.UTF_8); + } + } + } + return cachedString; + } + + public ByteBuffer getBuffer() { + return value.duplicate(); // zero-copy view + } + + @Override + public TypeCode getTypeCode() { return TypeCode.STRING; } + + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (obj == null) return false; + if (obj instanceof StringBufferValue) { + StringBufferValue that = (StringBufferValue) obj; + return value.equals(that.value); + } + if (obj instanceof StringValue) { + StringValue that = (StringValue) obj; + return getValue().equals(that.getValue()); + } + return false; + } + + @Override + public int hashCode() { + return getValue().hashCode(); // Use string hash for consistency + } + + @Override + public String toString() { + return "\"" + getValue() + "\""; + } + } + + // Array Value + @Getter + @EqualsAndHashCode(callSuper = false) + public static class ArrayValue extends Value { + private final List value; + + public ArrayValue(List value) { + this.value = List.copyOf(Objects.requireNonNull(value, "Array cannot be null")); + } + + @Override + public TypeCode getTypeCode() { return TypeCode.ARRAY; } + + @Override + public String toString() { + return value.toString(); + } + } + + // Map Value + @Getter + @EqualsAndHashCode(callSuper = false) + public static class MapValue extends Value { + private final Map value; + + public MapValue(Map value) { + this.value = Map.copyOf(Objects.requireNonNull(value, "Map cannot be null")); + } + + @Override + public TypeCode getTypeCode() { return TypeCode.MAP; } + + @Override + public String toString() { + return value.toString(); + } + } + + // Row Value + @Getter + @EqualsAndHashCode(callSuper = false) + public static class RowValue extends Value { + private final ImprintRecord value; + + public RowValue(ImprintRecord value) { + this.value = Objects.requireNonNull(value, "Record cannot be null"); + } + + @Override + public TypeCode getTypeCode() { return TypeCode.ROW; } + + @Override + public String toString() { + return "Row{" + value + "}"; + } + } + +} \ No newline at end of file diff --git a/src/main/java/com/imprint/util/VarInt.java b/src/main/java/com/imprint/util/VarInt.java new file mode 100644 index 0000000..5c9a7e5 --- /dev/null +++ b/src/main/java/com/imprint/util/VarInt.java @@ -0,0 +1,118 @@ +package com.imprint.util; + +import com.imprint.error.ImprintException; +import com.imprint.error.ErrorType; +import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; +import java.nio.ByteBuffer; + +/** + * Utility class for encoding and decoding variable-length integers (VarInt). + * Supports encoding/decoding of 32-bit unsigned integers. + */ +public final class VarInt { + + private static final byte CONTINUATION_BIT = (byte) 0x80; + private static final byte SEGMENT_BITS = 0x7f; + private static final int MAX_VARINT_LEN = 5; // Enough for u32 + + private VarInt() {} // utility class + + + /** + * Encode a 32-bit unsigned integer as a VarInt into the given ByteBuffer. + * @param value the value to encode (treated as unsigned) + * @param buffer the buffer to write to + */ + public static void encode(int value, ByteBuffer buffer) { + // Convert to unsigned long for proper bit manipulation + long val = Integer.toUnsignedLong(value); + + // Encode at least one byte, then continue while value has more bits + do { + byte b = (byte) (val & SEGMENT_BITS); + val >>>= 7; + if (val != 0) { + b |= CONTINUATION_BIT; + } + buffer.put(b); + } while (val != 0); + } + + + /** + * Decode a VarInt from a ByteBuffer. + * @param buffer the buffer to decode from + * @return a DecodeResult containing the decoded value and number of bytes consumed + * @throws ImprintException if the VarInt is malformed + */ + public static DecodeResult decode(ByteBuffer buffer) throws ImprintException { + long result = 0; + int shift = 0; + int bytesRead = 0; + + while (true) { + if (bytesRead >= MAX_VARINT_LEN) { + throw new ImprintException(ErrorType.MALFORMED_VARINT, "VarInt too long"); + } + if (!buffer.hasRemaining()) { + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, + "Unexpected end of data while reading VarInt"); + } + + byte b = buffer.get(); + bytesRead++; + + // Check if adding these 7 bits would overflow + long segment = b & SEGMENT_BITS; + if (shift >= 32 || (shift == 28 && segment > 0xF)) { + throw new ImprintException(ErrorType.MALFORMED_VARINT, "VarInt overflow"); + } + + // Add the bottom 7 bits to the result + result |= segment << shift; + + // If the high bit is not set, this is the last byte + if ((b & CONTINUATION_BIT) == 0) { + break; + } + + shift += 7; + } + + return new DecodeResult((int) result, bytesRead); + } + + /** + * Calculate the number of bytes needed to encode the given value as a VarInt. + * @param value the value to encode (treated as unsigned) + * @return the number of bytes needed + */ + public static int encodedLength(int value) { + // Convert to unsigned long for proper bit manipulation + long val = Integer.toUnsignedLong(value); + int length = 1; + + // Count additional bytes needed for values >= 128 + while (val >= 0x80) { + val >>>= 7; + length++; + } + + return length; + } + + /** + * Result of a VarInt decode operation. + */ + @Getter + @AllArgsConstructor + @EqualsAndHashCode + @ToString + public static class DecodeResult { + private final int value; + private final int bytesRead; + } +} \ No newline at end of file diff --git a/src/test/java/com/imprint/ByteBufferIntegrationTest.java b/src/test/java/com/imprint/ByteBufferIntegrationTest.java new file mode 100644 index 0000000..9460cbf --- /dev/null +++ b/src/test/java/com/imprint/ByteBufferIntegrationTest.java @@ -0,0 +1,87 @@ +package com.imprint; + +import com.imprint.core.*; +import com.imprint.types.*; +import com.imprint.error.ImprintException; +import java.nio.ByteBuffer; +import java.util.*; + +/** + * Integration test to verify ByteBuffer functionality and zero-copy benefits. + */ +public class ByteBufferIntegrationTest { + + public static void main(String[] args) { + try { + testByteBufferFunctionality(); + testZeroCopy(); + System.out.println("All ByteBuffer integration tests passed!"); + } catch (Exception e) { + System.err.println("ByteBuffer integration test failed: " + e.getMessage()); + e.printStackTrace(); + System.exit(1); + } + } + + static void testByteBufferFunctionality() throws ImprintException { + System.out.println("Testing ByteBuffer functionality..."); + + SchemaId schemaId = new SchemaId(1, 0xdeadbeef); + ImprintWriter writer = new ImprintWriter(schemaId); + + writer.addField(1, Value.fromInt32(42)) + .addField(2, Value.fromString("zero-copy test")) + .addField(3, Value.fromBytes(new byte[]{1, 2, 3, 4, 5})); + + ImprintRecord record = writer.build(); + + // Test ByteBuffer serialization + ByteBuffer serializedBuffer = record.serializeToBuffer(); + assert serializedBuffer.isReadOnly() : "Serialized buffer should be read-only"; + + // Test deserialization from ByteBuffer + ImprintRecord deserialized = ImprintRecord.deserialize(serializedBuffer); + + assert deserialized.getValue(1).get().equals(Value.fromInt32(42)); + assert deserialized.getValue(2).get().equals(Value.fromString("zero-copy test")); + + // Test raw bytes access returns ByteBuffer + Optional rawBytes = deserialized.getRawBytes(1); + assert rawBytes.isPresent() : "Raw bytes should be present for field 1"; + assert rawBytes.get().isReadOnly() : "Raw bytes buffer should be read-only"; + + System.out.println("ByteBuffer functionality test passed"); + } + + static void testZeroCopy() { + System.out.println("Testing zero-copy"); + + // Create a large payload to demonstrate zero-copy benefits + byte[] largePayload = new byte[1024 * 1024]; // 1MB + Arrays.fill(largePayload, (byte) 0xAB); + + SchemaId schemaId = new SchemaId(2, 0xcafebabe); + ImprintWriter writer = new ImprintWriter(schemaId); + + try { + writer.addField(1, Value.fromBytes(largePayload)); + ImprintRecord record = writer.build(); + + // Test that getRawBytes returns a view, not a copy + Optional rawBytes = record.getRawBytes(1); + assert rawBytes.isPresent() : "Raw bytes should be present"; + + ByteBuffer rawBuffer = rawBytes.get(); + assert rawBuffer.isReadOnly() : "Raw buffer should be read-only"; + + // The buffer should be positioned at the start of the actual data + // (after the VarInt length prefix) + assert rawBuffer.remaining() > largePayload.length : "Buffer should include length prefix"; + + System.out.println("Zero-copy benefits test passed"); + + } catch (ImprintException e) { + throw new RuntimeException("Failed zero-copy test", e); + } + } +} \ No newline at end of file diff --git a/src/test/java/com/imprint/ComprehensiveTest.java b/src/test/java/com/imprint/ComprehensiveTest.java new file mode 100644 index 0000000..af7f0b0 --- /dev/null +++ b/src/test/java/com/imprint/ComprehensiveTest.java @@ -0,0 +1,208 @@ +package com.imprint; + +import com.imprint.core.*; +import com.imprint.types.*; +import com.imprint.error.ImprintException; +import com.imprint.util.VarInt; +import java.nio.ByteBuffer; +import java.util.*; + +/** + * Comprehensive test to verify all functionality works correctly. + */ +public class ComprehensiveTest { + + public static void main(String[] args) { + try { + testVarIntFunctionality(); + testValueTypes(); + testMapKeys(); + testComplexSerialization(); + testErrorHandling(); + testByteBufferPerformance(); + System.out.println("All comprehensive tests passed!"); + } catch (Exception e) { + System.err.println("Comprehensive test failed: " + e.getMessage()); + e.printStackTrace(); + System.exit(1); + } + } + + static void testVarIntFunctionality() throws ImprintException { + System.out.println("Testing VarInt functionality..."); + + // Test encoding/decoding of various values + int[] testValues = {0, 1, 127, 128, 16383, 16384, Integer.MAX_VALUE}; + + for (int value : testValues) { + ByteBuffer buffer = ByteBuffer.allocate(10); + VarInt.encode(value, buffer); + int encodedLength = buffer.position(); + + buffer.flip(); + VarInt.DecodeResult result = VarInt.decode(buffer); + + assert result.getValue() == value : "VarInt roundtrip failed for " + value; + assert result.getBytesRead() == encodedLength : "Bytes read mismatch for " + value; + } + + System.out.println("✓ VarInt functionality test passed"); + } + + static void testValueTypes() { + System.out.println("Testing Value types"); + + // Test all value types + Value nullVal = Value.nullValue(); + Value boolVal = Value.fromBoolean(true); + Value int32Val = Value.fromInt32(42); + Value int64Val = Value.fromInt64(123456789L); + Value float32Val = Value.fromFloat32(3.14f); + Value float64Val = Value.fromFloat64(2.718281828); + Value bytesVal = Value.fromBytes(new byte[]{1, 2, 3, 4}); + Value stringVal = Value.fromString("test"); + + // Test type codes + assert nullVal.getTypeCode() == TypeCode.NULL; + assert boolVal.getTypeCode() == TypeCode.BOOL; + assert int32Val.getTypeCode() == TypeCode.INT32; + assert int64Val.getTypeCode() == TypeCode.INT64; + assert float32Val.getTypeCode() == TypeCode.FLOAT32; + assert float64Val.getTypeCode() == TypeCode.FLOAT64; + assert bytesVal.getTypeCode() == TypeCode.BYTES; + assert stringVal.getTypeCode() == TypeCode.STRING; + + // Test value extraction + assert ((Value.BoolValue) boolVal).getValue(); + assert ((Value.Int32Value) int32Val).getValue() == 42; + assert ((Value.Int64Value) int64Val).getValue() == 123456789L; + assert ((Value.Float32Value) float32Val).getValue() == 3.14f; + assert ((Value.Float64Value) float64Val).getValue() == 2.718281828; + assert Arrays.equals(((Value.BytesValue) bytesVal).getValue(), new byte[]{1, 2, 3, 4}); + assert ((Value.StringValue) stringVal).getValue().equals("test"); + + System.out.println("✓ Value types test passed"); + } + + static void testMapKeys() throws ImprintException { + System.out.println("Testing MapKey functionality..."); + + MapKey int32Key = MapKey.fromInt32(42); + MapKey int64Key = MapKey.fromInt64(123L); + MapKey bytesKey = MapKey.fromBytes(new byte[]{1, 2, 3}); + MapKey stringKey = MapKey.fromString("test"); + + // Test conversion to/from Values + Value int32Value = int32Key.toValue(); + Value int64Value = int64Key.toValue(); + Value bytesValue = bytesKey.toValue(); + Value stringValue = stringKey.toValue(); + + assert MapKey.fromValue(int32Value).equals(int32Key); + assert MapKey.fromValue(int64Value).equals(int64Key); + assert MapKey.fromValue(bytesValue).equals(bytesKey); + assert MapKey.fromValue(stringValue).equals(stringKey); + + System.out.println("✓ MapKey functionality test passed"); + } + + static void testComplexSerialization() throws ImprintException { + System.out.println("Testing complex serialization..."); + + SchemaId schemaId = new SchemaId(1, 0xdeadbeef); + ImprintWriter writer = new ImprintWriter(schemaId); + + // Create complex nested structure + List array = Arrays.asList( + Value.fromInt32(1), + Value.fromInt32(2), + Value.fromInt32(3) + ); + + Map map = new HashMap<>(); + map.put(MapKey.fromString("key1"), Value.fromString("value1")); + map.put(MapKey.fromString("key2"), Value.fromString("value2")); + + writer.addField(1, Value.fromArray(array)) + .addField(2, Value.fromMap(map)) + .addField(3, Value.fromString("complex test")); + + ImprintRecord record = writer.build(); + + // Test ByteBuffer serialization + ByteBuffer serialized = record.serializeToBuffer(); + ImprintRecord deserialized = ImprintRecord.deserialize(serialized); + + // Verify complex structures + Value deserializedArray = deserialized.getValue(1).get(); + assert deserializedArray instanceof Value.ArrayValue; + List deserializedList = ((Value.ArrayValue) deserializedArray).getValue(); + assert deserializedList.size() == 3; + assert deserializedList.get(0).equals(Value.fromInt32(1)); + + Value deserializedMap = deserialized.getValue(2).get(); + assert deserializedMap instanceof Value.MapValue; + Map deserializedMapValue = ((Value.MapValue) deserializedMap).getValue(); + assert deserializedMapValue.size() == 2; + assert deserializedMapValue.get(MapKey.fromString("key1")).equals(Value.fromString("value1")); + + System.out.println("✓ Complex serialization test passed"); + } + + static void testErrorHandling() { + System.out.println("Testing error handling..."); + + try { + // Test invalid type code + TypeCode.fromByte((byte) 0xFF); + assert false : "Should have thrown exception for invalid type code"; + } catch (ImprintException e) { + assert e.getErrorType() == com.imprint.error.ErrorType.INVALID_TYPE_CODE; + } + + try { + // Test invalid magic byte + byte[] invalidData = new byte[15]; + invalidData[0] = 0x00; // wrong magic + ImprintRecord.deserialize(invalidData); + assert false : "Should have thrown exception for invalid magic"; + } catch (ImprintException e) { + assert e.getErrorType() == com.imprint.error.ErrorType.INVALID_MAGIC; + } + + System.out.println("✓ Error handling test passed"); + } + + static void testByteBufferPerformance() throws ImprintException { + System.out.println("Testing ByteBuffer performance benefits..."); + + // Create a record with moderate-sized data + byte[] testData = new byte[1024]; + for (int i = 0; i < testData.length; i++) { + testData[i] = (byte) (i % 256); + } + + SchemaId schemaId = new SchemaId(1, 0x12345678); + ImprintWriter writer = new ImprintWriter(schemaId); + writer.addField(1, Value.fromBytes(testData)) + .addField(2, Value.fromString("performance test")); + + ImprintRecord record = writer.build(); + + // Test that raw bytes access is zero-copy + Optional rawBytes = record.getRawBytes(1); + assert rawBytes.isPresent(); + assert rawBytes.get().isReadOnly(); + + // Test ByteBuffer serialization + ByteBuffer serialized = record.serializeToBuffer(); + assert serialized.isReadOnly(); + + // Verify deserialization works + ImprintRecord deserialized = ImprintRecord.deserialize(serialized); + Value retrievedBytes = deserialized.getValue(1).get(); + assert Arrays.equals(((Value.BytesValue) retrievedBytes).getValue(), testData); + + System.out.println("✓ ByteBuffer performance test passed"); + } +} \ No newline at end of file diff --git a/src/test/java/com/imprint/IntegrationTest.java b/src/test/java/com/imprint/IntegrationTest.java new file mode 100644 index 0000000..49cfce7 --- /dev/null +++ b/src/test/java/com/imprint/IntegrationTest.java @@ -0,0 +1,144 @@ +package com.imprint; + +import com.imprint.core.*; +import com.imprint.types.*; +import com.imprint.error.ImprintException; +import java.util.*; + +/** + * Integration test to verify the complete Java implementation works. + * This can be run as a simple main method without JUnit. + */ +public class IntegrationTest { + + public static void main(String[] args) { + try { + testBasicFunctionality(); + testArraysAndMaps(); + testNestedRecords(); + System.out.println("All integration tests passed!"); + } catch (Exception e) { + System.err.println("Integration test failed: " + e.getMessage()); + e.printStackTrace(); + System.exit(1); + } + } + + static void testBasicFunctionality() throws ImprintException { + System.out.println("Testing basic functionality..."); + + SchemaId schemaId = new SchemaId(1, 0xdeadbeef); + ImprintWriter writer = new ImprintWriter(schemaId); + + writer.addField(1, Value.fromInt32(42)) + .addField(2, Value.fromString("testing java imprint spec")) + .addField(3, Value.fromBoolean(true)) + .addField(4, Value.fromFloat64(3.14159)) + .addField(5, Value.fromBytes(new byte[]{1, 2, 3, 4})); + + ImprintRecord record = writer.build(); + + // Verify we can read values back + assert record.getValue(1).get().equals(Value.fromInt32(42)); + assert record.getValue(2).get().equals(Value.fromString("testing java imprint spec")); + assert record.getValue(3).get().equals(Value.fromBoolean(true)); + assert record.getValue(999).isEmpty(); // non-existent field + + // Test serialization round-trip + var buffer = record.serializeToBuffer(); + byte[] serialized = new byte[buffer.remaining()]; + buffer.get(serialized); + ImprintRecord deserialized = ImprintRecord.deserialize(serialized); + + assert deserialized.getValue(1).get().equals(Value.fromInt32(42)); + assert deserialized.getValue(2).get().equals(Value.fromString("testing java imprint spec")); + assert deserialized.getValue(3).get().equals(Value.fromBoolean(true)); + + System.out.println("✓ Basic functionality test passed"); + } + + static void testArraysAndMaps() throws ImprintException { + System.out.println("Testing arrays and maps..."); + + SchemaId schemaId = new SchemaId(2, 0xcafebabe); + ImprintWriter writer = new ImprintWriter(schemaId); + + // Create an array + List intArray = Arrays.asList( + Value.fromInt32(1), + Value.fromInt32(2), + Value.fromInt32(3) + ); + + // Create a map + Map stringToIntMap = new HashMap<>(); + stringToIntMap.put(MapKey.fromString("one"), Value.fromInt32(1)); + stringToIntMap.put(MapKey.fromString("two"), Value.fromInt32(2)); + + writer.addField(1, Value.fromArray(intArray)) + .addField(2, Value.fromMap(stringToIntMap)); + + ImprintRecord record = writer.build(); + + // Test serialization round-trip + var buffer = record.serializeToBuffer(); + byte[] serialized = new byte[buffer.remaining()]; + buffer.get(serialized); + ImprintRecord deserialized = ImprintRecord.deserialize(serialized); + + // Verify array + Value arrayValue = deserialized.getValue(1).get(); + assert arrayValue instanceof Value.ArrayValue; + List deserializedArray = ((Value.ArrayValue) arrayValue).getValue(); + assert deserializedArray.size() == 3; + assert deserializedArray.get(0).equals(Value.fromInt32(1)); + + // Verify map + Value mapValue = deserialized.getValue(2).get(); + assert mapValue instanceof Value.MapValue; + Map deserializedMap = ((Value.MapValue) mapValue).getValue(); + assert deserializedMap.size() == 2; + assert deserializedMap.get(MapKey.fromString("one")).equals(Value.fromInt32(1)); + + System.out.println("✓ Arrays and maps test passed"); + } + + static void testNestedRecords() throws ImprintException { + System.out.println("Testing nested records..."); + + // Create inner record + SchemaId innerSchemaId = new SchemaId(3, 0x12345678); + ImprintWriter innerWriter = new ImprintWriter(innerSchemaId); + innerWriter.addField(1, Value.fromString("nested data")) + .addField(2, Value.fromInt64(9876543210L)); + ImprintRecord innerRecord = innerWriter.build(); + + // Create outer record + SchemaId outerSchemaId = new SchemaId(4, 0x87654321); + ImprintWriter outerWriter = new ImprintWriter(outerSchemaId); + outerWriter.addField(1, Value.fromRow(innerRecord)) + .addField(2, Value.fromString("outer data")); + ImprintRecord outerRecord = outerWriter.build(); + + // Test serialization round-trip + var buffer = outerRecord.serializeToBuffer(); + byte[] serialized = new byte[buffer.remaining()]; + buffer.get(serialized); + ImprintRecord deserialized = ImprintRecord.deserialize(serialized); + + // Verify outer record + assert deserialized.getHeader().getSchemaId().getFieldspaceId() == 4; + assert deserialized.getValue(2).get().equals(Value.fromString("outer data")); + + // Verify nested record + Value rowValue = deserialized.getValue(1).get(); + assert rowValue instanceof Value.RowValue; + ImprintRecord nestedRecord = ((Value.RowValue) rowValue).getValue(); + + assert nestedRecord.getHeader().getSchemaId().getFieldspaceId() == 3; + assert nestedRecord.getValue(1).get().equals(Value.fromString("nested data")); + assert nestedRecord.getValue(2).get().equals(Value.fromInt64(9876543210L)); + + System.out.println("✓ Nested records test passed"); + } +} \ No newline at end of file diff --git a/src/test/java/com/imprint/benchmark/ProfilerTest.java b/src/test/java/com/imprint/benchmark/ProfilerTest.java new file mode 100644 index 0000000..233d993 --- /dev/null +++ b/src/test/java/com/imprint/benchmark/ProfilerTest.java @@ -0,0 +1,226 @@ +package com.imprint.benchmark; + +import com.imprint.core.*; +import com.imprint.types.Value; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Disabled; + +import java.util.Random; + +/** + * A test designed for profiling hotspots during development. + *

+ * To use with a profiler: + * 1. Remove @Disabled annotation + * 2. Run with JProfiler, VisualVM, or async-profiler: + * - JProfiler: Attach to test JVM + * - VisualVM: jvisualvm, attach to process + * - async-profiler: java -jar async-profiler.jar -d 30 -f profile.html + * 3. Look for hotspots in CPU sampling + *

+ * Key areas to examine: + * - Object allocation (memory profiling) + * - Method call frequency (CPU sampling) + * - GC pressure (memory profiling) + * - String operations and UTF-8 encoding + * - ByteBuffer operations + */ +@Disabled("Enable manually for profiling") +public class ProfilerTest { + + private static final int ITERATIONS = 1_000_000; + private static final int RECORD_SIZE = 20; + + @Test + void profileFieldAccess() throws Exception { + System.out.println("Starting profiler test - attach profiler now..."); + Thread.sleep(5000); // Give time to attach profiler + + // Create a representative record + var record = createTestRecord(); + + System.out.println("Beginning field access profiling..."); + long start = System.nanoTime(); + + // Simulate real-world access patterns + Random random = new Random(42); + int hits = 0; + + for (int i = 0; i < ITERATIONS; i++) { + // Random field access (hotspot) + int fieldId = random.nextInt(RECORD_SIZE) + 1; + var value = record.getValue(fieldId); + if (value.isPresent()) { + hits++; + + // Trigger string decoding (potential hotspot) + if (value.get().getTypeCode() == com.imprint.types.TypeCode.STRING) { + if (value.get() instanceof Value.StringBufferValue) { + ((Value.StringBufferValue) value.get()).getValue(); + } else { + ((Value.StringValue) value.get()).getValue(); + } + } + } + + // Some raw access (zero-copy path) + if (i % 10 == 0) { + record.getRawBytes(fieldId); + } + } + + long duration = System.nanoTime() - start; + System.out.printf("Completed %,d field accesses in %.2f ms (avg: %.1f ns/op, hits: %d)%n", + ITERATIONS, duration / 1_000_000.0, (double) duration / ITERATIONS, hits); + } + + @Test + void profileSerialization() throws Exception { + System.out.println("Starting serialization profiler test..."); + Thread.sleep(3000); + + var schemaId = new SchemaId(1, 0x12345678); + + System.out.println("Beginning serialization profiling..."); + long start = System.nanoTime(); + + // Create and serialize many records (allocation hotspot) + for (int i = 0; i < 100_000; i++) { + var writer = new ImprintWriter(schemaId); + + // Add various field types + writer.addField(1, Value.fromInt32(i)) + .addField(2, Value.fromString("test-string-" + i)) + .addField(3, Value.fromFloat64(i * 3.14159)) + .addField(4, Value.fromBytes(("bytes-" + i).getBytes())); + + var record = writer.build(); + var serialized = record.serializeToBuffer(); // Potential hotspot + + // Trigger some deserialization + if (i % 1000 == 0) { + var deserialized = ImprintRecord.deserialize(serialized); + deserialized.getValue(2); // String decoding hotspot + } + } + + long duration = System.nanoTime() - start; + System.out.printf("Completed serialization test in %.2f ms%n", duration / 1_000_000.0); + } + + @Test + void profileProjection() throws Exception { + System.out.println("Starting projection profiler test..."); + Thread.sleep(3000); + + var record = createLargeRecord(); + + System.out.println("Beginning projection profiling..."); + long start = System.nanoTime(); + + // Simulate analytical workload - project subset of fields repeatedly + for (int i = 0; i < 50_000; i++) { + // Project 10 fields out of 100 (common analytical pattern) + for (int fieldId = 1; fieldId <= 10; fieldId++) { + var value = record.getValue(fieldId); + if (value.isPresent()) { + // Force materialization of string values + if (value.get().getTypeCode() == com.imprint.types.TypeCode.STRING) { + if (value.get() instanceof Value.StringBufferValue) { + ((Value.StringBufferValue) value.get()).getValue(); + } + } + } + } + } + + long duration = System.nanoTime() - start; + System.out.printf("Completed projection test in %.2f ms%n", duration / 1_000_000.0); + } + + @Test + void profileMemoryAllocation() throws Exception { + System.out.println("Starting allocation profiler test..."); + Thread.sleep(3000); + + System.out.println("Beginning allocation profiling - watch for GC events..."); + + // Force allocation pressure to reveal GC hotspots + for (int batch = 0; batch < 1000; batch++) { + for (int i = 0; i < 1000; i++) { + var schemaId = new SchemaId(batch, i); + var writer = new ImprintWriter(schemaId); + + // Create strings of varying sizes (allocation pressure) + writer.addField(1, Value.fromString("small")) + .addField(2, Value.fromString("medium-length-string-" + i)) + .addField(3, Value.fromString("very-long-string-that-will-cause-more-allocation-pressure-" + batch + "-" + i)) + .addField(4, Value.fromBytes(new byte[100 + i % 1000])); // Varying byte arrays + + var record = writer.build(); + + // Some deserialization to trigger string decoding allocations + record.getValue(2); + record.getValue(3); + } + + if (batch % 100 == 0) { + System.out.printf("Completed batch %d/1000%n", batch); + } + } + + System.out.println("Allocation test complete - check GC logs and memory profiler"); + } + + private ImprintRecord createTestRecord() throws Exception { + var schemaId = new SchemaId(1, 0xdeadbeef); + var writer = new ImprintWriter(schemaId); + + for (int i = 1; i <= RECORD_SIZE; i++) { + switch (i % 4) { + case 0: + writer.addField(i, Value.fromInt32(i * 100)); + break; + case 1: + writer.addField(i, Value.fromString("field-value-" + i)); + break; + case 2: + writer.addField(i, Value.fromFloat64(i * 3.14159)); + break; + case 3: + writer.addField(i, Value.fromBytes(("bytes-" + i).getBytes())); + break; + } + } + + return writer.build(); + } + + private ImprintRecord createLargeRecord() throws Exception { + var schemaId = new SchemaId(2, 0xcafebabe); + var writer = new ImprintWriter(schemaId); + + // Create 100 fields with realistic data + for (int i = 1; i <= 100; i++) { + switch (i % 5) { + case 0: + writer.addField(i, Value.fromInt32(i)); + break; + case 1: + writer.addField(i, Value.fromString("user-name-" + i + "@example.com")); + break; + case 2: + writer.addField(i, Value.fromString("Some longer descriptive text for field " + i + " that might represent a comment or description")); + break; + case 3: + writer.addField(i, Value.fromFloat64(i * 2.718281828)); + break; + case 4: + writer.addField(i, Value.fromBytes(String.format("binary-data-%04d", i).getBytes())); + break; + } + } + + return writer.build(); + } +} \ No newline at end of file diff --git a/src/test/java/com/imprint/core/ImprintRecordBuilderTest.java b/src/test/java/com/imprint/core/ImprintRecordBuilderTest.java new file mode 100644 index 0000000..54dcfae --- /dev/null +++ b/src/test/java/com/imprint/core/ImprintRecordBuilderTest.java @@ -0,0 +1,234 @@ +package com.imprint.core; + +import com.imprint.error.ImprintException; +import com.imprint.types.Value; +import org.junit.jupiter.api.Test; + +import java.util.*; + +import static org.assertj.core.api.Assertions.*; + +class ImprintRecordBuilderTest { + + private static final SchemaId TEST_SCHEMA = new SchemaId(1, 0x12345678); + + @Test + void shouldCreateRecordWithPrimitiveTypes() throws ImprintException { + var record = ImprintRecord.builder(TEST_SCHEMA) + .field(1, true) + .field(2, 42) + .field(3, 123L) + .field(4, 3.14f) + .field(5, 2.718) + .field(6, "hello world") + .field(7, new byte[]{1, 2, 3}) + .nullField(8) + .build(); + + assertThat(record.getHeader().getSchemaId()).isEqualTo(TEST_SCHEMA); + assertThat(record.getDirectory()).hasSize(8); + + // Verify field values + assertThat(getFieldValue(record, 1, Value.BoolValue.class).getValue()).isTrue(); + assertThat(getFieldValue(record, 2, Value.Int32Value.class).getValue()).isEqualTo(42); + assertThat(getFieldValue(record, 3, Value.Int64Value.class).getValue()).isEqualTo(123L); + assertThat(getFieldValue(record, 4, Value.Float32Value.class).getValue()).isEqualTo(3.14f); + assertThat(getFieldValue(record, 5, Value.Float64Value.class).getValue()).isEqualTo(2.718); + assertThat(getStringValue(record, 6)).isEqualTo("hello world"); + assertThat(getBytesValue(record, 7)).isEqualTo(new byte[]{1, 2, 3}); + assertThat(record.getValue(8).get()).isInstanceOf(Value.NullValue.class); + } + + @Test + void shouldCreateRecordWithCollections() throws ImprintException { + var list = List.of(1, 2, 3); + var map = Map.of("key1", 100, "key2", 200); + + var record = ImprintRecord.builder(TEST_SCHEMA) + .field(1, list) + .field(2, map) + .build(); + + // Verify array + var arrayValue = getFieldValue(record, 1, Value.ArrayValue.class); + assertThat(arrayValue.getValue()).hasSize(3); + assertThat(((Value.Int32Value) arrayValue.getValue().get(0)).getValue()).isEqualTo(1); + assertThat(((Value.Int32Value) arrayValue.getValue().get(1)).getValue()).isEqualTo(2); + assertThat(((Value.Int32Value) arrayValue.getValue().get(2)).getValue()).isEqualTo(3); + + // Verify map + var mapValue = getFieldValue(record, 2, Value.MapValue.class); + assertThat(mapValue.getValue()).hasSize(2); + } + + @Test + void shouldCreateRecordWithNestedRecord() throws ImprintException { + var nestedRecord = ImprintRecord.builder(new SchemaId(2, 0x87654321)) + .field(1, "nested") + .field(2, 999) + .build(); + + var record = ImprintRecord.builder(TEST_SCHEMA) + .field(1, "parent") + .field(2, nestedRecord) + .build(); + + var rowValue = getFieldValue(record, 2, Value.RowValue.class); + var nested = rowValue.getValue(); + assertThat(getStringValue(nested, 1)).isEqualTo("nested"); + assertThat(getFieldValue(nested, 2, Value.Int32Value.class).getValue()).isEqualTo(999); + } + + @Test + void shouldSupportConditionalFields() throws ImprintException { + boolean includeOptional = true; + String optionalValue = "optional"; + + var record = ImprintRecord.builder(TEST_SCHEMA) + .field(1, "required") + .fieldIf(includeOptional, 2, optionalValue) + .fieldIfNotNull(3, null) // Should not add field + .fieldIfNotNull(4, "not null") // Should add field + .build(); + + assertThat(record.getDirectory()).hasSize(3); // Only fields 1, 2, 4 + assertThat(getStringValue(record, 1)).isEqualTo("required"); + assertThat(getStringValue(record, 2)).isEqualTo("optional"); + assertThat(record.getValue(3)).isEmpty(); // Not added + assertThat(getStringValue(record, 4)).isEqualTo("not null"); + } + + @Test + void shouldSupportBulkOperations() throws ImprintException { + var fieldsMap = Map.of( + 1, "bulk1", + 2, 42, + 3, true + ); + + var record = ImprintRecord.builder(TEST_SCHEMA) + .fields(fieldsMap) + .field(4, "additional") + .build(); + + assertThat(record.getDirectory()).hasSize(4); + assertThat(getStringValue(record, 1)).isEqualTo("bulk1"); + assertThat(getFieldValue(record, 2, Value.Int32Value.class).getValue()).isEqualTo(42); + assertThat(getFieldValue(record, 3, Value.BoolValue.class).getValue()).isTrue(); + assertThat(getStringValue(record, 4)).isEqualTo("additional"); + } + + @Test + void shouldProvideBuilderUtilities() { + var builder = ImprintRecord.builder(TEST_SCHEMA) + .field(1, "test") + .field(2, 42); + + assertThat(builder.hasField(1)).isTrue(); + assertThat(builder.hasField(3)).isFalse(); + assertThat(builder.fieldCount()).isEqualTo(2); + assertThat(builder.fieldIds()).containsExactly(1, 2); + } + + @Test + void shouldSupportAlternativeSchemaConstructor() throws ImprintException { + var record = ImprintRecord.builder(1, 0x12345678) + .field(1, "test") + .build(); + + assertThat(record.getHeader().getSchemaId().getFieldspaceId()).isEqualTo(1); + assertThat(record.getHeader().getSchemaId().getSchemaHash()).isEqualTo(0x12345678); + } + + @Test + void shouldRoundTripThroughSerialization() throws ImprintException { + var original = ImprintRecord.builder(TEST_SCHEMA) + .field(1, "test string") + .field(2, 42) + .field(3, 3.14159) + .field(4, true) + .field(5, new byte[]{0x01, 0x02, 0x03}) + .build(); + + var serialized = original.serializeToBuffer(); + var deserialized = ImprintRecord.deserialize(serialized); + + assertThat(deserialized.getHeader().getSchemaId()).isEqualTo(TEST_SCHEMA); + assertThat(getStringValue(deserialized, 1)).isEqualTo("test string"); + assertThat(getFieldValue(deserialized, 2, Value.Int32Value.class).getValue()).isEqualTo(42); + assertThat(getFieldValue(deserialized, 3, Value.Float64Value.class).getValue()).isEqualTo(3.14159); + assertThat(getFieldValue(deserialized, 4, Value.BoolValue.class).getValue()).isTrue(); + assertThat(getBytesValue(deserialized, 5)).isEqualTo(new byte[]{0x01, 0x02, 0x03}); + } + + // Error cases + + @Test + void shouldRejectDuplicateFieldIds() { + assertThatThrownBy(() -> + ImprintRecord.builder(TEST_SCHEMA) + .field(1, "first") + .field(1, "duplicate") // Same field ID + ).isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Field ID 1 already exists"); + } + + @Test + void shouldRejectEmptyRecord() { + assertThatThrownBy(() -> + ImprintRecord.builder(TEST_SCHEMA).build() + ).isInstanceOf(ImprintException.class) + .hasMessageContaining("Cannot build empty record"); + } + + @Test + void shouldRejectInvalidMapKeys() { + var mapWithInvalidKey = Map.of(3.14, "value"); // Double key not supported + + assertThatThrownBy(() -> + ImprintRecord.builder(TEST_SCHEMA) + .field(1, mapWithInvalidKey) + ).isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Invalid map key type: Double"); + } + + @Test + void shouldRejectNullValueWithoutExplicitNullField() { + assertThatThrownBy(() -> + ImprintRecord.builder(TEST_SCHEMA) + .field(1, (Value) null) + ).isInstanceOf(NullPointerException.class) + .hasMessageContaining("Value cannot be null - use nullField()"); + } + + // Helper methods for cleaner test assertions + + private T getFieldValue(ImprintRecord record, int fieldId, Class valueType) throws ImprintException { + var value = record.getValue(fieldId); + assertThat(value).isPresent(); + assertThat(value.get()).isInstanceOf(valueType); + return valueType.cast(value.get()); + } + + private String getStringValue(ImprintRecord record, int fieldId) throws ImprintException { + var value = record.getValue(fieldId).get(); + if (value instanceof Value.StringValue) { + return ((Value.StringValue) value).getValue(); + } else if (value instanceof Value.StringBufferValue) { + return ((Value.StringBufferValue) value).getValue(); + } else { + throw new AssertionError("Expected string value, got: " + value.getClass()); + } + } + + private byte[] getBytesValue(ImprintRecord record, int fieldId) throws ImprintException { + var value = record.getValue(fieldId).get(); + if (value instanceof Value.BytesValue) { + return ((Value.BytesValue) value).getValue(); + } else if (value instanceof Value.BytesBufferValue) { + return ((Value.BytesBufferValue) value).getValue(); + } else { + throw new AssertionError("Expected bytes value, got: " + value.getClass()); + } + } +} \ No newline at end of file diff --git a/src/test/java/com/imprint/core/ImprintRecordTest.java b/src/test/java/com/imprint/core/ImprintRecordTest.java new file mode 100644 index 0000000..0772580 --- /dev/null +++ b/src/test/java/com/imprint/core/ImprintRecordTest.java @@ -0,0 +1,232 @@ +package com.imprint.core; + +import com.imprint.error.ImprintException; +import com.imprint.error.ErrorType; +import com.imprint.types.Value; +import com.imprint.types.MapKey; +import org.junit.jupiter.api.Test; +import java.util.*; +import static org.assertj.core.api.Assertions.*; + +class ImprintRecordTest { + + // Helper method to extract string value from either StringValue or StringBufferValue + private String getStringValue(Value value) { + if (value instanceof Value.StringValue) { + return ((Value.StringValue) value).getValue(); + } else if (value instanceof Value.StringBufferValue) { + return ((Value.StringBufferValue) value).getValue(); + } else { + throw new IllegalArgumentException("Expected string value, got: " + value.getClass()); + } + } + + @Test + void shouldCreateSimpleRecord() throws ImprintException { + var schemaId = new SchemaId(1, 0xdeadbeef); + var writer = new ImprintWriter(schemaId); + + writer.addField(1, Value.fromInt32(42)) + .addField(2, Value.fromString("hello")); + + var record = writer.build(); + + assertThat(record.getHeader().getSchemaId()).isEqualTo(schemaId); + assertThat(record.getDirectory()).hasSize(2); + + Optional field1 = record.getValue(1); + Optional field2 = record.getValue(2); + + assertThat(field1).isPresent(); + assertThat(field1.get()).isInstanceOf(Value.Int32Value.class); + assertThat(((Value.Int32Value) field1.get()).getValue()).isEqualTo(42); + + assertThat(field2).isPresent(); + assertThat(field2.get().getTypeCode()).isEqualTo(com.imprint.types.TypeCode.STRING); + String stringValue = getStringValue(field2.get()); + assertThat(stringValue).isEqualTo("hello"); + + // Non-existent field should return empty + assertThat(record.getValue(999)).isEmpty(); + } + + @Test + void shouldRoundtripThroughSerialization() throws ImprintException { + var schemaId = new SchemaId(1, 0xdeadbeef); + var writer = new ImprintWriter(schemaId); + + writer.addField(1, Value.nullValue()) + .addField(2, Value.fromBoolean(true)) + .addField(3, Value.fromInt32(42)) + .addField(4, Value.fromInt64(123456789L)) + .addField(5, Value.fromFloat32(3.14f)) + .addField(6, Value.fromFloat64(2.718281828)) + .addField(7, Value.fromBytes(new byte[]{1, 2, 3, 4})) + .addField(8, Value.fromString("test string")); + + var original = writer.build(); + + // Serialize and deserialize + var buffer = original.serializeToBuffer(); + byte[] serialized = new byte[buffer.remaining()]; + buffer.get(serialized); + var deserialized = ImprintRecord.deserialize(serialized); + + // Verify metadata + assertThat(deserialized.getHeader().getSchemaId().getFieldspaceId()).isEqualTo(1); + assertThat(deserialized.getHeader().getSchemaId().getSchemaHash()).isEqualTo(0xdeadbeef); + assertThat(deserialized.getDirectory()).hasSize(8); + + // Verify all values + assertThat(deserialized.getValue(1)).contains(Value.nullValue()); + assertThat(deserialized.getValue(2)).contains(Value.fromBoolean(true)); + assertThat(deserialized.getValue(3)).contains(Value.fromInt32(42)); + assertThat(deserialized.getValue(4)).contains(Value.fromInt64(123456789L)); + assertThat(deserialized.getValue(5)).contains(Value.fromFloat32(3.14f)); + assertThat(deserialized.getValue(6)).contains(Value.fromFloat64(2.718281828)); + assertThat(deserialized.getValue(7)).contains(Value.fromBytes(new byte[]{1, 2, 3, 4})); + assertThat(deserialized.getValue(8)).contains(Value.fromString("test string")); + + // Non-existent field + assertThat(deserialized.getValue(999)).isEmpty(); + } + + @Test + void shouldHandleArrays() throws ImprintException { + var schemaId = new SchemaId(1, 0xdeadbeef); + var writer = new ImprintWriter(schemaId); + + List intArray = Arrays.asList( + Value.fromInt32(1), + Value.fromInt32(2), + Value.fromInt32(3) + ); + + writer.addField(1, Value.fromArray(intArray)); + ImprintRecord record = writer.build(); + + // Serialize and deserialize + var buffer = record.serializeToBuffer(); + byte[] serialized = new byte[buffer.remaining()]; + buffer.get(serialized); + var deserialized = ImprintRecord.deserialize(serialized); + + Optional arrayValue = deserialized.getValue(1); + assertThat(arrayValue).isPresent(); + assertThat(arrayValue.get()).isInstanceOf(Value.ArrayValue.class); + + List deserializedArray = ((Value.ArrayValue) arrayValue.get()).getValue(); + assertThat(deserializedArray).hasSize(3); + assertThat(deserializedArray.get(0)).isEqualTo(Value.fromInt32(1)); + assertThat(deserializedArray.get(1)).isEqualTo(Value.fromInt32(2)); + assertThat(deserializedArray.get(2)).isEqualTo(Value.fromInt32(3)); + } + + @Test + void shouldHandleMaps() throws ImprintException { + var schemaId = new SchemaId(1, 0xdeadbeef); + var writer = new ImprintWriter(schemaId); + + var map = new HashMap(); + map.put(MapKey.fromString("key1"), Value.fromInt32(1)); + map.put(MapKey.fromString("key2"), Value.fromInt32(2)); + + writer.addField(1, Value.fromMap(map)); + var record = writer.build(); + + // Serialize and deserialize + var buffer = record.serializeToBuffer(); + byte[] serialized = new byte[buffer.remaining()]; + buffer.get(serialized); + var deserialized = ImprintRecord.deserialize(serialized); + + Optional mapValue = deserialized.getValue(1); + assertThat(mapValue).isPresent(); + assertThat(mapValue.get()).isInstanceOf(Value.MapValue.class); + + Map deserializedMap = ((Value.MapValue) mapValue.get()).getValue(); + assertThat(deserializedMap).hasSize(2); + assertThat(deserializedMap.get(MapKey.fromString("key1"))).isEqualTo(Value.fromInt32(1)); + assertThat(deserializedMap.get(MapKey.fromString("key2"))).isEqualTo(Value.fromInt32(2)); + } + + @Test + void shouldHandleNestedRecords() throws ImprintException { + // Create inner record + var innerSchemaId = new SchemaId(2, 0xcafebabe); + var innerWriter = new ImprintWriter(innerSchemaId); + innerWriter.addField(1, Value.fromInt32(42)) + .addField(2, Value.fromString("nested")); + var innerRecord = innerWriter.build(); + + // Create outer record containing inner record + var outerSchemaId = new SchemaId(1, 0xdeadbeef); + var outerWriter = new ImprintWriter(outerSchemaId); + outerWriter.addField(1, Value.fromRow(innerRecord)) + .addField(2, Value.fromInt64(123L)); + var outerRecord = outerWriter.build(); + + // Serialize and deserialize + var buffer = outerRecord.serializeToBuffer(); + byte[] serialized = new byte[buffer.remaining()]; + buffer.get(serialized); + var deserialized = ImprintRecord.deserialize(serialized); + + // Verify outer record metadata + assertThat(deserialized.getHeader().getSchemaId().getFieldspaceId()).isEqualTo(1); + assertThat(deserialized.getHeader().getSchemaId().getSchemaHash()).isEqualTo(0xdeadbeef); + + // Verify nested record + Optional rowValue = deserialized.getValue(1); + assertThat(rowValue).isPresent(); + assertThat(rowValue.get()).isInstanceOf(Value.RowValue.class); + + var nestedRecord = ((Value.RowValue) rowValue.get()).getValue(); + assertThat(nestedRecord.getHeader().getSchemaId().getFieldspaceId()).isEqualTo(2); + assertThat(nestedRecord.getHeader().getSchemaId().getSchemaHash()).isEqualTo(0xcafebabe); + + assertThat(nestedRecord.getValue(1)).contains(Value.fromInt32(42)); + assertThat(nestedRecord.getValue(2)).contains(Value.fromString("nested")); + + // Verify outer record field + assertThat(deserialized.getValue(2)).contains(Value.fromInt64(123L)); + } + + @Test + void shouldRejectInvalidMagic() { + byte[] invalidData = new byte[15]; + invalidData[0] = 0x00; // wrong magic + + assertThatThrownBy(() -> ImprintRecord.deserialize(invalidData)) + .isInstanceOf(ImprintException.class) + .extracting("errorType") + .isEqualTo(ErrorType.INVALID_MAGIC); + } + + @Test + void shouldRejectUnsupportedVersion() { + byte[] invalidData = new byte[15]; + invalidData[0] = (byte) 0x49; // correct magic + invalidData[1] = (byte) 0xFF; // wrong version + + assertThatThrownBy(() -> ImprintRecord.deserialize(invalidData)) + .isInstanceOf(ImprintException.class) + .extracting("errorType") + .isEqualTo(ErrorType.UNSUPPORTED_VERSION); + } + + @Test + void shouldHandleDuplicateFieldIds() throws ImprintException { + var schemaId = new SchemaId(1, 0xdeadbeef); + var writer = new ImprintWriter(schemaId); + + // Add duplicate field IDs - last one should win + writer.addField(1, Value.fromInt32(42)) + .addField(1, Value.fromInt32(43)); + + var record = writer.build(); + + assertThat(record.getDirectory()).hasSize(1); + assertThat(record.getValue(1)).contains(Value.fromInt32(43)); + } +} \ No newline at end of file diff --git a/src/test/java/com/imprint/types/MapKeyTest.java b/src/test/java/com/imprint/types/MapKeyTest.java new file mode 100644 index 0000000..08f4180 --- /dev/null +++ b/src/test/java/com/imprint/types/MapKeyTest.java @@ -0,0 +1,91 @@ +package com.imprint.types; + +import com.imprint.error.ImprintException; +import com.imprint.error.ErrorType; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.*; + +class MapKeyTest { + + @Test + void shouldCreateMapKeysFromValues() throws ImprintException { + var int32Key = MapKey.fromValue(Value.fromInt32(42)); + var int64Key = MapKey.fromValue(Value.fromInt64(123L)); + var bytesKey = MapKey.fromValue(Value.fromBytes(new byte[]{1, 2, 3})); + var stringKey = MapKey.fromValue(Value.fromString("test")); + + assertThat(int32Key).isInstanceOf(MapKey.Int32Key.class); + assertThat(((MapKey.Int32Key) int32Key).getValue()).isEqualTo(42); + + assertThat(int64Key).isInstanceOf(MapKey.Int64Key.class); + assertThat(((MapKey.Int64Key) int64Key).getValue()).isEqualTo(123L); + + assertThat(bytesKey).isInstanceOf(MapKey.BytesKey.class); + assertThat(((MapKey.BytesKey) bytesKey).getValue()).containsExactly(1, 2, 3); + + assertThat(stringKey).isInstanceOf(MapKey.StringKey.class); + assertThat(((MapKey.StringKey) stringKey).getValue()).isEqualTo("test"); + } + + @Test + void shouldConvertBackToValues() { + var int32Key = MapKey.fromInt32(42); + var stringKey = MapKey.fromString("test"); + + var int32Value = int32Key.toValue(); + var stringValue = stringKey.toValue(); + + assertThat(int32Value).isInstanceOf(Value.Int32Value.class); + assertThat(((Value.Int32Value) int32Value).getValue()).isEqualTo(42); + + assertThat(stringValue).isInstanceOf(Value.StringValue.class); + assertThat(((Value.StringValue) stringValue).getValue()).isEqualTo("test"); + } + + @Test + void shouldRejectInvalidValueTypes() { + var boolValue = Value.fromBoolean(true); + var arrayValue = Value.fromArray(java.util.Collections.emptyList()); + + assertThatThrownBy(() -> MapKey.fromValue(boolValue)) + .isInstanceOf(ImprintException.class) + .extracting("errorType") + .isEqualTo(ErrorType.TYPE_MISMATCH); + + assertThatThrownBy(() -> MapKey.fromValue(arrayValue)) + .isInstanceOf(ImprintException.class) + .extracting("errorType") + .isEqualTo(ErrorType.TYPE_MISMATCH); + } + + @Test + void shouldHandleEqualityAndHashing() { + var key1 = MapKey.fromString("test"); + var key2 = MapKey.fromString("test"); + var key3 = MapKey.fromString("different"); + + assertThat(key1).isEqualTo(key2); + assertThat(key1).isNotEqualTo(key3); + assertThat(key1.hashCode()).isEqualTo(key2.hashCode()); + } + + @Test + void shouldDefensiveCopyBytes() { + byte[] original = {1, 2, 3}; + var bytesKey = MapKey.fromBytes(original); + + // Modify original array + original[0] = 99; + + // Key should be unchanged + assertThat(((MapKey.BytesKey) bytesKey).getValue()).containsExactly(1, 2, 3); + } + + @Test + void shouldHaveCorrectTypeCodes() { + assertThat(MapKey.fromInt32(1).getTypeCode()).isEqualTo(TypeCode.INT32); + assertThat(MapKey.fromInt64(1L).getTypeCode()).isEqualTo(TypeCode.INT64); + assertThat(MapKey.fromBytes(new byte[]{1}).getTypeCode()).isEqualTo(TypeCode.BYTES); + assertThat(MapKey.fromString("test").getTypeCode()).isEqualTo(TypeCode.STRING); + } +} \ No newline at end of file diff --git a/src/test/java/com/imprint/types/TypeHandlerTest.java b/src/test/java/com/imprint/types/TypeHandlerTest.java new file mode 100644 index 0000000..9a4ae85 --- /dev/null +++ b/src/test/java/com/imprint/types/TypeHandlerTest.java @@ -0,0 +1,279 @@ +package com.imprint.types; + +import com.imprint.error.ImprintException; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +import static org.assertj.core.api.Assertions.*; + +/** + * Tests for individual TypeHandler implementations. + * Validates serialization, deserialization, and size estimation for each type. + */ +class TypeHandlerTest { + + @Test + void testNullHandler() throws ImprintException { + var handler = TypeHandler.NULL; + var value = Value.nullValue(); + + // Size estimation + assertThat(handler.estimateSize(value)).isEqualTo(0); + + // Serialization + var buffer = ByteBuffer.allocate(10); + handler.serialize(value, buffer); + assertThat(buffer.position()).isEqualTo(0); // NULL writes nothing + + // Deserialization + buffer.flip(); + var deserialized = handler.deserialize(buffer); + assertThat(deserialized).isEqualTo(value); + + // readValueBytes + buffer.clear(); + var valueBytes = handler.readValueBytes(buffer); + assertThat(valueBytes.remaining()).isEqualTo(0); + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void testBoolHandler(boolean testValue) throws ImprintException { + var handler = TypeHandler.BOOL; + var value = Value.fromBoolean(testValue); + + // Size estimation + assertThat(handler.estimateSize(value)).isEqualTo(1); + + // Round-trip test + var buffer = ByteBuffer.allocate(10); + handler.serialize(value, buffer); + assertThat(buffer.position()).isEqualTo(1); + + buffer.flip(); + var deserialized = handler.deserialize(buffer); + assertThat(deserialized).isEqualTo(value); + assertThat(((Value.BoolValue) deserialized).getValue()).isEqualTo(testValue); + } + + @ParameterizedTest + @ValueSource(ints = {0, 1, -1, Integer.MAX_VALUE, Integer.MIN_VALUE, 42, -42}) + void testInt32Handler(int testValue) throws ImprintException { + var handler = TypeHandler.INT32; + var value = Value.fromInt32(testValue); + + // Size estimation + assertThat(handler.estimateSize(value)).isEqualTo(4); + + // Round-trip test + var buffer = ByteBuffer.allocate(10).order(ByteOrder.LITTLE_ENDIAN); + handler.serialize(value, buffer); + assertThat(buffer.position()).isEqualTo(4); + + buffer.flip(); + var deserialized = handler.deserialize(buffer); + assertThat(deserialized).isEqualTo(value); + assertThat(((Value.Int32Value) deserialized).getValue()).isEqualTo(testValue); + } + + @ParameterizedTest + @ValueSource(longs = {0L, 1L, -1L, Long.MAX_VALUE, Long.MIN_VALUE, 123456789L}) + void testInt64Handler(long testValue) throws ImprintException { + var handler = TypeHandler.INT64; + var value = Value.fromInt64(testValue); + + // Size estimation + assertThat(handler.estimateSize(value)).isEqualTo(8); + + // Round-trip test + var buffer = ByteBuffer.allocate(20).order(ByteOrder.LITTLE_ENDIAN); + handler.serialize(value, buffer); + assertThat(buffer.position()).isEqualTo(8); + + buffer.flip(); + var deserialized = handler.deserialize(buffer); + assertThat(deserialized).isEqualTo(value); + assertThat(((Value.Int64Value) deserialized).getValue()).isEqualTo(testValue); + } + + @ParameterizedTest + @ValueSource(floats = {0.0f, 1.0f, -1.0f, Float.MAX_VALUE, Float.MIN_VALUE, 3.14159f, Float.NaN, Float.POSITIVE_INFINITY}) + void testFloat32Handler(float testValue) throws ImprintException { + var handler = TypeHandler.FLOAT32; + var value = Value.fromFloat32(testValue); + + // Size estimation + assertThat(handler.estimateSize(value)).isEqualTo(4); + + // Round-trip test + var buffer = ByteBuffer.allocate(10).order(ByteOrder.LITTLE_ENDIAN); + handler.serialize(value, buffer); + assertThat(buffer.position()).isEqualTo(4); + + buffer.flip(); + var deserialized = handler.deserialize(buffer); + assertThat(deserialized).isEqualTo(value); + + float deserializedValue = ((Value.Float32Value) deserialized).getValue(); + if (Float.isNaN(testValue)) { + assertThat(deserializedValue).isNaN(); + } else { + assertThat(deserializedValue).isEqualTo(testValue); + } + } + + @ParameterizedTest + @ValueSource(doubles = {0.0, 1.0, -1.0, Double.MAX_VALUE, Double.MIN_VALUE, Math.PI, Double.NaN, Double.POSITIVE_INFINITY}) + void testFloat64Handler(double testValue) throws ImprintException { + var handler = TypeHandler.FLOAT64; + var value = Value.fromFloat64(testValue); + + // Size estimation + assertThat(handler.estimateSize(value)).isEqualTo(8); + + // Round-trip test + var buffer = ByteBuffer.allocate(20).order(ByteOrder.LITTLE_ENDIAN); + handler.serialize(value, buffer); + assertThat(buffer.position()).isEqualTo(8); + + buffer.flip(); + var deserialized = handler.deserialize(buffer); + assertThat(deserialized).isEqualTo(value); + + double deserializedValue = ((Value.Float64Value) deserialized).getValue(); + if (Double.isNaN(testValue)) { + assertThat(deserializedValue).isNaN(); + } else { + assertThat(deserializedValue).isEqualTo(testValue); + } + } + + @ParameterizedTest + @ValueSource(strings = {"", "hello", "世界", "a very long string that exceeds typical buffer sizes and contains unicode: 🚀🎉", "null\0bytes"}) + void testStringHandler(String testValue) throws ImprintException { + var handler = TypeHandler.STRING; + var value = Value.fromString(testValue); + + byte[] utf8Bytes = testValue.getBytes(java.nio.charset.StandardCharsets.UTF_8); + int expectedSize = com.imprint.util.VarInt.encodedLength(utf8Bytes.length) + utf8Bytes.length; + + // Size estimation + assertThat(handler.estimateSize(value)).isEqualTo(expectedSize); + + // Round-trip test + var buffer = ByteBuffer.allocate(expectedSize + 20).order(ByteOrder.LITTLE_ENDIAN); + handler.serialize(value, buffer); + + buffer.flip(); + var deserialized = handler.deserialize(buffer); + + // Should return StringBufferValue (zero-copy implementation) + assertThat(deserialized).isInstanceOf(Value.StringBufferValue.class); + + String deserializedString; + if (deserialized instanceof Value.StringBufferValue) { + deserializedString = ((Value.StringBufferValue) deserialized).getValue(); + } else { + deserializedString = ((Value.StringValue) deserialized).getValue(); + } + + assertThat(deserializedString).isEqualTo(testValue); + } + + @Test + void testBytesHandlerWithArrayValue() throws ImprintException { + var handler = TypeHandler.BYTES; + byte[] testBytes = {0, 1, 2, (byte) 0xFF, 42, 127, -128}; + var value = Value.fromBytes(testBytes); + + int expectedSize = com.imprint.util.VarInt.encodedLength(testBytes.length) + testBytes.length; + + // Size estimation + assertThat(handler.estimateSize(value)).isEqualTo(expectedSize); + + // Round-trip test + var buffer = ByteBuffer.allocate(expectedSize + 20).order(ByteOrder.LITTLE_ENDIAN); + handler.serialize(value, buffer); + + buffer.flip(); + var deserialized = handler.deserialize(buffer); + + // Should return BytesBufferValue (zero-copy implementation) + assertThat(deserialized).isInstanceOf(Value.BytesBufferValue.class); + + byte[] deserializedBytes = ((Value.BytesBufferValue) deserialized).getValue(); + assertThat(deserializedBytes).isEqualTo(testBytes); + } + + @Test + void testBytesHandlerWithBufferValue() throws ImprintException { + var handler = TypeHandler.BYTES; + byte[] testBytes = {10, 20, 30, 40}; + var bufferValue = Value.fromBytesBuffer(ByteBuffer.wrap(testBytes).asReadOnlyBuffer()); + + int expectedSize = com.imprint.util.VarInt.encodedLength(testBytes.length) + testBytes.length; + + // Size estimation + assertThat(handler.estimateSize(bufferValue)).isEqualTo(expectedSize); + + // Round-trip test + var buffer = ByteBuffer.allocate(expectedSize + 20).order(ByteOrder.LITTLE_ENDIAN); + handler.serialize(bufferValue, buffer); + + buffer.flip(); + var deserialized = handler.deserialize(buffer); + + byte[] deserializedBytes = ((Value.BytesBufferValue) deserialized).getValue(); + assertThat(deserializedBytes).isEqualTo(testBytes); + } + + @Test + void testStringHandlerWithBufferValue() throws ImprintException { + var handler = TypeHandler.STRING; + String testString = "zero-copy string test"; + byte[] utf8Bytes = testString.getBytes(java.nio.charset.StandardCharsets.UTF_8); + var bufferValue = Value.fromStringBuffer(ByteBuffer.wrap(utf8Bytes).asReadOnlyBuffer()); + + int expectedSize = com.imprint.util.VarInt.encodedLength(utf8Bytes.length) + utf8Bytes.length; + + // Size estimation + assertThat(handler.estimateSize(bufferValue)).isEqualTo(expectedSize); + + // Round-trip test + var buffer = ByteBuffer.allocate(expectedSize + 20).order(ByteOrder.LITTLE_ENDIAN); + handler.serialize(bufferValue, buffer); + + buffer.flip(); + var deserialized = handler.deserialize(buffer); + + String deserializedString = ((Value.StringBufferValue) deserialized).getValue(); + assertThat(deserializedString).isEqualTo(testString); + } + + @Test + void testBoolHandlerInvalidValue() { + var handler = TypeHandler.BOOL; + var buffer = ByteBuffer.allocate(10); + buffer.put((byte) 2); // Invalid boolean value + buffer.flip(); + + assertThatThrownBy(() -> handler.deserialize(buffer)) + .isInstanceOf(ImprintException.class) + .hasMessageContaining("Invalid boolean value: 2"); + } + + @Test + void testHandlerBufferUnderflow() { + // Test that handlers properly detect buffer underflow + var int32Handler = TypeHandler.INT32; + var buffer = ByteBuffer.allocate(2); // Too small for int32 + + assertThatThrownBy(() -> int32Handler.deserialize(buffer)) + .isInstanceOf(ImprintException.class) + .hasMessageContaining("Not enough bytes for int32"); + } +} \ No newline at end of file diff --git a/src/test/java/com/imprint/types/ValueTest.java b/src/test/java/com/imprint/types/ValueTest.java new file mode 100644 index 0000000..9dd99c9 --- /dev/null +++ b/src/test/java/com/imprint/types/ValueTest.java @@ -0,0 +1,123 @@ +package com.imprint.types; + +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class ValueTest { + + @Test + void shouldCreateNullValue() { + Value value = Value.nullValue(); + + assertThat(value).isInstanceOf(Value.NullValue.class); + assertThat(value.getTypeCode()).isEqualTo(TypeCode.NULL); + assertThat(value.toString()).isEqualTo("null"); + } + + @Test + void shouldCreateBooleanValues() { + Value trueValue = Value.fromBoolean(true); + Value falseValue = Value.fromBoolean(false); + + assertThat(trueValue).isInstanceOf(Value.BoolValue.class); + assertThat(((Value.BoolValue) trueValue).getValue()).isTrue(); + assertThat(trueValue.getTypeCode()).isEqualTo(TypeCode.BOOL); + + assertThat(falseValue).isInstanceOf(Value.BoolValue.class); + assertThat(((Value.BoolValue) falseValue).getValue()).isFalse(); + assertThat(falseValue.getTypeCode()).isEqualTo(TypeCode.BOOL); + } + + @Test + void shouldCreateNumericValues() { + var int32 = Value.fromInt32(42); + var int64 = Value.fromInt64(123456789L); + var float32 = Value.fromFloat32(3.14f); + var float64 = Value.fromFloat64(2.718281828); + + assertThat(int32.getTypeCode()).isEqualTo(TypeCode.INT32); + assertThat(((Value.Int32Value) int32).getValue()).isEqualTo(42); + + assertThat(int64.getTypeCode()).isEqualTo(TypeCode.INT64); + assertThat(((Value.Int64Value) int64).getValue()).isEqualTo(123456789L); + + assertThat(float32.getTypeCode()).isEqualTo(TypeCode.FLOAT32); + assertThat(((Value.Float32Value) float32).getValue()).isEqualTo(3.14f); + + assertThat(float64.getTypeCode()).isEqualTo(TypeCode.FLOAT64); + assertThat(((Value.Float64Value) float64).getValue()).isEqualTo(2.718281828); + } + + @Test + void shouldCreateBytesAndStringValues() { + byte[] bytes = {1, 2, 3, 4}; + var bytesValue = Value.fromBytes(bytes); + var stringValue = Value.fromString("hello"); + + assertThat(bytesValue.getTypeCode()).isEqualTo(TypeCode.BYTES); + assertThat(((Value.BytesValue) bytesValue).getValue()).isEqualTo(bytes); + + assertThat(stringValue.getTypeCode()).isEqualTo(TypeCode.STRING); + assertThat(((Value.StringValue) stringValue).getValue()).isEqualTo("hello"); + } + + @Test + void shouldCreateArrayValues() { + List elements = Arrays.asList( + Value.fromInt32(1), + Value.fromInt32(2), + Value.fromInt32(3) + ); + Value arrayValue = Value.fromArray(elements); + + assertThat(arrayValue.getTypeCode()).isEqualTo(TypeCode.ARRAY); + assertThat(((Value.ArrayValue) arrayValue).getValue()).isEqualTo(elements); + } + + @Test + void shouldCreateMapValues() { + var map = new HashMap(); + map.put(MapKey.fromString("key1"), Value.fromInt32(1)); + map.put(MapKey.fromString("key2"), Value.fromInt32(2)); + + Value mapValue = Value.fromMap(map); + + assertThat(mapValue.getTypeCode()).isEqualTo(TypeCode.MAP); + assertThat(((Value.MapValue) mapValue).getValue()).isEqualTo(map); + } + + @Test + void shouldHandleEqualityCorrectly() { + var int1 = Value.fromInt32(42); + var int2 = Value.fromInt32(42); + var int3 = Value.fromInt32(43); + + assertThat(int1).isEqualTo(int2); + assertThat(int1).isNotEqualTo(int3); + assertThat(int1.hashCode()).isEqualTo(int2.hashCode()); + } + + @Test + void shouldDefensiveCopyArrays() { + byte[] original = {1, 2, 3}; + var bytesValue = Value.fromBytes(original); + + // Modify original array + original[0] = 99; + + // Value should be unchanged + assertThat(((Value.BytesValue) bytesValue).getValue()).containsExactly(1, 2, 3); + } + + @Test + void shouldRejectNullString() { + assertThatThrownBy(() -> Value.fromString(null)) + .isInstanceOf(NullPointerException.class); + } +} \ No newline at end of file diff --git a/src/test/java/com/imprint/util/VarIntTest.java b/src/test/java/com/imprint/util/VarIntTest.java new file mode 100644 index 0000000..677afb7 --- /dev/null +++ b/src/test/java/com/imprint/util/VarIntTest.java @@ -0,0 +1,115 @@ +package com.imprint.util; + +import com.imprint.error.ImprintException; +import com.imprint.error.ErrorType; +import org.junit.jupiter.api.Test; +import java.nio.ByteBuffer; +import static org.assertj.core.api.Assertions.*; + +class VarIntTest { + + @Test + void shouldRoundtripCommonValues() throws ImprintException { + int[] testCases = { + 0, 1, 127, 128, 16383, 16384, 2097151, 2097152, + 268435455, 268435456, -1 // -1 as unsigned is 0xFFFFFFFF + }; + + for (int value : testCases) { + ByteBuffer buffer = ByteBuffer.allocate(10); + VarInt.encode(value, buffer); + int encodedLength = buffer.position(); + + buffer.flip(); + VarInt.DecodeResult result = VarInt.decode(buffer); + + assertThat(result.getValue()).isEqualTo(value); + assertThat(result.getBytesRead()).isEqualTo(encodedLength); + } + } + + @Test + void shouldEncodeKnownValuesCorrectly() { + // Test cases with known encodings + assertEncodedBytes(0, 0x00); + assertEncodedBytes(1, 0x01); + assertEncodedBytes(127, 0x7f); + assertEncodedBytes(128, 0x80, 0x01); + assertEncodedBytes(16383, 0xff, 0x7f); + assertEncodedBytes(16384, 0x80, 0x80, 0x01); + } + + private void assertEncodedBytes(int value, int... expectedBytes) { + ByteBuffer buffer = ByteBuffer.allocate(10); + VarInt.encode(value, buffer); + buffer.flip(); + + byte[] actual = new byte[buffer.remaining()]; + buffer.get(actual); + + byte[] expected = new byte[expectedBytes.length]; + for (int i = 0; i < expectedBytes.length; i++) { + expected[i] = (byte) expectedBytes[i]; + } + + assertThat(actual).containsExactly(expected); + } + + @Test + void shouldWorkWithByteBuffer() throws ImprintException { + ByteBuffer buffer = ByteBuffer.allocate(10); + VarInt.encode(16384, buffer); + + buffer.flip(); + VarInt.DecodeResult result = VarInt.decode(buffer); + + assertThat(result.getValue()).isEqualTo(16384); + assertThat(result.getBytesRead()).isEqualTo(3); + } + + @Test + void shouldCalculateEncodedLength() { + assertThat(VarInt.encodedLength(0)).isEqualTo(1); + assertThat(VarInt.encodedLength(127)).isEqualTo(1); + assertThat(VarInt.encodedLength(128)).isEqualTo(2); + assertThat(VarInt.encodedLength(16383)).isEqualTo(2); + assertThat(VarInt.encodedLength(16384)).isEqualTo(3); + assertThat(VarInt.encodedLength(-1)).isEqualTo(5); // max u32 + } + + @Test + void shouldHandleBufferUnderflow() { + ByteBuffer buffer = ByteBuffer.allocate(1); + buffer.put((byte) 0x80); // incomplete varint + buffer.flip(); + + assertThatThrownBy(() -> VarInt.decode(buffer)) + .isInstanceOf(ImprintException.class) + .extracting("errorType") + .isEqualTo(ErrorType.BUFFER_UNDERFLOW); + } + + @Test + void shouldHandleOverlongEncoding() { + ByteBuffer buffer = ByteBuffer.allocate(10); + buffer.put(new byte[]{(byte) 0x80, (byte) 0x80, (byte) 0x80, (byte) 0x80, (byte) 0x80, 0x01}); + buffer.flip(); + + assertThatThrownBy(() -> VarInt.decode(buffer)) + .isInstanceOf(ImprintException.class) + .extracting("errorType") + .isEqualTo(ErrorType.MALFORMED_VARINT); + } + + @Test + void shouldHandleOverflow() { + ByteBuffer buffer = ByteBuffer.allocate(10); + buffer.put(new byte[]{(byte) 0x80, (byte) 0x80, (byte) 0x80, (byte) 0x80, 0x10}); + buffer.flip(); + + assertThatThrownBy(() -> VarInt.decode(buffer)) + .isInstanceOf(ImprintException.class) + .extracting("errorType") + .isEqualTo(ErrorType.MALFORMED_VARINT); + } +} \ No newline at end of file From dd4fdbc01cd78e4345a3c928d67466bae92b91f4 Mon Sep 17 00:00:00 2001 From: expanded-for-real Date: Sun, 1 Jun 2025 14:02:51 -0400 Subject: [PATCH 03/34] initial commit for imprint-java --- PROFILING.md | 106 +++++ gradle/wrapper/gradle-wrapper.properties | 7 + gradlew | 251 +++++++++++ gradlew.bat | 94 ++++ profile.sh | 98 +++++ settings.gradle | 1 + .../benchmark/ComparisonBenchmark.java | 344 +++++++++++++++ .../benchmark/FieldAccessBenchmark.java | 276 ++++++++++++ .../com/imprint/benchmark/MergeBenchmark.java | 163 +++++++ .../benchmark/SerializationBenchmark.java | 166 ++++++++ src/main/java/com/imprint/Constants.java | 10 + .../java/com/imprint/core/DirectoryEntry.java | 23 + src/main/java/com/imprint/core/Flags.java | 12 + src/main/java/com/imprint/core/Header.java | 13 + .../java/com/imprint/core/ImprintRecord.java | 365 ++++++++++++++++ .../imprint/core/ImprintRecordBuilder.java | 233 ++++++++++ .../java/com/imprint/core/ImprintWriter.java | 257 +++++++++++ src/main/java/com/imprint/core/SchemaId.java | 12 + .../java/com/imprint/error/ErrorType.java | 18 + .../com/imprint/error/ImprintException.java | 26 ++ src/main/java/com/imprint/types/MapKey.java | 163 +++++++ src/main/java/com/imprint/types/TypeCode.java | 48 +++ .../java/com/imprint/types/TypeHandler.java | 304 +++++++++++++ src/main/java/com/imprint/types/Value.java | 403 ++++++++++++++++++ src/main/java/com/imprint/util/VarInt.java | 118 +++++ .../imprint/ByteBufferIntegrationTest.java | 87 ++++ .../java/com/imprint/ComprehensiveTest.java | 208 +++++++++ .../java/com/imprint/IntegrationTest.java | 144 +++++++ .../com/imprint/benchmark/ProfilerTest.java | 226 ++++++++++ .../core/ImprintRecordBuilderTest.java | 234 ++++++++++ .../com/imprint/core/ImprintRecordTest.java | 232 ++++++++++ .../java/com/imprint/types/MapKeyTest.java | 91 ++++ .../com/imprint/types/TypeHandlerTest.java | 279 ++++++++++++ .../java/com/imprint/types/ValueTest.java | 123 ++++++ .../java/com/imprint/util/VarIntTest.java | 115 +++++ 35 files changed, 5250 insertions(+) create mode 100644 PROFILING.md create mode 100644 gradle/wrapper/gradle-wrapper.properties create mode 100644 gradlew create mode 100644 gradlew.bat create mode 100644 profile.sh create mode 100644 settings.gradle create mode 100644 src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java create mode 100644 src/jmh/java/com/imprint/benchmark/FieldAccessBenchmark.java create mode 100644 src/jmh/java/com/imprint/benchmark/MergeBenchmark.java create mode 100644 src/jmh/java/com/imprint/benchmark/SerializationBenchmark.java create mode 100644 src/main/java/com/imprint/Constants.java create mode 100644 src/main/java/com/imprint/core/DirectoryEntry.java create mode 100644 src/main/java/com/imprint/core/Flags.java create mode 100644 src/main/java/com/imprint/core/Header.java create mode 100644 src/main/java/com/imprint/core/ImprintRecord.java create mode 100644 src/main/java/com/imprint/core/ImprintRecordBuilder.java create mode 100644 src/main/java/com/imprint/core/ImprintWriter.java create mode 100644 src/main/java/com/imprint/core/SchemaId.java create mode 100644 src/main/java/com/imprint/error/ErrorType.java create mode 100644 src/main/java/com/imprint/error/ImprintException.java create mode 100644 src/main/java/com/imprint/types/MapKey.java create mode 100644 src/main/java/com/imprint/types/TypeCode.java create mode 100644 src/main/java/com/imprint/types/TypeHandler.java create mode 100644 src/main/java/com/imprint/types/Value.java create mode 100644 src/main/java/com/imprint/util/VarInt.java create mode 100644 src/test/java/com/imprint/ByteBufferIntegrationTest.java create mode 100644 src/test/java/com/imprint/ComprehensiveTest.java create mode 100644 src/test/java/com/imprint/IntegrationTest.java create mode 100644 src/test/java/com/imprint/benchmark/ProfilerTest.java create mode 100644 src/test/java/com/imprint/core/ImprintRecordBuilderTest.java create mode 100644 src/test/java/com/imprint/core/ImprintRecordTest.java create mode 100644 src/test/java/com/imprint/types/MapKeyTest.java create mode 100644 src/test/java/com/imprint/types/TypeHandlerTest.java create mode 100644 src/test/java/com/imprint/types/ValueTest.java create mode 100644 src/test/java/com/imprint/util/VarIntTest.java diff --git a/PROFILING.md b/PROFILING.md new file mode 100644 index 0000000..43f7b4e --- /dev/null +++ b/PROFILING.md @@ -0,0 +1,106 @@ +# Performance Profiling Guide + +This guide helps developers identify performance hotspots in the Imprint Java implementation. + +## Quick Start + +```bash +# Run field access profiling with async-profiler +./profile.sh profileFieldAccess asyncprofiler + +# Run memory allocation profiling with JFR +./profile.sh profileMemoryAllocation jfr +``` + +## Available Tests + +1. **`profileFieldAccess`** - Measures random field access patterns + - Focus: Binary search, TypeHandler dispatch, string decoding + - Good for: Optimizing read-heavy workloads + +2. **`profileSerialization`** - Tests record creation and serialization + - Focus: Object allocation, ByteBuffer operations, encoding + - Good for: Optimizing write-heavy workloads + +3. **`profileProjection`** - Simulates analytical field projection + - Focus: Bulk field access, string materialization + - Good for: Optimizing analytical workloads + +4. **`profileMemoryAllocation`** - Stress tests allocation patterns + - Focus: GC pressure, object lifecycle, string allocations + - Good for: Reducing memory footprint + +## Profiler Options + +### async-profiler (Recommended) +- **Setup**: Download from [async-profiler releases](https://github.com/jvm-profiling-tools/async-profiler/releases) +- **Output**: HTML flame graphs in `profiler-results/` +- **Best for**: CPU profiling, finding hot methods + +### Java Flight Recorder (JFR) +- **Setup**: Built into OpenJDK 11+ +- **Output**: `.jfr` files for Java Mission Control +- **Best for**: Memory profiling, GC analysis + +### VisualVM +- **Setup**: `jvisualvm` (usually pre-installed) +- **Output**: Real-time profiling UI +- **Best for**: Interactive profiling, heap dumps + +## Expected Hotspots + +Based on our optimizations, watch for: + +### CPU Hotspots +1. **Binary search** in `findDirectoryIndex()` - should be fast +2. **String decoding** in `StringBufferValue.getValue()` - lazy evaluation +3. **TypeHandler dispatch** - interface calls vs switch statements +4. **VarInt encoding/decoding** - variable-length integers +5. **ByteBuffer operations** - slicing and positioning + +### Memory Hotspots +1. **String allocations** during UTF-8 conversion +2. **Temporary objects** in binary search (should be eliminated) +3. **ByteBuffer slicing** (should be zero-copy) +4. **Array allocations** for BYTES values + +## Analyzing Results + +### async-profiler Flame Graphs +- **Wide bars** = high CPU usage (hotspots) +- **Deep stacks** = call overhead +- **Look for**: Red bars in `deserializeValue`, `findDirectoryIndex`, string operations + +### JFR Analysis +1. Open `.jfr` file in Java Mission Control +2. Check "Memory" tab for allocation hotspots +3. Check "Method Profiling" for CPU usage +4. Look at GC events for memory pressure + +### Memory Profiler Tips +- **Object allocation rate** should be low for zero-copy operations +- **String allocations** should be rare (lazy evaluation) +- **GC frequency** indicates allocation pressure + +## Performance Targets + +Based on our benchmarks: +- **Single field access**: < 50ns +- **Zero-copy operations**: < 30ns +- **String decoding**: Should be lazy, not in hot path +- **Binary search**: O(log n), ~10ns per comparison + +## Common Issues + +1. **High string allocation** → Enable lazy string decoding +2. **Object allocations in binary search** → Check DirectoryEntry creation +3. **ByteBuffer copying** → Ensure zero-copy slicing +4. **Switch statement overhead** → TypeHandler dispatch working? + +## Profiling Best Practices + +1. **Warm up JVM** - Run tests multiple times +2. **Use realistic data** - Match production patterns +3. **Profile different scenarios** - Read vs write heavy +4. **Check allocations** - Memory profiling reveals hidden costs +5. **Compare before/after** - Measure optimization impact \ No newline at end of file diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 0000000..37f853b --- /dev/null +++ b/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,7 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-8.13-bin.zip +networkTimeout=10000 +validateDistributionUrl=true +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/gradlew b/gradlew new file mode 100644 index 0000000..faf9300 --- /dev/null +++ b/gradlew @@ -0,0 +1,251 @@ +#!/bin/sh + +# +# Copyright © 2015-2021 the original authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# + +############################################################################## +# +# Gradle start up script for POSIX generated by Gradle. +# +# Important for running: +# +# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is +# noncompliant, but you have some other compliant shell such as ksh or +# bash, then to run this script, type that shell name before the whole +# command line, like: +# +# ksh Gradle +# +# Busybox and similar reduced shells will NOT work, because this script +# requires all of these POSIX shell features: +# * functions; +# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», +# «${var#prefix}», «${var%suffix}», and «$( cmd )»; +# * compound commands having a testable exit status, especially «case»; +# * various built-in commands including «command», «set», and «ulimit». +# +# Important for patching: +# +# (2) This script targets any POSIX shell, so it avoids extensions provided +# by Bash, Ksh, etc; in particular arrays are avoided. +# +# The "traditional" practice of packing multiple parameters into a +# space-separated string is a well documented source of bugs and security +# problems, so this is (mostly) avoided, by progressively accumulating +# options in "$@", and eventually passing that to Java. +# +# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, +# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; +# see the in-line comments for details. +# +# There are tweaks for specific operating systems such as AIX, CygWin, +# Darwin, MinGW, and NonStop. +# +# (3) This script is generated from the Groovy template +# https://github.com/gradle/gradle/blob/HEAD/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# within the Gradle project. +# +# You can find Gradle at https://github.com/gradle/gradle/. +# +############################################################################## + +# Attempt to set APP_HOME + +# Resolve links: $0 may be a link +app_path=$0 + +# Need this for daisy-chained symlinks. +while + APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path + [ -h "$app_path" ] +do + ls=$( ls -ld "$app_path" ) + link=${ls#*' -> '} + case $link in #( + /*) app_path=$link ;; #( + *) app_path=$APP_HOME$link ;; + esac +done + +# This is normally unused +# shellcheck disable=SC2034 +APP_BASE_NAME=${0##*/} +# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) +APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s\n' "$PWD" ) || exit + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD=maximum + +warn () { + echo "$*" +} >&2 + +die () { + echo + echo "$*" + echo + exit 1 +} >&2 + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "$( uname )" in #( + CYGWIN* ) cygwin=true ;; #( + Darwin* ) darwin=true ;; #( + MSYS* | MINGW* ) msys=true ;; #( + NONSTOP* ) nonstop=true ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD=$JAVA_HOME/jre/sh/java + else + JAVACMD=$JAVA_HOME/bin/java + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD=java + if ! command -v java >/dev/null 2>&1 + then + die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +fi + +# Increase the maximum file descriptors if we can. +if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then + case $MAX_FD in #( + max*) + # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + MAX_FD=$( ulimit -H -n ) || + warn "Could not query maximum file descriptor limit" + esac + case $MAX_FD in #( + '' | soft) :;; #( + *) + # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + ulimit -n "$MAX_FD" || + warn "Could not set maximum file descriptor limit to $MAX_FD" + esac +fi + +# Collect all arguments for the java command, stacking in reverse order: +# * args from the command line +# * the main class name +# * -classpath +# * -D...appname settings +# * --module-path (only if needed) +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. + +# For Cygwin or MSYS, switch paths to Windows format before running java +if "$cygwin" || "$msys" ; then + APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) + CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) + + JAVACMD=$( cygpath --unix "$JAVACMD" ) + + # Now convert the arguments - kludge to limit ourselves to /bin/sh + for arg do + if + case $arg in #( + -*) false ;; # don't mess with options #( + /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath + [ -e "$t" ] ;; #( + *) false ;; + esac + then + arg=$( cygpath --path --ignore --mixed "$arg" ) + fi + # Roll the args list around exactly as many times as the number of + # args, so each arg winds up back in the position where it started, but + # possibly modified. + # + # NB: a `for` loop captures its iteration list before it begins, so + # changing the positional parameters here affects neither the number of + # iterations, nor the values presented in `arg`. + shift # remove old arg + set -- "$@" "$arg" # push replacement arg + done +fi + + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Collect all arguments for the java command: +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, +# and any embedded shellness will be escaped. +# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be +# treated as '${Hostname}' itself on the command line. + +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -classpath "$CLASSPATH" \ + org.gradle.wrapper.GradleWrapperMain \ + "$@" + +# Stop when "xargs" is not available. +if ! command -v xargs >/dev/null 2>&1 +then + die "xargs is not available" +fi + +# Use "xargs" to parse quoted args. +# +# With -n1 it outputs one arg per line, with the quotes and backslashes removed. +# +# In Bash we could simply go: +# +# readarray ARGS < <( xargs -n1 <<<"$var" ) && +# set -- "${ARGS[@]}" "$@" +# +# but POSIX shell has neither arrays nor command substitution, so instead we +# post-process each arg (as a line of input to sed) to backslash-escape any +# character that might be a shell metacharacter, then use eval to reverse +# that process (while maintaining the separation between arguments), and wrap +# the whole thing up as a single "set" statement. +# +# This will of course break if any of these variables contains a newline or +# an unmatched quote. +# + +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | + tr '\n' ' ' + )" '"$@"' + +exec "$JAVACMD" "$@" diff --git a/gradlew.bat b/gradlew.bat new file mode 100644 index 0000000..9d21a21 --- /dev/null +++ b/gradlew.bat @@ -0,0 +1,94 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem +@rem SPDX-License-Identifier: Apache-2.0 +@rem + +@if "%DEBUG%"=="" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%"=="" set DIRNAME=. +@rem This is normally unused +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if %ERRORLEVEL% equ 0 goto execute + +echo. 1>&2 +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. 1>&2 +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 + +goto fail + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* + +:end +@rem End local scope for the variables with windows NT shell +if %ERRORLEVEL% equ 0 goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +set EXIT_CODE=%ERRORLEVEL% +if %EXIT_CODE% equ 0 set EXIT_CODE=1 +if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% +exit /b %EXIT_CODE% + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/profile.sh b/profile.sh new file mode 100644 index 0000000..1e618af --- /dev/null +++ b/profile.sh @@ -0,0 +1,98 @@ +#!/bin/bash + +# Profiling helper script for Imprint Java implementation +# +# Usage: +# ./profile.sh [test_method] [profiler] +# +# test_method: profileFieldAccess, profileSerialization, profileProjection, profileMemoryAllocation +# profiler: asyncprofiler, jfr, visualvm +# +# Examples: +# ./profile.sh profileFieldAccess asyncprofiler +# ./profile.sh profileSerialization jfr +# ./profile.sh profileMemoryAllocation + +set -e + +TEST_METHOD=${1:-profileFieldAccess} +PROFILER=${2:-asyncprofiler} + +echo "🔬 Starting profiling session for $TEST_METHOD using $PROFILER" + +# Enable the profiler test by removing @Disabled +sed -i 's/@Disabled.*/@Test/' src/test/java/com/imprint/benchmark/ProfilerTest.java + +case $PROFILER in + "asyncprofiler") + echo "📊 Using async-profiler (download from https://github.com/jvm-profiling-tools/async-profiler)" + echo " Will generate CPU profile in profiler-results/" + mkdir -p profiler-results + + # Run test in background and profile it + ./gradlew test --tests "*ProfilerTest.$TEST_METHOD" \ + -Dorg.gradle.jvmargs="-XX:+UnlockDiagnosticVMOptions -XX:+DebugNonSafepoints" & + + TEST_PID=$! + sleep 2 + + # Find the actual Java process (Gradle daemon) + JAVA_PID=$(pgrep -f "ProfilerTest.$TEST_METHOD" | head -1) + + if [ -n "$JAVA_PID" ]; then + echo " Profiling Java process $JAVA_PID" + if command -v async-profiler.jar >/dev/null 2>&1; then + java -jar async-profiler.jar -d 30 -f profiler-results/profile-$TEST_METHOD.html $JAVA_PID + else + echo " ⚠️ async-profiler.jar not found in PATH" + echo " 📥 Download from: https://github.com/jvm-profiling-tools/async-profiler/releases" + fi + fi + + wait $TEST_PID + ;; + + "jfr") + echo "📊 Using Java Flight Recorder" + mkdir -p profiler-results + + ./gradlew test --tests "*ProfilerTest.$TEST_METHOD" \ + -Dorg.gradle.jvmargs="-XX:+FlightRecorder -XX:StartFlightRecording=duration=60s,filename=profiler-results/profile-$TEST_METHOD.jfr,settings=profile" + + echo " 📂 JFR file saved to: profiler-results/profile-$TEST_METHOD.jfr" + echo " 🔍 Open with: jmc profiler-results/profile-$TEST_METHOD.jfr" + ;; + + "visualvm") + echo "📊 Using VisualVM" + echo " 1. Start VisualVM: jvisualvm" + echo " 2. Enable the ProfilerTest manually" + echo " 3. Run: ./gradlew test --tests '*ProfilerTest.$TEST_METHOD' --debug-jvm" + echo " 4. Attach VisualVM to the Gradle daemon process" + echo " 5. Start CPU/Memory profiling" + + read -p "Press Enter when VisualVM is ready..." + ./gradlew test --tests "*ProfilerTest.$TEST_METHOD" --debug-jvm + ;; + + *) + echo "❌ Unknown profiler: $PROFILER" + echo " Supported: asyncprofiler, jfr, visualvm" + exit 1 + ;; +esac + +# Restore @Disabled annotation +sed -i 's/@Test/@Disabled("Enable manually for profiling")/' src/test/java/com/imprint/benchmark/ProfilerTest.java + +echo "✅ Profiling complete!" +echo "" +echo "🔍 Key areas to examine:" +echo " • Object allocation hotspots (new, arrays, strings)" +echo " • ByteBuffer operations and slicing" +echo " • String UTF-8 encoding/decoding" +echo " • Binary search in directory lookup" +echo " • TypeHandler method dispatch" +echo " • VarInt encoding/decoding" +echo "" +echo "📊 Profile results in: profiler-results/" \ No newline at end of file diff --git a/settings.gradle b/settings.gradle new file mode 100644 index 0000000..0568c52 --- /dev/null +++ b/settings.gradle @@ -0,0 +1 @@ +rootProject.name = 'imprint-java' diff --git a/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java b/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java new file mode 100644 index 0000000..1293478 --- /dev/null +++ b/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java @@ -0,0 +1,344 @@ +package com.imprint.benchmark; + +import com.esotericsoftware.kryo.Kryo; +import com.esotericsoftware.kryo.io.Input; +import com.esotericsoftware.kryo.io.Output; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.imprint.core.ImprintRecord; +import com.imprint.core.ImprintWriter; +import com.imprint.core.SchemaId; +import com.imprint.types.MapKey; +import com.imprint.types.Value; +import org.openjdk.jmh.annotations.*; +import org.openjdk.jmh.infra.Blackhole; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.nio.ByteBuffer; +import java.util.*; +import java.util.concurrent.TimeUnit; + +/** + * Head-to-head benchmarks comparing Imprint against other serialization libraries. + * Tests the performance claims made in the documentation. + */ +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@State(Scope.Benchmark) +@Warmup(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS) +@Fork(1) +public class ComparisonBenchmark { + + // Test data + private TestRecord testData; + + // Serialized formats + private ByteBuffer imprintBytes; + private byte[] jacksonBytes; + private byte[] kryoBytes; + + // Library instances + private ObjectMapper jackson; + private Kryo kryo; + + @Setup + public void setup() throws Exception { + testData = createTestRecord(); + + // Initialize libraries + jackson = new ObjectMapper(); + kryo = new Kryo(); + kryo.register(TestRecord.class); + kryo.register(ArrayList.class); + kryo.register(HashMap.class); + + // Pre-serialize for deserialization benchmarks + imprintBytes = serializeWithImprint(testData); + jacksonBytes = serializeWithJackson(testData); + kryoBytes = serializeWithKryo(testData); + } + + // ===== SERIALIZATION BENCHMARKS ===== + + @Benchmark + public void serializeImprint(Blackhole bh) throws Exception { + ByteBuffer result = serializeWithImprint(testData); + bh.consume(result); + } + + @Benchmark + public void serializeJackson(Blackhole bh) throws Exception { + byte[] result = serializeWithJackson(testData); + bh.consume(result); + } + + @Benchmark + public void serializeKryo(Blackhole bh) throws Exception { + byte[] result = serializeWithKryo(testData); + bh.consume(result); + } + + // ===== DESERIALIZATION BENCHMARKS ===== + + @Benchmark + public void deserializeImprint(Blackhole bh) throws Exception { + ImprintRecord result = ImprintRecord.deserialize(imprintBytes.duplicate()); + bh.consume(result); + } + + @Benchmark + public void deserializeJackson(Blackhole bh) throws Exception { + TestRecord result = jackson.readValue(jacksonBytes, TestRecord.class); + bh.consume(result); + } + + @Benchmark + public void deserializeKryo(Blackhole bh) { + Input input = new Input(new ByteArrayInputStream(kryoBytes)); + TestRecord result = kryo.readObject(input, TestRecord.class); + input.close(); + bh.consume(result); + } + + // ===== FIELD ACCESS BENCHMARKS ===== + + @Benchmark + public void fieldAccessImprint(Blackhole bh) throws Exception { + ImprintRecord record = ImprintRecord.deserialize(imprintBytes.duplicate()); + + // Access multiple fields without full deserialization + var id = record.getValue(1); + var name = record.getValue(2); + var price = record.getValue(3); + var active = record.getValue(4); + var category = record.getValue(5); + + bh.consume(id); + bh.consume(name); + bh.consume(price); + bh.consume(active); + bh.consume(category); + } + + @Benchmark + public void fieldAccessJackson(Blackhole bh) throws Exception { + // Jackson requires full deserialization to access fields + TestRecord record = jackson.readValue(jacksonBytes, TestRecord.class); + + bh.consume(record.id); + bh.consume(record.name); + bh.consume(record.price); + bh.consume(record.active); + bh.consume(record.category); + } + + @Benchmark + public void fieldAccessKryo(Blackhole bh) { + // Kryo requires full deserialization to access fields + Input input = new Input(new ByteArrayInputStream(kryoBytes)); + TestRecord record = kryo.readObject(input, TestRecord.class); + input.close(); + + bh.consume(record.id); + bh.consume(record.name); + bh.consume(record.price); + bh.consume(record.active); + bh.consume(record.category); + } + + // ===== SIZE COMPARISON ===== + + @Benchmark + public void measureImprintSize(Blackhole bh) throws Exception { + ByteBuffer serialized = serializeWithImprint(testData); + bh.consume(serialized.remaining()); + } + + @Benchmark + public void measureJacksonSize(Blackhole bh) throws Exception { + byte[] serialized = serializeWithJackson(testData); + bh.consume(serialized.length); + } + + @Benchmark + public void measureKryoSize(Blackhole bh) throws Exception { + byte[] serialized = serializeWithKryo(testData); + bh.consume(serialized.length); + } + + // ===== MERGE SIMULATION BENCHMARKS ===== + + @Benchmark + public void mergeImprint(Blackhole bh) throws Exception { + // Simulate merge with Imprint (O(1) with proper API) + var record1 = serializeWithImprint(testData); + var record2 = serializeWithImprint(createTestRecord2()); + + // Current simulation - will be O(1) with actual merge API + var deserialized1 = ImprintRecord.deserialize(record1); + var deserialized2 = ImprintRecord.deserialize(record2); + var merged = simulateMerge(deserialized1, deserialized2); + + bh.consume(merged); + } + + @Benchmark + public void mergeJackson(Blackhole bh) throws Exception { + // Jackson merge requires full deserialization + merge + serialization + var record1 = jackson.readValue(jacksonBytes, TestRecord.class); + var record2 = jackson.readValue(serializeWithJackson(createTestRecord2()), TestRecord.class); + + var merged = mergeTestRecords(record1, record2); + byte[] result = jackson.writeValueAsBytes(merged); + + bh.consume(result); + } + + @Benchmark + public void mergeKryo(Blackhole bh) throws Exception { + // Kryo merge requires full deserialization + merge + serialization + Input input1 = new Input(new ByteArrayInputStream(kryoBytes)); + var record1 = kryo.readObject(input1, TestRecord.class); + input1.close(); + + Input input2 = new Input(new ByteArrayInputStream(serializeWithKryo(createTestRecord2()))); + var record2 = kryo.readObject(input2, TestRecord.class); + input2.close(); + + var merged = mergeTestRecords(record1, record2); + byte[] result = serializeWithKryo(merged); + + bh.consume(result); + } + + // ===== HELPER METHODS ===== + + private ByteBuffer serializeWithImprint(TestRecord data) throws Exception { + var writer = new ImprintWriter(new SchemaId(1, 0x12345678)); + + writer.addField(1, Value.fromInt32(data.id)); + writer.addField(2, Value.fromString(data.name)); + writer.addField(3, Value.fromFloat64(data.price)); + writer.addField(4, Value.fromBoolean(data.active)); + writer.addField(5, Value.fromString(data.category)); + + // Convert tags list + var tagValues = new ArrayList(); + for (String tag : data.tags) { + tagValues.add(Value.fromString(tag)); + } + writer.addField(6, Value.fromArray(tagValues)); + + // Convert metadata map + var metadataMap = new HashMap(); + for (var entry : data.metadata.entrySet()) { + metadataMap.put(MapKey.fromString(entry.getKey()), Value.fromString(entry.getValue())); + } + writer.addField(7, Value.fromMap(metadataMap)); + + return writer.build().serializeToBuffer(); + } + + private byte[] serializeWithJackson(TestRecord data) throws Exception { + return jackson.writeValueAsBytes(data); + } + + private byte[] serializeWithKryo(TestRecord data) { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + Output output = new Output(baos); + kryo.writeObject(output, data); + output.close(); + return baos.toByteArray(); + } + + private ImprintRecord simulateMerge(ImprintRecord first, ImprintRecord second) throws Exception { + var writer = new ImprintWriter(first.getHeader().getSchemaId()); + var usedFieldIds = new HashSet(); + + // Copy fields from first record (takes precedence) + copyFieldsToWriter(first, writer, usedFieldIds); + copyFieldsToWriter(second, writer, usedFieldIds); + + return writer.build(); + } + + private void copyFieldsToWriter(ImprintRecord record, ImprintWriter writer, Set usedFieldIds) throws Exception { + for (var entry : record.getDirectory()) { + int fieldId = entry.getId(); + if (!usedFieldIds.contains(fieldId)) { + var value = record.getValue(fieldId); + if (value.isPresent()) { + writer.addField(fieldId, value.get()); + usedFieldIds.add(fieldId); + } + } + } + } + + private TestRecord mergeTestRecords(TestRecord first, TestRecord second) { + // Simple merge logic - first record takes precedence + var merged = new TestRecord(); + merged.id = first.id; + merged.name = first.name != null ? first.name : second.name; + merged.price = first.price != 0.0 ? first.price : second.price; + merged.active = first.active; + merged.category = first.category != null ? first.category : second.category; + + merged.tags = new ArrayList<>(first.tags); + merged.tags.addAll(second.tags); + + merged.metadata = new HashMap<>(first.metadata); + merged.metadata.putAll(second.metadata); + + return merged; + } + + private TestRecord createTestRecord() { + var record = new TestRecord(); + record.id = 12345; + record.name = "Test Product"; + record.price = 99.99; + record.active = true; + record.category = "Electronics"; + + record.tags = Arrays.asList("popular", "trending", "bestseller"); + + record.metadata = new HashMap<>(); + record.metadata.put("manufacturer", "TechCorp"); + record.metadata.put("model", "TC-2024"); + record.metadata.put("warranty", "2 years"); + + return record; + } + + private TestRecord createTestRecord2() { + var record = new TestRecord(); + record.id = 67890; + record.name = "Test Product 2"; + record.price = 149.99; + record.active = false; + record.category = "Software"; + + record.tags = Arrays.asList("new", "premium"); + + record.metadata = new HashMap<>(); + record.metadata.put("vendor", "SoftCorp"); + record.metadata.put("version", "2.1"); + + return record; + } + + // Test data class for other serialization libraries + public static class TestRecord { + public int id; + public String name; + public double price; + public boolean active; + public String category; + public List tags = new ArrayList<>(); + public Map metadata = new HashMap<>(); + + public TestRecord() {} // Required for deserialization + } +} \ No newline at end of file diff --git a/src/jmh/java/com/imprint/benchmark/FieldAccessBenchmark.java b/src/jmh/java/com/imprint/benchmark/FieldAccessBenchmark.java new file mode 100644 index 0000000..f3abb7e --- /dev/null +++ b/src/jmh/java/com/imprint/benchmark/FieldAccessBenchmark.java @@ -0,0 +1,276 @@ +package com.imprint.benchmark; + +import com.imprint.core.ImprintRecord; +import com.imprint.core.ImprintWriter; +import com.imprint.core.SchemaId; +import com.imprint.types.MapKey; +import com.imprint.types.Value; +import org.openjdk.jmh.annotations.*; +import org.openjdk.jmh.infra.Blackhole; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.concurrent.TimeUnit; + +/** + * Benchmarks for ImprintRecord field access and projection operations. + * Tests the zero-copy field access performance claims. + */ +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@State(Scope.Benchmark) +@Warmup(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS) +@Fork(1) +public class FieldAccessBenchmark { + + private ImprintRecord sparseRecord; + private ImprintRecord denseRecord; + private ImprintRecord largeRecord; + + // Field IDs for testing different access patterns + private int[] firstFields; + private int[] middleFields; + private int[] lastFields; + private int[] randomFields; + private int[] allFields; + + @Setup + public void setup() throws Exception { + sparseRecord = createSparseRecord(); // Few fields, large field IDs + denseRecord = createDenseRecord(); // Many sequential fields + largeRecord = createLargeRecord(); // Large record with complex data + + // Setup field access patterns + firstFields = new int[]{1, 2, 3, 4, 5}; + middleFields = new int[]{45, 46, 47, 48, 49}; + lastFields = new int[]{95, 96, 97, 98, 99}; + randomFields = new int[]{7, 23, 41, 67, 89}; + allFields = new int[100]; + for (int i = 0; i < 100; i++) { + allFields[i] = i + 1; + } + } + + // ===== SINGLE FIELD ACCESS BENCHMARKS ===== + + @Benchmark + public void accessFirstField(Blackhole bh) throws Exception { + var value = denseRecord.getValue(1); + bh.consume(value); + } + + @Benchmark + public void accessMiddleField(Blackhole bh) throws Exception { + var value = denseRecord.getValue(50); + bh.consume(value); + } + + @Benchmark + public void accessLastField(Blackhole bh) throws Exception { + var value = denseRecord.getValue(100); + bh.consume(value); + } + + @Benchmark + public void accessNonExistentField(Blackhole bh) throws Exception { + var value = denseRecord.getValue(999); + bh.consume(value); + } + + // ===== MULTIPLE FIELD ACCESS PATTERNS ===== + + @Benchmark + public void accessFirstFields(Blackhole bh) throws Exception { + for (int fieldId : firstFields) { + var value = denseRecord.getValue(fieldId); + bh.consume(value); + } + } + + @Benchmark + public void accessMiddleFields(Blackhole bh) throws Exception { + for (int fieldId : middleFields) { + var value = denseRecord.getValue(fieldId); + bh.consume(value); + } + } + + @Benchmark + public void accessLastFields(Blackhole bh) throws Exception { + for (int fieldId : lastFields) { + var value = denseRecord.getValue(fieldId); + bh.consume(value); + } + } + + @Benchmark + public void accessRandomFields(Blackhole bh) throws Exception { + for (int fieldId : randomFields) { + var value = denseRecord.getValue(fieldId); + bh.consume(value); + } + } + + // ===== FIELD PROJECTION BENCHMARKS ===== + + @Benchmark + public void projectSmallSubset(Blackhole bh) throws Exception { + // Project 5 fields from a 100-field record + var projection = simulateProject(denseRecord, firstFields); + bh.consume(projection); + } + + @Benchmark + public void projectMediumSubset(Blackhole bh) throws Exception { + // Project 25 fields from a 100-field record + int[] fields = Arrays.copyOf(allFields, 25); + var projection = simulateProject(denseRecord, fields); + bh.consume(projection); + } + + @Benchmark + public void projectLargeSubset(Blackhole bh) throws Exception { + // Project 75 fields from a 100-field record + int[] fields = Arrays.copyOf(allFields, 75); + var projection = simulateProject(denseRecord, fields); + bh.consume(projection); + } + + @Benchmark + public void projectAllFields(Blackhole bh) throws Exception { + // Project all fields (should be nearly equivalent to full record) + var projection = simulateProject(denseRecord, allFields); + bh.consume(projection); + } + + // ===== RAW BYTES ACCESS BENCHMARKS ===== + + @Benchmark + public void getRawBytesFirstField(Blackhole bh) { + var rawBytes = denseRecord.getRawBytes(1); + bh.consume(rawBytes); + } + + @Benchmark + public void getRawBytesMiddleField(Blackhole bh) { + var rawBytes = denseRecord.getRawBytes(50); + bh.consume(rawBytes); + } + + @Benchmark + public void getRawBytesLastField(Blackhole bh) { + var rawBytes = denseRecord.getRawBytes(100); + bh.consume(rawBytes); + } + + // ===== SPARSE VS DENSE ACCESS PATTERNS ===== + + @Benchmark + public void accessSparseRecord(Blackhole bh) throws Exception { + // Access fields in sparse record (large field IDs, few fields) + var value1 = sparseRecord.getValue(1000); + var value2 = sparseRecord.getValue(5000); + var value3 = sparseRecord.getValue(10000); + bh.consume(value1); + bh.consume(value2); + bh.consume(value3); + } + + @Benchmark + public void accessDenseRecord(Blackhole bh) throws Exception { + // Access fields in dense record (sequential field IDs) + var value1 = denseRecord.getValue(1); + var value2 = denseRecord.getValue(2); + var value3 = denseRecord.getValue(3); + bh.consume(value1); + bh.consume(value2); + bh.consume(value3); + } + + // ===== HELPER METHODS ===== + + /** + * Simulates field projection by creating a new record with only specified fields. + * This should be replaced with actual project API when available. + */ + private ImprintRecord simulateProject(ImprintRecord source, int[] fieldIds) throws Exception { + var writer = new ImprintWriter(source.getHeader().getSchemaId()); + + for (int fieldId : fieldIds) { + var value = source.getValue(fieldId); + value.ifPresent(value1 -> writer.addField(fieldId, value1)); + } + + return writer.build(); + } + + private ImprintRecord createSparseRecord() throws Exception { + var writer = new ImprintWriter(new SchemaId(1, 0x12345678)); + + // Sparse record with large field IDs and few fields + writer.addField(1000, Value.fromString("sparse_field_1")); + writer.addField(5000, Value.fromInt32(42)); + writer.addField(10000, Value.fromFloat64(3.14159)); + writer.addField(15000, Value.fromBoolean(true)); + writer.addField(20000, Value.fromString("sparse_field_5")); + + return writer.build(); + } + + private ImprintRecord createDenseRecord() throws Exception { + var writer = new ImprintWriter(new SchemaId(2, 0x87654321)); + + // Dense record with 100 sequential fields + for (int i = 1; i <= 100; i++) { + switch (i % 5) { + case 0: + writer.addField(i, Value.fromString("string_field_" + i)); + break; + case 1: + writer.addField(i, Value.fromInt32(i * 10)); + break; + case 2: + writer.addField(i, Value.fromFloat64(i * 1.5)); + break; + case 3: + writer.addField(i, Value.fromBoolean(i % 2 == 0)); + break; + case 4: + writer.addField(i, Value.fromInt64(i * 1000L)); + break; + } + } + + return writer.build(); + } + + private ImprintRecord createLargeRecord() throws Exception { + var writer = new ImprintWriter(new SchemaId(3, 0x11223344)); + + // Large record with complex data types + writer.addField(1, Value.fromString("LargeRecord")); + + // Large array field + var largeArray = new ArrayList(); + for (int i = 0; i < 1000; i++) { + largeArray.add(Value.fromString("array_item_" + i)); + } + writer.addField(2, Value.fromArray(largeArray)); + + // Large map field + var largeMap = new HashMap(); + for (int i = 0; i < 100; i++) { + largeMap.put(MapKey.fromString("key_" + i), Value.fromString("map_value_" + i)); + } + writer.addField(3, Value.fromMap(largeMap)); + + // Many regular fields + for (int i = 4; i <= 50; i++) { + writer.addField(i, Value.fromString("large_record_field_" + i + "_with_substantial_content")); + } + + return writer.build(); + } +} \ No newline at end of file diff --git a/src/jmh/java/com/imprint/benchmark/MergeBenchmark.java b/src/jmh/java/com/imprint/benchmark/MergeBenchmark.java new file mode 100644 index 0000000..5c52908 --- /dev/null +++ b/src/jmh/java/com/imprint/benchmark/MergeBenchmark.java @@ -0,0 +1,163 @@ +package com.imprint.benchmark; + +import com.imprint.core.ImprintRecord; +import com.imprint.core.ImprintWriter; +import com.imprint.core.SchemaId; +import com.imprint.types.Value; +import org.openjdk.jmh.annotations.*; +import org.openjdk.jmh.infra.Blackhole; + +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +/** + * Benchmarks for ImprintRecord merge operations. + * NOTE: These benchmarks simulate merge operations until the actual merge API is implemented. + */ +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@State(Scope.Benchmark) +@Warmup(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS) +@Fork(1) +public class MergeBenchmark { + + private ImprintRecord productRecord; + private ImprintRecord orderRecord; + private ImprintRecord customerRecord; + + @Setup + public void setup() throws Exception { + productRecord = createProductRecord(); + orderRecord = createOrderRecord(); + customerRecord = createCustomerRecord(); + } + + // ===== SIMULATED MERGE BENCHMARKS ===== + // These will be replaced with actual merge API when implemented + + @Benchmark + public void mergeProductAndOrder(Blackhole bh) throws Exception { + // Simulate merge by creating a new record with fields from both + ImprintRecord result = simulateMerge(productRecord, orderRecord); + bh.consume(result); + } + + @Benchmark + public void mergeProductAndCustomer(Blackhole bh) throws Exception { + ImprintRecord result = simulateMerge(productRecord, customerRecord); + bh.consume(result); + } + + @Benchmark + public void mergeOrderAndCustomer(Blackhole bh) throws Exception { + ImprintRecord result = simulateMerge(orderRecord, customerRecord); + bh.consume(result); + } + + @Benchmark + public void mergeThreeRecords(Blackhole bh) throws Exception { + // Test merging multiple records + var temp = simulateMerge(productRecord, orderRecord); + ImprintRecord result = simulateMerge(temp, customerRecord); + bh.consume(result); + } + + // ===== MERGE CONFLICT HANDLING ===== + + @Benchmark + public void mergeWithConflicts(Blackhole bh) throws Exception { + // Create records with overlapping field IDs to test conflict resolution + var record1 = createRecordWithFields(1, 50, "record1_"); + var record2 = createRecordWithFields(25, 75, "record2_"); + + ImprintRecord result = simulateMerge(record1, record2); + bh.consume(result); + } + + // ===== HELPER METHODS ===== + + /** + * Simulates merge operation by manually copying fields. + * This should be replaced with actual merge API when available. + */ + private ImprintRecord simulateMerge(ImprintRecord first, ImprintRecord second) throws Exception { + var writer = new ImprintWriter(first.getHeader().getSchemaId()); + var usedFieldIds = new HashSet(); + + // Copy fields from first record (takes precedence) + copyFieldsToWriter(first, writer, usedFieldIds); + + // Copy non-conflicting fields from second record + copyFieldsToWriter(second, writer, usedFieldIds); + + return writer.build(); + } + + private void copyFieldsToWriter(ImprintRecord record, ImprintWriter writer, Set usedFieldIds) throws Exception { + for (var entry : record.getDirectory()) { + int fieldId = entry.getId(); + if (!usedFieldIds.contains(fieldId)) { + var value = record.getValue(fieldId); + if (value.isPresent()) { + writer.addField(fieldId, value.get()); + usedFieldIds.add(fieldId); + } + } + } + } + + private ImprintRecord createProductRecord() throws Exception { + var writer = new ImprintWriter(new SchemaId(1, 0x12345678)); + + writer.addField(1, Value.fromString("Product")); + writer.addField(2, Value.fromInt32(12345)); + writer.addField(3, Value.fromString("Laptop")); + writer.addField(4, Value.fromFloat64(999.99)); + writer.addField(5, Value.fromString("Electronics")); + writer.addField(6, Value.fromInt32(50)); // stock + writer.addField(7, Value.fromString("TechCorp")); + writer.addField(8, Value.fromBoolean(true)); // available + + return writer.build(); + } + + private ImprintRecord createOrderRecord() throws Exception { + var writer = new ImprintWriter(new SchemaId(2, 0x87654321)); + + writer.addField(10, Value.fromString("Order")); + writer.addField(11, Value.fromInt32(67890)); + writer.addField(12, Value.fromInt32(12345)); // product_id (overlaps with product) + writer.addField(13, Value.fromInt32(2)); // quantity + writer.addField(14, Value.fromFloat64(1999.98)); // total + writer.addField(15, Value.fromString("2024-01-15")); // order_date + writer.addField(16, Value.fromString("shipped")); // status + + return writer.build(); + } + + private ImprintRecord createCustomerRecord() throws Exception { + var writer = new ImprintWriter(new SchemaId(3, 0x11223344)); + + writer.addField(20, Value.fromString("Customer")); + writer.addField(21, Value.fromInt32(555)); + writer.addField(22, Value.fromString("John Doe")); + writer.addField(23, Value.fromString("john.doe@email.com")); + writer.addField(24, Value.fromString("123 Main St")); + writer.addField(25, Value.fromString("premium")); // tier + writer.addField(26, Value.fromBoolean(true)); // active + + return writer.build(); + } + + private ImprintRecord createRecordWithFields(int startId, int endId, String prefix) throws Exception { + var writer = new ImprintWriter(new SchemaId(1, 0x12345678)); + + for (int i = startId; i <= endId; i++) { + writer.addField(i, Value.fromString(prefix + "field_" + i)); + } + + return writer.build(); + } +} \ No newline at end of file diff --git a/src/jmh/java/com/imprint/benchmark/SerializationBenchmark.java b/src/jmh/java/com/imprint/benchmark/SerializationBenchmark.java new file mode 100644 index 0000000..2544b88 --- /dev/null +++ b/src/jmh/java/com/imprint/benchmark/SerializationBenchmark.java @@ -0,0 +1,166 @@ +package com.imprint.benchmark; + +import com.imprint.core.ImprintRecord; +import com.imprint.core.ImprintWriter; +import com.imprint.core.SchemaId; +import com.imprint.types.MapKey; +import com.imprint.types.Value; +import org.openjdk.jmh.annotations.*; +import org.openjdk.jmh.infra.Blackhole; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.concurrent.TimeUnit; + +/** + * Benchmarks for ImprintRecord serialization and deserialization operations. + */ +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@State(Scope.Benchmark) +@Warmup(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS) +@Fork(1) +public class SerializationBenchmark { + + private ImprintRecord smallRecord; + private ImprintRecord mediumRecord; + private ImprintRecord largeRecord; + + private ByteBuffer smallRecordBytes; + private ByteBuffer mediumRecordBytes; + private ByteBuffer largeRecordBytes; + + @Setup + public void setup() throws Exception { + // Create test records of varying sizes + smallRecord = createSmallRecord(); + mediumRecord = createMediumRecord(); + largeRecord = createLargeRecord(); + + // Pre-serialize for deserialization benchmarks + smallRecordBytes = smallRecord.serializeToBuffer(); + mediumRecordBytes = mediumRecord.serializeToBuffer(); + largeRecordBytes = largeRecord.serializeToBuffer(); + } + + // ===== SERIALIZATION BENCHMARKS ===== + + @Benchmark + public void serializeSmallRecord(Blackhole bh) { + ByteBuffer result = smallRecord.serializeToBuffer(); + bh.consume(result); + } + + @Benchmark + public void serializeMediumRecord(Blackhole bh) { + ByteBuffer result = mediumRecord.serializeToBuffer(); + bh.consume(result); + } + + @Benchmark + public void serializeLargeRecord(Blackhole bh) { + ByteBuffer result = largeRecord.serializeToBuffer(); + bh.consume(result); + } + + // ===== DESERIALIZATION BENCHMARKS ===== + + @Benchmark + public void deserializeSmallRecord(Blackhole bh) throws Exception { + ImprintRecord result = ImprintRecord.deserialize(smallRecordBytes.duplicate()); + bh.consume(result); + } + + @Benchmark + public void deserializeMediumRecord(Blackhole bh) throws Exception { + ImprintRecord result = ImprintRecord.deserialize(mediumRecordBytes.duplicate()); + bh.consume(result); + } + + @Benchmark + public void deserializeLargeRecord(Blackhole bh) throws Exception { + ImprintRecord result = ImprintRecord.deserialize(largeRecordBytes.duplicate()); + bh.consume(result); + } + + // ===== HELPER METHODS ===== + + private ImprintRecord createSmallRecord() throws Exception { + var writer = new ImprintWriter(new SchemaId(1, 0x12345678)); + + // Small record: ~10 fields, simple types + writer.addField(1, Value.fromString("Product")); + writer.addField(2, Value.fromInt32(12345)); + writer.addField(3, Value.fromFloat64(99.99)); + writer.addField(4, Value.fromBoolean(true)); + writer.addField(5, Value.fromString("Electronics")); + + return writer.build(); + } + + private ImprintRecord createMediumRecord() throws Exception { + var writer = new ImprintWriter(new SchemaId(1, 0x12345678)); + + // Medium record: ~50 fields, mixed types including arrays + writer.addField(1, Value.fromString("Product")); + writer.addField(2, Value.fromInt32(12345)); + writer.addField(3, Value.fromFloat64(99.99)); + writer.addField(4, Value.fromBoolean(true)); + writer.addField(5, Value.fromString("Electronics")); + + // Add array field + var tags = Arrays.asList( + Value.fromString("popular"), + Value.fromString("trending"), + Value.fromString("bestseller") + ); + writer.addField(6, Value.fromArray(tags)); + + // Add map field + var metadata = new HashMap(); + metadata.put(MapKey.fromString("manufacturer"), Value.fromString("TechCorp")); + metadata.put(MapKey.fromString("model"), Value.fromString("TC-2024")); + metadata.put(MapKey.fromString("year"), Value.fromInt32(2024)); + writer.addField(7, Value.fromMap(metadata)); + + // Add more fields for medium size + for (int i = 8; i <= 50; i++) { + writer.addField(i, Value.fromString("field_" + i + "_value")); + } + + return writer.build(); + } + + private ImprintRecord createLargeRecord() throws Exception { + var writer = new ImprintWriter(new SchemaId(1, 0x12345678)); + + // Large record: ~200 fields, complex nested structures + writer.addField(1, Value.fromString("LargeProduct")); + writer.addField(2, Value.fromInt32(12345)); + writer.addField(3, Value.fromFloat64(99.99)); + + // Large array + var largeArray = new ArrayList(); + for (int i = 0; i < 100; i++) { + largeArray.add(Value.fromString("item_" + i)); + } + writer.addField(4, Value.fromArray(largeArray)); + + // Large map + var largeMap = new HashMap(); + for (int i = 0; i < 50; i++) { + largeMap.put(MapKey.fromString("key_" + i), Value.fromString("value_" + i)); + } + writer.addField(5, Value.fromMap(largeMap)); + + // Many string fields + for (int i = 6; i <= 200; i++) { + writer.addField(i, Value.fromString("this_is_a_longer_field_value_for_field_" + i + "_to_increase_record_size")); + } + + return writer.build(); + } +} \ No newline at end of file diff --git a/src/main/java/com/imprint/Constants.java b/src/main/java/com/imprint/Constants.java new file mode 100644 index 0000000..78b91a0 --- /dev/null +++ b/src/main/java/com/imprint/Constants.java @@ -0,0 +1,10 @@ +package com.imprint; + +public final class Constants { + public static final byte MAGIC = 0x49; + public static final byte VERSION = 0x02; + public static final int HEADER_BYTES = 15; + public static final int DIR_ENTRY_BYTES = 7; + + private Constants() {} +} diff --git a/src/main/java/com/imprint/core/DirectoryEntry.java b/src/main/java/com/imprint/core/DirectoryEntry.java new file mode 100644 index 0000000..9556256 --- /dev/null +++ b/src/main/java/com/imprint/core/DirectoryEntry.java @@ -0,0 +1,23 @@ +package com.imprint.core; + +import com.imprint.types.TypeCode; +import lombok.Value; + +import java.util.Objects; + +/** + * A directory entry describing a single field in an Imprint record. + * Each entry has a fixed size of 7 bytes. + */ +@Value +public class DirectoryEntry { + short id; + TypeCode typeCode; + int offset; + + public DirectoryEntry(int id, TypeCode typeCode, int offset) { + this.id = (short) id; + this.typeCode = Objects.requireNonNull(typeCode, "TypeCode cannot be null"); + this.offset = offset; + } +} \ No newline at end of file diff --git a/src/main/java/com/imprint/core/Flags.java b/src/main/java/com/imprint/core/Flags.java new file mode 100644 index 0000000..ee6ffa7 --- /dev/null +++ b/src/main/java/com/imprint/core/Flags.java @@ -0,0 +1,12 @@ +package com.imprint.core; + +import lombok.Value; + +/** + * Bit flags for Imprint record header. + * Currently reserved for future use - field directory is always present. + */ +@Value +public class Flags { + byte value; +} \ No newline at end of file diff --git a/src/main/java/com/imprint/core/Header.java b/src/main/java/com/imprint/core/Header.java new file mode 100644 index 0000000..388d491 --- /dev/null +++ b/src/main/java/com/imprint/core/Header.java @@ -0,0 +1,13 @@ +package com.imprint.core; + +import lombok.Value; + +/** + * The header of an Imprint record. + */ +@Value +public class Header { + Flags flags; + SchemaId schemaId; + int payloadSize; +} \ No newline at end of file diff --git a/src/main/java/com/imprint/core/ImprintRecord.java b/src/main/java/com/imprint/core/ImprintRecord.java new file mode 100644 index 0000000..b7ed224 --- /dev/null +++ b/src/main/java/com/imprint/core/ImprintRecord.java @@ -0,0 +1,365 @@ + +package com.imprint.core; + +import com.imprint.Constants; +import com.imprint.error.ErrorType; +import com.imprint.error.ImprintException; +import com.imprint.types.MapKey; +import com.imprint.types.TypeCode; +import com.imprint.types.Value; +import com.imprint.util.VarInt; +import lombok.Getter; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.*; + +/** + * An Imprint record containing a header, optional field directory, and payload. + * Uses ByteBuffer for zero-copy operations to achieve low latency. + */ +@Getter +public final class ImprintRecord { + private final Header header; + private final List directory; + private final ByteBuffer payload; // Read-only view for zero-copy + + public ImprintRecord(Header header, List directory, ByteBuffer payload) { + this.header = Objects.requireNonNull(header, "Header cannot be null"); + this.directory = List.copyOf(Objects.requireNonNull(directory, "Directory cannot be null")); + this.payload = payload.asReadOnlyBuffer(); // Zero-copy read-only view + } + + /** + * Get a value by field ID, deserializing it on demand. + */ + public Optional getValue(int fieldId) throws ImprintException { + // Binary search for the field ID without allocation + int index = findDirectoryIndex(fieldId); + if (index < 0) return Optional.empty(); + + var entry = directory.get(index); + int startOffset = entry.getOffset(); + int endOffset = (index + 1 < directory.size()) ? + directory.get(index + 1).getOffset() : payload.remaining(); + + var valueBytes = payload.duplicate(); + valueBytes.position(startOffset).limit(endOffset); + var value = deserializeValue(entry.getTypeCode(), valueBytes.slice()); + return Optional.of(value); + } + + /** + * Get the raw bytes for a field without deserializing. + * Returns a zero-copy ByteBuffer view. + */ + public Optional getRawBytes(int fieldId) { + int index = findDirectoryIndex(fieldId); + if (index < 0) return Optional.empty(); + + var entry = directory.get(index); + int startOffset = entry.getOffset(); + int endOffset = (index + 1 < directory.size()) ? + directory.get(index + 1).getOffset() : payload.remaining(); + + var fieldBuffer = payload.duplicate(); + fieldBuffer.position(startOffset).limit(endOffset); + return Optional.of(fieldBuffer.slice().asReadOnlyBuffer()); + } + + /** + * Serialize this record to a ByteBuffer (zero-copy when possible). + */ + public ByteBuffer serializeToBuffer() { + var buffer = ByteBuffer.allocate(estimateSerializedSize()); + buffer.order(ByteOrder.LITTLE_ENDIAN); + + // Write header + serializeHeader(buffer); + + // Write directory (always present) + VarInt.encode(directory.size(), buffer); + for (var entry : directory) { + serializeDirectoryEntry(entry, buffer); + } + + // Write payload (shallow copy only) + var payloadCopy = payload.duplicate(); + buffer.put(payloadCopy); + + // Return read-only view of used portion + buffer.flip(); + return buffer.asReadOnlyBuffer(); + } + + /** + * Create a fluent builder for constructing ImprintRecord instances. + * + * @param schemaId the schema identifier for this record + * @return a new builder instance + */ + public static ImprintRecordBuilder builder(SchemaId schemaId) { + return new ImprintRecordBuilder(schemaId); + } + + /** + * Create a fluent builder for constructing ImprintRecord instances. + * + * @param fieldspaceId the fieldspace identifier + * @param schemaHash the schema hash + * @return a new builder instance + */ + public static ImprintRecordBuilder builder(int fieldspaceId, int schemaHash) { + return new ImprintRecordBuilder(new SchemaId(fieldspaceId, schemaHash)); + } + + /** + * Deserialize a record from bytes. + */ + public static ImprintRecord deserialize(byte[] bytes) throws ImprintException { + return deserialize(ByteBuffer.wrap(bytes)); + } + + /** + * Deserialize a record from a ByteBuffer (zero-copy when possible). + */ + public static ImprintRecord deserialize(ByteBuffer buffer) throws ImprintException { + buffer = buffer.duplicate().order(ByteOrder.LITTLE_ENDIAN); + + // Read header + var header = deserializeHeader(buffer); + + // Read directory (always present) + var directory = new ArrayList(); + VarInt.DecodeResult countResult = VarInt.decode(buffer); + int directoryCount = countResult.getValue(); + + for (int i = 0; i < directoryCount; i++) { + directory.add(deserializeDirectoryEntry(buffer)); + } + + // Read payload as ByteBuffer slice for zero-copy + var payload = buffer.slice(); + payload.limit(header.getPayloadSize()); + buffer.position(buffer.position() + header.getPayloadSize()); + + return new ImprintRecord(header, directory, payload); + } + + /** + * Binary search for field ID in directory without object allocation. + * Returns the index of the field if found, or a negative value if not found. + * + * @param fieldId the field ID to search for + * @return index if found, or negative insertion point - 1 if not found + */ + private int findDirectoryIndex(int fieldId) { + int low = 0; + int high = directory.size() - 1; + + while (low <= high) { + int mid = (low + high) >>> 1; // unsigned right shift to avoid overflow + int midFieldId = directory.get(mid).getId(); + + if (midFieldId < fieldId) { + low = mid + 1; + } else if (midFieldId > fieldId) { + high = mid - 1; + } else { + return mid; // field found + } + } + return -(low + 1); // field not found, return insertion point + } + + private int estimateSerializedSize() { + int size = Constants.HEADER_BYTES; // header + size += VarInt.encodedLength(directory.size()); // directory count + size += directory.size() * Constants.DIR_ENTRY_BYTES; // directory entries + size += payload.remaining(); // payload + return size; + } + + private void serializeHeader(ByteBuffer buffer) { + buffer.put(Constants.MAGIC); + buffer.put(Constants.VERSION); + buffer.put(header.getFlags().getValue()); + buffer.putInt(header.getSchemaId().getFieldspaceId()); + buffer.putInt(header.getSchemaId().getSchemaHash()); + buffer.putInt(header.getPayloadSize()); + } + + private static Header deserializeHeader(ByteBuffer buffer) throws ImprintException { + if (buffer.remaining() < Constants.HEADER_BYTES) { + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, + "Not enough bytes for header"); + } + + byte magic = buffer.get(); + if (magic != Constants.MAGIC) { + throw new ImprintException(ErrorType.INVALID_MAGIC, + "Invalid magic byte: expected 0x" + Integer.toHexString(Constants.MAGIC) + + ", got 0x" + Integer.toHexString(magic & 0xFF)); + } + + byte version = buffer.get(); + if (version != Constants.VERSION) { + throw new ImprintException(ErrorType.UNSUPPORTED_VERSION, + "Unsupported version: " + version); + } + + var flags = new Flags(buffer.get()); + int fieldspaceId = buffer.getInt(); + int schemaHash = buffer.getInt(); + int payloadSize = buffer.getInt(); + + return new Header(flags, new SchemaId(fieldspaceId, schemaHash), payloadSize); + } + + private void serializeDirectoryEntry(DirectoryEntry entry, ByteBuffer buffer) { + buffer.putShort(entry.getId()); + buffer.put(entry.getTypeCode().getCode()); + buffer.putInt(entry.getOffset()); + } + + private static DirectoryEntry deserializeDirectoryEntry(ByteBuffer buffer) throws ImprintException { + if (buffer.remaining() < Constants.DIR_ENTRY_BYTES) { + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, + "Not enough bytes for directory entry"); + } + + short id = buffer.getShort(); + var typeCode = TypeCode.fromByte(buffer.get()); + int offset = buffer.getInt(); + + return new DirectoryEntry(id, typeCode, offset); + } + + private Value deserializeValue(TypeCode typeCode, ByteBuffer buffer) throws ImprintException { + // Buffer is already positioned and limited correctly + buffer = buffer.duplicate().order(ByteOrder.LITTLE_ENDIAN); + + // Use TypeHandler for simple types + switch (typeCode) { + case NULL: + case BOOL: + case INT32: + case INT64: + case FLOAT32: + case FLOAT64: + case BYTES: + case STRING: + return typeCode.getHandler().deserialize(buffer); + + case ARRAY: + return deserializeArray(buffer); + + case MAP: + return deserializeMap(buffer); + + case ROW: + var remainingBuffer = buffer.slice(); + var nestedRecord = deserialize(remainingBuffer); + return Value.fromRow(nestedRecord); + + default: + throw new ImprintException(ErrorType.INVALID_TYPE_CODE, "Unknown type code: " + typeCode); + } + } + + private Value deserializeArray(ByteBuffer buffer) throws ImprintException { + VarInt.DecodeResult lengthResult = VarInt.decode(buffer); + int length = lengthResult.getValue(); + + if (length == 0) { + return Value.fromArray(Collections.emptyList()); + } + + var elementType = TypeCode.fromByte(buffer.get()); + var elements = new ArrayList(length); + + for (int i = 0; i < length; i++) { + var elementBytes = readValueBytes(elementType, buffer); + var element = deserializeValue(elementType, elementBytes); + elements.add(element); + } + + return Value.fromArray(elements); + } + + private Value deserializeMap(ByteBuffer buffer) throws ImprintException { + VarInt.DecodeResult lengthResult = VarInt.decode(buffer); + int length = lengthResult.getValue(); + + if (length == 0) { + return Value.fromMap(Collections.emptyMap()); + } + + var keyType = TypeCode.fromByte(buffer.get()); + var valueType = TypeCode.fromByte(buffer.get()); + var map = new HashMap(length); + + for (int i = 0; i < length; i++) { + // Read key + var keyBytes = readValueBytes(keyType, buffer); + var keyValue = deserializeValue(keyType, keyBytes); + var key = MapKey.fromValue(keyValue); + + // Read value + var valueBytes = readValueBytes(valueType, buffer); + var value = deserializeValue(valueType, valueBytes); + + map.put(key, value); + } + + return Value.fromMap(map); + } + + private ByteBuffer readValueBytes(TypeCode typeCode, ByteBuffer buffer) throws ImprintException { + // Use TypeHandler for simple types + switch (typeCode) { + case NULL: + case BOOL: + case INT32: + case INT64: + case FLOAT32: + case FLOAT64: + case BYTES: + case STRING: + return typeCode.getHandler().readValueBytes(buffer); + + case ARRAY: + case MAP: + case ROW: + // For complex types, return the entire remaining buffer for now + // The specific deserializer will handle parsing in the future + var remainingBuffer = buffer.slice(); + buffer.position(buffer.limit()); + return remainingBuffer.asReadOnlyBuffer(); + + default: + throw new ImprintException(ErrorType.INVALID_TYPE_CODE, "Unknown type code: " + typeCode); + } + } + + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (obj == null || getClass() != obj.getClass()) return false; + var that = (ImprintRecord) obj; + return header.equals(that.header) && + directory.equals(that.directory) && + payload.equals(that.payload); + } + + @Override + public int hashCode() { + return Objects.hash(header, directory, payload); + } + + @Override + public String toString() { + return String.format("ImprintRecord{header=%s, directorySize=%d, payloadSize=%d}", + header, directory.size(), payload.remaining()); + } +} \ No newline at end of file diff --git a/src/main/java/com/imprint/core/ImprintRecordBuilder.java b/src/main/java/com/imprint/core/ImprintRecordBuilder.java new file mode 100644 index 0000000..48b0998 --- /dev/null +++ b/src/main/java/com/imprint/core/ImprintRecordBuilder.java @@ -0,0 +1,233 @@ +package com.imprint.core; + +import com.imprint.error.ImprintException; +import com.imprint.types.MapKey; +import com.imprint.types.Value; + +import java.util.*; + +/** + * A fluent builder for creating ImprintRecord instances with type-safe, + * developer-friendly API that eliminates boilerplate Value.fromX() calls. + *

+ * Usage: + * var record = ImprintRecord.builder(schemaId) + * .field(1, 42) // int -> Int32Value + * .field(2, "hello") // String -> StringValue + * .field(3, 3.14) // double -> Float64Value + * .field(4, bytes) // byte[] -> BytesValue + * .field(5, true) // boolean -> BoolValue + * .nullField(6) // -> NullValue + * .build(); + */ +public final class ImprintRecordBuilder { + private final SchemaId schemaId; + private final Map fields = new TreeMap<>(); + + ImprintRecordBuilder(SchemaId schemaId) { + this.schemaId = Objects.requireNonNull(schemaId, "SchemaId cannot be null"); + } + + // Primitive types with automatic Value wrapping + public ImprintRecordBuilder field(int id, boolean value) { + return addField(id, Value.fromBoolean(value)); + } + + public ImprintRecordBuilder field(int id, int value) { + return addField(id, Value.fromInt32(value)); + } + + public ImprintRecordBuilder field(int id, long value) { + return addField(id, Value.fromInt64(value)); + } + + public ImprintRecordBuilder field(int id, float value) { + return addField(id, Value.fromFloat32(value)); + } + + public ImprintRecordBuilder field(int id, double value) { + return addField(id, Value.fromFloat64(value)); + } + + public ImprintRecordBuilder field(int id, String value) { + return addField(id, Value.fromString(value)); + } + + public ImprintRecordBuilder field(int id, byte[] value) { + return addField(id, Value.fromBytes(value)); + } + + // Collections with automatic conversion + public ImprintRecordBuilder field(int id, List values) { + var convertedValues = new ArrayList(values.size()); + for (var item : values) { + convertedValues.add(convertToValue(item)); + } + return addField(id, Value.fromArray(convertedValues)); + } + + public ImprintRecordBuilder field(int id, Map map) { + var convertedMap = new HashMap(map.size()); + for (var entry : map.entrySet()) { + var key = convertToMapKey(entry.getKey()); + var value = convertToValue(entry.getValue()); + convertedMap.put(key, value); + } + return addField(id, Value.fromMap(convertedMap)); + } + + // Nested records + public ImprintRecordBuilder field(int id, ImprintRecord nestedRecord) { + return addField(id, Value.fromRow(nestedRecord)); + } + + // Explicit null field + public ImprintRecordBuilder nullField(int id) { + return addField(id, Value.nullValue()); + } + + // Direct Value API (escape hatch for advanced usage) + public ImprintRecordBuilder field(int id, Value value) { + return addField(id, value); + } + + // Conditional field addition + public ImprintRecordBuilder fieldIf(boolean condition, int id, Object value) { + if (condition) { + return field(id, convertToValue(value)); + } + return this; + } + + public ImprintRecordBuilder fieldIfNotNull(int id, Object value) { + return fieldIf(value != null, id, value); + } + + // Bulk operations + public ImprintRecordBuilder fields(Map fieldsMap) { + for (var entry : fieldsMap.entrySet()) { + field(entry.getKey(), convertToValue(entry.getValue())); + } + return this; + } + + // Builder utilities + public boolean hasField(int id) { + return fields.containsKey(id); + } + + public int fieldCount() { + return fields.size(); + } + + public Set fieldIds() { + return new TreeSet<>(fields.keySet()); + } + + // Build the final record + public ImprintRecord build() throws ImprintException { + if (fields.isEmpty()) { + throw new ImprintException(com.imprint.error.ErrorType.SCHEMA_ERROR, + "Cannot build empty record - add at least one field"); + } + + var writer = new ImprintWriter(schemaId); + for (var entry : fields.entrySet()) { + writer.addField(entry.getKey(), entry.getValue()); + } + return writer.build(); + } + + // Internal helper methods + private ImprintRecordBuilder addField(int id, Value value) { + Objects.requireNonNull(value, "Value cannot be null - use nullField() for explicit null values"); + if (fields.containsKey(id)) { + throw new IllegalArgumentException("Field ID " + id + " already exists - field IDs must be unique"); + } + fields.put(id, value); + return this; + } + + private Value convertToValue(Object obj) { + if (obj == null) { + return Value.nullValue(); + } + + if (obj instanceof Value) { + return (Value) obj; + } + + // Auto-boxing conversion + if (obj instanceof Boolean) { + return Value.fromBoolean((Boolean) obj); + } + if (obj instanceof Integer) { + return Value.fromInt32((Integer) obj); + } + if (obj instanceof Long) { + return Value.fromInt64((Long) obj); + } + if (obj instanceof Float) { + return Value.fromFloat32((Float) obj); + } + if (obj instanceof Double) { + return Value.fromFloat64((Double) obj); + } + if (obj instanceof String) { + return Value.fromString((String) obj); + } + if (obj instanceof byte[]) { + return Value.fromBytes((byte[]) obj); + } + if (obj instanceof List) { + //test + @SuppressWarnings("unchecked") + List list = (List) obj; + var convertedValues = new ArrayList(list.size()); + for (var item : list) { + convertedValues.add(convertToValue(item)); + } + return Value.fromArray(convertedValues); + } + if (obj instanceof Map) { + @SuppressWarnings("unchecked") + Map map = (Map) obj; + var convertedMap = new HashMap(map.size()); + for (var entry : map.entrySet()) { + var key = convertToMapKey(entry.getKey()); + var value = convertToValue(entry.getValue()); + convertedMap.put(key, value); + } + return Value.fromMap(convertedMap); + } + if (obj instanceof ImprintRecord) { + return Value.fromRow((ImprintRecord) obj); + } + + throw new IllegalArgumentException("Cannot convert " + obj.getClass().getSimpleName() + + " to Imprint Value. Supported types: boolean, int, long, float, double, String, byte[], List, Map, ImprintRecord"); + } + + private MapKey convertToMapKey(Object obj) { + if (obj instanceof Integer) { + return MapKey.fromInt32((Integer) obj); + } + if (obj instanceof Long) { + return MapKey.fromInt64((Long) obj); + } + if (obj instanceof String) { + return MapKey.fromString((String) obj); + } + if (obj instanceof byte[]) { + return MapKey.fromBytes((byte[]) obj); + } + + throw new IllegalArgumentException("Invalid map key type: " + obj.getClass().getSimpleName() + + ". Map keys must be int, long, String, or byte[]"); + } + + @Override + public String toString() { + return String.format("ImprintRecordBuilder{schemaId=%s, fields=%d}", schemaId, fields.size()); + } +} \ No newline at end of file diff --git a/src/main/java/com/imprint/core/ImprintWriter.java b/src/main/java/com/imprint/core/ImprintWriter.java new file mode 100644 index 0000000..39ad9ea --- /dev/null +++ b/src/main/java/com/imprint/core/ImprintWriter.java @@ -0,0 +1,257 @@ +package com.imprint.core; + +import com.imprint.error.ErrorType; +import com.imprint.error.ImprintException; +import com.imprint.types.MapKey; +import com.imprint.types.Value; +import com.imprint.util.VarInt; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.util.*; + +/** + * A writer for constructing ImprintRecords by adding fields sequentially. + */ +public final class ImprintWriter { + private final SchemaId schemaId; + private final TreeMap fields; // keep fields in sorted order + + public ImprintWriter(SchemaId schemaId) { + this.schemaId = Objects.requireNonNull(schemaId, "SchemaId cannot be null"); + this.fields = new TreeMap<>(); + } + + /** + * Adds a field to the record being built. + */ + public ImprintWriter addField(int id, Value value) { + Objects.requireNonNull(value, "Value cannot be null"); + this.fields.put(id, value); + return this; + } + + /** + * Consumes the writer and builds an ImprintRecord. + */ + public ImprintRecord build() throws ImprintException { + var directory = new ArrayList(fields.size()); + var payloadBuffer = ByteBuffer.allocate(estimatePayloadSize()); + payloadBuffer.order(ByteOrder.LITTLE_ENDIAN); + + for (var entry : fields.entrySet()) { + int fieldId = entry.getKey(); + var value = entry.getValue(); + + directory.add(new DirectoryEntry(fieldId, value.getTypeCode(), payloadBuffer.position())); + serializeValue(value, payloadBuffer); + } + + // Create read-only view of the payload without copying + payloadBuffer.flip(); // limit = position, position = 0 + var payloadView = payloadBuffer.slice().asReadOnlyBuffer(); + + var header = new Header(new Flags((byte) 0), schemaId, payloadView.remaining()); + return new ImprintRecord(header, directory, payloadView); + } + + private int estimatePayloadSize() { + // More accurate estimation to reduce allocations + int estimatedSize = 0; + for (var value : fields.values()) { + estimatedSize += estimateValueSize(value); + } + // Add 25% buffer to reduce reallocations + return Math.max(estimatedSize + (estimatedSize / 4), fields.size() * 16); + } + + /** + * Estimates the serialized size in bytes for a given value. + * This method provides size estimates for payload buffer allocation, + * supporting both array-based and ByteBuffer-based value types. + * + * @param value the value to estimate size for + * @return estimated size in bytes including type-specific overhead + */ + private int estimateValueSize(Value value) { + // Use TypeHandler for simple types + switch (value.getTypeCode()) { + case NULL: + case BOOL: + case INT32: + case INT64: + case FLOAT32: + case FLOAT64: + case BYTES: + case STRING: + return value.getTypeCode().getHandler().estimateSize(value); + + case ARRAY: + List array = ((Value.ArrayValue) value).getValue(); + int arraySize = VarInt.encodedLength(array.size()) + 1; // length + type code + for (Value element : array) { + arraySize += estimateValueSize(element); + } + return arraySize; + + case MAP: + Map map = ((Value.MapValue) value).getValue(); + int mapSize = VarInt.encodedLength(map.size()) + 2; // length + 2 type codes + for (Map.Entry entry : map.entrySet()) { + mapSize += estimateMapKeySize(entry.getKey()); + mapSize += estimateValueSize(entry.getValue()); + } + return mapSize; + + case ROW: + // Estimate nested record size (rough approximation) + return 100; // Conservative estimate + + default: + return 32; // Default fallback + } + } + + private int estimateMapKeySize(MapKey key) { + switch (key.getTypeCode()) { + case INT32: return 4; + case INT64: return 8; + case BYTES: + byte[] bytes = ((MapKey.BytesKey) key).getValue(); + return VarInt.encodedLength(bytes.length) + bytes.length; + + case STRING: + var str = ((MapKey.StringKey) key).getValue(); + int utf8Length = str.getBytes(StandardCharsets.UTF_8).length; + return VarInt.encodedLength(utf8Length) + utf8Length; + + default: + return 16; // Default fallback + } + } + + private void serializeValue(Value value, ByteBuffer buffer) throws ImprintException { + switch (value.getTypeCode()) { + case NULL: + case BOOL: + case INT32: + case INT64: + case FLOAT32: + case FLOAT64: + case BYTES: + case STRING: + value.getTypeCode().getHandler().serialize(value, buffer); + break; + + case ARRAY: + serializeArray((Value.ArrayValue) value, buffer); + break; + + case MAP: + serializeMap((Value.MapValue) value, buffer); + break; + + case ROW: + Value.RowValue rowValue = (Value.RowValue) value; + var serializedRow = rowValue.getValue().serializeToBuffer(); + buffer.put(serializedRow); + break; + + default: + throw new ImprintException(ErrorType.SERIALIZATION_ERROR, + "Unknown type code: " + value.getTypeCode()); + } + } + + private void serializeArray(Value.ArrayValue arrayValue, ByteBuffer buffer) throws ImprintException { + var elements = arrayValue.getValue(); + VarInt.encode(elements.size(), buffer); + + if (elements.isEmpty()) return; + + // All elements must have the same type + var elementType = elements.get(0).getTypeCode(); + buffer.put(elementType.getCode()); + for (var element : elements) { + if (element.getTypeCode() != elementType) { + throw new ImprintException(ErrorType.SCHEMA_ERROR, + "Array elements must have same type code: " + + element.getTypeCode() + " != " + elementType); + } + serializeValue(element, buffer); + } + } + + private void serializeMap(Value.MapValue mapValue, ByteBuffer buffer) throws ImprintException { + var map = mapValue.getValue(); + VarInt.encode(map.size(), buffer); + + if (map.isEmpty()) { + return; + } + + // All keys and values must have consistent types + var iterator = map.entrySet().iterator(); + var first = iterator.next(); + var keyType = first.getKey().getTypeCode(); + var valueType = first.getValue().getTypeCode(); + + buffer.put(keyType.getCode()); + buffer.put(valueType.getCode()); + + // Serialize the first entry + serializeMapKey(first.getKey(), buffer); + serializeValue(first.getValue(), buffer); + + // Serialize remaining entries + while (iterator.hasNext()) { + var entry = iterator.next(); + if (entry.getKey().getTypeCode() != keyType) { + throw new ImprintException(ErrorType.SCHEMA_ERROR, + "Map keys must have same type code: " + + entry.getKey().getTypeCode() + " != " + keyType); + } + if (entry.getValue().getTypeCode() != valueType) { + throw new ImprintException(ErrorType.SCHEMA_ERROR, + "Map values must have same type code: " + + entry.getValue().getTypeCode() + " != " + valueType); + } + + serializeMapKey(entry.getKey(), buffer); + serializeValue(entry.getValue(), buffer); + } + } + + private void serializeMapKey(MapKey key, ByteBuffer buffer) throws ImprintException { + switch (key.getTypeCode()) { + case INT32: + MapKey.Int32Key int32Key = (MapKey.Int32Key) key; + buffer.putInt(int32Key.getValue()); + break; + + case INT64: + MapKey.Int64Key int64Key = (MapKey.Int64Key) key; + buffer.putLong(int64Key.getValue()); + break; + + case BYTES: + MapKey.BytesKey bytesKey = (MapKey.BytesKey) key; + byte[] bytes = bytesKey.getValue(); + VarInt.encode(bytes.length, buffer); + buffer.put(bytes); + break; + + case STRING: + MapKey.StringKey stringKey = (MapKey.StringKey) key; + byte[] stringBytes = stringKey.getValue().getBytes(StandardCharsets.UTF_8); + VarInt.encode(stringBytes.length, buffer); + buffer.put(stringBytes); + break; + + default: + throw new ImprintException(ErrorType.SERIALIZATION_ERROR, + "Invalid map key type: " + key.getTypeCode()); + } + } +} \ No newline at end of file diff --git a/src/main/java/com/imprint/core/SchemaId.java b/src/main/java/com/imprint/core/SchemaId.java new file mode 100644 index 0000000..cb03c1c --- /dev/null +++ b/src/main/java/com/imprint/core/SchemaId.java @@ -0,0 +1,12 @@ +package com.imprint.core; + +import lombok.Value; + +/** + * Schema identifier containing field-space ID and schema hash. + */ +@Value +public class SchemaId { + int fieldspaceId; + int schemaHash; +} \ No newline at end of file diff --git a/src/main/java/com/imprint/error/ErrorType.java b/src/main/java/com/imprint/error/ErrorType.java new file mode 100644 index 0000000..97b9772 --- /dev/null +++ b/src/main/java/com/imprint/error/ErrorType.java @@ -0,0 +1,18 @@ +package com.imprint.error; + +/** + * Types of errors that can occur in Imprint operations. + */ +public enum ErrorType { + INVALID_MAGIC, + UNSUPPORTED_VERSION, + BUFFER_UNDERFLOW, + FIELD_NOT_FOUND, + SCHEMA_ERROR, + INVALID_UTF8_STRING, + MALFORMED_VARINT, + TYPE_MISMATCH, + INVALID_TYPE_CODE, + SERIALIZATION_ERROR, + DESERIALIZATION_ERROR +} diff --git a/src/main/java/com/imprint/error/ImprintException.java b/src/main/java/com/imprint/error/ImprintException.java new file mode 100644 index 0000000..1b5a5d4 --- /dev/null +++ b/src/main/java/com/imprint/error/ImprintException.java @@ -0,0 +1,26 @@ +package com.imprint.error; + +import lombok.Getter; + +/** + * Exception thrown by Imprint operations. + */ +@Getter +public class ImprintException extends Exception { + private final ErrorType errorType; + + public ImprintException(ErrorType errorType, String message) { + super(message); + this.errorType = errorType; + } + + public ImprintException(ErrorType errorType, String message, Throwable cause) { + super(message, cause); + this.errorType = errorType; + } + + @Override + public String toString() { + return String.format("ImprintException{type=%s, message='%s'}", errorType, getMessage()); + } +} diff --git a/src/main/java/com/imprint/types/MapKey.java b/src/main/java/com/imprint/types/MapKey.java new file mode 100644 index 0000000..c0e0747 --- /dev/null +++ b/src/main/java/com/imprint/types/MapKey.java @@ -0,0 +1,163 @@ +package com.imprint.types; + +import com.imprint.error.ErrorType; +import com.imprint.error.ImprintException; +import lombok.EqualsAndHashCode; +import lombok.Getter; + +import java.util.Arrays; +import java.util.Objects; + +/** + * A subset of Value that's valid as a map key. + * Only Int32, Int64, Bytes, and String are valid map keys. + */ +public abstract class MapKey { + + public abstract TypeCode getTypeCode(); + public abstract boolean equals(Object obj); + public abstract int hashCode(); + public abstract String toString(); + + public static MapKey fromInt32(int value) { + return new Int32Key(value); + } + + public static MapKey fromInt64(long value) { + return new Int64Key(value); + } + + public static MapKey fromBytes(byte[] value) { + return new BytesKey(value); + } + + public static MapKey fromString(String value) { + return new StringKey(value); + } + + public static MapKey fromValue(Value value) throws ImprintException { + switch (value.getTypeCode()) { + case INT32: + return fromInt32(((Value.Int32Value) value).getValue()); + case INT64: + return fromInt64(((Value.Int64Value) value).getValue()); + case BYTES: + if (value instanceof Value.BytesBufferValue) { + return fromBytes(((Value.BytesBufferValue) value).getValue()); + } else { + return fromBytes(((Value.BytesValue) value).getValue()); + } + case STRING: + if (value instanceof Value.StringBufferValue) { + return fromString(((Value.StringBufferValue) value).getValue()); + } else { + return fromString(((Value.StringValue) value).getValue()); + } + default: + throw new ImprintException(ErrorType.TYPE_MISMATCH, + "Cannot convert " + value.getTypeCode() + " to MapKey"); + } + } + + public Value toValue() { + switch (getTypeCode()) { + case INT32: + return Value.fromInt32(((Int32Key) this).getValue()); + case INT64: + return Value.fromInt64(((Int64Key) this).getValue()); + case BYTES: + return Value.fromBytes(((BytesKey) this).getValue()); + case STRING: + return Value.fromString(((StringKey) this).getValue()); + default: + throw new IllegalStateException("Unknown MapKey type: " + getTypeCode()); + } + } + + @Getter + @EqualsAndHashCode(callSuper = false) + public static class Int32Key extends MapKey { + private final int value; + + public Int32Key(int value) { + this.value = value; + } + + @Override + public TypeCode getTypeCode() { return TypeCode.INT32; } + + @Override + public String toString() { + return String.valueOf(value); + } + } + + @Getter + @EqualsAndHashCode(callSuper = false) + public static class Int64Key extends MapKey { + private final long value; + + public Int64Key(long value) { + this.value = value; + } + + @Override + public TypeCode getTypeCode() { return TypeCode.INT64; } + + @Override + public String toString() { + return String.valueOf(value); + } + } + + public static class BytesKey extends MapKey { + private final byte[] value; + + public BytesKey(byte[] value) { + this.value = value.clone(); // defensive copy + } + + public byte[] getValue() { + return value.clone(); // defensive copy + } + + @Override + public TypeCode getTypeCode() { return TypeCode.BYTES; } + + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (obj == null || getClass() != obj.getClass()) return false; + BytesKey that = (BytesKey) obj; + return Arrays.equals(value, that.value); + } + + @Override + public int hashCode() { + return Arrays.hashCode(value); + } + + @Override + public String toString() { + return "bytes[" + value.length + "]"; + } + } + + @Getter + @EqualsAndHashCode(callSuper = false) + public static class StringKey extends MapKey { + private final String value; + + public StringKey(String value) { + this.value = Objects.requireNonNull(value, "String cannot be null"); + } + + @Override + public TypeCode getTypeCode() { return TypeCode.STRING; } + + @Override + public String toString() { + return "\"" + value + "\""; + } + } +} \ No newline at end of file diff --git a/src/main/java/com/imprint/types/TypeCode.java b/src/main/java/com/imprint/types/TypeCode.java new file mode 100644 index 0000000..6bf450d --- /dev/null +++ b/src/main/java/com/imprint/types/TypeCode.java @@ -0,0 +1,48 @@ +package com.imprint.types; + +import com.imprint.error.ImprintException; +import com.imprint.error.ErrorType; +import lombok.Getter; + +/** + * Type codes for Imprint values. + */ +public enum TypeCode { + NULL(0x0, TypeHandler.NULL), + BOOL(0x1, TypeHandler.BOOL), + INT32(0x2, TypeHandler.INT32), + INT64(0x3, TypeHandler.INT64), + FLOAT32(0x4, TypeHandler.FLOAT32), + FLOAT64(0x5, TypeHandler.FLOAT64), + BYTES(0x6, TypeHandler.BYTES), + STRING(0x7, TypeHandler.STRING), + ARRAY(0x8, null), // TODO: implement + MAP(0x9, null), // TODO: implement + ROW(0xA, null); // TODO: implement (basically a placeholder for user-defined type) + + @Getter + private final byte code; + private final TypeHandler handler; + + TypeCode(int code, TypeHandler handler) { + this.code = (byte) code; + this.handler = handler; + } + + public TypeHandler getHandler() { + if (handler == null) { + throw new UnsupportedOperationException("Handler not implemented for " + this); + } + return handler; + } + + public static TypeCode fromByte(byte code) throws ImprintException { + for (TypeCode type : values()) { + if (type.code == code) { + return type; + } + } + throw new ImprintException(ErrorType.INVALID_TYPE_CODE, + "Unknown type code: 0x" + Integer.toHexString(code & 0xFF)); + } +} \ No newline at end of file diff --git a/src/main/java/com/imprint/types/TypeHandler.java b/src/main/java/com/imprint/types/TypeHandler.java new file mode 100644 index 0000000..4b5830a --- /dev/null +++ b/src/main/java/com/imprint/types/TypeHandler.java @@ -0,0 +1,304 @@ +package com.imprint.types; + +import com.imprint.error.ImprintException; +import com.imprint.util.VarInt; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; + +/** + * Interface for handling type-specific serialization, deserialization, and size estimation. + * Note that primitives are potentially auto/un-boxed here which could impact performance slightly + * but having all the types in their own implementation helps keep things organized for now, especially + * for dealing with and testing more complex types in the future. + */ +public interface TypeHandler { + Value deserialize(ByteBuffer buffer) throws ImprintException; + void serialize(Value value, ByteBuffer buffer) throws ImprintException; + int estimateSize(Value value); + ByteBuffer readValueBytes(ByteBuffer buffer) throws ImprintException; + + // Static implementations for each type + TypeHandler NULL = new TypeHandler() { + @Override + public Value deserialize(ByteBuffer buffer) { + return Value.nullValue(); + } + + @Override + public void serialize(Value value, ByteBuffer buffer) { + // NULL values have no payload + } + + @Override + public int estimateSize(Value value) { + return 0; + } + + @Override + public ByteBuffer readValueBytes(ByteBuffer buffer) { + return ByteBuffer.allocate(0).asReadOnlyBuffer(); + } + }; + + TypeHandler BOOL = new TypeHandler() { + @Override + public Value deserialize(ByteBuffer buffer) throws ImprintException { + if (buffer.remaining() < 1) { + throw new ImprintException(com.imprint.error.ErrorType.BUFFER_UNDERFLOW, "Not enough bytes for bool"); + } + byte boolByte = buffer.get(); + if (boolByte == 0) return Value.fromBoolean(false); + if (boolByte == 1) return Value.fromBoolean(true); + throw new ImprintException(com.imprint.error.ErrorType.SCHEMA_ERROR, "Invalid boolean value: " + boolByte); + } + + @Override + public void serialize(Value value, ByteBuffer buffer) { + Value.BoolValue boolValue = (Value.BoolValue) value; + buffer.put((byte) (boolValue.getValue() ? 1 : 0)); + } + + @Override + public int estimateSize(Value value) { + return 1; + } + + @Override + public ByteBuffer readValueBytes(ByteBuffer buffer) { + var boolBuffer = buffer.slice(); + boolBuffer.limit(1); + buffer.position(buffer.position() + 1); + return boolBuffer.asReadOnlyBuffer(); + } + }; + + TypeHandler INT32 = new TypeHandler() { + @Override + public Value deserialize(ByteBuffer buffer) throws ImprintException { + if (buffer.remaining() < 4) { + throw new ImprintException(com.imprint.error.ErrorType.BUFFER_UNDERFLOW, "Not enough bytes for int32"); + } + return Value.fromInt32(buffer.getInt()); + } + + @Override + public void serialize(Value value, ByteBuffer buffer) { + Value.Int32Value int32Value = (Value.Int32Value) value; + buffer.putInt(int32Value.getValue()); + } + + @Override + public int estimateSize(Value value) { + return 4; + } + + @Override + public ByteBuffer readValueBytes(ByteBuffer buffer) { + var int32Buffer = buffer.slice(); + int32Buffer.limit(4); + buffer.position(buffer.position() + 4); + return int32Buffer.asReadOnlyBuffer(); + } + }; + + TypeHandler INT64 = new TypeHandler() { + @Override + public Value deserialize(ByteBuffer buffer) throws ImprintException { + if (buffer.remaining() < 8) { + throw new ImprintException(com.imprint.error.ErrorType.BUFFER_UNDERFLOW, "Not enough bytes for int64"); + } + return Value.fromInt64(buffer.getLong()); + } + + @Override + public void serialize(Value value, ByteBuffer buffer) { + Value.Int64Value int64Value = (Value.Int64Value) value; + buffer.putLong(int64Value.getValue()); + } + + @Override + public int estimateSize(Value value) { + return 8; + } + + @Override + public ByteBuffer readValueBytes(ByteBuffer buffer) { + var int64Buffer = buffer.slice(); + int64Buffer.limit(8); + buffer.position(buffer.position() + 8); + return int64Buffer.asReadOnlyBuffer(); + } + }; + + TypeHandler FLOAT32 = new TypeHandler() { + @Override + public Value deserialize(ByteBuffer buffer) throws ImprintException { + if (buffer.remaining() < 4) { + throw new ImprintException(com.imprint.error.ErrorType.BUFFER_UNDERFLOW, "Not enough bytes for float32"); + } + return Value.fromFloat32(buffer.getFloat()); + } + + @Override + public void serialize(Value value, ByteBuffer buffer) { + Value.Float32Value float32Value = (Value.Float32Value) value; + buffer.putFloat(float32Value.getValue()); + } + + @Override + public int estimateSize(Value value) { + return 4; + } + + @Override + public ByteBuffer readValueBytes(ByteBuffer buffer) { + var float32Buffer = buffer.slice(); + float32Buffer.limit(4); + buffer.position(buffer.position() + 4); + return float32Buffer.asReadOnlyBuffer(); + } + }; + + TypeHandler FLOAT64 = new TypeHandler() { + @Override + public Value deserialize(ByteBuffer buffer) throws ImprintException { + if (buffer.remaining() < 8) { + throw new ImprintException(com.imprint.error.ErrorType.BUFFER_UNDERFLOW, "Not enough bytes for float64"); + } + return Value.fromFloat64(buffer.getDouble()); + } + + @Override + public void serialize(Value value, ByteBuffer buffer) { + Value.Float64Value float64Value = (Value.Float64Value) value; + buffer.putDouble(float64Value.getValue()); + } + + @Override + public int estimateSize(Value value) { + return 8; + } + + @Override + public ByteBuffer readValueBytes(ByteBuffer buffer) { + var float64Buffer = buffer.slice(); + float64Buffer.limit(8); + buffer.position(buffer.position() + 8); + return float64Buffer.asReadOnlyBuffer(); + } + }; + + TypeHandler BYTES = new TypeHandler() { + @Override + public Value deserialize(ByteBuffer buffer) throws ImprintException { + VarInt.DecodeResult lengthResult = VarInt.decode(buffer); + int length = lengthResult.getValue(); + if (buffer.remaining() < length) { + throw new ImprintException(com.imprint.error.ErrorType.BUFFER_UNDERFLOW, "Not enough bytes for bytes value"); + } + var bytesView = buffer.slice(); + bytesView.limit(length); + buffer.position(buffer.position() + length); + return Value.fromBytesBuffer(bytesView.asReadOnlyBuffer()); + } + + @Override + public void serialize(Value value, ByteBuffer buffer) { + if (value instanceof Value.BytesBufferValue) { + Value.BytesBufferValue bufferValue = (Value.BytesBufferValue) value; + var bytesBuffer = bufferValue.getBuffer(); + VarInt.encode(bytesBuffer.remaining(), buffer); + buffer.put(bytesBuffer); + } else { + Value.BytesValue bytesValue = (Value.BytesValue) value; + byte[] bytes = bytesValue.getValue(); + VarInt.encode(bytes.length, buffer); + buffer.put(bytes); + } + } + + @Override + public int estimateSize(Value value) { + if (value instanceof Value.BytesBufferValue) { + Value.BytesBufferValue bufferValue = (Value.BytesBufferValue) value; + int length = bufferValue.getBuffer().remaining(); + return VarInt.encodedLength(length) + length; + } else { + byte[] bytes = ((Value.BytesValue) value).getValue(); + return VarInt.encodedLength(bytes.length) + bytes.length; + } + } + + @Override + public ByteBuffer readValueBytes(ByteBuffer buffer) throws ImprintException { + int originalPosition = buffer.position(); + VarInt.DecodeResult lengthResult = VarInt.decode(buffer); + int totalLength = lengthResult.getBytesRead() + lengthResult.getValue(); + buffer.position(originalPosition); + var valueBuffer = buffer.slice(); + valueBuffer.limit(totalLength); + buffer.position(buffer.position() + totalLength); + return valueBuffer.asReadOnlyBuffer(); + } + }; + + TypeHandler STRING = new TypeHandler() { + @Override + public Value deserialize(ByteBuffer buffer) throws ImprintException { + VarInt.DecodeResult strLengthResult = VarInt.decode(buffer); + int strLength = strLengthResult.getValue(); + if (buffer.remaining() < strLength) { + throw new ImprintException(com.imprint.error.ErrorType.BUFFER_UNDERFLOW, "Not enough bytes for string value"); + } + var stringBytesView = buffer.slice(); + stringBytesView.limit(strLength); + buffer.position(buffer.position() + strLength); + try { + return Value.fromStringBuffer(stringBytesView.asReadOnlyBuffer()); + } catch (Exception e) { + throw new ImprintException(com.imprint.error.ErrorType.INVALID_UTF8_STRING, "Invalid UTF-8 string"); + } + } + + @Override + public void serialize(Value value, ByteBuffer buffer) { + if (value instanceof Value.StringBufferValue) { + Value.StringBufferValue bufferValue = (Value.StringBufferValue) value; + var stringBuffer = bufferValue.getBuffer(); + VarInt.encode(stringBuffer.remaining(), buffer); + buffer.put(stringBuffer); + } else { + Value.StringValue stringValue = (Value.StringValue) value; + byte[] stringBytes = stringValue.getValue().getBytes(StandardCharsets.UTF_8); + VarInt.encode(stringBytes.length, buffer); + buffer.put(stringBytes); + } + } + + @Override + public int estimateSize(Value value) { + if (value instanceof Value.StringBufferValue) { + Value.StringBufferValue bufferValue = (Value.StringBufferValue) value; + int length = bufferValue.getBuffer().remaining(); + return VarInt.encodedLength(length) + length; + } else { + String str = ((Value.StringValue) value).getValue(); + int utf8Length = str.getBytes(StandardCharsets.UTF_8).length; + return VarInt.encodedLength(utf8Length) + utf8Length; + } + } + + @Override + public ByteBuffer readValueBytes(ByteBuffer buffer) throws ImprintException { + int originalPosition = buffer.position(); + VarInt.DecodeResult lengthResult = VarInt.decode(buffer); + int totalLength = lengthResult.getBytesRead() + lengthResult.getValue(); + buffer.position(originalPosition); + var valueBuffer = buffer.slice(); + valueBuffer.limit(totalLength); + buffer.position(buffer.position() + totalLength); + return valueBuffer.asReadOnlyBuffer(); + } + }; +} \ No newline at end of file diff --git a/src/main/java/com/imprint/types/Value.java b/src/main/java/com/imprint/types/Value.java new file mode 100644 index 0000000..4710ec5 --- /dev/null +++ b/src/main/java/com/imprint/types/Value.java @@ -0,0 +1,403 @@ +package com.imprint.types; + +import com.imprint.core.ImprintRecord; +import lombok.EqualsAndHashCode; +import lombok.Getter; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * A value that can be stored in an Imprint record. + */ +public abstract class Value { + + public abstract TypeCode getTypeCode(); + public abstract boolean equals(Object obj); + public abstract int hashCode(); + public abstract String toString(); + + // Factory methods + public static Value nullValue() { + return NullValue.INSTANCE; + } + + public static Value fromBoolean(boolean value) { + return new BoolValue(value); + } + + public static Value fromInt32(int value) { + return new Int32Value(value); + } + + public static Value fromInt64(long value) { + return new Int64Value(value); + } + + public static Value fromFloat32(float value) { + return new Float32Value(value); + } + + public static Value fromFloat64(double value) { + return new Float64Value(value); + } + + public static Value fromBytes(byte[] value) { + return new BytesValue(value); + } + + public static Value fromBytesBuffer(ByteBuffer value) { + return new BytesBufferValue(value); + } + + public static Value fromString(String value) { + return new StringValue(value); + } + + public static Value fromStringBuffer(ByteBuffer value) { + return new StringBufferValue(value); + } + + public static Value fromArray(List value) { + return new ArrayValue(value); + } + + public static Value fromMap(Map value) { + return new MapValue(value); + } + + public static Value fromRow(ImprintRecord value) { + return new RowValue(value); + } + + // Null Value + @EqualsAndHashCode(callSuper = false) + public static class NullValue extends Value { + public static final NullValue INSTANCE = new NullValue(); + + private NullValue() {} + + @Override + public TypeCode getTypeCode() { return TypeCode.NULL; } + + @Override + public String toString() { + return "null"; + } + } + + // Boolean Value + @Getter + @EqualsAndHashCode(callSuper = false) + public static class BoolValue extends Value { + private final boolean value; + + public BoolValue(boolean value) { + this.value = value; + } + + public boolean getValue() { return value; } + + @Override + public TypeCode getTypeCode() { return TypeCode.BOOL; } + + @Override + public String toString() { + return String.valueOf(value); + } + } + + // Int32 Value + @Getter + @EqualsAndHashCode(callSuper = false) + public static class Int32Value extends Value { + private final int value; + + public Int32Value(int value) { + this.value = value; + } + + @Override + public TypeCode getTypeCode() { return TypeCode.INT32; } + + @Override + public String toString() { + return String.valueOf(value); + } + } + + // Int64 Value + @Getter + @EqualsAndHashCode(callSuper = false) + public static class Int64Value extends Value { + private final long value; + + public Int64Value(long value) { + this.value = value; + } + + @Override + public TypeCode getTypeCode() { return TypeCode.INT64; } + + @Override + public String toString() { + return String.valueOf(value); + } + } + + // Float32 Value + @Getter + @EqualsAndHashCode(callSuper = false) + public static class Float32Value extends Value { + private final float value; + + public Float32Value(float value) { + this.value = value; + } + + @Override + public TypeCode getTypeCode() { return TypeCode.FLOAT32; } + + @Override + public String toString() { + return String.valueOf(value); + } + } + + // Float64 Value + @Getter + @EqualsAndHashCode(callSuper = false) + public static class Float64Value extends Value { + private final double value; + + public Float64Value(double value) { + this.value = value; + } + + @Override + public TypeCode getTypeCode() { return TypeCode.FLOAT64; } + + @Override + public String toString() { + return String.valueOf(value); + } + } + + // Bytes Value (array-based) + public static class BytesValue extends Value { + private final byte[] value; + + public BytesValue(byte[] value) { + this.value = value.clone(); // defensive copy + } + + public byte[] getValue() { + return value.clone(); // defensive copy + } + + @Override + public TypeCode getTypeCode() { return TypeCode.BYTES; } + + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (obj == null || getClass() != obj.getClass()) return false; + BytesValue that = (BytesValue) obj; + return Arrays.equals(value, that.value); + } + + @Override + public int hashCode() { + return Arrays.hashCode(value); + } + + @Override + public String toString() { + return "bytes[" + value.length + "]"; + } + } + + // Bytes Value (ByteBuffer-based, zero-copy) + public static class BytesBufferValue extends Value { + private final ByteBuffer value; + + public BytesBufferValue(ByteBuffer value) { + this.value = value.asReadOnlyBuffer(); // zero-copy read-only view + } + + public byte[] getValue() { + // Fallback to array when needed + byte[] array = new byte[value.remaining()]; + value.duplicate().get(array); + return array; + } + + public ByteBuffer getBuffer() { + return value.duplicate(); // zero-copy view + } + + @Override + public TypeCode getTypeCode() { return TypeCode.BYTES; } + + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (obj == null) return false; + if (obj instanceof BytesBufferValue) { + BytesBufferValue that = (BytesBufferValue) obj; + return value.equals(that.value); + } + if (obj instanceof BytesValue) { + BytesValue that = (BytesValue) obj; + return Arrays.equals(getValue(), that.getValue()); + } + return false; + } + + @Override + public int hashCode() { + return value.hashCode(); + } + + @Override + public String toString() { + return "bytes[" + value.remaining() + "]"; + } + } + + // String Value (String-based) + @Getter + @EqualsAndHashCode(callSuper = false) + public static class StringValue extends Value { + private final String value; + + public StringValue(String value) { + this.value = Objects.requireNonNull(value, "String cannot be null"); + } + + @Override + public TypeCode getTypeCode() { return TypeCode.STRING; } + + @Override + public String toString() { + return "\"" + value + "\""; + } + } + + // String Value (ByteBuffer-based, zero-copy) + public static class StringBufferValue extends Value { + private final ByteBuffer value; + private volatile String cachedString; // lazy decode + + public StringBufferValue(ByteBuffer value) { + this.value = value.asReadOnlyBuffer(); // zero-copy read-only view + } + + public String getValue() { + if (cachedString == null) { + synchronized (this) { + if (cachedString == null) { + var array = new byte[value.remaining()]; + value.duplicate().get(array); + cachedString = new String(array, StandardCharsets.UTF_8); + } + } + } + return cachedString; + } + + public ByteBuffer getBuffer() { + return value.duplicate(); // zero-copy view + } + + @Override + public TypeCode getTypeCode() { return TypeCode.STRING; } + + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (obj == null) return false; + if (obj instanceof StringBufferValue) { + StringBufferValue that = (StringBufferValue) obj; + return value.equals(that.value); + } + if (obj instanceof StringValue) { + StringValue that = (StringValue) obj; + return getValue().equals(that.getValue()); + } + return false; + } + + @Override + public int hashCode() { + return getValue().hashCode(); // Use string hash for consistency + } + + @Override + public String toString() { + return "\"" + getValue() + "\""; + } + } + + // Array Value + @Getter + @EqualsAndHashCode(callSuper = false) + public static class ArrayValue extends Value { + private final List value; + + public ArrayValue(List value) { + this.value = List.copyOf(Objects.requireNonNull(value, "Array cannot be null")); + } + + @Override + public TypeCode getTypeCode() { return TypeCode.ARRAY; } + + @Override + public String toString() { + return value.toString(); + } + } + + // Map Value + @Getter + @EqualsAndHashCode(callSuper = false) + public static class MapValue extends Value { + private final Map value; + + public MapValue(Map value) { + this.value = Map.copyOf(Objects.requireNonNull(value, "Map cannot be null")); + } + + @Override + public TypeCode getTypeCode() { return TypeCode.MAP; } + + @Override + public String toString() { + return value.toString(); + } + } + + // Row Value + @Getter + @EqualsAndHashCode(callSuper = false) + public static class RowValue extends Value { + private final ImprintRecord value; + + public RowValue(ImprintRecord value) { + this.value = Objects.requireNonNull(value, "Record cannot be null"); + } + + @Override + public TypeCode getTypeCode() { return TypeCode.ROW; } + + @Override + public String toString() { + return "Row{" + value + "}"; + } + } + +} \ No newline at end of file diff --git a/src/main/java/com/imprint/util/VarInt.java b/src/main/java/com/imprint/util/VarInt.java new file mode 100644 index 0000000..5c9a7e5 --- /dev/null +++ b/src/main/java/com/imprint/util/VarInt.java @@ -0,0 +1,118 @@ +package com.imprint.util; + +import com.imprint.error.ImprintException; +import com.imprint.error.ErrorType; +import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; +import java.nio.ByteBuffer; + +/** + * Utility class for encoding and decoding variable-length integers (VarInt). + * Supports encoding/decoding of 32-bit unsigned integers. + */ +public final class VarInt { + + private static final byte CONTINUATION_BIT = (byte) 0x80; + private static final byte SEGMENT_BITS = 0x7f; + private static final int MAX_VARINT_LEN = 5; // Enough for u32 + + private VarInt() {} // utility class + + + /** + * Encode a 32-bit unsigned integer as a VarInt into the given ByteBuffer. + * @param value the value to encode (treated as unsigned) + * @param buffer the buffer to write to + */ + public static void encode(int value, ByteBuffer buffer) { + // Convert to unsigned long for proper bit manipulation + long val = Integer.toUnsignedLong(value); + + // Encode at least one byte, then continue while value has more bits + do { + byte b = (byte) (val & SEGMENT_BITS); + val >>>= 7; + if (val != 0) { + b |= CONTINUATION_BIT; + } + buffer.put(b); + } while (val != 0); + } + + + /** + * Decode a VarInt from a ByteBuffer. + * @param buffer the buffer to decode from + * @return a DecodeResult containing the decoded value and number of bytes consumed + * @throws ImprintException if the VarInt is malformed + */ + public static DecodeResult decode(ByteBuffer buffer) throws ImprintException { + long result = 0; + int shift = 0; + int bytesRead = 0; + + while (true) { + if (bytesRead >= MAX_VARINT_LEN) { + throw new ImprintException(ErrorType.MALFORMED_VARINT, "VarInt too long"); + } + if (!buffer.hasRemaining()) { + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, + "Unexpected end of data while reading VarInt"); + } + + byte b = buffer.get(); + bytesRead++; + + // Check if adding these 7 bits would overflow + long segment = b & SEGMENT_BITS; + if (shift >= 32 || (shift == 28 && segment > 0xF)) { + throw new ImprintException(ErrorType.MALFORMED_VARINT, "VarInt overflow"); + } + + // Add the bottom 7 bits to the result + result |= segment << shift; + + // If the high bit is not set, this is the last byte + if ((b & CONTINUATION_BIT) == 0) { + break; + } + + shift += 7; + } + + return new DecodeResult((int) result, bytesRead); + } + + /** + * Calculate the number of bytes needed to encode the given value as a VarInt. + * @param value the value to encode (treated as unsigned) + * @return the number of bytes needed + */ + public static int encodedLength(int value) { + // Convert to unsigned long for proper bit manipulation + long val = Integer.toUnsignedLong(value); + int length = 1; + + // Count additional bytes needed for values >= 128 + while (val >= 0x80) { + val >>>= 7; + length++; + } + + return length; + } + + /** + * Result of a VarInt decode operation. + */ + @Getter + @AllArgsConstructor + @EqualsAndHashCode + @ToString + public static class DecodeResult { + private final int value; + private final int bytesRead; + } +} \ No newline at end of file diff --git a/src/test/java/com/imprint/ByteBufferIntegrationTest.java b/src/test/java/com/imprint/ByteBufferIntegrationTest.java new file mode 100644 index 0000000..9460cbf --- /dev/null +++ b/src/test/java/com/imprint/ByteBufferIntegrationTest.java @@ -0,0 +1,87 @@ +package com.imprint; + +import com.imprint.core.*; +import com.imprint.types.*; +import com.imprint.error.ImprintException; +import java.nio.ByteBuffer; +import java.util.*; + +/** + * Integration test to verify ByteBuffer functionality and zero-copy benefits. + */ +public class ByteBufferIntegrationTest { + + public static void main(String[] args) { + try { + testByteBufferFunctionality(); + testZeroCopy(); + System.out.println("All ByteBuffer integration tests passed!"); + } catch (Exception e) { + System.err.println("ByteBuffer integration test failed: " + e.getMessage()); + e.printStackTrace(); + System.exit(1); + } + } + + static void testByteBufferFunctionality() throws ImprintException { + System.out.println("Testing ByteBuffer functionality..."); + + SchemaId schemaId = new SchemaId(1, 0xdeadbeef); + ImprintWriter writer = new ImprintWriter(schemaId); + + writer.addField(1, Value.fromInt32(42)) + .addField(2, Value.fromString("zero-copy test")) + .addField(3, Value.fromBytes(new byte[]{1, 2, 3, 4, 5})); + + ImprintRecord record = writer.build(); + + // Test ByteBuffer serialization + ByteBuffer serializedBuffer = record.serializeToBuffer(); + assert serializedBuffer.isReadOnly() : "Serialized buffer should be read-only"; + + // Test deserialization from ByteBuffer + ImprintRecord deserialized = ImprintRecord.deserialize(serializedBuffer); + + assert deserialized.getValue(1).get().equals(Value.fromInt32(42)); + assert deserialized.getValue(2).get().equals(Value.fromString("zero-copy test")); + + // Test raw bytes access returns ByteBuffer + Optional rawBytes = deserialized.getRawBytes(1); + assert rawBytes.isPresent() : "Raw bytes should be present for field 1"; + assert rawBytes.get().isReadOnly() : "Raw bytes buffer should be read-only"; + + System.out.println("ByteBuffer functionality test passed"); + } + + static void testZeroCopy() { + System.out.println("Testing zero-copy"); + + // Create a large payload to demonstrate zero-copy benefits + byte[] largePayload = new byte[1024 * 1024]; // 1MB + Arrays.fill(largePayload, (byte) 0xAB); + + SchemaId schemaId = new SchemaId(2, 0xcafebabe); + ImprintWriter writer = new ImprintWriter(schemaId); + + try { + writer.addField(1, Value.fromBytes(largePayload)); + ImprintRecord record = writer.build(); + + // Test that getRawBytes returns a view, not a copy + Optional rawBytes = record.getRawBytes(1); + assert rawBytes.isPresent() : "Raw bytes should be present"; + + ByteBuffer rawBuffer = rawBytes.get(); + assert rawBuffer.isReadOnly() : "Raw buffer should be read-only"; + + // The buffer should be positioned at the start of the actual data + // (after the VarInt length prefix) + assert rawBuffer.remaining() > largePayload.length : "Buffer should include length prefix"; + + System.out.println("Zero-copy benefits test passed"); + + } catch (ImprintException e) { + throw new RuntimeException("Failed zero-copy test", e); + } + } +} \ No newline at end of file diff --git a/src/test/java/com/imprint/ComprehensiveTest.java b/src/test/java/com/imprint/ComprehensiveTest.java new file mode 100644 index 0000000..af7f0b0 --- /dev/null +++ b/src/test/java/com/imprint/ComprehensiveTest.java @@ -0,0 +1,208 @@ +package com.imprint; + +import com.imprint.core.*; +import com.imprint.types.*; +import com.imprint.error.ImprintException; +import com.imprint.util.VarInt; +import java.nio.ByteBuffer; +import java.util.*; + +/** + * Comprehensive test to verify all functionality works correctly. + */ +public class ComprehensiveTest { + + public static void main(String[] args) { + try { + testVarIntFunctionality(); + testValueTypes(); + testMapKeys(); + testComplexSerialization(); + testErrorHandling(); + testByteBufferPerformance(); + System.out.println("All comprehensive tests passed!"); + } catch (Exception e) { + System.err.println("Comprehensive test failed: " + e.getMessage()); + e.printStackTrace(); + System.exit(1); + } + } + + static void testVarIntFunctionality() throws ImprintException { + System.out.println("Testing VarInt functionality..."); + + // Test encoding/decoding of various values + int[] testValues = {0, 1, 127, 128, 16383, 16384, Integer.MAX_VALUE}; + + for (int value : testValues) { + ByteBuffer buffer = ByteBuffer.allocate(10); + VarInt.encode(value, buffer); + int encodedLength = buffer.position(); + + buffer.flip(); + VarInt.DecodeResult result = VarInt.decode(buffer); + + assert result.getValue() == value : "VarInt roundtrip failed for " + value; + assert result.getBytesRead() == encodedLength : "Bytes read mismatch for " + value; + } + + System.out.println("✓ VarInt functionality test passed"); + } + + static void testValueTypes() { + System.out.println("Testing Value types"); + + // Test all value types + Value nullVal = Value.nullValue(); + Value boolVal = Value.fromBoolean(true); + Value int32Val = Value.fromInt32(42); + Value int64Val = Value.fromInt64(123456789L); + Value float32Val = Value.fromFloat32(3.14f); + Value float64Val = Value.fromFloat64(2.718281828); + Value bytesVal = Value.fromBytes(new byte[]{1, 2, 3, 4}); + Value stringVal = Value.fromString("test"); + + // Test type codes + assert nullVal.getTypeCode() == TypeCode.NULL; + assert boolVal.getTypeCode() == TypeCode.BOOL; + assert int32Val.getTypeCode() == TypeCode.INT32; + assert int64Val.getTypeCode() == TypeCode.INT64; + assert float32Val.getTypeCode() == TypeCode.FLOAT32; + assert float64Val.getTypeCode() == TypeCode.FLOAT64; + assert bytesVal.getTypeCode() == TypeCode.BYTES; + assert stringVal.getTypeCode() == TypeCode.STRING; + + // Test value extraction + assert ((Value.BoolValue) boolVal).getValue(); + assert ((Value.Int32Value) int32Val).getValue() == 42; + assert ((Value.Int64Value) int64Val).getValue() == 123456789L; + assert ((Value.Float32Value) float32Val).getValue() == 3.14f; + assert ((Value.Float64Value) float64Val).getValue() == 2.718281828; + assert Arrays.equals(((Value.BytesValue) bytesVal).getValue(), new byte[]{1, 2, 3, 4}); + assert ((Value.StringValue) stringVal).getValue().equals("test"); + + System.out.println("✓ Value types test passed"); + } + + static void testMapKeys() throws ImprintException { + System.out.println("Testing MapKey functionality..."); + + MapKey int32Key = MapKey.fromInt32(42); + MapKey int64Key = MapKey.fromInt64(123L); + MapKey bytesKey = MapKey.fromBytes(new byte[]{1, 2, 3}); + MapKey stringKey = MapKey.fromString("test"); + + // Test conversion to/from Values + Value int32Value = int32Key.toValue(); + Value int64Value = int64Key.toValue(); + Value bytesValue = bytesKey.toValue(); + Value stringValue = stringKey.toValue(); + + assert MapKey.fromValue(int32Value).equals(int32Key); + assert MapKey.fromValue(int64Value).equals(int64Key); + assert MapKey.fromValue(bytesValue).equals(bytesKey); + assert MapKey.fromValue(stringValue).equals(stringKey); + + System.out.println("✓ MapKey functionality test passed"); + } + + static void testComplexSerialization() throws ImprintException { + System.out.println("Testing complex serialization..."); + + SchemaId schemaId = new SchemaId(1, 0xdeadbeef); + ImprintWriter writer = new ImprintWriter(schemaId); + + // Create complex nested structure + List array = Arrays.asList( + Value.fromInt32(1), + Value.fromInt32(2), + Value.fromInt32(3) + ); + + Map map = new HashMap<>(); + map.put(MapKey.fromString("key1"), Value.fromString("value1")); + map.put(MapKey.fromString("key2"), Value.fromString("value2")); + + writer.addField(1, Value.fromArray(array)) + .addField(2, Value.fromMap(map)) + .addField(3, Value.fromString("complex test")); + + ImprintRecord record = writer.build(); + + // Test ByteBuffer serialization + ByteBuffer serialized = record.serializeToBuffer(); + ImprintRecord deserialized = ImprintRecord.deserialize(serialized); + + // Verify complex structures + Value deserializedArray = deserialized.getValue(1).get(); + assert deserializedArray instanceof Value.ArrayValue; + List deserializedList = ((Value.ArrayValue) deserializedArray).getValue(); + assert deserializedList.size() == 3; + assert deserializedList.get(0).equals(Value.fromInt32(1)); + + Value deserializedMap = deserialized.getValue(2).get(); + assert deserializedMap instanceof Value.MapValue; + Map deserializedMapValue = ((Value.MapValue) deserializedMap).getValue(); + assert deserializedMapValue.size() == 2; + assert deserializedMapValue.get(MapKey.fromString("key1")).equals(Value.fromString("value1")); + + System.out.println("✓ Complex serialization test passed"); + } + + static void testErrorHandling() { + System.out.println("Testing error handling..."); + + try { + // Test invalid type code + TypeCode.fromByte((byte) 0xFF); + assert false : "Should have thrown exception for invalid type code"; + } catch (ImprintException e) { + assert e.getErrorType() == com.imprint.error.ErrorType.INVALID_TYPE_CODE; + } + + try { + // Test invalid magic byte + byte[] invalidData = new byte[15]; + invalidData[0] = 0x00; // wrong magic + ImprintRecord.deserialize(invalidData); + assert false : "Should have thrown exception for invalid magic"; + } catch (ImprintException e) { + assert e.getErrorType() == com.imprint.error.ErrorType.INVALID_MAGIC; + } + + System.out.println("✓ Error handling test passed"); + } + + static void testByteBufferPerformance() throws ImprintException { + System.out.println("Testing ByteBuffer performance benefits..."); + + // Create a record with moderate-sized data + byte[] testData = new byte[1024]; + for (int i = 0; i < testData.length; i++) { + testData[i] = (byte) (i % 256); + } + + SchemaId schemaId = new SchemaId(1, 0x12345678); + ImprintWriter writer = new ImprintWriter(schemaId); + writer.addField(1, Value.fromBytes(testData)) + .addField(2, Value.fromString("performance test")); + + ImprintRecord record = writer.build(); + + // Test that raw bytes access is zero-copy + Optional rawBytes = record.getRawBytes(1); + assert rawBytes.isPresent(); + assert rawBytes.get().isReadOnly(); + + // Test ByteBuffer serialization + ByteBuffer serialized = record.serializeToBuffer(); + assert serialized.isReadOnly(); + + // Verify deserialization works + ImprintRecord deserialized = ImprintRecord.deserialize(serialized); + Value retrievedBytes = deserialized.getValue(1).get(); + assert Arrays.equals(((Value.BytesValue) retrievedBytes).getValue(), testData); + + System.out.println("✓ ByteBuffer performance test passed"); + } +} \ No newline at end of file diff --git a/src/test/java/com/imprint/IntegrationTest.java b/src/test/java/com/imprint/IntegrationTest.java new file mode 100644 index 0000000..49cfce7 --- /dev/null +++ b/src/test/java/com/imprint/IntegrationTest.java @@ -0,0 +1,144 @@ +package com.imprint; + +import com.imprint.core.*; +import com.imprint.types.*; +import com.imprint.error.ImprintException; +import java.util.*; + +/** + * Integration test to verify the complete Java implementation works. + * This can be run as a simple main method without JUnit. + */ +public class IntegrationTest { + + public static void main(String[] args) { + try { + testBasicFunctionality(); + testArraysAndMaps(); + testNestedRecords(); + System.out.println("All integration tests passed!"); + } catch (Exception e) { + System.err.println("Integration test failed: " + e.getMessage()); + e.printStackTrace(); + System.exit(1); + } + } + + static void testBasicFunctionality() throws ImprintException { + System.out.println("Testing basic functionality..."); + + SchemaId schemaId = new SchemaId(1, 0xdeadbeef); + ImprintWriter writer = new ImprintWriter(schemaId); + + writer.addField(1, Value.fromInt32(42)) + .addField(2, Value.fromString("testing java imprint spec")) + .addField(3, Value.fromBoolean(true)) + .addField(4, Value.fromFloat64(3.14159)) + .addField(5, Value.fromBytes(new byte[]{1, 2, 3, 4})); + + ImprintRecord record = writer.build(); + + // Verify we can read values back + assert record.getValue(1).get().equals(Value.fromInt32(42)); + assert record.getValue(2).get().equals(Value.fromString("testing java imprint spec")); + assert record.getValue(3).get().equals(Value.fromBoolean(true)); + assert record.getValue(999).isEmpty(); // non-existent field + + // Test serialization round-trip + var buffer = record.serializeToBuffer(); + byte[] serialized = new byte[buffer.remaining()]; + buffer.get(serialized); + ImprintRecord deserialized = ImprintRecord.deserialize(serialized); + + assert deserialized.getValue(1).get().equals(Value.fromInt32(42)); + assert deserialized.getValue(2).get().equals(Value.fromString("testing java imprint spec")); + assert deserialized.getValue(3).get().equals(Value.fromBoolean(true)); + + System.out.println("✓ Basic functionality test passed"); + } + + static void testArraysAndMaps() throws ImprintException { + System.out.println("Testing arrays and maps..."); + + SchemaId schemaId = new SchemaId(2, 0xcafebabe); + ImprintWriter writer = new ImprintWriter(schemaId); + + // Create an array + List intArray = Arrays.asList( + Value.fromInt32(1), + Value.fromInt32(2), + Value.fromInt32(3) + ); + + // Create a map + Map stringToIntMap = new HashMap<>(); + stringToIntMap.put(MapKey.fromString("one"), Value.fromInt32(1)); + stringToIntMap.put(MapKey.fromString("two"), Value.fromInt32(2)); + + writer.addField(1, Value.fromArray(intArray)) + .addField(2, Value.fromMap(stringToIntMap)); + + ImprintRecord record = writer.build(); + + // Test serialization round-trip + var buffer = record.serializeToBuffer(); + byte[] serialized = new byte[buffer.remaining()]; + buffer.get(serialized); + ImprintRecord deserialized = ImprintRecord.deserialize(serialized); + + // Verify array + Value arrayValue = deserialized.getValue(1).get(); + assert arrayValue instanceof Value.ArrayValue; + List deserializedArray = ((Value.ArrayValue) arrayValue).getValue(); + assert deserializedArray.size() == 3; + assert deserializedArray.get(0).equals(Value.fromInt32(1)); + + // Verify map + Value mapValue = deserialized.getValue(2).get(); + assert mapValue instanceof Value.MapValue; + Map deserializedMap = ((Value.MapValue) mapValue).getValue(); + assert deserializedMap.size() == 2; + assert deserializedMap.get(MapKey.fromString("one")).equals(Value.fromInt32(1)); + + System.out.println("✓ Arrays and maps test passed"); + } + + static void testNestedRecords() throws ImprintException { + System.out.println("Testing nested records..."); + + // Create inner record + SchemaId innerSchemaId = new SchemaId(3, 0x12345678); + ImprintWriter innerWriter = new ImprintWriter(innerSchemaId); + innerWriter.addField(1, Value.fromString("nested data")) + .addField(2, Value.fromInt64(9876543210L)); + ImprintRecord innerRecord = innerWriter.build(); + + // Create outer record + SchemaId outerSchemaId = new SchemaId(4, 0x87654321); + ImprintWriter outerWriter = new ImprintWriter(outerSchemaId); + outerWriter.addField(1, Value.fromRow(innerRecord)) + .addField(2, Value.fromString("outer data")); + ImprintRecord outerRecord = outerWriter.build(); + + // Test serialization round-trip + var buffer = outerRecord.serializeToBuffer(); + byte[] serialized = new byte[buffer.remaining()]; + buffer.get(serialized); + ImprintRecord deserialized = ImprintRecord.deserialize(serialized); + + // Verify outer record + assert deserialized.getHeader().getSchemaId().getFieldspaceId() == 4; + assert deserialized.getValue(2).get().equals(Value.fromString("outer data")); + + // Verify nested record + Value rowValue = deserialized.getValue(1).get(); + assert rowValue instanceof Value.RowValue; + ImprintRecord nestedRecord = ((Value.RowValue) rowValue).getValue(); + + assert nestedRecord.getHeader().getSchemaId().getFieldspaceId() == 3; + assert nestedRecord.getValue(1).get().equals(Value.fromString("nested data")); + assert nestedRecord.getValue(2).get().equals(Value.fromInt64(9876543210L)); + + System.out.println("✓ Nested records test passed"); + } +} \ No newline at end of file diff --git a/src/test/java/com/imprint/benchmark/ProfilerTest.java b/src/test/java/com/imprint/benchmark/ProfilerTest.java new file mode 100644 index 0000000..233d993 --- /dev/null +++ b/src/test/java/com/imprint/benchmark/ProfilerTest.java @@ -0,0 +1,226 @@ +package com.imprint.benchmark; + +import com.imprint.core.*; +import com.imprint.types.Value; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Disabled; + +import java.util.Random; + +/** + * A test designed for profiling hotspots during development. + *

+ * To use with a profiler: + * 1. Remove @Disabled annotation + * 2. Run with JProfiler, VisualVM, or async-profiler: + * - JProfiler: Attach to test JVM + * - VisualVM: jvisualvm, attach to process + * - async-profiler: java -jar async-profiler.jar -d 30 -f profile.html + * 3. Look for hotspots in CPU sampling + *

+ * Key areas to examine: + * - Object allocation (memory profiling) + * - Method call frequency (CPU sampling) + * - GC pressure (memory profiling) + * - String operations and UTF-8 encoding + * - ByteBuffer operations + */ +@Disabled("Enable manually for profiling") +public class ProfilerTest { + + private static final int ITERATIONS = 1_000_000; + private static final int RECORD_SIZE = 20; + + @Test + void profileFieldAccess() throws Exception { + System.out.println("Starting profiler test - attach profiler now..."); + Thread.sleep(5000); // Give time to attach profiler + + // Create a representative record + var record = createTestRecord(); + + System.out.println("Beginning field access profiling..."); + long start = System.nanoTime(); + + // Simulate real-world access patterns + Random random = new Random(42); + int hits = 0; + + for (int i = 0; i < ITERATIONS; i++) { + // Random field access (hotspot) + int fieldId = random.nextInt(RECORD_SIZE) + 1; + var value = record.getValue(fieldId); + if (value.isPresent()) { + hits++; + + // Trigger string decoding (potential hotspot) + if (value.get().getTypeCode() == com.imprint.types.TypeCode.STRING) { + if (value.get() instanceof Value.StringBufferValue) { + ((Value.StringBufferValue) value.get()).getValue(); + } else { + ((Value.StringValue) value.get()).getValue(); + } + } + } + + // Some raw access (zero-copy path) + if (i % 10 == 0) { + record.getRawBytes(fieldId); + } + } + + long duration = System.nanoTime() - start; + System.out.printf("Completed %,d field accesses in %.2f ms (avg: %.1f ns/op, hits: %d)%n", + ITERATIONS, duration / 1_000_000.0, (double) duration / ITERATIONS, hits); + } + + @Test + void profileSerialization() throws Exception { + System.out.println("Starting serialization profiler test..."); + Thread.sleep(3000); + + var schemaId = new SchemaId(1, 0x12345678); + + System.out.println("Beginning serialization profiling..."); + long start = System.nanoTime(); + + // Create and serialize many records (allocation hotspot) + for (int i = 0; i < 100_000; i++) { + var writer = new ImprintWriter(schemaId); + + // Add various field types + writer.addField(1, Value.fromInt32(i)) + .addField(2, Value.fromString("test-string-" + i)) + .addField(3, Value.fromFloat64(i * 3.14159)) + .addField(4, Value.fromBytes(("bytes-" + i).getBytes())); + + var record = writer.build(); + var serialized = record.serializeToBuffer(); // Potential hotspot + + // Trigger some deserialization + if (i % 1000 == 0) { + var deserialized = ImprintRecord.deserialize(serialized); + deserialized.getValue(2); // String decoding hotspot + } + } + + long duration = System.nanoTime() - start; + System.out.printf("Completed serialization test in %.2f ms%n", duration / 1_000_000.0); + } + + @Test + void profileProjection() throws Exception { + System.out.println("Starting projection profiler test..."); + Thread.sleep(3000); + + var record = createLargeRecord(); + + System.out.println("Beginning projection profiling..."); + long start = System.nanoTime(); + + // Simulate analytical workload - project subset of fields repeatedly + for (int i = 0; i < 50_000; i++) { + // Project 10 fields out of 100 (common analytical pattern) + for (int fieldId = 1; fieldId <= 10; fieldId++) { + var value = record.getValue(fieldId); + if (value.isPresent()) { + // Force materialization of string values + if (value.get().getTypeCode() == com.imprint.types.TypeCode.STRING) { + if (value.get() instanceof Value.StringBufferValue) { + ((Value.StringBufferValue) value.get()).getValue(); + } + } + } + } + } + + long duration = System.nanoTime() - start; + System.out.printf("Completed projection test in %.2f ms%n", duration / 1_000_000.0); + } + + @Test + void profileMemoryAllocation() throws Exception { + System.out.println("Starting allocation profiler test..."); + Thread.sleep(3000); + + System.out.println("Beginning allocation profiling - watch for GC events..."); + + // Force allocation pressure to reveal GC hotspots + for (int batch = 0; batch < 1000; batch++) { + for (int i = 0; i < 1000; i++) { + var schemaId = new SchemaId(batch, i); + var writer = new ImprintWriter(schemaId); + + // Create strings of varying sizes (allocation pressure) + writer.addField(1, Value.fromString("small")) + .addField(2, Value.fromString("medium-length-string-" + i)) + .addField(3, Value.fromString("very-long-string-that-will-cause-more-allocation-pressure-" + batch + "-" + i)) + .addField(4, Value.fromBytes(new byte[100 + i % 1000])); // Varying byte arrays + + var record = writer.build(); + + // Some deserialization to trigger string decoding allocations + record.getValue(2); + record.getValue(3); + } + + if (batch % 100 == 0) { + System.out.printf("Completed batch %d/1000%n", batch); + } + } + + System.out.println("Allocation test complete - check GC logs and memory profiler"); + } + + private ImprintRecord createTestRecord() throws Exception { + var schemaId = new SchemaId(1, 0xdeadbeef); + var writer = new ImprintWriter(schemaId); + + for (int i = 1; i <= RECORD_SIZE; i++) { + switch (i % 4) { + case 0: + writer.addField(i, Value.fromInt32(i * 100)); + break; + case 1: + writer.addField(i, Value.fromString("field-value-" + i)); + break; + case 2: + writer.addField(i, Value.fromFloat64(i * 3.14159)); + break; + case 3: + writer.addField(i, Value.fromBytes(("bytes-" + i).getBytes())); + break; + } + } + + return writer.build(); + } + + private ImprintRecord createLargeRecord() throws Exception { + var schemaId = new SchemaId(2, 0xcafebabe); + var writer = new ImprintWriter(schemaId); + + // Create 100 fields with realistic data + for (int i = 1; i <= 100; i++) { + switch (i % 5) { + case 0: + writer.addField(i, Value.fromInt32(i)); + break; + case 1: + writer.addField(i, Value.fromString("user-name-" + i + "@example.com")); + break; + case 2: + writer.addField(i, Value.fromString("Some longer descriptive text for field " + i + " that might represent a comment or description")); + break; + case 3: + writer.addField(i, Value.fromFloat64(i * 2.718281828)); + break; + case 4: + writer.addField(i, Value.fromBytes(String.format("binary-data-%04d", i).getBytes())); + break; + } + } + + return writer.build(); + } +} \ No newline at end of file diff --git a/src/test/java/com/imprint/core/ImprintRecordBuilderTest.java b/src/test/java/com/imprint/core/ImprintRecordBuilderTest.java new file mode 100644 index 0000000..54dcfae --- /dev/null +++ b/src/test/java/com/imprint/core/ImprintRecordBuilderTest.java @@ -0,0 +1,234 @@ +package com.imprint.core; + +import com.imprint.error.ImprintException; +import com.imprint.types.Value; +import org.junit.jupiter.api.Test; + +import java.util.*; + +import static org.assertj.core.api.Assertions.*; + +class ImprintRecordBuilderTest { + + private static final SchemaId TEST_SCHEMA = new SchemaId(1, 0x12345678); + + @Test + void shouldCreateRecordWithPrimitiveTypes() throws ImprintException { + var record = ImprintRecord.builder(TEST_SCHEMA) + .field(1, true) + .field(2, 42) + .field(3, 123L) + .field(4, 3.14f) + .field(5, 2.718) + .field(6, "hello world") + .field(7, new byte[]{1, 2, 3}) + .nullField(8) + .build(); + + assertThat(record.getHeader().getSchemaId()).isEqualTo(TEST_SCHEMA); + assertThat(record.getDirectory()).hasSize(8); + + // Verify field values + assertThat(getFieldValue(record, 1, Value.BoolValue.class).getValue()).isTrue(); + assertThat(getFieldValue(record, 2, Value.Int32Value.class).getValue()).isEqualTo(42); + assertThat(getFieldValue(record, 3, Value.Int64Value.class).getValue()).isEqualTo(123L); + assertThat(getFieldValue(record, 4, Value.Float32Value.class).getValue()).isEqualTo(3.14f); + assertThat(getFieldValue(record, 5, Value.Float64Value.class).getValue()).isEqualTo(2.718); + assertThat(getStringValue(record, 6)).isEqualTo("hello world"); + assertThat(getBytesValue(record, 7)).isEqualTo(new byte[]{1, 2, 3}); + assertThat(record.getValue(8).get()).isInstanceOf(Value.NullValue.class); + } + + @Test + void shouldCreateRecordWithCollections() throws ImprintException { + var list = List.of(1, 2, 3); + var map = Map.of("key1", 100, "key2", 200); + + var record = ImprintRecord.builder(TEST_SCHEMA) + .field(1, list) + .field(2, map) + .build(); + + // Verify array + var arrayValue = getFieldValue(record, 1, Value.ArrayValue.class); + assertThat(arrayValue.getValue()).hasSize(3); + assertThat(((Value.Int32Value) arrayValue.getValue().get(0)).getValue()).isEqualTo(1); + assertThat(((Value.Int32Value) arrayValue.getValue().get(1)).getValue()).isEqualTo(2); + assertThat(((Value.Int32Value) arrayValue.getValue().get(2)).getValue()).isEqualTo(3); + + // Verify map + var mapValue = getFieldValue(record, 2, Value.MapValue.class); + assertThat(mapValue.getValue()).hasSize(2); + } + + @Test + void shouldCreateRecordWithNestedRecord() throws ImprintException { + var nestedRecord = ImprintRecord.builder(new SchemaId(2, 0x87654321)) + .field(1, "nested") + .field(2, 999) + .build(); + + var record = ImprintRecord.builder(TEST_SCHEMA) + .field(1, "parent") + .field(2, nestedRecord) + .build(); + + var rowValue = getFieldValue(record, 2, Value.RowValue.class); + var nested = rowValue.getValue(); + assertThat(getStringValue(nested, 1)).isEqualTo("nested"); + assertThat(getFieldValue(nested, 2, Value.Int32Value.class).getValue()).isEqualTo(999); + } + + @Test + void shouldSupportConditionalFields() throws ImprintException { + boolean includeOptional = true; + String optionalValue = "optional"; + + var record = ImprintRecord.builder(TEST_SCHEMA) + .field(1, "required") + .fieldIf(includeOptional, 2, optionalValue) + .fieldIfNotNull(3, null) // Should not add field + .fieldIfNotNull(4, "not null") // Should add field + .build(); + + assertThat(record.getDirectory()).hasSize(3); // Only fields 1, 2, 4 + assertThat(getStringValue(record, 1)).isEqualTo("required"); + assertThat(getStringValue(record, 2)).isEqualTo("optional"); + assertThat(record.getValue(3)).isEmpty(); // Not added + assertThat(getStringValue(record, 4)).isEqualTo("not null"); + } + + @Test + void shouldSupportBulkOperations() throws ImprintException { + var fieldsMap = Map.of( + 1, "bulk1", + 2, 42, + 3, true + ); + + var record = ImprintRecord.builder(TEST_SCHEMA) + .fields(fieldsMap) + .field(4, "additional") + .build(); + + assertThat(record.getDirectory()).hasSize(4); + assertThat(getStringValue(record, 1)).isEqualTo("bulk1"); + assertThat(getFieldValue(record, 2, Value.Int32Value.class).getValue()).isEqualTo(42); + assertThat(getFieldValue(record, 3, Value.BoolValue.class).getValue()).isTrue(); + assertThat(getStringValue(record, 4)).isEqualTo("additional"); + } + + @Test + void shouldProvideBuilderUtilities() { + var builder = ImprintRecord.builder(TEST_SCHEMA) + .field(1, "test") + .field(2, 42); + + assertThat(builder.hasField(1)).isTrue(); + assertThat(builder.hasField(3)).isFalse(); + assertThat(builder.fieldCount()).isEqualTo(2); + assertThat(builder.fieldIds()).containsExactly(1, 2); + } + + @Test + void shouldSupportAlternativeSchemaConstructor() throws ImprintException { + var record = ImprintRecord.builder(1, 0x12345678) + .field(1, "test") + .build(); + + assertThat(record.getHeader().getSchemaId().getFieldspaceId()).isEqualTo(1); + assertThat(record.getHeader().getSchemaId().getSchemaHash()).isEqualTo(0x12345678); + } + + @Test + void shouldRoundTripThroughSerialization() throws ImprintException { + var original = ImprintRecord.builder(TEST_SCHEMA) + .field(1, "test string") + .field(2, 42) + .field(3, 3.14159) + .field(4, true) + .field(5, new byte[]{0x01, 0x02, 0x03}) + .build(); + + var serialized = original.serializeToBuffer(); + var deserialized = ImprintRecord.deserialize(serialized); + + assertThat(deserialized.getHeader().getSchemaId()).isEqualTo(TEST_SCHEMA); + assertThat(getStringValue(deserialized, 1)).isEqualTo("test string"); + assertThat(getFieldValue(deserialized, 2, Value.Int32Value.class).getValue()).isEqualTo(42); + assertThat(getFieldValue(deserialized, 3, Value.Float64Value.class).getValue()).isEqualTo(3.14159); + assertThat(getFieldValue(deserialized, 4, Value.BoolValue.class).getValue()).isTrue(); + assertThat(getBytesValue(deserialized, 5)).isEqualTo(new byte[]{0x01, 0x02, 0x03}); + } + + // Error cases + + @Test + void shouldRejectDuplicateFieldIds() { + assertThatThrownBy(() -> + ImprintRecord.builder(TEST_SCHEMA) + .field(1, "first") + .field(1, "duplicate") // Same field ID + ).isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Field ID 1 already exists"); + } + + @Test + void shouldRejectEmptyRecord() { + assertThatThrownBy(() -> + ImprintRecord.builder(TEST_SCHEMA).build() + ).isInstanceOf(ImprintException.class) + .hasMessageContaining("Cannot build empty record"); + } + + @Test + void shouldRejectInvalidMapKeys() { + var mapWithInvalidKey = Map.of(3.14, "value"); // Double key not supported + + assertThatThrownBy(() -> + ImprintRecord.builder(TEST_SCHEMA) + .field(1, mapWithInvalidKey) + ).isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Invalid map key type: Double"); + } + + @Test + void shouldRejectNullValueWithoutExplicitNullField() { + assertThatThrownBy(() -> + ImprintRecord.builder(TEST_SCHEMA) + .field(1, (Value) null) + ).isInstanceOf(NullPointerException.class) + .hasMessageContaining("Value cannot be null - use nullField()"); + } + + // Helper methods for cleaner test assertions + + private T getFieldValue(ImprintRecord record, int fieldId, Class valueType) throws ImprintException { + var value = record.getValue(fieldId); + assertThat(value).isPresent(); + assertThat(value.get()).isInstanceOf(valueType); + return valueType.cast(value.get()); + } + + private String getStringValue(ImprintRecord record, int fieldId) throws ImprintException { + var value = record.getValue(fieldId).get(); + if (value instanceof Value.StringValue) { + return ((Value.StringValue) value).getValue(); + } else if (value instanceof Value.StringBufferValue) { + return ((Value.StringBufferValue) value).getValue(); + } else { + throw new AssertionError("Expected string value, got: " + value.getClass()); + } + } + + private byte[] getBytesValue(ImprintRecord record, int fieldId) throws ImprintException { + var value = record.getValue(fieldId).get(); + if (value instanceof Value.BytesValue) { + return ((Value.BytesValue) value).getValue(); + } else if (value instanceof Value.BytesBufferValue) { + return ((Value.BytesBufferValue) value).getValue(); + } else { + throw new AssertionError("Expected bytes value, got: " + value.getClass()); + } + } +} \ No newline at end of file diff --git a/src/test/java/com/imprint/core/ImprintRecordTest.java b/src/test/java/com/imprint/core/ImprintRecordTest.java new file mode 100644 index 0000000..0772580 --- /dev/null +++ b/src/test/java/com/imprint/core/ImprintRecordTest.java @@ -0,0 +1,232 @@ +package com.imprint.core; + +import com.imprint.error.ImprintException; +import com.imprint.error.ErrorType; +import com.imprint.types.Value; +import com.imprint.types.MapKey; +import org.junit.jupiter.api.Test; +import java.util.*; +import static org.assertj.core.api.Assertions.*; + +class ImprintRecordTest { + + // Helper method to extract string value from either StringValue or StringBufferValue + private String getStringValue(Value value) { + if (value instanceof Value.StringValue) { + return ((Value.StringValue) value).getValue(); + } else if (value instanceof Value.StringBufferValue) { + return ((Value.StringBufferValue) value).getValue(); + } else { + throw new IllegalArgumentException("Expected string value, got: " + value.getClass()); + } + } + + @Test + void shouldCreateSimpleRecord() throws ImprintException { + var schemaId = new SchemaId(1, 0xdeadbeef); + var writer = new ImprintWriter(schemaId); + + writer.addField(1, Value.fromInt32(42)) + .addField(2, Value.fromString("hello")); + + var record = writer.build(); + + assertThat(record.getHeader().getSchemaId()).isEqualTo(schemaId); + assertThat(record.getDirectory()).hasSize(2); + + Optional field1 = record.getValue(1); + Optional field2 = record.getValue(2); + + assertThat(field1).isPresent(); + assertThat(field1.get()).isInstanceOf(Value.Int32Value.class); + assertThat(((Value.Int32Value) field1.get()).getValue()).isEqualTo(42); + + assertThat(field2).isPresent(); + assertThat(field2.get().getTypeCode()).isEqualTo(com.imprint.types.TypeCode.STRING); + String stringValue = getStringValue(field2.get()); + assertThat(stringValue).isEqualTo("hello"); + + // Non-existent field should return empty + assertThat(record.getValue(999)).isEmpty(); + } + + @Test + void shouldRoundtripThroughSerialization() throws ImprintException { + var schemaId = new SchemaId(1, 0xdeadbeef); + var writer = new ImprintWriter(schemaId); + + writer.addField(1, Value.nullValue()) + .addField(2, Value.fromBoolean(true)) + .addField(3, Value.fromInt32(42)) + .addField(4, Value.fromInt64(123456789L)) + .addField(5, Value.fromFloat32(3.14f)) + .addField(6, Value.fromFloat64(2.718281828)) + .addField(7, Value.fromBytes(new byte[]{1, 2, 3, 4})) + .addField(8, Value.fromString("test string")); + + var original = writer.build(); + + // Serialize and deserialize + var buffer = original.serializeToBuffer(); + byte[] serialized = new byte[buffer.remaining()]; + buffer.get(serialized); + var deserialized = ImprintRecord.deserialize(serialized); + + // Verify metadata + assertThat(deserialized.getHeader().getSchemaId().getFieldspaceId()).isEqualTo(1); + assertThat(deserialized.getHeader().getSchemaId().getSchemaHash()).isEqualTo(0xdeadbeef); + assertThat(deserialized.getDirectory()).hasSize(8); + + // Verify all values + assertThat(deserialized.getValue(1)).contains(Value.nullValue()); + assertThat(deserialized.getValue(2)).contains(Value.fromBoolean(true)); + assertThat(deserialized.getValue(3)).contains(Value.fromInt32(42)); + assertThat(deserialized.getValue(4)).contains(Value.fromInt64(123456789L)); + assertThat(deserialized.getValue(5)).contains(Value.fromFloat32(3.14f)); + assertThat(deserialized.getValue(6)).contains(Value.fromFloat64(2.718281828)); + assertThat(deserialized.getValue(7)).contains(Value.fromBytes(new byte[]{1, 2, 3, 4})); + assertThat(deserialized.getValue(8)).contains(Value.fromString("test string")); + + // Non-existent field + assertThat(deserialized.getValue(999)).isEmpty(); + } + + @Test + void shouldHandleArrays() throws ImprintException { + var schemaId = new SchemaId(1, 0xdeadbeef); + var writer = new ImprintWriter(schemaId); + + List intArray = Arrays.asList( + Value.fromInt32(1), + Value.fromInt32(2), + Value.fromInt32(3) + ); + + writer.addField(1, Value.fromArray(intArray)); + ImprintRecord record = writer.build(); + + // Serialize and deserialize + var buffer = record.serializeToBuffer(); + byte[] serialized = new byte[buffer.remaining()]; + buffer.get(serialized); + var deserialized = ImprintRecord.deserialize(serialized); + + Optional arrayValue = deserialized.getValue(1); + assertThat(arrayValue).isPresent(); + assertThat(arrayValue.get()).isInstanceOf(Value.ArrayValue.class); + + List deserializedArray = ((Value.ArrayValue) arrayValue.get()).getValue(); + assertThat(deserializedArray).hasSize(3); + assertThat(deserializedArray.get(0)).isEqualTo(Value.fromInt32(1)); + assertThat(deserializedArray.get(1)).isEqualTo(Value.fromInt32(2)); + assertThat(deserializedArray.get(2)).isEqualTo(Value.fromInt32(3)); + } + + @Test + void shouldHandleMaps() throws ImprintException { + var schemaId = new SchemaId(1, 0xdeadbeef); + var writer = new ImprintWriter(schemaId); + + var map = new HashMap(); + map.put(MapKey.fromString("key1"), Value.fromInt32(1)); + map.put(MapKey.fromString("key2"), Value.fromInt32(2)); + + writer.addField(1, Value.fromMap(map)); + var record = writer.build(); + + // Serialize and deserialize + var buffer = record.serializeToBuffer(); + byte[] serialized = new byte[buffer.remaining()]; + buffer.get(serialized); + var deserialized = ImprintRecord.deserialize(serialized); + + Optional mapValue = deserialized.getValue(1); + assertThat(mapValue).isPresent(); + assertThat(mapValue.get()).isInstanceOf(Value.MapValue.class); + + Map deserializedMap = ((Value.MapValue) mapValue.get()).getValue(); + assertThat(deserializedMap).hasSize(2); + assertThat(deserializedMap.get(MapKey.fromString("key1"))).isEqualTo(Value.fromInt32(1)); + assertThat(deserializedMap.get(MapKey.fromString("key2"))).isEqualTo(Value.fromInt32(2)); + } + + @Test + void shouldHandleNestedRecords() throws ImprintException { + // Create inner record + var innerSchemaId = new SchemaId(2, 0xcafebabe); + var innerWriter = new ImprintWriter(innerSchemaId); + innerWriter.addField(1, Value.fromInt32(42)) + .addField(2, Value.fromString("nested")); + var innerRecord = innerWriter.build(); + + // Create outer record containing inner record + var outerSchemaId = new SchemaId(1, 0xdeadbeef); + var outerWriter = new ImprintWriter(outerSchemaId); + outerWriter.addField(1, Value.fromRow(innerRecord)) + .addField(2, Value.fromInt64(123L)); + var outerRecord = outerWriter.build(); + + // Serialize and deserialize + var buffer = outerRecord.serializeToBuffer(); + byte[] serialized = new byte[buffer.remaining()]; + buffer.get(serialized); + var deserialized = ImprintRecord.deserialize(serialized); + + // Verify outer record metadata + assertThat(deserialized.getHeader().getSchemaId().getFieldspaceId()).isEqualTo(1); + assertThat(deserialized.getHeader().getSchemaId().getSchemaHash()).isEqualTo(0xdeadbeef); + + // Verify nested record + Optional rowValue = deserialized.getValue(1); + assertThat(rowValue).isPresent(); + assertThat(rowValue.get()).isInstanceOf(Value.RowValue.class); + + var nestedRecord = ((Value.RowValue) rowValue.get()).getValue(); + assertThat(nestedRecord.getHeader().getSchemaId().getFieldspaceId()).isEqualTo(2); + assertThat(nestedRecord.getHeader().getSchemaId().getSchemaHash()).isEqualTo(0xcafebabe); + + assertThat(nestedRecord.getValue(1)).contains(Value.fromInt32(42)); + assertThat(nestedRecord.getValue(2)).contains(Value.fromString("nested")); + + // Verify outer record field + assertThat(deserialized.getValue(2)).contains(Value.fromInt64(123L)); + } + + @Test + void shouldRejectInvalidMagic() { + byte[] invalidData = new byte[15]; + invalidData[0] = 0x00; // wrong magic + + assertThatThrownBy(() -> ImprintRecord.deserialize(invalidData)) + .isInstanceOf(ImprintException.class) + .extracting("errorType") + .isEqualTo(ErrorType.INVALID_MAGIC); + } + + @Test + void shouldRejectUnsupportedVersion() { + byte[] invalidData = new byte[15]; + invalidData[0] = (byte) 0x49; // correct magic + invalidData[1] = (byte) 0xFF; // wrong version + + assertThatThrownBy(() -> ImprintRecord.deserialize(invalidData)) + .isInstanceOf(ImprintException.class) + .extracting("errorType") + .isEqualTo(ErrorType.UNSUPPORTED_VERSION); + } + + @Test + void shouldHandleDuplicateFieldIds() throws ImprintException { + var schemaId = new SchemaId(1, 0xdeadbeef); + var writer = new ImprintWriter(schemaId); + + // Add duplicate field IDs - last one should win + writer.addField(1, Value.fromInt32(42)) + .addField(1, Value.fromInt32(43)); + + var record = writer.build(); + + assertThat(record.getDirectory()).hasSize(1); + assertThat(record.getValue(1)).contains(Value.fromInt32(43)); + } +} \ No newline at end of file diff --git a/src/test/java/com/imprint/types/MapKeyTest.java b/src/test/java/com/imprint/types/MapKeyTest.java new file mode 100644 index 0000000..08f4180 --- /dev/null +++ b/src/test/java/com/imprint/types/MapKeyTest.java @@ -0,0 +1,91 @@ +package com.imprint.types; + +import com.imprint.error.ImprintException; +import com.imprint.error.ErrorType; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.*; + +class MapKeyTest { + + @Test + void shouldCreateMapKeysFromValues() throws ImprintException { + var int32Key = MapKey.fromValue(Value.fromInt32(42)); + var int64Key = MapKey.fromValue(Value.fromInt64(123L)); + var bytesKey = MapKey.fromValue(Value.fromBytes(new byte[]{1, 2, 3})); + var stringKey = MapKey.fromValue(Value.fromString("test")); + + assertThat(int32Key).isInstanceOf(MapKey.Int32Key.class); + assertThat(((MapKey.Int32Key) int32Key).getValue()).isEqualTo(42); + + assertThat(int64Key).isInstanceOf(MapKey.Int64Key.class); + assertThat(((MapKey.Int64Key) int64Key).getValue()).isEqualTo(123L); + + assertThat(bytesKey).isInstanceOf(MapKey.BytesKey.class); + assertThat(((MapKey.BytesKey) bytesKey).getValue()).containsExactly(1, 2, 3); + + assertThat(stringKey).isInstanceOf(MapKey.StringKey.class); + assertThat(((MapKey.StringKey) stringKey).getValue()).isEqualTo("test"); + } + + @Test + void shouldConvertBackToValues() { + var int32Key = MapKey.fromInt32(42); + var stringKey = MapKey.fromString("test"); + + var int32Value = int32Key.toValue(); + var stringValue = stringKey.toValue(); + + assertThat(int32Value).isInstanceOf(Value.Int32Value.class); + assertThat(((Value.Int32Value) int32Value).getValue()).isEqualTo(42); + + assertThat(stringValue).isInstanceOf(Value.StringValue.class); + assertThat(((Value.StringValue) stringValue).getValue()).isEqualTo("test"); + } + + @Test + void shouldRejectInvalidValueTypes() { + var boolValue = Value.fromBoolean(true); + var arrayValue = Value.fromArray(java.util.Collections.emptyList()); + + assertThatThrownBy(() -> MapKey.fromValue(boolValue)) + .isInstanceOf(ImprintException.class) + .extracting("errorType") + .isEqualTo(ErrorType.TYPE_MISMATCH); + + assertThatThrownBy(() -> MapKey.fromValue(arrayValue)) + .isInstanceOf(ImprintException.class) + .extracting("errorType") + .isEqualTo(ErrorType.TYPE_MISMATCH); + } + + @Test + void shouldHandleEqualityAndHashing() { + var key1 = MapKey.fromString("test"); + var key2 = MapKey.fromString("test"); + var key3 = MapKey.fromString("different"); + + assertThat(key1).isEqualTo(key2); + assertThat(key1).isNotEqualTo(key3); + assertThat(key1.hashCode()).isEqualTo(key2.hashCode()); + } + + @Test + void shouldDefensiveCopyBytes() { + byte[] original = {1, 2, 3}; + var bytesKey = MapKey.fromBytes(original); + + // Modify original array + original[0] = 99; + + // Key should be unchanged + assertThat(((MapKey.BytesKey) bytesKey).getValue()).containsExactly(1, 2, 3); + } + + @Test + void shouldHaveCorrectTypeCodes() { + assertThat(MapKey.fromInt32(1).getTypeCode()).isEqualTo(TypeCode.INT32); + assertThat(MapKey.fromInt64(1L).getTypeCode()).isEqualTo(TypeCode.INT64); + assertThat(MapKey.fromBytes(new byte[]{1}).getTypeCode()).isEqualTo(TypeCode.BYTES); + assertThat(MapKey.fromString("test").getTypeCode()).isEqualTo(TypeCode.STRING); + } +} \ No newline at end of file diff --git a/src/test/java/com/imprint/types/TypeHandlerTest.java b/src/test/java/com/imprint/types/TypeHandlerTest.java new file mode 100644 index 0000000..9a4ae85 --- /dev/null +++ b/src/test/java/com/imprint/types/TypeHandlerTest.java @@ -0,0 +1,279 @@ +package com.imprint.types; + +import com.imprint.error.ImprintException; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +import static org.assertj.core.api.Assertions.*; + +/** + * Tests for individual TypeHandler implementations. + * Validates serialization, deserialization, and size estimation for each type. + */ +class TypeHandlerTest { + + @Test + void testNullHandler() throws ImprintException { + var handler = TypeHandler.NULL; + var value = Value.nullValue(); + + // Size estimation + assertThat(handler.estimateSize(value)).isEqualTo(0); + + // Serialization + var buffer = ByteBuffer.allocate(10); + handler.serialize(value, buffer); + assertThat(buffer.position()).isEqualTo(0); // NULL writes nothing + + // Deserialization + buffer.flip(); + var deserialized = handler.deserialize(buffer); + assertThat(deserialized).isEqualTo(value); + + // readValueBytes + buffer.clear(); + var valueBytes = handler.readValueBytes(buffer); + assertThat(valueBytes.remaining()).isEqualTo(0); + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void testBoolHandler(boolean testValue) throws ImprintException { + var handler = TypeHandler.BOOL; + var value = Value.fromBoolean(testValue); + + // Size estimation + assertThat(handler.estimateSize(value)).isEqualTo(1); + + // Round-trip test + var buffer = ByteBuffer.allocate(10); + handler.serialize(value, buffer); + assertThat(buffer.position()).isEqualTo(1); + + buffer.flip(); + var deserialized = handler.deserialize(buffer); + assertThat(deserialized).isEqualTo(value); + assertThat(((Value.BoolValue) deserialized).getValue()).isEqualTo(testValue); + } + + @ParameterizedTest + @ValueSource(ints = {0, 1, -1, Integer.MAX_VALUE, Integer.MIN_VALUE, 42, -42}) + void testInt32Handler(int testValue) throws ImprintException { + var handler = TypeHandler.INT32; + var value = Value.fromInt32(testValue); + + // Size estimation + assertThat(handler.estimateSize(value)).isEqualTo(4); + + // Round-trip test + var buffer = ByteBuffer.allocate(10).order(ByteOrder.LITTLE_ENDIAN); + handler.serialize(value, buffer); + assertThat(buffer.position()).isEqualTo(4); + + buffer.flip(); + var deserialized = handler.deserialize(buffer); + assertThat(deserialized).isEqualTo(value); + assertThat(((Value.Int32Value) deserialized).getValue()).isEqualTo(testValue); + } + + @ParameterizedTest + @ValueSource(longs = {0L, 1L, -1L, Long.MAX_VALUE, Long.MIN_VALUE, 123456789L}) + void testInt64Handler(long testValue) throws ImprintException { + var handler = TypeHandler.INT64; + var value = Value.fromInt64(testValue); + + // Size estimation + assertThat(handler.estimateSize(value)).isEqualTo(8); + + // Round-trip test + var buffer = ByteBuffer.allocate(20).order(ByteOrder.LITTLE_ENDIAN); + handler.serialize(value, buffer); + assertThat(buffer.position()).isEqualTo(8); + + buffer.flip(); + var deserialized = handler.deserialize(buffer); + assertThat(deserialized).isEqualTo(value); + assertThat(((Value.Int64Value) deserialized).getValue()).isEqualTo(testValue); + } + + @ParameterizedTest + @ValueSource(floats = {0.0f, 1.0f, -1.0f, Float.MAX_VALUE, Float.MIN_VALUE, 3.14159f, Float.NaN, Float.POSITIVE_INFINITY}) + void testFloat32Handler(float testValue) throws ImprintException { + var handler = TypeHandler.FLOAT32; + var value = Value.fromFloat32(testValue); + + // Size estimation + assertThat(handler.estimateSize(value)).isEqualTo(4); + + // Round-trip test + var buffer = ByteBuffer.allocate(10).order(ByteOrder.LITTLE_ENDIAN); + handler.serialize(value, buffer); + assertThat(buffer.position()).isEqualTo(4); + + buffer.flip(); + var deserialized = handler.deserialize(buffer); + assertThat(deserialized).isEqualTo(value); + + float deserializedValue = ((Value.Float32Value) deserialized).getValue(); + if (Float.isNaN(testValue)) { + assertThat(deserializedValue).isNaN(); + } else { + assertThat(deserializedValue).isEqualTo(testValue); + } + } + + @ParameterizedTest + @ValueSource(doubles = {0.0, 1.0, -1.0, Double.MAX_VALUE, Double.MIN_VALUE, Math.PI, Double.NaN, Double.POSITIVE_INFINITY}) + void testFloat64Handler(double testValue) throws ImprintException { + var handler = TypeHandler.FLOAT64; + var value = Value.fromFloat64(testValue); + + // Size estimation + assertThat(handler.estimateSize(value)).isEqualTo(8); + + // Round-trip test + var buffer = ByteBuffer.allocate(20).order(ByteOrder.LITTLE_ENDIAN); + handler.serialize(value, buffer); + assertThat(buffer.position()).isEqualTo(8); + + buffer.flip(); + var deserialized = handler.deserialize(buffer); + assertThat(deserialized).isEqualTo(value); + + double deserializedValue = ((Value.Float64Value) deserialized).getValue(); + if (Double.isNaN(testValue)) { + assertThat(deserializedValue).isNaN(); + } else { + assertThat(deserializedValue).isEqualTo(testValue); + } + } + + @ParameterizedTest + @ValueSource(strings = {"", "hello", "世界", "a very long string that exceeds typical buffer sizes and contains unicode: 🚀🎉", "null\0bytes"}) + void testStringHandler(String testValue) throws ImprintException { + var handler = TypeHandler.STRING; + var value = Value.fromString(testValue); + + byte[] utf8Bytes = testValue.getBytes(java.nio.charset.StandardCharsets.UTF_8); + int expectedSize = com.imprint.util.VarInt.encodedLength(utf8Bytes.length) + utf8Bytes.length; + + // Size estimation + assertThat(handler.estimateSize(value)).isEqualTo(expectedSize); + + // Round-trip test + var buffer = ByteBuffer.allocate(expectedSize + 20).order(ByteOrder.LITTLE_ENDIAN); + handler.serialize(value, buffer); + + buffer.flip(); + var deserialized = handler.deserialize(buffer); + + // Should return StringBufferValue (zero-copy implementation) + assertThat(deserialized).isInstanceOf(Value.StringBufferValue.class); + + String deserializedString; + if (deserialized instanceof Value.StringBufferValue) { + deserializedString = ((Value.StringBufferValue) deserialized).getValue(); + } else { + deserializedString = ((Value.StringValue) deserialized).getValue(); + } + + assertThat(deserializedString).isEqualTo(testValue); + } + + @Test + void testBytesHandlerWithArrayValue() throws ImprintException { + var handler = TypeHandler.BYTES; + byte[] testBytes = {0, 1, 2, (byte) 0xFF, 42, 127, -128}; + var value = Value.fromBytes(testBytes); + + int expectedSize = com.imprint.util.VarInt.encodedLength(testBytes.length) + testBytes.length; + + // Size estimation + assertThat(handler.estimateSize(value)).isEqualTo(expectedSize); + + // Round-trip test + var buffer = ByteBuffer.allocate(expectedSize + 20).order(ByteOrder.LITTLE_ENDIAN); + handler.serialize(value, buffer); + + buffer.flip(); + var deserialized = handler.deserialize(buffer); + + // Should return BytesBufferValue (zero-copy implementation) + assertThat(deserialized).isInstanceOf(Value.BytesBufferValue.class); + + byte[] deserializedBytes = ((Value.BytesBufferValue) deserialized).getValue(); + assertThat(deserializedBytes).isEqualTo(testBytes); + } + + @Test + void testBytesHandlerWithBufferValue() throws ImprintException { + var handler = TypeHandler.BYTES; + byte[] testBytes = {10, 20, 30, 40}; + var bufferValue = Value.fromBytesBuffer(ByteBuffer.wrap(testBytes).asReadOnlyBuffer()); + + int expectedSize = com.imprint.util.VarInt.encodedLength(testBytes.length) + testBytes.length; + + // Size estimation + assertThat(handler.estimateSize(bufferValue)).isEqualTo(expectedSize); + + // Round-trip test + var buffer = ByteBuffer.allocate(expectedSize + 20).order(ByteOrder.LITTLE_ENDIAN); + handler.serialize(bufferValue, buffer); + + buffer.flip(); + var deserialized = handler.deserialize(buffer); + + byte[] deserializedBytes = ((Value.BytesBufferValue) deserialized).getValue(); + assertThat(deserializedBytes).isEqualTo(testBytes); + } + + @Test + void testStringHandlerWithBufferValue() throws ImprintException { + var handler = TypeHandler.STRING; + String testString = "zero-copy string test"; + byte[] utf8Bytes = testString.getBytes(java.nio.charset.StandardCharsets.UTF_8); + var bufferValue = Value.fromStringBuffer(ByteBuffer.wrap(utf8Bytes).asReadOnlyBuffer()); + + int expectedSize = com.imprint.util.VarInt.encodedLength(utf8Bytes.length) + utf8Bytes.length; + + // Size estimation + assertThat(handler.estimateSize(bufferValue)).isEqualTo(expectedSize); + + // Round-trip test + var buffer = ByteBuffer.allocate(expectedSize + 20).order(ByteOrder.LITTLE_ENDIAN); + handler.serialize(bufferValue, buffer); + + buffer.flip(); + var deserialized = handler.deserialize(buffer); + + String deserializedString = ((Value.StringBufferValue) deserialized).getValue(); + assertThat(deserializedString).isEqualTo(testString); + } + + @Test + void testBoolHandlerInvalidValue() { + var handler = TypeHandler.BOOL; + var buffer = ByteBuffer.allocate(10); + buffer.put((byte) 2); // Invalid boolean value + buffer.flip(); + + assertThatThrownBy(() -> handler.deserialize(buffer)) + .isInstanceOf(ImprintException.class) + .hasMessageContaining("Invalid boolean value: 2"); + } + + @Test + void testHandlerBufferUnderflow() { + // Test that handlers properly detect buffer underflow + var int32Handler = TypeHandler.INT32; + var buffer = ByteBuffer.allocate(2); // Too small for int32 + + assertThatThrownBy(() -> int32Handler.deserialize(buffer)) + .isInstanceOf(ImprintException.class) + .hasMessageContaining("Not enough bytes for int32"); + } +} \ No newline at end of file diff --git a/src/test/java/com/imprint/types/ValueTest.java b/src/test/java/com/imprint/types/ValueTest.java new file mode 100644 index 0000000..9dd99c9 --- /dev/null +++ b/src/test/java/com/imprint/types/ValueTest.java @@ -0,0 +1,123 @@ +package com.imprint.types; + +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class ValueTest { + + @Test + void shouldCreateNullValue() { + Value value = Value.nullValue(); + + assertThat(value).isInstanceOf(Value.NullValue.class); + assertThat(value.getTypeCode()).isEqualTo(TypeCode.NULL); + assertThat(value.toString()).isEqualTo("null"); + } + + @Test + void shouldCreateBooleanValues() { + Value trueValue = Value.fromBoolean(true); + Value falseValue = Value.fromBoolean(false); + + assertThat(trueValue).isInstanceOf(Value.BoolValue.class); + assertThat(((Value.BoolValue) trueValue).getValue()).isTrue(); + assertThat(trueValue.getTypeCode()).isEqualTo(TypeCode.BOOL); + + assertThat(falseValue).isInstanceOf(Value.BoolValue.class); + assertThat(((Value.BoolValue) falseValue).getValue()).isFalse(); + assertThat(falseValue.getTypeCode()).isEqualTo(TypeCode.BOOL); + } + + @Test + void shouldCreateNumericValues() { + var int32 = Value.fromInt32(42); + var int64 = Value.fromInt64(123456789L); + var float32 = Value.fromFloat32(3.14f); + var float64 = Value.fromFloat64(2.718281828); + + assertThat(int32.getTypeCode()).isEqualTo(TypeCode.INT32); + assertThat(((Value.Int32Value) int32).getValue()).isEqualTo(42); + + assertThat(int64.getTypeCode()).isEqualTo(TypeCode.INT64); + assertThat(((Value.Int64Value) int64).getValue()).isEqualTo(123456789L); + + assertThat(float32.getTypeCode()).isEqualTo(TypeCode.FLOAT32); + assertThat(((Value.Float32Value) float32).getValue()).isEqualTo(3.14f); + + assertThat(float64.getTypeCode()).isEqualTo(TypeCode.FLOAT64); + assertThat(((Value.Float64Value) float64).getValue()).isEqualTo(2.718281828); + } + + @Test + void shouldCreateBytesAndStringValues() { + byte[] bytes = {1, 2, 3, 4}; + var bytesValue = Value.fromBytes(bytes); + var stringValue = Value.fromString("hello"); + + assertThat(bytesValue.getTypeCode()).isEqualTo(TypeCode.BYTES); + assertThat(((Value.BytesValue) bytesValue).getValue()).isEqualTo(bytes); + + assertThat(stringValue.getTypeCode()).isEqualTo(TypeCode.STRING); + assertThat(((Value.StringValue) stringValue).getValue()).isEqualTo("hello"); + } + + @Test + void shouldCreateArrayValues() { + List elements = Arrays.asList( + Value.fromInt32(1), + Value.fromInt32(2), + Value.fromInt32(3) + ); + Value arrayValue = Value.fromArray(elements); + + assertThat(arrayValue.getTypeCode()).isEqualTo(TypeCode.ARRAY); + assertThat(((Value.ArrayValue) arrayValue).getValue()).isEqualTo(elements); + } + + @Test + void shouldCreateMapValues() { + var map = new HashMap(); + map.put(MapKey.fromString("key1"), Value.fromInt32(1)); + map.put(MapKey.fromString("key2"), Value.fromInt32(2)); + + Value mapValue = Value.fromMap(map); + + assertThat(mapValue.getTypeCode()).isEqualTo(TypeCode.MAP); + assertThat(((Value.MapValue) mapValue).getValue()).isEqualTo(map); + } + + @Test + void shouldHandleEqualityCorrectly() { + var int1 = Value.fromInt32(42); + var int2 = Value.fromInt32(42); + var int3 = Value.fromInt32(43); + + assertThat(int1).isEqualTo(int2); + assertThat(int1).isNotEqualTo(int3); + assertThat(int1.hashCode()).isEqualTo(int2.hashCode()); + } + + @Test + void shouldDefensiveCopyArrays() { + byte[] original = {1, 2, 3}; + var bytesValue = Value.fromBytes(original); + + // Modify original array + original[0] = 99; + + // Value should be unchanged + assertThat(((Value.BytesValue) bytesValue).getValue()).containsExactly(1, 2, 3); + } + + @Test + void shouldRejectNullString() { + assertThatThrownBy(() -> Value.fromString(null)) + .isInstanceOf(NullPointerException.class); + } +} \ No newline at end of file diff --git a/src/test/java/com/imprint/util/VarIntTest.java b/src/test/java/com/imprint/util/VarIntTest.java new file mode 100644 index 0000000..677afb7 --- /dev/null +++ b/src/test/java/com/imprint/util/VarIntTest.java @@ -0,0 +1,115 @@ +package com.imprint.util; + +import com.imprint.error.ImprintException; +import com.imprint.error.ErrorType; +import org.junit.jupiter.api.Test; +import java.nio.ByteBuffer; +import static org.assertj.core.api.Assertions.*; + +class VarIntTest { + + @Test + void shouldRoundtripCommonValues() throws ImprintException { + int[] testCases = { + 0, 1, 127, 128, 16383, 16384, 2097151, 2097152, + 268435455, 268435456, -1 // -1 as unsigned is 0xFFFFFFFF + }; + + for (int value : testCases) { + ByteBuffer buffer = ByteBuffer.allocate(10); + VarInt.encode(value, buffer); + int encodedLength = buffer.position(); + + buffer.flip(); + VarInt.DecodeResult result = VarInt.decode(buffer); + + assertThat(result.getValue()).isEqualTo(value); + assertThat(result.getBytesRead()).isEqualTo(encodedLength); + } + } + + @Test + void shouldEncodeKnownValuesCorrectly() { + // Test cases with known encodings + assertEncodedBytes(0, 0x00); + assertEncodedBytes(1, 0x01); + assertEncodedBytes(127, 0x7f); + assertEncodedBytes(128, 0x80, 0x01); + assertEncodedBytes(16383, 0xff, 0x7f); + assertEncodedBytes(16384, 0x80, 0x80, 0x01); + } + + private void assertEncodedBytes(int value, int... expectedBytes) { + ByteBuffer buffer = ByteBuffer.allocate(10); + VarInt.encode(value, buffer); + buffer.flip(); + + byte[] actual = new byte[buffer.remaining()]; + buffer.get(actual); + + byte[] expected = new byte[expectedBytes.length]; + for (int i = 0; i < expectedBytes.length; i++) { + expected[i] = (byte) expectedBytes[i]; + } + + assertThat(actual).containsExactly(expected); + } + + @Test + void shouldWorkWithByteBuffer() throws ImprintException { + ByteBuffer buffer = ByteBuffer.allocate(10); + VarInt.encode(16384, buffer); + + buffer.flip(); + VarInt.DecodeResult result = VarInt.decode(buffer); + + assertThat(result.getValue()).isEqualTo(16384); + assertThat(result.getBytesRead()).isEqualTo(3); + } + + @Test + void shouldCalculateEncodedLength() { + assertThat(VarInt.encodedLength(0)).isEqualTo(1); + assertThat(VarInt.encodedLength(127)).isEqualTo(1); + assertThat(VarInt.encodedLength(128)).isEqualTo(2); + assertThat(VarInt.encodedLength(16383)).isEqualTo(2); + assertThat(VarInt.encodedLength(16384)).isEqualTo(3); + assertThat(VarInt.encodedLength(-1)).isEqualTo(5); // max u32 + } + + @Test + void shouldHandleBufferUnderflow() { + ByteBuffer buffer = ByteBuffer.allocate(1); + buffer.put((byte) 0x80); // incomplete varint + buffer.flip(); + + assertThatThrownBy(() -> VarInt.decode(buffer)) + .isInstanceOf(ImprintException.class) + .extracting("errorType") + .isEqualTo(ErrorType.BUFFER_UNDERFLOW); + } + + @Test + void shouldHandleOverlongEncoding() { + ByteBuffer buffer = ByteBuffer.allocate(10); + buffer.put(new byte[]{(byte) 0x80, (byte) 0x80, (byte) 0x80, (byte) 0x80, (byte) 0x80, 0x01}); + buffer.flip(); + + assertThatThrownBy(() -> VarInt.decode(buffer)) + .isInstanceOf(ImprintException.class) + .extracting("errorType") + .isEqualTo(ErrorType.MALFORMED_VARINT); + } + + @Test + void shouldHandleOverflow() { + ByteBuffer buffer = ByteBuffer.allocate(10); + buffer.put(new byte[]{(byte) 0x80, (byte) 0x80, (byte) 0x80, (byte) 0x80, 0x10}); + buffer.flip(); + + assertThatThrownBy(() -> VarInt.decode(buffer)) + .isInstanceOf(ImprintException.class) + .extracting("errorType") + .isEqualTo(ErrorType.MALFORMED_VARINT); + } +} \ No newline at end of file From bce1d13dc70c6b1e95bf93d66548d693ab3073bb Mon Sep 17 00:00:00 2001 From: expand3d <> Date: Sun, 1 Jun 2025 20:33:38 -0400 Subject: [PATCH 04/34] Add GitHub Actions CI workflow for automated testing --- .github/workflows/ci.yml | 95 ++++++++++++++++++++++++++++++++++++++++ build.gradle | 75 +++++++++++++++++++++++++++++++ 2 files changed, 170 insertions(+) create mode 100644 .github/workflows/ci.yml create mode 100644 build.gradle diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..0650d7c --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,95 @@ +name: CI + +on: + push: + branches: [ main, dev ] + pull_request: + branches: [ main, dev ] + +jobs: + test: + runs-on: ubuntu-latest + + strategy: + matrix: + java-version: [11, 17, 21] + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up JDK ${{ matrix.java-version }} + uses: actions/setup-java@v4 + with: + java-version: ${{ matrix.java-version }} + distribution: 'temurin' + + - name: Cache Gradle dependencies + uses: actions/cache@v3 + with: + path: | + ~/.gradle/caches + ~/.gradle/wrapper + key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} + restore-keys: | + ${{ runner.os }}-gradle- + + - name: Make gradlew executable + run: chmod +x ./gradlew + + - name: Run tests + run: ./gradlew test + + - name: Run build + run: ./gradlew build + + - name: Upload test results + uses: actions/upload-artifact@v3 + if: always() + with: + name: test-results-java-${{ matrix.java-version }} + path: build/test-results/test/ + + - name: Upload build reports + uses: actions/upload-artifact@v3 + if: always() + with: + name: build-reports-java-${{ matrix.java-version }} + path: build/reports/ + + benchmark: + runs-on: ubuntu-latest + needs: test + if: github.event_name == 'pull_request' + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up JDK 17 + uses: actions/setup-java@v4 + with: + java-version: 17 + distribution: 'temurin' + + - name: Cache Gradle dependencies + uses: actions/cache@v3 + with: + path: | + ~/.gradle/caches + ~/.gradle/wrapper + key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} + restore-keys: | + ${{ runner.os }}-gradle- + + - name: Make gradlew executable + run: chmod +x ./gradlew + + - name: Run quick benchmark + run: ./gradlew jmh -Pjmh.fork=1 -Pjmh.warmupIterations=1 -Pjmh.iterations=1 + + - name: Upload benchmark results + uses: actions/upload-artifact@v3 + with: + name: benchmark-results + path: build/results/jmh/ \ No newline at end of file diff --git a/build.gradle b/build.gradle new file mode 100644 index 0000000..9262297 --- /dev/null +++ b/build.gradle @@ -0,0 +1,75 @@ +plugins { + id 'java-library' + id 'maven-publish' + id 'me.champeau.jmh' version '0.7.2' +} + +group = 'com.imprint' +version = '0.1.0-SNAPSHOT' + +java { + toolchain { + languageVersion = JavaLanguageVersion.of(11) + } + withJavadocJar() + withSourcesJar() +} + +repositories { + mavenCentral() +} + +dependencies { + // Lombok for reducing boilerplate + compileOnly 'org.projectlombok:lombok:1.18.30' + annotationProcessor 'org.projectlombok:lombok:1.18.30' + + // Test dependencies + testImplementation 'org.junit.jupiter:junit-jupiter:5.10.0' + testImplementation 'org.assertj:assertj-core:3.24.2' + testImplementation 'org.mockito:mockito-core:5.5.0' + + // Lombok for tests + testCompileOnly 'org.projectlombok:lombok:1.18.30' + testAnnotationProcessor 'org.projectlombok:lombok:1.18.30' + + // Performance testing with JMH + jmhImplementation 'org.openjdk.jmh:jmh-core:1.37' + jmhAnnotationProcessor 'org.openjdk.jmh:jmh-generator-annprocess:1.37' + + // Competitor libraries for benchmarking + jmhImplementation 'com.google.protobuf:protobuf-java:3.25.1' + jmhImplementation 'org.apache.avro:avro:1.11.3' + jmhImplementation 'com.fasterxml.jackson.core:jackson-databind:2.16.0' + jmhImplementation 'com.google.flatbuffers:flatbuffers-java:23.5.26' + jmhImplementation 'com.esotericsoftware:kryo:5.4.0' +} + +test { + useJUnitPlatform() + + // Enable detailed test output + testLogging { + events "passed", "skipped", "failed" + } +} + +// JMH configuration +jmh { + fork = 1 + warmupIterations = 3 + iterations = 3 + resultFormat = 'JSON' + includeTests = false +} + +compileJava { + options.compilerArgs << '-Xlint:unchecked' + options.deprecation = true +} + +javadoc { + if(JavaVersion.current().isJava9Compatible()) { + options.addBooleanOption('html5', true) + } +} From 72c468f1e052007e521864f0fc975e62e6c90bb0 Mon Sep 17 00:00:00 2001 From: expand3d <> Date: Sun, 1 Jun 2025 20:36:51 -0400 Subject: [PATCH 05/34] Update GitHub Actions workflow to use upload-artifact@v4 --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0650d7c..6e5a2a0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,14 +44,14 @@ jobs: run: ./gradlew build - name: Upload test results - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: always() with: name: test-results-java-${{ matrix.java-version }} path: build/test-results/test/ - name: Upload build reports - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: always() with: name: build-reports-java-${{ matrix.java-version }} @@ -89,7 +89,7 @@ jobs: run: ./gradlew jmh -Pjmh.fork=1 -Pjmh.warmupIterations=1 -Pjmh.iterations=1 - name: Upload benchmark results - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: benchmark-results path: build/results/jmh/ \ No newline at end of file From 468d68218bdc087c1be39f627fecbe799ae37e3a Mon Sep 17 00:00:00 2001 From: expand3d <> Date: Sun, 1 Jun 2025 20:42:37 -0400 Subject: [PATCH 06/34] Add Gradle wrapper validation to CI workflow --- .github/workflows/ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6e5a2a0..18842f6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,6 +37,9 @@ jobs: - name: Make gradlew executable run: chmod +x ./gradlew + - name: Validate Gradle wrapper + uses: gradle/wrapper-validation-action@v1 + - name: Run tests run: ./gradlew test @@ -85,6 +88,9 @@ jobs: - name: Make gradlew executable run: chmod +x ./gradlew + - name: Validate Gradle wrapper + uses: gradle/wrapper-validation-action@v1 + - name: Run quick benchmark run: ./gradlew jmh -Pjmh.fork=1 -Pjmh.warmupIterations=1 -Pjmh.iterations=1 From cf05b13275ac8878ef47f33713a8c396466c59e2 Mon Sep 17 00:00:00 2001 From: expand3d <> Date: Sun, 1 Jun 2025 20:45:02 -0400 Subject: [PATCH 07/34] Fix gitignore to include gradle-wrapper.jar for CI --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 09ea567..54c84dc 100644 --- a/.gitignore +++ b/.gitignore @@ -61,4 +61,5 @@ buildNumber.properties # JAR files (unless they're dependencies) *.jar +!gradle/wrapper/gradle-wrapper.jar !lombok.jar From d0d798388c77c1f33cacf410218753b179c000e9 Mon Sep 17 00:00:00 2001 From: expand3d <> Date: Sun, 1 Jun 2025 20:48:18 -0400 Subject: [PATCH 08/34] Force add gradle-wrapper.jar to repository --- gradle/wrapper/gradle-wrapper.jar | Bin 0 -> 43705 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 gradle/wrapper/gradle-wrapper.jar diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 0000000000000000000000000000000000000000..9bbc975c742b298b441bfb90dbc124400a3751b9 GIT binary patch literal 43705 zcma&Obx`DOvL%eWOXJW;V64viP??$)@wHcsJ68)>bJS6*&iHnskXE8MjvIPVl|FrmV}Npeql07fCw6`pw`0s zGauF(<*@v{3t!qoUU*=j)6;|-(yg@jvDx&fV^trtZt27?4Tkn729qrItVh@PMwG5$ z+oXHSPM??iHZ!cVP~gYact-CwV`}~Q+R}PPNRy+T-geK+>fHrijpllon_F4N{@b-} z1M0=a!VbVmJM8Xk@NRv)m&aRYN}FSJ{LS;}2ArQ5baSjfy40l@T5)1r-^0fAU6f_} zzScst%$Nd-^ElV~H0TetQhMc%S{}Q4lssln=|;LG?Ulo}*mhg8YvBAUY7YFdXs~vv zv~{duzVw%C#GxkBwX=TYp1Dh*Uaum2?RmsvPaLlzO^fIJ`L?&OV?Y&kKj~^kWC`Ly zfL-}J^4a0Ojuz9O{jUbIS;^JatJ5+YNNHe}6nG9Yd6P-lJiK2ms)A^xq^H2fKrTF) zp!6=`Ece~57>^9(RA4OB9;f1FAhV%zVss%#rDq$9ZW3N2cXC7dMz;|UcRFecBm`DA z1pCO!#6zKp#@mx{2>Qcme8y$Qg_gnA%(`Vtg3ccwgb~D(&@y8#Jg8nNYW*-P{_M#E zZ|wCsQoO1(iIKd-2B9xzI}?l#Q@G5d$m1Lfh0q;iS5FDQ&9_2X-H)VDKA*fa{b(sV zL--krNCXibi1+*C2;4qVjb0KWUVGjjRT{A}Q*!cFmj0tRip2ra>WYJ>ZK4C|V~RYs z6;~+*)5F^x^aQqk9tjh)L;DOLlD8j+0<>kHc8MN|68PxQV`tJFbgxSfq-}b(_h`luA0&;Vk<@51i0 z_cu6{_*=vlvYbKjDawLw+t^H?OV00_73Cn3goU5?})UYFuoSX6Xqw;TKcrsc|r# z$sMWYl@cs#SVopO$hpHZ)cdU-+Ui%z&Sa#lMI~zWW@vE%QDh@bTe0&V9nL>4Et9`N zGT8(X{l@A~loDx}BDz`m6@tLv@$mTlVJ;4MGuj!;9Y=%;;_kj#o8n5tX%@M)2I@}u z_{I!^7N1BxW9`g&Z+K#lZ@7_dXdsqp{W9_`)zgZ=sD~%WS5s$`7z#XR!Lfy(4se(m zR@a3twgMs19!-c4jh`PfpJOSU;vShBKD|I0@rmv_x|+ogqslnLLOepJpPMOxhRb*i zGHkwf#?ylQ@k9QJL?!}MY4i7joSzMcEhrDKJH&?2v{-tgCqJe+Y0njl7HYff z{&~M;JUXVR$qM1FPucIEY(IBAuCHC@^~QG6O!dAjzQBxDOR~lJEr4KS9R*idQ^p{D zS#%NQADGbAH~6wAt}(1=Uff-1O#ITe)31zCL$e9~{w)gx)g>?zFE{Bc9nJT6xR!i8 z)l)~9&~zSZTHk{?iQL^MQo$wLi}`B*qnvUy+Y*jEraZMnEhuj`Fu+>b5xD1_Tp z)8|wedv42#3AZUL7x&G@p@&zcUvPkvg=YJS6?1B7ZEXr4b>M+9Gli$gK-Sgh{O@>q7TUg+H zNJj`6q#O@>4HpPJEHvNij`sYW&u%#=215HKNg;C!0#hH1vlO5+dFq9& zS)8{5_%hz?#D#wn&nm@aB?1_|@kpA@{%jYcs{K%$a4W{k@F zPyTav?jb;F(|GaZhm6&M#g|`ckO+|mCtAU)5_(hn&Ogd z9Ku}orOMu@K^Ac>eRh3+0-y^F`j^noa*OkS3p^tLV`TY$F$cPXZJ48!xz1d7%vfA( zUx2+sDPqHfiD-_wJDb38K^LtpN2B0w=$A10z%F9f_P2aDX63w7zDG5CekVQJGy18I zB!tI`6rZr7TK10L(8bpiaQ>S@b7r_u@lh^vakd0e6USWw7W%d_Ob%M!a`K>#I3r-w zo2^+9Y)Sb?P9)x0iA#^ns+Kp{JFF|$09jb6ZS2}_<-=$?^#IUo5;g`4ICZknr!_aJ zd73%QP^e-$%Xjt|28xM}ftD|V@76V_qvNu#?Mt*A-OV{E4_zC4Ymo|(cb+w^`Wv== z>)c%_U0w`d$^`lZQp@midD89ta_qTJW~5lRrIVwjRG_9aRiQGug%f3p@;*%Y@J5uQ|#dJ+P{Omc`d2VR)DXM*=ukjVqIpkb<9gn9{*+&#p)Ek zN=4zwNWHF~=GqcLkd!q0p(S2_K=Q`$whZ}r@ec_cb9hhg9a z6CE=1n8Q;hC?;ujo0numJBSYY6)GTq^=kB~`-qE*h%*V6-ip=c4+Yqs*7C@@b4YAi zuLjsmD!5M7r7d5ZPe>4$;iv|zq=9=;B$lI|xuAJwi~j~^Wuv!Qj2iEPWjh9Z&#+G>lZQpZ@(xfBrhc{rlLwOC;optJZDj4Xfu3$u6rt_=YY0~lxoy~fq=*L_&RmD7dZWBUmY&12S;(Ui^y zBpHR0?Gk|`U&CooNm_(kkO~pK+cC%uVh^cnNn)MZjF@l{_bvn4`Jc}8QwC5_)k$zs zM2qW1Zda%bIgY^3NcfL)9ug`05r5c%8ck)J6{fluBQhVE>h+IA&Kb}~$55m-^c1S3 zJMXGlOk+01qTQUFlh5Jc3xq|7McY$nCs$5=`8Y;|il#Ypb{O9}GJZD8!kYh{TKqs@ z-mQn1K4q$yGeyMcryHQgD6Ra<6^5V(>6_qg`3uxbl|T&cJVA*M_+OC#>w(xL`RoPQ zf1ZCI3G%;o-x>RzO!mc}K!XX{1rih0$~9XeczHgHdPfL}4IPi~5EV#ZcT9 zdgkB3+NPbybS-d;{8%bZW^U+x@Ak+uw;a5JrZH!WbNvl!b~r4*vs#he^bqz`W93PkZna2oYO9dBrKh2QCWt{dGOw)%Su%1bIjtp4dKjZ^ zWfhb$M0MQiDa4)9rkip9DaH0_tv=XxNm>6MKeWv>`KNk@QVkp$Lhq_~>M6S$oliq2 zU6i7bK;TY)m>-}X7hDTie>cc$J|`*}t=MAMfWIALRh2=O{L57{#fA_9LMnrV(HrN6 zG0K_P5^#$eKt{J|#l~U0WN_3)p^LLY(XEqes0OvI?3)GTNY&S13X+9`6PLVFRf8K) z9x@c|2T72+-KOm|kZ@j4EDDec>03FdgQlJ!&FbUQQH+nU^=U3Jyrgu97&#-W4C*;_ z(WacjhBDp@&Yon<9(BWPb;Q?Kc0gR5ZH~aRNkPAWbDY!FiYVSu!~Ss^9067|JCrZk z-{Rn2KEBR|Wti_iy) zXnh2wiU5Yz2L!W{{_#LwNWXeNPHkF=jjXmHC@n*oiz zIoM~Wvo^T@@t!QQW?Ujql-GBOlnB|HjN@x~K8z)c(X}%%5Zcux09vC8=@tvgY>czq z3D(U&FiETaN9aP}FDP3ZSIXIffq>M3{~eTB{uauL07oYiM=~K(XA{SN!rJLyXeC+Y zOdeebgHOc2aCIgC=8>-Q>zfuXV*=a&gp{l#E@K|{qft@YtO>xaF>O7sZz%8);e86? z+jJlFB{0fu6%8ew^_<+v>>%6eB8|t*_v7gb{x=vLLQYJKo;p7^o9!9A1)fZZ8i#ZU z<|E?bZakjkEV8xGi?n+{Xh3EgFKdM^;4D;5fHmc04PI>6oU>>WuLy6jgpPhf8$K4M zjJo*MbN0rZbZ!5DmoC^@hbqXiP^1l7I5;Wtp2i9Jkh+KtDJoXP0O8qmN;Sp(+%upX zAxXs*qlr(ck+-QG_mMx?hQNXVV~LT{$Q$ShX+&x?Q7v z@8t|UDylH6@RZ?WsMVd3B0z5zf50BP6U<&X_}+y3uJ0c5OD}+J&2T8}A%2Hu#Nt_4 zoOoTI$A!hQ<2pk5wfZDv+7Z{yo+Etqry=$!*pvYyS+kA4xnJ~3b~TBmA8Qd){w_bE zqDaLIjnU8m$wG#&T!}{e0qmHHipA{$j`%KN{&#_Kmjd&#X-hQN+ju$5Ms$iHj4r?) z&5m8tI}L$ih&95AjQ9EDfPKSmMj-@j?Q+h~C3<|Lg2zVtfKz=ft{YaQ1i6Om&EMll zzov%MsjSg=u^%EfnO+W}@)O6u0LwoX709h3Cxdc2Rwgjd%LLTChQvHZ+y<1q6kbJXj3_pq1&MBE{8 zd;aFotyW>4WHB{JSD8Z9M@jBitC1RF;!B8;Rf-B4nOiVbGlh9w51(8WjL&e{_iXN( zAvuMDIm_>L?rJPxc>S`bqC|W$njA0MKWa?V$u6mN@PLKYqak!bR!b%c^ze(M`ec(x zv500337YCT4gO3+9>oVIJLv$pkf`01S(DUM+4u!HQob|IFHJHm#>eb#eB1X5;bMc| z>QA4Zv}$S?fWg~31?Lr(C>MKhZg>gplRm`2WZ--iw%&&YlneQYY|PXl;_4*>vkp;I z$VYTZq|B*(3(y17#@ud@o)XUZPYN*rStQg5U1Sm2gM}7hf_G<>*T%6ebK*tF(kbJc zNPH4*xMnJNgw!ff{YXrhL&V$6`ylY={qT_xg9znQWw9>PlG~IbhnpsG_94Kk_(V-o&v7#F znra%uD-}KOX2dkak**hJnZZQyp#ERyyV^lNe!Qrg=VHiyr7*%j#PMvZMuYNE8o;JM zGrnDWmGGy)(UX{rLzJ*QEBd(VwMBXnJ@>*F8eOFy|FK*Vi0tYDw;#E zu#6eS;%Nm2KY+7dHGT3m{TM7sl=z8|V0e!DzEkY-RG8vTWDdSQFE|?+&FYA146@|y zV(JP>LWL;TSL6rao@W5fWqM1-xr$gRci#RQV2DX-x4@`w{uEUgoH4G|`J%H!N?*Qn zy~rjzuf(E7E!A9R2bSF|{{U(zO+;e29K_dGmC^p7MCP!=Bzq@}&AdF5=rtCwka zTT1A?5o}i*sXCsRXBt)`?nOL$zxuP3i*rm3Gmbmr6}9HCLvL*45d|(zP;q&(v%}S5yBmRVdYQQ24zh z6qL2<2>StU$_Ft29IyF!6=!@;tW=o8vNzVy*hh}XhZhUbxa&;9~woye<_YmkUZ)S?PW{7t; zmr%({tBlRLx=ffLd60`e{PQR3NUniWN2W^~7Sy~MPJ>A#!6PLnlw7O0(`=PgA}JLZ ztqhiNcKvobCcBel2 z-N82?4-()eGOisnWcQ9Wp23|ybG?*g!2j#>m3~0__IX1o%dG4b;VF@^B+mRgKx|ij zWr5G4jiRy}5n*(qu!W`y54Y*t8g`$YrjSunUmOsqykYB4-D(*(A~?QpuFWh;)A;5= zPl|=x+-w&H9B7EZGjUMqXT}MkcSfF}bHeRFLttu!vHD{Aq)3HVhvtZY^&-lxYb2%` zDXk7>V#WzPfJs6u{?ZhXpsMdm3kZscOc<^P&e&684Rc1-d=+=VOB)NR;{?0NjTl~D z1MXak$#X4{VNJyD$b;U~Q@;zlGoPc@ny!u7Pe;N2l4;i8Q=8>R3H{>HU(z z%hV2?rSinAg6&wuv1DmXok`5@a3@H0BrqsF~L$pRYHNEXXuRIWom0l zR9hrZpn1LoYc+G@q@VsFyMDNX;>_Vf%4>6$Y@j;KSK#g)TZRmjJxB!_NmUMTY(cAV zmewn7H{z`M3^Z& z2O$pWlDuZHAQJ{xjA}B;fuojAj8WxhO}_9>qd0|p0nBXS6IIRMX|8Qa!YDD{9NYYK z%JZrk2!Ss(Ra@NRW<7U#%8SZdWMFDU@;q<}%F{|6n#Y|?FaBgV$7!@|=NSVoxlJI4G-G(rn}bh|?mKkaBF$-Yr zA;t0r?^5Nz;u6gwxURapQ0$(-su(S+24Ffmx-aP(@8d>GhMtC5x*iEXIKthE*mk$` zOj!Uri|EAb4>03C1xaC#(q_I<;t}U7;1JqISVHz3tO{) zD(Yu@=>I9FDmDtUiWt81;BeaU{_=es^#QI7>uYl@e$$lGeZ~Q(f$?^3>$<<{n`Bn$ zn8bamZlL@6r^RZHV_c5WV7m2(G6X|OI!+04eAnNA5=0v1Z3lxml2#p~Zo57ri;4>;#16sSXXEK#QlH>=b$inEH0`G#<_ zvp;{+iY)BgX$R!`HmB{S&1TrS=V;*5SB$7*&%4rf_2wQS2ed2E%Wtz@y$4ecq4w<) z-?1vz_&u>s?BMrCQG6t9;t&gvYz;@K@$k!Zi=`tgpw*v-#U1Pxy%S9%52`uf$XMv~ zU}7FR5L4F<#9i%$P=t29nX9VBVv)-y7S$ZW;gmMVBvT$BT8d}B#XV^@;wXErJ-W2A zA=JftQRL>vNO(!n4mcd3O27bHYZD!a0kI)6b4hzzL9)l-OqWn)a~{VP;=Uo|D~?AY z#8grAAASNOkFMbRDdlqVUfB;GIS-B-_YXNlT_8~a|LvRMVXf!<^uy;)d$^OR(u)!) zHHH=FqJF-*BXif9uP~`SXlt0pYx|W&7jQnCbjy|8b-i>NWb@!6bx;1L&$v&+!%9BZ z0nN-l`&}xvv|wwxmC-ZmoFT_B#BzgQZxtm|4N+|;+(YW&Jtj^g!)iqPG++Z%x0LmqnF875%Ry&2QcCamx!T@FgE@H zN39P6e#I5y6Yl&K4eUP{^biV`u9{&CiCG#U6xgGRQr)zew;Z%x+ z-gC>y%gvx|dM=OrO`N@P+h2klPtbYvjS!mNnk4yE0+I&YrSRi?F^plh}hIp_+OKd#o7ID;b;%*c0ES z!J))9D&YufGIvNVwT|qsGWiZAwFODugFQ$VsNS%gMi8OJ#i${a4!E3<-4Jj<9SdSY z&xe|D0V1c`dZv+$8>(}RE|zL{E3 z-$5Anhp#7}oO(xm#}tF+W=KE*3(xxKxhBt-uuJP}`_K#0A< zE%rhMg?=b$ot^i@BhE3&)bNBpt1V*O`g?8hhcsV-n#=|9wGCOYt8`^#T&H7{U`yt2 z{l9Xl5CVsE=`)w4A^%PbIR6uG_5Ww9k`=q<@t9Bu662;o{8PTjDBzzbY#tL;$wrpjONqZ{^Ds4oanFm~uyPm#y1Ll3(H57YDWk9TlC zq;kebC!e=`FU&q2ojmz~GeLxaJHfs0#F%c(i+~gg$#$XOHIi@1mA72g2pFEdZSvp}m0zgQb5u2?tSRp#oo!bp`FP}< zaK4iuMpH+Jg{bb7n9N6eR*NZfgL7QiLxI zk6{uKr>xxJ42sR%bJ%m8QgrL|fzo9@?9eQiMW8O`j3teoO_R8cXPe_XiLnlYkE3U4 zN!^F)Z4ZWcA8gekEPLtFqX-Q~)te`LZnJK_pgdKs)Dp50 zdUq)JjlJeELskKg^6KY!sIou-HUnSFRsqG^lsHuRs`Z{f(Ti9eyd3cwu*Kxp?Ws7l z3cN>hGPXTnQK@qBgqz(n*qdJ2wbafELi?b90fK~+#XIkFGU4+HihnWq;{{)1J zv*Txl@GlnIMOjzjA1z%g?GsB2(6Zb-8fooT*8b0KF2CdsIw}~Hir$d3TdVHRx1m3c z4C3#h@1Xi@{t4zge-#B6jo*ChO%s-R%+9%-E|y<*4;L>$766RiygaLR?X%izyqMXA zb|N=Z-0PSFeH;W6aQ3(5VZWVC>5Ibgi&cj*c%_3=o#VyUJv* zM&bjyFOzlaFq;ZW(q?|yyi|_zS%oIuH^T*MZ6NNXBj;&yM3eQ7!CqXY?`7+*+GN47 zNR#%*ZH<^x{(0@hS8l{seisY~IE*)BD+R6^OJX}<2HRzo^fC$n>#yTOAZbk4%=Bei=JEe=o$jm`or0YDw*G?d> z=i$eEL7^}_?UI^9$;1Tn9b>$KOM@NAnvWrcru)r`?LodV%lz55O3y(%FqN;cKgj7t zlJ7BmLTQ*NDX#uelGbCY>k+&H*iSK?x-{w;f5G%%!^e4QT9z<_0vHbXW^MLR} zeC*jezrU|{*_F`I0mi)9=sUj^G03i@MjXx@ePv@(Udt2CCXVOJhRh4yp~fpn>ssHZ z?k(C>2uOMWKW5FVsBo#Nk!oqYbL`?#i~#!{3w^qmCto05uS|hKkT+iPrC-}hU_nbL zO622#mJupB21nChpime}&M1+whF2XM?prT-Vv)|EjWYK(yGYwJLRRMCkx;nMSpu?0 zNwa*{0n+Yg6=SR3-S&;vq=-lRqN`s9~#)OOaIcy3GZ&~l4g@2h| zThAN#=dh{3UN7Xil;nb8@%)wx5t!l z0RSe_yJQ+_y#qEYy$B)m2yDlul^|m9V2Ia$1CKi6Q19~GTbzqk*{y4;ew=_B4V8zw zScDH&QedBl&M*-S+bH}@IZUSkUfleyM45G>CnYY{hx8J9q}ME?Iv%XK`#DJRNmAYt zk2uY?A*uyBA=nlYjkcNPMGi*552=*Q>%l?gDK_XYh*Rya_c)ve{=ps`QYE0n!n!)_$TrGi_}J|>1v}(VE7I~aP-wns#?>Y zu+O7`5kq32zM4mAQpJ50vJsUDT_^s&^k-llQMy9!@wRnxw@~kXV6{;z_wLu3i=F3m z&eVsJmuauY)8(<=pNUM5!!fQ4uA6hBkJoElL1asWNkYE#qaP?a+biwWw~vB48PRS7 zY;DSHvgbIB$)!uJU)xA!yLE*kP0owzYo`v@wfdux#~f!dv#uNc_$SF@Qq9#3q5R zfuQnPPN_(z;#X#nRHTV>TWL_Q%}5N-a=PhkQ^GL+$=QYfoDr2JO-zo#j;mCsZVUQ) zJ96e^OqdLW6b-T@CW@eQg)EgIS9*k`xr$1yDa1NWqQ|gF^2pn#dP}3NjfRYx$pTrb zwGrf8=bQAjXx*8?du*?rlH2x~^pXjiEmj^XwQo{`NMonBN=Q@Y21!H)D( zA~%|VhiTjaRQ%|#Q9d*K4j~JDXOa4wmHb0L)hn*;Eq#*GI}@#ux4}bt+olS(M4$>c z=v8x74V_5~xH$sP+LZCTrMxi)VC%(Dg!2)KvW|Wwj@pwmH6%8zd*x0rUUe$e(Z%AW z@Q{4LL9#(A-9QaY2*+q8Yq2P`pbk3!V3mJkh3uH~uN)+p?67d(r|Vo0CebgR#u}i? zBxa^w%U|7QytN%L9bKaeYhwdg7(z=AoMeP0)M3XZA)NnyqL%D_x-(jXp&tp*`%Qsx z6}=lGr;^m1<{;e=QQZ!FNxvLcvJVGPkJ63at5%*`W?46!6|5FHYV0qhizSMT>Zoe8 zsJ48kb2@=*txGRe;?~KhZgr-ZZ&c0rNV7eK+h$I-UvQ=552@psVrvj#Ys@EU4p8`3 zsNqJu-o=#@9N!Pq`}<=|((u)>^r0k^*%r<{YTMm+mOPL>EoSREuQc-e2~C#ZQ&Xve zZ}OUzmE4{N-7cqhJiUoO_V#(nHX11fdfVZJT>|6CJGX5RQ+Ng$Nq9xs-C86-)~`>p zW--X53J`O~vS{WWjsAuGq{K#8f#2iz` zzSSNIf6;?5sXrHig%X(}0q^Y=eYwvh{TWK-fT>($8Ex>!vo_oGFw#ncr{vmERi^m7lRi%8Imph})ZopLoIWt*eFWSPuBK zu>;Pu2B#+e_W|IZ0_Q9E9(s@0>C*1ft`V{*UWz^K<0Ispxi@4umgGXW!j%7n+NC~* zBDhZ~k6sS44(G}*zg||X#9Weto;u*Ty;fP!+v*7be%cYG|yEOBomch#m8Np!Sw`L)q+T` zmrTMf2^}7j=RPwgpO9@eXfb{Q>GW#{X=+xt`AwTl!=TgYm)aS2x5*`FSUaaP_I{Xi zA#irF%G33Bw>t?^1YqX%czv|JF0+@Pzi%!KJ?z!u$A`Catug*tYPO`_Zho5iip0@! z;`rR0-|Ao!YUO3yaujlSQ+j-@*{m9dHLtve!sY1Xq_T2L3&=8N;n!!Eb8P0Z^p4PL zQDdZ?An2uzbIakOpC|d@=xEA}v-srucnX3Ym{~I#Ghl~JZU(a~Ppo9Gy1oZH&Wh%y zI=KH_s!Lm%lAY&`_KGm*Ht)j*C{-t}Nn71drvS!o|I|g>ZKjE3&Mq0TCs6}W;p>%M zQ(e!h*U~b;rsZ1OPigud>ej=&hRzs@b>>sq6@Yjhnw?M26YLnDH_Wt#*7S$-BtL08 zVyIKBm$}^vp?ILpIJetMkW1VtIc&7P3z0M|{y5gA!Yi5x4}UNz5C0Wdh02!h zNS>923}vrkzl07CX`hi)nj-B?#n?BJ2Vk0zOGsF<~{Fo7OMCN_85daxhk*pO}x_8;-h>}pcw26V6CqR-=x2vRL?GB#y%tYqi;J}kvxaz}*iFO6YO0ha6!fHU9#UI2Nv z_(`F#QU1B+P;E!t#Lb)^KaQYYSewj4L!_w$RH%@IL-M($?DV@lGj%3ZgVdHe^q>n(x zyd5PDpGbvR-&p*eU9$#e5#g3-W_Z@loCSz}f~{94>k6VRG`e5lI=SE0AJ7Z_+=nnE zTuHEW)W|a8{fJS>2TaX zuRoa=LCP~kP)kx4L+OqTjtJOtXiF=y;*eUFgCn^Y@`gtyp?n14PvWF=zhNGGsM{R- z^DsGxtoDtx+g^hZi@E2Y(msb-hm{dWiHdoQvdX88EdM>^DS#f}&kCGpPFDu*KjEpv$FZtLpeT>@)mf|z#ZWEsueeW~hF78Hu zfY9a+Gp?<)s{Poh_qdcSATV2oZJo$OH~K@QzE2kCADZ@xX(; z)0i=kcAi%nvlsYagvUp(z0>3`39iKG9WBDu3z)h38p|hLGdD+Khk394PF3qkX!02H z#rNE`T~P9vwNQ_pNe0toMCRCBHuJUmNUl)KFn6Gu2je+p>{<9^oZ4Gfb!)rLZ3CR3 z-o&b;Bh>51JOt=)$-9+Z!P}c@cKev_4F1ZZGs$I(A{*PoK!6j@ZJrAt zv2LxN#p1z2_0Ox|Q8PVblp9N${kXkpsNVa^tNWhof)8x8&VxywcJz#7&P&d8vvxn` zt75mu>yV=Dl#SuiV!^1BPh5R)`}k@Nr2+s8VGp?%Le>+fa{3&(XYi~{k{ z-u4#CgYIdhp~GxLC+_wT%I*)tm4=w;ErgmAt<5i6c~)7JD2olIaK8by{u-!tZWT#RQddptXRfEZxmfpt|@bs<*uh?Y_< zD>W09Iy4iM@@80&!e^~gj!N`3lZwosC!!ydvJtc0nH==K)v#ta_I}4Tar|;TLb|+) zSF(;=?$Z0?ZFdG6>Qz)6oPM}y1&zx_Mf`A&chb znSERvt9%wdPDBIU(07X+CY74u`J{@SSgesGy~)!Mqr#yV6$=w-dO;C`JDmv=YciTH zvcrN1kVvq|(3O)NNdth>X?ftc`W2X|FGnWV%s})+uV*bw>aoJ#0|$pIqK6K0Lw!@- z3pkPbzd`ljS=H2Bt0NYe)u+%kU%DWwWa>^vKo=lzDZHr>ruL5Ky&#q7davj-_$C6J z>V8D-XJ}0cL$8}Xud{T_{19#W5y}D9HT~$&YY-@=Th219U+#nT{tu=d|B)3K`pL53 zf7`I*|L@^dPEIDJkI3_oA9vsH7n7O}JaR{G~8 zfi$?kmKvu20(l`dV7=0S43VwVKvtF!7njv1Q{Ju#ysj=|dASq&iTE8ZTbd-iiu|2& zmll%Ee1|M?n9pf~?_tdQ<7%JA53!ulo1b^h#s|Su2S4r{TH7BRB3iIOiX5|vc^;5( zKfE1+ah18YA9o1EPT(AhBtve5(%GMbspXV)|1wf5VdvzeYt8GVGt0e*3|ELBhwRaO zE|yMhl;Bm?8Ju3-;DNnxM3Roelg`^!S%e({t)jvYtJCKPqN`LmMg^V&S z$9OIFLF$%Py~{l?#ReyMzpWixvm(n(Y^Am*#>atEZ8#YD&?>NUU=zLxOdSh0m6mL? z_twklB0SjM!3+7U^>-vV=KyQZI-6<(EZiwmNBzGy;Sjc#hQk%D;bay$v#zczt%mFCHL*817X4R;E$~N5(N$1Tv{VZh7d4mhu?HgkE>O+^-C*R@ zR0ima8PsEV*WFvz`NaB+lhX3&LUZcWWJJrG7ZjQrOWD%_jxv=)`cbCk zMgelcftZ%1-p9u!I-Zf_LLz{hcn5NRbxkWby@sj2XmYfAV?iw^0?hM<$&ZDctdC`; zsL|C-7d;w$z2Gt0@hsltNlytoPnK&$>ksr(=>!7}Vk#;)Hp)LuA7(2(Hh(y3LcxRY zim!`~j6`~B+sRBv4 z<#B{@38kH;sLB4eH2+8IPWklhd25r5j2VR}YK$lpZ%7eVF5CBr#~=kUp`i zlb+>Z%i%BJH}5dmfg1>h7U5Q(-F{1d=aHDbMv9TugohX5lq#szPAvPE|HaokMQIi_ zTcTNsO53(oX=hg2w!XA&+qP}nwr$(C)pgG8emS@Mf7m0&*kiA!wPLS`88c=aD$niJ zp?3j%NI^uy|5*MzF`k4hFbsyQZ@wu!*IY+U&&9PwumdmyfL(S0#!2RFfmtzD3m9V7 zsNOw9RQofl-XBfKBF^~~{oUVouka#r3EqRf=SnleD=r1Hm@~`y8U7R)w16fgHvK-6?-TFth)f3WlklbZh+}0 zx*}7oDF4U^1tX4^$qd%987I}g;+o0*$Gsd=J>~Uae~XY6UtbdF)J8TzJXoSrqHVC) zJ@pMgE#;zmuz?N2MIC+{&)tx=7A%$yq-{GAzyz zLzZLf=%2Jqy8wGHD;>^x57VG)sDZxU+EMfe0L{@1DtxrFOp)=zKY1i%HUf~Dro#8} zUw_Mj10K7iDsX}+fThqhb@&GI7PwONx!5z;`yLmB_92z0sBd#HiqTzDvAsTdx+%W{ z2YL#U=9r!@3pNXMp_nvximh+@HV3psUaVa-lOBekVuMf1RUd26~P*|MLouQrb}XM-bEw(UgQxMI6M&l3Nha z{MBcV=tl(b_4}oFdAo}WX$~$Mj-z70FowdoB{TN|h2BdYs?$imcj{IQpEf9q z)rzpttc0?iwopSmEoB&V!1aoZqEWEeO-MKMx(4iK7&Fhc(94c zdy}SOnSCOHX+A8q@i>gB@mQ~Anv|yiUsW!bO9hb&5JqTfDit9X6xDEz*mQEiNu$ay zwqkTV%WLat|Ar+xCOfYs0UQNM`sdsnn*zJr>5T=qOU4#Z(d90!IL76DaHIZeWKyE1 zqwN%9+~lPf2d7)vN2*Q?En?DEPcM+GQwvA<#;X3v=fqsxmjYtLJpc3)A8~*g(KqFx zZEnqqruFDnEagXUM>TC7ngwKMjc2Gx%#Ll#=N4qkOuK|;>4%=0Xl7k`E69@QJ-*Vq zk9p5!+Ek#bjuPa<@Xv7ku4uiWo|_wy)6tIr`aO!)h>m5zaMS-@{HGIXJ0UilA7*I} z?|NZ!Tp8@o-lnyde*H+@8IHME8VTQOGh96&XX3E+}OB zA>VLAGW+urF&J{H{9Gj3&u+Gyn?JAVW84_XBeGs1;mm?2SQm9^!3UE@(_FiMwgkJI zZ*caE={wMm`7>9R?z3Ewg!{PdFDrbzCmz=RF<@(yQJ_A6?PCd_MdUf5vv6G#9Mf)i#G z($OxDT~8RNZ>1R-vw|nN699a}MQN4gJE_9gA-0%>a?Q<9;f3ymgoi$OI!=aE6Elw z2I`l!qe-1J$T$X&x9Zz#;3!P$I);jdOgYY1nqny-k=4|Q4F!mkqACSN`blRji>z1` zc8M57`~1lgL+Ha%@V9_G($HFBXH%k;Swyr>EsQvg%6rNi){Tr&+NAMga2;@85531V z_h+h{jdB&-l+%aY{$oy2hQfx`d{&?#psJ78iXrhrO)McOFt-o80(W^LKM{Zw93O}m z;}G!51qE?hi=Gk2VRUL2kYOBRuAzktql%_KYF4>944&lJKfbr+uo@)hklCHkC=i)E zE*%WbWr@9zoNjumq|kT<9Hm*%&ahcQ)|TCjp@uymEU!&mqqgS;d|v)QlBsE0Jw|+^ zFi9xty2hOk?rlGYT3)Q7i4k65@$RJ-d<38o<`}3KsOR}t8sAShiVWevR8z^Si4>dS z)$&ILfZ9?H#H&lumngpj7`|rKQQ`|tmMmFR+y-9PP`;-425w+#PRKKnx7o-Rw8;}*Ctyw zKh~1oJ5+0hNZ79!1fb(t7IqD8*O1I_hM;o*V~vd_LKqu7c_thyLalEF8Y3oAV=ODv z$F_m(Z>ucO(@?+g_vZ`S9+=~Msu6W-V5I-V6h7->50nQ@+TELlpl{SIfYYNvS6T6D z`9cq=at#zEZUmTfTiM3*vUamr!OB~g$#?9$&QiwDMbSaEmciWf3O2E8?oE0ApScg38hb&iN%K+kvRt#d))-tr^ zD+%!d`i!OOE3in0Q_HzNXE!JcZ<0;cu6P_@;_TIyMZ@Wv!J z)HSXAYKE%-oBk`Ye@W3ShYu-bfCAZ}1|J16hFnLy z?Bmg2_kLhlZ*?`5R8(1%Y?{O?xT)IMv{-)VWa9#1pKH|oVRm4!lLmls=u}Lxs44@g^Zwa0Z_h>Rk<(_mHN47=Id4oba zQ-=qXGz^cNX(b*=NT0<^23+hpS&#OXzzVO@$Z2)D`@oS=#(s+eQ@+FSQcpXD@9npp zlxNC&q-PFU6|!;RiM`?o&Sj&)<4xG3#ozRyQxcW4=EE;E)wcZ&zUG*5elg;{9!j}I z9slay#_bb<)N!IKO16`n3^@w=Y%duKA-{8q``*!w9SW|SRbxcNl50{k&CsV@b`5Xg zWGZ1lX)zs_M65Yt&lO%mG0^IFxzE_CL_6$rDFc&#xX5EXEKbV8E2FOAt>Ka@e0aHQ zMBf>J$FLrCGL@$VgPKSbRkkqo>sOXmU!Yx+Dp7E3SRfT`v~!mjU3qj-*!!YjgI*^) z+*05x78FVnVwSGKr^A|FW*0B|HYgc{c;e3Ld}z4rMI7hVBKaiJRL_e$rxDW^8!nGLdJ<7ex9dFoyj|EkODflJ#Xl`j&bTO%=$v)c+gJsLK_%H3}A_} z6%rfG?a7+k7Bl(HW;wQ7BwY=YFMSR3J43?!;#~E&)-RV_L!|S%XEPYl&#`s!LcF>l zn&K8eemu&CJp2hOHJKaYU#hxEutr+O161ze&=j3w12)UKS%+LAwbjqR8sDoZHnD=m0(p62!zg zxt!Sj65S?6WPmm zL&U9c`6G}T`irf=NcOiZ!V)qhnvMNOPjVkyO2^CGJ+dKTnNAPa?!AxZEpO7yL_LkB zWpolpaDfSaO-&Uv=dj7`03^BT3_HJOAjn~X;wz-}03kNs@D^()_{*BD|0mII!J>5p z1h06PTyM#3BWzAz1FPewjtrQfvecWhkRR=^gKeFDe$rmaYAo!np6iuio3>$w?az$E zwGH|zy@OgvuXok}C)o1_&N6B3P7ZX&-yimXc1hAbXr!K&vclCL%hjVF$yHpK6i_Wa z*CMg1RAH1(EuuA01@lA$sMfe*s@9- z$jNWqM;a%d3?(>Hzp*MiOUM*?8eJ$=(0fYFis!YA;0m8s^Q=M0Hx4ai3eLn%CBm14 zOb8lfI!^UAu_RkuHmKA-8gx8Z;##oCpZV{{NlNSe<i;9!MfIN!&;JI-{|n{(A19|s z9oiGesENcLf@NN^9R0uIrgg(46r%kjR{0SbnjBqPq()wDJ@LC2{kUu_j$VR=l`#RdaRe zxx;b7bu+@IntWaV$si1_nrQpo*IWGLBhhMS13qH zTy4NpK<-3aVc;M)5v(8JeksSAGQJ%6(PXGnQ-g^GQPh|xCop?zVXlFz>42%rbP@jg z)n)% zM9anq5(R=uo4tq~W7wES$g|Ko z1iNIw@-{x@xKxSXAuTx@SEcw(%E49+JJCpT(y=d+n9PO0Gv1SmHkYbcxPgDHF}4iY zkXU4rkqkwVBz<{mcv~A0K|{zpX}aJcty9s(u-$je2&=1u(e#Q~UA{gA!f;0EAaDzdQ=}x7g(9gWrWYe~ zV98=VkHbI!5Rr;+SM;*#tOgYNlfr7;nLU~MD^jSdSpn@gYOa$TQPv+e8DyJ&>aInB zDk>JmjH=}<4H4N4z&QeFx>1VPY8GU&^1c&71T*@2#dINft%ibtY(bAm%<2YwPL?J0Mt{ z7l7BR718o5=v|jB!<7PDBafdL>?cCdVmKC;)MCOobo5edt%RTWiReAMaIU5X9h`@El0sR&Z z7Ed+FiyA+QAyWn zf7=%(8XpcS*C4^-L24TBUu%0;@s!Nzy{e95qjgkzElf0#ou`sYng<}wG1M|L? zKl6ITA1X9mt6o@S(#R3B{uwJI8O$&<3{+A?T~t>Kapx6#QJDol6%?i-{b1aRu?&9B z*W@$T*o&IQ&5Kc*4LK_)MK-f&Ys^OJ9FfE?0SDbAPd(RB)Oju#S(LK)?EVandS1qb#KR;OP|86J?;TqI%E8`vszd&-kS%&~;1Als=NaLzRNnj4q=+ zu5H#z)BDKHo1EJTC?Cd_oq0qEqNAF8PwU7fK!-WwVEp4~4g z3SEmE3-$ddli))xY9KN$lxEIfyLzup@utHn=Q{OCoz9?>u%L^JjClW$M8OB`txg4r6Q-6UlVx3tR%%Z!VMb6#|BKRL`I))#g zij8#9gk|p&Iwv+4s+=XRDW7VQrI(+9>DikEq!_6vIX8$>poDjSYIPcju%=qluSS&j zI-~+ztl1f71O-B+s7Hf>AZ#}DNSf`7C7*)%(Xzf|ps6Dr7IOGSR417xsU=Rxb z1pgk9vv${17h7mZ{)*R{mc%R=!i}8EFV9pl8V=nXCZruBff`$cqN3tpB&RK^$yH!A8RL zJ5KltH$&5%xC7pLZD}6wjD2-uq3&XL8CM$@V9jqalF{mvZ)c4Vn?xXbvkB(q%xbSdjoXJXanVN@I;8I`)XlBX@6BjuQKD28Jrg05} z^ImmK-Ux*QMn_A|1ionE#AurP8Vi?x)7jG?v#YyVe_9^up@6^t_Zy^T1yKW*t* z&Z0+0Eo(==98ig=^`he&G^K$I!F~1l~gq}%o5#pR6?T+ zLmZu&_ekx%^nys<^tC@)s$kD`^r8)1^tUazRkWEYPw0P)=%cqnyeFo3nW zyV$^0DXPKn5^QiOtOi4MIX^#3wBPJjenU#2OIAgCHPKXv$OY=e;yf7+_vI7KcjKq% z?RVzC24ekYp2lEhIE^J$l&wNX0<}1Poir8PjM`m#zwk-AL0w6WvltT}*JN8WFmtP_ z6#rK7$6S!nS!}PSFTG6AF7giGJw5%A%14ECde3x95(%>&W3zUF!8x5%*h-zk8b@Bz zh`7@ixoCVCZ&$$*YUJpur90Yg0X-P82>c~NMzDy7@Ed|6(#`;{)%t7#Yb>*DBiXC3 zUFq(UDFjrgOsc%0KJ_L;WQKF0q!MINpQzSsqwv?#Wg+-NO; z84#4nk$+3C{2f#}TrRhin=Erdfs77TqBSvmxm0P?01Tn@V(}gI_ltHRzQKPyvQ2=M zX#i1-a(>FPaESNx+wZ6J{^m_q3i})1n~JG80c<%-Ky!ZdTs8cn{qWY%x%X^27-Or_ z`KjiUE$OG9K4lWS16+?aak__C*)XA{ z6HmS*8#t_3dl}4;7ZZgn4|Tyy1lOEM1~6Qgl(|BgfQF{Mfjktch zB5kc~4NeehRYO%)3Z!FFHhUVVcV@uEX$eft5Qn&V3g;}hScW_d)K_h5i)vxjKCxcf zL>XlZ^*pQNuX*RJQn)b6;blT3<7@Ap)55)aK3n-H08GIx65W zO9B%gE%`!fyT`)hKjm-&=on)l&!i-QH+mXQ&lbXg0d|F{Ac#U;6b$pqQcpqWSgAPo zmr$gOoE*0r#7J=cu1$5YZE%uylM!i3L{;GW{ae9uy)+EaV>GqW6QJ)*B2)-W`|kLL z)EeeBtpgm;79U_1;Ni5!c^0RbG8yZ0W98JiG~TC8rjFRjGc6Zi8BtoC);q1@8h7UV zFa&LRzYsq%6d!o5-yrqyjXi>jg&c8bu}{Bz9F2D(B%nnuVAz74zmBGv)PAdFXS2(A z=Z?uupM2f-ar0!A)C6l2o8a|+uT*~huH)!h3i!&$ zr>76mt|lwexD(W_+5R{e@2SwR15lGxsnEy|gbS-s5?U}l*kcfQlfnQKo5=LZXizrL zM=0ty+$#f_qGGri-*t@LfGS?%7&LigUIU#JXvwEdJZvIgPCWFBTPT`@Re5z%%tRDO zkMlJCoqf2A=hkU7Ih=IxmPF~fEL90)u76nfFRQwe{m7b&Ww$pnk~$4Lx#s9|($Cvt ze|p{Xozhb^g1MNh-PqS_dLY|Fex4|rhM#lmzq&mhebD$5P>M$eqLoV|z=VQY{)7&sR#tW zl(S1i!!Rrg7kv+V@EL51PGpm511he%MbX2-Jl+DtyYA(0gZyZQjPZP@`SAH{n&25@ zd)emg(p2T3$A!Nmzo|%=z%AhLX)W4hsZNFhmd4<1l6?b3&Fg)G(Zh%J{Cf8Q;?_++ zgO7O<(-)H|Es@QqUgcXNJEfC-BCB~#dhi6ADVZtL!)Mx|u7>ukD052z!QZ5UC-+rd zYXWNRpCmdM{&?M9OMa;OiN{Y#0+F>lBQ=W@M;OXq;-7v3niC$pM8p!agNmq7F04;| z@s-_98JJB&s`Pr6o$KZ=8}qO*7m6SMp7kVmmh$jfnG{r@O(auI7Z^jj!x}NTLS9>k zdo}&Qc2m4Ws3)5qFw#<$h=g%+QUKiYog33bE)e4*H~6tfd42q+|FT5+vmr6Y$6HGC zV!!q>B`1Ho|6E|D<2tYE;4`8WRfm2#AVBBn%_W)mi(~x@g;uyQV3_)~!#A6kmFy0p zY~#!R1%h5E{5;rehP%-#kjMLt*{g((o@0-9*8lKVu+t~CtnOxuaMgo2ssI6@kX09{ zkn~q8Gx<6T)l}7tWYS#q0&~x|-3ho@l}qIr79qOJQcm&Kfr7H54=BQto0)vd1A_*V z)8b2{xa5O^u95~TS=HcJF5b9gMV%&M6uaj<>E zPNM~qGjJ~xbg%QTy#(hPtfc46^nN=Y_GmPYY_hTL{q`W3NedZyRL^kgU@Q$_KMAjEzz*eip`3u6AhPDcWXzR=Io5EtZRPme>#K9 z4lN&87i%YYjoCKN_z9YK+{fJu{yrriba#oGM|2l$ir017UH86Eoig3x+;bz32R*;n zt)Eyg#PhQbbGr^naCv0?H<=@+Poz)Xw*3Gn00qdSL|zGiyYKOA0CP%qk=rBAlt~hr zEvd3Z4nfW%g|c`_sfK$z8fWsXTQm@@eI-FpLGrW<^PIjYw)XC-xFk+M<6>MfG;WJr zuN}7b;p^`uc0j(73^=XJcw;|D4B(`)Flm|qEbB?>qBBv2V?`mWA?Q3yRdLkK7b}y& z+!3!JBI{+&`~;%Pj#n&&y+<;IQzw5SvqlbC+V=kLZLAHOQb zS{{8E&JXy1p|B&$K!T*GKtSV^{|Uk;`oE*F;?@q1dX|>|KWb@|Dy*lbGV0Gx;gpA$ z*N16`v*gQ?6Skw(f^|SL;;^ox6jf2AQ$Zl?gvEV&H|-ep*hIS@0TmGu1X1ZmEPY&f zKCrV{UgRAiNU*=+Uw%gjIQhTAC@67m)6(_D+N>)(^gK74F%M2NUpWpho}aq|Kxh$3 zz#DWOmQV4Lg&}`XTU41Z|P~5;wN2c?2L{a=)Xi~!m#*=22c~&AW zgG#yc!_p##fI&E{xQD9l#^x|9`wSyCMxXe<3^kDIkS0N>=oAz7b`@M>aT?e$IGZR; zS;I{gnr4cS^u$#>D(sjkh^T6_$s=*o%vNLC5+6J=HA$&0v6(Y1lm|RDn&v|^CTV{= zjVrg_S}WZ|k=zzp>DX08AtfT@LhW&}!rv^);ds7|mKc5^zge_Li>FTNFoA8dbk@K$ zuuzmDQRL1leikp%m}2_`A7*7=1p2!HBlj0KjPC|WT?5{_aa%}rQ+9MqcfXI0NtjvXz1U)|H>0{6^JpHspI4MfXjV%1Tc1O!tdvd{!IpO+@ z!nh()i-J3`AXow^MP!oVLVhVW&!CDaQxlD9b|Zsc%IzsZ@d~OfMvTFXoEQg9Nj|_L zI+^=(GK9!FGck+y8!KF!nzw8ZCX>?kQr=p@7EL_^;2Mlu1e7@ixfZQ#pqpyCJ```(m;la2NpJNoLQR};i4E;hd+|QBL@GdQy(Cc zTSgZ)4O~hXj86x<7&ho5ePzDrVD`XL7{7PjjNM1|6d5>*1hFPY!E(XDMA+AS;_%E~ z(dOs)vy29&I`5_yEw0x{8Adg%wvmoW&Q;x?5`HJFB@KtmS+o0ZFkE@f)v>YYh-z&m z#>ze?@JK4oE7kFRFD%MPC@x$^p{aW}*CH9Y_(oJ~St#(2)4e-b34D>VG6giMGFA83 zpZTHM2I*c8HE}5G;?Y7RXMA2k{Y?RxHb2 zZFQv?!*Kr_q;jt3`{?B5Wf}_a7`roT&m1BN9{;5Vqo6JPh*gnN(gj}#=A$-F(SRJj zUih_ce0f%K19VLXi5(VBGOFbc(YF zLvvOJl+W<}>_6_4O?LhD>MRGlrk;~J{S#Q;Q9F^;Cu@>EgZAH=-5fp02(VND(v#7n zK-`CfxEdonk!!65?3Ry(s$=|CvNV}u$5YpUf?9kZl8h@M!AMR7RG<9#=`_@qF@})d ztJDH>=F!5I+h!4#^DN6C$pd6^)_;0Bz7|#^edb9_qFg&eI}x{Roovml5^Yf5;=ehZ zGqz-x{I`J$ejkmGTFipKrUbv-+1S_Yga=)I2ZsO16_ye@!%&Op^6;#*Bm;=I^#F;? z27Sz-pXm4x-ykSW*3`)y4$89wy6dNOP$(@VYuPfb97XPDTY2FE{Z+{6=}LLA23mAc zskjZJ05>b)I7^SfVc)LnKW(&*(kP*jBnj>jtph`ZD@&30362cnQpZW8juUWcDnghc zy|tN1T6m?R7E8iyrL%)53`ymXX~_;#r${G`4Q(&7=m7b#jN%wdLlS0lb~r9RMdSuU zJ{~>>zGA5N`^QmrzaqDJ(=9y*?@HZyE!yLFONJO!8q5Up#2v>fR6CkquE$PEcvw5q zC8FZX!15JgSn{Gqft&>A9r0e#be^C<%)psE*nyW^e>tsc8s4Q}OIm})rOhuc{3o)g1r>Q^w5mas) zDlZQyjQefhl0PmH%cK05*&v{-M1QCiK=rAP%c#pdCq_StgDW}mmw$S&K6ASE=`u4+ z5wcmtrP27nAlQCc4qazffZoFV7*l2=Va}SVJD6CgRY^=5Ul=VYLGqR7H^LHA;H^1g}ekn=4K8SPRCT+pel*@jUXnLz+AIePjz@mUsslCN2 z({jl?BWf&DS+FlE5Xwp%5zXC7{!C=k9oQLP5B;sLQxd`pg+B@qPRqZ6FU(k~QkQu{ zF~5P=kLhs+D}8qqa|CQo2=cv$wkqAzBRmz_HL9(HRBj&73T@+B{(zZahlkkJ>EQmQ zenp59dy+L;sSWYde!z_W+I~-+2Xnm;c;wI_wH=RTgxpMlCW@;Us*0}L74J#E z8XbDWJGpBscw?W$&ZxZNxUq(*DKDwNzW7_}AIw$HF6Ix|;AJ3t6lN=v(c9=?n9;Y0 zK9A0uW4Ib9|Mp-itnzS#5in=Ny+XhGO8#(1_H4%Z6yEBciBiHfn*h;^r9gWb^$UB4 zJtN8^++GfT`1!WfQt#3sXGi-p<~gIVdMM<#ZZ0e_kdPG%Q5s20NNt3Jj^t$(?5cJ$ zGZ#FT(Lt>-0fP4b5V3az4_byF12k%}Spc$WsRydi&H|9H5u1RbfPC#lq=z#a9W(r1 z!*}KST!Yhsem0tO#r!z`znSL-=NnP~f(pw-sE+Z$e7i7t9nBP^5ts1~WFmW+j+<@7 zIh@^zKO{1%Lpx^$w8-S+T_59v;%N;EZtJzcfN%&@(Ux5 z@YzX^MwbbXESD*d(&qT7-eOHD6iaH-^N>p2sVdq&(`C$;?#mgBANIc5$r| z^A$r)@c{Z}N%sbfo?T`tTHz9-YpiMW?6>kr&W9t$Cuk{q^g1<$I~L zo++o2!!$;|U93cI#p4hyc!_Mv2QKXxv419}Ej#w#%N+YIBDdnn8;35!f2QZkUG?8O zpP47Wf9rnoI^^!9!dy~XsZ&!DU4bVTAi3Fc<9$_krGR&3TI=Az9uMgYU5dd~ksx+} zP+bs9y+NgEL>c@l>H1R%@>5SWg2k&@QZL(qNUI4XwDl6(=!Q^U%o984{|0e|mR$p+ z9BcwttR#7?As?@Q{+j?K6H7R71PuiA^Dl$=f47nUKL|koCwutc_P<-m{|Al3C~o7w z=4S=}s5LcJFT1zjS)+10X_r$74`K78pz!nGGH%JV%w75!YSIt#hT7}}K>+@{{a+Im z5p#6%^X*txY?}|T17xWW*sa^?G2QHt#@tlcw0GIcy;|NR2vaCBDvn=`h)1il7E5Rx z%)mA4$`$OZx)NF5vXZnaJ1)*cA6ryx6Ll~t!LzhxvcTedxT;>JS&e=?-&DXUPaQ2~ zH*69ezE`hgV{K-|0z|m~ld}=X^-Ob={wpex&}*+Rz{gx)G}gn!C_VN{UN=>^EV=Xc zr$-HO09cW&p4^M}V3yBjTP_xrVcc8iU_^Y-JD~(bgw*@GXGB1gYKz5DWO+O`>})|N zWrC)MR93yA)3{&27-M)TJB6Ml3~?zZg#mYsF=#OSTaw&K z@hBftpt+2l@)YK@|3DvTjl(8wZtpLp9Ik!6G$CSL_idZ$Ti?R)4toe8bb)l|)lNb}?K;O2K9vyn1QG zd=v#y-Ld49UVkmfRU>Egc+(Y$^-;6vW;3Lcu*6~etz}0|@+b|+!UCal)DEYGLbHWJ zll5Wi^$Y<6@S%^y%hdjRh6&{!z1Py|lZ|q&Wub3l41uN2zEF8E&5H5?PL*&V}?*a}Lp% zCYi{ghjpRNT^^B+_U59No50Ghih5qn(W5`RkrsDWr{~A1dgtv{sRkH4RU2^A{jb&0 zxVRnrm|u<;$iI;M6A>$POP)TWGU-gSjAERk*EGmVT(aw$!XUSe~7Ql-oRA54^4V(JWS6Q1mG?!vZ zx+pE!FEtvqr|Xrcb3oR`%LHFLmU_&{=p%mGy6MRe2Yz_5WJ8p@IgU2 zdVvvhhQtiQkChK%*&PsiPCBL9oDOoJX8!$S(V>R}+1M}wzK*U*A{KJ`r=lM;mPrKU zQDqqN(W*u-5-?$(SIk<6A0E}34y&@-IVC%S!a1F4kz<3bIKjlyD)ooO_7ftl%S_(6w`!vX&1PZ!K`@D@L6JR)6zO@Dl!YF{RY}d3HZ7?Q5E>w=$ ze)H_)48Ds*Ov4?zoGb2fe3}{!5Ooc|KCIni1o)(Gj+CO?`*7jsV`hIv@8J(22o4Q? zu?Bvi)zDG(me?7XKeL|iF9ZRgZdT*}Ffsl62Cu;{Gv9j6dO zPt*H2GqC)-C`V`ceuu=tM{7!2yTEj=*5+T~5DYiZ)Hy)*PARYI6R2lZXoOj;v8M4W z*O-NX(7_~Q&A3>Oaw&1lBH_H%SwmISX-i3)HfHvBOeVwTT{LUM3}ZuZmg<(>)KE;d zbs2!0v6>J;1nQ0UJkUxnkE@Ibi~Q}M=-=Rk;hcOnxO$luOKEVxZc|!XECgex(2`}T z3Y;Q_6rL)e+SrOZhQj5_e}Lv>w7n*Pep$yWZNQl>ubBgb_NIWWDn3kNpn+MPQXV;8 zV|_Ba5jsQ(w&Ey^IM|@|y!AqcJ#3m0#Q6_qvgCG~eoF#mnGmbO(;DP+bW%_aOs1R_ z@9p#7X2UA^--#Nwx_Hvk2l1`eO{P*#j@q2UELtH|Uh6hxR`h_847wIJo0=5CQQ`6it|%a-I$^&a@we1rc&*;QIu5Ck^?) zx*5eSd*mG#=6Hi(5!;5uUi&{HfnT1S8X-)?gE5CZ6KWoqM5|CyrULmuFBKOU8SOp* z{IB1$OCcq`S-k*xs;4fmhKsIGZ;GYAY*%(@875NxhMq|j*m4CNLI(Vho|N|F);!E0cS5y^$H^Izje?z}oTgyr`9x9G&rlJZw&uqIoBMtz zzhU0(9;w02?m#0!)cFi*r+8YvooQ;(s2lLVvyLqAE%Xqe!vtWbIs!l1Bpp(FIht-Z zPn#CN-2C|J*GhA2fuHqYQ2mJiXlGTzD}mkr2;ia8Wp}h^;OS7+N^Mw|en!1${vN6 z-x{8N*4UekA~`IV2&K-GzhAqau|}d*pEQ$1MH$cFi03OG^1NetZ_jW^STaEzr&Xho zB452St%v3ez2#TFm~`gZh$vi=in+y2d!z<{OZ~Kty-5bQ;0O=k_ESi8Nx9{*T`LJy6jqR>&|+>OZ;+=0hA04 zE25t^sE9HG)3^KKR_A5WDkqispweP9!I-@dCO&N!JrD@i{WBHnfQ z95o8;d$`AFnca3;N-0iX-CmbbAp5yQ!GoH;h7Cn?m{ammZJI8igP{U73lFnl2&gCs zqJ4(Vo~^j`{zOAzScL5B_Sm?Mjtek1d(A6X5ObcZi$;aOYy|g$}BY z$GEP3#i60Ju_&3SHzryH!gUFwC9-295u??cf+aYRQ1$+!rc#42YNattd6mZEFI@?C zqFM>6+zxEunIHDZ>{Z15u##>N(28Dw!>G(k*dB{NHvip@aP}f`@=Q;!o;zRMWo{Cx zo?kyzh8n7#f1g0&g>Cd>O-2g?uPwy8sy8hZbHSsXPmU;@l=HL=zm7mN(=@*|D$i+u zs~TllkCTvD$f&-#b9B?}#Lg*-ibK13R_a$RyoN3m5`10tdhAq{+VW)K#Bht-ra1*J z+n$N%V>u0rVtx`aKJDwXXrxaD7nS<>$=c82v7@KVx^S@vT;h=SZE37K>iahpx3;VDzEr9GY=2(%uaqM;^76eSP0QLzo4sI z>p_Eei*T$K;|qK`sq;?Hesp}(@VvX2Q4sAMYAJ}b&d$htDMC{FG-$o4k9ApECi1$a zXdamjiOGKHBh(4M<3(2x6n-CrmZMCknkQxdSS!qlis#I}btfX;J`JU3RlvtLdrymP zG0ZzrsGXVFiq+Wk1=BFay&9ZiCE#(`h~CL+c-Hs@iGTU@YxM%vlg;)`Tf~IknA^02 zXkN#Txo6aR{j$wP5T#|UH#5AP2{rSY8p?jKFv zG3kn3y`FaV!*Jq%m39_TQEhD>M@l*bhEPGe1{ft3q#K5AknT=F2_=T^l#ou5ln@D# z5Tzs(kRG@qNDa~HLNvfv7Z0g=bSlb?`QAx|Gfoni|iHJ%K0cy z;~Nsaa+{8HP_qrb{nj+xzkdYhSI@W4N_1`z(eSGIkbDP)!Ko|M%}Rqp(~KI2hl~eE zvJ!j4m6iwMgKy>fkCLC)`M$z9EV}B+sq1}}kVf$(ig0pWTY?rHz1Sm=4srTGNb^JG z=2$9wz-C@aZZZ2!HY#HNejqZRmE=pN(D$Kui$NpfhU`!y_s{@MIxiJdHb1|{6xb`> zE74_@QtgtG{4=3P1$^vn&m}7Aw8!1DnT$2thO#~44wl(N#ao8S0@t@m+Z!KD2CfK; z)n5DAPKV_etmH1aLDK$?`;sL91iVt$D z*SG}=-LIAg(*+JON!-5ivqOMQ1S!OQUgHglDsKik&Mwg;vva523`JwQH6SRz9eTY# zTIi23145~kc3r1mSWC_RzD%hs$S#!pkI9!BU80jJCJcwo*FZolQG$q`8C1d9pP@ND zG^&-ZraIvhg_FDVSfKGwkcI=avIan%2sK4coUs~Nr8jC*&!G0#?}_^s3r-c}-uAqi zM-Lw>Y}I``T;IS%Y|qH;s{F*ZefM!4{I5awr!K+T@uPd*Vu*iPWI}>(-D{zxsN>LG z=@747a_Rb2>q?y8xYf?dq2HM5tFO8Y5e4N;Y=xy8yAhI zsm>oy%R5;7)7T3V_b2%`aH^tNlsQpFxIFW#iV#8?{6{^cGr{A0@1bA)|K z>MMTuZD(pd2t|7vmHtywGXb%%=)S<`OG~}U+jm#xd%H8 z$v8-C%F?ah3$;hn?{G3(LT!SgvCVi$vwsZssAQvUwT`Q%qSw!LSd!(I!64w1=%Sc1Mck)q1@pZ@)=SY zoX}d+L3-RA|c?G3_BQNm&( z!i$AZ7cI(z7q|e9VM##6T3Xorj1JG(9os$;(I$y%mBy(#8{|3l4|x*oBAQL^XhZ0g zy1FR1teRrpKq{uLAibTLx#n({qwjlkOvR{OdSAeT5ah4-sNN)n4Clg1T9lzF)&yj; zyal1%+s4n1IG;^VPWJ;#olpk8Z42Gj-tjFeQ&PlxB)`oCNoUYKj4U$AeG8rYiD{pK zndDf&2;2;)D|KvOZP+e7fcPU9k4M2sfhr@vC~Ly0?S-4dz)ZGAYpCsAhChgbxLd4g zhTrbIPkO5SEp_kD>Ha0m12h5n3s;mE8kn515&nzSf+^D= zyE{JnJ;43l&BH55CL<=W%CF;6iUI)V5C*6!`**KqvzR2=Fj*3Y4`HYwx}TYD445(K z-QtXwtL?m*(F=LVH*H4oM>dXHBW=38q_dZ-_Vr&qpEPxd9Fs95P5W~@Z|Rt+WZP6l zPSQ}~Dh4V?Pp1g&Hk*Px?lm16C@X6M29Vrk%Rw@E||E-v~$ zb_E~{z<}#8i`Mx9mkqtd#Z1lZ-E_J8I+2oumc#x1)jdvh{W76NKm6x-RYpM~v!P8$ zw3e|YVf|}Hse9~oC@N7^j}Fi$hNpyaYnu1}bdXsD=^oI*%WKvbme|BI}$G3>smu#6y)ls|j? zF7Bhu9Z)j)C;3cZb+I>0stSK^WLOYV^U{pUYkgv>?+Nt^5j*CUB=eGw-CvU&40>y~ zGoHLXxY^7k5Xgv62{iQy|5jJQuq0|LU`}lE@flQ2Z*Zn*VWcQjm4FTb>LSVox^S4q zLn`LfS@mrjKCmg$nb^af?d?0&$aX6#2u(JyzIJvuJ*lwPrh|0~aEnSACCTezSdG%h zmSQg`17j@$Iq)r1&?+eR@1nlX|H`<}_!?BQSF&N+QQnvEAqZe+mIFui!0V49R?|9*$ zv!K1A01{8xq;L()Tv*Qk0-$Oj6+vCT*TUD{HvxO@3JjxBwM!4g3ydy&eaJw4CoQBF zJtULJ!YxgNR7_Ls%LmogyI7uIs=!B&?=MYY^yX+v;j@D_xGeZg>eZk0C;4e|HRNSi z6KlD9>q=3v-$4Zik&^ZDhNm1X)+7LCH1k!s+T3tn zUn@={1U&NJLq@K?~w|(=Y<4W{ucX}FdRr6pLw(l2$iK)At%t3gYBMlJz#(K0Nqm;=KAML!&MMSNz=%k=j*zh77r34Rs37iCY` z=_kva_41bdrj(b=4Wc5MO0~q^z#pIWJ>)vDSgIQF=3JVJe1iDy%h)8oNy{s_r&;m` zL{DYKSB_5xRb9xKNOS{qAY3qv5sSXVrrf%~*q5HO|CQ&lbKMePa$M5D{vlJcoGrCZ zD?fKbZN$6rWwz)w7`9h4DAmh1ij2}EO|bO#A9L0_RW6l*$sPPUJrUbhLC75L9%W5iO$Iw5~Yut-qBeu~hF|xD7-eQ%l z412vpq_;t%^F*pYDk%Q35c-erK|6Ve=FxQbAv~ikZ4c9$Y4;ee#ciOD9{yRqf55Qk zumv}#+JciT|Gj$uFOxBUze)=?l{B}qaC0_7m`t82<$K53!4Xvi9Tr)ADp3Off?O8o zVDG0Yx|tfn@r((m?Nxrh(b0DGjg)$;DfO&$6uY;4&F!4jnxkhP}Y3x zS?WFFt>=HWzqlQhffVfvM$Ta8Sg*r3j!Eo&rUOW7SCL2~lG7<+XZ;+{&8h5g8ElI+P>>yR2U%S93NN!Xhm|C682t6ysH-=o1=Bd*N*VlnG%l+KZFtjG`UkL;%65qn0UYQ`h zh0{9jDQx(`aBe7J0Aj3Z)4}`A|4OMM0a;?{j}qkYwi)~O8$9D}ITiMH2buiU>ixYp zhL${nwj6X($*OwmpVG`y5b6v45tX*J8?og}Qju6eJ9H}`X87iEd%BUo7<`2q(HJx+ zMR}d-J4oAf{V1W^a2~`M-YAdZ81dd4o6NPO{cmZaAS@RS4ir#Sr zfFZO-VIL|VN<%nEXr2` z$0FK2L#8O_f1w~c@G70JrB@N}r(gJ!Vmkk6{r68w!o$qO?HrFcjeU0_3F5;*!E2%( zTx>4?gP8w z1B?3UVZmz^%d_dIps>>0{cB~mp3{9UoPR6uQFecVq&} zY{ebB?AlPAD_}(ll{fK99;Wh1cgRbnw)maD^F>*J!R}eHM*W0VYN1TADWMy9H=$00 z5bHY${oDgwX7(W9LZw?}{!8(_{JB~Xkje6{0x4fgC4kUmpfJ+LT1DYD*TWu4#h{Y7 zFLronmc=hS=W=j1ar3r1JNjQoWo2hMWsqW*e?TF%#&{GpsaLp}iN~$)ar+7Ti}E&X z-nq~+Gkp(`qF0F_4A22>VZn-x>I$?PDZSeG8h_ifoWf^DxIb5%T7UytYo3}F|4#RC zUHpg$=)qVqD~=m(!~?XwocuxU1u}9qhhM7d^eqmJPi_e-!IO`*{u7A zbu*?L$Mbj-X9n3G2>+Kc#l`@d8}Xb9{l*IN{#M*d;s+3Pdr8FO$EBELR=8{ zd?LJbSv9fI`{OqTH)5{b?WulgMb)psp+W|@cSp=jtl-&5C}9lw@*0H+gEW(}mAWNz zf{~U;;N}|wdSaphgqnH{FWUy!{y3^=AC*c?RJ5Eb<^ zCgH_v7^axIUVmHSFL^zlj2R$zow$|y#7>%#U7d#Vp_ezcp3lefMyd5ES=q$>4pWyA zp_Zso^^NP~lu2=S6nD(3Z5u=Uy&B&F1i$J*3;3KhEkD_lgscHGR*;T;U!9vgQa(hI}oh9IzEf_PU_8F+i77t-~gDX z490Sb)LyVZmf18N6w{+37$aO<2!Av0 ztLaPOv^J<2@p{WnMiDudoghX_`luFZt_4eNU}*~cF5i%eEcNLs;D>QVIwr8mH;=dc z09`}JV;aaF;13@&iS(w>Jc=k~|d_1hcpM(l|O zu>!@}me%isTT$xT#hNUvh(ATd0wT4fbv=6htcHNEZIw9%E6wlYmwfu2{j0kh1y=$;Yf!|NldgB9ul zB{dbE&LfRnr8ITm@;-68wo#VV?8lG3ed&9k1}QBS3}WGV9%26?A1rBkkDR9Z3o+g+ z)eQg8BY3y(Dh5&z?VLLNdDV`C=muUvCPpGg!oYxIgOI3^%4>5d7jTh~ni!Fg2;fhx z(*c%H6Je84kmQh;5tC3*l~7khLxK-e|Cz?FLh!yYe7g|*LwqU?2wv^_ZyKT$fYVkGJo@AK0$+ml?}zJeB~deT2WL1vz}dxB z)y??t!}%M@)u$_IyW~)6u1SttJ!awd6N5lx|xBrmyrBh>tb&D*=C+Z3nPfq$1%WgY0bY*?PZ#Hk|=xn zGM#0*w4CaB^y0G(J4q=;5NeM@m-P}#mv7QZNF)M!dK^w{mk_!n0`+Y3PQutu-%NBt zzgPXug?JLEbUL{e_dk;Vd896&yPe(hliVK!lj%5+@BKdcrEZ2Nc_*i@ve*2lB>u~{ zFozd2FM|_0+nAGR4TLNHanQn_Oeb!JrUcvzJ?7p9TTNB}ocO3j$7ij!li8#k6 z@2tSd1>K03K9A#_-MIq)S;T#oE^;>U$)&}okIvDf3lm?kI{d80$>~xKUoS!%q1Pi?WpsUUt(tI ztjNjY*y&Rm9(S(DC2GuPHBJs@5M{RGm`c1z<6nwyN^)rMo-AS{M2$oM9|y%fM|}G~ DHx0+F literal 0 HcmV?d00001 From f2cdd1b912feff0dcfb1f0b6820863e583b81dd9 Mon Sep 17 00:00:00 2001 From: expand3d <> Date: Sun, 1 Jun 2025 20:48:43 -0400 Subject: [PATCH 09/34] Update wrapper validation action to v3 --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 18842f6..2e906c6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,7 +38,7 @@ jobs: run: chmod +x ./gradlew - name: Validate Gradle wrapper - uses: gradle/wrapper-validation-action@v1 + uses: gradle/wrapper-validation-action@v3 - name: Run tests run: ./gradlew test @@ -89,7 +89,7 @@ jobs: run: chmod +x ./gradlew - name: Validate Gradle wrapper - uses: gradle/wrapper-validation-action@v1 + uses: gradle/wrapper-validation-action@v3 - name: Run quick benchmark run: ./gradlew jmh -Pjmh.fork=1 -Pjmh.warmupIterations=1 -Pjmh.iterations=1 From 57c824964a18f804cffdf458a746b20e0fcd68ae Mon Sep 17 00:00:00 2001 From: expand3d <> Date: Sun, 1 Jun 2025 20:53:03 -0400 Subject: [PATCH 10/34] Fix Javadoc syntax errors and disable strict Javadoc checking --- build.gradle | 2 ++ .../com/imprint/core/ImprintRecordBuilder.java | 14 ++++++++------ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/build.gradle b/build.gradle index 9262297..7e85806 100644 --- a/build.gradle +++ b/build.gradle @@ -72,4 +72,6 @@ javadoc { if(JavaVersion.current().isJava9Compatible()) { options.addBooleanOption('html5', true) } + // Don't fail build on missing javadoc + options.addStringOption('Xdoclint:none', '-quiet') } diff --git a/src/main/java/com/imprint/core/ImprintRecordBuilder.java b/src/main/java/com/imprint/core/ImprintRecordBuilder.java index 48b0998..202bd2a 100644 --- a/src/main/java/com/imprint/core/ImprintRecordBuilder.java +++ b/src/main/java/com/imprint/core/ImprintRecordBuilder.java @@ -11,14 +11,16 @@ * developer-friendly API that eliminates boilerplate Value.fromX() calls. *

* Usage: + * 

  *   var record = ImprintRecord.builder(schemaId)
- *       .field(1, 42)              // int -> Int32Value  
- *       .field(2, "hello")         // String -> StringValue
- *       .field(3, 3.14)            // double -> Float64Value
- *       .field(4, bytes)           // byte[] -> BytesValue
- *       .field(5, true)            // boolean -> BoolValue
- *       .nullField(6)              // -> NullValue
+ *       .field(1, 42)              // int to Int32Value  
+ *       .field(2, "hello")         // String to StringValue
+ *       .field(3, 3.14)            // double to Float64Value
+ *       .field(4, bytes)           // byte[] to BytesValue
+ *       .field(5, true)            // boolean to BoolValue
+ *       .nullField(6)              // to NullValue
  *       .build();
+ *
*/ public final class ImprintRecordBuilder { private final SchemaId schemaId; From edb30578f35d65522df2778eea62a9d0e35e0268 Mon Sep 17 00:00:00 2001 From: expand3d <> Date: Sun, 1 Jun 2025 23:33:47 -0400 Subject: [PATCH 11/34] Add JMH benchmark .bat and .sh for full suite benchmarking and performance tracking; add comprehensive String benchmark --- .gitignore | 3 + benchmark-results/README.md | 65 +++++ build.gradle | 1 + run-benchmarks.bat | 89 +++++++ run-benchmarks.sh | 148 +++++++++++ .../imprint/benchmark/StringBenchmark.java | 248 ++++++++++++++++++ 6 files changed, 554 insertions(+) create mode 100644 benchmark-results/README.md create mode 100644 run-benchmarks.bat create mode 100644 run-benchmarks.sh create mode 100644 src/jmh/java/com/imprint/benchmark/StringBenchmark.java diff --git a/.gitignore b/.gitignore index 54c84dc..3f1edb4 100644 --- a/.gitignore +++ b/.gitignore @@ -63,3 +63,6 @@ buildNumber.properties *.jar !gradle/wrapper/gradle-wrapper.jar !lombok.jar + +# Benchmark Results (keep timestamped results in repo for tracking) +# benchmark-results/ - Commented out to keep results in repo diff --git a/benchmark-results/README.md b/benchmark-results/README.md new file mode 100644 index 0000000..dc193d7 --- /dev/null +++ b/benchmark-results/README.md @@ -0,0 +1,65 @@ +# Benchmark Results + +This directory contains historical benchmark results for the Imprint Java implementation. + +## Files + +- `*.json` - Raw JMH benchmark results in JSON format +- `*-summary-*.txt` - Human-readable summaries of benchmark runs +- `system-info-*.txt` - System information for each benchmark run +- `overall-summary-*.txt` - Complete benchmark overview + +## Running Benchmarks + +### All Benchmarks +```bash +# Unix/Linux/macOS +./run-benchmarks.sh + +# Windows +run-benchmarks.bat +``` + +### Specific Benchmark Categories +```bash +# String performance tests +./gradlew jmh -Pjmh.include=StringBenchmark + +# Serialization tests +./gradlew jmh -Pjmh.include=".*[Ss]erial.*" + +# Field access tests +./gradlew jmh -Pjmh.include=".*[Aa]ccess.*" + +# Comparison tests (vs other formats) +./gradlew jmh -Pjmh.include=ComparisonBenchmark +``` + +## Analyzing Results + +### Online Visualization +Upload JSON files to [JMH Visualizer](https://jmh.morethan.io/) for interactive charts and analysis. + +### Command Line Analysis +```bash +# View benchmark names and scores +jq -r '.[] | select(.benchmark) | "\(.benchmark): \(.primaryMetric.score) \(.primaryMetric.scoreUnit)"' results.json + +# Find fastest operations +jq -r '.[] | select(.benchmark) | "\(.benchmark): \(.primaryMetric.score)"' results.json | sort -k2 -n + +# Compare specific benchmarks +jq '.[] | select(.benchmark | contains("String"))' results.json +``` + +## Performance Tracking + +Results are timestamped and committed to track performance changes over time. Compare results between commits to identify performance regressions or improvements. + +## System Requirements + +For consistent results: +- Run on dedicated hardware when possible +- Close unnecessary applications +- Run multiple times and compare results +- Note system configuration in commit messages \ No newline at end of file diff --git a/build.gradle b/build.gradle index 7e85806..2606710 100644 --- a/build.gradle +++ b/build.gradle @@ -61,6 +61,7 @@ jmh { iterations = 3 resultFormat = 'JSON' includeTests = false + resultsFile = file("${projectDir}/benchmark-results/jmh-results-${new Date().format('yyyy-MM-dd-HHmmss')}.json") } compileJava { diff --git a/run-benchmarks.bat b/run-benchmarks.bat new file mode 100644 index 0000000..750722d --- /dev/null +++ b/run-benchmarks.bat @@ -0,0 +1,89 @@ +@echo off +REM Benchmark execution script for Imprint Java implementation (Windows) +REM This script runs all benchmark suites and saves results with timestamps + +setlocal enabledelayedexpansion + +for /f "tokens=2 delims==" %%a in ('wmic OS Get localdatetime /value') do set "dt=%%a" +set "TIMESTAMP=%dt:~0,4%-%dt:~4,2%-%dt:~6,2%-%dt:~8,2%%dt:~10,2%%dt:~12,2%" +set "RESULTS_DIR=.\benchmark-results" +set "SYSTEM_INFO_FILE=%RESULTS_DIR%\system-info-%TIMESTAMP%.txt" + +echo 🏃 Running Imprint Java Benchmarks - %TIMESTAMP% +echo ================================================ + +REM Ensure results directory exists +if not exist "%RESULTS_DIR%" mkdir "%RESULTS_DIR%" + +REM Capture system information +echo 📊 Capturing system information... +( + echo Benchmark Run: %TIMESTAMP% + echo ============================== + echo. + echo System Information: + echo - Date: %date% %time% + echo - OS: %OS% + echo - Processor: %PROCESSOR_IDENTIFIER% + echo - Java Version: + java -version 2^>^&1 + echo. + echo - Gradle Version: + call gradlew --version + echo. + echo - Git Commit: + for /f %%i in ('git rev-parse HEAD') do echo - Hash: %%i + for /f %%i in ('git rev-parse --abbrev-ref HEAD') do echo - Branch: %%i + for /f "tokens=*" %%i in ('git log -1 --format^=%%cd') do echo - Date: %%i + for /f "tokens=*" %%i in ('git log -1 --format^=%%s') do echo - Message: %%i + echo. +) > "%SYSTEM_INFO_FILE%" + +echo ✅ System info saved to: %SYSTEM_INFO_FILE% + +echo 🚀 Starting benchmark execution... +echo. + +REM Function to run complete benchmark suite +echo 🔄 Running complete benchmark suite... +set "complete_output=%RESULTS_DIR%\complete-benchmarks-%TIMESTAMP%.json" +call gradlew jmh -Pjmh.resultsFile="%complete_output%" --console=plain + +if exist "%complete_output%" ( + echo ✅ Complete benchmark suite completed: %complete_output% + + REM Generate overall summary + set "overall_summary=%RESULTS_DIR%\overall-summary-%TIMESTAMP%.txt" + ( + echo Complete Imprint Java Benchmark Results - %TIMESTAMP% + echo ================================================== + echo. + echo Benchmark execution completed successfully. + echo Results saved to: %complete_output% + echo. + echo To analyze results: + echo - Use JMH Visualizer: https://jmh.morethan.io/ + echo - Import JSON into analysis tools + echo - Use jq for command-line analysis + echo. + ) > "!overall_summary!" + + echo 📊 Overall summary: !overall_summary! +) else ( + echo ❌ Complete benchmark suite failed +) + +echo. +echo 🎉 Benchmark execution completed! +echo 📁 All results saved in: %RESULTS_DIR% +echo 📄 Files created: +dir "%RESULTS_DIR%\*%TIMESTAMP%*" 2>nul + +echo. +echo 💡 To view results: +echo - JSON files can be analyzed with jq or imported into analysis tools +echo - Visit https://jmh.morethan.io/ to visualize results +echo - Summary files provide human-readable overviews +echo - System info file contains environment details for reproducibility + +pause \ No newline at end of file diff --git a/run-benchmarks.sh b/run-benchmarks.sh new file mode 100644 index 0000000..571ab5f --- /dev/null +++ b/run-benchmarks.sh @@ -0,0 +1,148 @@ +#!/bin/bash + +# Benchmark execution script for Imprint Java implementation +# This script runs all benchmark suites and saves results with timestamps + +set -e + +TIMESTAMP=$(date '+%Y-%m-%d-%H%M%S') +RESULTS_DIR="./benchmark-results" +SYSTEM_INFO_FILE="$RESULTS_DIR/system-info-$TIMESTAMP.txt" + +echo "🏃 Running Imprint Java Benchmarks - $TIMESTAMP" +echo "================================================" + +# Ensure results directory exists +mkdir -p "$RESULTS_DIR" + +# Capture system information +echo "📊 Capturing system information..." +{ + echo "Benchmark Run: $TIMESTAMP" + echo "==============================" + echo "" + echo "System Information:" + echo "- Date: $(date)" + echo "- OS: $(uname -a)" + echo "- Java Version:" + java -version 2>&1 | sed 's/^/ /' + echo "" + echo "- Gradle Version:" + ./gradlew --version | sed 's/^/ /' + echo "" + echo "- Git Commit:" + echo " - Hash: $(git rev-parse HEAD)" + echo " - Branch: $(git rev-parse --abbrev-ref HEAD)" + echo " - Date: $(git log -1 --format=%cd)" + echo " - Message: $(git log -1 --format=%s)" + echo "" + echo "Hardware Information:" + if command -v lscpu &> /dev/null; then + echo "- CPU:" + lscpu | grep -E "Model name|Architecture|CPU\(s\)|Thread|Core" | sed 's/^/ /' + fi + if command -v free &> /dev/null; then + echo "- Memory:" + free -h | sed 's/^/ /' + fi + echo "" +} > "$SYSTEM_INFO_FILE" + +echo "✅ System info saved to: $SYSTEM_INFO_FILE" + +# Function to run a specific benchmark suite +run_benchmark_suite() { + local suite_name=$1 + local suite_pattern=$2 + local output_file="$RESULTS_DIR/${suite_name}-$TIMESTAMP.json" + + echo "🔄 Running $suite_name benchmarks..." + echo " Pattern: $suite_pattern" + echo " Output: $output_file" + + ./gradlew jmh \ + -Pjmh.include="$suite_pattern" \ + -Pjmh.resultsFile="$output_file" \ + --console=plain + + if [ -f "$output_file" ]; then + echo "✅ $suite_name completed: $output_file" + + # Generate a human-readable summary + local summary_file="$RESULTS_DIR/${suite_name}-summary-$TIMESTAMP.txt" + { + echo "$suite_name Benchmark Summary - $TIMESTAMP" + echo "=======================================" + echo "" + echo "Top 10 Fastest Operations:" + jq -r '.[] | select(.benchmark) | "\(.benchmark): \(.primaryMetric.score | tonumber | . * 1000000 | floor / 1000000) \(.primaryMetric.scoreUnit)"' "$output_file" | sort -k2 -n | head -10 + echo "" + echo "Top 10 Slowest Operations:" + jq -r '.[] | select(.benchmark) | "\(.benchmark): \(.primaryMetric.score | tonumber | . * 1000000 | floor / 1000000) \(.primaryMetric.scoreUnit)"' "$output_file" | sort -k2 -nr | head -10 + echo "" + } > "$summary_file" 2>/dev/null || echo "⚠️ Could not generate summary (jq not available)" + else + echo "❌ $suite_name failed - no output file generated" + fi + echo "" +} + +# Run all benchmark suites +echo "🚀 Starting benchmark execution..." +echo "" + +# 1. Serialization benchmarks +run_benchmark_suite "serialization" ".*[Ss]erial.*" + +# 2. Field access benchmarks +run_benchmark_suite "field-access" ".*[Aa]ccess.*" + +# 3. Merge benchmarks +run_benchmark_suite "merge" ".*[Mm]erge.*" + +# 4. String benchmarks +run_benchmark_suite "string" ".*String.*" + +# 5. Comparison benchmarks (vs other formats) +run_benchmark_suite "comparison" ".*Comparison.*" + +# 6. Complete benchmark run (all benchmarks) +echo "🔄 Running complete benchmark suite..." +complete_output="$RESULTS_DIR/complete-benchmarks-$TIMESTAMP.json" +./gradlew jmh \ + -Pjmh.resultsFile="$complete_output" \ + --console=plain + +if [ -f "$complete_output" ]; then + echo "✅ Complete benchmark suite completed: $complete_output" + + # Generate overall summary + overall_summary="$RESULTS_DIR/overall-summary-$TIMESTAMP.txt" + { + echo "Complete Imprint Java Benchmark Results - $TIMESTAMP" + echo "==================================================" + echo "" + echo "Total Benchmarks Run: $(jq '[.[] | select(.benchmark)] | length' "$complete_output" 2>/dev/null || echo "Unknown")" + echo "" + echo "Performance Overview:" + echo "--------------------" + jq -r '.[] | select(.benchmark) | "\(.benchmark): \(.primaryMetric.score | tonumber | . * 1000000 | floor / 1000000) \(.primaryMetric.scoreUnit)"' "$complete_output" 2>/dev/null | sort -k2 -n || echo "Could not generate overview" + echo "" + } > "$overall_summary" 2>/dev/null + + echo "📊 Overall summary: $overall_summary" +else + echo "❌ Complete benchmark suite failed" +fi + +echo "" +echo "🎉 Benchmark execution completed!" +echo "📁 All results saved in: $RESULTS_DIR" +echo "📄 Files created:" +ls -la "$RESULTS_DIR"/*"$TIMESTAMP"* 2>/dev/null || echo " No files with timestamp $TIMESTAMP found" + +echo "" +echo "💡 To view results:" +echo " - JSON files can be analyzed with jq or imported into analysis tools" +echo " - Summary files provide human-readable overviews" +echo " - System info file contains environment details for reproducibility" \ No newline at end of file diff --git a/src/jmh/java/com/imprint/benchmark/StringBenchmark.java b/src/jmh/java/com/imprint/benchmark/StringBenchmark.java new file mode 100644 index 0000000..1891251 --- /dev/null +++ b/src/jmh/java/com/imprint/benchmark/StringBenchmark.java @@ -0,0 +1,248 @@ +package com.imprint.benchmark; + +import com.imprint.core.ImprintRecord; +import com.imprint.core.SchemaId; +import com.imprint.types.Value; +import org.openjdk.jmh.annotations.*; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; + +import java.nio.ByteBuffer; +import java.util.concurrent.TimeUnit; + +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@State(Scope.Benchmark) +@Fork(1) +@Warmup(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS) +public class StringBenchmark { + + private static final SchemaId SCHEMA_ID = new SchemaId(1, 42); + + // Small strings (typical field names, short values) + private String smallString5; + private String smallString20; + private String smallString50; + + // Medium strings (typical text content) + private String mediumString500; + private String mediumString2K; + + // Large strings (document content, JSON payloads) + private String largeString10K; + private String largeString100K; + private String largeString1M; + + // Pre-serialized records for deserialization benchmarks + private byte[] serializedSmall5; + private byte[] serializedSmall20; + private byte[] serializedSmall50; + private byte[] serializedMedium500; + private byte[] serializedMedium2K; + private byte[] serializedLarge10K; + private byte[] serializedLarge100K; + private byte[] serializedLarge1M; + + @Setup + public void setup() throws Exception { + // Generate strings of different sizes + smallString5 = generateString(5); + smallString20 = generateString(20); + smallString50 = generateString(50); + mediumString500 = generateString(500); + mediumString2K = generateString(2 * 1024); + largeString10K = generateString(10 * 1024); + largeString100K = generateString(100 * 1024); + largeString1M = generateString(1024 * 1024); + + // Pre-serialize records for deserialization benchmarks + serializedSmall5 = bufferToArray(createStringRecord(smallString5).serializeToBuffer()); + serializedSmall20 = bufferToArray(createStringRecord(smallString20).serializeToBuffer()); + serializedSmall50 = bufferToArray(createStringRecord(smallString50).serializeToBuffer()); + serializedMedium500 = bufferToArray(createStringRecord(mediumString500).serializeToBuffer()); + serializedMedium2K = bufferToArray(createStringRecord(mediumString2K).serializeToBuffer()); + serializedLarge10K = bufferToArray(createStringRecord(largeString10K).serializeToBuffer()); + serializedLarge100K = bufferToArray(createStringRecord(largeString100K).serializeToBuffer()); + serializedLarge1M = bufferToArray(createStringRecord(largeString1M).serializeToBuffer()); + } + + private String generateString(int length) { + StringBuilder sb = new StringBuilder(length); + String chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789 "; + for (int i = 0; i < length; i++) { + sb.append(chars.charAt(i % chars.length())); + } + return sb.toString(); + } + + private ImprintRecord createStringRecord(String value) throws Exception { + return ImprintRecord.builder(SCHEMA_ID) + .field(1, value) + .build(); + } + + private String extractString(Value value) { + if (value instanceof Value.StringValue) { + return ((Value.StringValue) value).getValue(); + } else if (value instanceof Value.StringBufferValue) { + return ((Value.StringBufferValue) value).getValue(); + } + return null; + } + + private byte[] bufferToArray(ByteBuffer buffer) { + byte[] array = new byte[buffer.remaining()]; + buffer.duplicate().get(array); + return array; + } + + // Serialization benchmarks + + @Benchmark + public ByteBuffer serializeSmallString5() throws Exception { + return createStringRecord(smallString5).serializeToBuffer(); + } + + @Benchmark + public ByteBuffer serializeSmallString20() throws Exception { + return createStringRecord(smallString20).serializeToBuffer(); + } + + @Benchmark + public ByteBuffer serializeSmallString50() throws Exception { + return createStringRecord(smallString50).serializeToBuffer(); + } + + @Benchmark + public ByteBuffer serializeMediumString500() throws Exception { + return createStringRecord(mediumString500).serializeToBuffer(); + } + + @Benchmark + public ByteBuffer serializeMediumString2K() throws Exception { + return createStringRecord(mediumString2K).serializeToBuffer(); + } + + @Benchmark + public ByteBuffer serializeLargeString10K() throws Exception { + return createStringRecord(largeString10K).serializeToBuffer(); + } + + @Benchmark + public ByteBuffer serializeLargeString100K() throws Exception { + return createStringRecord(largeString100K).serializeToBuffer(); + } + + @Benchmark + public ByteBuffer serializeLargeString1M() throws Exception { + return createStringRecord(largeString1M).serializeToBuffer(); + } + + // Deserialization benchmarks + + @Benchmark + public ImprintRecord deserializeSmallString5() throws Exception { + return ImprintRecord.deserialize(serializedSmall5); + } + + @Benchmark + public ImprintRecord deserializeSmallString20() throws Exception { + return ImprintRecord.deserialize(serializedSmall20); + } + + @Benchmark + public ImprintRecord deserializeSmallString50() throws Exception { + return ImprintRecord.deserialize(serializedSmall50); + } + + @Benchmark + public ImprintRecord deserializeMediumString500() throws Exception { + return ImprintRecord.deserialize(serializedMedium500); + } + + @Benchmark + public ImprintRecord deserializeMediumString2K() throws Exception { + return ImprintRecord.deserialize(serializedMedium2K); + } + + @Benchmark + public ImprintRecord deserializeLargeString10K() throws Exception { + return ImprintRecord.deserialize(serializedLarge10K); + } + + @Benchmark + public ImprintRecord deserializeLargeString100K() throws Exception { + return ImprintRecord.deserialize(serializedLarge100K); + } + + @Benchmark + public ImprintRecord deserializeLargeString1M() throws Exception { + return ImprintRecord.deserialize(serializedLarge1M); + } + + // String access benchmarks + + @Benchmark + public String accessSmallString5() throws Exception { + ImprintRecord record = ImprintRecord.deserialize(serializedSmall5); + return record.getValue(1).map(this::extractString).orElse(null); + } + + @Benchmark + public String accessMediumString500() throws Exception { + ImprintRecord record = ImprintRecord.deserialize(serializedMedium500); + return record.getValue(1).map(this::extractString).orElse(null); + } + + @Benchmark + public String accessLargeString100K() throws Exception { + ImprintRecord record = ImprintRecord.deserialize(serializedLarge100K); + return record.getValue(1).map(this::extractString).orElse(null); + } + + // Raw bytes access benchmarks (zero-copy) + + @Benchmark + public ByteBuffer getRawBytesSmallString5() throws Exception { + ImprintRecord record = ImprintRecord.deserialize(serializedSmall5); + return record.getRawBytes(1).orElse(null); + } + + @Benchmark + public ByteBuffer getRawBytesMediumString500() throws Exception { + ImprintRecord record = ImprintRecord.deserialize(serializedMedium500); + return record.getRawBytes(1).orElse(null); + } + + @Benchmark + public ByteBuffer getRawBytesLargeString100K() throws Exception { + ImprintRecord record = ImprintRecord.deserialize(serializedLarge100K); + return record.getRawBytes(1).orElse(null); + } + + // Size measurement benchmarks + + @Benchmark + public int measureSmallString5Size() throws Exception { + return createStringRecord(smallString5).serializeToBuffer().remaining(); + } + + @Benchmark + public int measureMediumString500Size() throws Exception { + return createStringRecord(mediumString500).serializeToBuffer().remaining(); + } + + @Benchmark + public int measureLargeString100KSize() throws Exception { + return createStringRecord(largeString100K).serializeToBuffer().remaining(); + } + + public static void main(String[] args) throws Exception { + Options opt = new OptionsBuilder() + .include(StringBenchmark.class.getSimpleName()) + .build(); + new Runner(opt).run(); + } +} \ No newline at end of file From 2853e3fd18b85eda7e14c25407670ecc8f22a5b2 Mon Sep 17 00:00:00 2001 From: expand3d <> Date: Sun, 1 Jun 2025 23:44:13 -0400 Subject: [PATCH 12/34] fix map serialization error in benchmark test and streamline ci file to remove a bunch of stuff --- .github/workflows/ci.yml | 64 +------------------ .../benchmark/SerializationBenchmark.java | 4 +- 2 files changed, 4 insertions(+), 64 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2e906c6..0ad6f6b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,7 +25,7 @@ jobs: distribution: 'temurin' - name: Cache Gradle dependencies - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: | ~/.gradle/caches @@ -34,68 +34,8 @@ jobs: restore-keys: | ${{ runner.os }}-gradle- - - name: Make gradlew executable - run: chmod +x ./gradlew - - - name: Validate Gradle wrapper - uses: gradle/wrapper-validation-action@v3 - - name: Run tests run: ./gradlew test - name: Run build - run: ./gradlew build - - - name: Upload test results - uses: actions/upload-artifact@v4 - if: always() - with: - name: test-results-java-${{ matrix.java-version }} - path: build/test-results/test/ - - - name: Upload build reports - uses: actions/upload-artifact@v4 - if: always() - with: - name: build-reports-java-${{ matrix.java-version }} - path: build/reports/ - - benchmark: - runs-on: ubuntu-latest - needs: test - if: github.event_name == 'pull_request' - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up JDK 17 - uses: actions/setup-java@v4 - with: - java-version: 17 - distribution: 'temurin' - - - name: Cache Gradle dependencies - uses: actions/cache@v3 - with: - path: | - ~/.gradle/caches - ~/.gradle/wrapper - key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} - restore-keys: | - ${{ runner.os }}-gradle- - - - name: Make gradlew executable - run: chmod +x ./gradlew - - - name: Validate Gradle wrapper - uses: gradle/wrapper-validation-action@v3 - - - name: Run quick benchmark - run: ./gradlew jmh -Pjmh.fork=1 -Pjmh.warmupIterations=1 -Pjmh.iterations=1 - - - name: Upload benchmark results - uses: actions/upload-artifact@v4 - with: - name: benchmark-results - path: build/results/jmh/ \ No newline at end of file + run: ./gradlew build \ No newline at end of file diff --git a/src/jmh/java/com/imprint/benchmark/SerializationBenchmark.java b/src/jmh/java/com/imprint/benchmark/SerializationBenchmark.java index 2544b88..3275843 100644 --- a/src/jmh/java/com/imprint/benchmark/SerializationBenchmark.java +++ b/src/jmh/java/com/imprint/benchmark/SerializationBenchmark.java @@ -119,11 +119,11 @@ private ImprintRecord createMediumRecord() throws Exception { ); writer.addField(6, Value.fromArray(tags)); - // Add map field + // Add map field (all string values for consistency) var metadata = new HashMap(); metadata.put(MapKey.fromString("manufacturer"), Value.fromString("TechCorp")); metadata.put(MapKey.fromString("model"), Value.fromString("TC-2024")); - metadata.put(MapKey.fromString("year"), Value.fromInt32(2024)); + metadata.put(MapKey.fromString("year"), Value.fromString("2024")); writer.addField(7, Value.fromMap(metadata)); // Add more fields for medium size From 3a5a113ff093933da07e4c9c5fded1c8109b7fca Mon Sep 17 00:00:00 2001 From: expand3d <> Date: Sun, 1 Jun 2025 23:45:45 -0400 Subject: [PATCH 13/34] Add execute permissions back for gradlew in CI --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0ad6f6b..d4c8bde 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,6 +34,9 @@ jobs: restore-keys: | ${{ runner.os }}-gradle- + - name: Make gradlew executable + run: chmod +x ./gradlew + - name: Run tests run: ./gradlew test From 50a288b19e0eb142f7dcd1b8d0d57b334b540b07 Mon Sep 17 00:00:00 2001 From: expand3d <> Date: Mon, 2 Jun 2025 01:18:43 -0400 Subject: [PATCH 14/34] Add some more string based performance benchmarks and try to make string deserialization a bit faster --- .../imprint/benchmark/StringBenchmark.java | 32 ++++++++++ .../java/com/imprint/core/ImprintRecord.java | 4 +- .../java/com/imprint/types/TypeHandler.java | 9 ++- src/main/java/com/imprint/types/Value.java | 58 +++++++++++++------ 4 files changed, 78 insertions(+), 25 deletions(-) diff --git a/src/jmh/java/com/imprint/benchmark/StringBenchmark.java b/src/jmh/java/com/imprint/benchmark/StringBenchmark.java index 1891251..dda7f1c 100644 --- a/src/jmh/java/com/imprint/benchmark/StringBenchmark.java +++ b/src/jmh/java/com/imprint/benchmark/StringBenchmark.java @@ -44,6 +44,10 @@ public class StringBenchmark { private byte[] serializedLarge10K; private byte[] serializedLarge100K; private byte[] serializedLarge1M; + + private ImprintRecord preDeserializedSmall5; + private ImprintRecord preDeserializedMedium500; + private ImprintRecord preDeserializedLarge100K; @Setup public void setup() throws Exception { @@ -66,6 +70,10 @@ public void setup() throws Exception { serializedLarge10K = bufferToArray(createStringRecord(largeString10K).serializeToBuffer()); serializedLarge100K = bufferToArray(createStringRecord(largeString100K).serializeToBuffer()); serializedLarge1M = bufferToArray(createStringRecord(largeString1M).serializeToBuffer()); + + preDeserializedSmall5 = ImprintRecord.deserialize(serializedSmall5); + preDeserializedMedium500 = ImprintRecord.deserialize(serializedMedium500); + preDeserializedLarge100K = ImprintRecord.deserialize(serializedLarge100K); } private String generateString(int length) { @@ -238,6 +246,30 @@ public int measureMediumString500Size() throws Exception { public int measureLargeString100KSize() throws Exception { return createStringRecord(largeString100K).serializeToBuffer().remaining(); } + + // Pure string access benchmarks (no record deserialization overhead) + @Benchmark + public String pureStringAccessSmall5() throws Exception { + return preDeserializedSmall5.getValue(1).map(this::extractString).orElse(null); + } + + @Benchmark + public String pureStringAccessMedium500() throws Exception { + return preDeserializedMedium500.getValue(1).map(this::extractString).orElse(null); + } + + @Benchmark + public String pureStringAccessLarge100K() throws Exception { + return preDeserializedLarge100K.getValue(1).map(this::extractString).orElse(null); + } + + // Test cached vs uncached access + @Benchmark + public String cachedStringAccessSmall5() throws Exception { + // Second access should hit cache + preDeserializedSmall5.getValue(1).map(this::extractString).orElse(null); + return preDeserializedSmall5.getValue(1).map(this::extractString).orElse(null); + } public static void main(String[] args) throws Exception { Options opt = new OptionsBuilder() diff --git a/src/main/java/com/imprint/core/ImprintRecord.java b/src/main/java/com/imprint/core/ImprintRecord.java index b7ed224..d9d5659 100644 --- a/src/main/java/com/imprint/core/ImprintRecord.java +++ b/src/main/java/com/imprint/core/ImprintRecord.java @@ -114,14 +114,14 @@ public static ImprintRecordBuilder builder(int fieldspaceId, int schemaHash) { } /** - * Deserialize a record from bytes. + * Deserialize a record from bytes through an array backed ByteBuffer. */ public static ImprintRecord deserialize(byte[] bytes) throws ImprintException { return deserialize(ByteBuffer.wrap(bytes)); } /** - * Deserialize a record from a ByteBuffer (zero-copy when possible). + * Deserialize a record from a ByteBuffer. */ public static ImprintRecord deserialize(ByteBuffer buffer) throws ImprintException { buffer = buffer.duplicate().order(ByteOrder.LITTLE_ENDIAN); diff --git a/src/main/java/com/imprint/types/TypeHandler.java b/src/main/java/com/imprint/types/TypeHandler.java index 4b5830a..07ef7a1 100644 --- a/src/main/java/com/imprint/types/TypeHandler.java +++ b/src/main/java/com/imprint/types/TypeHandler.java @@ -4,7 +4,6 @@ import com.imprint.util.VarInt; import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; /** * Interface for handling type-specific serialization, deserialization, and size estimation. @@ -270,7 +269,7 @@ public void serialize(Value value, ByteBuffer buffer) { buffer.put(stringBuffer); } else { Value.StringValue stringValue = (Value.StringValue) value; - byte[] stringBytes = stringValue.getValue().getBytes(StandardCharsets.UTF_8); + byte[] stringBytes = stringValue.getUtf8Bytes(); // Use cached UTF-8 bytes VarInt.encode(stringBytes.length, buffer); buffer.put(stringBytes); } @@ -283,9 +282,9 @@ public int estimateSize(Value value) { int length = bufferValue.getBuffer().remaining(); return VarInt.encodedLength(length) + length; } else { - String str = ((Value.StringValue) value).getValue(); - int utf8Length = str.getBytes(StandardCharsets.UTF_8).length; - return VarInt.encodedLength(utf8Length) + utf8Length; + Value.StringValue stringValue = (Value.StringValue) value; + byte[] utf8Bytes = stringValue.getUtf8Bytes(); // Use cached UTF-8 bytes + return VarInt.encodedLength(utf8Bytes.length) + utf8Bytes.length; } } diff --git a/src/main/java/com/imprint/types/Value.java b/src/main/java/com/imprint/types/Value.java index 4710ec5..64eae91 100644 --- a/src/main/java/com/imprint/types/Value.java +++ b/src/main/java/com/imprint/types/Value.java @@ -270,15 +270,26 @@ public String toString() { } // String Value (String-based) - @Getter @EqualsAndHashCode(callSuper = false) public static class StringValue extends Value { + @Getter private final String value; + private volatile byte[] cachedUtf8Bytes; // Cache UTF-8 encoding public StringValue(String value) { this.value = Objects.requireNonNull(value, "String cannot be null"); } + public byte[] getUtf8Bytes() { + byte[] cached = cachedUtf8Bytes; + if (cached == null) { + // Multiple threads may compute this - that's OK since it's idempotent + cached = value.getBytes(StandardCharsets.UTF_8); + cachedUtf8Bytes = cached; // Benign race - last writer wins + } + return cached; // Return our computed value, not re-read from volatile field + } + @Override public TypeCode getTypeCode() { return TypeCode.STRING; } @@ -288,35 +299,46 @@ public String toString() { } } - // String Value (ByteBuffer-based, zero-copy) + // String Value (ByteBuffer-based) public static class StringBufferValue extends Value { private final ByteBuffer value; private volatile String cachedString; // lazy decode - + public StringBufferValue(ByteBuffer value) { this.value = value.asReadOnlyBuffer(); // zero-copy read-only view } - + public String getValue() { - if (cachedString == null) { - synchronized (this) { - if (cachedString == null) { - var array = new byte[value.remaining()]; - value.duplicate().get(array); - cachedString = new String(array, StandardCharsets.UTF_8); - } - } + String result = cachedString; + if (result == null) { + // Simple, fast decoding - no thread-local overhead + result = decodeUtf8(); + cachedString = result; } - return cachedString; + return result; } - + + private String decodeUtf8() { + // Fast path: zero-copy for array-backed ByteBuffers + if (value.hasArray() && value.arrayOffset() == 0) { + return new String(value.array(), value.position(), + value.remaining(), StandardCharsets.UTF_8); + } + + // Fallback path - should be impossible since deserialize uses wrap() to create an array-backed ByteBuffer. + // Allocation required for direct ByteBuffers since Java's String API doesn't provide ByteBuffer constructors + var array = new byte[value.remaining()]; + value.duplicate().get(array); + return new String(array, StandardCharsets.UTF_8); + } + public ByteBuffer getBuffer() { return value.duplicate(); // zero-copy view } - + @Override public TypeCode getTypeCode() { return TypeCode.STRING; } - + @Override public boolean equals(Object obj) { if (this == obj) return true; @@ -331,12 +353,12 @@ public boolean equals(Object obj) { } return false; } - + @Override public int hashCode() { return getValue().hashCode(); // Use string hash for consistency } - + @Override public String toString() { return "\"" + getValue() + "\""; From 43cab28d2b532be7b03e74533a7e1c1c1d7fdc8c Mon Sep 17 00:00:00 2001 From: Brent Johnson Date: Tue, 3 Jun 2025 18:28:49 -0400 Subject: [PATCH 15/34] second main commit to address initial commits A full list of enhancements can be found here - https://github.com/imprint-serde/imprint-java/issues/3 --- .../benchmark/ComparisonBenchmark.java | 69 ++-- .../benchmark/FieldAccessBenchmark.java | 4 +- .../com/imprint/benchmark/MergeBenchmark.java | 4 +- .../imprint/benchmark/StringBenchmark.java | 67 ++- src/main/java/com/imprint/Constants.java | 4 +- .../java/com/imprint/core/ImprintRecord.java | 168 ++------ .../imprint/core/ImprintRecordBuilder.java | 19 +- .../java/com/imprint/core/ImprintWriter.java | 155 +------ src/main/java/com/imprint/core/SchemaId.java | 2 +- src/main/java/com/imprint/types/TypeCode.java | 4 +- .../java/com/imprint/types/TypeHandler.java | 386 ++++++++++++++++-- src/main/java/com/imprint/types/Value.java | 47 ++- src/main/java/com/imprint/util/VarInt.java | 20 + .../imprint/ByteBufferIntegrationTest.java | 157 +++++-- .../java/com/imprint/ComprehensiveTest.java | 208 ---------- .../java/com/imprint/IntegrationTest.java | 32 +- .../com/imprint/benchmark/ProfilerTest.java | 18 +- .../core/ImprintRecordBuilderTest.java | 234 ----------- .../com/imprint/core/ImprintRecordTest.java | 76 ++-- .../java/com/imprint/types/ValueTest.java | 115 +++++- 20 files changed, 871 insertions(+), 918 deletions(-) delete mode 100644 src/test/java/com/imprint/ComprehensiveTest.java delete mode 100644 src/test/java/com/imprint/core/ImprintRecordBuilderTest.java diff --git a/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java b/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java index 1293478..152bb6d 100644 --- a/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java +++ b/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java @@ -74,7 +74,7 @@ public void serializeJackson(Blackhole bh) throws Exception { } @Benchmark - public void serializeKryo(Blackhole bh) throws Exception { + public void serializeKryo(Blackhole bh) { byte[] result = serializeWithKryo(testData); bh.consume(result); } @@ -102,49 +102,36 @@ public void deserializeKryo(Blackhole bh) { } // ===== FIELD ACCESS BENCHMARKS ===== + // Tests accessing a single field near the end of a large record + // This showcases Imprint's O(1) directory lookup vs sequential deserialization @Benchmark - public void fieldAccessImprint(Blackhole bh) throws Exception { + public void singleFieldAccessImprint(Blackhole bh) throws Exception { ImprintRecord record = ImprintRecord.deserialize(imprintBytes.duplicate()); - // Access multiple fields without full deserialization - var id = record.getValue(1); - var name = record.getValue(2); - var price = record.getValue(3); - var active = record.getValue(4); - var category = record.getValue(5); - - bh.consume(id); - bh.consume(name); - bh.consume(price); - bh.consume(active); - bh.consume(category); + // Access field 15 directly via directory lookup - O(1) + var field15 = record.getValue(15); + bh.consume(field15); } @Benchmark - public void fieldAccessJackson(Blackhole bh) throws Exception { - // Jackson requires full deserialization to access fields + public void singleFieldAccessJackson(Blackhole bh) throws Exception { + // Jackson must deserialize entire object to access any field TestRecord record = jackson.readValue(jacksonBytes, TestRecord.class); - bh.consume(record.id); - bh.consume(record.name); - bh.consume(record.price); - bh.consume(record.active); - bh.consume(record.category); + // Access field15 equivalent (extraData[4]) after full deserialization + bh.consume(record.extraData.get(4)); } @Benchmark - public void fieldAccessKryo(Blackhole bh) { - // Kryo requires full deserialization to access fields + public void singleFieldAccessKryo(Blackhole bh) { + // Kryo must deserialize entire object to access any field Input input = new Input(new ByteArrayInputStream(kryoBytes)); TestRecord record = kryo.readObject(input, TestRecord.class); input.close(); - bh.consume(record.id); - bh.consume(record.name); - bh.consume(record.price); - bh.consume(record.active); - bh.consume(record.category); + // Access field15 equivalent (extraData[4]) after full deserialization + bh.consume(record.extraData.get(4)); } // ===== SIZE COMPARISON ===== @@ -162,7 +149,7 @@ public void measureJacksonSize(Blackhole bh) throws Exception { } @Benchmark - public void measureKryoSize(Blackhole bh) throws Exception { + public void measureKryoSize(Blackhole bh) { byte[] serialized = serializeWithKryo(testData); bh.consume(serialized.length); } @@ -196,7 +183,7 @@ public void mergeJackson(Blackhole bh) throws Exception { } @Benchmark - public void mergeKryo(Blackhole bh) throws Exception { + public void mergeKryo(Blackhole bh) { // Kryo merge requires full deserialization + merge + serialization Input input1 = new Input(new ByteArrayInputStream(kryoBytes)); var record1 = kryo.readObject(input1, TestRecord.class); @@ -237,6 +224,11 @@ private ByteBuffer serializeWithImprint(TestRecord data) throws Exception { } writer.addField(7, Value.fromMap(metadataMap)); + // Add extra fields (8-20) to create a larger record + for (int i = 0; i < data.extraData.size(); i++) { + writer.addField(8 + i, Value.fromString(data.extraData.get(i))); + } + return writer.build().serializeToBuffer(); } @@ -268,8 +260,8 @@ private void copyFieldsToWriter(ImprintRecord record, ImprintWriter writer, Set< int fieldId = entry.getId(); if (!usedFieldIds.contains(fieldId)) { var value = record.getValue(fieldId); - if (value.isPresent()) { - writer.addField(fieldId, value.get()); + if (value != null) { + writer.addField(fieldId, value); usedFieldIds.add(fieldId); } } @@ -309,6 +301,12 @@ var record = new TestRecord(); record.metadata.put("model", "TC-2024"); record.metadata.put("warranty", "2 years"); + // Add extra data to create a larger record (fields 8-20) + record.extraData = new ArrayList<>(); + for (int i = 0; i < 13; i++) { + record.extraData.add("extraField" + i + "_value_" + (1000 + i)); + } + return record; } @@ -326,6 +324,12 @@ var record = new TestRecord(); record.metadata.put("vendor", "SoftCorp"); record.metadata.put("version", "2.1"); + // Add extra data to match the structure + record.extraData = new ArrayList<>(); + for (int i = 0; i < 13; i++) { + record.extraData.add("extraField" + i + "_value2_" + (2000 + i)); + } + return record; } @@ -338,6 +342,7 @@ public static class TestRecord { public String category; public List tags = new ArrayList<>(); public Map metadata = new HashMap<>(); + public List extraData = new ArrayList<>(); // Fields 8-20 for large record test public TestRecord() {} // Required for deserialization } diff --git a/src/jmh/java/com/imprint/benchmark/FieldAccessBenchmark.java b/src/jmh/java/com/imprint/benchmark/FieldAccessBenchmark.java index f3abb7e..1ead21f 100644 --- a/src/jmh/java/com/imprint/benchmark/FieldAccessBenchmark.java +++ b/src/jmh/java/com/imprint/benchmark/FieldAccessBenchmark.java @@ -200,7 +200,9 @@ private ImprintRecord simulateProject(ImprintRecord source, int[] fieldIds) thro for (int fieldId : fieldIds) { var value = source.getValue(fieldId); - value.ifPresent(value1 -> writer.addField(fieldId, value1)); + if (value != null) { + writer.addField(fieldId, value); + } } return writer.build(); diff --git a/src/jmh/java/com/imprint/benchmark/MergeBenchmark.java b/src/jmh/java/com/imprint/benchmark/MergeBenchmark.java index 5c52908..f93092a 100644 --- a/src/jmh/java/com/imprint/benchmark/MergeBenchmark.java +++ b/src/jmh/java/com/imprint/benchmark/MergeBenchmark.java @@ -100,8 +100,8 @@ private void copyFieldsToWriter(ImprintRecord record, ImprintWriter writer, Set< int fieldId = entry.getId(); if (!usedFieldIds.contains(fieldId)) { var value = record.getValue(fieldId); - if (value.isPresent()) { - writer.addField(fieldId, value.get()); + if (value != null) { + writer.addField(fieldId, value); usedFieldIds.add(fieldId); } } diff --git a/src/jmh/java/com/imprint/benchmark/StringBenchmark.java b/src/jmh/java/com/imprint/benchmark/StringBenchmark.java index dda7f1c..e156c8c 100644 --- a/src/jmh/java/com/imprint/benchmark/StringBenchmark.java +++ b/src/jmh/java/com/imprint/benchmark/StringBenchmark.java @@ -5,7 +5,6 @@ import com.imprint.types.Value; import org.openjdk.jmh.annotations.*; import org.openjdk.jmh.runner.Runner; -import org.openjdk.jmh.runner.options.Options; import org.openjdk.jmh.runner.options.OptionsBuilder; import java.nio.ByteBuffer; @@ -15,8 +14,8 @@ @OutputTimeUnit(TimeUnit.NANOSECONDS) @State(Scope.Benchmark) @Fork(1) -@Warmup(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS) -@Measurement(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS) +@Warmup(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 10, time = 1, timeUnit = TimeUnit.SECONDS) public class StringBenchmark { private static final SchemaId SCHEMA_ID = new SchemaId(1, 42); @@ -195,19 +194,22 @@ public ImprintRecord deserializeLargeString1M() throws Exception { @Benchmark public String accessSmallString5() throws Exception { ImprintRecord record = ImprintRecord.deserialize(serializedSmall5); - return record.getValue(1).map(this::extractString).orElse(null); + Value value = record.getValue(1); + return value != null ? extractString(value) : null; } @Benchmark public String accessMediumString500() throws Exception { ImprintRecord record = ImprintRecord.deserialize(serializedMedium500); - return record.getValue(1).map(this::extractString).orElse(null); + Value value = record.getValue(1); + return value != null ? extractString(value) : null; } @Benchmark public String accessLargeString100K() throws Exception { ImprintRecord record = ImprintRecord.deserialize(serializedLarge100K); - return record.getValue(1).map(this::extractString).orElse(null); + Value value = record.getValue(1); + return value != null ? extractString(value) : null; } // Raw bytes access benchmarks (zero-copy) @@ -215,19 +217,19 @@ public String accessLargeString100K() throws Exception { @Benchmark public ByteBuffer getRawBytesSmallString5() throws Exception { ImprintRecord record = ImprintRecord.deserialize(serializedSmall5); - return record.getRawBytes(1).orElse(null); + return record.getRawBytes(1); } @Benchmark public ByteBuffer getRawBytesMediumString500() throws Exception { ImprintRecord record = ImprintRecord.deserialize(serializedMedium500); - return record.getRawBytes(1).orElse(null); + return record.getRawBytes(1); } @Benchmark public ByteBuffer getRawBytesLargeString100K() throws Exception { ImprintRecord record = ImprintRecord.deserialize(serializedLarge100K); - return record.getRawBytes(1).orElse(null); + return record.getRawBytes(1); } // Size measurement benchmarks @@ -250,30 +252,63 @@ public int measureLargeString100KSize() throws Exception { // Pure string access benchmarks (no record deserialization overhead) @Benchmark public String pureStringAccessSmall5() throws Exception { - return preDeserializedSmall5.getValue(1).map(this::extractString).orElse(null); + Value value = preDeserializedSmall5.getValue(1); + return value != null ? extractString(value) : null; } @Benchmark public String pureStringAccessMedium500() throws Exception { - return preDeserializedMedium500.getValue(1).map(this::extractString).orElse(null); + Value value = preDeserializedMedium500.getValue(1); + return value != null ? extractString(value) : null; } @Benchmark public String pureStringAccessLarge100K() throws Exception { - return preDeserializedLarge100K.getValue(1).map(this::extractString).orElse(null); + Value value = preDeserializedLarge100K.getValue(1); + return value != null ? extractString(value) : null; } // Test cached vs uncached access @Benchmark public String cachedStringAccessSmall5() throws Exception { // Second access should hit cache - preDeserializedSmall5.getValue(1).map(this::extractString).orElse(null); - return preDeserializedSmall5.getValue(1).map(this::extractString).orElse(null); + Value value1 = preDeserializedSmall5.getValue(1); + String result1 = value1 != null ? extractString(value1) : null; + Value value2 = preDeserializedSmall5.getValue(1); + return value2 != null ? extractString(value2) : null; } public static void main(String[] args) throws Exception { - Options opt = new OptionsBuilder() - .include(StringBenchmark.class.getSimpleName()) + runDeserializationOnly(); + } + + public static void runAll() throws Exception { + var opt = new OptionsBuilder() + .include(StringBenchmark.class.getSimpleName()) + .build(); + new Runner(opt).run(); + } + + /** + * Run only string deserialization benchmarks to measure the impact of + * ThreadLocal buffer pool optimization and fast/fallback path performance. + */ + public static void runDeserializationOnly() throws Exception { + var opt = new OptionsBuilder() + .include(StringBenchmark.class.getSimpleName() + ".*deserialize.*") // Only deserialize methods + .forks(0) // Run in same JVM to avoid serialization issues + .build(); + new Runner(opt).run(); + } + + /** + * Run only pure string access benchmarks (no record deserialization overhead) + * to isolate string decode performance with ThreadLocal buffer optimization. + */ + public static void runStringAccessOnly() throws Exception { + var opt = new OptionsBuilder() + .include(StringBenchmark.class.getSimpleName() + ".*(pureStringAccess|cachedStringAccess).*") // Only pure string access methods + .forks(0) // Run in same JVM to avoid serialization issues .build(); new Runner(opt).run(); } diff --git a/src/main/java/com/imprint/Constants.java b/src/main/java/com/imprint/Constants.java index 78b91a0..3c84a28 100644 --- a/src/main/java/com/imprint/Constants.java +++ b/src/main/java/com/imprint/Constants.java @@ -2,9 +2,7 @@ public final class Constants { public static final byte MAGIC = 0x49; - public static final byte VERSION = 0x02; + public static final byte VERSION = 0x01; public static final int HEADER_BYTES = 15; public static final int DIR_ENTRY_BYTES = 7; - - private Constants() {} } diff --git a/src/main/java/com/imprint/core/ImprintRecord.java b/src/main/java/com/imprint/core/ImprintRecord.java index d9d5659..1915707 100644 --- a/src/main/java/com/imprint/core/ImprintRecord.java +++ b/src/main/java/com/imprint/core/ImprintRecord.java @@ -1,10 +1,8 @@ - package com.imprint.core; import com.imprint.Constants; import com.imprint.error.ErrorType; import com.imprint.error.ImprintException; -import com.imprint.types.MapKey; import com.imprint.types.TypeCode; import com.imprint.types.Value; import com.imprint.util.VarInt; @@ -15,8 +13,12 @@ import java.util.*; /** - * An Imprint record containing a header, optional field directory, and payload. + * An Imprint record containing a header, field directory, and payload. * Uses ByteBuffer for zero-copy operations to achieve low latency. + * + *

Performance Note: All ByteBuffers should be array-backed + * (hasArray() == true) for optimal zero-copy performance. Direct buffers + * may cause performance degradation.

*/ @Getter public final class ImprintRecord { @@ -24,6 +26,11 @@ public final class ImprintRecord { private final List directory; private final ByteBuffer payload; // Read-only view for zero-copy + /** + * Creates a new ImprintRecord. + * + * @param payload the payload buffer. Should be array-backed for optimal performance. + */ public ImprintRecord(Header header, List directory, ByteBuffer payload) { this.header = Objects.requireNonNull(header, "Header cannot be null"); this.directory = List.copyOf(Objects.requireNonNull(directory, "Directory cannot be null")); @@ -32,39 +39,44 @@ public ImprintRecord(Header header, List directory, ByteBuffer p /** * Get a value by field ID, deserializing it on demand. + * Returns null if the field is not found. */ - public Optional getValue(int fieldId) throws ImprintException { - // Binary search for the field ID without allocation - int index = findDirectoryIndex(fieldId); - if (index < 0) return Optional.empty(); + public Value getValue(int fieldId) throws ImprintException { + var fieldBuffer = getFieldBuffer(fieldId); + if (fieldBuffer == null) return null; - var entry = directory.get(index); - int startOffset = entry.getOffset(); - int endOffset = (index + 1 < directory.size()) ? - directory.get(index + 1).getOffset() : payload.remaining(); - - var valueBytes = payload.duplicate(); - valueBytes.position(startOffset).limit(endOffset); - var value = deserializeValue(entry.getTypeCode(), valueBytes.slice()); - return Optional.of(value); + var entry = directory.get(findDirectoryIndex(fieldId)); + return deserializeValue(entry.getTypeCode(), fieldBuffer); } /** * Get the raw bytes for a field without deserializing. - * Returns a zero-copy ByteBuffer view. + * Returns a zero-copy ByteBuffer view, or null if field not found. + */ + public ByteBuffer getRawBytes(int fieldId) { + var fieldBuffer = getFieldBuffer(fieldId); + return fieldBuffer != null ? fieldBuffer.asReadOnlyBuffer() : null; + } + + /** + * Get a ByteBuffer view of a field's data. + * Returns null if the field is not found. */ - public Optional getRawBytes(int fieldId) { + private ByteBuffer getFieldBuffer(int fieldId) { int index = findDirectoryIndex(fieldId); - if (index < 0) return Optional.empty(); - + if (index < 0) return null; + var entry = directory.get(index); int startOffset = entry.getOffset(); int endOffset = (index + 1 < directory.size()) ? - directory.get(index + 1).getOffset() : payload.remaining(); + directory.get(index + 1).getOffset() : payload.limit(); var fieldBuffer = payload.duplicate(); + if (startOffset > payload.limit() || endOffset > payload.limit() || startOffset > endOffset) { + return null; + } fieldBuffer.position(startOffset).limit(endOffset); - return Optional.of(fieldBuffer.slice().asReadOnlyBuffer()); + return fieldBuffer.slice(); } /** @@ -122,6 +134,9 @@ public static ImprintRecord deserialize(byte[] bytes) throws ImprintException { /** * Deserialize a record from a ByteBuffer. + * + * @param buffer the buffer to deserialize from. Must be array-backed + * (buffer.hasArray() == true) for optimal zero-copy performance. */ public static ImprintRecord deserialize(ByteBuffer buffer) throws ImprintException { buffer = buffer.duplicate().order(ByteOrder.LITTLE_ENDIAN); @@ -172,7 +187,7 @@ private int findDirectoryIndex(int fieldId) { return -(low + 1); // field not found, return insertion point } - private int estimateSerializedSize() { + public int estimateSerializedSize() { int size = Constants.HEADER_BYTES; // header size += VarInt.encodedLength(directory.size()); // directory count size += directory.size() * Constants.DIR_ENTRY_BYTES; // directory entries @@ -184,7 +199,7 @@ private void serializeHeader(ByteBuffer buffer) { buffer.put(Constants.MAGIC); buffer.put(Constants.VERSION); buffer.put(header.getFlags().getValue()); - buffer.putInt(header.getSchemaId().getFieldspaceId()); + buffer.putInt(header.getSchemaId().getFieldSpaceId()); buffer.putInt(header.getSchemaId().getSchemaHash()); buffer.putInt(header.getPayloadSize()); } @@ -236,10 +251,9 @@ private static DirectoryEntry deserializeDirectoryEntry(ByteBuffer buffer) throw } private Value deserializeValue(TypeCode typeCode, ByteBuffer buffer) throws ImprintException { - // Buffer is already positioned and limited correctly - buffer = buffer.duplicate().order(ByteOrder.LITTLE_ENDIAN); + ByteBuffer valueSpecificBuffer = buffer.duplicate(); + valueSpecificBuffer.order(ByteOrder.LITTLE_ENDIAN); - // Use TypeHandler for simple types switch (typeCode) { case NULL: case BOOL: @@ -249,17 +263,11 @@ private Value deserializeValue(TypeCode typeCode, ByteBuffer buffer) throws Impr case FLOAT64: case BYTES: case STRING: - return typeCode.getHandler().deserialize(buffer); - case ARRAY: - return deserializeArray(buffer); - case MAP: - return deserializeMap(buffer); - + return typeCode.getHandler().deserialize(valueSpecificBuffer); case ROW: - var remainingBuffer = buffer.slice(); - var nestedRecord = deserialize(remainingBuffer); + var nestedRecord = deserialize(valueSpecificBuffer); return Value.fromRow(nestedRecord); default: @@ -267,96 +275,6 @@ private Value deserializeValue(TypeCode typeCode, ByteBuffer buffer) throws Impr } } - private Value deserializeArray(ByteBuffer buffer) throws ImprintException { - VarInt.DecodeResult lengthResult = VarInt.decode(buffer); - int length = lengthResult.getValue(); - - if (length == 0) { - return Value.fromArray(Collections.emptyList()); - } - - var elementType = TypeCode.fromByte(buffer.get()); - var elements = new ArrayList(length); - - for (int i = 0; i < length; i++) { - var elementBytes = readValueBytes(elementType, buffer); - var element = deserializeValue(elementType, elementBytes); - elements.add(element); - } - - return Value.fromArray(elements); - } - - private Value deserializeMap(ByteBuffer buffer) throws ImprintException { - VarInt.DecodeResult lengthResult = VarInt.decode(buffer); - int length = lengthResult.getValue(); - - if (length == 0) { - return Value.fromMap(Collections.emptyMap()); - } - - var keyType = TypeCode.fromByte(buffer.get()); - var valueType = TypeCode.fromByte(buffer.get()); - var map = new HashMap(length); - - for (int i = 0; i < length; i++) { - // Read key - var keyBytes = readValueBytes(keyType, buffer); - var keyValue = deserializeValue(keyType, keyBytes); - var key = MapKey.fromValue(keyValue); - - // Read value - var valueBytes = readValueBytes(valueType, buffer); - var value = deserializeValue(valueType, valueBytes); - - map.put(key, value); - } - - return Value.fromMap(map); - } - - private ByteBuffer readValueBytes(TypeCode typeCode, ByteBuffer buffer) throws ImprintException { - // Use TypeHandler for simple types - switch (typeCode) { - case NULL: - case BOOL: - case INT32: - case INT64: - case FLOAT32: - case FLOAT64: - case BYTES: - case STRING: - return typeCode.getHandler().readValueBytes(buffer); - - case ARRAY: - case MAP: - case ROW: - // For complex types, return the entire remaining buffer for now - // The specific deserializer will handle parsing in the future - var remainingBuffer = buffer.slice(); - buffer.position(buffer.limit()); - return remainingBuffer.asReadOnlyBuffer(); - - default: - throw new ImprintException(ErrorType.INVALID_TYPE_CODE, "Unknown type code: " + typeCode); - } - } - - @Override - public boolean equals(Object obj) { - if (this == obj) return true; - if (obj == null || getClass() != obj.getClass()) return false; - var that = (ImprintRecord) obj; - return header.equals(that.header) && - directory.equals(that.directory) && - payload.equals(that.payload); - } - - @Override - public int hashCode() { - return Objects.hash(header, directory, payload); - } - @Override public String toString() { return String.format("ImprintRecord{header=%s, directorySize=%d, payloadSize=%d}", diff --git a/src/main/java/com/imprint/core/ImprintRecordBuilder.java b/src/main/java/com/imprint/core/ImprintRecordBuilder.java index 202bd2a..8c04bf5 100644 --- a/src/main/java/com/imprint/core/ImprintRecordBuilder.java +++ b/src/main/java/com/imprint/core/ImprintRecordBuilder.java @@ -10,11 +10,16 @@ * A fluent builder for creating ImprintRecord instances with type-safe, * developer-friendly API that eliminates boilerplate Value.fromX() calls. *

+ * Field IDs can be overwritten - calling field() with the same ID multiple times + * will replace the previous value. This allows for flexible builder patterns and + * conditional field updates. + *

* Usage: * 

  *   var record = ImprintRecord.builder(schemaId)
  *       .field(1, 42)              // int to Int32Value  
  *       .field(2, "hello")         // String to StringValue
+ *       .field(1, 100)             // overwrites field 1 with new value
  *       .field(3, 3.14)            // double to Float64Value
  *       .field(4, bytes)           // byte[] to BytesValue
  *       .field(5, true)            // boolean to BoolValue
@@ -22,6 +27,7 @@
  *       .build();
  *
*/ +@SuppressWarnings("unused") public final class ImprintRecordBuilder { private final SchemaId schemaId; private final Map fields = new TreeMap<>(); @@ -141,12 +147,17 @@ public ImprintRecord build() throws ImprintException { } // Internal helper methods + /** + * Adds or overwrites a field in the record being built. + * If a field with the given ID already exists, it will be replaced. + * + * @param id the field ID + * @param value the field value (cannot be null - use nullField() for explicit nulls) + * @return this builder for method chaining + */ private ImprintRecordBuilder addField(int id, Value value) { Objects.requireNonNull(value, "Value cannot be null - use nullField() for explicit null values"); - if (fields.containsKey(id)) { - throw new IllegalArgumentException("Field ID " + id + " already exists - field IDs must be unique"); - } - fields.put(id, value); + fields.put(id, value); // TreeMap.put() overwrites existing values return this; } diff --git a/src/main/java/com/imprint/core/ImprintWriter.java b/src/main/java/com/imprint/core/ImprintWriter.java index 39ad9ea..b1d5f53 100644 --- a/src/main/java/com/imprint/core/ImprintWriter.java +++ b/src/main/java/com/imprint/core/ImprintWriter.java @@ -2,14 +2,13 @@ import com.imprint.error.ErrorType; import com.imprint.error.ImprintException; -import com.imprint.types.MapKey; import com.imprint.types.Value; -import com.imprint.util.VarInt; import java.nio.ByteBuffer; import java.nio.ByteOrder; -import java.nio.charset.StandardCharsets; -import java.util.*; +import java.util.ArrayList; +import java.util.Objects; +import java.util.TreeMap; /** * A writer for constructing ImprintRecords by adding fields sequentially. @@ -56,7 +55,7 @@ public ImprintRecord build() throws ImprintException { return new ImprintRecord(header, directory, payloadView); } - private int estimatePayloadSize() { + private int estimatePayloadSize() throws ImprintException { // More accurate estimation to reduce allocations int estimatedSize = 0; for (var value : fields.values()) { @@ -74,7 +73,7 @@ private int estimatePayloadSize() { * @param value the value to estimate size for * @return estimated size in bytes including type-specific overhead */ - private int estimateValueSize(Value value) { + private int estimateValueSize(Value value) throws ImprintException { // Use TypeHandler for simple types switch (value.getTypeCode()) { case NULL: @@ -85,52 +84,20 @@ private int estimateValueSize(Value value) { case FLOAT64: case BYTES: case STRING: - return value.getTypeCode().getHandler().estimateSize(value); - case ARRAY: - List array = ((Value.ArrayValue) value).getValue(); - int arraySize = VarInt.encodedLength(array.size()) + 1; // length + type code - for (Value element : array) { - arraySize += estimateValueSize(element); - } - return arraySize; - case MAP: - Map map = ((Value.MapValue) value).getValue(); - int mapSize = VarInt.encodedLength(map.size()) + 2; // length + 2 type codes - for (Map.Entry entry : map.entrySet()) { - mapSize += estimateMapKeySize(entry.getKey()); - mapSize += estimateValueSize(entry.getValue()); - } - return mapSize; + return value.getTypeCode().getHandler().estimateSize(value); case ROW: - // Estimate nested record size (rough approximation) - return 100; // Conservative estimate + Value.RowValue rowValue = (Value.RowValue) value; + return rowValue.getValue().estimateSerializedSize(); default: - return 32; // Default fallback + throw new ImprintException(ErrorType.SERIALIZATION_ERROR, "Unknown type code: " + value.getTypeCode()); } } - - private int estimateMapKeySize(MapKey key) { - switch (key.getTypeCode()) { - case INT32: return 4; - case INT64: return 8; - case BYTES: - byte[] bytes = ((MapKey.BytesKey) key).getValue(); - return VarInt.encodedLength(bytes.length) + bytes.length; - case STRING: - var str = ((MapKey.StringKey) key).getValue(); - int utf8Length = str.getBytes(StandardCharsets.UTF_8).length; - return VarInt.encodedLength(utf8Length) + utf8Length; - default: - return 16; // Default fallback - } - } - private void serializeValue(Value value, ByteBuffer buffer) throws ImprintException { switch (value.getTypeCode()) { case NULL: @@ -141,17 +108,11 @@ private void serializeValue(Value value, ByteBuffer buffer) throws ImprintExcept case FLOAT64: case BYTES: case STRING: - value.getTypeCode().getHandler().serialize(value, buffer); - break; - case ARRAY: - serializeArray((Value.ArrayValue) value, buffer); - break; - case MAP: - serializeMap((Value.MapValue) value, buffer); + value.getTypeCode().getHandler().serialize(value, buffer); break; - + //TODO eliminate this switch entirely by implementing a ROW TypeHandler case ROW: Value.RowValue rowValue = (Value.RowValue) value; var serializedRow = rowValue.getValue().serializeToBuffer(); @@ -159,99 +120,7 @@ private void serializeValue(Value value, ByteBuffer buffer) throws ImprintExcept break; default: - throw new ImprintException(ErrorType.SERIALIZATION_ERROR, - "Unknown type code: " + value.getTypeCode()); - } - } - - private void serializeArray(Value.ArrayValue arrayValue, ByteBuffer buffer) throws ImprintException { - var elements = arrayValue.getValue(); - VarInt.encode(elements.size(), buffer); - - if (elements.isEmpty()) return; - - // All elements must have the same type - var elementType = elements.get(0).getTypeCode(); - buffer.put(elementType.getCode()); - for (var element : elements) { - if (element.getTypeCode() != elementType) { - throw new ImprintException(ErrorType.SCHEMA_ERROR, - "Array elements must have same type code: " + - element.getTypeCode() + " != " + elementType); - } - serializeValue(element, buffer); - } - } - - private void serializeMap(Value.MapValue mapValue, ByteBuffer buffer) throws ImprintException { - var map = mapValue.getValue(); - VarInt.encode(map.size(), buffer); - - if (map.isEmpty()) { - return; - } - - // All keys and values must have consistent types - var iterator = map.entrySet().iterator(); - var first = iterator.next(); - var keyType = first.getKey().getTypeCode(); - var valueType = first.getValue().getTypeCode(); - - buffer.put(keyType.getCode()); - buffer.put(valueType.getCode()); - - // Serialize the first entry - serializeMapKey(first.getKey(), buffer); - serializeValue(first.getValue(), buffer); - - // Serialize remaining entries - while (iterator.hasNext()) { - var entry = iterator.next(); - if (entry.getKey().getTypeCode() != keyType) { - throw new ImprintException(ErrorType.SCHEMA_ERROR, - "Map keys must have same type code: " + - entry.getKey().getTypeCode() + " != " + keyType); - } - if (entry.getValue().getTypeCode() != valueType) { - throw new ImprintException(ErrorType.SCHEMA_ERROR, - "Map values must have same type code: " + - entry.getValue().getTypeCode() + " != " + valueType); - } - - serializeMapKey(entry.getKey(), buffer); - serializeValue(entry.getValue(), buffer); - } - } - - private void serializeMapKey(MapKey key, ByteBuffer buffer) throws ImprintException { - switch (key.getTypeCode()) { - case INT32: - MapKey.Int32Key int32Key = (MapKey.Int32Key) key; - buffer.putInt(int32Key.getValue()); - break; - - case INT64: - MapKey.Int64Key int64Key = (MapKey.Int64Key) key; - buffer.putLong(int64Key.getValue()); - break; - - case BYTES: - MapKey.BytesKey bytesKey = (MapKey.BytesKey) key; - byte[] bytes = bytesKey.getValue(); - VarInt.encode(bytes.length, buffer); - buffer.put(bytes); - break; - - case STRING: - MapKey.StringKey stringKey = (MapKey.StringKey) key; - byte[] stringBytes = stringKey.getValue().getBytes(StandardCharsets.UTF_8); - VarInt.encode(stringBytes.length, buffer); - buffer.put(stringBytes); - break; - - default: - throw new ImprintException(ErrorType.SERIALIZATION_ERROR, - "Invalid map key type: " + key.getTypeCode()); + throw new ImprintException(ErrorType.SERIALIZATION_ERROR, "Unknown type code: " + value.getTypeCode()); } } } \ No newline at end of file diff --git a/src/main/java/com/imprint/core/SchemaId.java b/src/main/java/com/imprint/core/SchemaId.java index cb03c1c..b6dae3b 100644 --- a/src/main/java/com/imprint/core/SchemaId.java +++ b/src/main/java/com/imprint/core/SchemaId.java @@ -7,6 +7,6 @@ */ @Value public class SchemaId { - int fieldspaceId; + int fieldSpaceId; int schemaHash; } \ No newline at end of file diff --git a/src/main/java/com/imprint/types/TypeCode.java b/src/main/java/com/imprint/types/TypeCode.java index 6bf450d..a81b199 100644 --- a/src/main/java/com/imprint/types/TypeCode.java +++ b/src/main/java/com/imprint/types/TypeCode.java @@ -16,8 +16,8 @@ public enum TypeCode { FLOAT64(0x5, TypeHandler.FLOAT64), BYTES(0x6, TypeHandler.BYTES), STRING(0x7, TypeHandler.STRING), - ARRAY(0x8, null), // TODO: implement - MAP(0x9, null), // TODO: implement + ARRAY(0x8, TypeHandler.ARRAY), + MAP(0x9, TypeHandler.MAP), ROW(0xA, null); // TODO: implement (basically a placeholder for user-defined type) @Getter diff --git a/src/main/java/com/imprint/types/TypeHandler.java b/src/main/java/com/imprint/types/TypeHandler.java index 07ef7a1..573aed3 100644 --- a/src/main/java/com/imprint/types/TypeHandler.java +++ b/src/main/java/com/imprint/types/TypeHandler.java @@ -1,9 +1,11 @@ package com.imprint.types; +import com.imprint.error.ErrorType; import com.imprint.error.ImprintException; import com.imprint.util.VarInt; import java.nio.ByteBuffer; +import java.util.*; /** * Interface for handling type-specific serialization, deserialization, and size estimation. @@ -14,9 +16,52 @@ public interface TypeHandler { Value deserialize(ByteBuffer buffer) throws ImprintException; void serialize(Value value, ByteBuffer buffer) throws ImprintException; - int estimateSize(Value value); + int estimateSize(Value value) throws ImprintException; ByteBuffer readValueBytes(ByteBuffer buffer) throws ImprintException; + // Helper method to eliminate duplication in ARRAY/MAP readValueBytes + static ByteBuffer readComplexValueBytes(ByteBuffer buffer, String typeName, + ComplexValueMeasurer measurer) throws ImprintException { + int initialPosition = buffer.position(); + ByteBuffer tempBuffer = buffer.duplicate(); + tempBuffer.order(buffer.order()); + + VarInt.DecodeResult lengthResult = VarInt.decode(tempBuffer); + int numElements = lengthResult.getValue(); + int varIntLength = tempBuffer.position() - initialPosition; + + if (numElements == 0) { + if (buffer.remaining() < varIntLength) { + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, + "Not enough bytes for empty " + typeName + " VarInt. Needed: " + + varIntLength + ", available: " + buffer.remaining()); + } + ByteBuffer valueSlice = buffer.slice(); + valueSlice.limit(varIntLength); + buffer.position(initialPosition + varIntLength); + return valueSlice.asReadOnlyBuffer(); + } + + int dataLength = measurer.measureDataLength(tempBuffer, numElements); + int totalLength = varIntLength + dataLength; + + if (buffer.remaining() < totalLength) { + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, + "Not enough bytes for " + typeName + " value. Needed: " + totalLength + + ", available: " + buffer.remaining() + " at position " + initialPosition); + } + + ByteBuffer valueSlice = buffer.slice(); + valueSlice.limit(totalLength); + buffer.position(initialPosition + totalLength); + return valueSlice.asReadOnlyBuffer(); + } + + @FunctionalInterface + interface ComplexValueMeasurer { + int measureDataLength(ByteBuffer tempBuffer, int numElements) throws ImprintException; + } + // Static implementations for each type TypeHandler NULL = new TypeHandler() { @Override @@ -54,7 +99,7 @@ public Value deserialize(ByteBuffer buffer) throws ImprintException { @Override public void serialize(Value value, ByteBuffer buffer) { - Value.BoolValue boolValue = (Value.BoolValue) value; + var boolValue = (Value.BoolValue) value; buffer.put((byte) (boolValue.getValue() ? 1 : 0)); } @@ -76,14 +121,14 @@ public ByteBuffer readValueBytes(ByteBuffer buffer) { @Override public Value deserialize(ByteBuffer buffer) throws ImprintException { if (buffer.remaining() < 4) { - throw new ImprintException(com.imprint.error.ErrorType.BUFFER_UNDERFLOW, "Not enough bytes for int32"); + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, "Not enough bytes for int32"); } return Value.fromInt32(buffer.getInt()); } @Override public void serialize(Value value, ByteBuffer buffer) { - Value.Int32Value int32Value = (Value.Int32Value) value; + var int32Value = (Value.Int32Value) value; buffer.putInt(int32Value.getValue()); } @@ -105,7 +150,7 @@ public ByteBuffer readValueBytes(ByteBuffer buffer) { @Override public Value deserialize(ByteBuffer buffer) throws ImprintException { if (buffer.remaining() < 8) { - throw new ImprintException(com.imprint.error.ErrorType.BUFFER_UNDERFLOW, "Not enough bytes for int64"); + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, "Not enough bytes for int64"); } return Value.fromInt64(buffer.getLong()); } @@ -134,14 +179,14 @@ public ByteBuffer readValueBytes(ByteBuffer buffer) { @Override public Value deserialize(ByteBuffer buffer) throws ImprintException { if (buffer.remaining() < 4) { - throw new ImprintException(com.imprint.error.ErrorType.BUFFER_UNDERFLOW, "Not enough bytes for float32"); + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, "Not enough bytes for float32"); } return Value.fromFloat32(buffer.getFloat()); } @Override public void serialize(Value value, ByteBuffer buffer) { - Value.Float32Value float32Value = (Value.Float32Value) value; + var float32Value = (Value.Float32Value) value; buffer.putFloat(float32Value.getValue()); } @@ -163,14 +208,14 @@ public ByteBuffer readValueBytes(ByteBuffer buffer) { @Override public Value deserialize(ByteBuffer buffer) throws ImprintException { if (buffer.remaining() < 8) { - throw new ImprintException(com.imprint.error.ErrorType.BUFFER_UNDERFLOW, "Not enough bytes for float64"); + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, "Not enough bytes for float64"); } return Value.fromFloat64(buffer.getDouble()); } @Override public void serialize(Value value, ByteBuffer buffer) { - Value.Float64Value float64Value = (Value.Float64Value) value; + var float64Value = (Value.Float64Value) value; buffer.putDouble(float64Value.getValue()); } @@ -194,7 +239,7 @@ public Value deserialize(ByteBuffer buffer) throws ImprintException { VarInt.DecodeResult lengthResult = VarInt.decode(buffer); int length = lengthResult.getValue(); if (buffer.remaining() < length) { - throw new ImprintException(com.imprint.error.ErrorType.BUFFER_UNDERFLOW, "Not enough bytes for bytes value"); + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, "Not enough bytes for bytes value data after VarInt. Slice from readValueBytes is too short. Needed: " + length + ", available: " + buffer.remaining()); } var bytesView = buffer.slice(); bytesView.limit(length); @@ -231,14 +276,25 @@ public int estimateSize(Value value) { @Override public ByteBuffer readValueBytes(ByteBuffer buffer) throws ImprintException { - int originalPosition = buffer.position(); - VarInt.DecodeResult lengthResult = VarInt.decode(buffer); - int totalLength = lengthResult.getBytesRead() + lengthResult.getValue(); - buffer.position(originalPosition); - var valueBuffer = buffer.slice(); - valueBuffer.limit(totalLength); - buffer.position(buffer.position() + totalLength); - return valueBuffer.asReadOnlyBuffer(); + int initialPos = buffer.position(); + ByteBuffer tempMeasureBuffer = buffer.duplicate(); + VarInt.DecodeResult dr = VarInt.decode(tempMeasureBuffer); + + int varIntByteLength = tempMeasureBuffer.position() - initialPos; + int payloadByteLength = dr.getValue(); + int totalValueLength = varIntByteLength + payloadByteLength; + + if (buffer.remaining() < totalValueLength) { + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, + "Not enough bytes for VarInt-prefixed data. Needed: " + totalValueLength + + ", available: " + buffer.remaining() + " at position " + initialPos); + } + + ByteBuffer resultSlice = buffer.slice(); + resultSlice.limit(totalValueLength); + + buffer.position(initialPos + totalValueLength); + return resultSlice.asReadOnlyBuffer(); } }; @@ -248,28 +304,28 @@ public Value deserialize(ByteBuffer buffer) throws ImprintException { VarInt.DecodeResult strLengthResult = VarInt.decode(buffer); int strLength = strLengthResult.getValue(); if (buffer.remaining() < strLength) { - throw new ImprintException(com.imprint.error.ErrorType.BUFFER_UNDERFLOW, "Not enough bytes for string value"); + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, "Not enough bytes for string value data after VarInt. Slice from readValueBytes is too short. Needed: " + strLength + ", available: " + buffer.remaining()); } var stringBytesView = buffer.slice(); stringBytesView.limit(strLength); buffer.position(buffer.position() + strLength); try { - return Value.fromStringBuffer(stringBytesView.asReadOnlyBuffer()); + return Value.fromStringBuffer(stringBytesView); } catch (Exception e) { - throw new ImprintException(com.imprint.error.ErrorType.INVALID_UTF8_STRING, "Invalid UTF-8 string"); + throw new ImprintException(ErrorType.INVALID_UTF8_STRING, "Invalid UTF-8 string or buffer issue: " + e.getMessage()); } } @Override public void serialize(Value value, ByteBuffer buffer) { if (value instanceof Value.StringBufferValue) { - Value.StringBufferValue bufferValue = (Value.StringBufferValue) value; + var bufferValue = (Value.StringBufferValue) value; var stringBuffer = bufferValue.getBuffer(); VarInt.encode(stringBuffer.remaining(), buffer); buffer.put(stringBuffer); } else { - Value.StringValue stringValue = (Value.StringValue) value; - byte[] stringBytes = stringValue.getUtf8Bytes(); // Use cached UTF-8 bytes + var stringValue = (Value.StringValue) value; + byte[] stringBytes = stringValue.getUtf8Bytes(); VarInt.encode(stringBytes.length, buffer); buffer.put(stringBytes); } @@ -283,21 +339,287 @@ public int estimateSize(Value value) { return VarInt.encodedLength(length) + length; } else { Value.StringValue stringValue = (Value.StringValue) value; - byte[] utf8Bytes = stringValue.getUtf8Bytes(); // Use cached UTF-8 bytes + byte[] utf8Bytes = stringValue.getUtf8Bytes(); return VarInt.encodedLength(utf8Bytes.length) + utf8Bytes.length; } } @Override public ByteBuffer readValueBytes(ByteBuffer buffer) throws ImprintException { - int originalPosition = buffer.position(); + int initialPos = buffer.position(); + ByteBuffer tempMeasureBuffer = buffer.duplicate(); + VarInt.DecodeResult dr = VarInt.decode(tempMeasureBuffer); + + int varIntByteLength = tempMeasureBuffer.position() - initialPos; + int payloadByteLength = dr.getValue(); + int totalValueLength = varIntByteLength + payloadByteLength; + + if (buffer.remaining() < totalValueLength) { + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, + "Not enough bytes for VarInt-prefixed string. Needed: " + totalValueLength + + ", available: " + buffer.remaining() + " at position " + initialPos); + } + + ByteBuffer resultSlice = buffer.slice(); + resultSlice.limit(totalValueLength); + + buffer.position(initialPos + totalValueLength); + return resultSlice.asReadOnlyBuffer(); + } + }; + + TypeHandler ARRAY = new TypeHandler() { + @Override + public Value deserialize(ByteBuffer buffer) throws ImprintException { VarInt.DecodeResult lengthResult = VarInt.decode(buffer); - int totalLength = lengthResult.getBytesRead() + lengthResult.getValue(); - buffer.position(originalPosition); - var valueBuffer = buffer.slice(); - valueBuffer.limit(totalLength); - buffer.position(buffer.position() + totalLength); - return valueBuffer.asReadOnlyBuffer(); + int length = lengthResult.getValue(); + + if (length == 0) { + return Value.fromArray(Collections.emptyList()); + } + + if (buffer.remaining() < 1) { + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, "Not enough bytes for ARRAY element type code."); + } + var elementType = TypeCode.fromByte(buffer.get()); + var elements = new ArrayList(length); + var elementHandler = elementType.getHandler(); + + for (int i = 0; i < length; i++) { + var elementValueBytes = elementHandler.readValueBytes(buffer); + elementValueBytes.order(buffer.order()); + var element = elementHandler.deserialize(elementValueBytes); + elements.add(element); + } + + return Value.fromArray(elements); + } + + @Override + public void serialize(Value value, ByteBuffer buffer) throws ImprintException { + var arrayValue = (Value.ArrayValue) value; + var elements = arrayValue.getValue(); + VarInt.encode(elements.size(), buffer); + + if (elements.isEmpty()) return; + + var elementType = elements.get(0).getTypeCode(); + buffer.put(elementType.getCode()); + var elementHandler = elementType.getHandler(); + for (var element : elements) { + if (element.getTypeCode() != elementType) { + throw new ImprintException(ErrorType.SCHEMA_ERROR, + "Array elements must have same type code: " + + element.getTypeCode() + " != " + elementType); + } + elementHandler.serialize(element, buffer); + } + } + + @Override + public int estimateSize(Value value) throws ImprintException { + var arrayValue = (Value.ArrayValue) value; + var elements = arrayValue.getValue(); + int sizeOfLength = VarInt.encodedLength(elements.size()); + if (elements.isEmpty()) { + return sizeOfLength; + } + int sizeOfElementTypeCode = 1; + int arraySize = sizeOfLength + sizeOfElementTypeCode; + var elementHandler = elements.get(0).getTypeCode().getHandler(); + for (var element : elements) { + arraySize += elementHandler.estimateSize(element); + } + return arraySize; + } + + @Override + public ByteBuffer readValueBytes(ByteBuffer buffer) throws ImprintException { + return readComplexValueBytes(buffer, "ARRAY", (tempBuffer, numElements) -> { + if (tempBuffer.remaining() < 1) { + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, + "Not enough bytes for ARRAY element type code in temp buffer during measurement."); + } + byte elementTypeCodeByte = tempBuffer.get(); + int typeCodeLength = 1; + + TypeHandler elementHandler = TypeCode.fromByte(elementTypeCodeByte).getHandler(); + int elementsDataLength = 0; + for (int i = 0; i < numElements; i++) { + int elementStartPos = tempBuffer.position(); + elementHandler.readValueBytes(tempBuffer); + elementsDataLength += (tempBuffer.position() - elementStartPos); + } + + return typeCodeLength + elementsDataLength; + }); + } + }; + + TypeHandler MAP = new TypeHandler() { + @Override + public Value deserialize(ByteBuffer buffer) throws ImprintException { + VarInt.DecodeResult lengthResult = VarInt.decode(buffer); + int length = lengthResult.getValue(); + + if (length == 0) { + return Value.fromMap(Collections.emptyMap()); + } + + if (buffer.remaining() < 2) { + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, "Not enough bytes for MAP key/value type codes."); + } + var keyType = TypeCode.fromByte(buffer.get()); + var valueType = TypeCode.fromByte(buffer.get()); + var map = new HashMap(length); + + var keyHandler = keyType.getHandler(); + var valueHandler = valueType.getHandler(); + + for (int i = 0; i < length; i++) { + var keyBytes = keyHandler.readValueBytes(buffer); + keyBytes.order(buffer.order()); + var keyValue = keyHandler.deserialize(keyBytes); + var key = MapKey.fromValue(keyValue); + + var valueBytes = valueHandler.readValueBytes(buffer); + valueBytes.order(buffer.order()); + var mapInternalValue = valueHandler.deserialize(valueBytes); + + map.put(key, mapInternalValue); + } + + return Value.fromMap(map); + } + + @Override + public void serialize(Value value, ByteBuffer buffer) throws ImprintException { + var mapValue = (Value.MapValue) value; + var map = mapValue.getValue(); + VarInt.encode(map.size(), buffer); + + if (map.isEmpty()) { + return; + } + + var iterator = map.entrySet().iterator(); + var first = iterator.next(); + var keyType = first.getKey().getTypeCode(); + var valueType = first.getValue().getTypeCode(); + + buffer.put(keyType.getCode()); + buffer.put(valueType.getCode()); + + serializeMapKey(first.getKey(), buffer); + first.getValue().getTypeCode().getHandler().serialize(first.getValue(), buffer); + + while (iterator.hasNext()) { + var entry = iterator.next(); + if (entry.getKey().getTypeCode() != keyType) { + throw new ImprintException(ErrorType.SCHEMA_ERROR, + "Map keys must have same type code: " + + entry.getKey().getTypeCode() + " != " + keyType); + } + if (entry.getValue().getTypeCode() != valueType) { + throw new ImprintException(ErrorType.SCHEMA_ERROR, + "Map values must have same type code: " + + entry.getValue().getTypeCode() + " != " + valueType); + } + + serializeMapKey(entry.getKey(), buffer); + entry.getValue().getTypeCode().getHandler().serialize(entry.getValue(), buffer); + } + } + + @Override + public int estimateSize(Value value) throws ImprintException { + var mapValue = (Value.MapValue) value; + var map = mapValue.getValue(); + int sizeOfLength = VarInt.encodedLength(map.size()); + if (map.isEmpty()) { + return sizeOfLength; + } + int sizeOfTypeCodes = 2; + int mapSize = sizeOfLength + sizeOfTypeCodes; + + for (var entry : map.entrySet()) { + mapSize += estimateMapKeySize(entry.getKey()); + mapSize += entry.getValue().getTypeCode().getHandler().estimateSize(entry.getValue()); + } + return mapSize; + } + + @Override + public ByteBuffer readValueBytes(ByteBuffer buffer) throws ImprintException { + return readComplexValueBytes(buffer, "MAP", (tempBuffer, numEntries) -> { + if (tempBuffer.remaining() < 2) { + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, + "Not enough bytes for MAP key/value type codes in temp buffer during measurement."); + } + byte keyTypeCodeByte = tempBuffer.get(); + byte valueTypeCodeByte = tempBuffer.get(); + int typeCodesLength = 2; + int entriesDataLength = 0; + for (int i = 0; i < numEntries; i++) { + int entryStartPos = tempBuffer.position(); + TypeCode.fromByte(keyTypeCodeByte).getHandler().readValueBytes(tempBuffer); + TypeCode.fromByte(valueTypeCodeByte).getHandler().readValueBytes(tempBuffer); + entriesDataLength += (tempBuffer.position() - entryStartPos); + } + + return typeCodesLength + entriesDataLength; + }); + } + + private void serializeMapKey(MapKey key, ByteBuffer buffer) throws ImprintException { + switch (key.getTypeCode()) { + case INT32: + MapKey.Int32Key int32Key = (MapKey.Int32Key) key; + buffer.putInt(int32Key.getValue()); + break; + + case INT64: + MapKey.Int64Key int64Key = (MapKey.Int64Key) key; + buffer.putLong(int64Key.getValue()); + break; + + case BYTES: + MapKey.BytesKey bytesKey = (MapKey.BytesKey) key; + byte[] bytes = bytesKey.getValue(); + VarInt.encode(bytes.length, buffer); + buffer.put(bytes); + break; + + case STRING: + MapKey.StringKey stringKey = (MapKey.StringKey) key; + byte[] stringBytes = stringKey.getValue().getBytes(java.nio.charset.StandardCharsets.UTF_8); + VarInt.encode(stringBytes.length, buffer); + buffer.put(stringBytes); + break; + + default: + throw new ImprintException(ErrorType.SERIALIZATION_ERROR, + "Invalid map key type: " + key.getTypeCode()); + } + } + + private int estimateMapKeySize(MapKey key) throws ImprintException { + switch (key.getTypeCode()) { + case INT32: return 4; + case INT64: return 8; + case BYTES: + byte[] bytes = ((MapKey.BytesKey) key).getValue(); + return VarInt.encodedLength(bytes.length) + bytes.length; + + case STRING: + var str = ((MapKey.StringKey) key).getValue(); + int utf8Length = str.getBytes(java.nio.charset.StandardCharsets.UTF_8).length; + return VarInt.encodedLength(utf8Length) + utf8Length; + + default: + throw new ImprintException(ErrorType.SERIALIZATION_ERROR, + "Invalid map key type: " + key.getTypeCode()); + } } }; } \ No newline at end of file diff --git a/src/main/java/com/imprint/types/Value.java b/src/main/java/com/imprint/types/Value.java index 64eae91..7f3bbb9 100644 --- a/src/main/java/com/imprint/types/Value.java +++ b/src/main/java/com/imprint/types/Value.java @@ -205,9 +205,16 @@ public byte[] getValue() { @Override public boolean equals(Object obj) { if (this == obj) return true; - if (obj == null || getClass() != obj.getClass()) return false; - BytesValue that = (BytesValue) obj; - return Arrays.equals(value, that.value); + if (obj == null) return false; + if (obj instanceof BytesValue) { + BytesValue that = (BytesValue) obj; + return Arrays.equals(value, that.value); + } + if (obj instanceof BytesBufferValue) { + BytesBufferValue that = (BytesBufferValue) obj; + return Arrays.equals(value, that.getValue()); + } + return false; } @Override @@ -270,7 +277,6 @@ public String toString() { } // String Value (String-based) - @EqualsAndHashCode(callSuper = false) public static class StringValue extends Value { @Getter private final String value; @@ -281,11 +287,11 @@ public StringValue(String value) { } public byte[] getUtf8Bytes() { - byte[] cached = cachedUtf8Bytes; + var cached = cachedUtf8Bytes; if (cached == null) { // Multiple threads may compute this - that's OK since it's idempotent cached = value.getBytes(StandardCharsets.UTF_8); - cachedUtf8Bytes = cached; // Benign race - last writer wins + cachedUtf8Bytes = cached; } return cached; // Return our computed value, not re-read from volatile field } @@ -293,6 +299,26 @@ public byte[] getUtf8Bytes() { @Override public TypeCode getTypeCode() { return TypeCode.STRING; } + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (obj == null) return false; + if (obj instanceof StringValue) { + StringValue that = (StringValue) obj; + return value.equals(that.value); + } + if (obj instanceof StringBufferValue) { + StringBufferValue that = (StringBufferValue) obj; + return value.equals(that.getValue()); + } + return false; + } + + @Override + public int hashCode() { + return value.hashCode(); + } + @Override public String toString() { return "\"" + value + "\""; @@ -320,13 +346,14 @@ public String getValue() { private String decodeUtf8() { // Fast path: zero-copy for array-backed ByteBuffers - if (value.hasArray() && value.arrayOffset() == 0) { - return new String(value.array(), value.position(), + if (value.hasArray()) { + return new String(value.array(), value.arrayOffset() + value.position(), value.remaining(), StandardCharsets.UTF_8); } - // Fallback path - should be impossible since deserialize uses wrap() to create an array-backed ByteBuffer. - // Allocation required for direct ByteBuffers since Java's String API doesn't provide ByteBuffer constructors + // Fallback path for non-array-backed ByteBuffers (e.g., direct buffers). + // Allocation is required here as Java's String(byte[],...) constructor needs a heap array. + // Data is copied from the ByteBuffer to a new byte array. var array = new byte[value.remaining()]; value.duplicate().get(array); return new String(array, StandardCharsets.UTF_8); diff --git a/src/main/java/com/imprint/util/VarInt.java b/src/main/java/com/imprint/util/VarInt.java index 5c9a7e5..f4c22f2 100644 --- a/src/main/java/com/imprint/util/VarInt.java +++ b/src/main/java/com/imprint/util/VarInt.java @@ -104,6 +104,26 @@ public static int encodedLength(int value) { return length; } + /** + * Read VarInt-prefixed data from a ByteBuffer. + * The data format is: VarInt(length) + data(length bytes). + * Returns a read-only ByteBuffer containing the entire VarInt + data. + * + * @param buffer the buffer to read from + * @return a read-only ByteBuffer view of the VarInt + data + * @throws ImprintException if the VarInt is malformed or buffer underflow + */ + public static ByteBuffer readVarIntPrefixedBytes(ByteBuffer buffer) throws ImprintException { + int originalPosition = buffer.position(); + VarInt.DecodeResult lengthResult = VarInt.decode(buffer); + int totalLength = lengthResult.getBytesRead() + lengthResult.getValue(); + buffer.position(originalPosition); + var valueBuffer = buffer.slice(); + valueBuffer.limit(totalLength); + buffer.position(buffer.position() + totalLength); + return valueBuffer.asReadOnlyBuffer(); + } + /** * Result of a VarInt decode operation. */ diff --git a/src/test/java/com/imprint/ByteBufferIntegrationTest.java b/src/test/java/com/imprint/ByteBufferIntegrationTest.java index 9460cbf..56ec3b0 100644 --- a/src/test/java/com/imprint/ByteBufferIntegrationTest.java +++ b/src/test/java/com/imprint/ByteBufferIntegrationTest.java @@ -1,20 +1,25 @@ package com.imprint; -import com.imprint.core.*; -import com.imprint.types.*; +import com.imprint.core.ImprintRecord; +import com.imprint.core.ImprintWriter; +import com.imprint.core.SchemaId; import com.imprint.error.ImprintException; +import com.imprint.types.MapKey; +import com.imprint.types.Value; + import java.nio.ByteBuffer; import java.util.*; /** * Integration test to verify ByteBuffer functionality and zero-copy benefits. */ -public class ByteBufferIntegrationTest { - +class ByteBufferIntegrationTest { + public static void main(String[] args) { try { testByteBufferFunctionality(); testZeroCopy(); + testArrayBackedBuffers(); System.out.println("All ByteBuffer integration tests passed!"); } catch (Exception e) { System.err.println("ByteBuffer integration test failed: " + e.getMessage()); @@ -22,66 +27,154 @@ public static void main(String[] args) { System.exit(1); } } - + static void testByteBufferFunctionality() throws ImprintException { System.out.println("Testing ByteBuffer functionality..."); - + SchemaId schemaId = new SchemaId(1, 0xdeadbeef); ImprintWriter writer = new ImprintWriter(schemaId); - + writer.addField(1, Value.fromInt32(42)) .addField(2, Value.fromString("zero-copy test")) .addField(3, Value.fromBytes(new byte[]{1, 2, 3, 4, 5})); - + ImprintRecord record = writer.build(); - + // Test ByteBuffer serialization ByteBuffer serializedBuffer = record.serializeToBuffer(); assert serializedBuffer.isReadOnly() : "Serialized buffer should be read-only"; - + // Test deserialization from ByteBuffer ImprintRecord deserialized = ImprintRecord.deserialize(serializedBuffer); - - assert deserialized.getValue(1).get().equals(Value.fromInt32(42)); - assert deserialized.getValue(2).get().equals(Value.fromString("zero-copy test")); - + + assert Objects.equals(deserialized.getValue(1), Value.fromInt32(42)); + assert Objects.equals(deserialized.getValue(2), Value.fromString("zero-copy test")); + // Test raw bytes access returns ByteBuffer - Optional rawBytes = deserialized.getRawBytes(1); - assert rawBytes.isPresent() : "Raw bytes should be present for field 1"; - assert rawBytes.get().isReadOnly() : "Raw bytes buffer should be read-only"; - + var rawBytes = deserialized.getRawBytes(1); + assert rawBytes != null : "Raw bytes should be present for field 1"; + assert rawBytes.isReadOnly() : "Raw bytes buffer should be read-only"; + System.out.println("ByteBuffer functionality test passed"); } - + static void testZeroCopy() { System.out.println("Testing zero-copy"); - + // Create a large payload to demonstrate zero-copy benefits byte[] largePayload = new byte[1024 * 1024]; // 1MB Arrays.fill(largePayload, (byte) 0xAB); - + SchemaId schemaId = new SchemaId(2, 0xcafebabe); ImprintWriter writer = new ImprintWriter(schemaId); - + try { writer.addField(1, Value.fromBytes(largePayload)); ImprintRecord record = writer.build(); - + // Test that getRawBytes returns a view, not a copy - Optional rawBytes = record.getRawBytes(1); - assert rawBytes.isPresent() : "Raw bytes should be present"; - - ByteBuffer rawBuffer = rawBytes.get(); - assert rawBuffer.isReadOnly() : "Raw buffer should be read-only"; - + var rawBytes = record.getRawBytes(1); + assert rawBytes !=null : "Raw bytes should be present"; + assert rawBytes.isReadOnly() : "Raw buffer should be read-only"; + // The buffer should be positioned at the start of the actual data // (after the VarInt length prefix) - assert rawBuffer.remaining() > largePayload.length : "Buffer should include length prefix"; - + assert rawBytes.remaining() > largePayload.length : "Buffer should include length prefix"; + System.out.println("Zero-copy benefits test passed"); - + } catch (ImprintException e) { throw new RuntimeException("Failed zero-copy test", e); } } + + static void testArrayBackedBuffers() throws ImprintException { + System.out.println("Testing array-backed buffers for zero-copy performance..."); + + // Test serialized buffers are array-backed + SchemaId schemaId = new SchemaId(1, 0xdeadbeef); + ImprintWriter writer = new ImprintWriter(schemaId); + + writer.addField(1, Value.fromInt32(42)) + .addField(2, Value.fromString("test string")) + .addField(3, Value.fromBytes(new byte[]{1, 2, 3, 4})) + .addField(4, Value.fromBoolean(true)); + + ImprintRecord record = writer.build(); + + // Test that serializeToBuffer() returns array-backed buffer + ByteBuffer serializedBuffer = record.serializeToBuffer(); + assert serializedBuffer.hasArray() : "Serialized buffer should be array-backed for zero-copy performance"; + + // Test that the internal payload is array-backed + assert record.getPayload().hasArray() : "Record payload should be array-backed for zero-copy performance"; + + // Test deserialized buffers are array-backed + byte[] bytes = new byte[serializedBuffer.remaining()]; + serializedBuffer.get(bytes); + ImprintRecord deserialized = ImprintRecord.deserialize(bytes); + + assert deserialized.getPayload().hasArray() : "Deserialized record payload should be array-backed"; + + // Test that getRawBytes() returns array-backed buffers + ByteBuffer rawBytes1 = deserialized.getRawBytes(1); + ByteBuffer rawBytes2 = deserialized.getRawBytes(2); + + assert rawBytes1 != null && rawBytes1.hasArray() : "Raw bytes buffer for int field should be array-backed"; + assert rawBytes2 != null && rawBytes2.hasArray() : "Raw bytes buffer for string field should be array-backed"; + + // Test complex types use array-backed buffers + ImprintWriter complexWriter = new ImprintWriter(new SchemaId(2, 0xcafebabe)); + + // Create array value + List arrayValues = Arrays.asList( + Value.fromInt32(1), + Value.fromInt32(2), + Value.fromInt32(3) + ); + + // Create map value + Map mapValues = new HashMap<>(); + mapValues.put(MapKey.fromString("key1"), Value.fromString("value1")); + mapValues.put(MapKey.fromString("key2"), Value.fromString("value2")); + + complexWriter.addField(1, Value.fromArray(arrayValues)) + .addField(2, Value.fromMap(mapValues)); + + ImprintRecord complexRecord = complexWriter.build(); + + assert complexRecord.getPayload().hasArray() : "Record with complex types should use array-backed payload"; + + ByteBuffer complexSerialized = complexRecord.serializeToBuffer(); + assert complexSerialized.hasArray() : "Serialized buffer with complex types should be array-backed"; + + // Test nested records use array-backed buffers + ImprintWriter innerWriter = new ImprintWriter(new SchemaId(3, 0x12345678)); + innerWriter.addField(1, Value.fromString("nested data")); + ImprintRecord innerRecord = innerWriter.build(); + + ImprintWriter outerWriter = new ImprintWriter(new SchemaId(4, 0x87654321)); + outerWriter.addField(1, Value.fromRow(innerRecord)); + ImprintRecord outerRecord = outerWriter.build(); + + assert innerRecord.getPayload().hasArray() : "Inner record payload should be array-backed"; + assert outerRecord.getPayload().hasArray() : "Outer record payload should be array-backed"; + + ByteBuffer nestedSerialized = outerRecord.serializeToBuffer(); + assert nestedSerialized.hasArray() : "Serialized nested record should be array-backed"; + + // Test builder pattern uses array-backed buffers + ImprintRecord builderRecord = ImprintRecord.builder(1, 0xabcdef00) + .field(1, "test string") + .field(2, 42) + .field(3, new byte[]{1, 2, 3}) + .build(); + + assert builderRecord.getPayload().hasArray() : "Builder-created record should use array-backed payload"; + + ByteBuffer builderSerialized = builderRecord.serializeToBuffer(); + assert builderSerialized.hasArray() : "Builder-created serialized buffer should be array-backed"; + + System.out.println("✓ Array-backed buffers test passed"); + } } \ No newline at end of file diff --git a/src/test/java/com/imprint/ComprehensiveTest.java b/src/test/java/com/imprint/ComprehensiveTest.java deleted file mode 100644 index af7f0b0..0000000 --- a/src/test/java/com/imprint/ComprehensiveTest.java +++ /dev/null @@ -1,208 +0,0 @@ -package com.imprint; - -import com.imprint.core.*; -import com.imprint.types.*; -import com.imprint.error.ImprintException; -import com.imprint.util.VarInt; -import java.nio.ByteBuffer; -import java.util.*; - -/** - * Comprehensive test to verify all functionality works correctly. - */ -public class ComprehensiveTest { - - public static void main(String[] args) { - try { - testVarIntFunctionality(); - testValueTypes(); - testMapKeys(); - testComplexSerialization(); - testErrorHandling(); - testByteBufferPerformance(); - System.out.println("All comprehensive tests passed!"); - } catch (Exception e) { - System.err.println("Comprehensive test failed: " + e.getMessage()); - e.printStackTrace(); - System.exit(1); - } - } - - static void testVarIntFunctionality() throws ImprintException { - System.out.println("Testing VarInt functionality..."); - - // Test encoding/decoding of various values - int[] testValues = {0, 1, 127, 128, 16383, 16384, Integer.MAX_VALUE}; - - for (int value : testValues) { - ByteBuffer buffer = ByteBuffer.allocate(10); - VarInt.encode(value, buffer); - int encodedLength = buffer.position(); - - buffer.flip(); - VarInt.DecodeResult result = VarInt.decode(buffer); - - assert result.getValue() == value : "VarInt roundtrip failed for " + value; - assert result.getBytesRead() == encodedLength : "Bytes read mismatch for " + value; - } - - System.out.println("✓ VarInt functionality test passed"); - } - - static void testValueTypes() { - System.out.println("Testing Value types"); - - // Test all value types - Value nullVal = Value.nullValue(); - Value boolVal = Value.fromBoolean(true); - Value int32Val = Value.fromInt32(42); - Value int64Val = Value.fromInt64(123456789L); - Value float32Val = Value.fromFloat32(3.14f); - Value float64Val = Value.fromFloat64(2.718281828); - Value bytesVal = Value.fromBytes(new byte[]{1, 2, 3, 4}); - Value stringVal = Value.fromString("test"); - - // Test type codes - assert nullVal.getTypeCode() == TypeCode.NULL; - assert boolVal.getTypeCode() == TypeCode.BOOL; - assert int32Val.getTypeCode() == TypeCode.INT32; - assert int64Val.getTypeCode() == TypeCode.INT64; - assert float32Val.getTypeCode() == TypeCode.FLOAT32; - assert float64Val.getTypeCode() == TypeCode.FLOAT64; - assert bytesVal.getTypeCode() == TypeCode.BYTES; - assert stringVal.getTypeCode() == TypeCode.STRING; - - // Test value extraction - assert ((Value.BoolValue) boolVal).getValue(); - assert ((Value.Int32Value) int32Val).getValue() == 42; - assert ((Value.Int64Value) int64Val).getValue() == 123456789L; - assert ((Value.Float32Value) float32Val).getValue() == 3.14f; - assert ((Value.Float64Value) float64Val).getValue() == 2.718281828; - assert Arrays.equals(((Value.BytesValue) bytesVal).getValue(), new byte[]{1, 2, 3, 4}); - assert ((Value.StringValue) stringVal).getValue().equals("test"); - - System.out.println("✓ Value types test passed"); - } - - static void testMapKeys() throws ImprintException { - System.out.println("Testing MapKey functionality..."); - - MapKey int32Key = MapKey.fromInt32(42); - MapKey int64Key = MapKey.fromInt64(123L); - MapKey bytesKey = MapKey.fromBytes(new byte[]{1, 2, 3}); - MapKey stringKey = MapKey.fromString("test"); - - // Test conversion to/from Values - Value int32Value = int32Key.toValue(); - Value int64Value = int64Key.toValue(); - Value bytesValue = bytesKey.toValue(); - Value stringValue = stringKey.toValue(); - - assert MapKey.fromValue(int32Value).equals(int32Key); - assert MapKey.fromValue(int64Value).equals(int64Key); - assert MapKey.fromValue(bytesValue).equals(bytesKey); - assert MapKey.fromValue(stringValue).equals(stringKey); - - System.out.println("✓ MapKey functionality test passed"); - } - - static void testComplexSerialization() throws ImprintException { - System.out.println("Testing complex serialization..."); - - SchemaId schemaId = new SchemaId(1, 0xdeadbeef); - ImprintWriter writer = new ImprintWriter(schemaId); - - // Create complex nested structure - List array = Arrays.asList( - Value.fromInt32(1), - Value.fromInt32(2), - Value.fromInt32(3) - ); - - Map map = new HashMap<>(); - map.put(MapKey.fromString("key1"), Value.fromString("value1")); - map.put(MapKey.fromString("key2"), Value.fromString("value2")); - - writer.addField(1, Value.fromArray(array)) - .addField(2, Value.fromMap(map)) - .addField(3, Value.fromString("complex test")); - - ImprintRecord record = writer.build(); - - // Test ByteBuffer serialization - ByteBuffer serialized = record.serializeToBuffer(); - ImprintRecord deserialized = ImprintRecord.deserialize(serialized); - - // Verify complex structures - Value deserializedArray = deserialized.getValue(1).get(); - assert deserializedArray instanceof Value.ArrayValue; - List deserializedList = ((Value.ArrayValue) deserializedArray).getValue(); - assert deserializedList.size() == 3; - assert deserializedList.get(0).equals(Value.fromInt32(1)); - - Value deserializedMap = deserialized.getValue(2).get(); - assert deserializedMap instanceof Value.MapValue; - Map deserializedMapValue = ((Value.MapValue) deserializedMap).getValue(); - assert deserializedMapValue.size() == 2; - assert deserializedMapValue.get(MapKey.fromString("key1")).equals(Value.fromString("value1")); - - System.out.println("✓ Complex serialization test passed"); - } - - static void testErrorHandling() { - System.out.println("Testing error handling..."); - - try { - // Test invalid type code - TypeCode.fromByte((byte) 0xFF); - assert false : "Should have thrown exception for invalid type code"; - } catch (ImprintException e) { - assert e.getErrorType() == com.imprint.error.ErrorType.INVALID_TYPE_CODE; - } - - try { - // Test invalid magic byte - byte[] invalidData = new byte[15]; - invalidData[0] = 0x00; // wrong magic - ImprintRecord.deserialize(invalidData); - assert false : "Should have thrown exception for invalid magic"; - } catch (ImprintException e) { - assert e.getErrorType() == com.imprint.error.ErrorType.INVALID_MAGIC; - } - - System.out.println("✓ Error handling test passed"); - } - - static void testByteBufferPerformance() throws ImprintException { - System.out.println("Testing ByteBuffer performance benefits..."); - - // Create a record with moderate-sized data - byte[] testData = new byte[1024]; - for (int i = 0; i < testData.length; i++) { - testData[i] = (byte) (i % 256); - } - - SchemaId schemaId = new SchemaId(1, 0x12345678); - ImprintWriter writer = new ImprintWriter(schemaId); - writer.addField(1, Value.fromBytes(testData)) - .addField(2, Value.fromString("performance test")); - - ImprintRecord record = writer.build(); - - // Test that raw bytes access is zero-copy - Optional rawBytes = record.getRawBytes(1); - assert rawBytes.isPresent(); - assert rawBytes.get().isReadOnly(); - - // Test ByteBuffer serialization - ByteBuffer serialized = record.serializeToBuffer(); - assert serialized.isReadOnly(); - - // Verify deserialization works - ImprintRecord deserialized = ImprintRecord.deserialize(serialized); - Value retrievedBytes = deserialized.getValue(1).get(); - assert Arrays.equals(((Value.BytesValue) retrievedBytes).getValue(), testData); - - System.out.println("✓ ByteBuffer performance test passed"); - } -} \ No newline at end of file diff --git a/src/test/java/com/imprint/IntegrationTest.java b/src/test/java/com/imprint/IntegrationTest.java index 49cfce7..565b7cd 100644 --- a/src/test/java/com/imprint/IntegrationTest.java +++ b/src/test/java/com/imprint/IntegrationTest.java @@ -39,10 +39,10 @@ static void testBasicFunctionality() throws ImprintException { ImprintRecord record = writer.build(); // Verify we can read values back - assert record.getValue(1).get().equals(Value.fromInt32(42)); - assert record.getValue(2).get().equals(Value.fromString("testing java imprint spec")); - assert record.getValue(3).get().equals(Value.fromBoolean(true)); - assert record.getValue(999).isEmpty(); // non-existent field + assert Objects.equals(record.getValue(1), Value.fromInt32(42)); + assert Objects.equals(record.getValue(2), Value.fromString("testing java imprint spec")); + assert Objects.equals(record.getValue(3), Value.fromBoolean(true)); + assert record.getValue(999) == null; // non-existent field // Test serialization round-trip var buffer = record.serializeToBuffer(); @@ -50,11 +50,11 @@ static void testBasicFunctionality() throws ImprintException { buffer.get(serialized); ImprintRecord deserialized = ImprintRecord.deserialize(serialized); - assert deserialized.getValue(1).get().equals(Value.fromInt32(42)); - assert deserialized.getValue(2).get().equals(Value.fromString("testing java imprint spec")); - assert deserialized.getValue(3).get().equals(Value.fromBoolean(true)); + assert Objects.equals(deserialized.getValue(1), Value.fromInt32(42)); + assert Objects.equals(deserialized.getValue(2), Value.fromString("testing java imprint spec")); + assert Objects.equals(deserialized.getValue(3), Value.fromBoolean(true)); - System.out.println("✓ Basic functionality test passed"); + System.out.println("Basic functionality test passed"); } static void testArraysAndMaps() throws ImprintException { @@ -87,14 +87,14 @@ static void testArraysAndMaps() throws ImprintException { ImprintRecord deserialized = ImprintRecord.deserialize(serialized); // Verify array - Value arrayValue = deserialized.getValue(1).get(); + Value arrayValue = deserialized.getValue(1); assert arrayValue instanceof Value.ArrayValue; List deserializedArray = ((Value.ArrayValue) arrayValue).getValue(); assert deserializedArray.size() == 3; assert deserializedArray.get(0).equals(Value.fromInt32(1)); // Verify map - Value mapValue = deserialized.getValue(2).get(); + Value mapValue = deserialized.getValue(2); assert mapValue instanceof Value.MapValue; Map deserializedMap = ((Value.MapValue) mapValue).getValue(); assert deserializedMap.size() == 2; @@ -127,17 +127,17 @@ static void testNestedRecords() throws ImprintException { ImprintRecord deserialized = ImprintRecord.deserialize(serialized); // Verify outer record - assert deserialized.getHeader().getSchemaId().getFieldspaceId() == 4; - assert deserialized.getValue(2).get().equals(Value.fromString("outer data")); + assert deserialized.getHeader().getSchemaId().getFieldSpaceId() == 4; + assert Objects.equals(deserialized.getValue(2), Value.fromString("outer data")); // Verify nested record - Value rowValue = deserialized.getValue(1).get(); + Value rowValue = deserialized.getValue(1); assert rowValue instanceof Value.RowValue; ImprintRecord nestedRecord = ((Value.RowValue) rowValue).getValue(); - assert nestedRecord.getHeader().getSchemaId().getFieldspaceId() == 3; - assert nestedRecord.getValue(1).get().equals(Value.fromString("nested data")); - assert nestedRecord.getValue(2).get().equals(Value.fromInt64(9876543210L)); + assert nestedRecord.getHeader().getSchemaId().getFieldSpaceId() == 3; + assert Objects.equals(nestedRecord.getValue(1), Value.fromString("nested data")); + assert Objects.equals(nestedRecord.getValue(2), Value.fromInt64(9876543210L)); System.out.println("✓ Nested records test passed"); } diff --git a/src/test/java/com/imprint/benchmark/ProfilerTest.java b/src/test/java/com/imprint/benchmark/ProfilerTest.java index 233d993..5b531a9 100644 --- a/src/test/java/com/imprint/benchmark/ProfilerTest.java +++ b/src/test/java/com/imprint/benchmark/ProfilerTest.java @@ -50,15 +50,15 @@ var record = createTestRecord(); // Random field access (hotspot) int fieldId = random.nextInt(RECORD_SIZE) + 1; var value = record.getValue(fieldId); - if (value.isPresent()) { + if (value != null) { hits++; // Trigger string decoding (potential hotspot) - if (value.get().getTypeCode() == com.imprint.types.TypeCode.STRING) { - if (value.get() instanceof Value.StringBufferValue) { - ((Value.StringBufferValue) value.get()).getValue(); + if (value.getTypeCode() == com.imprint.types.TypeCode.STRING) { + if (value instanceof Value.StringBufferValue) { + ((Value.StringBufferValue) value).getValue(); } else { - ((Value.StringValue) value.get()).getValue(); + ((Value.StringValue) value).getValue(); } } } @@ -123,11 +123,11 @@ var record = createLargeRecord(); // Project 10 fields out of 100 (common analytical pattern) for (int fieldId = 1; fieldId <= 10; fieldId++) { var value = record.getValue(fieldId); - if (value.isPresent()) { + if (value != null) { // Force materialization of string values - if (value.get().getTypeCode() == com.imprint.types.TypeCode.STRING) { - if (value.get() instanceof Value.StringBufferValue) { - ((Value.StringBufferValue) value.get()).getValue(); + if (value.getTypeCode() == com.imprint.types.TypeCode.STRING) { + if (value instanceof Value.StringBufferValue) { + ((Value.StringBufferValue) value).getValue(); } } } diff --git a/src/test/java/com/imprint/core/ImprintRecordBuilderTest.java b/src/test/java/com/imprint/core/ImprintRecordBuilderTest.java deleted file mode 100644 index 54dcfae..0000000 --- a/src/test/java/com/imprint/core/ImprintRecordBuilderTest.java +++ /dev/null @@ -1,234 +0,0 @@ -package com.imprint.core; - -import com.imprint.error.ImprintException; -import com.imprint.types.Value; -import org.junit.jupiter.api.Test; - -import java.util.*; - -import static org.assertj.core.api.Assertions.*; - -class ImprintRecordBuilderTest { - - private static final SchemaId TEST_SCHEMA = new SchemaId(1, 0x12345678); - - @Test - void shouldCreateRecordWithPrimitiveTypes() throws ImprintException { - var record = ImprintRecord.builder(TEST_SCHEMA) - .field(1, true) - .field(2, 42) - .field(3, 123L) - .field(4, 3.14f) - .field(5, 2.718) - .field(6, "hello world") - .field(7, new byte[]{1, 2, 3}) - .nullField(8) - .build(); - - assertThat(record.getHeader().getSchemaId()).isEqualTo(TEST_SCHEMA); - assertThat(record.getDirectory()).hasSize(8); - - // Verify field values - assertThat(getFieldValue(record, 1, Value.BoolValue.class).getValue()).isTrue(); - assertThat(getFieldValue(record, 2, Value.Int32Value.class).getValue()).isEqualTo(42); - assertThat(getFieldValue(record, 3, Value.Int64Value.class).getValue()).isEqualTo(123L); - assertThat(getFieldValue(record, 4, Value.Float32Value.class).getValue()).isEqualTo(3.14f); - assertThat(getFieldValue(record, 5, Value.Float64Value.class).getValue()).isEqualTo(2.718); - assertThat(getStringValue(record, 6)).isEqualTo("hello world"); - assertThat(getBytesValue(record, 7)).isEqualTo(new byte[]{1, 2, 3}); - assertThat(record.getValue(8).get()).isInstanceOf(Value.NullValue.class); - } - - @Test - void shouldCreateRecordWithCollections() throws ImprintException { - var list = List.of(1, 2, 3); - var map = Map.of("key1", 100, "key2", 200); - - var record = ImprintRecord.builder(TEST_SCHEMA) - .field(1, list) - .field(2, map) - .build(); - - // Verify array - var arrayValue = getFieldValue(record, 1, Value.ArrayValue.class); - assertThat(arrayValue.getValue()).hasSize(3); - assertThat(((Value.Int32Value) arrayValue.getValue().get(0)).getValue()).isEqualTo(1); - assertThat(((Value.Int32Value) arrayValue.getValue().get(1)).getValue()).isEqualTo(2); - assertThat(((Value.Int32Value) arrayValue.getValue().get(2)).getValue()).isEqualTo(3); - - // Verify map - var mapValue = getFieldValue(record, 2, Value.MapValue.class); - assertThat(mapValue.getValue()).hasSize(2); - } - - @Test - void shouldCreateRecordWithNestedRecord() throws ImprintException { - var nestedRecord = ImprintRecord.builder(new SchemaId(2, 0x87654321)) - .field(1, "nested") - .field(2, 999) - .build(); - - var record = ImprintRecord.builder(TEST_SCHEMA) - .field(1, "parent") - .field(2, nestedRecord) - .build(); - - var rowValue = getFieldValue(record, 2, Value.RowValue.class); - var nested = rowValue.getValue(); - assertThat(getStringValue(nested, 1)).isEqualTo("nested"); - assertThat(getFieldValue(nested, 2, Value.Int32Value.class).getValue()).isEqualTo(999); - } - - @Test - void shouldSupportConditionalFields() throws ImprintException { - boolean includeOptional = true; - String optionalValue = "optional"; - - var record = ImprintRecord.builder(TEST_SCHEMA) - .field(1, "required") - .fieldIf(includeOptional, 2, optionalValue) - .fieldIfNotNull(3, null) // Should not add field - .fieldIfNotNull(4, "not null") // Should add field - .build(); - - assertThat(record.getDirectory()).hasSize(3); // Only fields 1, 2, 4 - assertThat(getStringValue(record, 1)).isEqualTo("required"); - assertThat(getStringValue(record, 2)).isEqualTo("optional"); - assertThat(record.getValue(3)).isEmpty(); // Not added - assertThat(getStringValue(record, 4)).isEqualTo("not null"); - } - - @Test - void shouldSupportBulkOperations() throws ImprintException { - var fieldsMap = Map.of( - 1, "bulk1", - 2, 42, - 3, true - ); - - var record = ImprintRecord.builder(TEST_SCHEMA) - .fields(fieldsMap) - .field(4, "additional") - .build(); - - assertThat(record.getDirectory()).hasSize(4); - assertThat(getStringValue(record, 1)).isEqualTo("bulk1"); - assertThat(getFieldValue(record, 2, Value.Int32Value.class).getValue()).isEqualTo(42); - assertThat(getFieldValue(record, 3, Value.BoolValue.class).getValue()).isTrue(); - assertThat(getStringValue(record, 4)).isEqualTo("additional"); - } - - @Test - void shouldProvideBuilderUtilities() { - var builder = ImprintRecord.builder(TEST_SCHEMA) - .field(1, "test") - .field(2, 42); - - assertThat(builder.hasField(1)).isTrue(); - assertThat(builder.hasField(3)).isFalse(); - assertThat(builder.fieldCount()).isEqualTo(2); - assertThat(builder.fieldIds()).containsExactly(1, 2); - } - - @Test - void shouldSupportAlternativeSchemaConstructor() throws ImprintException { - var record = ImprintRecord.builder(1, 0x12345678) - .field(1, "test") - .build(); - - assertThat(record.getHeader().getSchemaId().getFieldspaceId()).isEqualTo(1); - assertThat(record.getHeader().getSchemaId().getSchemaHash()).isEqualTo(0x12345678); - } - - @Test - void shouldRoundTripThroughSerialization() throws ImprintException { - var original = ImprintRecord.builder(TEST_SCHEMA) - .field(1, "test string") - .field(2, 42) - .field(3, 3.14159) - .field(4, true) - .field(5, new byte[]{0x01, 0x02, 0x03}) - .build(); - - var serialized = original.serializeToBuffer(); - var deserialized = ImprintRecord.deserialize(serialized); - - assertThat(deserialized.getHeader().getSchemaId()).isEqualTo(TEST_SCHEMA); - assertThat(getStringValue(deserialized, 1)).isEqualTo("test string"); - assertThat(getFieldValue(deserialized, 2, Value.Int32Value.class).getValue()).isEqualTo(42); - assertThat(getFieldValue(deserialized, 3, Value.Float64Value.class).getValue()).isEqualTo(3.14159); - assertThat(getFieldValue(deserialized, 4, Value.BoolValue.class).getValue()).isTrue(); - assertThat(getBytesValue(deserialized, 5)).isEqualTo(new byte[]{0x01, 0x02, 0x03}); - } - - // Error cases - - @Test - void shouldRejectDuplicateFieldIds() { - assertThatThrownBy(() -> - ImprintRecord.builder(TEST_SCHEMA) - .field(1, "first") - .field(1, "duplicate") // Same field ID - ).isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("Field ID 1 already exists"); - } - - @Test - void shouldRejectEmptyRecord() { - assertThatThrownBy(() -> - ImprintRecord.builder(TEST_SCHEMA).build() - ).isInstanceOf(ImprintException.class) - .hasMessageContaining("Cannot build empty record"); - } - - @Test - void shouldRejectInvalidMapKeys() { - var mapWithInvalidKey = Map.of(3.14, "value"); // Double key not supported - - assertThatThrownBy(() -> - ImprintRecord.builder(TEST_SCHEMA) - .field(1, mapWithInvalidKey) - ).isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("Invalid map key type: Double"); - } - - @Test - void shouldRejectNullValueWithoutExplicitNullField() { - assertThatThrownBy(() -> - ImprintRecord.builder(TEST_SCHEMA) - .field(1, (Value) null) - ).isInstanceOf(NullPointerException.class) - .hasMessageContaining("Value cannot be null - use nullField()"); - } - - // Helper methods for cleaner test assertions - - private T getFieldValue(ImprintRecord record, int fieldId, Class valueType) throws ImprintException { - var value = record.getValue(fieldId); - assertThat(value).isPresent(); - assertThat(value.get()).isInstanceOf(valueType); - return valueType.cast(value.get()); - } - - private String getStringValue(ImprintRecord record, int fieldId) throws ImprintException { - var value = record.getValue(fieldId).get(); - if (value instanceof Value.StringValue) { - return ((Value.StringValue) value).getValue(); - } else if (value instanceof Value.StringBufferValue) { - return ((Value.StringBufferValue) value).getValue(); - } else { - throw new AssertionError("Expected string value, got: " + value.getClass()); - } - } - - private byte[] getBytesValue(ImprintRecord record, int fieldId) throws ImprintException { - var value = record.getValue(fieldId).get(); - if (value instanceof Value.BytesValue) { - return ((Value.BytesValue) value).getValue(); - } else if (value instanceof Value.BytesBufferValue) { - return ((Value.BytesBufferValue) value).getValue(); - } else { - throw new AssertionError("Expected bytes value, got: " + value.getClass()); - } - } -} \ No newline at end of file diff --git a/src/test/java/com/imprint/core/ImprintRecordTest.java b/src/test/java/com/imprint/core/ImprintRecordTest.java index 0772580..3e37473 100644 --- a/src/test/java/com/imprint/core/ImprintRecordTest.java +++ b/src/test/java/com/imprint/core/ImprintRecordTest.java @@ -34,20 +34,20 @@ var record = writer.build(); assertThat(record.getHeader().getSchemaId()).isEqualTo(schemaId); assertThat(record.getDirectory()).hasSize(2); - Optional field1 = record.getValue(1); - Optional field2 = record.getValue(2); + Value field1 = record.getValue(1); + Value field2 = record.getValue(2); - assertThat(field1).isPresent(); - assertThat(field1.get()).isInstanceOf(Value.Int32Value.class); - assertThat(((Value.Int32Value) field1.get()).getValue()).isEqualTo(42); + assertThat(field1).isNotNull(); + assertThat(field1).isInstanceOf(Value.Int32Value.class); + assertThat(((Value.Int32Value) field1).getValue()).isEqualTo(42); - assertThat(field2).isPresent(); - assertThat(field2.get().getTypeCode()).isEqualTo(com.imprint.types.TypeCode.STRING); - String stringValue = getStringValue(field2.get()); + assertThat(field2).isNotNull(); + assertThat(field2.getTypeCode()).isEqualTo(com.imprint.types.TypeCode.STRING); + String stringValue = getStringValue(field2); assertThat(stringValue).isEqualTo("hello"); - // Non-existent field should return empty - assertThat(record.getValue(999)).isEmpty(); + // Non-existent field should return null + assertThat(record.getValue(999)).isNull(); } @Test @@ -73,22 +73,22 @@ void shouldRoundtripThroughSerialization() throws ImprintException { var deserialized = ImprintRecord.deserialize(serialized); // Verify metadata - assertThat(deserialized.getHeader().getSchemaId().getFieldspaceId()).isEqualTo(1); + assertThat(deserialized.getHeader().getSchemaId().getFieldSpaceId()).isEqualTo(1); assertThat(deserialized.getHeader().getSchemaId().getSchemaHash()).isEqualTo(0xdeadbeef); assertThat(deserialized.getDirectory()).hasSize(8); // Verify all values - assertThat(deserialized.getValue(1)).contains(Value.nullValue()); - assertThat(deserialized.getValue(2)).contains(Value.fromBoolean(true)); - assertThat(deserialized.getValue(3)).contains(Value.fromInt32(42)); - assertThat(deserialized.getValue(4)).contains(Value.fromInt64(123456789L)); - assertThat(deserialized.getValue(5)).contains(Value.fromFloat32(3.14f)); - assertThat(deserialized.getValue(6)).contains(Value.fromFloat64(2.718281828)); - assertThat(deserialized.getValue(7)).contains(Value.fromBytes(new byte[]{1, 2, 3, 4})); - assertThat(deserialized.getValue(8)).contains(Value.fromString("test string")); + assertThat(deserialized.getValue(1)).isEqualTo(Value.nullValue()); + assertThat(deserialized.getValue(2)).isEqualTo(Value.fromBoolean(true)); + assertThat(deserialized.getValue(3)).isEqualTo(Value.fromInt32(42)); + assertThat(deserialized.getValue(4)).isEqualTo(Value.fromInt64(123456789L)); + assertThat(deserialized.getValue(5)).isEqualTo(Value.fromFloat32(3.14f)); + assertThat(deserialized.getValue(6)).isEqualTo(Value.fromFloat64(2.718281828)); + assertThat(deserialized.getValue(7)).isEqualTo(Value.fromBytes(new byte[]{1, 2, 3, 4})); + assertThat(deserialized.getValue(8)).isEqualTo(Value.fromString("test string")); // Non-existent field - assertThat(deserialized.getValue(999)).isEmpty(); + assertThat(deserialized.getValue(999)).isNull(); } @Test @@ -111,11 +111,11 @@ void shouldHandleArrays() throws ImprintException { buffer.get(serialized); var deserialized = ImprintRecord.deserialize(serialized); - Optional arrayValue = deserialized.getValue(1); - assertThat(arrayValue).isPresent(); - assertThat(arrayValue.get()).isInstanceOf(Value.ArrayValue.class); + Value arrayValue = deserialized.getValue(1); + assertThat(arrayValue).isNotNull(); + assertThat(arrayValue).isInstanceOf(Value.ArrayValue.class); - List deserializedArray = ((Value.ArrayValue) arrayValue.get()).getValue(); + List deserializedArray = ((Value.ArrayValue) arrayValue).getValue(); assertThat(deserializedArray).hasSize(3); assertThat(deserializedArray.get(0)).isEqualTo(Value.fromInt32(1)); assertThat(deserializedArray.get(1)).isEqualTo(Value.fromInt32(2)); @@ -140,11 +140,11 @@ var record = writer.build(); buffer.get(serialized); var deserialized = ImprintRecord.deserialize(serialized); - Optional mapValue = deserialized.getValue(1); - assertThat(mapValue).isPresent(); - assertThat(mapValue.get()).isInstanceOf(Value.MapValue.class); + Value mapValue = deserialized.getValue(1); + assertThat(mapValue).isNotNull(); + assertThat(mapValue).isInstanceOf(Value.MapValue.class); - Map deserializedMap = ((Value.MapValue) mapValue.get()).getValue(); + Map deserializedMap = ((Value.MapValue) mapValue).getValue(); assertThat(deserializedMap).hasSize(2); assertThat(deserializedMap.get(MapKey.fromString("key1"))).isEqualTo(Value.fromInt32(1)); assertThat(deserializedMap.get(MapKey.fromString("key2"))).isEqualTo(Value.fromInt32(2)); @@ -173,23 +173,23 @@ void shouldHandleNestedRecords() throws ImprintException { var deserialized = ImprintRecord.deserialize(serialized); // Verify outer record metadata - assertThat(deserialized.getHeader().getSchemaId().getFieldspaceId()).isEqualTo(1); + assertThat(deserialized.getHeader().getSchemaId().getFieldSpaceId()).isEqualTo(1); assertThat(deserialized.getHeader().getSchemaId().getSchemaHash()).isEqualTo(0xdeadbeef); // Verify nested record - Optional rowValue = deserialized.getValue(1); - assertThat(rowValue).isPresent(); - assertThat(rowValue.get()).isInstanceOf(Value.RowValue.class); + Value rowValue = deserialized.getValue(1); + assertThat(rowValue).isNotNull(); + assertThat(rowValue).isInstanceOf(Value.RowValue.class); - var nestedRecord = ((Value.RowValue) rowValue.get()).getValue(); - assertThat(nestedRecord.getHeader().getSchemaId().getFieldspaceId()).isEqualTo(2); + var nestedRecord = ((Value.RowValue) rowValue).getValue(); + assertThat(nestedRecord.getHeader().getSchemaId().getFieldSpaceId()).isEqualTo(2); assertThat(nestedRecord.getHeader().getSchemaId().getSchemaHash()).isEqualTo(0xcafebabe); - assertThat(nestedRecord.getValue(1)).contains(Value.fromInt32(42)); - assertThat(nestedRecord.getValue(2)).contains(Value.fromString("nested")); + assertThat(nestedRecord.getValue(1)).isEqualTo(Value.fromInt32(42)); + assertThat(nestedRecord.getValue(2)).isEqualTo(Value.fromString("nested")); // Verify outer record field - assertThat(deserialized.getValue(2)).contains(Value.fromInt64(123L)); + assertThat(deserialized.getValue(2)).isEqualTo(Value.fromInt64(123L)); } @Test @@ -227,6 +227,6 @@ void shouldHandleDuplicateFieldIds() throws ImprintException { var record = writer.build(); assertThat(record.getDirectory()).hasSize(1); - assertThat(record.getValue(1)).contains(Value.fromInt32(43)); + assertThat(record.getValue(1)).isEqualTo(Value.fromInt32(43)); } } \ No newline at end of file diff --git a/src/test/java/com/imprint/types/ValueTest.java b/src/test/java/com/imprint/types/ValueTest.java index 9dd99c9..c400a44 100644 --- a/src/test/java/com/imprint/types/ValueTest.java +++ b/src/test/java/com/imprint/types/ValueTest.java @@ -2,6 +2,8 @@ import org.junit.jupiter.api.Test; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.HashMap; import java.util.List; @@ -104,20 +106,113 @@ void shouldHandleEqualityCorrectly() { } @Test - void shouldDefensiveCopyArrays() { - byte[] original = {1, 2, 3}; - var bytesValue = Value.fromBytes(original); + void shouldRejectNullString() { + assertThatThrownBy(() -> Value.fromString(null)) + .isInstanceOf(NullPointerException.class); + } + + @Test + void shouldCreateStringBufferValue() { + String testString = "hello world"; + byte[] utf8Bytes = testString.getBytes(StandardCharsets.UTF_8); + ByteBuffer buffer = ByteBuffer.wrap(utf8Bytes); - // Modify original array - original[0] = 99; + Value stringBufferValue = Value.fromStringBuffer(buffer); - // Value should be unchanged - assertThat(((Value.BytesValue) bytesValue).getValue()).containsExactly(1, 2, 3); + assertThat(stringBufferValue).isInstanceOf(Value.StringBufferValue.class); + assertThat(stringBufferValue.getTypeCode()).isEqualTo(TypeCode.STRING); + assertThat(((Value.StringBufferValue) stringBufferValue).getValue()).isEqualTo(testString); } @Test - void shouldRejectNullString() { - assertThatThrownBy(() -> Value.fromString(null)) - .isInstanceOf(NullPointerException.class); + void shouldCreateBytesBufferValue() { + byte[] testBytes = {1, 2, 3, 4, 5}; + ByteBuffer buffer = ByteBuffer.wrap(testBytes); + + Value bytesBufferValue = Value.fromBytesBuffer(buffer); + + assertThat(bytesBufferValue).isInstanceOf(Value.BytesBufferValue.class); + assertThat(bytesBufferValue.getTypeCode()).isEqualTo(TypeCode.BYTES); + assertThat(((Value.BytesBufferValue) bytesBufferValue).getValue()).isEqualTo(testBytes); + } + + @Test + void shouldHandleStringBufferValueFastPath() { + // Array-backed buffer with arrayOffset() == 0 should use fast path + String testString = "fast path test"; + byte[] utf8Bytes = testString.getBytes(StandardCharsets.UTF_8); + ByteBuffer buffer = ByteBuffer.wrap(utf8Bytes); + + Value stringBufferValue = Value.fromStringBuffer(buffer); + + // Should work correctly regardless of path taken + assertThat(((Value.StringBufferValue) stringBufferValue).getValue()).isEqualTo(testString); + } + + @Test + void shouldHandleStringBufferValueFallbackPath() { + // Sliced buffer will have non-zero arrayOffset, forcing fallback path + String testString = "fallback path test"; + byte[] utf8Bytes = testString.getBytes(StandardCharsets.UTF_8); + ByteBuffer buffer = ByteBuffer.wrap(utf8Bytes); + ByteBuffer sliced = buffer.slice(); // This may break arrayOffset() == 0 + + Value stringBufferValue = Value.fromStringBuffer(sliced); + + // Should work correctly regardless of path taken + assertThat(((Value.StringBufferValue) stringBufferValue).getValue()).isEqualTo(testString); + } + + @Test + void shouldHandleLargeStringWithoutCaching() { + // Create string > 1KB to test the no-cache path + String largeString = "x".repeat(2000); + byte[] utf8Bytes = largeString.getBytes(StandardCharsets.UTF_8); + ByteBuffer buffer = ByteBuffer.wrap(utf8Bytes).slice(); // Force fallback path + + Value stringBufferValue = Value.fromStringBuffer(buffer); + + assertThat(((Value.StringBufferValue) stringBufferValue).getValue()).isEqualTo(largeString); + } + + @Test + void shouldCacheStringDecoding() { + String testString = "cache test"; + byte[] utf8Bytes = testString.getBytes(StandardCharsets.UTF_8); + ByteBuffer buffer = ByteBuffer.wrap(utf8Bytes); + + Value.StringBufferValue stringBufferValue = (Value.StringBufferValue) Value.fromStringBuffer(buffer); + + // First call should decode and cache + String result1 = stringBufferValue.getValue(); + // Second call should return cached value + String result2 = stringBufferValue.getValue(); + + assertThat(result1).isEqualTo(testString); + assertThat(result2).isEqualTo(testString); + assertThat(result1).isSameAs(result2); // Should be same object reference due to caching + } + + @Test + void shouldHandleStringValueEquality() { + String testString = "equality test"; + + Value stringValue = Value.fromString(testString); + Value stringBufferValue = Value.fromStringBuffer(ByteBuffer.wrap(testString.getBytes(StandardCharsets.UTF_8))); + + assertThat(stringValue).isEqualTo(stringBufferValue); + assertThat(stringBufferValue).isEqualTo(stringValue); + assertThat(stringValue.hashCode()).isEqualTo(stringBufferValue.hashCode()); + } + + @Test + void shouldHandleBytesValueEquality() { + byte[] testBytes = {1, 2, 3, 4, 5}; + + Value bytesValue = Value.fromBytes(testBytes); + Value bytesBufferValue = Value.fromBytesBuffer(ByteBuffer.wrap(testBytes)); + + assertThat(bytesValue).isEqualTo(bytesBufferValue); + assertThat(bytesBufferValue).isEqualTo(bytesValue); } } \ No newline at end of file From fdb8a56ee7e81ba635a349031e28b64e1ab91a44 Mon Sep 17 00:00:00 2001 From: expand3d <> Date: Tue, 3 Jun 2025 18:44:27 -0400 Subject: [PATCH 16/34] additional cleanup to address concerns in https://github.com/imprint-serde/imprint-java/issues/3 --- .../java/com/imprint/core/ImprintRecord.java | 283 +++++++++++--- .../imprint/core/ImprintRecordBuilder.java | 73 ++-- .../java/com/imprint/error/ErrorType.java | 3 +- .../java/com/imprint/types/TypeHandler.java | 5 +- src/main/java/com/imprint/util/VarInt.java | 27 +- .../imprint/ByteBufferIntegrationTest.java | 180 --------- .../java/com/imprint/IntegrationTest.java | 368 +++++++++++++----- 7 files changed, 526 insertions(+), 413 deletions(-) delete mode 100644 src/test/java/com/imprint/ByteBufferIntegrationTest.java diff --git a/src/main/java/com/imprint/core/ImprintRecord.java b/src/main/java/com/imprint/core/ImprintRecord.java index 1915707..d667039 100644 --- a/src/main/java/com/imprint/core/ImprintRecord.java +++ b/src/main/java/com/imprint/core/ImprintRecord.java @@ -3,6 +3,7 @@ import com.imprint.Constants; import com.imprint.error.ErrorType; import com.imprint.error.ImprintException; +import com.imprint.types.MapKey; import com.imprint.types.TypeCode; import com.imprint.types.Value; import com.imprint.util.VarInt; @@ -15,7 +16,7 @@ /** * An Imprint record containing a header, field directory, and payload. * Uses ByteBuffer for zero-copy operations to achieve low latency. - * + * *

Performance Note: All ByteBuffers should be array-backed * (hasArray() == true) for optimal zero-copy performance. Direct buffers * may cause performance degradation.

@@ -25,10 +26,10 @@ public final class ImprintRecord { private final Header header; private final List directory; private final ByteBuffer payload; // Read-only view for zero-copy - + /** * Creates a new ImprintRecord. - * + * * @param payload the payload buffer. Should be array-backed for optimal performance. */ public ImprintRecord(Header header, List directory, ByteBuffer payload) { @@ -40,15 +41,29 @@ public ImprintRecord(Header header, List directory, ByteBuffer p /** * Get a value by field ID, deserializing it on demand. * Returns null if the field is not found. + * Note: If the field exists and is an explicit NULL type, this will return Value.NullValue.INSTANCE. */ public Value getValue(int fieldId) throws ImprintException { var fieldBuffer = getFieldBuffer(fieldId); - if (fieldBuffer == null) return null; - - var entry = directory.get(findDirectoryIndex(fieldId)); + if (fieldBuffer == null) { + // If getFieldBuffer returns null, it means the fieldId was not in the directory, + // or an issue occurred slicing the payload (e.g., bad offsets). + return null; + } + + // findDirectoryIndex should not be negative here if fieldBuffer is not null, + // but a defensive check or ensuring findDirectoryIndex is called once is good. + // For simplicity, assume getFieldBuffer implies a valid index. + int directoryIndex = findDirectoryIndex(fieldId); + if (directoryIndex < 0) { + // This case should ideally be caught by getFieldBuffer returning null. + // If it happens, indicates an inconsistency. + throw new ImprintException(ErrorType.INTERNAL_ERROR, "Field ID " + fieldId + " found buffer but not in directory."); + } + var entry = directory.get(directoryIndex); return deserializeValue(entry.getTypeCode(), fieldBuffer); } - + /** * Get the raw bytes for a field without deserializing. * Returns a zero-copy ByteBuffer view, or null if field not found. @@ -57,7 +72,7 @@ public ByteBuffer getRawBytes(int fieldId) { var fieldBuffer = getFieldBuffer(fieldId); return fieldBuffer != null ? fieldBuffer.asReadOnlyBuffer() : null; } - + /** * Get a ByteBuffer view of a field's data. * Returns null if the field is not found. @@ -65,12 +80,12 @@ public ByteBuffer getRawBytes(int fieldId) { private ByteBuffer getFieldBuffer(int fieldId) { int index = findDirectoryIndex(fieldId); if (index < 0) return null; - + var entry = directory.get(index); int startOffset = entry.getOffset(); - int endOffset = (index + 1 < directory.size()) ? - directory.get(index + 1).getOffset() : payload.limit(); - + int endOffset = (index + 1 < directory.size()) ? + directory.get(index + 1).getOffset() : payload.limit(); + var fieldBuffer = payload.duplicate(); if (startOffset > payload.limit() || endOffset > payload.limit() || startOffset > endOffset) { return null; @@ -78,104 +93,106 @@ private ByteBuffer getFieldBuffer(int fieldId) { fieldBuffer.position(startOffset).limit(endOffset); return fieldBuffer.slice(); } - + /** - * Serialize this record to a ByteBuffer (zero-copy when possible). + * Serialize this record to a ByteBuffer. + * The returned buffer will be array-backed. */ public ByteBuffer serializeToBuffer() { var buffer = ByteBuffer.allocate(estimateSerializedSize()); buffer.order(ByteOrder.LITTLE_ENDIAN); - + // Write header serializeHeader(buffer); - + // Write directory (always present) VarInt.encode(directory.size(), buffer); for (var entry : directory) { serializeDirectoryEntry(entry, buffer); } - + // Write payload (shallow copy only) var payloadCopy = payload.duplicate(); buffer.put(payloadCopy); - - // Return read-only view of used portion + + // Prepare buffer for reading buffer.flip(); - return buffer.asReadOnlyBuffer(); + return buffer; } - + /** * Create a fluent builder for constructing ImprintRecord instances. - * + * * @param schemaId the schema identifier for this record * @return a new builder instance */ public static ImprintRecordBuilder builder(SchemaId schemaId) { return new ImprintRecordBuilder(schemaId); } - + /** * Create a fluent builder for constructing ImprintRecord instances. - * + * * @param fieldspaceId the fieldspace identifier * @param schemaHash the schema hash * @return a new builder instance */ + @SuppressWarnings("unused") public static ImprintRecordBuilder builder(int fieldspaceId, int schemaHash) { return new ImprintRecordBuilder(new SchemaId(fieldspaceId, schemaHash)); } - + /** * Deserialize a record from bytes through an array backed ByteBuffer. */ public static ImprintRecord deserialize(byte[] bytes) throws ImprintException { return deserialize(ByteBuffer.wrap(bytes)); } - + /** * Deserialize a record from a ByteBuffer. - * - * @param buffer the buffer to deserialize from. Must be array-backed + * + * @param buffer the buffer to deserialize from. Must be array-backed * (buffer.hasArray() == true) for optimal zero-copy performance. */ public static ImprintRecord deserialize(ByteBuffer buffer) throws ImprintException { buffer = buffer.duplicate().order(ByteOrder.LITTLE_ENDIAN); - + // Read header var header = deserializeHeader(buffer); - + // Read directory (always present) var directory = new ArrayList(); VarInt.DecodeResult countResult = VarInt.decode(buffer); int directoryCount = countResult.getValue(); - + for (int i = 0; i < directoryCount; i++) { directory.add(deserializeDirectoryEntry(buffer)); } - + // Read payload as ByteBuffer slice for zero-copy var payload = buffer.slice(); payload.limit(header.getPayloadSize()); buffer.position(buffer.position() + header.getPayloadSize()); - + return new ImprintRecord(header, directory, payload); } - + /** * Binary search for field ID in directory without object allocation. * Returns the index of the field if found, or a negative value if not found. - * + * * @param fieldId the field ID to search for * @return index if found, or negative insertion point - 1 if not found */ private int findDirectoryIndex(int fieldId) { int low = 0; int high = directory.size() - 1; - + while (low <= high) { int mid = (low + high) >>> 1; // unsigned right shift to avoid overflow int midFieldId = directory.get(mid).getId(); - + if (midFieldId < fieldId) { low = mid + 1; } else if (midFieldId > fieldId) { @@ -186,7 +203,7 @@ private int findDirectoryIndex(int fieldId) { } return -(low + 1); // field not found, return insertion point } - + public int estimateSerializedSize() { int size = Constants.HEADER_BYTES; // header size += VarInt.encodedLength(directory.size()); // directory count @@ -194,7 +211,7 @@ public int estimateSerializedSize() { size += payload.remaining(); // payload return size; } - + private void serializeHeader(ByteBuffer buffer) { buffer.put(Constants.MAGIC); buffer.put(Constants.VERSION); @@ -203,57 +220,57 @@ private void serializeHeader(ByteBuffer buffer) { buffer.putInt(header.getSchemaId().getSchemaHash()); buffer.putInt(header.getPayloadSize()); } - + private static Header deserializeHeader(ByteBuffer buffer) throws ImprintException { if (buffer.remaining() < Constants.HEADER_BYTES) { - throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, - "Not enough bytes for header"); + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, + "Not enough bytes for header"); } - + byte magic = buffer.get(); if (magic != Constants.MAGIC) { - throw new ImprintException(ErrorType.INVALID_MAGIC, - "Invalid magic byte: expected 0x" + Integer.toHexString(Constants.MAGIC) + - ", got 0x" + Integer.toHexString(magic & 0xFF)); + throw new ImprintException(ErrorType.INVALID_MAGIC, + "Invalid magic byte: expected 0x" + Integer.toHexString(Constants.MAGIC) + + ", got 0x" + Integer.toHexString(magic & 0xFF)); } - + byte version = buffer.get(); if (version != Constants.VERSION) { - throw new ImprintException(ErrorType.UNSUPPORTED_VERSION, - "Unsupported version: " + version); + throw new ImprintException(ErrorType.UNSUPPORTED_VERSION, + "Unsupported version: " + version); } - + var flags = new Flags(buffer.get()); int fieldspaceId = buffer.getInt(); int schemaHash = buffer.getInt(); int payloadSize = buffer.getInt(); - + return new Header(flags, new SchemaId(fieldspaceId, schemaHash), payloadSize); } - + private void serializeDirectoryEntry(DirectoryEntry entry, ByteBuffer buffer) { buffer.putShort(entry.getId()); buffer.put(entry.getTypeCode().getCode()); buffer.putInt(entry.getOffset()); } - + private static DirectoryEntry deserializeDirectoryEntry(ByteBuffer buffer) throws ImprintException { if (buffer.remaining() < Constants.DIR_ENTRY_BYTES) { - throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, - "Not enough bytes for directory entry"); + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, + "Not enough bytes for directory entry"); } - + short id = buffer.getShort(); var typeCode = TypeCode.fromByte(buffer.get()); int offset = buffer.getInt(); - + return new DirectoryEntry(id, typeCode, offset); } - + private Value deserializeValue(TypeCode typeCode, ByteBuffer buffer) throws ImprintException { ByteBuffer valueSpecificBuffer = buffer.duplicate(); valueSpecificBuffer.order(ByteOrder.LITTLE_ENDIAN); - + switch (typeCode) { case NULL: case BOOL: @@ -274,10 +291,152 @@ private Value deserializeValue(TypeCode typeCode, ByteBuffer buffer) throws Impr throw new ImprintException(ErrorType.INVALID_TYPE_CODE, "Unknown type code: " + typeCode); } } - + @Override public String toString() { - return String.format("ImprintRecord{header=%s, directorySize=%d, payloadSize=%d}", - header, directory.size(), payload.remaining()); + return String.format("ImprintRecord{header=%s, directorySize=%d, payloadSize=%d}", + header, directory.size(), payload.remaining()); + } + + private T getTypedValueOrThrow(int fieldId, TypeCode expectedTypeCode, Class expectedValueClass, String expectedTypeName) throws ImprintException { + Value value = getValue(fieldId); + + if (value == null) { + throw new ImprintException(ErrorType.FIELD_NOT_FOUND, + "Field " + fieldId + " not found, cannot retrieve as " + expectedTypeName + "."); + } + + if (value.getTypeCode() == TypeCode.NULL) { + throw new ImprintException(ErrorType.TYPE_MISMATCH, + "Field " + fieldId + " is NULL, cannot retrieve as " + expectedTypeName + "."); + } + + if (value.getTypeCode() == expectedTypeCode && expectedValueClass.isInstance(value)) { + return expectedValueClass.cast(value); + } + + throw new ImprintException(ErrorType.TYPE_MISMATCH, + "Field " + fieldId + " is of type " + value.getTypeCode() + ", expected " + expectedTypeName + "."); + } + + /** + * Retrieves the boolean value for the given field ID. + * @throws ImprintException if the field is not found, is null, or is not of type BOOL. + */ + public boolean getBoolean(int fieldId) throws ImprintException { + return getTypedValueOrThrow(fieldId, TypeCode.BOOL, Value.BoolValue.class, "boolean").getValue(); + } + + /** + * Retrieves the int (int32) value for the given field ID. + * @throws ImprintException if the field is not found, is null, or is not of type INT32. + */ + public int getInt32(int fieldId) throws ImprintException { + return getTypedValueOrThrow(fieldId, TypeCode.INT32, Value.Int32Value.class, "int32").getValue(); + } + + /** + * Retrieves the long (int64) value for the given field ID. + * @throws ImprintException if the field is not found, is null, or is not of type INT64. + */ + public long getInt64(int fieldId) throws ImprintException { + return getTypedValueOrThrow(fieldId, TypeCode.INT64, Value.Int64Value.class, "int64").getValue(); + } + + /** + * Retrieves the float (float32) value for the given field ID. + * @throws ImprintException if the field is not found, is null, or is not of type FLOAT32. + */ + public float getFloat32(int fieldId) throws ImprintException { + return getTypedValueOrThrow(fieldId, TypeCode.FLOAT32, Value.Float32Value.class, "float32").getValue(); + } + + /** + * Retrieves the double (float64) value for the given field ID. + * @throws ImprintException if the field is not found, is null, or is not of type FLOAT64. + */ + public double getFloat64(int fieldId) throws ImprintException { + return getTypedValueOrThrow(fieldId, TypeCode.FLOAT64, Value.Float64Value.class, "float64").getValue(); + } + + /** + * Retrieves the String value for the given field ID. + * @throws ImprintException if the field is not found, is null, or is not of type STRING. + */ + public String getString(int fieldId) throws ImprintException { + Value value = getValue(fieldId); + + if (value == null) { + throw new ImprintException(ErrorType.FIELD_NOT_FOUND, + "Field " + fieldId + " not found, cannot retrieve as String."); + } + if (value.getTypeCode() == TypeCode.NULL) { + throw new ImprintException(ErrorType.TYPE_MISMATCH, + "Field " + fieldId + " is NULL, cannot retrieve as String."); + } + + if (value instanceof Value.StringValue) { + return ((Value.StringValue) value).getValue(); + } + if (value instanceof Value.StringBufferValue) { + return ((Value.StringBufferValue) value).getValue(); + } + + throw new ImprintException(ErrorType.TYPE_MISMATCH, + "Field " + fieldId + " is of type " + value.getTypeCode() + ", expected STRING."); + } + + /** + * Retrieves the byte array (byte[]) value for the given field ID. + * Note: This may involve a defensive copy depending on the underlying Value type. + * @throws ImprintException if the field is not found, is null, or is not of type BYTES. + */ + public byte[] getBytes(int fieldId) throws ImprintException { + Value value = getValue(fieldId); + + if (value == null) { + throw new ImprintException(ErrorType.FIELD_NOT_FOUND, + "Field " + fieldId + " not found, cannot retrieve as byte[]."); + } + if (value.getTypeCode() == TypeCode.NULL) { + throw new ImprintException(ErrorType.TYPE_MISMATCH, + "Field " + fieldId + " is NULL, cannot retrieve as byte[]."); + } + + if (value instanceof Value.BytesValue) { + return ((Value.BytesValue) value).getValue(); // getValue() in BytesValue returns a clone + } + if (value instanceof Value.BytesBufferValue) { + return ((Value.BytesBufferValue) value).getValue(); // getValue() in BytesBufferValue creates a new array + } + + throw new ImprintException(ErrorType.TYPE_MISMATCH, + "Field " + fieldId + " is of type " + value.getTypeCode() + ", expected BYTES."); + } + + /** + * Retrieves the List for the given field ID. + * The list itself is a copy; modifications to it will not affect the record. + * @throws ImprintException if the field is not found, is null, or is not of type ARRAY. + */ + public List getArray(int fieldId) throws ImprintException { + return getTypedValueOrThrow(fieldId, TypeCode.ARRAY, Value.ArrayValue.class, "ARRAY").getValue(); + } + + /** + * Retrieves the Map for the given field ID. + * The map itself is a copy; modifications to it will not affect the record. + * @throws ImprintException if the field is not found, is null, or is not of type MAP. + */ + public Map getMap(int fieldId) throws ImprintException { + return getTypedValueOrThrow(fieldId, TypeCode.MAP, Value.MapValue.class, "MAP").getValue(); + } + + /** + * Retrieves the nested ImprintRecord for the given field ID. + * @throws ImprintException if the field is not found, is null, or is not of type ROW. + */ + public ImprintRecord getRow(int fieldId) throws ImprintException { + return getTypedValueOrThrow(fieldId, TypeCode.ROW, Value.RowValue.class, "ROW").getValue(); } } \ No newline at end of file diff --git a/src/main/java/com/imprint/core/ImprintRecordBuilder.java b/src/main/java/com/imprint/core/ImprintRecordBuilder.java index 8c04bf5..51a3525 100644 --- a/src/main/java/com/imprint/core/ImprintRecordBuilder.java +++ b/src/main/java/com/imprint/core/ImprintRecordBuilder.java @@ -31,40 +31,40 @@ public final class ImprintRecordBuilder { private final SchemaId schemaId; private final Map fields = new TreeMap<>(); - + ImprintRecordBuilder(SchemaId schemaId) { this.schemaId = Objects.requireNonNull(schemaId, "SchemaId cannot be null"); } - + // Primitive types with automatic Value wrapping public ImprintRecordBuilder field(int id, boolean value) { return addField(id, Value.fromBoolean(value)); } - + public ImprintRecordBuilder field(int id, int value) { return addField(id, Value.fromInt32(value)); } - + public ImprintRecordBuilder field(int id, long value) { return addField(id, Value.fromInt64(value)); } - + public ImprintRecordBuilder field(int id, float value) { return addField(id, Value.fromFloat32(value)); } - + public ImprintRecordBuilder field(int id, double value) { return addField(id, Value.fromFloat64(value)); } - + public ImprintRecordBuilder field(int id, String value) { return addField(id, Value.fromString(value)); } - + public ImprintRecordBuilder field(int id, byte[] value) { return addField(id, Value.fromBytes(value)); } - + // Collections with automatic conversion public ImprintRecordBuilder field(int id, List values) { var convertedValues = new ArrayList(values.size()); @@ -73,7 +73,7 @@ public ImprintRecordBuilder field(int id, List values) { } return addField(id, Value.fromArray(convertedValues)); } - + public ImprintRecordBuilder field(int id, Map map) { var convertedMap = new HashMap(map.size()); for (var entry : map.entrySet()) { @@ -83,22 +83,22 @@ public ImprintRecordBuilder field(int id, Map fieldsMap) { for (var entry : fieldsMap.entrySet()) { @@ -118,58 +118,53 @@ public ImprintRecordBuilder fields(Map fieldsMap) { } return this; } - + // Builder utilities public boolean hasField(int id) { return fields.containsKey(id); } - + public int fieldCount() { return fields.size(); } - + public Set fieldIds() { return new TreeSet<>(fields.keySet()); } - + // Build the final record public ImprintRecord build() throws ImprintException { - if (fields.isEmpty()) { - throw new ImprintException(com.imprint.error.ErrorType.SCHEMA_ERROR, - "Cannot build empty record - add at least one field"); - } - var writer = new ImprintWriter(schemaId); for (var entry : fields.entrySet()) { writer.addField(entry.getKey(), entry.getValue()); } return writer.build(); } - + // Internal helper methods /** * Adds or overwrites a field in the record being built. * If a field with the given ID already exists, it will be replaced. - * + * * @param id the field ID * @param value the field value (cannot be null - use nullField() for explicit nulls) * @return this builder for method chaining */ private ImprintRecordBuilder addField(int id, Value value) { Objects.requireNonNull(value, "Value cannot be null - use nullField() for explicit null values"); - fields.put(id, value); // TreeMap.put() overwrites existing values + fields.put(id, value); return this; } - + private Value convertToValue(Object obj) { if (obj == null) { return Value.nullValue(); } - + if (obj instanceof Value) { return (Value) obj; } - + // Auto-boxing conversion if (obj instanceof Boolean) { return Value.fromBoolean((Boolean) obj); @@ -216,11 +211,11 @@ private Value convertToValue(Object obj) { if (obj instanceof ImprintRecord) { return Value.fromRow((ImprintRecord) obj); } - - throw new IllegalArgumentException("Cannot convert " + obj.getClass().getSimpleName() + - " to Imprint Value. Supported types: boolean, int, long, float, double, String, byte[], List, Map, ImprintRecord"); + + throw new IllegalArgumentException("Cannot convert " + obj.getClass().getSimpleName() + + " to Imprint Value. Supported types: boolean, int, long, float, double, String, byte[], List, Map, ImprintRecord"); } - + private MapKey convertToMapKey(Object obj) { if (obj instanceof Integer) { return MapKey.fromInt32((Integer) obj); @@ -234,11 +229,11 @@ private MapKey convertToMapKey(Object obj) { if (obj instanceof byte[]) { return MapKey.fromBytes((byte[]) obj); } - - throw new IllegalArgumentException("Invalid map key type: " + obj.getClass().getSimpleName() + - ". Map keys must be int, long, String, or byte[]"); + + throw new IllegalArgumentException("Invalid map key type: " + obj.getClass().getSimpleName() + + ". Map keys must be int, long, String, or byte[]"); } - + @Override public String toString() { return String.format("ImprintRecordBuilder{schemaId=%s, fields=%d}", schemaId, fields.size()); diff --git a/src/main/java/com/imprint/error/ErrorType.java b/src/main/java/com/imprint/error/ErrorType.java index 97b9772..49784ef 100644 --- a/src/main/java/com/imprint/error/ErrorType.java +++ b/src/main/java/com/imprint/error/ErrorType.java @@ -14,5 +14,6 @@ public enum ErrorType { TYPE_MISMATCH, INVALID_TYPE_CODE, SERIALIZATION_ERROR, - DESERIALIZATION_ERROR + DESERIALIZATION_ERROR, + INTERNAL_ERROR } diff --git a/src/main/java/com/imprint/types/TypeHandler.java b/src/main/java/com/imprint/types/TypeHandler.java index 573aed3..d21403b 100644 --- a/src/main/java/com/imprint/types/TypeHandler.java +++ b/src/main/java/com/imprint/types/TypeHandler.java @@ -20,8 +20,7 @@ public interface TypeHandler { ByteBuffer readValueBytes(ByteBuffer buffer) throws ImprintException; // Helper method to eliminate duplication in ARRAY/MAP readValueBytes - static ByteBuffer readComplexValueBytes(ByteBuffer buffer, String typeName, - ComplexValueMeasurer measurer) throws ImprintException { + static ByteBuffer readComplexValueBytes(ByteBuffer buffer, String typeName, BufferViewer measurer) throws ImprintException { int initialPosition = buffer.position(); ByteBuffer tempBuffer = buffer.duplicate(); tempBuffer.order(buffer.order()); @@ -58,7 +57,7 @@ static ByteBuffer readComplexValueBytes(ByteBuffer buffer, String typeName, } @FunctionalInterface - interface ComplexValueMeasurer { + interface BufferViewer { int measureDataLength(ByteBuffer tempBuffer, int numElements) throws ImprintException; } diff --git a/src/main/java/com/imprint/util/VarInt.java b/src/main/java/com/imprint/util/VarInt.java index f4c22f2..75bd132 100644 --- a/src/main/java/com/imprint/util/VarInt.java +++ b/src/main/java/com/imprint/util/VarInt.java @@ -6,20 +6,21 @@ import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.ToString; +import lombok.experimental.UtilityClass; + import java.nio.ByteBuffer; /** * Utility class for encoding and decoding variable-length integers (VarInt). * Supports encoding/decoding of 32-bit unsigned integers. */ +@UtilityClass public final class VarInt { private static final byte CONTINUATION_BIT = (byte) 0x80; private static final byte SEGMENT_BITS = 0x7f; private static final int MAX_VARINT_LEN = 5; // Enough for u32 - private VarInt() {} // utility class - /** * Encode a 32-bit unsigned integer as a VarInt into the given ByteBuffer. @@ -103,27 +104,7 @@ public static int encodedLength(int value) { return length; } - - /** - * Read VarInt-prefixed data from a ByteBuffer. - * The data format is: VarInt(length) + data(length bytes). - * Returns a read-only ByteBuffer containing the entire VarInt + data. - * - * @param buffer the buffer to read from - * @return a read-only ByteBuffer view of the VarInt + data - * @throws ImprintException if the VarInt is malformed or buffer underflow - */ - public static ByteBuffer readVarIntPrefixedBytes(ByteBuffer buffer) throws ImprintException { - int originalPosition = buffer.position(); - VarInt.DecodeResult lengthResult = VarInt.decode(buffer); - int totalLength = lengthResult.getBytesRead() + lengthResult.getValue(); - buffer.position(originalPosition); - var valueBuffer = buffer.slice(); - valueBuffer.limit(totalLength); - buffer.position(buffer.position() + totalLength); - return valueBuffer.asReadOnlyBuffer(); - } - + /** * Result of a VarInt decode operation. */ diff --git a/src/test/java/com/imprint/ByteBufferIntegrationTest.java b/src/test/java/com/imprint/ByteBufferIntegrationTest.java deleted file mode 100644 index 56ec3b0..0000000 --- a/src/test/java/com/imprint/ByteBufferIntegrationTest.java +++ /dev/null @@ -1,180 +0,0 @@ -package com.imprint; - -import com.imprint.core.ImprintRecord; -import com.imprint.core.ImprintWriter; -import com.imprint.core.SchemaId; -import com.imprint.error.ImprintException; -import com.imprint.types.MapKey; -import com.imprint.types.Value; - -import java.nio.ByteBuffer; -import java.util.*; - -/** - * Integration test to verify ByteBuffer functionality and zero-copy benefits. - */ -class ByteBufferIntegrationTest { - - public static void main(String[] args) { - try { - testByteBufferFunctionality(); - testZeroCopy(); - testArrayBackedBuffers(); - System.out.println("All ByteBuffer integration tests passed!"); - } catch (Exception e) { - System.err.println("ByteBuffer integration test failed: " + e.getMessage()); - e.printStackTrace(); - System.exit(1); - } - } - - static void testByteBufferFunctionality() throws ImprintException { - System.out.println("Testing ByteBuffer functionality..."); - - SchemaId schemaId = new SchemaId(1, 0xdeadbeef); - ImprintWriter writer = new ImprintWriter(schemaId); - - writer.addField(1, Value.fromInt32(42)) - .addField(2, Value.fromString("zero-copy test")) - .addField(3, Value.fromBytes(new byte[]{1, 2, 3, 4, 5})); - - ImprintRecord record = writer.build(); - - // Test ByteBuffer serialization - ByteBuffer serializedBuffer = record.serializeToBuffer(); - assert serializedBuffer.isReadOnly() : "Serialized buffer should be read-only"; - - // Test deserialization from ByteBuffer - ImprintRecord deserialized = ImprintRecord.deserialize(serializedBuffer); - - assert Objects.equals(deserialized.getValue(1), Value.fromInt32(42)); - assert Objects.equals(deserialized.getValue(2), Value.fromString("zero-copy test")); - - // Test raw bytes access returns ByteBuffer - var rawBytes = deserialized.getRawBytes(1); - assert rawBytes != null : "Raw bytes should be present for field 1"; - assert rawBytes.isReadOnly() : "Raw bytes buffer should be read-only"; - - System.out.println("ByteBuffer functionality test passed"); - } - - static void testZeroCopy() { - System.out.println("Testing zero-copy"); - - // Create a large payload to demonstrate zero-copy benefits - byte[] largePayload = new byte[1024 * 1024]; // 1MB - Arrays.fill(largePayload, (byte) 0xAB); - - SchemaId schemaId = new SchemaId(2, 0xcafebabe); - ImprintWriter writer = new ImprintWriter(schemaId); - - try { - writer.addField(1, Value.fromBytes(largePayload)); - ImprintRecord record = writer.build(); - - // Test that getRawBytes returns a view, not a copy - var rawBytes = record.getRawBytes(1); - assert rawBytes !=null : "Raw bytes should be present"; - assert rawBytes.isReadOnly() : "Raw buffer should be read-only"; - - // The buffer should be positioned at the start of the actual data - // (after the VarInt length prefix) - assert rawBytes.remaining() > largePayload.length : "Buffer should include length prefix"; - - System.out.println("Zero-copy benefits test passed"); - - } catch (ImprintException e) { - throw new RuntimeException("Failed zero-copy test", e); - } - } - - static void testArrayBackedBuffers() throws ImprintException { - System.out.println("Testing array-backed buffers for zero-copy performance..."); - - // Test serialized buffers are array-backed - SchemaId schemaId = new SchemaId(1, 0xdeadbeef); - ImprintWriter writer = new ImprintWriter(schemaId); - - writer.addField(1, Value.fromInt32(42)) - .addField(2, Value.fromString("test string")) - .addField(3, Value.fromBytes(new byte[]{1, 2, 3, 4})) - .addField(4, Value.fromBoolean(true)); - - ImprintRecord record = writer.build(); - - // Test that serializeToBuffer() returns array-backed buffer - ByteBuffer serializedBuffer = record.serializeToBuffer(); - assert serializedBuffer.hasArray() : "Serialized buffer should be array-backed for zero-copy performance"; - - // Test that the internal payload is array-backed - assert record.getPayload().hasArray() : "Record payload should be array-backed for zero-copy performance"; - - // Test deserialized buffers are array-backed - byte[] bytes = new byte[serializedBuffer.remaining()]; - serializedBuffer.get(bytes); - ImprintRecord deserialized = ImprintRecord.deserialize(bytes); - - assert deserialized.getPayload().hasArray() : "Deserialized record payload should be array-backed"; - - // Test that getRawBytes() returns array-backed buffers - ByteBuffer rawBytes1 = deserialized.getRawBytes(1); - ByteBuffer rawBytes2 = deserialized.getRawBytes(2); - - assert rawBytes1 != null && rawBytes1.hasArray() : "Raw bytes buffer for int field should be array-backed"; - assert rawBytes2 != null && rawBytes2.hasArray() : "Raw bytes buffer for string field should be array-backed"; - - // Test complex types use array-backed buffers - ImprintWriter complexWriter = new ImprintWriter(new SchemaId(2, 0xcafebabe)); - - // Create array value - List arrayValues = Arrays.asList( - Value.fromInt32(1), - Value.fromInt32(2), - Value.fromInt32(3) - ); - - // Create map value - Map mapValues = new HashMap<>(); - mapValues.put(MapKey.fromString("key1"), Value.fromString("value1")); - mapValues.put(MapKey.fromString("key2"), Value.fromString("value2")); - - complexWriter.addField(1, Value.fromArray(arrayValues)) - .addField(2, Value.fromMap(mapValues)); - - ImprintRecord complexRecord = complexWriter.build(); - - assert complexRecord.getPayload().hasArray() : "Record with complex types should use array-backed payload"; - - ByteBuffer complexSerialized = complexRecord.serializeToBuffer(); - assert complexSerialized.hasArray() : "Serialized buffer with complex types should be array-backed"; - - // Test nested records use array-backed buffers - ImprintWriter innerWriter = new ImprintWriter(new SchemaId(3, 0x12345678)); - innerWriter.addField(1, Value.fromString("nested data")); - ImprintRecord innerRecord = innerWriter.build(); - - ImprintWriter outerWriter = new ImprintWriter(new SchemaId(4, 0x87654321)); - outerWriter.addField(1, Value.fromRow(innerRecord)); - ImprintRecord outerRecord = outerWriter.build(); - - assert innerRecord.getPayload().hasArray() : "Inner record payload should be array-backed"; - assert outerRecord.getPayload().hasArray() : "Outer record payload should be array-backed"; - - ByteBuffer nestedSerialized = outerRecord.serializeToBuffer(); - assert nestedSerialized.hasArray() : "Serialized nested record should be array-backed"; - - // Test builder pattern uses array-backed buffers - ImprintRecord builderRecord = ImprintRecord.builder(1, 0xabcdef00) - .field(1, "test string") - .field(2, 42) - .field(3, new byte[]{1, 2, 3}) - .build(); - - assert builderRecord.getPayload().hasArray() : "Builder-created record should use array-backed payload"; - - ByteBuffer builderSerialized = builderRecord.serializeToBuffer(); - assert builderSerialized.hasArray() : "Builder-created serialized buffer should be array-backed"; - - System.out.println("✓ Array-backed buffers test passed"); - } -} \ No newline at end of file diff --git a/src/test/java/com/imprint/IntegrationTest.java b/src/test/java/com/imprint/IntegrationTest.java index 565b7cd..76efcc5 100644 --- a/src/test/java/com/imprint/IntegrationTest.java +++ b/src/test/java/com/imprint/IntegrationTest.java @@ -2,143 +2,301 @@ import com.imprint.core.*; import com.imprint.types.*; +import com.imprint.error.ErrorType; import com.imprint.error.ImprintException; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.DisplayName; + import java.util.*; +import static org.junit.jupiter.api.Assertions.*; + /** - * Integration test to verify the complete Java implementation works. - * This can be run as a simple main method without JUnit. + * Integration tests for Imprint core functionality. */ public class IntegrationTest { - - public static void main(String[] args) { - try { - testBasicFunctionality(); - testArraysAndMaps(); - testNestedRecords(); - System.out.println("All integration tests passed!"); - } catch (Exception e) { - System.err.println("Integration test failed: " + e.getMessage()); - e.printStackTrace(); - System.exit(1); - } - } - - static void testBasicFunctionality() throws ImprintException { - System.out.println("Testing basic functionality..."); - + + // Removed main method, individual methods are now JUnit tests. + + @Test + @DisplayName("Basic functionality: create, serialize, deserialize primitive types") + void testBasicFunctionality() throws ImprintException { + System.out.println("Testing basic functionality..."); // Keep for now if desired, or remove + SchemaId schemaId = new SchemaId(1, 0xdeadbeef); - ImprintWriter writer = new ImprintWriter(schemaId); - - writer.addField(1, Value.fromInt32(42)) - .addField(2, Value.fromString("testing java imprint spec")) - .addField(3, Value.fromBoolean(true)) - .addField(4, Value.fromFloat64(3.14159)) - .addField(5, Value.fromBytes(new byte[]{1, 2, 3, 4})); - - ImprintRecord record = writer.build(); - - // Verify we can read values back - assert Objects.equals(record.getValue(1), Value.fromInt32(42)); - assert Objects.equals(record.getValue(2), Value.fromString("testing java imprint spec")); - assert Objects.equals(record.getValue(3), Value.fromBoolean(true)); - assert record.getValue(999) == null; // non-existent field - + // Using ImprintRecordBuilder for consistency with other tests + ImprintRecord record = ImprintRecord.builder(schemaId) + .field(1, 42) + .field(2, "testing java imprint spec") + .field(3, true) + .field(4, 3.14159) // double + .field(5, new byte[]{1, 2, 3, 4}) + .build(); + + // Verify we can read values back using ergonomic getters + assertEquals(42, record.getInt32(1)); + assertEquals("testing java imprint spec", record.getString(2)); + assertTrue(record.getBoolean(3)); + assertEquals(3.14159, record.getFloat64(4)); + assertArrayEquals(new byte[]{1,2,3,4}, record.getBytes(5)); + + assertNull(record.getValue(999), "Non-existent field should return null from getValue()"); + assertThrows(ImprintException.class, () -> record.getInt32(999), "Accessing non-existent field with getInt32 should throw"); + // Test serialization round-trip var buffer = record.serializeToBuffer(); byte[] serialized = new byte[buffer.remaining()]; buffer.get(serialized); ImprintRecord deserialized = ImprintRecord.deserialize(serialized); - - assert Objects.equals(deserialized.getValue(1), Value.fromInt32(42)); - assert Objects.equals(deserialized.getValue(2), Value.fromString("testing java imprint spec")); - assert Objects.equals(deserialized.getValue(3), Value.fromBoolean(true)); - + + assertEquals(42, deserialized.getInt32(1)); + assertEquals("testing java imprint spec", deserialized.getString(2)); + assertTrue(deserialized.getBoolean(3)); + assertEquals(3.14159, deserialized.getFloat64(4)); + assertArrayEquals(new byte[]{1,2,3,4}, deserialized.getBytes(5)); + System.out.println("Basic functionality test passed"); } - - static void testArraysAndMaps() throws ImprintException { + + @Test + @DisplayName("Collections: create, serialize, deserialize arrays and maps") + void testArraysAndMaps() throws ImprintException { System.out.println("Testing arrays and maps..."); - + SchemaId schemaId = new SchemaId(2, 0xcafebabe); - ImprintWriter writer = new ImprintWriter(schemaId); - - // Create an array - List intArray = Arrays.asList( - Value.fromInt32(1), - Value.fromInt32(2), - Value.fromInt32(3) - ); - + + // Create an array using builder for convenience + List sourceIntList = Arrays.asList(1, 2, 3); + // Create a map - Map stringToIntMap = new HashMap<>(); - stringToIntMap.put(MapKey.fromString("one"), Value.fromInt32(1)); - stringToIntMap.put(MapKey.fromString("two"), Value.fromInt32(2)); - - writer.addField(1, Value.fromArray(intArray)) - .addField(2, Value.fromMap(stringToIntMap)); - - ImprintRecord record = writer.build(); - + Map sourceStringToIntMap = new HashMap<>(); + sourceStringToIntMap.put("one", 1); + sourceStringToIntMap.put("two", 2); + + ImprintRecord record = ImprintRecord.builder(schemaId) + .field(1, sourceIntList) // Builder converts List to List + .field(2, sourceStringToIntMap) // Builder converts Map + .build(); + // Test serialization round-trip var buffer = record.serializeToBuffer(); byte[] serialized = new byte[buffer.remaining()]; buffer.get(serialized); ImprintRecord deserialized = ImprintRecord.deserialize(serialized); - + // Verify array - Value arrayValue = deserialized.getValue(1); - assert arrayValue instanceof Value.ArrayValue; - List deserializedArray = ((Value.ArrayValue) arrayValue).getValue(); - assert deserializedArray.size() == 3; - assert deserializedArray.get(0).equals(Value.fromInt32(1)); - + List deserializedArray = deserialized.getArray(1); + assertNotNull(deserializedArray); + assertEquals(3, deserializedArray.size()); + assertEquals(Value.fromInt32(1), deserializedArray.get(0)); + assertEquals(Value.fromInt32(2), deserializedArray.get(1)); + assertEquals(Value.fromInt32(3), deserializedArray.get(2)); + // Verify map - Value mapValue = deserialized.getValue(2); - assert mapValue instanceof Value.MapValue; - Map deserializedMap = ((Value.MapValue) mapValue).getValue(); - assert deserializedMap.size() == 2; - assert deserializedMap.get(MapKey.fromString("one")).equals(Value.fromInt32(1)); - - System.out.println("✓ Arrays and maps test passed"); + Map deserializedMap = deserialized.getMap(2); + assertNotNull(deserializedMap); + assertEquals(2, deserializedMap.size()); + assertEquals(Value.fromInt32(1), deserializedMap.get(MapKey.fromString("one"))); + assertEquals(Value.fromInt32(2), deserializedMap.get(MapKey.fromString("two"))); + + System.out.println("Arrays and maps test passed"); } - - static void testNestedRecords() throws ImprintException { + + @Test + @DisplayName("Nested Records: create, serialize, deserialize records within records") + void testNestedRecords() throws ImprintException { System.out.println("Testing nested records..."); - - // Create inner record + SchemaId innerSchemaId = new SchemaId(3, 0x12345678); - ImprintWriter innerWriter = new ImprintWriter(innerSchemaId); - innerWriter.addField(1, Value.fromString("nested data")) - .addField(2, Value.fromInt64(9876543210L)); - ImprintRecord innerRecord = innerWriter.build(); - - // Create outer record + ImprintRecord innerRecord = ImprintRecord.builder(innerSchemaId) + .field(1, "nested data") + .field(2, 9876543210L) + .build(); + SchemaId outerSchemaId = new SchemaId(4, 0x87654321); - ImprintWriter outerWriter = new ImprintWriter(outerSchemaId); - outerWriter.addField(1, Value.fromRow(innerRecord)) - .addField(2, Value.fromString("outer data")); - ImprintRecord outerRecord = outerWriter.build(); - - // Test serialization round-trip + ImprintRecord outerRecord = ImprintRecord.builder(outerSchemaId) + .field(1, innerRecord) // Builder handles ImprintRecord directly + .field(2, "outer data") + .build(); + var buffer = outerRecord.serializeToBuffer(); byte[] serialized = new byte[buffer.remaining()]; buffer.get(serialized); ImprintRecord deserialized = ImprintRecord.deserialize(serialized); - - // Verify outer record - assert deserialized.getHeader().getSchemaId().getFieldSpaceId() == 4; - assert Objects.equals(deserialized.getValue(2), Value.fromString("outer data")); - - // Verify nested record - Value rowValue = deserialized.getValue(1); - assert rowValue instanceof Value.RowValue; - ImprintRecord nestedRecord = ((Value.RowValue) rowValue).getValue(); - - assert nestedRecord.getHeader().getSchemaId().getFieldSpaceId() == 3; - assert Objects.equals(nestedRecord.getValue(1), Value.fromString("nested data")); - assert Objects.equals(nestedRecord.getValue(2), Value.fromInt64(9876543210L)); - + + assertEquals(4, deserialized.getHeader().getSchemaId().getFieldSpaceId()); + assertEquals("outer data", deserialized.getString(2)); + + ImprintRecord nestedDeserialized = deserialized.getRow(1); + assertNotNull(nestedDeserialized); + assertEquals(3, nestedDeserialized.getHeader().getSchemaId().getFieldSpaceId()); + assertEquals("nested data", nestedDeserialized.getString(1)); + assertEquals(9876543210L, nestedDeserialized.getInt64(2)); + System.out.println("✓ Nested records test passed"); } + + // --- Start of broken down tests for ErgonomicGettersAndNestedTypes --- + + private ImprintRecord createTestRecordForGetters() throws ImprintException { + SchemaId schemaId = new SchemaId(5, 0xabcdef01); + + List innerList1 = Arrays.asList(Value.fromInt32(10), Value.fromInt32(20)); + List innerList2 = Arrays.asList(Value.fromInt32(30), Value.fromInt32(40)); + List listOfLists = Arrays.asList(Value.fromArray(innerList1), Value.fromArray(innerList2)); + + Map mapWithArrayValue = new HashMap<>(); + mapWithArrayValue.put(MapKey.fromString("list1"), Value.fromArray(innerList1)); + + return ImprintRecord.builder(schemaId) + .field(1, true) + .field(2, 12345) + .field(3, 9876543210L) + .field(4, 3.14f) + .field(5, 2.718281828) + .field(6, "hello type world") + .field(7, new byte[]{10, 20, 30}) + .nullField(8) + .field(9, Value.fromArray(listOfLists)) // Array of Arrays (using Value directly for test setup) + .field(10, Value.fromMap(mapWithArrayValue)) // Map with Array value + .field(11, Collections.emptyList()) // Empty Array via builder + .field(12, Collections.emptyMap()) // Empty Map via builder + .build(); + } + + private ImprintRecord serializeAndDeserialize(ImprintRecord record) throws ImprintException { + var buffer = record.serializeToBuffer(); + byte[] serialized = new byte[buffer.remaining()]; + buffer.get(serialized); + return ImprintRecord.deserialize(serialized); + } + + @Test + @DisplayName("Type Getters: Basic primitive and String types") + void testBasicTypeGetters() throws ImprintException { + ImprintRecord originalRecord = createTestRecordForGetters(); + ImprintRecord record = serializeAndDeserialize(originalRecord); + + assertTrue(record.getBoolean(1)); + assertEquals(12345, record.getInt32(2)); + assertEquals(9876543210L, record.getInt64(3)); + assertEquals(3.14f, record.getFloat32(4)); + assertEquals(2.718281828, record.getFloat64(5)); + assertEquals("hello type world", record.getString(6)); + assertArrayEquals(new byte[]{10, 20, 30}, record.getBytes(7)); + } + + @Test + @DisplayName("Type Getters: Array of Arrays") + void testTypeGetterArrayOfArrays() throws ImprintException { + ImprintRecord originalRecord = createTestRecordForGetters(); + ImprintRecord record = serializeAndDeserialize(originalRecord); + + List arrOfArr = record.getArray(9); + assertNotNull(arrOfArr); + assertEquals(2, arrOfArr.size()); + assertInstanceOf(Value.ArrayValue.class, arrOfArr.get(0)); + Value.ArrayValue firstInnerArray = (Value.ArrayValue) arrOfArr.get(0); + assertEquals(2, firstInnerArray.getValue().size()); + assertEquals(Value.fromInt32(10), firstInnerArray.getValue().get(0)); + assertEquals(Value.fromInt32(20), firstInnerArray.getValue().get(1)); + + assertInstanceOf(Value.ArrayValue.class, arrOfArr.get(1)); + Value.ArrayValue secondInnerArray = (Value.ArrayValue) arrOfArr.get(1); + assertEquals(2, secondInnerArray.getValue().size()); + assertEquals(Value.fromInt32(30), secondInnerArray.getValue().get(0)); + assertEquals(Value.fromInt32(40), secondInnerArray.getValue().get(1)); + } + + @Test + @DisplayName("Type Getters: Map with Array Value") + void testTypeGetterMapWithArrayValue() throws ImprintException { + ImprintRecord originalRecord = createTestRecordForGetters(); + ImprintRecord record = serializeAndDeserialize(originalRecord); + + Map mapWithArr = record.getMap(10); + assertNotNull(mapWithArr); + assertEquals(1, mapWithArr.size()); + assertInstanceOf(Value.ArrayValue.class, mapWithArr.get(MapKey.fromString("list1"))); + Value.ArrayValue innerArray = (Value.ArrayValue) mapWithArr.get(MapKey.fromString("list1")); + assertNotNull(innerArray); + assertEquals(2, innerArray.getValue().size()); + assertEquals(Value.fromInt32(10), innerArray.getValue().get(0)); + } + + @Test + @DisplayName("Type Getters: Empty Collections (Array and Map)") + void testErgonomicGettersEmptyCollections() throws ImprintException { + ImprintRecord originalRecord = createTestRecordForGetters(); + ImprintRecord record = serializeAndDeserialize(originalRecord); + + List emptyArr = record.getArray(11); + assertNotNull(emptyArr); + assertTrue(emptyArr.isEmpty()); + + Map emptyMap = record.getMap(12); + assertNotNull(emptyMap); + assertTrue(emptyMap.isEmpty()); + } + + @Test + @DisplayName("Type Getters: Exception for Field Not Found") + void testErgonomicGetterExceptionFieldNotFound() throws ImprintException { + ImprintRecord originalRecord = createTestRecordForGetters(); + ImprintRecord record = serializeAndDeserialize(originalRecord); + + ImprintException ex = assertThrows(ImprintException.class, () -> record.getInt32(99)); + assertEquals(ErrorType.FIELD_NOT_FOUND, ex.getErrorType()); + } + + @Test + @DisplayName("Type Getters: Exception for Null Field accessed as primitive") + void testErgonomicGetterExceptionNullField() throws ImprintException { + ImprintRecord originalRecord = createTestRecordForGetters(); + ImprintRecord record = serializeAndDeserialize(originalRecord); + + ImprintException ex = assertThrows(ImprintException.class, () -> record.getString(8)); + assertEquals(ErrorType.TYPE_MISMATCH, ex.getErrorType()); // getString throws TYPE_MISMATCH for null + assertTrue(ex.getMessage().contains("Field 8 is NULL")); + + + // Also test getValue for a null field returns Value.NullValue + Value nullValueField = record.getValue(8); + assertNotNull(nullValueField); + assertInstanceOf(Value.NullValue.class, nullValueField, "Field 8 should be Value.NullValue"); + } + + @Test + @DisplayName("Type Getters: Exception for Type Mismatch") + void testErgonomicGetterExceptionTypeMismatch() throws ImprintException { + ImprintRecord originalRecord = createTestRecordForGetters(); + ImprintRecord record = serializeAndDeserialize(originalRecord); + + ImprintException ex = assertThrows(ImprintException.class, () -> record.getInt32(6)); // Field 6 is a String + assertEquals(ErrorType.TYPE_MISMATCH, ex.getErrorType()); + } + + @Test + @DisplayName("Type Getters: Row (Nested Record)") + void testErgonomicGetterRow() throws ImprintException { + SchemaId innerSchemaId = new SchemaId(6, 0x12345678); + ImprintRecord innerRecord = ImprintRecord.builder(innerSchemaId) + .field(101, "nested string") + .field(102, 999L) + .build(); + + ImprintRecord recordWithRow = ImprintRecord.builder(new SchemaId(7, 0x87654321)) + .field(201, innerRecord) // Using builder to add row + .field(202, "outer field") + .build(); + + ImprintRecord deserializedWithRow = serializeAndDeserialize(recordWithRow); + + ImprintRecord retrievedRow = deserializedWithRow.getRow(201); + assertNotNull(retrievedRow); + assertEquals(innerSchemaId, retrievedRow.getHeader().getSchemaId()); + assertEquals("nested string", retrievedRow.getString(101)); + assertEquals(999L, retrievedRow.getInt64(102)); + assertEquals("outer field", deserializedWithRow.getString(202)); + } } \ No newline at end of file From 2e56688ecbdefa4dca64237b72f2ea3eed4983d9 Mon Sep 17 00:00:00 2001 From: expand3d <> Date: Tue, 3 Jun 2025 18:53:18 -0400 Subject: [PATCH 17/34] minor style fixes --- .../java/com/imprint/core/ImprintRecord.java | 13 +- .../com/imprint/types/TypeHandlerTest.java | 122 +++++++++--------- .../java/com/imprint/types/ValueTest.java | 100 +++++++------- 3 files changed, 118 insertions(+), 117 deletions(-) diff --git a/src/main/java/com/imprint/core/ImprintRecord.java b/src/main/java/com/imprint/core/ImprintRecord.java index d667039..dd66389 100644 --- a/src/main/java/com/imprint/core/ImprintRecord.java +++ b/src/main/java/com/imprint/core/ImprintRecord.java @@ -292,12 +292,6 @@ private Value deserializeValue(TypeCode typeCode, ByteBuffer buffer) throws Impr } } - @Override - public String toString() { - return String.format("ImprintRecord{header=%s, directorySize=%d, payloadSize=%d}", - header, directory.size(), payload.remaining()); - } - private T getTypedValueOrThrow(int fieldId, TypeCode expectedTypeCode, Class expectedValueClass, String expectedTypeName) throws ImprintException { Value value = getValue(fieldId); @@ -439,4 +433,11 @@ public Map getMap(int fieldId) throws ImprintException { public ImprintRecord getRow(int fieldId) throws ImprintException { return getTypedValueOrThrow(fieldId, TypeCode.ROW, Value.RowValue.class, "ROW").getValue(); } + + @Override + public String toString() { + return String.format("ImprintRecord{header=%s, directorySize=%d, payloadSize=%d}", + header, directory.size(), payload.remaining()); + } + } \ No newline at end of file diff --git a/src/test/java/com/imprint/types/TypeHandlerTest.java b/src/test/java/com/imprint/types/TypeHandlerTest.java index 9a4ae85..f131a0f 100644 --- a/src/test/java/com/imprint/types/TypeHandlerTest.java +++ b/src/test/java/com/imprint/types/TypeHandlerTest.java @@ -20,104 +20,104 @@ class TypeHandlerTest { void testNullHandler() throws ImprintException { var handler = TypeHandler.NULL; var value = Value.nullValue(); - + // Size estimation assertThat(handler.estimateSize(value)).isEqualTo(0); - + // Serialization var buffer = ByteBuffer.allocate(10); handler.serialize(value, buffer); assertThat(buffer.position()).isEqualTo(0); // NULL writes nothing - + // Deserialization buffer.flip(); var deserialized = handler.deserialize(buffer); assertThat(deserialized).isEqualTo(value); - + // readValueBytes buffer.clear(); var valueBytes = handler.readValueBytes(buffer); assertThat(valueBytes.remaining()).isEqualTo(0); } - + @ParameterizedTest @ValueSource(booleans = {true, false}) void testBoolHandler(boolean testValue) throws ImprintException { var handler = TypeHandler.BOOL; var value = Value.fromBoolean(testValue); - + // Size estimation assertThat(handler.estimateSize(value)).isEqualTo(1); - + // Round-trip test var buffer = ByteBuffer.allocate(10); handler.serialize(value, buffer); assertThat(buffer.position()).isEqualTo(1); - + buffer.flip(); var deserialized = handler.deserialize(buffer); assertThat(deserialized).isEqualTo(value); assertThat(((Value.BoolValue) deserialized).getValue()).isEqualTo(testValue); } - + @ParameterizedTest @ValueSource(ints = {0, 1, -1, Integer.MAX_VALUE, Integer.MIN_VALUE, 42, -42}) void testInt32Handler(int testValue) throws ImprintException { var handler = TypeHandler.INT32; var value = Value.fromInt32(testValue); - + // Size estimation assertThat(handler.estimateSize(value)).isEqualTo(4); - + // Round-trip test var buffer = ByteBuffer.allocate(10).order(ByteOrder.LITTLE_ENDIAN); handler.serialize(value, buffer); assertThat(buffer.position()).isEqualTo(4); - + buffer.flip(); var deserialized = handler.deserialize(buffer); assertThat(deserialized).isEqualTo(value); assertThat(((Value.Int32Value) deserialized).getValue()).isEqualTo(testValue); } - + @ParameterizedTest @ValueSource(longs = {0L, 1L, -1L, Long.MAX_VALUE, Long.MIN_VALUE, 123456789L}) void testInt64Handler(long testValue) throws ImprintException { var handler = TypeHandler.INT64; var value = Value.fromInt64(testValue); - + // Size estimation assertThat(handler.estimateSize(value)).isEqualTo(8); - + // Round-trip test var buffer = ByteBuffer.allocate(20).order(ByteOrder.LITTLE_ENDIAN); handler.serialize(value, buffer); assertThat(buffer.position()).isEqualTo(8); - + buffer.flip(); var deserialized = handler.deserialize(buffer); assertThat(deserialized).isEqualTo(value); assertThat(((Value.Int64Value) deserialized).getValue()).isEqualTo(testValue); } - + @ParameterizedTest @ValueSource(floats = {0.0f, 1.0f, -1.0f, Float.MAX_VALUE, Float.MIN_VALUE, 3.14159f, Float.NaN, Float.POSITIVE_INFINITY}) void testFloat32Handler(float testValue) throws ImprintException { var handler = TypeHandler.FLOAT32; var value = Value.fromFloat32(testValue); - + // Size estimation assertThat(handler.estimateSize(value)).isEqualTo(4); - + // Round-trip test var buffer = ByteBuffer.allocate(10).order(ByteOrder.LITTLE_ENDIAN); handler.serialize(value, buffer); assertThat(buffer.position()).isEqualTo(4); - + buffer.flip(); var deserialized = handler.deserialize(buffer); assertThat(deserialized).isEqualTo(value); - + float deserializedValue = ((Value.Float32Value) deserialized).getValue(); if (Float.isNaN(testValue)) { assertThat(deserializedValue).isNaN(); @@ -125,25 +125,25 @@ void testFloat32Handler(float testValue) throws ImprintException { assertThat(deserializedValue).isEqualTo(testValue); } } - + @ParameterizedTest @ValueSource(doubles = {0.0, 1.0, -1.0, Double.MAX_VALUE, Double.MIN_VALUE, Math.PI, Double.NaN, Double.POSITIVE_INFINITY}) void testFloat64Handler(double testValue) throws ImprintException { var handler = TypeHandler.FLOAT64; var value = Value.fromFloat64(testValue); - + // Size estimation assertThat(handler.estimateSize(value)).isEqualTo(8); - + // Round-trip test var buffer = ByteBuffer.allocate(20).order(ByteOrder.LITTLE_ENDIAN); handler.serialize(value, buffer); assertThat(buffer.position()).isEqualTo(8); - + buffer.flip(); var deserialized = handler.deserialize(buffer); assertThat(deserialized).isEqualTo(value); - + double deserializedValue = ((Value.Float64Value) deserialized).getValue(); if (Double.isNaN(testValue)) { assertThat(deserializedValue).isNaN(); @@ -151,129 +151,129 @@ void testFloat64Handler(double testValue) throws ImprintException { assertThat(deserializedValue).isEqualTo(testValue); } } - + @ParameterizedTest @ValueSource(strings = {"", "hello", "世界", "a very long string that exceeds typical buffer sizes and contains unicode: 🚀🎉", "null\0bytes"}) void testStringHandler(String testValue) throws ImprintException { var handler = TypeHandler.STRING; var value = Value.fromString(testValue); - + byte[] utf8Bytes = testValue.getBytes(java.nio.charset.StandardCharsets.UTF_8); int expectedSize = com.imprint.util.VarInt.encodedLength(utf8Bytes.length) + utf8Bytes.length; - + // Size estimation assertThat(handler.estimateSize(value)).isEqualTo(expectedSize); - + // Round-trip test var buffer = ByteBuffer.allocate(expectedSize + 20).order(ByteOrder.LITTLE_ENDIAN); handler.serialize(value, buffer); - + buffer.flip(); var deserialized = handler.deserialize(buffer); - + // Should return StringBufferValue (zero-copy implementation) assertThat(deserialized).isInstanceOf(Value.StringBufferValue.class); - + String deserializedString; if (deserialized instanceof Value.StringBufferValue) { deserializedString = ((Value.StringBufferValue) deserialized).getValue(); } else { deserializedString = ((Value.StringValue) deserialized).getValue(); } - + assertThat(deserializedString).isEqualTo(testValue); } - + @Test void testBytesHandlerWithArrayValue() throws ImprintException { var handler = TypeHandler.BYTES; byte[] testBytes = {0, 1, 2, (byte) 0xFF, 42, 127, -128}; var value = Value.fromBytes(testBytes); - + int expectedSize = com.imprint.util.VarInt.encodedLength(testBytes.length) + testBytes.length; - + // Size estimation assertThat(handler.estimateSize(value)).isEqualTo(expectedSize); - + // Round-trip test var buffer = ByteBuffer.allocate(expectedSize + 20).order(ByteOrder.LITTLE_ENDIAN); handler.serialize(value, buffer); - + buffer.flip(); var deserialized = handler.deserialize(buffer); - + // Should return BytesBufferValue (zero-copy implementation) assertThat(deserialized).isInstanceOf(Value.BytesBufferValue.class); - + byte[] deserializedBytes = ((Value.BytesBufferValue) deserialized).getValue(); assertThat(deserializedBytes).isEqualTo(testBytes); } - + @Test void testBytesHandlerWithBufferValue() throws ImprintException { var handler = TypeHandler.BYTES; byte[] testBytes = {10, 20, 30, 40}; var bufferValue = Value.fromBytesBuffer(ByteBuffer.wrap(testBytes).asReadOnlyBuffer()); - + int expectedSize = com.imprint.util.VarInt.encodedLength(testBytes.length) + testBytes.length; - + // Size estimation assertThat(handler.estimateSize(bufferValue)).isEqualTo(expectedSize); - + // Round-trip test var buffer = ByteBuffer.allocate(expectedSize + 20).order(ByteOrder.LITTLE_ENDIAN); handler.serialize(bufferValue, buffer); - + buffer.flip(); var deserialized = handler.deserialize(buffer); - + byte[] deserializedBytes = ((Value.BytesBufferValue) deserialized).getValue(); assertThat(deserializedBytes).isEqualTo(testBytes); } - + @Test void testStringHandlerWithBufferValue() throws ImprintException { var handler = TypeHandler.STRING; String testString = "zero-copy string test"; byte[] utf8Bytes = testString.getBytes(java.nio.charset.StandardCharsets.UTF_8); var bufferValue = Value.fromStringBuffer(ByteBuffer.wrap(utf8Bytes).asReadOnlyBuffer()); - + int expectedSize = com.imprint.util.VarInt.encodedLength(utf8Bytes.length) + utf8Bytes.length; - + // Size estimation assertThat(handler.estimateSize(bufferValue)).isEqualTo(expectedSize); - + // Round-trip test var buffer = ByteBuffer.allocate(expectedSize + 20).order(ByteOrder.LITTLE_ENDIAN); handler.serialize(bufferValue, buffer); - + buffer.flip(); var deserialized = handler.deserialize(buffer); - + String deserializedString = ((Value.StringBufferValue) deserialized).getValue(); assertThat(deserializedString).isEqualTo(testString); } - + @Test void testBoolHandlerInvalidValue() { var handler = TypeHandler.BOOL; var buffer = ByteBuffer.allocate(10); buffer.put((byte) 2); // Invalid boolean value buffer.flip(); - + assertThatThrownBy(() -> handler.deserialize(buffer)) - .isInstanceOf(ImprintException.class) - .hasMessageContaining("Invalid boolean value: 2"); + .isInstanceOf(ImprintException.class) + .hasMessageContaining("Invalid boolean value: 2"); } - + @Test void testHandlerBufferUnderflow() { // Test that handlers properly detect buffer underflow var int32Handler = TypeHandler.INT32; var buffer = ByteBuffer.allocate(2); // Too small for int32 - + assertThatThrownBy(() -> int32Handler.deserialize(buffer)) - .isInstanceOf(ImprintException.class) - .hasMessageContaining("Not enough bytes for int32"); + .isInstanceOf(ImprintException.class) + .hasMessageContaining("Not enough bytes for int32"); } } \ No newline at end of file diff --git a/src/test/java/com/imprint/types/ValueTest.java b/src/test/java/com/imprint/types/ValueTest.java index c400a44..b092bb7 100644 --- a/src/test/java/com/imprint/types/ValueTest.java +++ b/src/test/java/com/imprint/types/ValueTest.java @@ -12,143 +12,143 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; class ValueTest { - + @Test void shouldCreateNullValue() { Value value = Value.nullValue(); - + assertThat(value).isInstanceOf(Value.NullValue.class); assertThat(value.getTypeCode()).isEqualTo(TypeCode.NULL); assertThat(value.toString()).isEqualTo("null"); } - + @Test void shouldCreateBooleanValues() { Value trueValue = Value.fromBoolean(true); Value falseValue = Value.fromBoolean(false); - + assertThat(trueValue).isInstanceOf(Value.BoolValue.class); assertThat(((Value.BoolValue) trueValue).getValue()).isTrue(); assertThat(trueValue.getTypeCode()).isEqualTo(TypeCode.BOOL); - + assertThat(falseValue).isInstanceOf(Value.BoolValue.class); assertThat(((Value.BoolValue) falseValue).getValue()).isFalse(); assertThat(falseValue.getTypeCode()).isEqualTo(TypeCode.BOOL); } - + @Test void shouldCreateNumericValues() { var int32 = Value.fromInt32(42); var int64 = Value.fromInt64(123456789L); var float32 = Value.fromFloat32(3.14f); var float64 = Value.fromFloat64(2.718281828); - + assertThat(int32.getTypeCode()).isEqualTo(TypeCode.INT32); assertThat(((Value.Int32Value) int32).getValue()).isEqualTo(42); - + assertThat(int64.getTypeCode()).isEqualTo(TypeCode.INT64); assertThat(((Value.Int64Value) int64).getValue()).isEqualTo(123456789L); - + assertThat(float32.getTypeCode()).isEqualTo(TypeCode.FLOAT32); assertThat(((Value.Float32Value) float32).getValue()).isEqualTo(3.14f); - + assertThat(float64.getTypeCode()).isEqualTo(TypeCode.FLOAT64); assertThat(((Value.Float64Value) float64).getValue()).isEqualTo(2.718281828); } - + @Test void shouldCreateBytesAndStringValues() { byte[] bytes = {1, 2, 3, 4}; var bytesValue = Value.fromBytes(bytes); var stringValue = Value.fromString("hello"); - + assertThat(bytesValue.getTypeCode()).isEqualTo(TypeCode.BYTES); assertThat(((Value.BytesValue) bytesValue).getValue()).isEqualTo(bytes); - + assertThat(stringValue.getTypeCode()).isEqualTo(TypeCode.STRING); assertThat(((Value.StringValue) stringValue).getValue()).isEqualTo("hello"); } - + @Test void shouldCreateArrayValues() { List elements = Arrays.asList( - Value.fromInt32(1), - Value.fromInt32(2), - Value.fromInt32(3) + Value.fromInt32(1), + Value.fromInt32(2), + Value.fromInt32(3) ); Value arrayValue = Value.fromArray(elements); - + assertThat(arrayValue.getTypeCode()).isEqualTo(TypeCode.ARRAY); assertThat(((Value.ArrayValue) arrayValue).getValue()).isEqualTo(elements); } - + @Test void shouldCreateMapValues() { var map = new HashMap(); map.put(MapKey.fromString("key1"), Value.fromInt32(1)); map.put(MapKey.fromString("key2"), Value.fromInt32(2)); - + Value mapValue = Value.fromMap(map); - + assertThat(mapValue.getTypeCode()).isEqualTo(TypeCode.MAP); assertThat(((Value.MapValue) mapValue).getValue()).isEqualTo(map); } - + @Test void shouldHandleEqualityCorrectly() { var int1 = Value.fromInt32(42); var int2 = Value.fromInt32(42); var int3 = Value.fromInt32(43); - + assertThat(int1).isEqualTo(int2); assertThat(int1).isNotEqualTo(int3); assertThat(int1.hashCode()).isEqualTo(int2.hashCode()); } - + @Test void shouldRejectNullString() { assertThatThrownBy(() -> Value.fromString(null)) - .isInstanceOf(NullPointerException.class); + .isInstanceOf(NullPointerException.class); } - + @Test void shouldCreateStringBufferValue() { String testString = "hello world"; byte[] utf8Bytes = testString.getBytes(StandardCharsets.UTF_8); ByteBuffer buffer = ByteBuffer.wrap(utf8Bytes); - + Value stringBufferValue = Value.fromStringBuffer(buffer); - + assertThat(stringBufferValue).isInstanceOf(Value.StringBufferValue.class); assertThat(stringBufferValue.getTypeCode()).isEqualTo(TypeCode.STRING); assertThat(((Value.StringBufferValue) stringBufferValue).getValue()).isEqualTo(testString); } - + @Test void shouldCreateBytesBufferValue() { byte[] testBytes = {1, 2, 3, 4, 5}; ByteBuffer buffer = ByteBuffer.wrap(testBytes); - + Value bytesBufferValue = Value.fromBytesBuffer(buffer); - + assertThat(bytesBufferValue).isInstanceOf(Value.BytesBufferValue.class); assertThat(bytesBufferValue.getTypeCode()).isEqualTo(TypeCode.BYTES); assertThat(((Value.BytesBufferValue) bytesBufferValue).getValue()).isEqualTo(testBytes); } - + @Test void shouldHandleStringBufferValueFastPath() { // Array-backed buffer with arrayOffset() == 0 should use fast path String testString = "fast path test"; byte[] utf8Bytes = testString.getBytes(StandardCharsets.UTF_8); ByteBuffer buffer = ByteBuffer.wrap(utf8Bytes); - + Value stringBufferValue = Value.fromStringBuffer(buffer); - + // Should work correctly regardless of path taken assertThat(((Value.StringBufferValue) stringBufferValue).getValue()).isEqualTo(testString); } - + @Test void shouldHandleStringBufferValueFallbackPath() { // Sliced buffer will have non-zero arrayOffset, forcing fallback path @@ -156,62 +156,62 @@ void shouldHandleStringBufferValueFallbackPath() { byte[] utf8Bytes = testString.getBytes(StandardCharsets.UTF_8); ByteBuffer buffer = ByteBuffer.wrap(utf8Bytes); ByteBuffer sliced = buffer.slice(); // This may break arrayOffset() == 0 - + Value stringBufferValue = Value.fromStringBuffer(sliced); - + // Should work correctly regardless of path taken assertThat(((Value.StringBufferValue) stringBufferValue).getValue()).isEqualTo(testString); } - + @Test void shouldHandleLargeStringWithoutCaching() { // Create string > 1KB to test the no-cache path String largeString = "x".repeat(2000); byte[] utf8Bytes = largeString.getBytes(StandardCharsets.UTF_8); ByteBuffer buffer = ByteBuffer.wrap(utf8Bytes).slice(); // Force fallback path - + Value stringBufferValue = Value.fromStringBuffer(buffer); - + assertThat(((Value.StringBufferValue) stringBufferValue).getValue()).isEqualTo(largeString); } - + @Test void shouldCacheStringDecoding() { String testString = "cache test"; byte[] utf8Bytes = testString.getBytes(StandardCharsets.UTF_8); ByteBuffer buffer = ByteBuffer.wrap(utf8Bytes); - + Value.StringBufferValue stringBufferValue = (Value.StringBufferValue) Value.fromStringBuffer(buffer); - + // First call should decode and cache String result1 = stringBufferValue.getValue(); // Second call should return cached value String result2 = stringBufferValue.getValue(); - + assertThat(result1).isEqualTo(testString); assertThat(result2).isEqualTo(testString); assertThat(result1).isSameAs(result2); // Should be same object reference due to caching } - + @Test void shouldHandleStringValueEquality() { String testString = "equality test"; - + Value stringValue = Value.fromString(testString); Value stringBufferValue = Value.fromStringBuffer(ByteBuffer.wrap(testString.getBytes(StandardCharsets.UTF_8))); - + assertThat(stringValue).isEqualTo(stringBufferValue); assertThat(stringBufferValue).isEqualTo(stringValue); assertThat(stringValue.hashCode()).isEqualTo(stringBufferValue.hashCode()); } - + @Test void shouldHandleBytesValueEquality() { byte[] testBytes = {1, 2, 3, 4, 5}; - + Value bytesValue = Value.fromBytes(testBytes); Value bytesBufferValue = Value.fromBytesBuffer(ByteBuffer.wrap(testBytes)); - + assertThat(bytesValue).isEqualTo(bytesBufferValue); assertThat(bytesBufferValue).isEqualTo(bytesValue); } From 9353388b22d44653b9b061f30de1add7dc284258 Mon Sep 17 00:00:00 2001 From: expand3d <> Date: Tue, 3 Jun 2025 18:57:15 -0400 Subject: [PATCH 18/34] minor style fixes again --- src/main/java/com/imprint/core/ImprintRecord.java | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/imprint/core/ImprintRecord.java b/src/main/java/com/imprint/core/ImprintRecord.java index dd66389..5d4719f 100644 --- a/src/main/java/com/imprint/core/ImprintRecord.java +++ b/src/main/java/com/imprint/core/ImprintRecord.java @@ -41,23 +41,16 @@ public ImprintRecord(Header header, List directory, ByteBuffer p /** * Get a value by field ID, deserializing it on demand. * Returns null if the field is not found. - * Note: If the field exists and is an explicit NULL type, this will return Value.NullValue.INSTANCE. + * Note: If the field exists and is an explicit NULL type, this will return Value.NullValue.INSTANCE */ public Value getValue(int fieldId) throws ImprintException { var fieldBuffer = getFieldBuffer(fieldId); if (fieldBuffer == null) { - // If getFieldBuffer returns null, it means the fieldId was not in the directory, - // or an issue occurred slicing the payload (e.g., bad offsets). return null; } - // findDirectoryIndex should not be negative here if fieldBuffer is not null, - // but a defensive check or ensuring findDirectoryIndex is called once is good. - // For simplicity, assume getFieldBuffer implies a valid index. int directoryIndex = findDirectoryIndex(fieldId); if (directoryIndex < 0) { - // This case should ideally be caught by getFieldBuffer returning null. - // If it happens, indicates an inconsistency. throw new ImprintException(ErrorType.INTERNAL_ERROR, "Field ID " + fieldId + " found buffer but not in directory."); } var entry = directory.get(directoryIndex); @@ -293,7 +286,7 @@ private Value deserializeValue(TypeCode typeCode, ByteBuffer buffer) throws Impr } private T getTypedValueOrThrow(int fieldId, TypeCode expectedTypeCode, Class expectedValueClass, String expectedTypeName) throws ImprintException { - Value value = getValue(fieldId); + var value = getValue(fieldId); if (value == null) { throw new ImprintException(ErrorType.FIELD_NOT_FOUND, From 09d0377922236cfc60799a75d8397a1cc75c67dc Mon Sep 17 00:00:00 2001 From: expand3d <> Date: Tue, 3 Jun 2025 19:00:32 -0400 Subject: [PATCH 19/34] minor style fixes on benchmark tests and supress unused --- src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java | 6 +----- src/jmh/java/com/imprint/benchmark/StringBenchmark.java | 1 + 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java b/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java index 152bb6d..8163522 100644 --- a/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java +++ b/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java @@ -103,8 +103,6 @@ public void deserializeKryo(Blackhole bh) { // ===== FIELD ACCESS BENCHMARKS ===== // Tests accessing a single field near the end of a large record - // This showcases Imprint's O(1) directory lookup vs sequential deserialization - @Benchmark public void singleFieldAccessImprint(Blackhole bh) throws Exception { ImprintRecord record = ImprintRecord.deserialize(imprintBytes.duplicate()); @@ -158,11 +156,9 @@ public void measureKryoSize(Blackhole bh) { @Benchmark public void mergeImprint(Blackhole bh) throws Exception { - // Simulate merge with Imprint (O(1) with proper API) var record1 = serializeWithImprint(testData); var record2 = serializeWithImprint(createTestRecord2()); - - // Current simulation - will be O(1) with actual merge API + var deserialized1 = ImprintRecord.deserialize(record1); var deserialized2 = ImprintRecord.deserialize(record2); var merged = simulateMerge(deserialized1, deserialized2); diff --git a/src/jmh/java/com/imprint/benchmark/StringBenchmark.java b/src/jmh/java/com/imprint/benchmark/StringBenchmark.java index e156c8c..045940e 100644 --- a/src/jmh/java/com/imprint/benchmark/StringBenchmark.java +++ b/src/jmh/java/com/imprint/benchmark/StringBenchmark.java @@ -16,6 +16,7 @@ @Fork(1) @Warmup(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS) @Measurement(iterations = 10, time = 1, timeUnit = TimeUnit.SECONDS) +@SuppressWarnings("unused") public class StringBenchmark { private static final SchemaId SCHEMA_ID = new SchemaId(1, 42); From 6209bb1217239a167b26660a690427629ef854d2 Mon Sep 17 00:00:00 2001 From: expand3d <> Date: Wed, 4 Jun 2025 01:09:26 -0400 Subject: [PATCH 20/34] minor reordering --- .../java/com/imprint/types/TypeHandler.java | 28 ++++++++++--------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/src/main/java/com/imprint/types/TypeHandler.java b/src/main/java/com/imprint/types/TypeHandler.java index d21403b..be4fc7b 100644 --- a/src/main/java/com/imprint/types/TypeHandler.java +++ b/src/main/java/com/imprint/types/TypeHandler.java @@ -19,6 +19,13 @@ public interface TypeHandler { int estimateSize(Value value) throws ImprintException; ByteBuffer readValueBytes(ByteBuffer buffer) throws ImprintException; + + + @FunctionalInterface + interface BufferViewer { + int measureDataLength(ByteBuffer tempBuffer, int numElements) throws ImprintException; + } + // Helper method to eliminate duplication in ARRAY/MAP readValueBytes static ByteBuffer readComplexValueBytes(ByteBuffer buffer, String typeName, BufferViewer measurer) throws ImprintException { int initialPosition = buffer.position(); @@ -31,9 +38,9 @@ static ByteBuffer readComplexValueBytes(ByteBuffer buffer, String typeName, Buff if (numElements == 0) { if (buffer.remaining() < varIntLength) { - throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, - "Not enough bytes for empty " + typeName + " VarInt. Needed: " + - varIntLength + ", available: " + buffer.remaining()); + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, + "Not enough bytes for empty " + typeName + " VarInt. Needed: " + + varIntLength + ", available: " + buffer.remaining()); } ByteBuffer valueSlice = buffer.slice(); valueSlice.limit(varIntLength); @@ -43,24 +50,19 @@ static ByteBuffer readComplexValueBytes(ByteBuffer buffer, String typeName, Buff int dataLength = measurer.measureDataLength(tempBuffer, numElements); int totalLength = varIntLength + dataLength; - + if (buffer.remaining() < totalLength) { - throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, - "Not enough bytes for " + typeName + " value. Needed: " + totalLength + - ", available: " + buffer.remaining() + " at position " + initialPosition); + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, + "Not enough bytes for " + typeName + " value. Needed: " + totalLength + + ", available: " + buffer.remaining() + " at position " + initialPosition); } - + ByteBuffer valueSlice = buffer.slice(); valueSlice.limit(totalLength); buffer.position(initialPosition + totalLength); return valueSlice.asReadOnlyBuffer(); } - @FunctionalInterface - interface BufferViewer { - int measureDataLength(ByteBuffer tempBuffer, int numElements) throws ImprintException; - } - // Static implementations for each type TypeHandler NULL = new TypeHandler() { @Override From 4632e013e4b2bd209feb9466cce9d4d260fe5636 Mon Sep 17 00:00:00 2001 From: expand3d <> Date: Thu, 5 Jun 2025 10:19:37 -0400 Subject: [PATCH 21/34] Full comprehensive comparison tests with a lot of other libraries + some micro-optimizations added that were found along the way --- .github/workflows/ci.yml | 244 ++++++- build.gradle | 265 ++++++- src/jmh/flatbuffers/test_record.fbs | 15 + .../benchmark/ComparisonBenchmark.java | 661 +++++++++++++++--- src/jmh/proto/test_record.proto | 15 + .../java/com/imprint/core/ImprintRecord.java | 12 +- .../java/com/imprint/types/TypeHandler.java | 84 ++- src/main/java/com/imprint/types/Value.java | 36 +- .../{benchmark => profile}/ProfilerTest.java | 11 +- 9 files changed, 1169 insertions(+), 174 deletions(-) create mode 100644 src/jmh/flatbuffers/test_record.fbs create mode 100644 src/jmh/proto/test_record.proto rename src/test/java/com/imprint/{benchmark => profile}/ProfilerTest.java (97%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d4c8bde..62ac6f5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,30 +15,220 @@ jobs: java-version: [11, 17, 21] steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up JDK ${{ matrix.java-version }} - uses: actions/setup-java@v4 - with: - java-version: ${{ matrix.java-version }} - distribution: 'temurin' - - - name: Cache Gradle dependencies - uses: actions/cache@v4 - with: - path: | - ~/.gradle/caches - ~/.gradle/wrapper - key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} - restore-keys: | - ${{ runner.os }}-gradle- - - - name: Make gradlew executable - run: chmod +x ./gradlew - - - name: Run tests - run: ./gradlew test - - - name: Run build - run: ./gradlew build \ No newline at end of file + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up JDK ${{ matrix.java-version }} + uses: actions/setup-java@v4 + with: + java-version: ${{ matrix.java-version }} + distribution: 'temurin' + + - name: Cache Gradle dependencies + uses: actions/cache@v4 + with: + path: | + ~/.gradle/caches + ~/.gradle/wrapper + key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} + restore-keys: | + ${{ runner.os }}-gradle- + + - name: Make gradlew executable + run: chmod +x ./gradlew + + - name: Run tests + run: ./gradlew test + + - name: Run build + run: ./gradlew build + + benchmark: + runs-on: ubuntu-latest + needs: test + # Only run benchmarks on main branch pushes and PRs to main to avoid excessive CI time + if: github.ref == 'refs/heads/main' || github.base_ref == 'main' + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up JDK 11 + uses: actions/setup-java@v4 + with: + java-version: '11' + distribution: 'temurin' + + - name: Cache Gradle dependencies + uses: actions/cache@v4 + with: + path: | + ~/.gradle/caches + ~/.gradle/wrapper + key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} + restore-keys: | + ${{ runner.os }}-gradle- + + - name: Make gradlew executable + run: chmod +x ./gradlew + + - name: Create benchmark results directory + run: mkdir -p benchmark-results + + - name: Run serialization benchmarks + run: | + ./gradlew jmhRunSerializationBenchmarks + continue-on-error: true + + - name: Run deserialization benchmarks + run: | + ./gradlew jmhRunDeserializationBenchmarks + continue-on-error: true + + - name: Run field access benchmarks + run: | + ./gradlew jmhRunFieldAccessBenchmarks + continue-on-error: true + + - name: Run size comparison benchmarks + run: | + ./gradlew jmhRunSizeComparisonBenchmarks + continue-on-error: true + + - name: Upload benchmark results + uses: actions/upload-artifact@v4 + if: always() + with: + name: benchmark-results-${{ github.sha }} + path: benchmark-results/ + retention-days: 30 + + - name: Comment benchmark results on PR + if: github.event_name == 'pull_request' + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const path = require('path'); + + // Find the latest benchmark results file + const resultsDir = 'benchmark-results'; + let latestFile = null; + let latestTime = 0; + + if (fs.existsSync(resultsDir)) { + const files = fs.readdirSync(resultsDir); + for (const file of files) { + if (file.endsWith('.json')) { + const filePath = path.join(resultsDir, file); + const stats = fs.statSync(filePath); + if (stats.mtime.getTime() > latestTime) { + latestTime = stats.mtime.getTime(); + latestFile = filePath; + } + } + } + } + + if (latestFile) { + const results = JSON.parse(fs.readFileSync(latestFile, 'utf8')); + + // Group results by benchmark type + const serialization = results.filter(r => r.benchmark.includes('serialize')); + const deserialization = results.filter(r => r.benchmark.includes('deserialize')); + const fieldAccess = results.filter(r => r.benchmark.includes('singleFieldAccess')); + const sizes = results.filter(r => r.benchmark.includes('measure')); + + // Format results into a table + const formatResults = (benchmarks, title) => { + if (benchmarks.length === 0) return ''; + + let table = `\n### ${title}\n\n| Library | Score (ns/op) | Error | Unit |\n|---------|---------------|-------|------|\n`; + + benchmarks + .sort((a, b) => a.primaryMetric.score - b.primaryMetric.score) + .forEach(benchmark => { + const name = benchmark.benchmark.split('.').pop().replace(/serialize|deserialize|singleFieldAccess|measure/, '').replace(/Imprint|JacksonJson|Kryo|MessagePack|Avro|Protobuf|FlatBuffers/, (match) => match); + const score = benchmark.primaryMetric.score.toFixed(2); + const error = benchmark.primaryMetric.scoreError.toFixed(2); + const unit = benchmark.primaryMetric.scoreUnit; + table += `| ${name} | ${score} | ±${error} | ${unit} |\n`; + }); + + return table; + }; + + const comment = `##Benchmark Results + + Benchmark comparison between Imprint and other serialization libraries: + ${formatResults(serialization, 'Serialization Performance')} + ${formatResults(deserialization, 'Deserialization Performance')} + ${formatResults(fieldAccess, 'Single Field Access Performance')} + ${formatResults(sizes, 'Serialized Size Comparison')} + +
+ View detailed results + + Results generated from commit: \`${context.sha.substring(0, 7)}\` + + Lower scores are better for performance benchmarks. + +
`; + + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: comment + }); + } else { + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: '## Benchmark Results\n\nBenchmark execution completed but no results file was found. Check the [workflow logs](' + + `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}` + ') for details.' + }); + } + + # Optional: Run full benchmark suite on releases + benchmark-full: + runs-on: ubuntu-latest + if: startsWith(github.ref, 'refs/tags/') + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up JDK 11 + uses: actions/setup-java@v4 + with: + java-version: '11' + distribution: 'temurin' + + - name: Cache Gradle dependencies + uses: actions/cache@v4 + with: + path: | + ~/.gradle/caches + ~/.gradle/wrapper + key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} + restore-keys: | + ${{ runner.os }}-gradle- + + - name: Make gradlew executable + run: chmod +x ./gradlew + + - name: Create benchmark results directory + run: mkdir -p benchmark-results + + - name: Run full benchmark suite + run: | + ./gradlew jmh + + - name: Upload full benchmark results + uses: actions/upload-artifact@v4 + with: + name: full-benchmark-results-${{ github.ref_name }} + path: benchmark-results/ + retention-days: 90 \ No newline at end of file diff --git a/build.gradle b/build.gradle index 2606710..8406676 100644 --- a/build.gradle +++ b/build.gradle @@ -2,6 +2,8 @@ plugins { id 'java-library' id 'maven-publish' id 'me.champeau.jmh' version '0.7.2' + id 'com.google.protobuf' version '0.9.4' + id 'io.netifi.flatbuffers' version '1.0.7' } group = 'com.imprint' @@ -23,31 +25,141 @@ dependencies { // Lombok for reducing boilerplate compileOnly 'org.projectlombok:lombok:1.18.30' annotationProcessor 'org.projectlombok:lombok:1.18.30' - + // Test dependencies testImplementation 'org.junit.jupiter:junit-jupiter:5.10.0' testImplementation 'org.assertj:assertj-core:3.24.2' testImplementation 'org.mockito:mockito-core:5.5.0' - + // Lombok for tests testCompileOnly 'org.projectlombok:lombok:1.18.30' testAnnotationProcessor 'org.projectlombok:lombok:1.18.30' - + // Performance testing with JMH jmhImplementation 'org.openjdk.jmh:jmh-core:1.37' jmhAnnotationProcessor 'org.openjdk.jmh:jmh-generator-annprocess:1.37' - - // Competitor libraries for benchmarking + + // Suppress SLF4J warnings + jmhImplementation 'org.slf4j:slf4j-nop:1.7.36' + + // Competitor libraries for benchmarking (JMH only) jmhImplementation 'com.google.protobuf:protobuf-java:3.25.1' jmhImplementation 'org.apache.avro:avro:1.11.3' jmhImplementation 'com.fasterxml.jackson.core:jackson-databind:2.16.0' jmhImplementation 'com.google.flatbuffers:flatbuffers-java:23.5.26' jmhImplementation 'com.esotericsoftware:kryo:5.4.0' + jmhImplementation 'org.msgpack:msgpack-core:0.9.8' + jmhImplementation 'org.msgpack:jackson-dataformat-msgpack:0.9.8' +} + +protobuf { + protoc { + artifact = "com.google.protobuf:protoc:3.25.1" + } + generateProtoTasks { + // Only generate for JMH, not main + all().each { task -> + task.enabled = false + } + ofSourceSet('jmh').each { task -> + task.enabled = true + task.builtins { + java { + outputSubDir = 'java' + } + } + } + } +} + +// Download and setup FlatBuffers compiler for Linux (CI environment) +task downloadFlatc { + description = 'Download FlatBuffers compiler' + group = 'build setup' + + def flatcDir = file("${buildDir}/flatc") + def flatcExe = file("${flatcDir}/flatc") + def isWindows = System.getProperty('os.name').toLowerCase().contains('windows') + def flatcUrl = isWindows ? + "https://github.com/google/flatbuffers/releases/download/v23.5.26/Windows.flatc.binary.zip" : + "https://github.com/google/flatbuffers/releases/download/v23.5.26/Linux.flatc.binary.clang++-12.zip" + def flatcZip = file("${buildDir}/flatc.zip") + + outputs.file(flatcExe) + + doLast { + if (!flatcExe.exists()) { + println "Downloading FlatBuffers compiler for ${isWindows ? 'Windows' : 'Linux'}..." + flatcDir.mkdirs() + + // Download + new URL(flatcUrl).withInputStream { i -> + flatcZip.withOutputStream { it << i } + } + + // Extract + copy { + from zipTree(flatcZip) + into flatcDir + } + + // Make executable on Unix systems + if (!isWindows) { + exec { + commandLine 'chmod', '+x', flatcExe.absolutePath + } + } + + flatcZip.delete() + println "FlatBuffers compiler downloaded to: ${flatcExe}" + } + } +} + +// Generate FlatBuffers sources +task generateFlatBuffers(type: Exec) { + dependsOn downloadFlatc + description = 'Generate Java classes from FlatBuffers schema' + group = 'build' + + def isWindows = System.getProperty('os.name').toLowerCase().contains('windows') + def flatcExe = file("${buildDir}/flatc/${isWindows ? 'flatc.exe' : 'flatc'}") + def schemaFile = file('src/jmh/flatbuffers/test_record.fbs') + def outputDir = file('build/generated/source/flatbuffers/jmh/java') + + commandLine flatcExe.absolutePath, '--java', '-o', outputDir.absolutePath, schemaFile.absolutePath + + inputs.file(schemaFile) + outputs.dir(outputDir) + + doFirst { + outputDir.mkdirs() + } +} + +// Add generated FlatBuffers sources to JMH source set +sourceSets { + jmh { + java { + srcDir 'build/generated/source/flatbuffers/jmh/java' + } + proto { + srcDir 'src/jmh/proto' + } + } +} + +// Make JMH compilation depend on FlatBuffers generation +compileJmhJava.dependsOn generateFlatBuffers + +// Handle duplicate proto files +tasks.named('processJmhResources') { + duplicatesStrategy = DuplicatesStrategy.EXCLUDE } test { useJUnitPlatform() - + // Enable detailed test output testLogging { events "passed", "skipped", "failed" @@ -57,11 +169,146 @@ test { // JMH configuration jmh { fork = 1 - warmupIterations = 3 - iterations = 3 + warmupIterations = 2 // Reduced for faster CI + iterations = 3 // Reduced for faster CI resultFormat = 'JSON' includeTests = false resultsFile = file("${projectDir}/benchmark-results/jmh-results-${new Date().format('yyyy-MM-dd-HHmmss')}.json") + + // Java 11 specific JVM args + jvmArgs = [ + '-XX:+UseG1GC', + '-Xmx2g', + '-XX:+UnlockExperimentalVMOptions', + '-XX:+UseJVMCICompiler' + ] +} + +// Create individual benchmark tasks for CI pipeline +task jmhRunSerializationBenchmarks(type: JavaExec) { + dependsOn compileJmhJava + description = 'Run serialization benchmarks' + group = 'benchmarking' + + classpath = sourceSets.jmh.runtimeClasspath + mainClass = 'com.imprint.benchmark.ComparisonBenchmark' + args = ['runSerializationBenchmarks'] + + // Java 11 optimized JVM settings + jvmArgs = [ + '-XX:+UseG1GC', + '-Xmx2g', + '-XX:+UnlockExperimentalVMOptions' + ] + + doFirst { + file("${projectDir}/benchmark-results").mkdirs() + } +} + +task jmhRunDeserializationBenchmarks(type: JavaExec) { + dependsOn compileJmhJava + description = 'Run deserialization benchmarks' + group = 'benchmarking' + + classpath = sourceSets.jmh.runtimeClasspath + mainClass = 'com.imprint.benchmark.ComparisonBenchmark' + args = ['runDeserializationBenchmarks'] + + // Java 11 optimized JVM settings + jvmArgs = [ + '-XX:+UseG1GC', + '-Xmx2g', + '-XX:+UnlockExperimentalVMOptions' + ] + + doFirst { + file("${projectDir}/benchmark-results").mkdirs() + } +} + +task jmhRunFieldAccessBenchmarks(type: JavaExec) { + dependsOn compileJmhJava + description = 'Run field access benchmarks' + group = 'benchmarking' + + classpath = sourceSets.jmh.runtimeClasspath + mainClass = 'com.imprint.benchmark.ComparisonBenchmark' + args = ['runFieldAccessBenchmarks'] + + // Java 11 optimized JVM settings + jvmArgs = [ + '-XX:+UseG1GC', + '-Xmx2g', + '-XX:+UnlockExperimentalVMOptions' + ] + + doFirst { + file("${projectDir}/benchmark-results").mkdirs() + } +} + +task jmhRunSizeComparisonBenchmarks(type: JavaExec) { + dependsOn compileJmhJava + description = 'Run size comparison benchmarks' + group = 'benchmarking' + + classpath = sourceSets.jmh.runtimeClasspath + mainClass = 'com.imprint.benchmark.ComparisonBenchmark' + args = ['runSizeComparisonBenchmarks'] + + // Java 11 optimized JVM settings + jvmArgs = [ + '-XX:+UseG1GC', + '-Xmx2g', + '-XX:+UnlockExperimentalVMOptions' + ] + + doFirst { + file("${projectDir}/benchmark-results").mkdirs() + } +} + +task jmhRunMergeBenchmarks(type: JavaExec) { + dependsOn compileJmhJava + description = 'Run merge operation benchmarks' + group = 'benchmarking' + + classpath = sourceSets.jmh.runtimeClasspath + mainClass = 'com.imprint.benchmark.ComparisonBenchmark' + args = ['runMergeBenchmarks'] + + // Java 11 optimized JVM settings + jvmArgs = [ + '-XX:+UseG1GC', + '-Xmx2g', + '-XX:+UnlockExperimentalVMOptions' + ] + + doFirst { + file("${projectDir}/benchmark-results").mkdirs() + } +} + +task jmhRunAllBenchmarks(type: JavaExec) { + dependsOn compileJmhJava + description = 'Run all comparison benchmarks' + group = 'benchmarking' + + classpath = sourceSets.jmh.runtimeClasspath + mainClass = 'com.imprint.benchmark.ComparisonBenchmark' + args = ['runAll'] + + // Java 11 optimized JVM settings + jvmArgs = [ + '-XX:+UseG1GC', + '-Xmx2g', + '-XX:+UnlockExperimentalVMOptions' + ] + + doFirst { + file("${projectDir}/benchmark-results").mkdirs() + } } compileJava { @@ -75,4 +322,4 @@ javadoc { } // Don't fail build on missing javadoc options.addStringOption('Xdoclint:none', '-quiet') -} +} \ No newline at end of file diff --git a/src/jmh/flatbuffers/test_record.fbs b/src/jmh/flatbuffers/test_record.fbs new file mode 100644 index 0000000..ccc31d0 --- /dev/null +++ b/src/jmh/flatbuffers/test_record.fbs @@ -0,0 +1,15 @@ +namespace com.imprint.benchmark; + +table TestRecordFB { + id: int; + name: string; + price: double; + active: bool; + category: string; + tags: [string]; + metadata_keys: [string]; + metadata_values: [string]; + extra_data: [string]; +} + +root_type TestRecordFB; \ No newline at end of file diff --git a/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java b/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java index 8163522..49260b1 100644 --- a/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java +++ b/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java @@ -4,13 +4,25 @@ import com.esotericsoftware.kryo.io.Input; import com.esotericsoftware.kryo.io.Output; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.flatbuffers.FlatBufferBuilder; import com.imprint.core.ImprintRecord; import com.imprint.core.ImprintWriter; import com.imprint.core.SchemaId; import com.imprint.types.MapKey; import com.imprint.types.Value; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericDatumReader; +import org.apache.avro.generic.GenericDatumWriter; +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.io.*; +import org.msgpack.jackson.dataformat.MessagePackFactory; import org.openjdk.jmh.annotations.*; import org.openjdk.jmh.infra.Blackhole; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; @@ -25,38 +37,56 @@ @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.NANOSECONDS) @State(Scope.Benchmark) -@Warmup(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS) -@Measurement(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS) +@Warmup(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 10, time = 1, timeUnit = TimeUnit.SECONDS) @Fork(1) +@SuppressWarnings("unused") public class ComparisonBenchmark { // Test data private TestRecord testData; - + // Serialized formats - private ByteBuffer imprintBytes; - private byte[] jacksonBytes; + private ByteBuffer imprintBytesBuffer; + private byte[] jacksonJsonBytes; private byte[] kryoBytes; - + private byte[] messagePackBytes; + private byte[] avroBytes; + private byte[] protobufBytes; + private ByteBuffer flatbuffersBytes; + // Library instances - private ObjectMapper jackson; + private Schema avroSchema; + private DatumWriter avroWriter; + private DatumReader avroReader; + private ObjectMapper jacksonJsonMapper; private Kryo kryo; + private ObjectMapper messagePackMapper; @Setup public void setup() throws Exception { testData = createTestRecord(); - + // Initialize libraries - jackson = new ObjectMapper(); + jacksonJsonMapper = new ObjectMapper(); kryo = new Kryo(); kryo.register(TestRecord.class); kryo.register(ArrayList.class); kryo.register(HashMap.class); - + kryo.register(Arrays.asList().getClass()); + + // Initialize MessagePack ObjectMapper + messagePackMapper = new ObjectMapper(new MessagePackFactory()); + setupAvro(); + // Pre-serialize for deserialization benchmarks - imprintBytes = serializeWithImprint(testData); - jacksonBytes = serializeWithJackson(testData); + imprintBytesBuffer = serializeWithImprint(testData); + jacksonJsonBytes = serializeWithJacksonJson(testData); kryoBytes = serializeWithKryo(testData); + messagePackBytes = serializeWithMessagePack(testData); + avroBytes = serializeWithAvro(testData); + protobufBytes = serializeWithProtobuf(testData); + flatbuffersBytes = serializeWithFlatBuffers(testData); } // ===== SERIALIZATION BENCHMARKS ===== @@ -68,8 +98,8 @@ public void serializeImprint(Blackhole bh) throws Exception { } @Benchmark - public void serializeJackson(Blackhole bh) throws Exception { - byte[] result = serializeWithJackson(testData); + public void serializeJacksonJson(Blackhole bh) throws Exception { + byte[] result = serializeWithJacksonJson(testData); bh.consume(result); } @@ -79,17 +109,41 @@ public void serializeKryo(Blackhole bh) { bh.consume(result); } + @Benchmark + public void serializeMessagePack(Blackhole bh) throws Exception { + byte[] result = serializeWithMessagePack(testData); + bh.consume(result); + } + + @Benchmark + public void serializeAvro(Blackhole bh) throws Exception { + byte[] result = serializeWithAvro(testData); + bh.consume(result); + } + + @Benchmark + public void serializeProtobuf(Blackhole bh) { + byte[] result = serializeWithProtobuf(testData); + bh.consume(result); + } + + @Benchmark + public void serializeFlatBuffers(Blackhole bh) { + ByteBuffer result = serializeWithFlatBuffers(testData); + bh.consume(result); + } + // ===== DESERIALIZATION BENCHMARKS ===== @Benchmark public void deserializeImprint(Blackhole bh) throws Exception { - ImprintRecord result = ImprintRecord.deserialize(imprintBytes.duplicate()); + ImprintRecord result = ImprintRecord.deserialize(imprintBytesBuffer.duplicate()); bh.consume(result); } @Benchmark - public void deserializeJackson(Blackhole bh) throws Exception { - TestRecord result = jackson.readValue(jacksonBytes, TestRecord.class); + public void deserializeJacksonJson(Blackhole bh) throws Exception { + TestRecord result = jacksonJsonMapper.readValue(jacksonJsonBytes, TestRecord.class); bh.consume(result); } @@ -101,135 +155,361 @@ public void deserializeKryo(Blackhole bh) { bh.consume(result); } + @Benchmark + public void deserializeMessagePack(Blackhole bh) throws Exception { + TestRecord result = messagePackMapper.readValue(messagePackBytes, TestRecord.class); + bh.consume(result); + } + + @Benchmark + public void deserializeAvro(Blackhole bh) throws Exception { + GenericRecord result = deserializeWithAvro(avroBytes); + bh.consume(result); + } + + @Benchmark + public void deserializeProtobuf(Blackhole bh) throws Exception { + TestRecordProto.TestRecord result = TestRecordProto.TestRecord.parseFrom(protobufBytes); + bh.consume(result); + } + + @Benchmark + public void deserializeFlatBuffers(Blackhole bh) { + TestRecordFB result = TestRecordFB.getRootAsTestRecordFB(flatbuffersBytes.duplicate()); + bh.consume(result); + } + // ===== FIELD ACCESS BENCHMARKS ===== // Tests accessing a single field near the end of a large record + // This showcases Imprint's O(1) directory lookup vs sequential deserialization + @Benchmark public void singleFieldAccessImprint(Blackhole bh) throws Exception { - ImprintRecord record = ImprintRecord.deserialize(imprintBytes.duplicate()); - - // Access field 15 directly via directory lookup - O(1) - var field15 = record.getValue(15); + ImprintRecord record = ImprintRecord.deserialize(imprintBytesBuffer.duplicate()); + var field15 = record.getString(15); bh.consume(field15); } @Benchmark - public void singleFieldAccessJackson(Blackhole bh) throws Exception { - // Jackson must deserialize entire object to access any field - TestRecord record = jackson.readValue(jacksonBytes, TestRecord.class); - - // Access field15 equivalent (extraData[4]) after full deserialization + public void singleFieldAccessJacksonJson(Blackhole bh) throws Exception { + TestRecord record = jacksonJsonMapper.readValue(jacksonJsonBytes, TestRecord.class); bh.consume(record.extraData.get(4)); } @Benchmark public void singleFieldAccessKryo(Blackhole bh) { - // Kryo must deserialize entire object to access any field Input input = new Input(new ByteArrayInputStream(kryoBytes)); TestRecord record = kryo.readObject(input, TestRecord.class); input.close(); - - // Access field15 equivalent (extraData[4]) after full deserialization bh.consume(record.extraData.get(4)); } + @Benchmark + public void singleFieldAccessMessagePack(Blackhole bh) throws Exception { + TestRecord record = messagePackMapper.readValue(messagePackBytes, TestRecord.class); + bh.consume(record.extraData.get(4)); + } + + @Benchmark + public void singleFieldAccessAvro(Blackhole bh) throws Exception { + GenericRecord record = deserializeWithAvro(avroBytes); + bh.consume(record.get("extraData4")); // Accessing field near end + } + + @Benchmark + public void singleFieldAccessProtobuf(Blackhole bh) throws Exception { + TestRecordProto.TestRecord record = TestRecordProto.TestRecord.parseFrom(protobufBytes); + bh.consume(record.getExtraData(4)); // Accessing field near end + } + + @Benchmark + public void singleFieldAccessFlatBuffers(Blackhole bh) { + TestRecordFB record = TestRecordFB.getRootAsTestRecordFB(flatbuffersBytes.duplicate()); + bh.consume(record.extraData(4)); // Accessing field near end - zero copy! + } + // ===== SIZE COMPARISON ===== @Benchmark - public void measureImprintSize(Blackhole bh) throws Exception { - ByteBuffer serialized = serializeWithImprint(testData); - bh.consume(serialized.remaining()); + public void measureImprintSize(Blackhole bh) { + bh.consume(imprintBytesBuffer.remaining()); } @Benchmark - public void measureJacksonSize(Blackhole bh) throws Exception { - byte[] serialized = serializeWithJackson(testData); - bh.consume(serialized.length); + public void measureJacksonJsonSize(Blackhole bh) { + bh.consume(jacksonJsonBytes.length); } @Benchmark public void measureKryoSize(Blackhole bh) { - byte[] serialized = serializeWithKryo(testData); - bh.consume(serialized.length); + bh.consume(kryoBytes.length); + } + + @Benchmark + public void measureMessagePackSize(Blackhole bh) { + bh.consume(messagePackBytes.length); + } + + @Benchmark + public void measureAvroSize(Blackhole bh) { + bh.consume(avroBytes.length); + } + + @Benchmark + public void measureProtobufSize(Blackhole bh) { + bh.consume(protobufBytes.length); + } + + @Benchmark + public void measureFlatBuffersSize(Blackhole bh) { + bh.consume(flatbuffersBytes.remaining()); } // ===== MERGE SIMULATION BENCHMARKS ===== @Benchmark public void mergeImprint(Blackhole bh) throws Exception { - var record1 = serializeWithImprint(testData); - var record2 = serializeWithImprint(createTestRecord2()); + var record1Buffer = imprintBytesBuffer.duplicate(); + var record2Data = createTestRecord2(); + var record2Buffer = serializeWithImprint(record2Data); - var deserialized1 = ImprintRecord.deserialize(record1); - var deserialized2 = ImprintRecord.deserialize(record2); + var deserialized1 = ImprintRecord.deserialize(record1Buffer); + var deserialized2 = ImprintRecord.deserialize(record2Buffer); var merged = simulateMerge(deserialized1, deserialized2); - + bh.consume(merged); } @Benchmark - public void mergeJackson(Blackhole bh) throws Exception { - // Jackson merge requires full deserialization + merge + serialization - var record1 = jackson.readValue(jacksonBytes, TestRecord.class); - var record2 = jackson.readValue(serializeWithJackson(createTestRecord2()), TestRecord.class); - - var merged = mergeTestRecords(record1, record2); - byte[] result = jackson.writeValueAsBytes(merged); - + public void mergeJacksonJson(Blackhole bh) throws Exception { + var record1 = jacksonJsonMapper.readValue(jacksonJsonBytes, TestRecord.class); + var record2Data = createTestRecord2(); + var record2Bytes = serializeWithJacksonJson(record2Data); + var record2 = jacksonJsonMapper.readValue(record2Bytes, TestRecord.class); + + var mergedPojo = mergeTestRecords(record1, record2); + byte[] result = jacksonJsonMapper.writeValueAsBytes(mergedPojo); bh.consume(result); } @Benchmark public void mergeKryo(Blackhole bh) { - // Kryo merge requires full deserialization + merge + serialization Input input1 = new Input(new ByteArrayInputStream(kryoBytes)); var record1 = kryo.readObject(input1, TestRecord.class); input1.close(); - - Input input2 = new Input(new ByteArrayInputStream(serializeWithKryo(createTestRecord2()))); + + var record2Data = createTestRecord2(); + var record2Bytes = serializeWithKryo(record2Data); + Input input2 = new Input(new ByteArrayInputStream(record2Bytes)); var record2 = kryo.readObject(input2, TestRecord.class); input2.close(); - - var merged = mergeTestRecords(record1, record2); - byte[] result = serializeWithKryo(merged); - + + var mergedPojo = mergeTestRecords(record1, record2); + byte[] result = serializeWithKryo(mergedPojo); + bh.consume(result); + } + + @Benchmark + public void mergeMessagePack(Blackhole bh) throws Exception { + var record1 = messagePackMapper.readValue(messagePackBytes, TestRecord.class); + var record2Data = createTestRecord2(); + var record2Bytes = serializeWithMessagePack(record2Data); + var record2 = messagePackMapper.readValue(record2Bytes, TestRecord.class); + + var mergedPojo = mergeTestRecords(record1, record2); + byte[] result = messagePackMapper.writeValueAsBytes(mergedPojo); bh.consume(result); } + @Benchmark + public void mergeAvro(Blackhole bh) throws Exception { + var record1 = deserializeWithAvro(avroBytes); + var record2Data = createTestRecord2(); + var record2Bytes = serializeWithAvro(record2Data); + var record2 = deserializeWithAvro(record2Bytes); + + var merged = mergeAvroRecords(record1, record2); + byte[] result = serializeAvroRecord(merged); + bh.consume(result); + } + + @Benchmark + public void mergeProtobuf(Blackhole bh) throws Exception { + var record1 = TestRecordProto.TestRecord.parseFrom(protobufBytes); + var record2Data = createTestRecord2(); + var record2Bytes = serializeWithProtobuf(record2Data); + var record2 = TestRecordProto.TestRecord.parseFrom(record2Bytes); + + var merged = mergeProtobufRecords(record1, record2); + byte[] result = merged.toByteArray(); + bh.consume(result); + } + + @Benchmark + public void mergeFlatBuffers(Blackhole bh) { + var record1 = TestRecordFB.getRootAsTestRecordFB(flatbuffersBytes.duplicate()); + var record2Data = createTestRecord2(); + var record2Buffer = serializeWithFlatBuffers(record2Data); + var record2 = TestRecordFB.getRootAsTestRecordFB(record2Buffer); + + var merged = mergeFlatBuffersRecords(record1, record2); + bh.consume(merged); + } + + // ===== MAIN METHOD TO RUN BENCHMARKS ===== + + public static void main(String[] args) throws RunnerException { + runAll(); + // Or, uncomment specific runner methods to execute subsets: + // runSerializationBenchmarks(); + // runDeserializationBenchmarks(); + // runFieldAccessBenchmarks(); + // runSizeComparisonBenchmarks(); + // runMergeBenchmarks(); + // runMessagePackBenchmarks(); + } + + public static void runAll() throws RunnerException { + Options opt = new OptionsBuilder() + .include(ComparisonBenchmark.class.getSimpleName()) + .build(); + new Runner(opt).run(); + } + + public static void runSerializationBenchmarks() throws RunnerException { + Options opt = new OptionsBuilder() + .include(ComparisonBenchmark.class.getSimpleName() + ".serialize.*") + .build(); + new Runner(opt).run(); + } + + public static void runDeserializationBenchmarks() throws RunnerException { + Options opt = new OptionsBuilder() + .include(ComparisonBenchmark.class.getSimpleName() + ".deserialize.*") + .build(); + new Runner(opt).run(); + } + + public static void runFieldAccessBenchmarks() throws RunnerException { + Options opt = new OptionsBuilder() + .include(ComparisonBenchmark.class.getSimpleName() + ".singleFieldAccess.*") + .build(); + new Runner(opt).run(); + } + + public static void runSizeComparisonBenchmarks() throws RunnerException { + Options opt = new OptionsBuilder() + .include(ComparisonBenchmark.class.getSimpleName() + ".measure.*") + .build(); + new Runner(opt).run(); + } + + public static void runMergeBenchmarks() throws RunnerException { + Options opt = new OptionsBuilder() + .include(ComparisonBenchmark.class.getSimpleName() + ".merge.*") + .build(); + new Runner(opt).run(); + } + + public static void runMessagePackBenchmarks() throws RunnerException { + Options opt = new OptionsBuilder() + .include(ComparisonBenchmark.class.getSimpleName() + ".*MessagePack.*") + .build(); + new Runner(opt).run(); + } + + public static void runAvroBenchmarks() throws RunnerException { + Options opt = new OptionsBuilder() + .include(ComparisonBenchmark.class.getSimpleName() + ".*Avro.*") + .build(); + new Runner(opt).run(); + } + + public static void runProtobufBenchmarks() throws RunnerException { + Options opt = new OptionsBuilder() + .include(ComparisonBenchmark.class.getSimpleName() + ".*Protobuf.*") + .build(); + new Runner(opt).run(); + } + + public static void runFlatBuffersBenchmarks() throws RunnerException { + Options opt = new OptionsBuilder() + .include(ComparisonBenchmark.class.getSimpleName() + ".*FlatBuffers.*") + .build(); + new Runner(opt).run(); + } + // ===== HELPER METHODS ===== + private void setupAvro() { + String schemaJson = "{\n" + + " \"type\": \"record\",\n" + + " \"name\": \"TestRecord\",\n" + + " \"fields\": [\n" + + " {\"name\": \"id\", \"type\": \"int\"},\n" + + " {\"name\": \"name\", \"type\": \"string\"},\n" + + " {\"name\": \"price\", \"type\": \"double\"},\n" + + " {\"name\": \"active\", \"type\": \"boolean\"},\n" + + " {\"name\": \"category\", \"type\": \"string\"},\n" + + " {\"name\": \"tags\", \"type\": {\"type\": \"array\", \"items\": \"string\"}},\n" + + " {\"name\": \"metadata\", \"type\": {\"type\": \"map\", \"values\": \"string\"}},\n" + + " {\"name\": \"extraData0\", \"type\": \"string\"},\n" + + " {\"name\": \"extraData1\", \"type\": \"string\"},\n" + + " {\"name\": \"extraData2\", \"type\": \"string\"},\n" + + " {\"name\": \"extraData3\", \"type\": \"string\"},\n" + + " {\"name\": \"extraData4\", \"type\": \"string\"},\n" + + " {\"name\": \"extraData5\", \"type\": \"string\"},\n" + + " {\"name\": \"extraData6\", \"type\": \"string\"},\n" + + " {\"name\": \"extraData7\", \"type\": \"string\"},\n" + + " {\"name\": \"extraData8\", \"type\": \"string\"},\n" + + " {\"name\": \"extraData9\", \"type\": \"string\"},\n" + + " {\"name\": \"extraData10\", \"type\": \"string\"},\n" + + " {\"name\": \"extraData11\", \"type\": \"string\"},\n" + + " {\"name\": \"extraData12\", \"type\": \"string\"}\n" + + " ]\n" + + "}"; + + avroSchema = new Schema.Parser().parse(schemaJson); + avroWriter = new GenericDatumWriter<>(avroSchema); + avroReader = new GenericDatumReader<>(avroSchema); + } + private ByteBuffer serializeWithImprint(TestRecord data) throws Exception { var writer = new ImprintWriter(new SchemaId(1, 0x12345678)); - + writer.addField(1, Value.fromInt32(data.id)); writer.addField(2, Value.fromString(data.name)); writer.addField(3, Value.fromFloat64(data.price)); writer.addField(4, Value.fromBoolean(data.active)); writer.addField(5, Value.fromString(data.category)); - - // Convert tags list + var tagValues = new ArrayList(); - for (String tag : data.tags) { - tagValues.add(Value.fromString(tag)); + if (data.tags != null) { + for (String tag : data.tags) { + tagValues.add(Value.fromString(tag)); + } } writer.addField(6, Value.fromArray(tagValues)); - - // Convert metadata map + var metadataMap = new HashMap(); - for (var entry : data.metadata.entrySet()) { - metadataMap.put(MapKey.fromString(entry.getKey()), Value.fromString(entry.getValue())); + if (data.metadata != null) { + for (var entry : data.metadata.entrySet()) { + metadataMap.put(MapKey.fromString(entry.getKey()), Value.fromString(entry.getValue())); + } } writer.addField(7, Value.fromMap(metadataMap)); - - // Add extra fields (8-20) to create a larger record - for (int i = 0; i < data.extraData.size(); i++) { - writer.addField(8 + i, Value.fromString(data.extraData.get(i))); + + if (data.extraData != null) { + for (int i = 0; i < data.extraData.size(); i++) { + writer.addField(8 + i, Value.fromString(data.extraData.get(i))); + } } - + return writer.build().serializeToBuffer(); } - private byte[] serializeWithJackson(TestRecord data) throws Exception { - return jackson.writeValueAsBytes(data); + private byte[] serializeWithJacksonJson(TestRecord data) throws Exception { + return jacksonJsonMapper.writeValueAsBytes(data); } private byte[] serializeWithKryo(TestRecord data) { @@ -240,14 +520,117 @@ private byte[] serializeWithKryo(TestRecord data) { return baos.toByteArray(); } + private byte[] serializeWithMessagePack(TestRecord data) throws Exception { + return messagePackMapper.writeValueAsBytes(data); + } + + private byte[] serializeWithAvro(TestRecord data) throws Exception { + GenericRecord record = new GenericData.Record(avroSchema); + record.put("id", data.id); + record.put("name", data.name); + record.put("price", data.price); + record.put("active", data.active); + record.put("category", data.category); + record.put("tags", data.tags); + record.put("metadata", data.metadata); + + for (int i = 0; i < data.extraData.size(); i++) { + record.put("extraData" + i, data.extraData.get(i)); + } + + return serializeAvroRecord(record); + } + + private byte[] serializeAvroRecord(GenericRecord record) throws Exception { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + Encoder encoder = EncoderFactory.get().binaryEncoder(baos, null); + avroWriter.write(record, encoder); + encoder.flush(); + return baos.toByteArray(); + } + + private GenericRecord deserializeWithAvro(byte[] data) throws Exception { + Decoder decoder = DecoderFactory.get().binaryDecoder(data, null); + return avroReader.read(null, decoder); + } + + private byte[] serializeWithProtobuf(TestRecord data) { + var builder = TestRecordProto.TestRecord.newBuilder() + .setId(data.id) + .setName(data.name) + .setPrice(data.price) + .setActive(data.active) + .setCategory(data.category) + .addAllTags(data.tags) + .putAllMetadata(data.metadata); + + for (String extraData : data.extraData) { + builder.addExtraData(extraData); + } + + return builder.build().toByteArray(); + } + + private ByteBuffer serializeWithFlatBuffers(TestRecord data) { + FlatBufferBuilder builder = new FlatBufferBuilder(1024); + + // Create strings (must be created before the object that uses them) + int nameOffset = builder.createString(data.name); + int categoryOffset = builder.createString(data.category); + + // Create tags array + int[] tagOffsets = new int[data.tags.size()]; + for (int i = 0; i < data.tags.size(); i++) { + tagOffsets[i] = builder.createString(data.tags.get(i)); + } + int tagsOffset = TestRecordFB.createTagsVector(builder, tagOffsets); + + // Create metadata (as parallel arrays for keys and values) + String[] metadataKeys = data.metadata.keySet().toArray(new String[0]); + String[] metadataValues = new String[metadataKeys.length]; + int[] keyOffsets = new int[metadataKeys.length]; + int[] valueOffsets = new int[metadataKeys.length]; + + for (int i = 0; i < metadataKeys.length; i++) { + metadataValues[i] = data.metadata.get(metadataKeys[i]); + keyOffsets[i] = builder.createString(metadataKeys[i]); + valueOffsets[i] = builder.createString(metadataValues[i]); + } + int metadataKeysOffset = TestRecordFB.createMetadataKeysVector(builder, keyOffsets); + int metadataValuesOffset = TestRecordFB.createMetadataValuesVector(builder, valueOffsets); + + // Create extra data array + int[] extraDataOffsets = new int[data.extraData.size()]; + for (int i = 0; i < data.extraData.size(); i++) { + extraDataOffsets[i] = builder.createString(data.extraData.get(i)); + } + int extraDataOffset = TestRecordFB.createExtraDataVector(builder, extraDataOffsets); + + // Create the main object + TestRecordFB.startTestRecordFB(builder); + TestRecordFB.addId(builder, data.id); + TestRecordFB.addName(builder, nameOffset); + TestRecordFB.addPrice(builder, data.price); + TestRecordFB.addActive(builder, data.active); + TestRecordFB.addCategory(builder, categoryOffset); + TestRecordFB.addTags(builder, tagsOffset); + TestRecordFB.addMetadataKeys(builder, metadataKeysOffset); + TestRecordFB.addMetadataValues(builder, metadataValuesOffset); + TestRecordFB.addExtraData(builder, extraDataOffset); + int recordOffset = TestRecordFB.endTestRecordFB(builder); + + // Finish and return + builder.finish(recordOffset); + return builder.dataBuffer().slice(); + } + private ImprintRecord simulateMerge(ImprintRecord first, ImprintRecord second) throws Exception { var writer = new ImprintWriter(first.getHeader().getSchemaId()); var usedFieldIds = new HashSet(); - - // Copy fields from first record (takes precedence) + copyFieldsToWriter(first, writer, usedFieldIds); copyFieldsToWriter(second, writer, usedFieldIds); - + return writer.build(); } @@ -265,23 +648,121 @@ private void copyFieldsToWriter(ImprintRecord record, ImprintWriter writer, Set< } private TestRecord mergeTestRecords(TestRecord first, TestRecord second) { - // Simple merge logic - first record takes precedence var merged = new TestRecord(); merged.id = first.id; merged.name = first.name != null ? first.name : second.name; merged.price = first.price != 0.0 ? first.price : second.price; merged.active = first.active; merged.category = first.category != null ? first.category : second.category; - + merged.tags = new ArrayList<>(first.tags); merged.tags.addAll(second.tags); - + merged.metadata = new HashMap<>(first.metadata); merged.metadata.putAll(second.metadata); - + return merged; } + private GenericRecord mergeAvroRecords(GenericRecord first, GenericRecord second) { + GenericRecord merged = new GenericData.Record(avroSchema); + + // Copy all fields from first record + for (Schema.Field field : avroSchema.getFields()) { + merged.put(field.name(), first.get(field.name())); + } + + // Override with non-null values from second record + for (Schema.Field field : avroSchema.getFields()) { + Object secondValue = second.get(field.name()); + if (secondValue != null && !secondValue.toString().isEmpty()) { + merged.put(field.name(), secondValue); + } + } + + return merged; + } + + private TestRecordProto.TestRecord mergeProtobufRecords(TestRecordProto.TestRecord first, TestRecordProto.TestRecord second) { + return TestRecordProto.TestRecord.newBuilder() + .mergeFrom(first) + .mergeFrom(second) + .build(); + } + + private ByteBuffer mergeFlatBuffersRecords(TestRecordFB first, TestRecordFB second) { + FlatBufferBuilder builder = new FlatBufferBuilder(1024); + + // Use second record's values if they exist, otherwise first record's values + String name = second.name() != null && !second.name().isEmpty() ? second.name() : first.name(); + String category = second.category() != null && !second.category().isEmpty() ? second.category() : first.category(); + double price = second.price() != 0.0 ? second.price() : first.price(); + boolean active = second.active(); // Use second's boolean value + int id = first.id(); // Keep first record's ID + + // Create merged strings + int nameOffset = builder.createString(name); + int categoryOffset = builder.createString(category); + + // Merge tags (combine both arrays) + List mergedTags = new ArrayList<>(); + for (int i = 0; i < first.tagsLength(); i++) { + mergedTags.add(first.tags(i)); + } + for (int i = 0; i < second.tagsLength(); i++) { + mergedTags.add(second.tags(i)); + } + + int[] tagOffsets = new int[mergedTags.size()]; + for (int i = 0; i < mergedTags.size(); i++) { + tagOffsets[i] = builder.createString(mergedTags.get(i)); + } + int tagsOffset = TestRecordFB.createTagsVector(builder, tagOffsets); + + // Merge metadata (second overwrites first) + Map mergedMetadata = new HashMap<>(); + for (int i = 0; i < first.metadataKeysLength(); i++) { + mergedMetadata.put(first.metadataKeys(i), first.metadataValues(i)); + } + for (int i = 0; i < second.metadataKeysLength(); i++) { + mergedMetadata.put(second.metadataKeys(i), second.metadataValues(i)); + } + + String[] metadataKeys = mergedMetadata.keySet().toArray(new String[0]); + int[] keyOffsets = new int[metadataKeys.length]; + int[] valueOffsets = new int[metadataKeys.length]; + + for (int i = 0; i < metadataKeys.length; i++) { + keyOffsets[i] = builder.createString(metadataKeys[i]); + valueOffsets[i] = builder.createString(mergedMetadata.get(metadataKeys[i])); + } + int metadataKeysOffset = TestRecordFB.createMetadataKeysVector(builder, keyOffsets); + int metadataValuesOffset = TestRecordFB.createMetadataValuesVector(builder, valueOffsets); + + // Use first record's extra data (or could merge both) + int[] extraDataOffsets = new int[first.extraDataLength()]; + for (int i = 0; i < first.extraDataLength(); i++) { + extraDataOffsets[i] = builder.createString(first.extraData(i)); + } + int extraDataOffset = TestRecordFB.createExtraDataVector(builder, extraDataOffsets); + + // Create the merged object + TestRecordFB.startTestRecordFB(builder); + TestRecordFB.addId(builder, id); + TestRecordFB.addName(builder, nameOffset); + TestRecordFB.addPrice(builder, price); + TestRecordFB.addActive(builder, active); + TestRecordFB.addCategory(builder, categoryOffset); + TestRecordFB.addTags(builder, tagsOffset); + TestRecordFB.addMetadataKeys(builder, metadataKeysOffset); + TestRecordFB.addMetadataValues(builder, metadataValuesOffset); + TestRecordFB.addExtraData(builder, extraDataOffset); + int recordOffset = TestRecordFB.endTestRecordFB(builder); + + builder.finish(recordOffset); + return builder.dataBuffer().slice(); + } + private TestRecord createTestRecord() { var record = new TestRecord(); record.id = 12345; @@ -289,20 +770,19 @@ var record = new TestRecord(); record.price = 99.99; record.active = true; record.category = "Electronics"; - + record.tags = Arrays.asList("popular", "trending", "bestseller"); - + record.metadata = new HashMap<>(); record.metadata.put("manufacturer", "TechCorp"); record.metadata.put("model", "TC-2024"); record.metadata.put("warranty", "2 years"); - - // Add extra data to create a larger record (fields 8-20) + record.extraData = new ArrayList<>(); for (int i = 0; i < 13; i++) { record.extraData.add("extraField" + i + "_value_" + (1000 + i)); } - + return record; } @@ -313,19 +793,18 @@ var record = new TestRecord(); record.price = 149.99; record.active = false; record.category = "Software"; - + record.tags = Arrays.asList("new", "premium"); - + record.metadata = new HashMap<>(); record.metadata.put("vendor", "SoftCorp"); record.metadata.put("version", "2.1"); - - // Add extra data to match the structure + record.extraData = new ArrayList<>(); for (int i = 0; i < 13; i++) { record.extraData.add("extraField" + i + "_value2_" + (2000 + i)); } - + return record; } @@ -339,7 +818,7 @@ public static class TestRecord { public List tags = new ArrayList<>(); public Map metadata = new HashMap<>(); public List extraData = new ArrayList<>(); // Fields 8-20 for large record test - + public TestRecord() {} // Required for deserialization } } \ No newline at end of file diff --git a/src/jmh/proto/test_record.proto b/src/jmh/proto/test_record.proto new file mode 100644 index 0000000..7a76f71 --- /dev/null +++ b/src/jmh/proto/test_record.proto @@ -0,0 +1,15 @@ +syntax = "proto3"; + +option java_package = "com.imprint.benchmark"; +option java_outer_classname = "TestRecordProto"; + +message TestRecord { + int32 id = 1; + string name = 2; + double price = 3; + bool active = 4; + string category = 5; + repeated string tags = 6; + map metadata = 7; + repeated string extra_data = 8; +} \ No newline at end of file diff --git a/src/main/java/com/imprint/core/ImprintRecord.java b/src/main/java/com/imprint/core/ImprintRecord.java index 5d4719f..2291550 100644 --- a/src/main/java/com/imprint/core/ImprintRecord.java +++ b/src/main/java/com/imprint/core/ImprintRecord.java @@ -34,7 +34,7 @@ public final class ImprintRecord { */ public ImprintRecord(Header header, List directory, ByteBuffer payload) { this.header = Objects.requireNonNull(header, "Header cannot be null"); - this.directory = List.copyOf(Objects.requireNonNull(directory, "Directory cannot be null")); + this.directory = Collections.unmodifiableList(Objects.requireNonNull(directory, "Directory cannot be null")); this.payload = payload.asReadOnlyBuffer(); // Zero-copy read-only view } @@ -79,12 +79,14 @@ private ByteBuffer getFieldBuffer(int fieldId) { int endOffset = (index + 1 < directory.size()) ? directory.get(index + 1).getOffset() : payload.limit(); - var fieldBuffer = payload.duplicate(); if (startOffset > payload.limit() || endOffset > payload.limit() || startOffset > endOffset) { return null; } + + // OPTIMIZATION: Single allocation instead of duplicate + slice + var fieldBuffer = payload.duplicate(); fieldBuffer.position(startOffset).limit(endOffset); - return fieldBuffer.slice(); + return fieldBuffer; } /** @@ -261,7 +263,7 @@ private static DirectoryEntry deserializeDirectoryEntry(ByteBuffer buffer) throw } private Value deserializeValue(TypeCode typeCode, ByteBuffer buffer) throws ImprintException { - ByteBuffer valueSpecificBuffer = buffer.duplicate(); + var valueSpecificBuffer = buffer.duplicate(); valueSpecificBuffer.order(ByteOrder.LITTLE_ENDIAN); switch (typeCode) { @@ -351,7 +353,7 @@ public double getFloat64(int fieldId) throws ImprintException { * @throws ImprintException if the field is not found, is null, or is not of type STRING. */ public String getString(int fieldId) throws ImprintException { - Value value = getValue(fieldId); + var value = getValue(fieldId); if (value == null) { throw new ImprintException(ErrorType.FIELD_NOT_FOUND, diff --git a/src/main/java/com/imprint/types/TypeHandler.java b/src/main/java/com/imprint/types/TypeHandler.java index be4fc7b..e58c355 100644 --- a/src/main/java/com/imprint/types/TypeHandler.java +++ b/src/main/java/com/imprint/types/TypeHandler.java @@ -433,26 +433,38 @@ public int estimateSize(Value value) throws ImprintException { } return arraySize; } - + @Override public ByteBuffer readValueBytes(ByteBuffer buffer) throws ImprintException { return readComplexValueBytes(buffer, "ARRAY", (tempBuffer, numElements) -> { if (tempBuffer.remaining() < 1) { - throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, - "Not enough bytes for ARRAY element type code in temp buffer during measurement."); + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, + "Not enough bytes for ARRAY element type code"); } byte elementTypeCodeByte = tempBuffer.get(); - int typeCodeLength = 1; + var elementType = TypeCode.fromByte(elementTypeCodeByte); - TypeHandler elementHandler = TypeCode.fromByte(elementTypeCodeByte).getHandler(); - int elementsDataLength = 0; - for (int i = 0; i < numElements; i++) { - int elementStartPos = tempBuffer.position(); - elementHandler.readValueBytes(tempBuffer); - elementsDataLength += (tempBuffer.position() - elementStartPos); + switch (elementType) { + case NULL: + return 1; + case BOOL: + return 1 + numElements; + case INT32: + case FLOAT32: + return 1 + (numElements * 4); + case INT64: + case FLOAT64: + return 1 + (numElements * 8); + default: + var elementHandler = elementType.getHandler(); + int elementsDataLength = 0; + for (int i = 0; i < numElements; i++) { + int elementStartPos = tempBuffer.position(); + elementHandler.readValueBytes(tempBuffer); + elementsDataLength += (tempBuffer.position() - elementStartPos); + } + return 1 + elementsDataLength; } - - return typeCodeLength + elementsDataLength; }); } }; @@ -549,28 +561,52 @@ public int estimateSize(Value value) throws ImprintException { } return mapSize; } - + @Override public ByteBuffer readValueBytes(ByteBuffer buffer) throws ImprintException { return readComplexValueBytes(buffer, "MAP", (tempBuffer, numEntries) -> { if (tempBuffer.remaining() < 2) { - throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, - "Not enough bytes for MAP key/value type codes in temp buffer during measurement."); + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, + "Not enough bytes for MAP key/value type codes"); } byte keyTypeCodeByte = tempBuffer.get(); byte valueTypeCodeByte = tempBuffer.get(); - int typeCodesLength = 2; - int entriesDataLength = 0; - for (int i = 0; i < numEntries; i++) { - int entryStartPos = tempBuffer.position(); - TypeCode.fromByte(keyTypeCodeByte).getHandler().readValueBytes(tempBuffer); - TypeCode.fromByte(valueTypeCodeByte).getHandler().readValueBytes(tempBuffer); - entriesDataLength += (tempBuffer.position() - entryStartPos); - } + TypeCode keyType = TypeCode.fromByte(keyTypeCodeByte); + TypeCode valueType = TypeCode.fromByte(valueTypeCodeByte); + + // OPTIMIZATION: Calculate sizes directly for fixed-size types + int keySize = getFixedTypeSize(keyType); + int valueSize = getFixedTypeSize(valueType); - return typeCodesLength + entriesDataLength; + if (keySize > 0 && valueSize > 0) { + // Both are fixed-size: O(1) calculation + return 2 + (numEntries * (keySize + valueSize)); + } else { + // At least one is variable-size: fall back to traversal + int entriesDataLength = 0; + for (int i = 0; i < numEntries; i++) { + int entryStartPos = tempBuffer.position(); + keyType.getHandler().readValueBytes(tempBuffer); + valueType.getHandler().readValueBytes(tempBuffer); + entriesDataLength += (tempBuffer.position() - entryStartPos); + } + return 2 + entriesDataLength; + } }); } + + private int getFixedTypeSize(TypeCode type) { + switch (type) { + case NULL: return 0; + case BOOL: return 1; + case INT32: + case FLOAT32: return 4; + case INT64: + case FLOAT64: return 8; + default: return -1; // Variable size + } + } + private void serializeMapKey(MapKey key, ByteBuffer buffer) throws ImprintException { switch (key.getTypeCode()) { diff --git a/src/main/java/com/imprint/types/Value.java b/src/main/java/com/imprint/types/Value.java index 7f3bbb9..ba747de 100644 --- a/src/main/java/com/imprint/types/Value.java +++ b/src/main/java/com/imprint/types/Value.java @@ -328,16 +328,19 @@ public String toString() { // String Value (ByteBuffer-based) public static class StringBufferValue extends Value { private final ByteBuffer value; - private volatile String cachedString; // lazy decode + private volatile String cachedString; + + private static final int THREAD_LOCAL_BUFFER_SIZE = 1024; + private static final ThreadLocal DECODE_BUFFER_CACHE = + ThreadLocal.withInitial(() -> new byte[THREAD_LOCAL_BUFFER_SIZE]); public StringBufferValue(ByteBuffer value) { - this.value = value.asReadOnlyBuffer(); // zero-copy read-only view + this.value = value.asReadOnlyBuffer(); } public String getValue() { String result = cachedString; if (result == null) { - // Simple, fast decoding - no thread-local overhead result = decodeUtf8(); cachedString = result; } @@ -345,18 +348,25 @@ public String getValue() { } private String decodeUtf8() { - // Fast path: zero-copy for array-backed ByteBuffers + final byte[] array; + final int offset; + final int length = value.remaining(); + if (value.hasArray()) { - return new String(value.array(), value.arrayOffset() + value.position(), - value.remaining(), StandardCharsets.UTF_8); + array = value.array(); + offset = value.arrayOffset() + value.position(); + } else { + byte[] threadLocalBuffer = DECODE_BUFFER_CACHE.get(); + if (length <= threadLocalBuffer.length) { + array = threadLocalBuffer; + } else { + // Fallback: copy bytes from the ByteBuffer to a new heap array (if too large for cache) + array = new byte[length]; + } + value.duplicate().get(array, 0, length); // Get bytes from current position into chosen array + offset = 0; } - - // Fallback path for non-array-backed ByteBuffers (e.g., direct buffers). - // Allocation is required here as Java's String(byte[],...) constructor needs a heap array. - // Data is copied from the ByteBuffer to a new byte array. - var array = new byte[value.remaining()]; - value.duplicate().get(array); - return new String(array, StandardCharsets.UTF_8); + return new String(array, offset, length, StandardCharsets.UTF_8); } public ByteBuffer getBuffer() { diff --git a/src/test/java/com/imprint/benchmark/ProfilerTest.java b/src/test/java/com/imprint/profile/ProfilerTest.java similarity index 97% rename from src/test/java/com/imprint/benchmark/ProfilerTest.java rename to src/test/java/com/imprint/profile/ProfilerTest.java index 5b531a9..d48c1aa 100644 --- a/src/test/java/com/imprint/benchmark/ProfilerTest.java +++ b/src/test/java/com/imprint/profile/ProfilerTest.java @@ -1,9 +1,10 @@ -package com.imprint.benchmark; +package com.imprint.profile; -import com.imprint.core.*; +import com.imprint.core.ImprintRecord; +import com.imprint.core.ImprintWriter; +import com.imprint.core.SchemaId; import com.imprint.types.Value; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.Disabled; import java.util.Random; @@ -25,11 +26,11 @@ * - String operations and UTF-8 encoding * - ByteBuffer operations */ -@Disabled("Enable manually for profiling") +//@Disabled("Enable manually for profiling") public class ProfilerTest { private static final int ITERATIONS = 1_000_000; - private static final int RECORD_SIZE = 20; + private static final int RECORD_SIZE = 50; @Test void profileFieldAccess() throws Exception { From 3738861fb994922a5fd6ed1a8fd647b8f658e933 Mon Sep 17 00:00:00 2001 From: expand3d <> Date: Thu, 5 Jun 2025 10:22:08 -0400 Subject: [PATCH 22/34] replace deprecated gradle methods with latest --- build.gradle | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/build.gradle b/build.gradle index 8406676..66f2e3e 100644 --- a/build.gradle +++ b/build.gradle @@ -73,7 +73,7 @@ protobuf { } // Download and setup FlatBuffers compiler for Linux (CI environment) -task downloadFlatc { +tasks.register('downloadFlatc') { description = 'Download FlatBuffers compiler' group = 'build setup' @@ -117,7 +117,7 @@ task downloadFlatc { } // Generate FlatBuffers sources -task generateFlatBuffers(type: Exec) { +tasks.register('generateFlatBuffers', Exec) { dependsOn downloadFlatc description = 'Generate Java classes from FlatBuffers schema' group = 'build' @@ -166,11 +166,11 @@ test { } } -// JMH configuration +// JMH configuration - optimized for Java 11 jmh { fork = 1 warmupIterations = 2 // Reduced for faster CI - iterations = 3 // Reduced for faster CI + iterations = 3 resultFormat = 'JSON' includeTests = false resultsFile = file("${projectDir}/benchmark-results/jmh-results-${new Date().format('yyyy-MM-dd-HHmmss')}.json") @@ -180,12 +180,12 @@ jmh { '-XX:+UseG1GC', '-Xmx2g', '-XX:+UnlockExperimentalVMOptions', - '-XX:+UseJVMCICompiler' + '-XX:+UseJVMCICompiler' // Use Graal if available for better performance ] } // Create individual benchmark tasks for CI pipeline -task jmhRunSerializationBenchmarks(type: JavaExec) { +tasks.register('jmhRunSerializationBenchmarks', JavaExec) { dependsOn compileJmhJava description = 'Run serialization benchmarks' group = 'benchmarking' @@ -206,7 +206,7 @@ task jmhRunSerializationBenchmarks(type: JavaExec) { } } -task jmhRunDeserializationBenchmarks(type: JavaExec) { +tasks.register('jmhRunDeserializationBenchmarks', JavaExec) { dependsOn compileJmhJava description = 'Run deserialization benchmarks' group = 'benchmarking' @@ -227,7 +227,7 @@ task jmhRunDeserializationBenchmarks(type: JavaExec) { } } -task jmhRunFieldAccessBenchmarks(type: JavaExec) { +tasks.register('jmhRunFieldAccessBenchmarks', JavaExec) { dependsOn compileJmhJava description = 'Run field access benchmarks' group = 'benchmarking' @@ -248,7 +248,7 @@ task jmhRunFieldAccessBenchmarks(type: JavaExec) { } } -task jmhRunSizeComparisonBenchmarks(type: JavaExec) { +tasks.register('jmhRunSizeComparisonBenchmarks', JavaExec) { dependsOn compileJmhJava description = 'Run size comparison benchmarks' group = 'benchmarking' @@ -269,7 +269,7 @@ task jmhRunSizeComparisonBenchmarks(type: JavaExec) { } } -task jmhRunMergeBenchmarks(type: JavaExec) { +tasks.register('jmhRunMergeBenchmarks', JavaExec) { dependsOn compileJmhJava description = 'Run merge operation benchmarks' group = 'benchmarking' @@ -290,7 +290,7 @@ task jmhRunMergeBenchmarks(type: JavaExec) { } } -task jmhRunAllBenchmarks(type: JavaExec) { +tasks.register('jmhRunAllBenchmarks', JavaExec) { dependsOn compileJmhJava description = 'Run all comparison benchmarks' group = 'benchmarking' From 12d28233e9f73c0d23687d9977356e6d35ec0f88 Mon Sep 17 00:00:00 2001 From: Brent Johnson Date: Thu, 5 Jun 2025 14:57:56 -0400 Subject: [PATCH 23/34] Merge Comparisons into dev branch (#8) * Full comprehensive comparison tests with a lot of other libraries + some micro-optimizations added that were found along the way * replace deprecated gradle methods with latest --------- Co-authored-by: expand3d <> --- .github/workflows/ci.yml | 244 ++++++- build.gradle | 265 ++++++- src/jmh/flatbuffers/test_record.fbs | 15 + .../benchmark/ComparisonBenchmark.java | 661 +++++++++++++++--- src/jmh/proto/test_record.proto | 15 + .../java/com/imprint/core/ImprintRecord.java | 12 +- .../java/com/imprint/types/TypeHandler.java | 84 ++- src/main/java/com/imprint/types/Value.java | 36 +- .../{benchmark => profile}/ProfilerTest.java | 11 +- 9 files changed, 1169 insertions(+), 174 deletions(-) create mode 100644 src/jmh/flatbuffers/test_record.fbs create mode 100644 src/jmh/proto/test_record.proto rename src/test/java/com/imprint/{benchmark => profile}/ProfilerTest.java (97%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d4c8bde..62ac6f5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,30 +15,220 @@ jobs: java-version: [11, 17, 21] steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up JDK ${{ matrix.java-version }} - uses: actions/setup-java@v4 - with: - java-version: ${{ matrix.java-version }} - distribution: 'temurin' - - - name: Cache Gradle dependencies - uses: actions/cache@v4 - with: - path: | - ~/.gradle/caches - ~/.gradle/wrapper - key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} - restore-keys: | - ${{ runner.os }}-gradle- - - - name: Make gradlew executable - run: chmod +x ./gradlew - - - name: Run tests - run: ./gradlew test - - - name: Run build - run: ./gradlew build \ No newline at end of file + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up JDK ${{ matrix.java-version }} + uses: actions/setup-java@v4 + with: + java-version: ${{ matrix.java-version }} + distribution: 'temurin' + + - name: Cache Gradle dependencies + uses: actions/cache@v4 + with: + path: | + ~/.gradle/caches + ~/.gradle/wrapper + key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} + restore-keys: | + ${{ runner.os }}-gradle- + + - name: Make gradlew executable + run: chmod +x ./gradlew + + - name: Run tests + run: ./gradlew test + + - name: Run build + run: ./gradlew build + + benchmark: + runs-on: ubuntu-latest + needs: test + # Only run benchmarks on main branch pushes and PRs to main to avoid excessive CI time + if: github.ref == 'refs/heads/main' || github.base_ref == 'main' + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up JDK 11 + uses: actions/setup-java@v4 + with: + java-version: '11' + distribution: 'temurin' + + - name: Cache Gradle dependencies + uses: actions/cache@v4 + with: + path: | + ~/.gradle/caches + ~/.gradle/wrapper + key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} + restore-keys: | + ${{ runner.os }}-gradle- + + - name: Make gradlew executable + run: chmod +x ./gradlew + + - name: Create benchmark results directory + run: mkdir -p benchmark-results + + - name: Run serialization benchmarks + run: | + ./gradlew jmhRunSerializationBenchmarks + continue-on-error: true + + - name: Run deserialization benchmarks + run: | + ./gradlew jmhRunDeserializationBenchmarks + continue-on-error: true + + - name: Run field access benchmarks + run: | + ./gradlew jmhRunFieldAccessBenchmarks + continue-on-error: true + + - name: Run size comparison benchmarks + run: | + ./gradlew jmhRunSizeComparisonBenchmarks + continue-on-error: true + + - name: Upload benchmark results + uses: actions/upload-artifact@v4 + if: always() + with: + name: benchmark-results-${{ github.sha }} + path: benchmark-results/ + retention-days: 30 + + - name: Comment benchmark results on PR + if: github.event_name == 'pull_request' + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const path = require('path'); + + // Find the latest benchmark results file + const resultsDir = 'benchmark-results'; + let latestFile = null; + let latestTime = 0; + + if (fs.existsSync(resultsDir)) { + const files = fs.readdirSync(resultsDir); + for (const file of files) { + if (file.endsWith('.json')) { + const filePath = path.join(resultsDir, file); + const stats = fs.statSync(filePath); + if (stats.mtime.getTime() > latestTime) { + latestTime = stats.mtime.getTime(); + latestFile = filePath; + } + } + } + } + + if (latestFile) { + const results = JSON.parse(fs.readFileSync(latestFile, 'utf8')); + + // Group results by benchmark type + const serialization = results.filter(r => r.benchmark.includes('serialize')); + const deserialization = results.filter(r => r.benchmark.includes('deserialize')); + const fieldAccess = results.filter(r => r.benchmark.includes('singleFieldAccess')); + const sizes = results.filter(r => r.benchmark.includes('measure')); + + // Format results into a table + const formatResults = (benchmarks, title) => { + if (benchmarks.length === 0) return ''; + + let table = `\n### ${title}\n\n| Library | Score (ns/op) | Error | Unit |\n|---------|---------------|-------|------|\n`; + + benchmarks + .sort((a, b) => a.primaryMetric.score - b.primaryMetric.score) + .forEach(benchmark => { + const name = benchmark.benchmark.split('.').pop().replace(/serialize|deserialize|singleFieldAccess|measure/, '').replace(/Imprint|JacksonJson|Kryo|MessagePack|Avro|Protobuf|FlatBuffers/, (match) => match); + const score = benchmark.primaryMetric.score.toFixed(2); + const error = benchmark.primaryMetric.scoreError.toFixed(2); + const unit = benchmark.primaryMetric.scoreUnit; + table += `| ${name} | ${score} | ±${error} | ${unit} |\n`; + }); + + return table; + }; + + const comment = `##Benchmark Results + + Benchmark comparison between Imprint and other serialization libraries: + ${formatResults(serialization, 'Serialization Performance')} + ${formatResults(deserialization, 'Deserialization Performance')} + ${formatResults(fieldAccess, 'Single Field Access Performance')} + ${formatResults(sizes, 'Serialized Size Comparison')} + +
+ View detailed results + + Results generated from commit: \`${context.sha.substring(0, 7)}\` + + Lower scores are better for performance benchmarks. + +
`; + + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: comment + }); + } else { + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: '## Benchmark Results\n\nBenchmark execution completed but no results file was found. Check the [workflow logs](' + + `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}` + ') for details.' + }); + } + + # Optional: Run full benchmark suite on releases + benchmark-full: + runs-on: ubuntu-latest + if: startsWith(github.ref, 'refs/tags/') + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up JDK 11 + uses: actions/setup-java@v4 + with: + java-version: '11' + distribution: 'temurin' + + - name: Cache Gradle dependencies + uses: actions/cache@v4 + with: + path: | + ~/.gradle/caches + ~/.gradle/wrapper + key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} + restore-keys: | + ${{ runner.os }}-gradle- + + - name: Make gradlew executable + run: chmod +x ./gradlew + + - name: Create benchmark results directory + run: mkdir -p benchmark-results + + - name: Run full benchmark suite + run: | + ./gradlew jmh + + - name: Upload full benchmark results + uses: actions/upload-artifact@v4 + with: + name: full-benchmark-results-${{ github.ref_name }} + path: benchmark-results/ + retention-days: 90 \ No newline at end of file diff --git a/build.gradle b/build.gradle index 2606710..66f2e3e 100644 --- a/build.gradle +++ b/build.gradle @@ -2,6 +2,8 @@ plugins { id 'java-library' id 'maven-publish' id 'me.champeau.jmh' version '0.7.2' + id 'com.google.protobuf' version '0.9.4' + id 'io.netifi.flatbuffers' version '1.0.7' } group = 'com.imprint' @@ -23,45 +25,290 @@ dependencies { // Lombok for reducing boilerplate compileOnly 'org.projectlombok:lombok:1.18.30' annotationProcessor 'org.projectlombok:lombok:1.18.30' - + // Test dependencies testImplementation 'org.junit.jupiter:junit-jupiter:5.10.0' testImplementation 'org.assertj:assertj-core:3.24.2' testImplementation 'org.mockito:mockito-core:5.5.0' - + // Lombok for tests testCompileOnly 'org.projectlombok:lombok:1.18.30' testAnnotationProcessor 'org.projectlombok:lombok:1.18.30' - + // Performance testing with JMH jmhImplementation 'org.openjdk.jmh:jmh-core:1.37' jmhAnnotationProcessor 'org.openjdk.jmh:jmh-generator-annprocess:1.37' - - // Competitor libraries for benchmarking + + // Suppress SLF4J warnings + jmhImplementation 'org.slf4j:slf4j-nop:1.7.36' + + // Competitor libraries for benchmarking (JMH only) jmhImplementation 'com.google.protobuf:protobuf-java:3.25.1' jmhImplementation 'org.apache.avro:avro:1.11.3' jmhImplementation 'com.fasterxml.jackson.core:jackson-databind:2.16.0' jmhImplementation 'com.google.flatbuffers:flatbuffers-java:23.5.26' jmhImplementation 'com.esotericsoftware:kryo:5.4.0' + jmhImplementation 'org.msgpack:msgpack-core:0.9.8' + jmhImplementation 'org.msgpack:jackson-dataformat-msgpack:0.9.8' +} + +protobuf { + protoc { + artifact = "com.google.protobuf:protoc:3.25.1" + } + generateProtoTasks { + // Only generate for JMH, not main + all().each { task -> + task.enabled = false + } + ofSourceSet('jmh').each { task -> + task.enabled = true + task.builtins { + java { + outputSubDir = 'java' + } + } + } + } +} + +// Download and setup FlatBuffers compiler for Linux (CI environment) +tasks.register('downloadFlatc') { + description = 'Download FlatBuffers compiler' + group = 'build setup' + + def flatcDir = file("${buildDir}/flatc") + def flatcExe = file("${flatcDir}/flatc") + def isWindows = System.getProperty('os.name').toLowerCase().contains('windows') + def flatcUrl = isWindows ? + "https://github.com/google/flatbuffers/releases/download/v23.5.26/Windows.flatc.binary.zip" : + "https://github.com/google/flatbuffers/releases/download/v23.5.26/Linux.flatc.binary.clang++-12.zip" + def flatcZip = file("${buildDir}/flatc.zip") + + outputs.file(flatcExe) + + doLast { + if (!flatcExe.exists()) { + println "Downloading FlatBuffers compiler for ${isWindows ? 'Windows' : 'Linux'}..." + flatcDir.mkdirs() + + // Download + new URL(flatcUrl).withInputStream { i -> + flatcZip.withOutputStream { it << i } + } + + // Extract + copy { + from zipTree(flatcZip) + into flatcDir + } + + // Make executable on Unix systems + if (!isWindows) { + exec { + commandLine 'chmod', '+x', flatcExe.absolutePath + } + } + + flatcZip.delete() + println "FlatBuffers compiler downloaded to: ${flatcExe}" + } + } +} + +// Generate FlatBuffers sources +tasks.register('generateFlatBuffers', Exec) { + dependsOn downloadFlatc + description = 'Generate Java classes from FlatBuffers schema' + group = 'build' + + def isWindows = System.getProperty('os.name').toLowerCase().contains('windows') + def flatcExe = file("${buildDir}/flatc/${isWindows ? 'flatc.exe' : 'flatc'}") + def schemaFile = file('src/jmh/flatbuffers/test_record.fbs') + def outputDir = file('build/generated/source/flatbuffers/jmh/java') + + commandLine flatcExe.absolutePath, '--java', '-o', outputDir.absolutePath, schemaFile.absolutePath + + inputs.file(schemaFile) + outputs.dir(outputDir) + + doFirst { + outputDir.mkdirs() + } +} + +// Add generated FlatBuffers sources to JMH source set +sourceSets { + jmh { + java { + srcDir 'build/generated/source/flatbuffers/jmh/java' + } + proto { + srcDir 'src/jmh/proto' + } + } +} + +// Make JMH compilation depend on FlatBuffers generation +compileJmhJava.dependsOn generateFlatBuffers + +// Handle duplicate proto files +tasks.named('processJmhResources') { + duplicatesStrategy = DuplicatesStrategy.EXCLUDE } test { useJUnitPlatform() - + // Enable detailed test output testLogging { events "passed", "skipped", "failed" } } -// JMH configuration +// JMH configuration - optimized for Java 11 jmh { fork = 1 - warmupIterations = 3 + warmupIterations = 2 // Reduced for faster CI iterations = 3 resultFormat = 'JSON' includeTests = false resultsFile = file("${projectDir}/benchmark-results/jmh-results-${new Date().format('yyyy-MM-dd-HHmmss')}.json") + + // Java 11 specific JVM args + jvmArgs = [ + '-XX:+UseG1GC', + '-Xmx2g', + '-XX:+UnlockExperimentalVMOptions', + '-XX:+UseJVMCICompiler' // Use Graal if available for better performance + ] +} + +// Create individual benchmark tasks for CI pipeline +tasks.register('jmhRunSerializationBenchmarks', JavaExec) { + dependsOn compileJmhJava + description = 'Run serialization benchmarks' + group = 'benchmarking' + + classpath = sourceSets.jmh.runtimeClasspath + mainClass = 'com.imprint.benchmark.ComparisonBenchmark' + args = ['runSerializationBenchmarks'] + + // Java 11 optimized JVM settings + jvmArgs = [ + '-XX:+UseG1GC', + '-Xmx2g', + '-XX:+UnlockExperimentalVMOptions' + ] + + doFirst { + file("${projectDir}/benchmark-results").mkdirs() + } +} + +tasks.register('jmhRunDeserializationBenchmarks', JavaExec) { + dependsOn compileJmhJava + description = 'Run deserialization benchmarks' + group = 'benchmarking' + + classpath = sourceSets.jmh.runtimeClasspath + mainClass = 'com.imprint.benchmark.ComparisonBenchmark' + args = ['runDeserializationBenchmarks'] + + // Java 11 optimized JVM settings + jvmArgs = [ + '-XX:+UseG1GC', + '-Xmx2g', + '-XX:+UnlockExperimentalVMOptions' + ] + + doFirst { + file("${projectDir}/benchmark-results").mkdirs() + } +} + +tasks.register('jmhRunFieldAccessBenchmarks', JavaExec) { + dependsOn compileJmhJava + description = 'Run field access benchmarks' + group = 'benchmarking' + + classpath = sourceSets.jmh.runtimeClasspath + mainClass = 'com.imprint.benchmark.ComparisonBenchmark' + args = ['runFieldAccessBenchmarks'] + + // Java 11 optimized JVM settings + jvmArgs = [ + '-XX:+UseG1GC', + '-Xmx2g', + '-XX:+UnlockExperimentalVMOptions' + ] + + doFirst { + file("${projectDir}/benchmark-results").mkdirs() + } +} + +tasks.register('jmhRunSizeComparisonBenchmarks', JavaExec) { + dependsOn compileJmhJava + description = 'Run size comparison benchmarks' + group = 'benchmarking' + + classpath = sourceSets.jmh.runtimeClasspath + mainClass = 'com.imprint.benchmark.ComparisonBenchmark' + args = ['runSizeComparisonBenchmarks'] + + // Java 11 optimized JVM settings + jvmArgs = [ + '-XX:+UseG1GC', + '-Xmx2g', + '-XX:+UnlockExperimentalVMOptions' + ] + + doFirst { + file("${projectDir}/benchmark-results").mkdirs() + } +} + +tasks.register('jmhRunMergeBenchmarks', JavaExec) { + dependsOn compileJmhJava + description = 'Run merge operation benchmarks' + group = 'benchmarking' + + classpath = sourceSets.jmh.runtimeClasspath + mainClass = 'com.imprint.benchmark.ComparisonBenchmark' + args = ['runMergeBenchmarks'] + + // Java 11 optimized JVM settings + jvmArgs = [ + '-XX:+UseG1GC', + '-Xmx2g', + '-XX:+UnlockExperimentalVMOptions' + ] + + doFirst { + file("${projectDir}/benchmark-results").mkdirs() + } +} + +tasks.register('jmhRunAllBenchmarks', JavaExec) { + dependsOn compileJmhJava + description = 'Run all comparison benchmarks' + group = 'benchmarking' + + classpath = sourceSets.jmh.runtimeClasspath + mainClass = 'com.imprint.benchmark.ComparisonBenchmark' + args = ['runAll'] + + // Java 11 optimized JVM settings + jvmArgs = [ + '-XX:+UseG1GC', + '-Xmx2g', + '-XX:+UnlockExperimentalVMOptions' + ] + + doFirst { + file("${projectDir}/benchmark-results").mkdirs() + } } compileJava { @@ -75,4 +322,4 @@ javadoc { } // Don't fail build on missing javadoc options.addStringOption('Xdoclint:none', '-quiet') -} +} \ No newline at end of file diff --git a/src/jmh/flatbuffers/test_record.fbs b/src/jmh/flatbuffers/test_record.fbs new file mode 100644 index 0000000..ccc31d0 --- /dev/null +++ b/src/jmh/flatbuffers/test_record.fbs @@ -0,0 +1,15 @@ +namespace com.imprint.benchmark; + +table TestRecordFB { + id: int; + name: string; + price: double; + active: bool; + category: string; + tags: [string]; + metadata_keys: [string]; + metadata_values: [string]; + extra_data: [string]; +} + +root_type TestRecordFB; \ No newline at end of file diff --git a/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java b/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java index 8163522..49260b1 100644 --- a/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java +++ b/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java @@ -4,13 +4,25 @@ import com.esotericsoftware.kryo.io.Input; import com.esotericsoftware.kryo.io.Output; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.flatbuffers.FlatBufferBuilder; import com.imprint.core.ImprintRecord; import com.imprint.core.ImprintWriter; import com.imprint.core.SchemaId; import com.imprint.types.MapKey; import com.imprint.types.Value; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericDatumReader; +import org.apache.avro.generic.GenericDatumWriter; +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.io.*; +import org.msgpack.jackson.dataformat.MessagePackFactory; import org.openjdk.jmh.annotations.*; import org.openjdk.jmh.infra.Blackhole; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; @@ -25,38 +37,56 @@ @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.NANOSECONDS) @State(Scope.Benchmark) -@Warmup(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS) -@Measurement(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS) +@Warmup(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 10, time = 1, timeUnit = TimeUnit.SECONDS) @Fork(1) +@SuppressWarnings("unused") public class ComparisonBenchmark { // Test data private TestRecord testData; - + // Serialized formats - private ByteBuffer imprintBytes; - private byte[] jacksonBytes; + private ByteBuffer imprintBytesBuffer; + private byte[] jacksonJsonBytes; private byte[] kryoBytes; - + private byte[] messagePackBytes; + private byte[] avroBytes; + private byte[] protobufBytes; + private ByteBuffer flatbuffersBytes; + // Library instances - private ObjectMapper jackson; + private Schema avroSchema; + private DatumWriter avroWriter; + private DatumReader avroReader; + private ObjectMapper jacksonJsonMapper; private Kryo kryo; + private ObjectMapper messagePackMapper; @Setup public void setup() throws Exception { testData = createTestRecord(); - + // Initialize libraries - jackson = new ObjectMapper(); + jacksonJsonMapper = new ObjectMapper(); kryo = new Kryo(); kryo.register(TestRecord.class); kryo.register(ArrayList.class); kryo.register(HashMap.class); - + kryo.register(Arrays.asList().getClass()); + + // Initialize MessagePack ObjectMapper + messagePackMapper = new ObjectMapper(new MessagePackFactory()); + setupAvro(); + // Pre-serialize for deserialization benchmarks - imprintBytes = serializeWithImprint(testData); - jacksonBytes = serializeWithJackson(testData); + imprintBytesBuffer = serializeWithImprint(testData); + jacksonJsonBytes = serializeWithJacksonJson(testData); kryoBytes = serializeWithKryo(testData); + messagePackBytes = serializeWithMessagePack(testData); + avroBytes = serializeWithAvro(testData); + protobufBytes = serializeWithProtobuf(testData); + flatbuffersBytes = serializeWithFlatBuffers(testData); } // ===== SERIALIZATION BENCHMARKS ===== @@ -68,8 +98,8 @@ public void serializeImprint(Blackhole bh) throws Exception { } @Benchmark - public void serializeJackson(Blackhole bh) throws Exception { - byte[] result = serializeWithJackson(testData); + public void serializeJacksonJson(Blackhole bh) throws Exception { + byte[] result = serializeWithJacksonJson(testData); bh.consume(result); } @@ -79,17 +109,41 @@ public void serializeKryo(Blackhole bh) { bh.consume(result); } + @Benchmark + public void serializeMessagePack(Blackhole bh) throws Exception { + byte[] result = serializeWithMessagePack(testData); + bh.consume(result); + } + + @Benchmark + public void serializeAvro(Blackhole bh) throws Exception { + byte[] result = serializeWithAvro(testData); + bh.consume(result); + } + + @Benchmark + public void serializeProtobuf(Blackhole bh) { + byte[] result = serializeWithProtobuf(testData); + bh.consume(result); + } + + @Benchmark + public void serializeFlatBuffers(Blackhole bh) { + ByteBuffer result = serializeWithFlatBuffers(testData); + bh.consume(result); + } + // ===== DESERIALIZATION BENCHMARKS ===== @Benchmark public void deserializeImprint(Blackhole bh) throws Exception { - ImprintRecord result = ImprintRecord.deserialize(imprintBytes.duplicate()); + ImprintRecord result = ImprintRecord.deserialize(imprintBytesBuffer.duplicate()); bh.consume(result); } @Benchmark - public void deserializeJackson(Blackhole bh) throws Exception { - TestRecord result = jackson.readValue(jacksonBytes, TestRecord.class); + public void deserializeJacksonJson(Blackhole bh) throws Exception { + TestRecord result = jacksonJsonMapper.readValue(jacksonJsonBytes, TestRecord.class); bh.consume(result); } @@ -101,135 +155,361 @@ public void deserializeKryo(Blackhole bh) { bh.consume(result); } + @Benchmark + public void deserializeMessagePack(Blackhole bh) throws Exception { + TestRecord result = messagePackMapper.readValue(messagePackBytes, TestRecord.class); + bh.consume(result); + } + + @Benchmark + public void deserializeAvro(Blackhole bh) throws Exception { + GenericRecord result = deserializeWithAvro(avroBytes); + bh.consume(result); + } + + @Benchmark + public void deserializeProtobuf(Blackhole bh) throws Exception { + TestRecordProto.TestRecord result = TestRecordProto.TestRecord.parseFrom(protobufBytes); + bh.consume(result); + } + + @Benchmark + public void deserializeFlatBuffers(Blackhole bh) { + TestRecordFB result = TestRecordFB.getRootAsTestRecordFB(flatbuffersBytes.duplicate()); + bh.consume(result); + } + // ===== FIELD ACCESS BENCHMARKS ===== // Tests accessing a single field near the end of a large record + // This showcases Imprint's O(1) directory lookup vs sequential deserialization + @Benchmark public void singleFieldAccessImprint(Blackhole bh) throws Exception { - ImprintRecord record = ImprintRecord.deserialize(imprintBytes.duplicate()); - - // Access field 15 directly via directory lookup - O(1) - var field15 = record.getValue(15); + ImprintRecord record = ImprintRecord.deserialize(imprintBytesBuffer.duplicate()); + var field15 = record.getString(15); bh.consume(field15); } @Benchmark - public void singleFieldAccessJackson(Blackhole bh) throws Exception { - // Jackson must deserialize entire object to access any field - TestRecord record = jackson.readValue(jacksonBytes, TestRecord.class); - - // Access field15 equivalent (extraData[4]) after full deserialization + public void singleFieldAccessJacksonJson(Blackhole bh) throws Exception { + TestRecord record = jacksonJsonMapper.readValue(jacksonJsonBytes, TestRecord.class); bh.consume(record.extraData.get(4)); } @Benchmark public void singleFieldAccessKryo(Blackhole bh) { - // Kryo must deserialize entire object to access any field Input input = new Input(new ByteArrayInputStream(kryoBytes)); TestRecord record = kryo.readObject(input, TestRecord.class); input.close(); - - // Access field15 equivalent (extraData[4]) after full deserialization bh.consume(record.extraData.get(4)); } + @Benchmark + public void singleFieldAccessMessagePack(Blackhole bh) throws Exception { + TestRecord record = messagePackMapper.readValue(messagePackBytes, TestRecord.class); + bh.consume(record.extraData.get(4)); + } + + @Benchmark + public void singleFieldAccessAvro(Blackhole bh) throws Exception { + GenericRecord record = deserializeWithAvro(avroBytes); + bh.consume(record.get("extraData4")); // Accessing field near end + } + + @Benchmark + public void singleFieldAccessProtobuf(Blackhole bh) throws Exception { + TestRecordProto.TestRecord record = TestRecordProto.TestRecord.parseFrom(protobufBytes); + bh.consume(record.getExtraData(4)); // Accessing field near end + } + + @Benchmark + public void singleFieldAccessFlatBuffers(Blackhole bh) { + TestRecordFB record = TestRecordFB.getRootAsTestRecordFB(flatbuffersBytes.duplicate()); + bh.consume(record.extraData(4)); // Accessing field near end - zero copy! + } + // ===== SIZE COMPARISON ===== @Benchmark - public void measureImprintSize(Blackhole bh) throws Exception { - ByteBuffer serialized = serializeWithImprint(testData); - bh.consume(serialized.remaining()); + public void measureImprintSize(Blackhole bh) { + bh.consume(imprintBytesBuffer.remaining()); } @Benchmark - public void measureJacksonSize(Blackhole bh) throws Exception { - byte[] serialized = serializeWithJackson(testData); - bh.consume(serialized.length); + public void measureJacksonJsonSize(Blackhole bh) { + bh.consume(jacksonJsonBytes.length); } @Benchmark public void measureKryoSize(Blackhole bh) { - byte[] serialized = serializeWithKryo(testData); - bh.consume(serialized.length); + bh.consume(kryoBytes.length); + } + + @Benchmark + public void measureMessagePackSize(Blackhole bh) { + bh.consume(messagePackBytes.length); + } + + @Benchmark + public void measureAvroSize(Blackhole bh) { + bh.consume(avroBytes.length); + } + + @Benchmark + public void measureProtobufSize(Blackhole bh) { + bh.consume(protobufBytes.length); + } + + @Benchmark + public void measureFlatBuffersSize(Blackhole bh) { + bh.consume(flatbuffersBytes.remaining()); } // ===== MERGE SIMULATION BENCHMARKS ===== @Benchmark public void mergeImprint(Blackhole bh) throws Exception { - var record1 = serializeWithImprint(testData); - var record2 = serializeWithImprint(createTestRecord2()); + var record1Buffer = imprintBytesBuffer.duplicate(); + var record2Data = createTestRecord2(); + var record2Buffer = serializeWithImprint(record2Data); - var deserialized1 = ImprintRecord.deserialize(record1); - var deserialized2 = ImprintRecord.deserialize(record2); + var deserialized1 = ImprintRecord.deserialize(record1Buffer); + var deserialized2 = ImprintRecord.deserialize(record2Buffer); var merged = simulateMerge(deserialized1, deserialized2); - + bh.consume(merged); } @Benchmark - public void mergeJackson(Blackhole bh) throws Exception { - // Jackson merge requires full deserialization + merge + serialization - var record1 = jackson.readValue(jacksonBytes, TestRecord.class); - var record2 = jackson.readValue(serializeWithJackson(createTestRecord2()), TestRecord.class); - - var merged = mergeTestRecords(record1, record2); - byte[] result = jackson.writeValueAsBytes(merged); - + public void mergeJacksonJson(Blackhole bh) throws Exception { + var record1 = jacksonJsonMapper.readValue(jacksonJsonBytes, TestRecord.class); + var record2Data = createTestRecord2(); + var record2Bytes = serializeWithJacksonJson(record2Data); + var record2 = jacksonJsonMapper.readValue(record2Bytes, TestRecord.class); + + var mergedPojo = mergeTestRecords(record1, record2); + byte[] result = jacksonJsonMapper.writeValueAsBytes(mergedPojo); bh.consume(result); } @Benchmark public void mergeKryo(Blackhole bh) { - // Kryo merge requires full deserialization + merge + serialization Input input1 = new Input(new ByteArrayInputStream(kryoBytes)); var record1 = kryo.readObject(input1, TestRecord.class); input1.close(); - - Input input2 = new Input(new ByteArrayInputStream(serializeWithKryo(createTestRecord2()))); + + var record2Data = createTestRecord2(); + var record2Bytes = serializeWithKryo(record2Data); + Input input2 = new Input(new ByteArrayInputStream(record2Bytes)); var record2 = kryo.readObject(input2, TestRecord.class); input2.close(); - - var merged = mergeTestRecords(record1, record2); - byte[] result = serializeWithKryo(merged); - + + var mergedPojo = mergeTestRecords(record1, record2); + byte[] result = serializeWithKryo(mergedPojo); + bh.consume(result); + } + + @Benchmark + public void mergeMessagePack(Blackhole bh) throws Exception { + var record1 = messagePackMapper.readValue(messagePackBytes, TestRecord.class); + var record2Data = createTestRecord2(); + var record2Bytes = serializeWithMessagePack(record2Data); + var record2 = messagePackMapper.readValue(record2Bytes, TestRecord.class); + + var mergedPojo = mergeTestRecords(record1, record2); + byte[] result = messagePackMapper.writeValueAsBytes(mergedPojo); bh.consume(result); } + @Benchmark + public void mergeAvro(Blackhole bh) throws Exception { + var record1 = deserializeWithAvro(avroBytes); + var record2Data = createTestRecord2(); + var record2Bytes = serializeWithAvro(record2Data); + var record2 = deserializeWithAvro(record2Bytes); + + var merged = mergeAvroRecords(record1, record2); + byte[] result = serializeAvroRecord(merged); + bh.consume(result); + } + + @Benchmark + public void mergeProtobuf(Blackhole bh) throws Exception { + var record1 = TestRecordProto.TestRecord.parseFrom(protobufBytes); + var record2Data = createTestRecord2(); + var record2Bytes = serializeWithProtobuf(record2Data); + var record2 = TestRecordProto.TestRecord.parseFrom(record2Bytes); + + var merged = mergeProtobufRecords(record1, record2); + byte[] result = merged.toByteArray(); + bh.consume(result); + } + + @Benchmark + public void mergeFlatBuffers(Blackhole bh) { + var record1 = TestRecordFB.getRootAsTestRecordFB(flatbuffersBytes.duplicate()); + var record2Data = createTestRecord2(); + var record2Buffer = serializeWithFlatBuffers(record2Data); + var record2 = TestRecordFB.getRootAsTestRecordFB(record2Buffer); + + var merged = mergeFlatBuffersRecords(record1, record2); + bh.consume(merged); + } + + // ===== MAIN METHOD TO RUN BENCHMARKS ===== + + public static void main(String[] args) throws RunnerException { + runAll(); + // Or, uncomment specific runner methods to execute subsets: + // runSerializationBenchmarks(); + // runDeserializationBenchmarks(); + // runFieldAccessBenchmarks(); + // runSizeComparisonBenchmarks(); + // runMergeBenchmarks(); + // runMessagePackBenchmarks(); + } + + public static void runAll() throws RunnerException { + Options opt = new OptionsBuilder() + .include(ComparisonBenchmark.class.getSimpleName()) + .build(); + new Runner(opt).run(); + } + + public static void runSerializationBenchmarks() throws RunnerException { + Options opt = new OptionsBuilder() + .include(ComparisonBenchmark.class.getSimpleName() + ".serialize.*") + .build(); + new Runner(opt).run(); + } + + public static void runDeserializationBenchmarks() throws RunnerException { + Options opt = new OptionsBuilder() + .include(ComparisonBenchmark.class.getSimpleName() + ".deserialize.*") + .build(); + new Runner(opt).run(); + } + + public static void runFieldAccessBenchmarks() throws RunnerException { + Options opt = new OptionsBuilder() + .include(ComparisonBenchmark.class.getSimpleName() + ".singleFieldAccess.*") + .build(); + new Runner(opt).run(); + } + + public static void runSizeComparisonBenchmarks() throws RunnerException { + Options opt = new OptionsBuilder() + .include(ComparisonBenchmark.class.getSimpleName() + ".measure.*") + .build(); + new Runner(opt).run(); + } + + public static void runMergeBenchmarks() throws RunnerException { + Options opt = new OptionsBuilder() + .include(ComparisonBenchmark.class.getSimpleName() + ".merge.*") + .build(); + new Runner(opt).run(); + } + + public static void runMessagePackBenchmarks() throws RunnerException { + Options opt = new OptionsBuilder() + .include(ComparisonBenchmark.class.getSimpleName() + ".*MessagePack.*") + .build(); + new Runner(opt).run(); + } + + public static void runAvroBenchmarks() throws RunnerException { + Options opt = new OptionsBuilder() + .include(ComparisonBenchmark.class.getSimpleName() + ".*Avro.*") + .build(); + new Runner(opt).run(); + } + + public static void runProtobufBenchmarks() throws RunnerException { + Options opt = new OptionsBuilder() + .include(ComparisonBenchmark.class.getSimpleName() + ".*Protobuf.*") + .build(); + new Runner(opt).run(); + } + + public static void runFlatBuffersBenchmarks() throws RunnerException { + Options opt = new OptionsBuilder() + .include(ComparisonBenchmark.class.getSimpleName() + ".*FlatBuffers.*") + .build(); + new Runner(opt).run(); + } + // ===== HELPER METHODS ===== + private void setupAvro() { + String schemaJson = "{\n" + + " \"type\": \"record\",\n" + + " \"name\": \"TestRecord\",\n" + + " \"fields\": [\n" + + " {\"name\": \"id\", \"type\": \"int\"},\n" + + " {\"name\": \"name\", \"type\": \"string\"},\n" + + " {\"name\": \"price\", \"type\": \"double\"},\n" + + " {\"name\": \"active\", \"type\": \"boolean\"},\n" + + " {\"name\": \"category\", \"type\": \"string\"},\n" + + " {\"name\": \"tags\", \"type\": {\"type\": \"array\", \"items\": \"string\"}},\n" + + " {\"name\": \"metadata\", \"type\": {\"type\": \"map\", \"values\": \"string\"}},\n" + + " {\"name\": \"extraData0\", \"type\": \"string\"},\n" + + " {\"name\": \"extraData1\", \"type\": \"string\"},\n" + + " {\"name\": \"extraData2\", \"type\": \"string\"},\n" + + " {\"name\": \"extraData3\", \"type\": \"string\"},\n" + + " {\"name\": \"extraData4\", \"type\": \"string\"},\n" + + " {\"name\": \"extraData5\", \"type\": \"string\"},\n" + + " {\"name\": \"extraData6\", \"type\": \"string\"},\n" + + " {\"name\": \"extraData7\", \"type\": \"string\"},\n" + + " {\"name\": \"extraData8\", \"type\": \"string\"},\n" + + " {\"name\": \"extraData9\", \"type\": \"string\"},\n" + + " {\"name\": \"extraData10\", \"type\": \"string\"},\n" + + " {\"name\": \"extraData11\", \"type\": \"string\"},\n" + + " {\"name\": \"extraData12\", \"type\": \"string\"}\n" + + " ]\n" + + "}"; + + avroSchema = new Schema.Parser().parse(schemaJson); + avroWriter = new GenericDatumWriter<>(avroSchema); + avroReader = new GenericDatumReader<>(avroSchema); + } + private ByteBuffer serializeWithImprint(TestRecord data) throws Exception { var writer = new ImprintWriter(new SchemaId(1, 0x12345678)); - + writer.addField(1, Value.fromInt32(data.id)); writer.addField(2, Value.fromString(data.name)); writer.addField(3, Value.fromFloat64(data.price)); writer.addField(4, Value.fromBoolean(data.active)); writer.addField(5, Value.fromString(data.category)); - - // Convert tags list + var tagValues = new ArrayList(); - for (String tag : data.tags) { - tagValues.add(Value.fromString(tag)); + if (data.tags != null) { + for (String tag : data.tags) { + tagValues.add(Value.fromString(tag)); + } } writer.addField(6, Value.fromArray(tagValues)); - - // Convert metadata map + var metadataMap = new HashMap(); - for (var entry : data.metadata.entrySet()) { - metadataMap.put(MapKey.fromString(entry.getKey()), Value.fromString(entry.getValue())); + if (data.metadata != null) { + for (var entry : data.metadata.entrySet()) { + metadataMap.put(MapKey.fromString(entry.getKey()), Value.fromString(entry.getValue())); + } } writer.addField(7, Value.fromMap(metadataMap)); - - // Add extra fields (8-20) to create a larger record - for (int i = 0; i < data.extraData.size(); i++) { - writer.addField(8 + i, Value.fromString(data.extraData.get(i))); + + if (data.extraData != null) { + for (int i = 0; i < data.extraData.size(); i++) { + writer.addField(8 + i, Value.fromString(data.extraData.get(i))); + } } - + return writer.build().serializeToBuffer(); } - private byte[] serializeWithJackson(TestRecord data) throws Exception { - return jackson.writeValueAsBytes(data); + private byte[] serializeWithJacksonJson(TestRecord data) throws Exception { + return jacksonJsonMapper.writeValueAsBytes(data); } private byte[] serializeWithKryo(TestRecord data) { @@ -240,14 +520,117 @@ private byte[] serializeWithKryo(TestRecord data) { return baos.toByteArray(); } + private byte[] serializeWithMessagePack(TestRecord data) throws Exception { + return messagePackMapper.writeValueAsBytes(data); + } + + private byte[] serializeWithAvro(TestRecord data) throws Exception { + GenericRecord record = new GenericData.Record(avroSchema); + record.put("id", data.id); + record.put("name", data.name); + record.put("price", data.price); + record.put("active", data.active); + record.put("category", data.category); + record.put("tags", data.tags); + record.put("metadata", data.metadata); + + for (int i = 0; i < data.extraData.size(); i++) { + record.put("extraData" + i, data.extraData.get(i)); + } + + return serializeAvroRecord(record); + } + + private byte[] serializeAvroRecord(GenericRecord record) throws Exception { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + Encoder encoder = EncoderFactory.get().binaryEncoder(baos, null); + avroWriter.write(record, encoder); + encoder.flush(); + return baos.toByteArray(); + } + + private GenericRecord deserializeWithAvro(byte[] data) throws Exception { + Decoder decoder = DecoderFactory.get().binaryDecoder(data, null); + return avroReader.read(null, decoder); + } + + private byte[] serializeWithProtobuf(TestRecord data) { + var builder = TestRecordProto.TestRecord.newBuilder() + .setId(data.id) + .setName(data.name) + .setPrice(data.price) + .setActive(data.active) + .setCategory(data.category) + .addAllTags(data.tags) + .putAllMetadata(data.metadata); + + for (String extraData : data.extraData) { + builder.addExtraData(extraData); + } + + return builder.build().toByteArray(); + } + + private ByteBuffer serializeWithFlatBuffers(TestRecord data) { + FlatBufferBuilder builder = new FlatBufferBuilder(1024); + + // Create strings (must be created before the object that uses them) + int nameOffset = builder.createString(data.name); + int categoryOffset = builder.createString(data.category); + + // Create tags array + int[] tagOffsets = new int[data.tags.size()]; + for (int i = 0; i < data.tags.size(); i++) { + tagOffsets[i] = builder.createString(data.tags.get(i)); + } + int tagsOffset = TestRecordFB.createTagsVector(builder, tagOffsets); + + // Create metadata (as parallel arrays for keys and values) + String[] metadataKeys = data.metadata.keySet().toArray(new String[0]); + String[] metadataValues = new String[metadataKeys.length]; + int[] keyOffsets = new int[metadataKeys.length]; + int[] valueOffsets = new int[metadataKeys.length]; + + for (int i = 0; i < metadataKeys.length; i++) { + metadataValues[i] = data.metadata.get(metadataKeys[i]); + keyOffsets[i] = builder.createString(metadataKeys[i]); + valueOffsets[i] = builder.createString(metadataValues[i]); + } + int metadataKeysOffset = TestRecordFB.createMetadataKeysVector(builder, keyOffsets); + int metadataValuesOffset = TestRecordFB.createMetadataValuesVector(builder, valueOffsets); + + // Create extra data array + int[] extraDataOffsets = new int[data.extraData.size()]; + for (int i = 0; i < data.extraData.size(); i++) { + extraDataOffsets[i] = builder.createString(data.extraData.get(i)); + } + int extraDataOffset = TestRecordFB.createExtraDataVector(builder, extraDataOffsets); + + // Create the main object + TestRecordFB.startTestRecordFB(builder); + TestRecordFB.addId(builder, data.id); + TestRecordFB.addName(builder, nameOffset); + TestRecordFB.addPrice(builder, data.price); + TestRecordFB.addActive(builder, data.active); + TestRecordFB.addCategory(builder, categoryOffset); + TestRecordFB.addTags(builder, tagsOffset); + TestRecordFB.addMetadataKeys(builder, metadataKeysOffset); + TestRecordFB.addMetadataValues(builder, metadataValuesOffset); + TestRecordFB.addExtraData(builder, extraDataOffset); + int recordOffset = TestRecordFB.endTestRecordFB(builder); + + // Finish and return + builder.finish(recordOffset); + return builder.dataBuffer().slice(); + } + private ImprintRecord simulateMerge(ImprintRecord first, ImprintRecord second) throws Exception { var writer = new ImprintWriter(first.getHeader().getSchemaId()); var usedFieldIds = new HashSet(); - - // Copy fields from first record (takes precedence) + copyFieldsToWriter(first, writer, usedFieldIds); copyFieldsToWriter(second, writer, usedFieldIds); - + return writer.build(); } @@ -265,23 +648,121 @@ private void copyFieldsToWriter(ImprintRecord record, ImprintWriter writer, Set< } private TestRecord mergeTestRecords(TestRecord first, TestRecord second) { - // Simple merge logic - first record takes precedence var merged = new TestRecord(); merged.id = first.id; merged.name = first.name != null ? first.name : second.name; merged.price = first.price != 0.0 ? first.price : second.price; merged.active = first.active; merged.category = first.category != null ? first.category : second.category; - + merged.tags = new ArrayList<>(first.tags); merged.tags.addAll(second.tags); - + merged.metadata = new HashMap<>(first.metadata); merged.metadata.putAll(second.metadata); - + return merged; } + private GenericRecord mergeAvroRecords(GenericRecord first, GenericRecord second) { + GenericRecord merged = new GenericData.Record(avroSchema); + + // Copy all fields from first record + for (Schema.Field field : avroSchema.getFields()) { + merged.put(field.name(), first.get(field.name())); + } + + // Override with non-null values from second record + for (Schema.Field field : avroSchema.getFields()) { + Object secondValue = second.get(field.name()); + if (secondValue != null && !secondValue.toString().isEmpty()) { + merged.put(field.name(), secondValue); + } + } + + return merged; + } + + private TestRecordProto.TestRecord mergeProtobufRecords(TestRecordProto.TestRecord first, TestRecordProto.TestRecord second) { + return TestRecordProto.TestRecord.newBuilder() + .mergeFrom(first) + .mergeFrom(second) + .build(); + } + + private ByteBuffer mergeFlatBuffersRecords(TestRecordFB first, TestRecordFB second) { + FlatBufferBuilder builder = new FlatBufferBuilder(1024); + + // Use second record's values if they exist, otherwise first record's values + String name = second.name() != null && !second.name().isEmpty() ? second.name() : first.name(); + String category = second.category() != null && !second.category().isEmpty() ? second.category() : first.category(); + double price = second.price() != 0.0 ? second.price() : first.price(); + boolean active = second.active(); // Use second's boolean value + int id = first.id(); // Keep first record's ID + + // Create merged strings + int nameOffset = builder.createString(name); + int categoryOffset = builder.createString(category); + + // Merge tags (combine both arrays) + List mergedTags = new ArrayList<>(); + for (int i = 0; i < first.tagsLength(); i++) { + mergedTags.add(first.tags(i)); + } + for (int i = 0; i < second.tagsLength(); i++) { + mergedTags.add(second.tags(i)); + } + + int[] tagOffsets = new int[mergedTags.size()]; + for (int i = 0; i < mergedTags.size(); i++) { + tagOffsets[i] = builder.createString(mergedTags.get(i)); + } + int tagsOffset = TestRecordFB.createTagsVector(builder, tagOffsets); + + // Merge metadata (second overwrites first) + Map mergedMetadata = new HashMap<>(); + for (int i = 0; i < first.metadataKeysLength(); i++) { + mergedMetadata.put(first.metadataKeys(i), first.metadataValues(i)); + } + for (int i = 0; i < second.metadataKeysLength(); i++) { + mergedMetadata.put(second.metadataKeys(i), second.metadataValues(i)); + } + + String[] metadataKeys = mergedMetadata.keySet().toArray(new String[0]); + int[] keyOffsets = new int[metadataKeys.length]; + int[] valueOffsets = new int[metadataKeys.length]; + + for (int i = 0; i < metadataKeys.length; i++) { + keyOffsets[i] = builder.createString(metadataKeys[i]); + valueOffsets[i] = builder.createString(mergedMetadata.get(metadataKeys[i])); + } + int metadataKeysOffset = TestRecordFB.createMetadataKeysVector(builder, keyOffsets); + int metadataValuesOffset = TestRecordFB.createMetadataValuesVector(builder, valueOffsets); + + // Use first record's extra data (or could merge both) + int[] extraDataOffsets = new int[first.extraDataLength()]; + for (int i = 0; i < first.extraDataLength(); i++) { + extraDataOffsets[i] = builder.createString(first.extraData(i)); + } + int extraDataOffset = TestRecordFB.createExtraDataVector(builder, extraDataOffsets); + + // Create the merged object + TestRecordFB.startTestRecordFB(builder); + TestRecordFB.addId(builder, id); + TestRecordFB.addName(builder, nameOffset); + TestRecordFB.addPrice(builder, price); + TestRecordFB.addActive(builder, active); + TestRecordFB.addCategory(builder, categoryOffset); + TestRecordFB.addTags(builder, tagsOffset); + TestRecordFB.addMetadataKeys(builder, metadataKeysOffset); + TestRecordFB.addMetadataValues(builder, metadataValuesOffset); + TestRecordFB.addExtraData(builder, extraDataOffset); + int recordOffset = TestRecordFB.endTestRecordFB(builder); + + builder.finish(recordOffset); + return builder.dataBuffer().slice(); + } + private TestRecord createTestRecord() { var record = new TestRecord(); record.id = 12345; @@ -289,20 +770,19 @@ var record = new TestRecord(); record.price = 99.99; record.active = true; record.category = "Electronics"; - + record.tags = Arrays.asList("popular", "trending", "bestseller"); - + record.metadata = new HashMap<>(); record.metadata.put("manufacturer", "TechCorp"); record.metadata.put("model", "TC-2024"); record.metadata.put("warranty", "2 years"); - - // Add extra data to create a larger record (fields 8-20) + record.extraData = new ArrayList<>(); for (int i = 0; i < 13; i++) { record.extraData.add("extraField" + i + "_value_" + (1000 + i)); } - + return record; } @@ -313,19 +793,18 @@ var record = new TestRecord(); record.price = 149.99; record.active = false; record.category = "Software"; - + record.tags = Arrays.asList("new", "premium"); - + record.metadata = new HashMap<>(); record.metadata.put("vendor", "SoftCorp"); record.metadata.put("version", "2.1"); - - // Add extra data to match the structure + record.extraData = new ArrayList<>(); for (int i = 0; i < 13; i++) { record.extraData.add("extraField" + i + "_value2_" + (2000 + i)); } - + return record; } @@ -339,7 +818,7 @@ public static class TestRecord { public List tags = new ArrayList<>(); public Map metadata = new HashMap<>(); public List extraData = new ArrayList<>(); // Fields 8-20 for large record test - + public TestRecord() {} // Required for deserialization } } \ No newline at end of file diff --git a/src/jmh/proto/test_record.proto b/src/jmh/proto/test_record.proto new file mode 100644 index 0000000..7a76f71 --- /dev/null +++ b/src/jmh/proto/test_record.proto @@ -0,0 +1,15 @@ +syntax = "proto3"; + +option java_package = "com.imprint.benchmark"; +option java_outer_classname = "TestRecordProto"; + +message TestRecord { + int32 id = 1; + string name = 2; + double price = 3; + bool active = 4; + string category = 5; + repeated string tags = 6; + map metadata = 7; + repeated string extra_data = 8; +} \ No newline at end of file diff --git a/src/main/java/com/imprint/core/ImprintRecord.java b/src/main/java/com/imprint/core/ImprintRecord.java index 5d4719f..2291550 100644 --- a/src/main/java/com/imprint/core/ImprintRecord.java +++ b/src/main/java/com/imprint/core/ImprintRecord.java @@ -34,7 +34,7 @@ public final class ImprintRecord { */ public ImprintRecord(Header header, List directory, ByteBuffer payload) { this.header = Objects.requireNonNull(header, "Header cannot be null"); - this.directory = List.copyOf(Objects.requireNonNull(directory, "Directory cannot be null")); + this.directory = Collections.unmodifiableList(Objects.requireNonNull(directory, "Directory cannot be null")); this.payload = payload.asReadOnlyBuffer(); // Zero-copy read-only view } @@ -79,12 +79,14 @@ private ByteBuffer getFieldBuffer(int fieldId) { int endOffset = (index + 1 < directory.size()) ? directory.get(index + 1).getOffset() : payload.limit(); - var fieldBuffer = payload.duplicate(); if (startOffset > payload.limit() || endOffset > payload.limit() || startOffset > endOffset) { return null; } + + // OPTIMIZATION: Single allocation instead of duplicate + slice + var fieldBuffer = payload.duplicate(); fieldBuffer.position(startOffset).limit(endOffset); - return fieldBuffer.slice(); + return fieldBuffer; } /** @@ -261,7 +263,7 @@ private static DirectoryEntry deserializeDirectoryEntry(ByteBuffer buffer) throw } private Value deserializeValue(TypeCode typeCode, ByteBuffer buffer) throws ImprintException { - ByteBuffer valueSpecificBuffer = buffer.duplicate(); + var valueSpecificBuffer = buffer.duplicate(); valueSpecificBuffer.order(ByteOrder.LITTLE_ENDIAN); switch (typeCode) { @@ -351,7 +353,7 @@ public double getFloat64(int fieldId) throws ImprintException { * @throws ImprintException if the field is not found, is null, or is not of type STRING. */ public String getString(int fieldId) throws ImprintException { - Value value = getValue(fieldId); + var value = getValue(fieldId); if (value == null) { throw new ImprintException(ErrorType.FIELD_NOT_FOUND, diff --git a/src/main/java/com/imprint/types/TypeHandler.java b/src/main/java/com/imprint/types/TypeHandler.java index be4fc7b..e58c355 100644 --- a/src/main/java/com/imprint/types/TypeHandler.java +++ b/src/main/java/com/imprint/types/TypeHandler.java @@ -433,26 +433,38 @@ public int estimateSize(Value value) throws ImprintException { } return arraySize; } - + @Override public ByteBuffer readValueBytes(ByteBuffer buffer) throws ImprintException { return readComplexValueBytes(buffer, "ARRAY", (tempBuffer, numElements) -> { if (tempBuffer.remaining() < 1) { - throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, - "Not enough bytes for ARRAY element type code in temp buffer during measurement."); + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, + "Not enough bytes for ARRAY element type code"); } byte elementTypeCodeByte = tempBuffer.get(); - int typeCodeLength = 1; + var elementType = TypeCode.fromByte(elementTypeCodeByte); - TypeHandler elementHandler = TypeCode.fromByte(elementTypeCodeByte).getHandler(); - int elementsDataLength = 0; - for (int i = 0; i < numElements; i++) { - int elementStartPos = tempBuffer.position(); - elementHandler.readValueBytes(tempBuffer); - elementsDataLength += (tempBuffer.position() - elementStartPos); + switch (elementType) { + case NULL: + return 1; + case BOOL: + return 1 + numElements; + case INT32: + case FLOAT32: + return 1 + (numElements * 4); + case INT64: + case FLOAT64: + return 1 + (numElements * 8); + default: + var elementHandler = elementType.getHandler(); + int elementsDataLength = 0; + for (int i = 0; i < numElements; i++) { + int elementStartPos = tempBuffer.position(); + elementHandler.readValueBytes(tempBuffer); + elementsDataLength += (tempBuffer.position() - elementStartPos); + } + return 1 + elementsDataLength; } - - return typeCodeLength + elementsDataLength; }); } }; @@ -549,28 +561,52 @@ public int estimateSize(Value value) throws ImprintException { } return mapSize; } - + @Override public ByteBuffer readValueBytes(ByteBuffer buffer) throws ImprintException { return readComplexValueBytes(buffer, "MAP", (tempBuffer, numEntries) -> { if (tempBuffer.remaining() < 2) { - throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, - "Not enough bytes for MAP key/value type codes in temp buffer during measurement."); + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, + "Not enough bytes for MAP key/value type codes"); } byte keyTypeCodeByte = tempBuffer.get(); byte valueTypeCodeByte = tempBuffer.get(); - int typeCodesLength = 2; - int entriesDataLength = 0; - for (int i = 0; i < numEntries; i++) { - int entryStartPos = tempBuffer.position(); - TypeCode.fromByte(keyTypeCodeByte).getHandler().readValueBytes(tempBuffer); - TypeCode.fromByte(valueTypeCodeByte).getHandler().readValueBytes(tempBuffer); - entriesDataLength += (tempBuffer.position() - entryStartPos); - } + TypeCode keyType = TypeCode.fromByte(keyTypeCodeByte); + TypeCode valueType = TypeCode.fromByte(valueTypeCodeByte); + + // OPTIMIZATION: Calculate sizes directly for fixed-size types + int keySize = getFixedTypeSize(keyType); + int valueSize = getFixedTypeSize(valueType); - return typeCodesLength + entriesDataLength; + if (keySize > 0 && valueSize > 0) { + // Both are fixed-size: O(1) calculation + return 2 + (numEntries * (keySize + valueSize)); + } else { + // At least one is variable-size: fall back to traversal + int entriesDataLength = 0; + for (int i = 0; i < numEntries; i++) { + int entryStartPos = tempBuffer.position(); + keyType.getHandler().readValueBytes(tempBuffer); + valueType.getHandler().readValueBytes(tempBuffer); + entriesDataLength += (tempBuffer.position() - entryStartPos); + } + return 2 + entriesDataLength; + } }); } + + private int getFixedTypeSize(TypeCode type) { + switch (type) { + case NULL: return 0; + case BOOL: return 1; + case INT32: + case FLOAT32: return 4; + case INT64: + case FLOAT64: return 8; + default: return -1; // Variable size + } + } + private void serializeMapKey(MapKey key, ByteBuffer buffer) throws ImprintException { switch (key.getTypeCode()) { diff --git a/src/main/java/com/imprint/types/Value.java b/src/main/java/com/imprint/types/Value.java index 7f3bbb9..ba747de 100644 --- a/src/main/java/com/imprint/types/Value.java +++ b/src/main/java/com/imprint/types/Value.java @@ -328,16 +328,19 @@ public String toString() { // String Value (ByteBuffer-based) public static class StringBufferValue extends Value { private final ByteBuffer value; - private volatile String cachedString; // lazy decode + private volatile String cachedString; + + private static final int THREAD_LOCAL_BUFFER_SIZE = 1024; + private static final ThreadLocal DECODE_BUFFER_CACHE = + ThreadLocal.withInitial(() -> new byte[THREAD_LOCAL_BUFFER_SIZE]); public StringBufferValue(ByteBuffer value) { - this.value = value.asReadOnlyBuffer(); // zero-copy read-only view + this.value = value.asReadOnlyBuffer(); } public String getValue() { String result = cachedString; if (result == null) { - // Simple, fast decoding - no thread-local overhead result = decodeUtf8(); cachedString = result; } @@ -345,18 +348,25 @@ public String getValue() { } private String decodeUtf8() { - // Fast path: zero-copy for array-backed ByteBuffers + final byte[] array; + final int offset; + final int length = value.remaining(); + if (value.hasArray()) { - return new String(value.array(), value.arrayOffset() + value.position(), - value.remaining(), StandardCharsets.UTF_8); + array = value.array(); + offset = value.arrayOffset() + value.position(); + } else { + byte[] threadLocalBuffer = DECODE_BUFFER_CACHE.get(); + if (length <= threadLocalBuffer.length) { + array = threadLocalBuffer; + } else { + // Fallback: copy bytes from the ByteBuffer to a new heap array (if too large for cache) + array = new byte[length]; + } + value.duplicate().get(array, 0, length); // Get bytes from current position into chosen array + offset = 0; } - - // Fallback path for non-array-backed ByteBuffers (e.g., direct buffers). - // Allocation is required here as Java's String(byte[],...) constructor needs a heap array. - // Data is copied from the ByteBuffer to a new byte array. - var array = new byte[value.remaining()]; - value.duplicate().get(array); - return new String(array, StandardCharsets.UTF_8); + return new String(array, offset, length, StandardCharsets.UTF_8); } public ByteBuffer getBuffer() { diff --git a/src/test/java/com/imprint/benchmark/ProfilerTest.java b/src/test/java/com/imprint/profile/ProfilerTest.java similarity index 97% rename from src/test/java/com/imprint/benchmark/ProfilerTest.java rename to src/test/java/com/imprint/profile/ProfilerTest.java index 5b531a9..d48c1aa 100644 --- a/src/test/java/com/imprint/benchmark/ProfilerTest.java +++ b/src/test/java/com/imprint/profile/ProfilerTest.java @@ -1,9 +1,10 @@ -package com.imprint.benchmark; +package com.imprint.profile; -import com.imprint.core.*; +import com.imprint.core.ImprintRecord; +import com.imprint.core.ImprintWriter; +import com.imprint.core.SchemaId; import com.imprint.types.Value; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.Disabled; import java.util.Random; @@ -25,11 +26,11 @@ * - String operations and UTF-8 encoding * - ByteBuffer operations */ -@Disabled("Enable manually for profiling") +//@Disabled("Enable manually for profiling") public class ProfilerTest { private static final int ITERATIONS = 1_000_000; - private static final int RECORD_SIZE = 20; + private static final int RECORD_SIZE = 50; @Test void profileFieldAccess() throws Exception { From f7a6e8e02cb2d907c412362e306da1774449b4e7 Mon Sep 17 00:00:00 2001 From: expand3d <> Date: Thu, 5 Jun 2025 14:59:54 -0400 Subject: [PATCH 24/34] Lazy load of directory and header data --- .../benchmark/ComparisonBenchmark.java | 68 ++- .../java/com/imprint/core/ImprintRecord.java | 405 +++++++++++++----- .../java/com/imprint/IntegrationTest.java | 68 ++- .../com/imprint/profile/ProfilerTest.java | 3 +- 4 files changed, 399 insertions(+), 145 deletions(-) diff --git a/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java b/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java index 49260b1..ce2fbcb 100644 --- a/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java +++ b/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java @@ -133,14 +133,34 @@ public void serializeFlatBuffers(Blackhole bh) { bh.consume(result); } - // ===== DESERIALIZATION BENCHMARKS ===== + // ===== PARTIAL DESERIALIZATION (SETUP ONLY) ===== +// These benchmarks measure the cost of preparing a record for field access, +// not the cost of accessing the actual data. This is important because +// +// 1. Imprint: Only parses header + stores raw directory bytes +// 2. FlatBuffers: Only wraps the buffer with minimal validation +// 3. Others (eager): Parse and construct all field objects upfront +// +// This comparison shows the advantage of lazy loading approaches when you +// only need to access a subset of fields. In real streaming workloads, +// records are often filtered/routed based on just a few key fields. +// +// For a fair "full deserialization" comparison, see FULL DESERIALIZATION BENCHMARKS. + + @Benchmark + public void deserializeSetupImprint(Blackhole bh) throws Exception { + ImprintRecord result = ImprintRecord.deserialize(imprintBytesBuffer.duplicate()); + bh.consume(result); + } @Benchmark - public void deserializeImprint(Blackhole bh) throws Exception { - ImprintRecord result = ImprintRecord.deserialize(imprintBytesBuffer.duplicate()); + public void deserializeSetupFlatBuffers(Blackhole bh) { + TestRecordFB result = TestRecordFB.getRootAsTestRecordFB(flatbuffersBytes.duplicate()); bh.consume(result); } + // ===== FULL DESERIALIZATION BENCHMARKS ===== + @Benchmark public void deserializeJacksonJson(Blackhole bh) throws Exception { TestRecord result = jacksonJsonMapper.readValue(jacksonJsonBytes, TestRecord.class); @@ -173,15 +193,53 @@ public void deserializeProtobuf(Blackhole bh) throws Exception { bh.consume(result); } + @Benchmark + public void deserializeImprint(Blackhole bh) throws Exception { + ImprintRecord result = ImprintRecord.deserialize(imprintBytesBuffer.duplicate()); + // Access all fields to force full deserialization + result.getInt32(1); // id + result.getString(2); // name + result.getFloat64(3); // price + result.getBoolean(4); // active + result.getString(5); // category + result.getArray(6); // tags + result.getMap(7); // metadata + for (int i = 8; i < 21; i++) { + result.getString(i); // extraData fields + } + + bh.consume(result); + } + @Benchmark public void deserializeFlatBuffers(Blackhole bh) { TestRecordFB result = TestRecordFB.getRootAsTestRecordFB(flatbuffersBytes.duplicate()); + + // Access all fields + result.id(); + result.name(); + result.price(); + result.active(); + result.category(); + // Access all tags + for (int i = 0; i < result.tagsLength(); i++) { + result.tags(i); + } + // Access all metadata + for (int i = 0; i < result.metadataKeysLength(); i++) { + result.metadataKeys(i); + result.metadataValues(i); + } + // Access all extra data + for (int i = 0; i < result.extraDataLength(); i++) { + result.extraData(i); + } + bh.consume(result); } // ===== FIELD ACCESS BENCHMARKS ===== - // Tests accessing a single field near the end of a large record - // This showcases Imprint's O(1) directory lookup vs sequential deserialization + // Tests accessing a single field near the end of a record @Benchmark public void singleFieldAccessImprint(Blackhole bh) throws Exception { diff --git a/src/main/java/com/imprint/core/ImprintRecord.java b/src/main/java/com/imprint/core/ImprintRecord.java index 2291550..da6b6e0 100644 --- a/src/main/java/com/imprint/core/ImprintRecord.java +++ b/src/main/java/com/imprint/core/ImprintRecord.java @@ -17,6 +17,10 @@ * An Imprint record containing a header, field directory, and payload. * Uses ByteBuffer for zero-copy operations to achieve low latency. * + *

This implementation uses lazy directory parsing for optimal single field access performance. + * The directory is only parsed when needed, and binary search is performed directly on raw bytes + * when possible.

+ * *

Performance Note: All ByteBuffers should be array-backed * (hasArray() == true) for optimal zero-copy performance. Direct buffers * may cause performance degradation.

@@ -24,37 +28,64 @@ @Getter public final class ImprintRecord { private final Header header; - private final List directory; + private final ByteBuffer directoryBuffer; // Raw directory bytes private final ByteBuffer payload; // Read-only view for zero-copy + // Lazy-loaded directory state + private List parsedDirectory; + private boolean directoryParsed = false; + + // Cache for parsed directory count to avoid repeated VarInt decoding + private int directoryCount = -1; + /** - * Creates a new ImprintRecord. + * Creates a new ImprintRecord with lazy directory parsing. * + * @param header the record header + * @param directoryBuffer raw directory bytes (including count) * @param payload the payload buffer. Should be array-backed for optimal performance. */ - public ImprintRecord(Header header, List directory, ByteBuffer payload) { + private ImprintRecord(Header header, ByteBuffer directoryBuffer, ByteBuffer payload) { this.header = Objects.requireNonNull(header, "Header cannot be null"); - this.directory = Collections.unmodifiableList(Objects.requireNonNull(directory, "Directory cannot be null")); + this.directoryBuffer = directoryBuffer.asReadOnlyBuffer(); this.payload = payload.asReadOnlyBuffer(); // Zero-copy read-only view } + /** + * Creates a new ImprintRecord with pre-parsed directory (used by ImprintWriter). + * This constructor is used when the directory is already known and parsed. + * + * @param header the record header + * @param directory the parsed directory entries + * @param payload the payload buffer. Should be array-backed for optimal performance. + */ + ImprintRecord(Header header, List directory, ByteBuffer payload) { + this.header = Objects.requireNonNull(header, "Header cannot be null"); + this.parsedDirectory = Collections.unmodifiableList(Objects.requireNonNull(directory, "Directory cannot be null")); + this.directoryParsed = true; + this.directoryCount = directory.size(); + this.payload = payload.asReadOnlyBuffer(); + + // Create directory buffer for serialization compatibility + this.directoryBuffer = createDirectoryBuffer(directory); + } + /** * Get a value by field ID, deserializing it on demand. * Returns null if the field is not found. * Note: If the field exists and is an explicit NULL type, this will return Value.NullValue.INSTANCE + * + *

Performance Note: Accessing fields one-by-one is optimized for single field access. + * If you need to access many fields from the same record, consider calling getDirectory() first + * to parse the full directory once, then access fields normally.

*/ public Value getValue(int fieldId) throws ImprintException { - var fieldBuffer = getFieldBuffer(fieldId); - if (fieldBuffer == null) { + DirectoryEntry entry = findDirectoryEntry(fieldId); + if (entry == null) { return null; } - int directoryIndex = findDirectoryIndex(fieldId); - if (directoryIndex < 0) { - throw new ImprintException(ErrorType.INTERNAL_ERROR, "Field ID " + fieldId + " found buffer but not in directory."); - } - var entry = directory.get(directoryIndex); - return deserializeValue(entry.getTypeCode(), fieldBuffer); + return deserializeValue(entry.getTypeCode(), getFieldBufferFromEntry(entry)); } /** @@ -62,38 +93,237 @@ public Value getValue(int fieldId) throws ImprintException { * Returns a zero-copy ByteBuffer view, or null if field not found. */ public ByteBuffer getRawBytes(int fieldId) { - var fieldBuffer = getFieldBuffer(fieldId); - return fieldBuffer != null ? fieldBuffer.asReadOnlyBuffer() : null; + try { + DirectoryEntry entry = findDirectoryEntry(fieldId); + if (entry == null) { + return null; + } + + return getFieldBufferFromEntry(entry).asReadOnlyBuffer(); + } catch (ImprintException e) { + return null; + } } /** - * Get a ByteBuffer view of a field's data. - * Returns null if the field is not found. + * Find a directory entry for the given field ID. + * Uses the most efficient method based on current state. */ - private ByteBuffer getFieldBuffer(int fieldId) { - int index = findDirectoryIndex(fieldId); - if (index < 0) return null; + private DirectoryEntry findDirectoryEntry(int fieldId) throws ImprintException { + if (directoryParsed) { + // Use parsed directory + int index = findDirectoryIndexInParsed(fieldId); + return index >= 0 ? parsedDirectory.get(index) : null; + } else { + // Use fast binary search on raw bytes + return findFieldEntryFast(fieldId); + } + } - var entry = directory.get(index); - int startOffset = entry.getOffset(); - int endOffset = (index + 1 < directory.size()) ? - directory.get(index + 1).getOffset() : payload.limit(); + /** + * Fast binary search directly on raw directory bytes. + * This avoids parsing the entire directory for single field access. + */ + private DirectoryEntry findFieldEntryFast(int fieldId) throws ImprintException { + ByteBuffer searchBuffer = directoryBuffer.duplicate(); + searchBuffer.order(ByteOrder.LITTLE_ENDIAN); + + // Decode directory count (cache it to avoid repeated decoding) + if (directoryCount < 0) { + directoryCount = VarInt.decode(searchBuffer).getValue(); + } else { + // Skip past the VarInt count + VarInt.decode(searchBuffer); + } - if (startOffset > payload.limit() || endOffset > payload.limit() || startOffset > endOffset) { + if (directoryCount == 0) { return null; } - // OPTIMIZATION: Single allocation instead of duplicate + slice + // Now searchBuffer.position() points to the first directory entry + int directoryStartPos = searchBuffer.position(); + + int low = 0; + int high = directoryCount - 1; + + while (low <= high) { + int mid = (low + high) >>> 1; + + // Calculate position of mid entry + int entryPos = directoryStartPos + (mid * Constants.DIR_ENTRY_BYTES); + + // Bounds check + if (entryPos + Constants.DIR_ENTRY_BYTES > searchBuffer.limit()) { + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, + "Directory entry at position " + entryPos + " exceeds buffer limit " + searchBuffer.limit()); + } + + searchBuffer.position(entryPos); + short midFieldId = searchBuffer.getShort(); + + if (midFieldId < fieldId) { + low = mid + 1; + } else if (midFieldId > fieldId) { + high = mid - 1; + } else { + // Found it - read the complete entry + searchBuffer.position(entryPos); + return deserializeDirectoryEntry(searchBuffer); + } + } + + return null; // Not found + } + + /** + * Get the directory (parsing it if necessary). + * This maintains backward compatibility with existing code. + * + *

Performance Tip: If you plan to access many fields from this record, + * call this method first to parse the directory once, then use the field accessor methods. + * This is more efficient than accessing fields one-by-one when you need multiple fields.

+ */ + public List getDirectory() { + ensureDirectoryParsed(); + return parsedDirectory; + } + + /** + * Get a ByteBuffer view of a field's data from a DirectoryEntry. + */ + private ByteBuffer getFieldBufferFromEntry(DirectoryEntry entry) throws ImprintException { + int startOffset = entry.getOffset(); + + // Find end offset + int endOffset; + if (directoryParsed) { + // Use parsed directory to find next entry + int entryIndex = findDirectoryIndexInParsed(entry.getId()); + endOffset = (entryIndex + 1 < parsedDirectory.size()) ? + parsedDirectory.get(entryIndex + 1).getOffset() : payload.limit(); + } else { + // Calculate end offset by finding the next field in the directory + endOffset = findNextOffsetInRawDirectory(entry.getId()); + } + + if (startOffset < 0 || endOffset < 0 || startOffset > payload.limit() || + endOffset > payload.limit() || startOffset > endOffset) { + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, + "Invalid field buffer range: start=" + startOffset + ", end=" + endOffset + + ", payloadLimit=" + payload.limit()); + } + var fieldBuffer = payload.duplicate(); fieldBuffer.position(startOffset).limit(endOffset); return fieldBuffer; } + /** + * Find the next field's offset by scanning the raw directory. + * This is used when the directory isn't fully parsed yet. + */ + private int findNextOffsetInRawDirectory(int currentFieldId) throws ImprintException { + ByteBuffer scanBuffer = directoryBuffer.duplicate(); + scanBuffer.order(ByteOrder.LITTLE_ENDIAN); + + // Get directory count + int count = (directoryCount >= 0) ? directoryCount : VarInt.decode(scanBuffer).getValue(); + if (count == 0) { + return payload.limit(); + } + + // Skip past count if we just decoded it + if (directoryCount < 0) { + // VarInt.decode already advanced the position + } else { + VarInt.decode(scanBuffer); // Skip past the count + } + + int directoryStartPos = scanBuffer.position(); + + for (int i = 0; i < count; i++) { + int entryPos = directoryStartPos + (i * Constants.DIR_ENTRY_BYTES); + + // Bounds check + if (entryPos + Constants.DIR_ENTRY_BYTES > scanBuffer.limit()) { + return payload.limit(); + } + + scanBuffer.position(entryPos); + short fieldId = scanBuffer.getShort(); + scanBuffer.get(); // skip type + int offset = scanBuffer.getInt(); + + if (fieldId > currentFieldId) { + return offset; // Found next field's offset + } + } + + return payload.limit(); // No next field, use payload end + } + + /** + * Ensure the directory is fully parsed (thread-safe). + */ + private synchronized void ensureDirectoryParsed() { + if (directoryParsed) { + return; + } + + try { + ByteBuffer parseBuffer = directoryBuffer.duplicate(); + parseBuffer.order(ByteOrder.LITTLE_ENDIAN); + + VarInt.DecodeResult countResult = VarInt.decode(parseBuffer); + int count = countResult.getValue(); + this.directoryCount = count; // Cache the count + + List directory = new ArrayList<>(count); + for (int i = 0; i < count; i++) { + directory.add(deserializeDirectoryEntry(parseBuffer)); + } + + this.parsedDirectory = Collections.unmodifiableList(directory); + this.directoryParsed = true; + } catch (ImprintException e) { + throw new RuntimeException("Failed to parse directory", e); + } + } + + /** + * Creates a directory buffer from parsed directory entries. + * This is used when creating records with pre-parsed directories (e.g., from ImprintWriter). + */ + private ByteBuffer createDirectoryBuffer(List directory) { + try { + int bufferSize = VarInt.encodedLength(directory.size()) + (directory.size() * Constants.DIR_ENTRY_BYTES); + ByteBuffer buffer = ByteBuffer.allocate(bufferSize); + buffer.order(ByteOrder.LITTLE_ENDIAN); + + // Write directory count + VarInt.encode(directory.size(), buffer); + + // Write directory entries + for (DirectoryEntry entry : directory) { + serializeDirectoryEntry(entry, buffer); + } + + buffer.flip(); + return buffer.asReadOnlyBuffer(); + } catch (Exception e) { + // Fallback to empty buffer if creation fails + return ByteBuffer.allocate(0).asReadOnlyBuffer(); + } + } + /** * Serialize this record to a ByteBuffer. * The returned buffer will be array-backed. */ public ByteBuffer serializeToBuffer() { + // Ensure directory is parsed for serialization + ensureDirectoryParsed(); + var buffer = ByteBuffer.allocate(estimateSerializedSize()); buffer.order(ByteOrder.LITTLE_ENDIAN); @@ -101,8 +331,8 @@ public ByteBuffer serializeToBuffer() { serializeHeader(buffer); // Write directory (always present) - VarInt.encode(directory.size(), buffer); - for (var entry : directory) { + VarInt.encode(parsedDirectory.size(), buffer); + for (var entry : parsedDirectory) { serializeDirectoryEntry(entry, buffer); } @@ -117,9 +347,6 @@ public ByteBuffer serializeToBuffer() { /** * Create a fluent builder for constructing ImprintRecord instances. - * - * @param schemaId the schema identifier for this record - * @return a new builder instance */ public static ImprintRecordBuilder builder(SchemaId schemaId) { return new ImprintRecordBuilder(schemaId); @@ -127,10 +354,6 @@ public static ImprintRecordBuilder builder(SchemaId schemaId) { /** * Create a fluent builder for constructing ImprintRecord instances. - * - * @param fieldspaceId the fieldspace identifier - * @param schemaHash the schema hash - * @return a new builder instance */ @SuppressWarnings("unused") public static ImprintRecordBuilder builder(int fieldspaceId, int schemaHash) { @@ -145,7 +368,7 @@ public static ImprintRecord deserialize(byte[] bytes) throws ImprintException { } /** - * Deserialize a record from a ByteBuffer. + * Deserialize a record from a ByteBuffer with lazy directory parsing. * * @param buffer the buffer to deserialize from. Must be array-backed * (buffer.hasArray() == true) for optimal zero-copy performance. @@ -156,37 +379,43 @@ public static ImprintRecord deserialize(ByteBuffer buffer) throws ImprintExcepti // Read header var header = deserializeHeader(buffer); - // Read directory (always present) - var directory = new ArrayList(); + // Read directory count but don't parse entries yet + int directoryStartPos = buffer.position(); VarInt.DecodeResult countResult = VarInt.decode(buffer); int directoryCount = countResult.getValue(); - for (int i = 0; i < directoryCount; i++) { - directory.add(deserializeDirectoryEntry(buffer)); - } + // Calculate directory buffer (includes count + all entries) + int directorySize = countResult.getBytesRead() + (directoryCount * Constants.DIR_ENTRY_BYTES); + buffer.position(directoryStartPos); // Reset to include count in directory buffer + + var directoryBuffer = buffer.slice(); + directoryBuffer.limit(directorySize); + + // Advance buffer past directory + buffer.position(buffer.position() + directorySize); // Read payload as ByteBuffer slice for zero-copy var payload = buffer.slice(); payload.limit(header.getPayloadSize()); - buffer.position(buffer.position() + header.getPayloadSize()); - return new ImprintRecord(header, directory, payload); + return new ImprintRecord(header, directoryBuffer, payload); } /** - * Binary search for field ID in directory without object allocation. + * Binary search for field ID in parsed directory. * Returns the index of the field if found, or a negative value if not found. - * - * @param fieldId the field ID to search for - * @return index if found, or negative insertion point - 1 if not found */ - private int findDirectoryIndex(int fieldId) { + private int findDirectoryIndexInParsed(int fieldId) { + if (!directoryParsed) { + return -1; + } + int low = 0; - int high = directory.size() - 1; + int high = parsedDirectory.size() - 1; while (low <= high) { - int mid = (low + high) >>> 1; // unsigned right shift to avoid overflow - int midFieldId = directory.get(mid).getId(); + int mid = (low + high) >>> 1; + int midFieldId = parsedDirectory.get(mid).getId(); if (midFieldId < fieldId) { low = mid + 1; @@ -201,12 +430,30 @@ private int findDirectoryIndex(int fieldId) { public int estimateSerializedSize() { int size = Constants.HEADER_BYTES; // header - size += VarInt.encodedLength(directory.size()); // directory count - size += directory.size() * Constants.DIR_ENTRY_BYTES; // directory entries + size += VarInt.encodedLength(getDirectoryCount()); // directory count + size += getDirectoryCount() * Constants.DIR_ENTRY_BYTES; // directory entries size += payload.remaining(); // payload return size; } + private int getDirectoryCount() { + if (directoryCount >= 0) { + return directoryCount; + } + if (directoryParsed) { + return parsedDirectory.size(); + } + // Last resort: decode from buffer + try { + ByteBuffer countBuffer = directoryBuffer.duplicate(); + return VarInt.decode(countBuffer).getValue(); + } catch (Exception e) { + return 0; + } + } + + // ===== EXISTING HELPER METHODS (unchanged) ===== + private void serializeHeader(ByteBuffer buffer) { buffer.put(Constants.MAGIC); buffer.put(Constants.VERSION); @@ -287,6 +534,8 @@ private Value deserializeValue(TypeCode typeCode, ByteBuffer buffer) throws Impr } } + // ===== TYPE-SPECIFIC GETTERS (unchanged API, improved performance) ===== + private T getTypedValueOrThrow(int fieldId, TypeCode expectedTypeCode, Class expectedValueClass, String expectedTypeName) throws ImprintException { var value = getValue(fieldId); @@ -308,50 +557,26 @@ private T getTypedValueOrThrow(int fieldId, TypeCode expectedT "Field " + fieldId + " is of type " + value.getTypeCode() + ", expected " + expectedTypeName + "."); } - /** - * Retrieves the boolean value for the given field ID. - * @throws ImprintException if the field is not found, is null, or is not of type BOOL. - */ public boolean getBoolean(int fieldId) throws ImprintException { return getTypedValueOrThrow(fieldId, TypeCode.BOOL, Value.BoolValue.class, "boolean").getValue(); } - /** - * Retrieves the int (int32) value for the given field ID. - * @throws ImprintException if the field is not found, is null, or is not of type INT32. - */ public int getInt32(int fieldId) throws ImprintException { return getTypedValueOrThrow(fieldId, TypeCode.INT32, Value.Int32Value.class, "int32").getValue(); } - /** - * Retrieves the long (int64) value for the given field ID. - * @throws ImprintException if the field is not found, is null, or is not of type INT64. - */ public long getInt64(int fieldId) throws ImprintException { return getTypedValueOrThrow(fieldId, TypeCode.INT64, Value.Int64Value.class, "int64").getValue(); } - /** - * Retrieves the float (float32) value for the given field ID. - * @throws ImprintException if the field is not found, is null, or is not of type FLOAT32. - */ public float getFloat32(int fieldId) throws ImprintException { return getTypedValueOrThrow(fieldId, TypeCode.FLOAT32, Value.Float32Value.class, "float32").getValue(); } - /** - * Retrieves the double (float64) value for the given field ID. - * @throws ImprintException if the field is not found, is null, or is not of type FLOAT64. - */ public double getFloat64(int fieldId) throws ImprintException { return getTypedValueOrThrow(fieldId, TypeCode.FLOAT64, Value.Float64Value.class, "float64").getValue(); } - /** - * Retrieves the String value for the given field ID. - * @throws ImprintException if the field is not found, is null, or is not of type STRING. - */ public String getString(int fieldId) throws ImprintException { var value = getValue(fieldId); @@ -375,11 +600,6 @@ public String getString(int fieldId) throws ImprintException { "Field " + fieldId + " is of type " + value.getTypeCode() + ", expected STRING."); } - /** - * Retrieves the byte array (byte[]) value for the given field ID. - * Note: This may involve a defensive copy depending on the underlying Value type. - * @throws ImprintException if the field is not found, is null, or is not of type BYTES. - */ public byte[] getBytes(int fieldId) throws ImprintException { Value value = getValue(fieldId); @@ -393,46 +613,31 @@ public byte[] getBytes(int fieldId) throws ImprintException { } if (value instanceof Value.BytesValue) { - return ((Value.BytesValue) value).getValue(); // getValue() in BytesValue returns a clone + return ((Value.BytesValue) value).getValue(); } if (value instanceof Value.BytesBufferValue) { - return ((Value.BytesBufferValue) value).getValue(); // getValue() in BytesBufferValue creates a new array + return ((Value.BytesBufferValue) value).getValue(); } throw new ImprintException(ErrorType.TYPE_MISMATCH, "Field " + fieldId + " is of type " + value.getTypeCode() + ", expected BYTES."); } - /** - * Retrieves the List for the given field ID. - * The list itself is a copy; modifications to it will not affect the record. - * @throws ImprintException if the field is not found, is null, or is not of type ARRAY. - */ public List getArray(int fieldId) throws ImprintException { return getTypedValueOrThrow(fieldId, TypeCode.ARRAY, Value.ArrayValue.class, "ARRAY").getValue(); } - /** - * Retrieves the Map for the given field ID. - * The map itself is a copy; modifications to it will not affect the record. - * @throws ImprintException if the field is not found, is null, or is not of type MAP. - */ public Map getMap(int fieldId) throws ImprintException { return getTypedValueOrThrow(fieldId, TypeCode.MAP, Value.MapValue.class, "MAP").getValue(); } - /** - * Retrieves the nested ImprintRecord for the given field ID. - * @throws ImprintException if the field is not found, is null, or is not of type ROW. - */ public ImprintRecord getRow(int fieldId) throws ImprintException { return getTypedValueOrThrow(fieldId, TypeCode.ROW, Value.RowValue.class, "ROW").getValue(); } @Override public String toString() { - return String.format("ImprintRecord{header=%s, directorySize=%d, payloadSize=%d}", - header, directory.size(), payload.remaining()); + return String.format("ImprintRecord{header=%s, directorySize=%d, payloadSize=%d, directoryParsed=%s}", + header, getDirectoryCount(), payload.remaining(), directoryParsed); } - } \ No newline at end of file diff --git a/src/test/java/com/imprint/IntegrationTest.java b/src/test/java/com/imprint/IntegrationTest.java index 76efcc5..898adfb 100644 --- a/src/test/java/com/imprint/IntegrationTest.java +++ b/src/test/java/com/imprint/IntegrationTest.java @@ -16,16 +16,11 @@ */ public class IntegrationTest { - // Removed main method, individual methods are now JUnit tests. - @Test @DisplayName("Basic functionality: create, serialize, deserialize primitive types") void testBasicFunctionality() throws ImprintException { - System.out.println("Testing basic functionality..."); // Keep for now if desired, or remove - SchemaId schemaId = new SchemaId(1, 0xdeadbeef); - // Using ImprintRecordBuilder for consistency with other tests - ImprintRecord record = ImprintRecord.builder(schemaId) + var record = ImprintRecord.builder(schemaId) .field(1, 42) .field(2, "testing java imprint spec") .field(3, true) @@ -33,7 +28,7 @@ void testBasicFunctionality() throws ImprintException { .field(5, new byte[]{1, 2, 3, 4}) .build(); - // Verify we can read values back using ergonomic getters + // Verify we can read values back using type getters assertEquals(42, record.getInt32(1)); assertEquals("testing java imprint spec", record.getString(2)); assertTrue(record.getBoolean(3)); @@ -47,7 +42,7 @@ void testBasicFunctionality() throws ImprintException { var buffer = record.serializeToBuffer(); byte[] serialized = new byte[buffer.remaining()]; buffer.get(serialized); - ImprintRecord deserialized = ImprintRecord.deserialize(serialized); + var deserialized = ImprintRecord.deserialize(serialized); assertEquals(42, deserialized.getInt32(1)); assertEquals("testing java imprint spec", deserialized.getString(2)); @@ -61,8 +56,6 @@ void testBasicFunctionality() throws ImprintException { @Test @DisplayName("Collections: create, serialize, deserialize arrays and maps") void testArraysAndMaps() throws ImprintException { - System.out.println("Testing arrays and maps..."); - SchemaId schemaId = new SchemaId(2, 0xcafebabe); // Create an array using builder for convenience @@ -72,8 +65,7 @@ void testArraysAndMaps() throws ImprintException { Map sourceStringToIntMap = new HashMap<>(); sourceStringToIntMap.put("one", 1); sourceStringToIntMap.put("two", 2); - - ImprintRecord record = ImprintRecord.builder(schemaId) + var record = ImprintRecord.builder(schemaId) .field(1, sourceIntList) // Builder converts List to List .field(2, sourceStringToIntMap) // Builder converts Map .build(); @@ -107,14 +99,14 @@ void testArraysAndMaps() throws ImprintException { void testNestedRecords() throws ImprintException { System.out.println("Testing nested records..."); - SchemaId innerSchemaId = new SchemaId(3, 0x12345678); - ImprintRecord innerRecord = ImprintRecord.builder(innerSchemaId) + var innerSchemaId = new SchemaId(3, 0x12345678); + var innerRecord = ImprintRecord.builder(innerSchemaId) .field(1, "nested data") .field(2, 9876543210L) .build(); - SchemaId outerSchemaId = new SchemaId(4, 0x87654321); - ImprintRecord outerRecord = ImprintRecord.builder(outerSchemaId) + var outerSchemaId = new SchemaId(4, 0x87654321); + var outerRecord = ImprintRecord.builder(outerSchemaId) .field(1, innerRecord) // Builder handles ImprintRecord directly .field(2, "outer data") .build(); @@ -122,12 +114,12 @@ void testNestedRecords() throws ImprintException { var buffer = outerRecord.serializeToBuffer(); byte[] serialized = new byte[buffer.remaining()]; buffer.get(serialized); - ImprintRecord deserialized = ImprintRecord.deserialize(serialized); + var deserialized = ImprintRecord.deserialize(serialized); assertEquals(4, deserialized.getHeader().getSchemaId().getFieldSpaceId()); assertEquals("outer data", deserialized.getString(2)); - ImprintRecord nestedDeserialized = deserialized.getRow(1); + var nestedDeserialized = deserialized.getRow(1); assertNotNull(nestedDeserialized); assertEquals(3, nestedDeserialized.getHeader().getSchemaId().getFieldSpaceId()); assertEquals("nested data", nestedDeserialized.getString(1)); @@ -136,8 +128,6 @@ void testNestedRecords() throws ImprintException { System.out.println("✓ Nested records test passed"); } - // --- Start of broken down tests for ErgonomicGettersAndNestedTypes --- - private ImprintRecord createTestRecordForGetters() throws ImprintException { SchemaId schemaId = new SchemaId(5, 0xabcdef01); @@ -174,8 +164,8 @@ private ImprintRecord serializeAndDeserialize(ImprintRecord record) throws Impri @Test @DisplayName("Type Getters: Basic primitive and String types") void testBasicTypeGetters() throws ImprintException { - ImprintRecord originalRecord = createTestRecordForGetters(); - ImprintRecord record = serializeAndDeserialize(originalRecord); + var originalRecord = createTestRecordForGetters(); + var record = serializeAndDeserialize(originalRecord); assertTrue(record.getBoolean(1)); assertEquals(12345, record.getInt32(2)); @@ -189,8 +179,8 @@ void testBasicTypeGetters() throws ImprintException { @Test @DisplayName("Type Getters: Array of Arrays") void testTypeGetterArrayOfArrays() throws ImprintException { - ImprintRecord originalRecord = createTestRecordForGetters(); - ImprintRecord record = serializeAndDeserialize(originalRecord); + var originalRecord = createTestRecordForGetters(); + var record = serializeAndDeserialize(originalRecord); List arrOfArr = record.getArray(9); assertNotNull(arrOfArr); @@ -211,8 +201,8 @@ void testTypeGetterArrayOfArrays() throws ImprintException { @Test @DisplayName("Type Getters: Map with Array Value") void testTypeGetterMapWithArrayValue() throws ImprintException { - ImprintRecord originalRecord = createTestRecordForGetters(); - ImprintRecord record = serializeAndDeserialize(originalRecord); + var originalRecord = createTestRecordForGetters(); + var record = serializeAndDeserialize(originalRecord); Map mapWithArr = record.getMap(10); assertNotNull(mapWithArr); @@ -227,8 +217,8 @@ void testTypeGetterMapWithArrayValue() throws ImprintException { @Test @DisplayName("Type Getters: Empty Collections (Array and Map)") void testErgonomicGettersEmptyCollections() throws ImprintException { - ImprintRecord originalRecord = createTestRecordForGetters(); - ImprintRecord record = serializeAndDeserialize(originalRecord); + var originalRecord = createTestRecordForGetters(); + var record = serializeAndDeserialize(originalRecord); List emptyArr = record.getArray(11); assertNotNull(emptyArr); @@ -242,8 +232,8 @@ void testErgonomicGettersEmptyCollections() throws ImprintException { @Test @DisplayName("Type Getters: Exception for Field Not Found") void testErgonomicGetterExceptionFieldNotFound() throws ImprintException { - ImprintRecord originalRecord = createTestRecordForGetters(); - ImprintRecord record = serializeAndDeserialize(originalRecord); + var originalRecord = createTestRecordForGetters(); + var record = serializeAndDeserialize(originalRecord); ImprintException ex = assertThrows(ImprintException.class, () -> record.getInt32(99)); assertEquals(ErrorType.FIELD_NOT_FOUND, ex.getErrorType()); @@ -252,8 +242,8 @@ void testErgonomicGetterExceptionFieldNotFound() throws ImprintException { @Test @DisplayName("Type Getters: Exception for Null Field accessed as primitive") void testErgonomicGetterExceptionNullField() throws ImprintException { - ImprintRecord originalRecord = createTestRecordForGetters(); - ImprintRecord record = serializeAndDeserialize(originalRecord); + var originalRecord = createTestRecordForGetters(); + var record = serializeAndDeserialize(originalRecord); ImprintException ex = assertThrows(ImprintException.class, () -> record.getString(8)); assertEquals(ErrorType.TYPE_MISMATCH, ex.getErrorType()); // getString throws TYPE_MISMATCH for null @@ -269,8 +259,8 @@ void testErgonomicGetterExceptionNullField() throws ImprintException { @Test @DisplayName("Type Getters: Exception for Type Mismatch") void testErgonomicGetterExceptionTypeMismatch() throws ImprintException { - ImprintRecord originalRecord = createTestRecordForGetters(); - ImprintRecord record = serializeAndDeserialize(originalRecord); + var originalRecord = createTestRecordForGetters(); + var record = serializeAndDeserialize(originalRecord); ImprintException ex = assertThrows(ImprintException.class, () -> record.getInt32(6)); // Field 6 is a String assertEquals(ErrorType.TYPE_MISMATCH, ex.getErrorType()); @@ -279,20 +269,20 @@ void testErgonomicGetterExceptionTypeMismatch() throws ImprintException { @Test @DisplayName("Type Getters: Row (Nested Record)") void testErgonomicGetterRow() throws ImprintException { - SchemaId innerSchemaId = new SchemaId(6, 0x12345678); - ImprintRecord innerRecord = ImprintRecord.builder(innerSchemaId) + var innerSchemaId = new SchemaId(6, 0x12345678); + var innerRecord = ImprintRecord.builder(innerSchemaId) .field(101, "nested string") .field(102, 999L) .build(); - ImprintRecord recordWithRow = ImprintRecord.builder(new SchemaId(7, 0x87654321)) + var recordWithRow = ImprintRecord.builder(new SchemaId(7, 0x87654321)) .field(201, innerRecord) // Using builder to add row .field(202, "outer field") .build(); - ImprintRecord deserializedWithRow = serializeAndDeserialize(recordWithRow); + var deserializedWithRow = serializeAndDeserialize(recordWithRow); - ImprintRecord retrievedRow = deserializedWithRow.getRow(201); + var retrievedRow = deserializedWithRow.getRow(201); assertNotNull(retrievedRow); assertEquals(innerSchemaId, retrievedRow.getHeader().getSchemaId()); assertEquals("nested string", retrievedRow.getString(101)); diff --git a/src/test/java/com/imprint/profile/ProfilerTest.java b/src/test/java/com/imprint/profile/ProfilerTest.java index d48c1aa..3b9f371 100644 --- a/src/test/java/com/imprint/profile/ProfilerTest.java +++ b/src/test/java/com/imprint/profile/ProfilerTest.java @@ -4,6 +4,7 @@ import com.imprint.core.ImprintWriter; import com.imprint.core.SchemaId; import com.imprint.types.Value; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import java.util.Random; @@ -26,7 +27,7 @@ * - String operations and UTF-8 encoding * - ByteBuffer operations */ -//@Disabled("Enable manually for profiling") +@Disabled("Enable manually for profiling") public class ProfilerTest { private static final int ITERATIONS = 1_000_000; From 83ed96192fa421f7cf51815a5c43d730de7727e4 Mon Sep 17 00:00:00 2001 From: expand3d <> Date: Thu, 5 Jun 2025 15:13:59 -0400 Subject: [PATCH 25/34] minor cleanup --- .../benchmark/ComparisonBenchmark.java | 21 ++++++++----------- .../java/com/imprint/types/TypeHandler.java | 10 ++++----- src/main/java/com/imprint/types/Value.java | 16 +++++++------- 3 files changed, 21 insertions(+), 26 deletions(-) diff --git a/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java b/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java index 49260b1..a7ffd3c 100644 --- a/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java +++ b/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java @@ -10,6 +10,7 @@ import com.imprint.core.SchemaId; import com.imprint.types.MapKey; import com.imprint.types.Value; +import lombok.NoArgsConstructor; import org.apache.avro.Schema; import org.apache.avro.generic.GenericData; import org.apache.avro.generic.GenericDatumReader; @@ -90,7 +91,6 @@ public void setup() throws Exception { } // ===== SERIALIZATION BENCHMARKS ===== - @Benchmark public void serializeImprint(Blackhole bh) throws Exception { ByteBuffer result = serializeWithImprint(testData); @@ -134,7 +134,6 @@ public void serializeFlatBuffers(Blackhole bh) { } // ===== DESERIALIZATION BENCHMARKS ===== - @Benchmark public void deserializeImprint(Blackhole bh) throws Exception { ImprintRecord result = ImprintRecord.deserialize(imprintBytesBuffer.duplicate()); @@ -181,8 +180,6 @@ public void deserializeFlatBuffers(Blackhole bh) { // ===== FIELD ACCESS BENCHMARKS ===== // Tests accessing a single field near the end of a large record - // This showcases Imprint's O(1) directory lookup vs sequential deserialization - @Benchmark public void singleFieldAccessImprint(Blackhole bh) throws Exception { ImprintRecord record = ImprintRecord.deserialize(imprintBytesBuffer.duplicate()); @@ -213,19 +210,19 @@ public void singleFieldAccessMessagePack(Blackhole bh) throws Exception { @Benchmark public void singleFieldAccessAvro(Blackhole bh) throws Exception { GenericRecord record = deserializeWithAvro(avroBytes); - bh.consume(record.get("extraData4")); // Accessing field near end + bh.consume(record.get("extraData4")); } @Benchmark public void singleFieldAccessProtobuf(Blackhole bh) throws Exception { TestRecordProto.TestRecord record = TestRecordProto.TestRecord.parseFrom(protobufBytes); - bh.consume(record.getExtraData(4)); // Accessing field near end + bh.consume(record.getExtraData(4)); } @Benchmark public void singleFieldAccessFlatBuffers(Blackhole bh) { TestRecordFB record = TestRecordFB.getRootAsTestRecordFB(flatbuffersBytes.duplicate()); - bh.consume(record.extraData(4)); // Accessing field near end - zero copy! + bh.consume(record.extraData(4)); } // ===== SIZE COMPARISON ===== @@ -694,8 +691,8 @@ private ByteBuffer mergeFlatBuffersRecords(TestRecordFB first, TestRecordFB seco FlatBufferBuilder builder = new FlatBufferBuilder(1024); // Use second record's values if they exist, otherwise first record's values - String name = second.name() != null && !second.name().isEmpty() ? second.name() : first.name(); - String category = second.category() != null && !second.category().isEmpty() ? second.category() : first.category(); + String name = second.name() != null && !Objects.requireNonNull(second.name()).isEmpty() ? second.name() : first.name(); + String category = second.category() != null && !Objects.requireNonNull(second.category()).isEmpty() ? second.category() : first.category(); double price = second.price() != 0.0 ? second.price() : first.price(); boolean active = second.active(); // Use second's boolean value int id = first.id(); // Keep first record's ID @@ -809,6 +806,7 @@ var record = new TestRecord(); } // Test data class for other serialization libraries + @NoArgsConstructor public static class TestRecord { public int id; public String name; @@ -817,8 +815,7 @@ public static class TestRecord { public String category; public List tags = new ArrayList<>(); public Map metadata = new HashMap<>(); - public List extraData = new ArrayList<>(); // Fields 8-20 for large record test - - public TestRecord() {} // Required for deserialization + // Fields 8-20 for large record test + public List extraData = new ArrayList<>(); } } \ No newline at end of file diff --git a/src/main/java/com/imprint/types/TypeHandler.java b/src/main/java/com/imprint/types/TypeHandler.java index e58c355..dce6973 100644 --- a/src/main/java/com/imprint/types/TypeHandler.java +++ b/src/main/java/com/imprint/types/TypeHandler.java @@ -9,7 +9,7 @@ /** * Interface for handling type-specific serialization, deserialization, and size estimation. - * Note that primitives are potentially auto/un-boxed here which could impact performance slightly + * Note that primitives are basically boxed here which could impact performance slightly * but having all the types in their own implementation helps keep things organized for now, especially * for dealing with and testing more complex types in the future. */ @@ -26,7 +26,7 @@ interface BufferViewer { int measureDataLength(ByteBuffer tempBuffer, int numElements) throws ImprintException; } - // Helper method to eliminate duplication in ARRAY/MAP readValueBytes + // Helper method for complex buffer positioning in MAP and ARRAY static ByteBuffer readComplexValueBytes(ByteBuffer buffer, String typeName, BufferViewer measurer) throws ImprintException { int initialPosition = buffer.position(); ByteBuffer tempBuffer = buffer.duplicate(); @@ -571,15 +571,13 @@ public ByteBuffer readValueBytes(ByteBuffer buffer) throws ImprintException { } byte keyTypeCodeByte = tempBuffer.get(); byte valueTypeCodeByte = tempBuffer.get(); - TypeCode keyType = TypeCode.fromByte(keyTypeCodeByte); - TypeCode valueType = TypeCode.fromByte(valueTypeCodeByte); + var keyType = TypeCode.fromByte(keyTypeCodeByte); + var valueType = TypeCode.fromByte(valueTypeCodeByte); - // OPTIMIZATION: Calculate sizes directly for fixed-size types int keySize = getFixedTypeSize(keyType); int valueSize = getFixedTypeSize(valueType); if (keySize > 0 && valueSize > 0) { - // Both are fixed-size: O(1) calculation return 2 + (numEntries * (keySize + valueSize)); } else { // At least one is variable-size: fall back to traversal diff --git a/src/main/java/com/imprint/types/Value.java b/src/main/java/com/imprint/types/Value.java index ba747de..fbb988c 100644 --- a/src/main/java/com/imprint/types/Value.java +++ b/src/main/java/com/imprint/types/Value.java @@ -192,11 +192,11 @@ public static class BytesValue extends Value { private final byte[] value; public BytesValue(byte[] value) { - this.value = value.clone(); // defensive copy + this.value = value.clone(); } public byte[] getValue() { - return value.clone(); // defensive copy + return value.clone(); } @Override @@ -233,7 +233,7 @@ public static class BytesBufferValue extends Value { private final ByteBuffer value; public BytesBufferValue(ByteBuffer value) { - this.value = value.asReadOnlyBuffer(); // zero-copy read-only view + this.value = value.asReadOnlyBuffer(); } public byte[] getValue() { @@ -244,7 +244,7 @@ public byte[] getValue() { } public ByteBuffer getBuffer() { - return value.duplicate(); // zero-copy view + return value.duplicate(); } @Override @@ -289,11 +289,11 @@ public StringValue(String value) { public byte[] getUtf8Bytes() { var cached = cachedUtf8Bytes; if (cached == null) { - // Multiple threads may compute this - that's OK since it's idempotent + // UTF8 is idempotent so no need to synchronize cached = value.getBytes(StandardCharsets.UTF_8); cachedUtf8Bytes = cached; } - return cached; // Return our computed value, not re-read from volatile field + return cached; // Return computed value } @Override @@ -363,14 +363,14 @@ private String decodeUtf8() { // Fallback: copy bytes from the ByteBuffer to a new heap array (if too large for cache) array = new byte[length]; } - value.duplicate().get(array, 0, length); // Get bytes from current position into chosen array + value.duplicate().get(array, 0, length); offset = 0; } return new String(array, offset, length, StandardCharsets.UTF_8); } public ByteBuffer getBuffer() { - return value.duplicate(); // zero-copy view + return value.duplicate(); } @Override From a605b652eec81478b7f1d7f4cf25529ea00cebfb Mon Sep 17 00:00:00 2001 From: expand3d <> Date: Thu, 5 Jun 2025 15:22:06 -0400 Subject: [PATCH 26/34] minor cleanup --- build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index 66f2e3e..d9093f9 100644 --- a/build.gradle +++ b/build.gradle @@ -170,7 +170,7 @@ test { jmh { fork = 1 warmupIterations = 2 // Reduced for faster CI - iterations = 3 + iterations = 3 // Reduced for faster CI resultFormat = 'JSON' includeTests = false resultsFile = file("${projectDir}/benchmark-results/jmh-results-${new Date().format('yyyy-MM-dd-HHmmss')}.json") @@ -180,7 +180,7 @@ jmh { '-XX:+UseG1GC', '-Xmx2g', '-XX:+UnlockExperimentalVMOptions', - '-XX:+UseJVMCICompiler' // Use Graal if available for better performance + '-XX:+UseJVMCICompiler' ] } From aacddeb0ec5791176ec57a7927366364f87acda3 Mon Sep 17 00:00:00 2001 From: expand3d <> Date: Thu, 5 Jun 2025 15:23:46 -0400 Subject: [PATCH 27/34] minor cleanup --- src/main/java/com/imprint/core/ImprintRecord.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/imprint/core/ImprintRecord.java b/src/main/java/com/imprint/core/ImprintRecord.java index 2291550..e7dab70 100644 --- a/src/main/java/com/imprint/core/ImprintRecord.java +++ b/src/main/java/com/imprint/core/ImprintRecord.java @@ -83,7 +83,7 @@ private ByteBuffer getFieldBuffer(int fieldId) { return null; } - // OPTIMIZATION: Single allocation instead of duplicate + slice + //Single allocation instead of duplicate + slice var fieldBuffer = payload.duplicate(); fieldBuffer.position(startOffset).limit(endOffset); return fieldBuffer; From 3bf81ade2bea81333d3acef7f2e80979490f4c7e Mon Sep 17 00:00:00 2001 From: expand3d <> Date: Thu, 5 Jun 2025 15:41:43 -0400 Subject: [PATCH 28/34] Actually fixes offsets and read Byte Values for Maps and Arrays even with nested objects --- .../java/com/imprint/types/TypeHandler.java | 252 ++---------------- .../com/imprint/profile/ProfilerTest.java | 3 +- .../com/imprint/types/TypeHandlerTest.java | 5 - 3 files changed, 19 insertions(+), 241 deletions(-) diff --git a/src/main/java/com/imprint/types/TypeHandler.java b/src/main/java/com/imprint/types/TypeHandler.java index dce6973..634867b 100644 --- a/src/main/java/com/imprint/types/TypeHandler.java +++ b/src/main/java/com/imprint/types/TypeHandler.java @@ -17,52 +17,7 @@ public interface TypeHandler { Value deserialize(ByteBuffer buffer) throws ImprintException; void serialize(Value value, ByteBuffer buffer) throws ImprintException; int estimateSize(Value value) throws ImprintException; - ByteBuffer readValueBytes(ByteBuffer buffer) throws ImprintException; - - - - @FunctionalInterface - interface BufferViewer { - int measureDataLength(ByteBuffer tempBuffer, int numElements) throws ImprintException; - } - - // Helper method for complex buffer positioning in MAP and ARRAY - static ByteBuffer readComplexValueBytes(ByteBuffer buffer, String typeName, BufferViewer measurer) throws ImprintException { - int initialPosition = buffer.position(); - ByteBuffer tempBuffer = buffer.duplicate(); - tempBuffer.order(buffer.order()); - - VarInt.DecodeResult lengthResult = VarInt.decode(tempBuffer); - int numElements = lengthResult.getValue(); - int varIntLength = tempBuffer.position() - initialPosition; - - if (numElements == 0) { - if (buffer.remaining() < varIntLength) { - throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, - "Not enough bytes for empty " + typeName + " VarInt. Needed: " + - varIntLength + ", available: " + buffer.remaining()); - } - ByteBuffer valueSlice = buffer.slice(); - valueSlice.limit(varIntLength); - buffer.position(initialPosition + varIntLength); - return valueSlice.asReadOnlyBuffer(); - } - - int dataLength = measurer.measureDataLength(tempBuffer, numElements); - int totalLength = varIntLength + dataLength; - - if (buffer.remaining() < totalLength) { - throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, - "Not enough bytes for " + typeName + " value. Needed: " + totalLength + - ", available: " + buffer.remaining() + " at position " + initialPosition); - } - ByteBuffer valueSlice = buffer.slice(); - valueSlice.limit(totalLength); - buffer.position(initialPosition + totalLength); - return valueSlice.asReadOnlyBuffer(); - } - // Static implementations for each type TypeHandler NULL = new TypeHandler() { @Override @@ -79,11 +34,6 @@ public void serialize(Value value, ByteBuffer buffer) { public int estimateSize(Value value) { return 0; } - - @Override - public ByteBuffer readValueBytes(ByteBuffer buffer) { - return ByteBuffer.allocate(0).asReadOnlyBuffer(); - } }; TypeHandler BOOL = new TypeHandler() { @@ -108,14 +58,6 @@ public void serialize(Value value, ByteBuffer buffer) { public int estimateSize(Value value) { return 1; } - - @Override - public ByteBuffer readValueBytes(ByteBuffer buffer) { - var boolBuffer = buffer.slice(); - boolBuffer.limit(1); - buffer.position(buffer.position() + 1); - return boolBuffer.asReadOnlyBuffer(); - } }; TypeHandler INT32 = new TypeHandler() { @@ -137,14 +79,6 @@ public void serialize(Value value, ByteBuffer buffer) { public int estimateSize(Value value) { return 4; } - - @Override - public ByteBuffer readValueBytes(ByteBuffer buffer) { - var int32Buffer = buffer.slice(); - int32Buffer.limit(4); - buffer.position(buffer.position() + 4); - return int32Buffer.asReadOnlyBuffer(); - } }; TypeHandler INT64 = new TypeHandler() { @@ -166,14 +100,6 @@ public void serialize(Value value, ByteBuffer buffer) { public int estimateSize(Value value) { return 8; } - - @Override - public ByteBuffer readValueBytes(ByteBuffer buffer) { - var int64Buffer = buffer.slice(); - int64Buffer.limit(8); - buffer.position(buffer.position() + 8); - return int64Buffer.asReadOnlyBuffer(); - } }; TypeHandler FLOAT32 = new TypeHandler() { @@ -195,14 +121,6 @@ public void serialize(Value value, ByteBuffer buffer) { public int estimateSize(Value value) { return 4; } - - @Override - public ByteBuffer readValueBytes(ByteBuffer buffer) { - var float32Buffer = buffer.slice(); - float32Buffer.limit(4); - buffer.position(buffer.position() + 4); - return float32Buffer.asReadOnlyBuffer(); - } }; TypeHandler FLOAT64 = new TypeHandler() { @@ -224,14 +142,6 @@ public void serialize(Value value, ByteBuffer buffer) { public int estimateSize(Value value) { return 8; } - - @Override - public ByteBuffer readValueBytes(ByteBuffer buffer) { - var float64Buffer = buffer.slice(); - float64Buffer.limit(8); - buffer.position(buffer.position() + 8); - return float64Buffer.asReadOnlyBuffer(); - } }; TypeHandler BYTES = new TypeHandler() { @@ -274,29 +184,6 @@ public int estimateSize(Value value) { return VarInt.encodedLength(bytes.length) + bytes.length; } } - - @Override - public ByteBuffer readValueBytes(ByteBuffer buffer) throws ImprintException { - int initialPos = buffer.position(); - ByteBuffer tempMeasureBuffer = buffer.duplicate(); - VarInt.DecodeResult dr = VarInt.decode(tempMeasureBuffer); - - int varIntByteLength = tempMeasureBuffer.position() - initialPos; - int payloadByteLength = dr.getValue(); - int totalValueLength = varIntByteLength + payloadByteLength; - - if (buffer.remaining() < totalValueLength) { - throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, - "Not enough bytes for VarInt-prefixed data. Needed: " + totalValueLength + - ", available: " + buffer.remaining() + " at position " + initialPos); - } - - ByteBuffer resultSlice = buffer.slice(); - resultSlice.limit(totalValueLength); - - buffer.position(initialPos + totalValueLength); - return resultSlice.asReadOnlyBuffer(); - } }; TypeHandler STRING = new TypeHandler() { @@ -344,29 +231,6 @@ public int estimateSize(Value value) { return VarInt.encodedLength(utf8Bytes.length) + utf8Bytes.length; } } - - @Override - public ByteBuffer readValueBytes(ByteBuffer buffer) throws ImprintException { - int initialPos = buffer.position(); - ByteBuffer tempMeasureBuffer = buffer.duplicate(); - VarInt.DecodeResult dr = VarInt.decode(tempMeasureBuffer); - - int varIntByteLength = tempMeasureBuffer.position() - initialPos; - int payloadByteLength = dr.getValue(); - int totalValueLength = varIntByteLength + payloadByteLength; - - if (buffer.remaining() < totalValueLength) { - throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, - "Not enough bytes for VarInt-prefixed string. Needed: " + totalValueLength + - ", available: " + buffer.remaining() + " at position " + initialPos); - } - - ByteBuffer resultSlice = buffer.slice(); - resultSlice.limit(totalValueLength); - - buffer.position(initialPos + totalValueLength); - return resultSlice.asReadOnlyBuffer(); - } }; TypeHandler ARRAY = new TypeHandler() { @@ -374,25 +238,24 @@ public ByteBuffer readValueBytes(ByteBuffer buffer) throws ImprintException { public Value deserialize(ByteBuffer buffer) throws ImprintException { VarInt.DecodeResult lengthResult = VarInt.decode(buffer); int length = lengthResult.getValue(); - + if (length == 0) { return Value.fromArray(Collections.emptyList()); } - + if (buffer.remaining() < 1) { - throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, "Not enough bytes for ARRAY element type code."); + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, "Not enough bytes for ARRAY element type code."); } var elementType = TypeCode.fromByte(buffer.get()); var elements = new ArrayList(length); var elementHandler = elementType.getHandler(); - + + //Let each element handler consume what it needs from the buffer for (int i = 0; i < length; i++) { - var elementValueBytes = elementHandler.readValueBytes(buffer); - elementValueBytes.order(buffer.order()); - var element = elementHandler.deserialize(elementValueBytes); + var element = elementHandler.deserialize(buffer); //Handler advances buffer position elements.add(element); } - + return Value.fromArray(elements); } @@ -433,40 +296,6 @@ public int estimateSize(Value value) throws ImprintException { } return arraySize; } - - @Override - public ByteBuffer readValueBytes(ByteBuffer buffer) throws ImprintException { - return readComplexValueBytes(buffer, "ARRAY", (tempBuffer, numElements) -> { - if (tempBuffer.remaining() < 1) { - throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, - "Not enough bytes for ARRAY element type code"); - } - byte elementTypeCodeByte = tempBuffer.get(); - var elementType = TypeCode.fromByte(elementTypeCodeByte); - - switch (elementType) { - case NULL: - return 1; - case BOOL: - return 1 + numElements; - case INT32: - case FLOAT32: - return 1 + (numElements * 4); - case INT64: - case FLOAT64: - return 1 + (numElements * 8); - default: - var elementHandler = elementType.getHandler(); - int elementsDataLength = 0; - for (int i = 0; i < numElements; i++) { - int elementStartPos = tempBuffer.position(); - elementHandler.readValueBytes(tempBuffer); - elementsDataLength += (tempBuffer.position() - elementStartPos); - } - return 1 + elementsDataLength; - } - }); - } }; TypeHandler MAP = new TypeHandler() { @@ -474,13 +303,13 @@ public ByteBuffer readValueBytes(ByteBuffer buffer) throws ImprintException { public Value deserialize(ByteBuffer buffer) throws ImprintException { VarInt.DecodeResult lengthResult = VarInt.decode(buffer); int length = lengthResult.getValue(); - + if (length == 0) { return Value.fromMap(Collections.emptyMap()); } - + if (buffer.remaining() < 2) { - throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, "Not enough bytes for MAP key/value type codes."); + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, "Not enough bytes for MAP key/value type codes."); } var keyType = TypeCode.fromByte(buffer.get()); var valueType = TypeCode.fromByte(buffer.get()); @@ -488,20 +317,17 @@ public Value deserialize(ByteBuffer buffer) throws ImprintException { var keyHandler = keyType.getHandler(); var valueHandler = valueType.getHandler(); - + + //Let handlers consume directly from buffer for (int i = 0; i < length; i++) { - var keyBytes = keyHandler.readValueBytes(buffer); - keyBytes.order(buffer.order()); - var keyValue = keyHandler.deserialize(keyBytes); + var keyValue = keyHandler.deserialize(buffer);// Advances buffer var key = MapKey.fromValue(keyValue); - - var valueBytes = valueHandler.readValueBytes(buffer); - valueBytes.order(buffer.order()); - var mapInternalValue = valueHandler.deserialize(valueBytes); - + + var mapInternalValue = valueHandler.deserialize(buffer);//Advances buffer + map.put(key, mapInternalValue); } - + return Value.fromMap(map); } @@ -562,50 +388,6 @@ public int estimateSize(Value value) throws ImprintException { return mapSize; } - @Override - public ByteBuffer readValueBytes(ByteBuffer buffer) throws ImprintException { - return readComplexValueBytes(buffer, "MAP", (tempBuffer, numEntries) -> { - if (tempBuffer.remaining() < 2) { - throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, - "Not enough bytes for MAP key/value type codes"); - } - byte keyTypeCodeByte = tempBuffer.get(); - byte valueTypeCodeByte = tempBuffer.get(); - var keyType = TypeCode.fromByte(keyTypeCodeByte); - var valueType = TypeCode.fromByte(valueTypeCodeByte); - - int keySize = getFixedTypeSize(keyType); - int valueSize = getFixedTypeSize(valueType); - - if (keySize > 0 && valueSize > 0) { - return 2 + (numEntries * (keySize + valueSize)); - } else { - // At least one is variable-size: fall back to traversal - int entriesDataLength = 0; - for (int i = 0; i < numEntries; i++) { - int entryStartPos = tempBuffer.position(); - keyType.getHandler().readValueBytes(tempBuffer); - valueType.getHandler().readValueBytes(tempBuffer); - entriesDataLength += (tempBuffer.position() - entryStartPos); - } - return 2 + entriesDataLength; - } - }); - } - - private int getFixedTypeSize(TypeCode type) { - switch (type) { - case NULL: return 0; - case BOOL: return 1; - case INT32: - case FLOAT32: return 4; - case INT64: - case FLOAT64: return 8; - default: return -1; // Variable size - } - } - - private void serializeMapKey(MapKey key, ByteBuffer buffer) throws ImprintException { switch (key.getTypeCode()) { case INT32: diff --git a/src/test/java/com/imprint/profile/ProfilerTest.java b/src/test/java/com/imprint/profile/ProfilerTest.java index d48c1aa..3b9f371 100644 --- a/src/test/java/com/imprint/profile/ProfilerTest.java +++ b/src/test/java/com/imprint/profile/ProfilerTest.java @@ -4,6 +4,7 @@ import com.imprint.core.ImprintWriter; import com.imprint.core.SchemaId; import com.imprint.types.Value; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import java.util.Random; @@ -26,7 +27,7 @@ * - String operations and UTF-8 encoding * - ByteBuffer operations */ -//@Disabled("Enable manually for profiling") +@Disabled("Enable manually for profiling") public class ProfilerTest { private static final int ITERATIONS = 1_000_000; diff --git a/src/test/java/com/imprint/types/TypeHandlerTest.java b/src/test/java/com/imprint/types/TypeHandlerTest.java index f131a0f..75d118f 100644 --- a/src/test/java/com/imprint/types/TypeHandlerTest.java +++ b/src/test/java/com/imprint/types/TypeHandlerTest.java @@ -33,11 +33,6 @@ void testNullHandler() throws ImprintException { buffer.flip(); var deserialized = handler.deserialize(buffer); assertThat(deserialized).isEqualTo(value); - - // readValueBytes - buffer.clear(); - var valueBytes = handler.readValueBytes(buffer); - assertThat(valueBytes.remaining()).isEqualTo(0); } @ParameterizedTest From 7eaa6e9ec299fe72f43a19af766e180002716f91 Mon Sep 17 00:00:00 2001 From: expand3d <> Date: Thu, 5 Jun 2025 16:04:42 -0400 Subject: [PATCH 29/34] change CI file to use JMH plugin to respect iteration and warmup values in gradle file. Also fix permission issue --- .github/workflows/ci.yml | 148 +++++++++++++++++++++++---------------- 1 file changed, 89 insertions(+), 59 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 62ac6f5..ec052ca 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,6 +46,11 @@ jobs: benchmark: runs-on: ubuntu-latest needs: test + # Add explicit permissions for commenting on PRs + permissions: + contents: read + pull-requests: write + issues: write # Only run benchmarks on main branch pushes and PRs to main to avoid excessive CI time if: github.ref == 'refs/heads/main' || github.base_ref == 'main' @@ -77,22 +82,22 @@ jobs: - name: Run serialization benchmarks run: | - ./gradlew jmhRunSerializationBenchmarks + ./gradlew jmh --include=".*serialize.*" --exclude=".*deserialize.*" continue-on-error: true - name: Run deserialization benchmarks run: | - ./gradlew jmhRunDeserializationBenchmarks + ./gradlew jmh --include=".*deserialize.*" continue-on-error: true - name: Run field access benchmarks run: | - ./gradlew jmhRunFieldAccessBenchmarks + ./gradlew jmh --include=".*singleFieldAccess.*" continue-on-error: true - name: Run size comparison benchmarks run: | - ./gradlew jmhRunSizeComparisonBenchmarks + ./gradlew jmh --include=".*measure.*" continue-on-error: true - name: Upload benchmark results @@ -106,59 +111,63 @@ jobs: - name: Comment benchmark results on PR if: github.event_name == 'pull_request' uses: actions/github-script@v7 + continue-on-error: true with: + github-token: ${{ secrets.GITHUB_TOKEN }} script: | - const fs = require('fs'); - const path = require('path'); - - // Find the latest benchmark results file - const resultsDir = 'benchmark-results'; - let latestFile = null; - let latestTime = 0; - - if (fs.existsSync(resultsDir)) { - const files = fs.readdirSync(resultsDir); - for (const file of files) { - if (file.endsWith('.json')) { - const filePath = path.join(resultsDir, file); - const stats = fs.statSync(filePath); - if (stats.mtime.getTime() > latestTime) { - latestTime = stats.mtime.getTime(); - latestFile = filePath; + try { + const fs = require('fs'); + const path = require('path'); + + // Find the latest benchmark results file + const resultsDir = 'benchmark-results'; + let latestFile = null; + let latestTime = 0; + + if (fs.existsSync(resultsDir)) { + const files = fs.readdirSync(resultsDir); + for (const file of files) { + if (file.endsWith('.json')) { + const filePath = path.join(resultsDir, file); + const stats = fs.statSync(filePath); + if (stats.mtime.getTime() > latestTime) { + latestTime = stats.mtime.getTime(); + latestFile = filePath; + } } } } - } - if (latestFile) { - const results = JSON.parse(fs.readFileSync(latestFile, 'utf8')); + if (latestFile) { + console.log(`Found benchmark results: ${latestFile}`); + const results = JSON.parse(fs.readFileSync(latestFile, 'utf8')); - // Group results by benchmark type - const serialization = results.filter(r => r.benchmark.includes('serialize')); - const deserialization = results.filter(r => r.benchmark.includes('deserialize')); - const fieldAccess = results.filter(r => r.benchmark.includes('singleFieldAccess')); - const sizes = results.filter(r => r.benchmark.includes('measure')); + // Group results by benchmark type + const serialization = results.filter(r => r.benchmark.includes('serialize')); + const deserialization = results.filter(r => r.benchmark.includes('deserialize')); + const fieldAccess = results.filter(r => r.benchmark.includes('singleFieldAccess')); + const sizes = results.filter(r => r.benchmark.includes('measure')); - // Format results into a table - const formatResults = (benchmarks, title) => { - if (benchmarks.length === 0) return ''; + // Format results into a table + const formatResults = (benchmarks, title) => { + if (benchmarks.length === 0) return ''; - let table = `\n### ${title}\n\n| Library | Score (ns/op) | Error | Unit |\n|---------|---------------|-------|------|\n`; + let table = `\n### ${title}\n\n| Library | Score (ns/op) | Error | Unit |\n|---------|---------------|-------|------|\n`; - benchmarks - .sort((a, b) => a.primaryMetric.score - b.primaryMetric.score) - .forEach(benchmark => { - const name = benchmark.benchmark.split('.').pop().replace(/serialize|deserialize|singleFieldAccess|measure/, '').replace(/Imprint|JacksonJson|Kryo|MessagePack|Avro|Protobuf|FlatBuffers/, (match) => match); - const score = benchmark.primaryMetric.score.toFixed(2); - const error = benchmark.primaryMetric.scoreError.toFixed(2); - const unit = benchmark.primaryMetric.scoreUnit; - table += `| ${name} | ${score} | ±${error} | ${unit} |\n`; - }); + benchmarks + .sort((a, b) => a.primaryMetric.score - b.primaryMetric.score) + .forEach(benchmark => { + const name = benchmark.benchmark.split('.').pop().replace(/serialize|deserialize|singleFieldAccess|measure/, '').replace(/Imprint|JacksonJson|Kryo|MessagePack|Avro|Protobuf|FlatBuffers/, (match) => match); + const score = benchmark.primaryMetric.score.toFixed(2); + const error = benchmark.primaryMetric.scoreError.toFixed(2); + const unit = benchmark.primaryMetric.scoreUnit; + table += `| ${name} | ${score} | ±${error} | ${unit} |\n`; + }); - return table; - }; + return table; + }; - const comment = `##Benchmark Results + const comment = `## Benchmark Results Benchmark comparison between Imprint and other serialization libraries: ${formatResults(serialization, 'Serialization Performance')} @@ -175,26 +184,47 @@ jobs: `; - github.rest.issues.createComment({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - body: comment - }); - } else { - github.rest.issues.createComment({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - body: '## Benchmark Results\n\nBenchmark execution completed but no results file was found. Check the [workflow logs](' + - `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}` + ') for details.' - }); + await github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: comment + }); + + console.log('Successfully posted benchmark results'); + } else { + console.log('No benchmark results found'); + await github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: '## Benchmark Results\n\nBenchmark execution completed but no results file was found. Check the [workflow logs](' + + `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}` + ') for details.' + }); + } + } catch (error) { + console.log('Failed to post benchmark comment:', error.message); + console.log('Benchmark results are still available in workflow artifacts'); + + // Try to post a simple error message + try { + await github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: `## Benchmark Results\n\n Failed to process benchmark results automatically.\n\nResults are available in the [workflow artifacts](https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}).` + }); + } catch (commentError) { + console.log('Also failed to post error comment:', commentError.message); + } } # Optional: Run full benchmark suite on releases benchmark-full: runs-on: ubuntu-latest if: startsWith(github.ref, 'refs/tags/') + permissions: + contents: read steps: - name: Checkout code From 32640cdb551ddd9dd137f0c21be4973dac5234ad Mon Sep 17 00:00:00 2001 From: expand3d <> Date: Thu, 5 Jun 2025 16:12:02 -0400 Subject: [PATCH 30/34] ok plugin didn't work apparently so reverting that and just reducing Comparison tests iterations manually --- .github/workflows/ci.yml | 28 +++++++++---------- .../benchmark/ComparisonBenchmark.java | 4 +-- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ec052ca..d0e43cb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -82,22 +82,22 @@ jobs: - name: Run serialization benchmarks run: | - ./gradlew jmh --include=".*serialize.*" --exclude=".*deserialize.*" + ./gradlew jmhRunSerializationBenchmarks continue-on-error: true - name: Run deserialization benchmarks run: | - ./gradlew jmh --include=".*deserialize.*" + ./gradlew jmhRunDeserializationBenchmarks continue-on-error: true - name: Run field access benchmarks run: | - ./gradlew jmh --include=".*singleFieldAccess.*" + ./gradlew jmhRunFieldAccessBenchmarks continue-on-error: true - name: Run size comparison benchmarks run: | - ./gradlew jmh --include=".*measure.*" + ./gradlew jmhRunSizeComparisonBenchmarks continue-on-error: true - name: Upload benchmark results @@ -139,7 +139,7 @@ jobs: } if (latestFile) { - console.log(`Found benchmark results: ${latestFile}`); + console.log(`📊 Found benchmark results: ${latestFile}`); const results = JSON.parse(fs.readFileSync(latestFile, 'utf8')); // Group results by benchmark type @@ -167,7 +167,7 @@ jobs: return table; }; - const comment = `## Benchmark Results + const comment = `## 📊 Benchmark Results Benchmark comparison between Imprint and other serialization libraries: ${formatResults(serialization, 'Serialization Performance')} @@ -191,20 +191,20 @@ jobs: body: comment }); - console.log('Successfully posted benchmark results'); + console.log('✅ Successfully posted benchmark results to PR'); } else { - console.log('No benchmark results found'); + console.log('⚠️ No benchmark results found'); await github.rest.issues.createComment({ issue_number: context.issue.number, owner: context.repo.owner, repo: context.repo.repo, - body: '## Benchmark Results\n\nBenchmark execution completed but no results file was found. Check the [workflow logs](' + + body: '## 📊 Benchmark Results\n\nBenchmark execution completed but no results file was found. Check the [workflow logs](' + `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}` + ') for details.' }); } } catch (error) { - console.log('Failed to post benchmark comment:', error.message); - console.log('Benchmark results are still available in workflow artifacts'); + console.log('❌ Failed to post benchmark comment:', error.message); + console.log('📁 Benchmark results are still available in workflow artifacts'); // Try to post a simple error message try { @@ -212,10 +212,10 @@ jobs: issue_number: context.issue.number, owner: context.repo.owner, repo: context.repo.repo, - body: `## Benchmark Results\n\n Failed to process benchmark results automatically.\n\nResults are available in the [workflow artifacts](https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}).` + body: `## 📊 Benchmark Results\n\n⚠️ Failed to process benchmark results automatically.\n\nResults are available in the [workflow artifacts](https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}).` }); } catch (commentError) { - console.log('Also failed to post error comment:', commentError.message); + console.log('❌ Also failed to post error comment:', commentError.message); } } @@ -254,7 +254,7 @@ jobs: - name: Run full benchmark suite run: | - ./gradlew jmh + ./gradlew jmhRunAllBenchmarks - name: Upload full benchmark results uses: actions/upload-artifact@v4 diff --git a/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java b/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java index a7ffd3c..ee32ff0 100644 --- a/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java +++ b/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java @@ -38,8 +38,8 @@ @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.NANOSECONDS) @State(Scope.Benchmark) -@Warmup(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS) -@Measurement(iterations = 10, time = 1, timeUnit = TimeUnit.SECONDS) +@Warmup(iterations = 2, time = 1, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS) @Fork(1) @SuppressWarnings("unused") public class ComparisonBenchmark { From 880aeb0c6bdfd24d7893c3204d43eeb59059745b Mon Sep 17 00:00:00 2001 From: expand3d <> Date: Thu, 5 Jun 2025 16:43:08 -0400 Subject: [PATCH 31/34] trying to update github ci to make jmh actually work correctly --- .github/workflows/ci.yml | 140 +++------------------------- build.gradle | 192 ++++++++++++++++++++++++--------------- 2 files changed, 132 insertions(+), 200 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d0e43cb..a80f529 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,11 +46,6 @@ jobs: benchmark: runs-on: ubuntu-latest needs: test - # Add explicit permissions for commenting on PRs - permissions: - contents: read - pull-requests: write - issues: write # Only run benchmarks on main branch pushes and PRs to main to avoid excessive CI time if: github.ref == 'refs/heads/main' || github.base_ref == 'main' @@ -81,25 +76,28 @@ jobs: run: mkdir -p benchmark-results - name: Run serialization benchmarks - run: | - ./gradlew jmhRunSerializationBenchmarks + run: ./gradlew jmhRunSerializationBenchmarks continue-on-error: true - name: Run deserialization benchmarks - run: | - ./gradlew jmhRunDeserializationBenchmarks + run: ./gradlew jmhRunDeserializationBenchmarks continue-on-error: true - name: Run field access benchmarks - run: | - ./gradlew jmhRunFieldAccessBenchmarks + run: ./gradlew jmhRunFieldAccessBenchmarks continue-on-error: true - name: Run size comparison benchmarks - run: | - ./gradlew jmhRunSizeComparisonBenchmarks + run: ./gradlew jmhRunSizeComparisonBenchmarks continue-on-error: true + - name: List benchmark results + run: | + echo "Contents of benchmark-results directory:" + ls -la benchmark-results/ || echo "benchmark-results directory not found" + echo "Working directory contents:" + ls -la + - name: Upload benchmark results uses: actions/upload-artifact@v4 if: always() @@ -108,123 +106,10 @@ jobs: path: benchmark-results/ retention-days: 30 - - name: Comment benchmark results on PR - if: github.event_name == 'pull_request' - uses: actions/github-script@v7 - continue-on-error: true - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - try { - const fs = require('fs'); - const path = require('path'); - - // Find the latest benchmark results file - const resultsDir = 'benchmark-results'; - let latestFile = null; - let latestTime = 0; - - if (fs.existsSync(resultsDir)) { - const files = fs.readdirSync(resultsDir); - for (const file of files) { - if (file.endsWith('.json')) { - const filePath = path.join(resultsDir, file); - const stats = fs.statSync(filePath); - if (stats.mtime.getTime() > latestTime) { - latestTime = stats.mtime.getTime(); - latestFile = filePath; - } - } - } - } - - if (latestFile) { - console.log(`📊 Found benchmark results: ${latestFile}`); - const results = JSON.parse(fs.readFileSync(latestFile, 'utf8')); - - // Group results by benchmark type - const serialization = results.filter(r => r.benchmark.includes('serialize')); - const deserialization = results.filter(r => r.benchmark.includes('deserialize')); - const fieldAccess = results.filter(r => r.benchmark.includes('singleFieldAccess')); - const sizes = results.filter(r => r.benchmark.includes('measure')); - - // Format results into a table - const formatResults = (benchmarks, title) => { - if (benchmarks.length === 0) return ''; - - let table = `\n### ${title}\n\n| Library | Score (ns/op) | Error | Unit |\n|---------|---------------|-------|------|\n`; - - benchmarks - .sort((a, b) => a.primaryMetric.score - b.primaryMetric.score) - .forEach(benchmark => { - const name = benchmark.benchmark.split('.').pop().replace(/serialize|deserialize|singleFieldAccess|measure/, '').replace(/Imprint|JacksonJson|Kryo|MessagePack|Avro|Protobuf|FlatBuffers/, (match) => match); - const score = benchmark.primaryMetric.score.toFixed(2); - const error = benchmark.primaryMetric.scoreError.toFixed(2); - const unit = benchmark.primaryMetric.scoreUnit; - table += `| ${name} | ${score} | ±${error} | ${unit} |\n`; - }); - - return table; - }; - - const comment = `## 📊 Benchmark Results - - Benchmark comparison between Imprint and other serialization libraries: - ${formatResults(serialization, 'Serialization Performance')} - ${formatResults(deserialization, 'Deserialization Performance')} - ${formatResults(fieldAccess, 'Single Field Access Performance')} - ${formatResults(sizes, 'Serialized Size Comparison')} - -
- View detailed results - - Results generated from commit: \`${context.sha.substring(0, 7)}\` - - Lower scores are better for performance benchmarks. - -
`; - - await github.rest.issues.createComment({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - body: comment - }); - - console.log('✅ Successfully posted benchmark results to PR'); - } else { - console.log('⚠️ No benchmark results found'); - await github.rest.issues.createComment({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - body: '## 📊 Benchmark Results\n\nBenchmark execution completed but no results file was found. Check the [workflow logs](' + - `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}` + ') for details.' - }); - } - } catch (error) { - console.log('❌ Failed to post benchmark comment:', error.message); - console.log('📁 Benchmark results are still available in workflow artifacts'); - - // Try to post a simple error message - try { - await github.rest.issues.createComment({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - body: `## 📊 Benchmark Results\n\n⚠️ Failed to process benchmark results automatically.\n\nResults are available in the [workflow artifacts](https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}).` - }); - } catch (commentError) { - console.log('❌ Also failed to post error comment:', commentError.message); - } - } - # Optional: Run full benchmark suite on releases benchmark-full: runs-on: ubuntu-latest if: startsWith(github.ref, 'refs/tags/') - permissions: - contents: read steps: - name: Checkout code @@ -253,8 +138,7 @@ jobs: run: mkdir -p benchmark-results - name: Run full benchmark suite - run: | - ./gradlew jmhRunAllBenchmarks + run: ./gradlew jmhRunAllBenchmarks - name: Upload full benchmark results uses: actions/upload-artifact@v4 diff --git a/build.gradle b/build.gradle index d9093f9..6c97a24 100644 --- a/build.gradle +++ b/build.gradle @@ -185,130 +185,178 @@ jmh { } // Create individual benchmark tasks for CI pipeline -tasks.register('jmhRunSerializationBenchmarks', JavaExec) { +tasks.register('jmhRunSerializationBenchmarks') { dependsOn compileJmhJava description = 'Run serialization benchmarks' group = 'benchmarking' - classpath = sourceSets.jmh.runtimeClasspath - mainClass = 'com.imprint.benchmark.ComparisonBenchmark' - args = ['runSerializationBenchmarks'] - - // Java 11 optimized JVM settings - jvmArgs = [ - '-XX:+UseG1GC', - '-Xmx2g', - '-XX:+UnlockExperimentalVMOptions' - ] - doFirst { file("${projectDir}/benchmark-results").mkdirs() } + + doLast { + javaexec { + classpath = sourceSets.jmh.runtimeClasspath + mainClass = 'org.openjdk.jmh.Main' + args = [ + '.*serialize.*', + '-f', '1', + '-wi', '2', + '-i', '3', + '-rf', 'json', + '-rff', "${projectDir}/benchmark-results/serialization-results.json" + ] + jvmArgs = [ + '-XX:+UseG1GC', + '-Xmx2g' + ] + } + } } -tasks.register('jmhRunDeserializationBenchmarks', JavaExec) { +tasks.register('jmhRunDeserializationBenchmarks') { dependsOn compileJmhJava description = 'Run deserialization benchmarks' group = 'benchmarking' - classpath = sourceSets.jmh.runtimeClasspath - mainClass = 'com.imprint.benchmark.ComparisonBenchmark' - args = ['runDeserializationBenchmarks'] - - // Java 11 optimized JVM settings - jvmArgs = [ - '-XX:+UseG1GC', - '-Xmx2g', - '-XX:+UnlockExperimentalVMOptions' - ] - doFirst { file("${projectDir}/benchmark-results").mkdirs() } + + doLast { + javaexec { + classpath = sourceSets.jmh.runtimeClasspath + mainClass = 'org.openjdk.jmh.Main' + args = [ + '.*deserialize.*', + '-f', '1', + '-wi', '2', + '-i', '3', + '-rf', 'json', + '-rff', "${projectDir}/benchmark-results/deserialization-results.json" + ] + jvmArgs = [ + '-XX:+UseG1GC', + '-Xmx2g' + ] + } + } } -tasks.register('jmhRunFieldAccessBenchmarks', JavaExec) { +tasks.register('jmhRunFieldAccessBenchmarks') { dependsOn compileJmhJava description = 'Run field access benchmarks' group = 'benchmarking' - classpath = sourceSets.jmh.runtimeClasspath - mainClass = 'com.imprint.benchmark.ComparisonBenchmark' - args = ['runFieldAccessBenchmarks'] - - // Java 11 optimized JVM settings - jvmArgs = [ - '-XX:+UseG1GC', - '-Xmx2g', - '-XX:+UnlockExperimentalVMOptions' - ] - doFirst { file("${projectDir}/benchmark-results").mkdirs() } + + doLast { + javaexec { + classpath = sourceSets.jmh.runtimeClasspath + mainClass = 'org.openjdk.jmh.Main' + args = [ + '.*singleFieldAccess.*', + '-f', '1', + '-wi', '2', + '-i', '3', + '-rf', 'json', + '-rff', "${projectDir}/benchmark-results/fieldaccess-results.json" + ] + jvmArgs = [ + '-XX:+UseG1GC', + '-Xmx2g' + ] + } + } } -tasks.register('jmhRunSizeComparisonBenchmarks', JavaExec) { +tasks.register('jmhRunSizeComparisonBenchmarks') { dependsOn compileJmhJava description = 'Run size comparison benchmarks' group = 'benchmarking' - classpath = sourceSets.jmh.runtimeClasspath - mainClass = 'com.imprint.benchmark.ComparisonBenchmark' - args = ['runSizeComparisonBenchmarks'] - - // Java 11 optimized JVM settings - jvmArgs = [ - '-XX:+UseG1GC', - '-Xmx2g', - '-XX:+UnlockExperimentalVMOptions' - ] - doFirst { file("${projectDir}/benchmark-results").mkdirs() } + + doLast { + javaexec { + classpath = sourceSets.jmh.runtimeClasspath + mainClass = 'org.openjdk.jmh.Main' + args = [ + '.*measure.*', + '-f', '1', + '-wi', '2', + '-i', '3', + '-rf', 'json', + '-rff', "${projectDir}/benchmark-results/size-results.json" + ] + jvmArgs = [ + '-XX:+UseG1GC', + '-Xmx2g' + ] + } + } } -tasks.register('jmhRunMergeBenchmarks', JavaExec) { +tasks.register('jmhRunMergeBenchmarks') { dependsOn compileJmhJava description = 'Run merge operation benchmarks' group = 'benchmarking' - classpath = sourceSets.jmh.runtimeClasspath - mainClass = 'com.imprint.benchmark.ComparisonBenchmark' - args = ['runMergeBenchmarks'] - - // Java 11 optimized JVM settings - jvmArgs = [ - '-XX:+UseG1GC', - '-Xmx2g', - '-XX:+UnlockExperimentalVMOptions' - ] - doFirst { file("${projectDir}/benchmark-results").mkdirs() } + + doLast { + javaexec { + classpath = sourceSets.jmh.runtimeClasspath + mainClass = 'org.openjdk.jmh.Main' + args = [ + '.*merge.*', + '-f', '1', + '-wi', '2', + '-i', '3', + '-rf', 'json', + '-rff', "${projectDir}/benchmark-results/merge-results.json" + ] + jvmArgs = [ + '-XX:+UseG1GC', + '-Xmx2g' + ] + } + } } -tasks.register('jmhRunAllBenchmarks', JavaExec) { +tasks.register('jmhRunAllBenchmarks') { dependsOn compileJmhJava description = 'Run all comparison benchmarks' group = 'benchmarking' - classpath = sourceSets.jmh.runtimeClasspath - mainClass = 'com.imprint.benchmark.ComparisonBenchmark' - args = ['runAll'] - - // Java 11 optimized JVM settings - jvmArgs = [ - '-XX:+UseG1GC', - '-Xmx2g', - '-XX:+UnlockExperimentalVMOptions' - ] - doFirst { file("${projectDir}/benchmark-results").mkdirs() } + + doLast { + javaexec { + classpath = sourceSets.jmh.runtimeClasspath + mainClass = 'org.openjdk.jmh.Main' + args = [ + 'ComparisonBenchmark', + '-f', '1', + '-wi', '2', + '-i', '3', + '-rf', 'json', + '-rff', "${projectDir}/benchmark-results/all-results.json" + ] + jvmArgs = [ + '-XX:+UseG1GC', + '-Xmx2g' + ] + } + } } compileJava { From 8831922bc690b490f3acb1ba963525c61260f27f Mon Sep 17 00:00:00 2001 From: expand3d <> Date: Thu, 5 Jun 2025 21:56:09 -0400 Subject: [PATCH 32/34] lazy directory deserialization --- .github/workflows/ci.yml | 110 +--- build.gradle | 183 +----- .../benchmark/ComparisonBenchmark.java | 18 +- .../java/com/imprint/core/ImprintBuffers.java | 451 +++++++++++++ .../java/com/imprint/core/ImprintRecord.java | 612 ++++-------------- src/main/java/com/imprint/util/VarInt.java | 55 +- 6 files changed, 632 insertions(+), 797 deletions(-) create mode 100644 src/main/java/com/imprint/core/ImprintBuffers.java diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a80f529..378ebb7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,9 +2,9 @@ name: CI on: push: - branches: [ main, dev ] + branches: [ main ] pull_request: - branches: [ main, dev ] + branches: [ main ] jobs: test: @@ -41,108 +41,4 @@ jobs: run: ./gradlew test - name: Run build - run: ./gradlew build - - benchmark: - runs-on: ubuntu-latest - needs: test - # Only run benchmarks on main branch pushes and PRs to main to avoid excessive CI time - if: github.ref == 'refs/heads/main' || github.base_ref == 'main' - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up JDK 11 - uses: actions/setup-java@v4 - with: - java-version: '11' - distribution: 'temurin' - - - name: Cache Gradle dependencies - uses: actions/cache@v4 - with: - path: | - ~/.gradle/caches - ~/.gradle/wrapper - key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} - restore-keys: | - ${{ runner.os }}-gradle- - - - name: Make gradlew executable - run: chmod +x ./gradlew - - - name: Create benchmark results directory - run: mkdir -p benchmark-results - - - name: Run serialization benchmarks - run: ./gradlew jmhRunSerializationBenchmarks - continue-on-error: true - - - name: Run deserialization benchmarks - run: ./gradlew jmhRunDeserializationBenchmarks - continue-on-error: true - - - name: Run field access benchmarks - run: ./gradlew jmhRunFieldAccessBenchmarks - continue-on-error: true - - - name: Run size comparison benchmarks - run: ./gradlew jmhRunSizeComparisonBenchmarks - continue-on-error: true - - - name: List benchmark results - run: | - echo "Contents of benchmark-results directory:" - ls -la benchmark-results/ || echo "benchmark-results directory not found" - echo "Working directory contents:" - ls -la - - - name: Upload benchmark results - uses: actions/upload-artifact@v4 - if: always() - with: - name: benchmark-results-${{ github.sha }} - path: benchmark-results/ - retention-days: 30 - - # Optional: Run full benchmark suite on releases - benchmark-full: - runs-on: ubuntu-latest - if: startsWith(github.ref, 'refs/tags/') - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up JDK 11 - uses: actions/setup-java@v4 - with: - java-version: '11' - distribution: 'temurin' - - - name: Cache Gradle dependencies - uses: actions/cache@v4 - with: - path: | - ~/.gradle/caches - ~/.gradle/wrapper - key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} - restore-keys: | - ${{ runner.os }}-gradle- - - - name: Make gradlew executable - run: chmod +x ./gradlew - - - name: Create benchmark results directory - run: mkdir -p benchmark-results - - - name: Run full benchmark suite - run: ./gradlew jmhRunAllBenchmarks - - - name: Upload full benchmark results - uses: actions/upload-artifact@v4 - with: - name: full-benchmark-results-${{ github.ref_name }} - path: benchmark-results/ - retention-days: 90 \ No newline at end of file + run: ./gradlew build \ No newline at end of file diff --git a/build.gradle b/build.gradle index 6c97a24..33b1645 100644 --- a/build.gradle +++ b/build.gradle @@ -166,11 +166,11 @@ test { } } -// JMH configuration - optimized for Java 11 +// JMH configuration jmh { - fork = 1 - warmupIterations = 2 // Reduced for faster CI - iterations = 3 // Reduced for faster CI + fork = 2 + warmupIterations = 3 + iterations = 5 resultFormat = 'JSON' includeTests = false resultsFile = file("${projectDir}/benchmark-results/jmh-results-${new Date().format('yyyy-MM-dd-HHmmss')}.json") @@ -184,181 +184,6 @@ jmh { ] } -// Create individual benchmark tasks for CI pipeline -tasks.register('jmhRunSerializationBenchmarks') { - dependsOn compileJmhJava - description = 'Run serialization benchmarks' - group = 'benchmarking' - - doFirst { - file("${projectDir}/benchmark-results").mkdirs() - } - - doLast { - javaexec { - classpath = sourceSets.jmh.runtimeClasspath - mainClass = 'org.openjdk.jmh.Main' - args = [ - '.*serialize.*', - '-f', '1', - '-wi', '2', - '-i', '3', - '-rf', 'json', - '-rff', "${projectDir}/benchmark-results/serialization-results.json" - ] - jvmArgs = [ - '-XX:+UseG1GC', - '-Xmx2g' - ] - } - } -} - -tasks.register('jmhRunDeserializationBenchmarks') { - dependsOn compileJmhJava - description = 'Run deserialization benchmarks' - group = 'benchmarking' - - doFirst { - file("${projectDir}/benchmark-results").mkdirs() - } - - doLast { - javaexec { - classpath = sourceSets.jmh.runtimeClasspath - mainClass = 'org.openjdk.jmh.Main' - args = [ - '.*deserialize.*', - '-f', '1', - '-wi', '2', - '-i', '3', - '-rf', 'json', - '-rff', "${projectDir}/benchmark-results/deserialization-results.json" - ] - jvmArgs = [ - '-XX:+UseG1GC', - '-Xmx2g' - ] - } - } -} - -tasks.register('jmhRunFieldAccessBenchmarks') { - dependsOn compileJmhJava - description = 'Run field access benchmarks' - group = 'benchmarking' - - doFirst { - file("${projectDir}/benchmark-results").mkdirs() - } - - doLast { - javaexec { - classpath = sourceSets.jmh.runtimeClasspath - mainClass = 'org.openjdk.jmh.Main' - args = [ - '.*singleFieldAccess.*', - '-f', '1', - '-wi', '2', - '-i', '3', - '-rf', 'json', - '-rff', "${projectDir}/benchmark-results/fieldaccess-results.json" - ] - jvmArgs = [ - '-XX:+UseG1GC', - '-Xmx2g' - ] - } - } -} - -tasks.register('jmhRunSizeComparisonBenchmarks') { - dependsOn compileJmhJava - description = 'Run size comparison benchmarks' - group = 'benchmarking' - - doFirst { - file("${projectDir}/benchmark-results").mkdirs() - } - - doLast { - javaexec { - classpath = sourceSets.jmh.runtimeClasspath - mainClass = 'org.openjdk.jmh.Main' - args = [ - '.*measure.*', - '-f', '1', - '-wi', '2', - '-i', '3', - '-rf', 'json', - '-rff', "${projectDir}/benchmark-results/size-results.json" - ] - jvmArgs = [ - '-XX:+UseG1GC', - '-Xmx2g' - ] - } - } -} - -tasks.register('jmhRunMergeBenchmarks') { - dependsOn compileJmhJava - description = 'Run merge operation benchmarks' - group = 'benchmarking' - - doFirst { - file("${projectDir}/benchmark-results").mkdirs() - } - - doLast { - javaexec { - classpath = sourceSets.jmh.runtimeClasspath - mainClass = 'org.openjdk.jmh.Main' - args = [ - '.*merge.*', - '-f', '1', - '-wi', '2', - '-i', '3', - '-rf', 'json', - '-rff', "${projectDir}/benchmark-results/merge-results.json" - ] - jvmArgs = [ - '-XX:+UseG1GC', - '-Xmx2g' - ] - } - } -} - -tasks.register('jmhRunAllBenchmarks') { - dependsOn compileJmhJava - description = 'Run all comparison benchmarks' - group = 'benchmarking' - - doFirst { - file("${projectDir}/benchmark-results").mkdirs() - } - - doLast { - javaexec { - classpath = sourceSets.jmh.runtimeClasspath - mainClass = 'org.openjdk.jmh.Main' - args = [ - 'ComparisonBenchmark', - '-f', '1', - '-wi', '2', - '-i', '3', - '-rf', 'json', - '-rff', "${projectDir}/benchmark-results/all-results.json" - ] - jvmArgs = [ - '-XX:+UseG1GC', - '-Xmx2g' - ] - } - } -} - compileJava { options.compilerArgs << '-Xlint:unchecked' options.deprecation = true diff --git a/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java b/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java index ce2fbcb..dd62457 100644 --- a/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java +++ b/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java @@ -37,8 +37,8 @@ @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.NANOSECONDS) @State(Scope.Benchmark) -@Warmup(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS) -@Measurement(iterations = 10, time = 1, timeUnit = TimeUnit.SECONDS) +@Warmup(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS) @Fork(1) @SuppressWarnings("unused") public class ComparisonBenchmark { @@ -325,7 +325,7 @@ public void measureFlatBuffersSize(Blackhole bh) { // ===== MERGE SIMULATION BENCHMARKS ===== - @Benchmark + //@Benchmark public void mergeImprint(Blackhole bh) throws Exception { var record1Buffer = imprintBytesBuffer.duplicate(); var record2Data = createTestRecord2(); @@ -338,7 +338,7 @@ public void mergeImprint(Blackhole bh) throws Exception { bh.consume(merged); } - @Benchmark + //@Benchmark public void mergeJacksonJson(Blackhole bh) throws Exception { var record1 = jacksonJsonMapper.readValue(jacksonJsonBytes, TestRecord.class); var record2Data = createTestRecord2(); @@ -350,7 +350,7 @@ public void mergeJacksonJson(Blackhole bh) throws Exception { bh.consume(result); } - @Benchmark + //@Benchmark public void mergeKryo(Blackhole bh) { Input input1 = new Input(new ByteArrayInputStream(kryoBytes)); var record1 = kryo.readObject(input1, TestRecord.class); @@ -367,7 +367,7 @@ public void mergeKryo(Blackhole bh) { bh.consume(result); } - @Benchmark + //@Benchmark public void mergeMessagePack(Blackhole bh) throws Exception { var record1 = messagePackMapper.readValue(messagePackBytes, TestRecord.class); var record2Data = createTestRecord2(); @@ -379,7 +379,7 @@ public void mergeMessagePack(Blackhole bh) throws Exception { bh.consume(result); } - @Benchmark + //@Benchmark public void mergeAvro(Blackhole bh) throws Exception { var record1 = deserializeWithAvro(avroBytes); var record2Data = createTestRecord2(); @@ -391,7 +391,7 @@ public void mergeAvro(Blackhole bh) throws Exception { bh.consume(result); } - @Benchmark + //@Benchmark public void mergeProtobuf(Blackhole bh) throws Exception { var record1 = TestRecordProto.TestRecord.parseFrom(protobufBytes); var record2Data = createTestRecord2(); @@ -403,7 +403,7 @@ public void mergeProtobuf(Blackhole bh) throws Exception { bh.consume(result); } - @Benchmark + //@Benchmark public void mergeFlatBuffers(Blackhole bh) { var record1 = TestRecordFB.getRootAsTestRecordFB(flatbuffersBytes.duplicate()); var record2Data = createTestRecord2(); diff --git a/src/main/java/com/imprint/core/ImprintBuffers.java b/src/main/java/com/imprint/core/ImprintBuffers.java new file mode 100644 index 0000000..f6a341b --- /dev/null +++ b/src/main/java/com/imprint/core/ImprintBuffers.java @@ -0,0 +1,451 @@ +package com.imprint.core; + +import com.imprint.Constants; +import com.imprint.error.ErrorType; +import com.imprint.error.ImprintException; +import com.imprint.types.TypeCode; +import com.imprint.util.VarInt; +import lombok.Getter; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +/** + * Manages the raw buffers for an Imprint record with lazy directory parsing. + * Encapsulates all buffer operations and provides zero-copy field access. + * + *

Buffer Layout Overview:

+ * 
+ * directoryBuffer: [VarInt count][DirectoryEntry 1][DirectoryEntry 2]...[DirectoryEntry N]
+ * payload:         [Field 1 data][Field 2 data]...[Field N data]
+ *
+ * + *

Each DirectoryEntry contains: [fieldId:2bytes][typeCode:1byte][offset:4bytes]

+ */ +@Getter +public final class ImprintBuffers { + private final ByteBuffer directoryBuffer; // Raw directory bytes (includes count) + private final ByteBuffer payload; // Read-only payload view + + // Lazy-loaded directory state + private List parsedDirectory; + private boolean directoryParsed = false; + private int directoryCount = -1; // Cached count to avoid repeated VarInt decoding + + /** + * Creates buffers from raw data (used during deserialization). + * + * @param directoryBuffer Raw directory bytes including VarInt count and all entries. + * Format: [VarInt count][Entry1][Entry2]...[EntryN] + * @param payload Raw payload data containing all field values sequentially + */ + public ImprintBuffers(ByteBuffer directoryBuffer, ByteBuffer payload) { + this.directoryBuffer = directoryBuffer.asReadOnlyBuffer(); + this.payload = payload.asReadOnlyBuffer(); + } + + /** + * Creates buffers from pre-parsed directory (used during construction). + * This is more efficient when the directory is already known. + * + * @param directory Parsed directory entries, must be sorted by fieldId + * @param payload Raw payload data containing all field values + */ + public ImprintBuffers(List directory, ByteBuffer payload) { + this.parsedDirectory = Collections.unmodifiableList(Objects.requireNonNull(directory)); + this.directoryParsed = true; + this.directoryCount = directory.size(); + this.payload = payload.asReadOnlyBuffer(); + this.directoryBuffer = createDirectoryBuffer(directory); + } + + /** + * Get a zero-copy ByteBuffer view of a field's data. + * + *

Buffer Positioning Logic:

+ *
    + *
  1. Find the directory entry for the requested fieldId
    2. + *
  2. Use entry.offset as start position in payload
    3. + *
  3. Find end position by looking at next field's offset (or payload end)
    4. + *
  4. Create a slice view: payload[startOffset:endOffset]
    5. + *
+ * + * @param fieldId The field identifier to retrieve + * @return Zero-copy ByteBuffer positioned at field data, or null if field not found + * @throws ImprintException if buffer bounds are invalid or directory is corrupted + */ + public ByteBuffer getFieldBuffer(int fieldId) throws ImprintException { + var entry = findDirectoryEntry(fieldId); + if (entry == null) + return null; + + int startOffset = entry.getOffset(); + int endOffset = findEndOffset(entry); + + if (startOffset < 0 || endOffset < 0 || startOffset > payload.limit() || + endOffset > payload.limit() || startOffset > endOffset) { + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, + "Invalid field buffer range: start=" + startOffset + ", end=" + endOffset + ", payloadLimit=" + payload.limit()); + } + + ByteBuffer fieldBuffer = payload.duplicate(); + fieldBuffer.position(startOffset).limit(endOffset); + return fieldBuffer; + } + + /** + * Find a directory entry for the given field ID using the most efficient method. + * + *

Search Strategy:

+ *
    + *
  • If directory is parsed: binary search on in-memory List<DirectoryEntry>
    • + *
  • If directory is raw: binary search directly on raw bytes (faster for single lookups)
    • + *
+ * @param fieldId The field identifier to find + * @return DirectoryEntry if found, null otherwise + * @throws ImprintException if directory buffer is corrupted or truncated + */ + public DirectoryEntry findDirectoryEntry(int fieldId) throws ImprintException { + if (directoryParsed) { + int index = findDirectoryIndexInParsed(fieldId); + return index >= 0 ? parsedDirectory.get(index) : null; + } else { + return findFieldEntryInRawDirectory(fieldId); + } + } + + /** + * Get the full directory, parsing it if necessary. + * + *

Lazy Parsing Behavior:

+ *
    + *
  • First call: Parses entire directory from raw bytes into List<DirectoryEntry>
    • + *
  • Subsequent calls: Returns cached parsed directory
    • + *
  • Note - the method is not synchronized and assumes single-threaded usage.
    • + *
+ * + *

When to use: Call this if you need to access multiple fields + * from the same record. For single field access, direct field getters are more efficient.

+ * + * @return Immutable list of directory entries, sorted by fieldId + */ + public List getDirectory() { + ensureDirectoryParsed(); + return parsedDirectory; + } + + /** + * Get the directory count without fully parsing the directory. + *

+ * This method avoids parsing the entire directory when only the count is needed. + *

    + *
  1. Return cached count if available (directoryCount >= 0)
    2. + *
  2. Return parsed directory size if directory is already parsed
    3. + *
  3. Decode VarInt from raw buffer and cache the result
    4. + *
+ * + *

VarInt Decoding: The count is stored as a VarInt at the beginning + * of the directoryBuffer. This method reads just enough bytes to decode the count.

+ * + * @return Number of fields in the directory, or 0 if decoding fails + */ + public int getDirectoryCount() { + if (directoryCount >= 0) + return directoryCount; + if (directoryParsed) + return parsedDirectory.size(); + + // Decode from buffer and cache + try { + var countBuffer = directoryBuffer.duplicate(); + directoryCount = VarInt.decode(countBuffer).getValue(); + return directoryCount; + } catch (Exception e) { + return 0; + } + } + + /** + * Create a new buffer containing the serialized directory. + * + *

Output Format:

+ * 
+     * [VarInt count][DirectoryEntry 1][DirectoryEntry 2]...[DirectoryEntry N]
+     *
+ * + *

Each DirectoryEntry is serialized as: [fieldId:2bytes][typeCode:1byte][offset:4bytes]

+ * + * + * @return New ByteBuffer containing the complete serialized directory + */ + public ByteBuffer serializeDirectory() { + ensureDirectoryParsed(); + return createDirectoryBuffer(parsedDirectory); + } + + // ========== PRIVATE METHODS ========== + + /** + * Binary search on raw directory bytes to find a specific field. + * + *
    + *
  1. Position buffer at start and decode VarInt count (cache for future use)
    2. + *
  2. Calculate directory start position after VarInt
    3. + *
  3. For binary search mid-point: entryPos = startPos + (mid * DIR_ENTRY_BYTES)
    4. + *
  4. Read fieldId from calculated position (first 2 bytes of entry)
    5. + *
  5. Compare fieldId and adjust search bounds
    6. + *
  6. When found: reposition buffer and deserialize complete entry
    7. + *
+ * + *

All buffer positions are bounds-checked before access.

+ * + * @param fieldId Field identifier to search for + * @return Complete DirectoryEntry if found, null if not found + * @throws ImprintException if buffer is truncated or corrupted + */ + private DirectoryEntry findFieldEntryInRawDirectory(int fieldId) throws ImprintException { + var searchBuffer = directoryBuffer.duplicate(); + searchBuffer.order(ByteOrder.LITTLE_ENDIAN); + + // Decode directory count (cache it) + if (directoryCount < 0) + directoryCount = VarInt.decode(searchBuffer).getValue(); + else + VarInt.decode(searchBuffer); // Skip past the count + + if (directoryCount == 0) + return null; + + int directoryStartPos = searchBuffer.position(); + int low = 0; + int high = directoryCount - 1; + + while (low <= high) { + int mid = (low + high) >>> 1; + int entryPos = directoryStartPos + (mid * Constants.DIR_ENTRY_BYTES); + + if (entryPos + Constants.DIR_ENTRY_BYTES > searchBuffer.limit()) { + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, + "Directory entry at position " + entryPos + " exceeds buffer limit"); + } + + searchBuffer.position(entryPos); + short midFieldId = searchBuffer.getShort(); + + if (midFieldId < fieldId) { + low = mid + 1; + } else if (midFieldId > fieldId) { + high = mid - 1; + } else { + // Found it - read the complete entry + searchBuffer.position(entryPos); + return deserializeDirectoryEntry(searchBuffer); + } + } + + return null; + } + + /** + * + * @param fieldId Field identifier to find + * @return Index of the field if found, or negative insertion point if not found + */ + private int findDirectoryIndexInParsed(int fieldId) { + if (!directoryParsed) + return -1; + int low = 0; + int high = parsedDirectory.size() - 1; + while (low <= high) { + int mid = (low + high) >>> 1; + int midFieldId = parsedDirectory.get(mid).getId(); + if (midFieldId < fieldId) + low = mid + 1; + else if (midFieldId > fieldId) + high = mid - 1; + else + return mid; + } + return -(low + 1); + } + + /** + * Find the end offset for a field by looking at the next field's offset. + * + *
    + *
  • Field data spans from: entry.offset to nextField.offset (exclusive)
    • + *
  • Last field spans from: entry.offset to payload.limit()
    • + *
  • This works because directory entries are sorted by fieldId
    • + *
+ * + *

Search Strategy:

+ *
    + *
  • If directory parsed: Use binary search result + 1 to get next entry
    • + *
  • If directory raw: Scan raw entries until fieldId > currentFieldId
    • + *
+ * + * @param entry The directory entry whose end offset we need to find + * @return End offset (exclusive) for the field data + * @throws ImprintException if directory scanning fails + */ + private int findEndOffset(DirectoryEntry entry) throws ImprintException { + if (directoryParsed) { + int entryIndex = findDirectoryIndexInParsed(entry.getId()); + return (entryIndex + 1 < parsedDirectory.size()) ? + parsedDirectory.get(entryIndex + 1).getOffset() : payload.limit(); + } else + return findNextOffsetInRawDirectory(entry.getId()); + } + + /** + * Scan raw directory to find the next field's offset after currentFieldId. + * + *
    + *
  1. Position buffer after VarInt count
    2. + *
  2. For each directory entry at position: startPos + (i * DIR_ENTRY_BYTES)
    3. + *
  3. Read fieldId (first 2 bytes) and offset (bytes 3-6)
    4. + *
  4. Return offset of first field where fieldId > currentFieldId
    5. + *
  5. If no next field found, return payload.limit()
    6. + *
+ * + * @param currentFieldId Find the next field after this fieldId + * @return Offset where the next field starts, or payload.limit() if this is the last field + * @throws ImprintException if directory buffer is corrupted + */ + private int findNextOffsetInRawDirectory(int currentFieldId) throws ImprintException { + var scanBuffer = directoryBuffer.duplicate(); + scanBuffer.order(ByteOrder.LITTLE_ENDIAN); + + int count = (directoryCount >= 0) ? directoryCount : VarInt.decode(scanBuffer).getValue(); + if (count == 0) + return payload.limit(); + if (directoryCount >= 0) + VarInt.decode(scanBuffer); // Skip count if cached + + int directoryStartPos = scanBuffer.position(); + + for (int i = 0; i < count; i++) { + int entryPos = directoryStartPos + (i * Constants.DIR_ENTRY_BYTES); + + if (entryPos + Constants.DIR_ENTRY_BYTES > scanBuffer.limit()) + return payload.limit(); + + scanBuffer.position(entryPos); + short fieldId = scanBuffer.getShort(); + scanBuffer.get(); // skip type + int offset = scanBuffer.getInt(); + + if (fieldId > currentFieldId) + return offset; + } + + return payload.limit(); + } + + /** + * Parse the full directory if not already parsed. + * + *
    + *
  1. Duplicate directoryBuffer to avoid affecting original position
    2. + *
  2. Set byte order to LITTLE_ENDIAN for consistent reading
    3. + *
  3. Decode VarInt count and cache it
    4. + *
  4. Read 'count' directory entries sequentially
    5. + *
  5. Each entry: [fieldId:2bytes][typeCode:1byte][offset:4bytes]
    6. + *
  6. Store as immutable list and mark as parsed
    7. + *
+ * + *

Error Handling: If parsing fails, throws RuntimeException + * since this indicates corrupted data that should never happen in normal operation.

+ * + *

Will return immediately if directory has already been parsed.

+ */ + private void ensureDirectoryParsed() { + if (directoryParsed) + return; + try { + var parseBuffer = directoryBuffer.duplicate(); + parseBuffer.order(ByteOrder.LITTLE_ENDIAN); + + var countResult = VarInt.decode(parseBuffer); + int count = countResult.getValue(); + this.directoryCount = count; + + var directory = new ArrayList(count); + for (int i = 0; i < count; i++) { + directory.add(deserializeDirectoryEntry(parseBuffer)); + } + + this.parsedDirectory = Collections.unmodifiableList(directory); + this.directoryParsed = true; + } catch (ImprintException e) { + throw new RuntimeException("Failed to parse directory", e); + } + } + + /** + * Create directory buffer from parsed entries. + * + *

Serialization Format:

+ *
    + *
  1. Calculate buffer size: VarInt.encodedLength(count) + (count * DIR_ENTRY_BYTES)
    2. + *
  2. Allocate ByteBuffer with LITTLE_ENDIAN byte order
    3. + *
  3. Write VarInt count
    4. + *
  4. Write each directory entry: [fieldId:2][typeCode:1][offset:4]
    5. + *
  5. Flip buffer and return read-only view
    6. + *
+ * + * @param directory List of directory entries to serialize + * @return Read-only ByteBuffer containing serialized directory, or empty buffer on error + */ + private ByteBuffer createDirectoryBuffer(List directory) { + try { + int bufferSize = VarInt.encodedLength(directory.size()) + + (directory.size() * Constants.DIR_ENTRY_BYTES); + var buffer = ByteBuffer.allocate(bufferSize); + buffer.order(ByteOrder.LITTLE_ENDIAN); + + VarInt.encode(directory.size(), buffer); + for (var entry : directory) { + serializeDirectoryEntry(entry, buffer); + } + + buffer.flip(); + return buffer.asReadOnlyBuffer(); + } catch (Exception e) { + return ByteBuffer.allocate(0).asReadOnlyBuffer(); + } + } + + /** + * Serialize a single directory entry to the buffer. + * Format: [fieldId:2bytes][typeCode:1byte][offset:4bytes] + */ + private void serializeDirectoryEntry(DirectoryEntry entry, ByteBuffer buffer) { + buffer.putShort(entry.getId()); + buffer.put(entry.getTypeCode().getCode()); + buffer.putInt(entry.getOffset()); + } + + /** + * Deserialize a single directory entry from the buffer. + * Reads: [fieldId:2bytes][typeCode:1byte][offset:4bytes] + * + * @param buffer Buffer positioned at the start of a directory entry + * @return Parsed DirectoryEntry + * @throws ImprintException if buffer doesn't contain enough bytes + */ + private DirectoryEntry deserializeDirectoryEntry(ByteBuffer buffer) throws ImprintException { + if (buffer.remaining() < Constants.DIR_ENTRY_BYTES) + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, "Not enough bytes for directory entry"); + + short id = buffer.getShort(); + var typeCode = TypeCode.fromByte(buffer.get()); + int offset = buffer.getInt(); + + return new DirectoryEntry(id, typeCode, offset); + } +} \ No newline at end of file diff --git a/src/main/java/com/imprint/core/ImprintRecord.java b/src/main/java/com/imprint/core/ImprintRecord.java index da6b6e0..6abc9cf 100644 --- a/src/main/java/com/imprint/core/ImprintRecord.java +++ b/src/main/java/com/imprint/core/ImprintRecord.java @@ -11,507 +11,224 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; -import java.util.*; +import java.util.List; +import java.util.Map; +import java.util.Objects; /** - * An Imprint record containing a header, field directory, and payload. - * Uses ByteBuffer for zero-copy operations to achieve low latency. - * - *

This implementation uses lazy directory parsing for optimal single field access performance. - * The directory is only parsed when needed, and binary search is performed directly on raw bytes - * when possible.

- * - *

Performance Note: All ByteBuffers should be array-backed - * (hasArray() == true) for optimal zero-copy performance. Direct buffers - * may cause performance degradation.

+ * An Imprint record containing a header and buffer management. + * Delegates all buffer operations to ImprintBuffers for cleaner separation. */ @Getter public final class ImprintRecord { private final Header header; - private final ByteBuffer directoryBuffer; // Raw directory bytes - private final ByteBuffer payload; // Read-only view for zero-copy - - // Lazy-loaded directory state - private List parsedDirectory; - private boolean directoryParsed = false; - - // Cache for parsed directory count to avoid repeated VarInt decoding - private int directoryCount = -1; + private final ImprintBuffers buffers; /** - * Creates a new ImprintRecord with lazy directory parsing. - * - * @param header the record header - * @param directoryBuffer raw directory bytes (including count) - * @param payload the payload buffer. Should be array-backed for optimal performance. + * Creates a record from deserialized components. */ - private ImprintRecord(Header header, ByteBuffer directoryBuffer, ByteBuffer payload) { + private ImprintRecord(Header header, ImprintBuffers buffers) { this.header = Objects.requireNonNull(header, "Header cannot be null"); - this.directoryBuffer = directoryBuffer.asReadOnlyBuffer(); - this.payload = payload.asReadOnlyBuffer(); // Zero-copy read-only view + this.buffers = Objects.requireNonNull(buffers, "Buffers cannot be null"); } /** - * Creates a new ImprintRecord with pre-parsed directory (used by ImprintWriter). - * This constructor is used when the directory is already known and parsed. - * - * @param header the record header - * @param directory the parsed directory entries - * @param payload the payload buffer. Should be array-backed for optimal performance. + * Creates a record from pre-parsed directory (used by ImprintWriter). */ ImprintRecord(Header header, List directory, ByteBuffer payload) { this.header = Objects.requireNonNull(header, "Header cannot be null"); - this.parsedDirectory = Collections.unmodifiableList(Objects.requireNonNull(directory, "Directory cannot be null")); - this.directoryParsed = true; - this.directoryCount = directory.size(); - this.payload = payload.asReadOnlyBuffer(); - - // Create directory buffer for serialization compatibility - this.directoryBuffer = createDirectoryBuffer(directory); + this.buffers = new ImprintBuffers(directory, payload); } + // ========== FIELD ACCESS METHODS ========== + /** * Get a value by field ID, deserializing it on demand. * Returns null if the field is not found. - * Note: If the field exists and is an explicit NULL type, this will return Value.NullValue.INSTANCE - * - *

Performance Note: Accessing fields one-by-one is optimized for single field access. - * If you need to access many fields from the same record, consider calling getDirectory() first - * to parse the full directory once, then access fields normally.

*/ public Value getValue(int fieldId) throws ImprintException { - DirectoryEntry entry = findDirectoryEntry(fieldId); - if (entry == null) { + var entry = buffers.findDirectoryEntry(fieldId); + if (entry == null) return null; - } - return deserializeValue(entry.getTypeCode(), getFieldBufferFromEntry(entry)); + var fieldBuffer = buffers.getFieldBuffer(fieldId); + if (fieldBuffer == null) + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, "Failed to get buffer for field " + fieldId); + + return deserializeValue(entry.getTypeCode(), fieldBuffer); } /** - * Get the raw bytes for a field without deserializing. - * Returns a zero-copy ByteBuffer view, or null if field not found. + * Get raw bytes for a field without deserializing. */ public ByteBuffer getRawBytes(int fieldId) { try { - DirectoryEntry entry = findDirectoryEntry(fieldId); - if (entry == null) { - return null; - } - - return getFieldBufferFromEntry(entry).asReadOnlyBuffer(); + return buffers.getFieldBuffer(fieldId); } catch (ImprintException e) { return null; } } - /** - * Find a directory entry for the given field ID. - * Uses the most efficient method based on current state. - */ - private DirectoryEntry findDirectoryEntry(int fieldId) throws ImprintException { - if (directoryParsed) { - // Use parsed directory - int index = findDirectoryIndexInParsed(fieldId); - return index >= 0 ? parsedDirectory.get(index) : null; - } else { - // Use fast binary search on raw bytes - return findFieldEntryFast(fieldId); - } - } - - /** - * Fast binary search directly on raw directory bytes. - * This avoids parsing the entire directory for single field access. - */ - private DirectoryEntry findFieldEntryFast(int fieldId) throws ImprintException { - ByteBuffer searchBuffer = directoryBuffer.duplicate(); - searchBuffer.order(ByteOrder.LITTLE_ENDIAN); - - // Decode directory count (cache it to avoid repeated decoding) - if (directoryCount < 0) { - directoryCount = VarInt.decode(searchBuffer).getValue(); - } else { - // Skip past the VarInt count - VarInt.decode(searchBuffer); - } - - if (directoryCount == 0) { - return null; - } - - // Now searchBuffer.position() points to the first directory entry - int directoryStartPos = searchBuffer.position(); - - int low = 0; - int high = directoryCount - 1; - - while (low <= high) { - int mid = (low + high) >>> 1; - - // Calculate position of mid entry - int entryPos = directoryStartPos + (mid * Constants.DIR_ENTRY_BYTES); - - // Bounds check - if (entryPos + Constants.DIR_ENTRY_BYTES > searchBuffer.limit()) { - throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, - "Directory entry at position " + entryPos + " exceeds buffer limit " + searchBuffer.limit()); - } - - searchBuffer.position(entryPos); - short midFieldId = searchBuffer.getShort(); - - if (midFieldId < fieldId) { - low = mid + 1; - } else if (midFieldId > fieldId) { - high = mid - 1; - } else { - // Found it - read the complete entry - searchBuffer.position(entryPos); - return deserializeDirectoryEntry(searchBuffer); - } - } - - return null; // Not found - } - /** * Get the directory (parsing it if necessary). - * This maintains backward compatibility with existing code. - * - *

Performance Tip: If you plan to access many fields from this record, - * call this method first to parse the directory once, then use the field accessor methods. - * This is more efficient than accessing fields one-by-one when you need multiple fields.

*/ public List getDirectory() { - ensureDirectoryParsed(); - return parsedDirectory; + return buffers.getDirectory(); } - /** - * Get a ByteBuffer view of a field's data from a DirectoryEntry. - */ - private ByteBuffer getFieldBufferFromEntry(DirectoryEntry entry) throws ImprintException { - int startOffset = entry.getOffset(); - - // Find end offset - int endOffset; - if (directoryParsed) { - // Use parsed directory to find next entry - int entryIndex = findDirectoryIndexInParsed(entry.getId()); - endOffset = (entryIndex + 1 < parsedDirectory.size()) ? - parsedDirectory.get(entryIndex + 1).getOffset() : payload.limit(); - } else { - // Calculate end offset by finding the next field in the directory - endOffset = findNextOffsetInRawDirectory(entry.getId()); - } + // ========== TYPED GETTERS ========== - if (startOffset < 0 || endOffset < 0 || startOffset > payload.limit() || - endOffset > payload.limit() || startOffset > endOffset) { - throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, - "Invalid field buffer range: start=" + startOffset + ", end=" + endOffset + - ", payloadLimit=" + payload.limit()); - } - - var fieldBuffer = payload.duplicate(); - fieldBuffer.position(startOffset).limit(endOffset); - return fieldBuffer; + public boolean getBoolean(int fieldId) throws ImprintException { + return getTypedValueOrThrow(fieldId, TypeCode.BOOL, Value.BoolValue.class, "boolean").getValue(); } - /** - * Find the next field's offset by scanning the raw directory. - * This is used when the directory isn't fully parsed yet. - */ - private int findNextOffsetInRawDirectory(int currentFieldId) throws ImprintException { - ByteBuffer scanBuffer = directoryBuffer.duplicate(); - scanBuffer.order(ByteOrder.LITTLE_ENDIAN); - - // Get directory count - int count = (directoryCount >= 0) ? directoryCount : VarInt.decode(scanBuffer).getValue(); - if (count == 0) { - return payload.limit(); - } - - // Skip past count if we just decoded it - if (directoryCount < 0) { - // VarInt.decode already advanced the position - } else { - VarInt.decode(scanBuffer); // Skip past the count - } - - int directoryStartPos = scanBuffer.position(); - - for (int i = 0; i < count; i++) { - int entryPos = directoryStartPos + (i * Constants.DIR_ENTRY_BYTES); - - // Bounds check - if (entryPos + Constants.DIR_ENTRY_BYTES > scanBuffer.limit()) { - return payload.limit(); - } - - scanBuffer.position(entryPos); - short fieldId = scanBuffer.getShort(); - scanBuffer.get(); // skip type - int offset = scanBuffer.getInt(); + public int getInt32(int fieldId) throws ImprintException { + return getTypedValueOrThrow(fieldId, TypeCode.INT32, Value.Int32Value.class, "int32").getValue(); + } - if (fieldId > currentFieldId) { - return offset; // Found next field's offset - } - } + public long getInt64(int fieldId) throws ImprintException { + return getTypedValueOrThrow(fieldId, TypeCode.INT64, Value.Int64Value.class, "int64").getValue(); + } - return payload.limit(); // No next field, use payload end + public float getFloat32(int fieldId) throws ImprintException { + return getTypedValueOrThrow(fieldId, TypeCode.FLOAT32, Value.Float32Value.class, "float32").getValue(); } - /** - * Ensure the directory is fully parsed (thread-safe). - */ - private synchronized void ensureDirectoryParsed() { - if (directoryParsed) { - return; - } + public double getFloat64(int fieldId) throws ImprintException { + return getTypedValueOrThrow(fieldId, TypeCode.FLOAT64, Value.Float64Value.class, "float64").getValue(); + } - try { - ByteBuffer parseBuffer = directoryBuffer.duplicate(); - parseBuffer.order(ByteOrder.LITTLE_ENDIAN); + public String getString(int fieldId) throws ImprintException { + var value = getValidatedValue(fieldId, "STRING"); + if (value instanceof Value.StringValue) + return ((Value.StringValue) value).getValue(); + if (value instanceof Value.StringBufferValue) + return ((Value.StringBufferValue) value).getValue(); + throw new ImprintException(ErrorType.TYPE_MISMATCH, "Field " + fieldId + " is not a STRING"); + } - VarInt.DecodeResult countResult = VarInt.decode(parseBuffer); - int count = countResult.getValue(); - this.directoryCount = count; // Cache the count + public byte[] getBytes(int fieldId) throws ImprintException { + var value = getValidatedValue(fieldId, "BYTES"); + if (value instanceof Value.BytesValue) + return ((Value.BytesValue) value).getValue(); + if (value instanceof Value.BytesBufferValue) + return ((Value.BytesBufferValue) value).getValue(); + throw new ImprintException(ErrorType.TYPE_MISMATCH, "Field " + fieldId + " is not BYTES"); + } - List directory = new ArrayList<>(count); - for (int i = 0; i < count; i++) { - directory.add(deserializeDirectoryEntry(parseBuffer)); - } + public List getArray(int fieldId) throws ImprintException { + return getTypedValueOrThrow(fieldId, TypeCode.ARRAY, Value.ArrayValue.class, "ARRAY").getValue(); + } - this.parsedDirectory = Collections.unmodifiableList(directory); - this.directoryParsed = true; - } catch (ImprintException e) { - throw new RuntimeException("Failed to parse directory", e); - } + public Map getMap(int fieldId) throws ImprintException { + return getTypedValueOrThrow(fieldId, TypeCode.MAP, Value.MapValue.class, "MAP").getValue(); } - /** - * Creates a directory buffer from parsed directory entries. - * This is used when creating records with pre-parsed directories (e.g., from ImprintWriter). - */ - private ByteBuffer createDirectoryBuffer(List directory) { - try { - int bufferSize = VarInt.encodedLength(directory.size()) + (directory.size() * Constants.DIR_ENTRY_BYTES); - ByteBuffer buffer = ByteBuffer.allocate(bufferSize); - buffer.order(ByteOrder.LITTLE_ENDIAN); - - // Write directory count - VarInt.encode(directory.size(), buffer); - - // Write directory entries - for (DirectoryEntry entry : directory) { - serializeDirectoryEntry(entry, buffer); - } - - buffer.flip(); - return buffer.asReadOnlyBuffer(); - } catch (Exception e) { - // Fallback to empty buffer if creation fails - return ByteBuffer.allocate(0).asReadOnlyBuffer(); - } + public ImprintRecord getRow(int fieldId) throws ImprintException { + return getTypedValueOrThrow(fieldId, TypeCode.ROW, Value.RowValue.class, "ROW").getValue(); } + // ========== SERIALIZATION ========== + /** * Serialize this record to a ByteBuffer. - * The returned buffer will be array-backed. */ public ByteBuffer serializeToBuffer() { - // Ensure directory is parsed for serialization - ensureDirectoryParsed(); - var buffer = ByteBuffer.allocate(estimateSerializedSize()); buffer.order(ByteOrder.LITTLE_ENDIAN); // Write header serializeHeader(buffer); - // Write directory (always present) - VarInt.encode(parsedDirectory.size(), buffer); - for (var entry : parsedDirectory) { - serializeDirectoryEntry(entry, buffer); - } + // Write directory + var directoryBuffer = buffers.serializeDirectory(); + buffer.put(directoryBuffer); - // Write payload (shallow copy only) + // Write payload + var payload = buffers.getPayload(); var payloadCopy = payload.duplicate(); buffer.put(payloadCopy); - // Prepare buffer for reading buffer.flip(); return buffer; } - /** - * Create a fluent builder for constructing ImprintRecord instances. - */ + public int estimateSerializedSize() { + int size = Constants.HEADER_BYTES; // header + size += buffers.serializeDirectory().remaining(); // directory + size += buffers.getPayload().remaining(); // payload + return size; + } + + // ========== STATIC FACTORY METHODS ========== + public static ImprintRecordBuilder builder(SchemaId schemaId) { return new ImprintRecordBuilder(schemaId); } - /** - * Create a fluent builder for constructing ImprintRecord instances. - */ - @SuppressWarnings("unused") public static ImprintRecordBuilder builder(int fieldspaceId, int schemaHash) { return new ImprintRecordBuilder(new SchemaId(fieldspaceId, schemaHash)); } - /** - * Deserialize a record from bytes through an array backed ByteBuffer. - */ public static ImprintRecord deserialize(byte[] bytes) throws ImprintException { return deserialize(ByteBuffer.wrap(bytes)); } - /** - * Deserialize a record from a ByteBuffer with lazy directory parsing. - * - * @param buffer the buffer to deserialize from. Must be array-backed - * (buffer.hasArray() == true) for optimal zero-copy performance. - */ public static ImprintRecord deserialize(ByteBuffer buffer) throws ImprintException { buffer = buffer.duplicate().order(ByteOrder.LITTLE_ENDIAN); // Read header var header = deserializeHeader(buffer); - // Read directory count but don't parse entries yet + // Calculate directory size int directoryStartPos = buffer.position(); - VarInt.DecodeResult countResult = VarInt.decode(buffer); + var countResult = VarInt.decode(buffer); int directoryCount = countResult.getValue(); - - // Calculate directory buffer (includes count + all entries) int directorySize = countResult.getBytesRead() + (directoryCount * Constants.DIR_ENTRY_BYTES); - buffer.position(directoryStartPos); // Reset to include count in directory buffer + // Create directory buffer + buffer.position(directoryStartPos); var directoryBuffer = buffer.slice(); directoryBuffer.limit(directorySize); - // Advance buffer past directory + // Advance past directory buffer.position(buffer.position() + directorySize); - // Read payload as ByteBuffer slice for zero-copy + // Create payload buffer var payload = buffer.slice(); payload.limit(header.getPayloadSize()); - return new ImprintRecord(header, directoryBuffer, payload); - } - - /** - * Binary search for field ID in parsed directory. - * Returns the index of the field if found, or a negative value if not found. - */ - private int findDirectoryIndexInParsed(int fieldId) { - if (!directoryParsed) { - return -1; - } - - int low = 0; - int high = parsedDirectory.size() - 1; - - while (low <= high) { - int mid = (low + high) >>> 1; - int midFieldId = parsedDirectory.get(mid).getId(); - - if (midFieldId < fieldId) { - low = mid + 1; - } else if (midFieldId > fieldId) { - high = mid - 1; - } else { - return mid; // field found - } - } - return -(low + 1); // field not found, return insertion point - } - - public int estimateSerializedSize() { - int size = Constants.HEADER_BYTES; // header - size += VarInt.encodedLength(getDirectoryCount()); // directory count - size += getDirectoryCount() * Constants.DIR_ENTRY_BYTES; // directory entries - size += payload.remaining(); // payload - return size; - } - - private int getDirectoryCount() { - if (directoryCount >= 0) { - return directoryCount; - } - if (directoryParsed) { - return parsedDirectory.size(); - } - // Last resort: decode from buffer - try { - ByteBuffer countBuffer = directoryBuffer.duplicate(); - return VarInt.decode(countBuffer).getValue(); - } catch (Exception e) { - return 0; - } - } - - // ===== EXISTING HELPER METHODS (unchanged) ===== + // Create buffers wrapper + var buffers = new ImprintBuffers(directoryBuffer, payload); - private void serializeHeader(ByteBuffer buffer) { - buffer.put(Constants.MAGIC); - buffer.put(Constants.VERSION); - buffer.put(header.getFlags().getValue()); - buffer.putInt(header.getSchemaId().getFieldSpaceId()); - buffer.putInt(header.getSchemaId().getSchemaHash()); - buffer.putInt(header.getPayloadSize()); + return new ImprintRecord(header, buffers); } - private static Header deserializeHeader(ByteBuffer buffer) throws ImprintException { - if (buffer.remaining() < Constants.HEADER_BYTES) { - throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, - "Not enough bytes for header"); - } - - byte magic = buffer.get(); - if (magic != Constants.MAGIC) { - throw new ImprintException(ErrorType.INVALID_MAGIC, - "Invalid magic byte: expected 0x" + Integer.toHexString(Constants.MAGIC) + - ", got 0x" + Integer.toHexString(magic & 0xFF)); - } + // ========== PRIVATE HELPER METHODS ========== - byte version = buffer.get(); - if (version != Constants.VERSION) { - throw new ImprintException(ErrorType.UNSUPPORTED_VERSION, - "Unsupported version: " + version); - } - - var flags = new Flags(buffer.get()); - int fieldspaceId = buffer.getInt(); - int schemaHash = buffer.getInt(); - int payloadSize = buffer.getInt(); - - return new Header(flags, new SchemaId(fieldspaceId, schemaHash), payloadSize); - } - - private void serializeDirectoryEntry(DirectoryEntry entry, ByteBuffer buffer) { - buffer.putShort(entry.getId()); - buffer.put(entry.getTypeCode().getCode()); - buffer.putInt(entry.getOffset()); + /** + * Get and validate a value exists and is not null. + */ + private Value getValidatedValue(int fieldId, String typeName) throws ImprintException { + var value = getValue(fieldId); + if (value == null) + throw new ImprintException(ErrorType.FIELD_NOT_FOUND, "Field " + fieldId + " not found"); + if (value.getTypeCode() == TypeCode.NULL) + throw new ImprintException(ErrorType.TYPE_MISMATCH, "Field " + fieldId + " is NULL, cannot retrieve as " + typeName); + return value; } - private static DirectoryEntry deserializeDirectoryEntry(ByteBuffer buffer) throws ImprintException { - if (buffer.remaining() < Constants.DIR_ENTRY_BYTES) { - throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, - "Not enough bytes for directory entry"); - } - - short id = buffer.getShort(); - var typeCode = TypeCode.fromByte(buffer.get()); - int offset = buffer.getInt(); - - return new DirectoryEntry(id, typeCode, offset); + private T getTypedValueOrThrow(int fieldId, TypeCode expectedTypeCode, Class expectedValueClass, String expectedTypeName) + throws ImprintException { + var value = getValidatedValue(fieldId, expectedTypeName); + if (value.getTypeCode() == expectedTypeCode && expectedValueClass.isInstance(value)) + return expectedValueClass.cast(value); + throw new ImprintException(ErrorType.TYPE_MISMATCH, "Field " + fieldId + " is of type " + value.getTypeCode() + ", expected " + expectedTypeName); } private Value deserializeValue(TypeCode typeCode, ByteBuffer buffer) throws ImprintException { - var valueSpecificBuffer = buffer.duplicate(); - valueSpecificBuffer.order(ByteOrder.LITTLE_ENDIAN); + var valueBuffer = buffer.duplicate(); + valueBuffer.order(ByteOrder.LITTLE_ENDIAN); switch (typeCode) { case NULL: @@ -524,120 +241,51 @@ private Value deserializeValue(TypeCode typeCode, ByteBuffer buffer) throws Impr case STRING: case ARRAY: case MAP: - return typeCode.getHandler().deserialize(valueSpecificBuffer); + return typeCode.getHandler().deserialize(valueBuffer); case ROW: - var nestedRecord = deserialize(valueSpecificBuffer); + var nestedRecord = deserialize(valueBuffer); return Value.fromRow(nestedRecord); - default: throw new ImprintException(ErrorType.INVALID_TYPE_CODE, "Unknown type code: " + typeCode); } } - // ===== TYPE-SPECIFIC GETTERS (unchanged API, improved performance) ===== - - private T getTypedValueOrThrow(int fieldId, TypeCode expectedTypeCode, Class expectedValueClass, String expectedTypeName) throws ImprintException { - var value = getValue(fieldId); - - if (value == null) { - throw new ImprintException(ErrorType.FIELD_NOT_FOUND, - "Field " + fieldId + " not found, cannot retrieve as " + expectedTypeName + "."); - } - - if (value.getTypeCode() == TypeCode.NULL) { - throw new ImprintException(ErrorType.TYPE_MISMATCH, - "Field " + fieldId + " is NULL, cannot retrieve as " + expectedTypeName + "."); - } - - if (value.getTypeCode() == expectedTypeCode && expectedValueClass.isInstance(value)) { - return expectedValueClass.cast(value); - } - - throw new ImprintException(ErrorType.TYPE_MISMATCH, - "Field " + fieldId + " is of type " + value.getTypeCode() + ", expected " + expectedTypeName + "."); - } - - public boolean getBoolean(int fieldId) throws ImprintException { - return getTypedValueOrThrow(fieldId, TypeCode.BOOL, Value.BoolValue.class, "boolean").getValue(); - } - - public int getInt32(int fieldId) throws ImprintException { - return getTypedValueOrThrow(fieldId, TypeCode.INT32, Value.Int32Value.class, "int32").getValue(); - } - - public long getInt64(int fieldId) throws ImprintException { - return getTypedValueOrThrow(fieldId, TypeCode.INT64, Value.Int64Value.class, "int64").getValue(); - } - - public float getFloat32(int fieldId) throws ImprintException { - return getTypedValueOrThrow(fieldId, TypeCode.FLOAT32, Value.Float32Value.class, "float32").getValue(); - } - - public double getFloat64(int fieldId) throws ImprintException { - return getTypedValueOrThrow(fieldId, TypeCode.FLOAT64, Value.Float64Value.class, "float64").getValue(); + private void serializeHeader(ByteBuffer buffer) { + buffer.put(Constants.MAGIC); + buffer.put(Constants.VERSION); + buffer.put(header.getFlags().getValue()); + buffer.putInt(header.getSchemaId().getFieldSpaceId()); + buffer.putInt(header.getSchemaId().getSchemaHash()); + buffer.putInt(header.getPayloadSize()); } - public String getString(int fieldId) throws ImprintException { - var value = getValue(fieldId); - - if (value == null) { - throw new ImprintException(ErrorType.FIELD_NOT_FOUND, - "Field " + fieldId + " not found, cannot retrieve as String."); - } - if (value.getTypeCode() == TypeCode.NULL) { - throw new ImprintException(ErrorType.TYPE_MISMATCH, - "Field " + fieldId + " is NULL, cannot retrieve as String."); - } - - if (value instanceof Value.StringValue) { - return ((Value.StringValue) value).getValue(); - } - if (value instanceof Value.StringBufferValue) { - return ((Value.StringBufferValue) value).getValue(); - } + private static Header deserializeHeader(ByteBuffer buffer) throws ImprintException { + if (buffer.remaining() < Constants.HEADER_BYTES) + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, "Not enough bytes for header"); - throw new ImprintException(ErrorType.TYPE_MISMATCH, - "Field " + fieldId + " is of type " + value.getTypeCode() + ", expected STRING."); - } - public byte[] getBytes(int fieldId) throws ImprintException { - Value value = getValue(fieldId); - - if (value == null) { - throw new ImprintException(ErrorType.FIELD_NOT_FOUND, - "Field " + fieldId + " not found, cannot retrieve as byte[]."); - } - if (value.getTypeCode() == TypeCode.NULL) { - throw new ImprintException(ErrorType.TYPE_MISMATCH, - "Field " + fieldId + " is NULL, cannot retrieve as byte[]."); + byte magic = buffer.get(); + if (magic != Constants.MAGIC) { + throw new ImprintException(ErrorType.INVALID_MAGIC, "Invalid magic byte: expected 0x" + Integer.toHexString(Constants.MAGIC) + + ", got 0x" + Integer.toHexString(magic & 0xFF)); } - if (value instanceof Value.BytesValue) { - return ((Value.BytesValue) value).getValue(); - } - if (value instanceof Value.BytesBufferValue) { - return ((Value.BytesBufferValue) value).getValue(); + byte version = buffer.get(); + if (version != Constants.VERSION) { + throw new ImprintException(ErrorType.UNSUPPORTED_VERSION, "Unsupported version: " + version); } - throw new ImprintException(ErrorType.TYPE_MISMATCH, - "Field " + fieldId + " is of type " + value.getTypeCode() + ", expected BYTES."); - } - - public List getArray(int fieldId) throws ImprintException { - return getTypedValueOrThrow(fieldId, TypeCode.ARRAY, Value.ArrayValue.class, "ARRAY").getValue(); - } - - public Map getMap(int fieldId) throws ImprintException { - return getTypedValueOrThrow(fieldId, TypeCode.MAP, Value.MapValue.class, "MAP").getValue(); - } + var flags = new Flags(buffer.get()); + int fieldSpaceId = buffer.getInt(); + int schemaHash = buffer.getInt(); + int payloadSize = buffer.getInt(); - public ImprintRecord getRow(int fieldId) throws ImprintException { - return getTypedValueOrThrow(fieldId, TypeCode.ROW, Value.RowValue.class, "ROW").getValue(); + return new Header(flags, new SchemaId(fieldSpaceId, schemaHash), payloadSize); } @Override public String toString() { - return String.format("ImprintRecord{header=%s, directorySize=%d, payloadSize=%d, directoryParsed=%s}", - header, getDirectoryCount(), payload.remaining(), directoryParsed); + return String.format("ImprintRecord{header=%s, directorySize=%d, payloadSize=%d}", + header, buffers.getDirectoryCount(), buffers.getPayload().remaining()); } } \ No newline at end of file diff --git a/src/main/java/com/imprint/util/VarInt.java b/src/main/java/com/imprint/util/VarInt.java index 75bd132..f43683b 100644 --- a/src/main/java/com/imprint/util/VarInt.java +++ b/src/main/java/com/imprint/util/VarInt.java @@ -16,12 +16,28 @@ */ @UtilityClass public final class VarInt { - + private static final byte CONTINUATION_BIT = (byte) 0x80; private static final byte SEGMENT_BITS = 0x7f; private static final int MAX_VARINT_LEN = 5; // Enough for u32 - - + + // Simple cache for values 0-1023 + private static final int CACHE_SIZE = 1024; + private static final int[] ENCODED_LENGTHS = new int[CACHE_SIZE]; + + static { + // Pre-compute encoded lengths for cached values + for (int i = 0; i < CACHE_SIZE; i++) { + long val = Integer.toUnsignedLong(i); + int length = 1; + while (val >= 0x80) { + val >>>= 7; + length++; + } + ENCODED_LENGTHS[i] = length; + } + } + /** * Encode a 32-bit unsigned integer as a VarInt into the given ByteBuffer. * @param value the value to encode (treated as unsigned) @@ -30,7 +46,7 @@ public final class VarInt { public static void encode(int value, ByteBuffer buffer) { // Convert to unsigned long for proper bit manipulation long val = Integer.toUnsignedLong(value); - + // Encode at least one byte, then continue while value has more bits do { byte b = (byte) (val & SEGMENT_BITS); @@ -41,8 +57,7 @@ public static void encode(int value, ByteBuffer buffer) { buffer.put(b); } while (val != 0); } - - + /** * Decode a VarInt from a ByteBuffer. * @param buffer the buffer to decode from @@ -53,55 +68,55 @@ public static DecodeResult decode(ByteBuffer buffer) throws ImprintException { long result = 0; int shift = 0; int bytesRead = 0; - + while (true) { if (bytesRead >= MAX_VARINT_LEN) { throw new ImprintException(ErrorType.MALFORMED_VARINT, "VarInt too long"); } if (!buffer.hasRemaining()) { - throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, - "Unexpected end of data while reading VarInt"); + throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, + "Unexpected end of data while reading VarInt"); } - + byte b = buffer.get(); bytesRead++; - + // Check if adding these 7 bits would overflow long segment = b & SEGMENT_BITS; if (shift >= 32 || (shift == 28 && segment > 0xF)) { throw new ImprintException(ErrorType.MALFORMED_VARINT, "VarInt overflow"); } - + // Add the bottom 7 bits to the result result |= segment << shift; - + // If the high bit is not set, this is the last byte if ((b & CONTINUATION_BIT) == 0) { break; } - + shift += 7; } - + return new DecodeResult((int) result, bytesRead); } - + /** * Calculate the number of bytes needed to encode the given value as a VarInt. * @param value the value to encode (treated as unsigned) * @return the number of bytes needed */ public static int encodedLength(int value) { - // Convert to unsigned long for proper bit manipulation + if (value >= 0 && value < CACHE_SIZE) { + return ENCODED_LENGTHS[value]; + } + long val = Integer.toUnsignedLong(value); int length = 1; - - // Count additional bytes needed for values >= 128 while (val >= 0x80) { val >>>= 7; length++; } - return length; } From 73eade6f6c55af6d75d738a0882cc924b71cc825 Mon Sep 17 00:00:00 2001 From: expand3d <> Date: Thu, 5 Jun 2025 22:12:44 -0400 Subject: [PATCH 33/34] remove extra comments --- .../com/imprint/benchmark/ComparisonBenchmark.java | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java b/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java index dd62457..a666ffe 100644 --- a/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java +++ b/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java @@ -133,19 +133,7 @@ public void serializeFlatBuffers(Blackhole bh) { bh.consume(result); } - // ===== PARTIAL DESERIALIZATION (SETUP ONLY) ===== -// These benchmarks measure the cost of preparing a record for field access, -// not the cost of accessing the actual data. This is important because -// -// 1. Imprint: Only parses header + stores raw directory bytes -// 2. FlatBuffers: Only wraps the buffer with minimal validation -// 3. Others (eager): Parse and construct all field objects upfront -// -// This comparison shows the advantage of lazy loading approaches when you -// only need to access a subset of fields. In real streaming workloads, -// records are often filtered/routed based on just a few key fields. -// -// For a fair "full deserialization" comparison, see FULL DESERIALIZATION BENCHMARKS. + // ===== SETUP ONLY ===== @Benchmark public void deserializeSetupImprint(Blackhole bh) throws Exception { From 02866d5f9fe9abfcd1c543d224db0410098324ad Mon Sep 17 00:00:00 2001 From: expand3d <> Date: Thu, 5 Jun 2025 22:17:01 -0400 Subject: [PATCH 34/34] remove extra comments --- src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java b/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java index a666ffe..4d9c01c 100644 --- a/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java +++ b/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java @@ -259,19 +259,19 @@ public void singleFieldAccessMessagePack(Blackhole bh) throws Exception { @Benchmark public void singleFieldAccessAvro(Blackhole bh) throws Exception { GenericRecord record = deserializeWithAvro(avroBytes); - bh.consume(record.get("extraData4")); // Accessing field near end + bh.consume(record.get("extraData4")); } @Benchmark public void singleFieldAccessProtobuf(Blackhole bh) throws Exception { TestRecordProto.TestRecord record = TestRecordProto.TestRecord.parseFrom(protobufBytes); - bh.consume(record.getExtraData(4)); // Accessing field near end + bh.consume(record.getExtraData(4)); } @Benchmark public void singleFieldAccessFlatBuffers(Blackhole bh) { TestRecordFB record = TestRecordFB.getRootAsTestRecordFB(flatbuffersBytes.duplicate()); - bh.consume(record.extraData(4)); // Accessing field near end - zero copy! + bh.consume(record.extraData(4)); } // ===== SIZE COMPARISON =====