diff --git a/build.gradle b/build.gradle index b5f9126..c5b0361 100644 --- a/build.gradle +++ b/build.gradle @@ -52,6 +52,7 @@ dependencies { jmhImplementation 'org.msgpack:jackson-dataformat-msgpack:0.9.8' jmhImplementation 'org.apache.thrift:libthrift:0.19.0' jmhImplementation 'javax.annotation:javax.annotation-api:1.3.2' + jmhImplementation 'net.openhft:chronicle-wire:2.25ea5' } protobuf { @@ -236,7 +237,13 @@ jmh { '-XX:+UseG1GC', '-Xmx2g', '-XX:+UnlockExperimentalVMOptions', - '-XX:+UseJVMCICompiler' + '-XX:+UseJVMCICompiler', + '--illegal-access=permit', + '--add-exports=java.base/jdk.internal.ref=ALL-UNNAMED', + '--add-opens=java.base/java.lang=ALL-UNNAMED', + '--add-opens=java.base/java.lang.reflect=ALL-UNNAMED', + '--add-opens=java.base/java.util=ALL-UNNAMED', + '--add-opens=java.base/sun.nio.ch=ALL-UNNAMED' ] } diff --git a/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java b/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java index d8fbcde..925c48a 100644 --- a/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java +++ b/src/jmh/java/com/imprint/benchmark/ComparisonBenchmark.java @@ -16,8 +16,15 @@ @OutputTimeUnit(TimeUnit.NANOSECONDS) @State(Scope.Benchmark) @Warmup(iterations = 3, time = 1) -@Measurement(iterations = 25, time = 1) -@Fork(value = 1, jvmArgs = {"-Xms4g", "-Xmx4g"}) +@Measurement(iterations = 10, time = 1) +@Fork(value = 1, jvmArgs = {"-Xms4g", "-Xmx4g", + "--illegal-access=permit", + "--add-exports=java.base/jdk.internal.ref=ALL-UNNAMED", + "--add-opens=java.base/java.lang=ALL-UNNAMED", + "--add-opens=java.base/java.lang.reflect=ALL-UNNAMED", + "--add-opens=java.base/java.util=ALL-UNNAMED", + "--add-opens=java.base/sun.nio.ch=ALL-UNNAMED", + "-Dimprint.buffer.bounds.check=false"}) public class ComparisonBenchmark { private static final List FRAMEWORKS = List.of( @@ -28,9 +35,10 @@ public class ComparisonBenchmark { new AvroSerializingBenchmark(), new ThriftSerializingBenchmark(), new KryoSerializingBenchmark(), - new MessagePackSerializingBenchmark()); + new MessagePackSerializingBenchmark(), + new ChronicleWireSerializingBenchmark()); - @Param({"Imprint"}) + @Param({"Imprint", "Jackson-JSON", "Protobuf", "FlatBuffers", "Avro-Generic", "Thrift", "Kryo", "MessagePack", "Chronicle-Wire"}) public String framework; private SerializingBenchmark serializingBenchmark; @@ -51,27 +59,27 @@ public void setup() { } @Benchmark - public void serialize(Blackhole bh) { + public void serializeRecord(Blackhole bh) { serializingBenchmark.serialize(bh); } - //@Benchmark - public void deserialize(Blackhole bh) { + @Benchmark + public void deserializeRecord(Blackhole bh) { serializingBenchmark.deserialize(bh); } - //@Benchmark - public void projectAndSerialize(Blackhole bh) { + @Benchmark + public void projectThenSerialize(Blackhole bh) { serializingBenchmark.projectAndSerialize(bh); } - //@Benchmark - public void mergeAndSerialize(Blackhole bh) { + @Benchmark + public void mergeThenSerialize(Blackhole bh) { serializingBenchmark.mergeAndSerialize(bh); } - //@Benchmark - public void accessField(Blackhole bh) { + @Benchmark + public void accessSingleField(Blackhole bh) { serializingBenchmark.accessField(bh); } diff --git a/src/jmh/java/com/imprint/benchmark/ImprintDetailedBenchmark.java b/src/jmh/java/com/imprint/benchmark/ImprintDetailedBenchmark.java index c9d4514..3d47077 100644 --- a/src/jmh/java/com/imprint/benchmark/ImprintDetailedBenchmark.java +++ b/src/jmh/java/com/imprint/benchmark/ImprintDetailedBenchmark.java @@ -26,18 +26,12 @@ public class ImprintDetailedBenchmark { private DataGenerator.TestRecord testData; private ImprintRecordBuilder preBuiltBuilder; - private ImprintRecord preBuiltRecord; private static final SchemaId SCHEMA_ID = new SchemaId(1, 1); @Setup(Level.Trial) public void setup() { testData = DataGenerator.createTestRecord(); - try { - preBuiltBuilder = buildRecord(testData); - preBuiltRecord = preBuiltBuilder.build(); - } catch (ImprintException e) { - throw new RuntimeException(e); - } + preBuiltBuilder = buildRecord(testData); } private ImprintRecordBuilder buildRecord(DataGenerator.TestRecord pojo) { @@ -74,12 +68,6 @@ public void buildToBuffer(Blackhole bh) { } } - @Benchmark - public void serializeToBuffer(Blackhole bh) { - // Benchmark: Record → Bytes (just buffer copy) - bh.consume(preBuiltRecord.serializeToBuffer()); - } - @Benchmark public void fullPipeline(Blackhole bh) { // Benchmark: POJO → Builder → Bytes (complete pipeline) @@ -97,7 +85,6 @@ public static void main(String[] args) throws RunnerException { .mode(Mode.AverageTime) .timeUnit(TimeUnit.NANOSECONDS) .build(); - new Runner(opt).run(); } } \ No newline at end of file diff --git a/src/jmh/java/com/imprint/benchmark/serializers/AvroSerializingBenchmark.java b/src/jmh/java/com/imprint/benchmark/serializers/AvroSerializingBenchmark.java index f3e5b8a..3cc199e 100644 --- a/src/jmh/java/com/imprint/benchmark/serializers/AvroSerializingBenchmark.java +++ b/src/jmh/java/com/imprint/benchmark/serializers/AvroSerializingBenchmark.java @@ -11,6 +11,7 @@ import java.io.ByteArrayOutputStream; import java.nio.ByteBuffer; +import java.util.List; public class AvroSerializingBenchmark extends AbstractSerializingBenchmark { @@ -101,7 +102,7 @@ public void projectAndSerialize(Blackhole bh) { GenericRecord projected = new GenericData.Record(projectedSchema); projected.put("id", original.get("id")); projected.put("timestamp", original.get("timestamp")); - projected.put("tags", ((java.util.List)original.get("tags")).subList(0, 5)); + projected.put("tags", ((List)original.get("tags")).subList(0, 5)); BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(out, null); projectedWriter.write(projected, encoder); @@ -132,19 +133,6 @@ public void mergeAndSerialize(Blackhole bh) { bh.consume(buildBytes(merged)); } - private GenericRecord buildAvroRecord(DataGenerator.TestRecord pojo) { - GenericRecord record = new GenericData.Record(schema); - record.put("id", pojo.id); - record.put("timestamp", pojo.timestamp); - record.put("flags", pojo.flags); - record.put("active", pojo.active); - record.put("value", pojo.value); - record.put("data", ByteBuffer.wrap(pojo.data)); - record.put("tags", pojo.tags); - record.put("metadata", pojo.metadata); - return record; - } - private GenericRecord buildAvroRecordFromBytes(byte[] bytes) { try { BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(bytes, null); diff --git a/src/jmh/java/com/imprint/benchmark/serializers/ChronicleWireSerializingBenchmark.java b/src/jmh/java/com/imprint/benchmark/serializers/ChronicleWireSerializingBenchmark.java new file mode 100644 index 0000000..0c2f00f --- /dev/null +++ b/src/jmh/java/com/imprint/benchmark/serializers/ChronicleWireSerializingBenchmark.java @@ -0,0 +1,163 @@ +package com.imprint.benchmark.serializers; + +import com.imprint.benchmark.DataGenerator; +import net.openhft.chronicle.bytes.Bytes; +import net.openhft.chronicle.wire.BinaryWire; +import net.openhft.chronicle.wire.Wire; +import net.openhft.chronicle.wire.WireType; +import org.openjdk.jmh.infra.Blackhole; + +import java.util.List; +import java.util.Map; + +public class ChronicleWireSerializingBenchmark extends AbstractSerializingBenchmark { + + private byte[] serializedRecord1; + + public ChronicleWireSerializingBenchmark() { + super("Chronicle-Wire"); + } + + @Override + public void setup(DataGenerator.TestRecord record1, DataGenerator.TestRecord record2) { + super.setup(record1, record2); + + // Pre-serialize for deserialize benchmarks + this.serializedRecord1 = serializeRecord(record1); + byte[] serializedRecord2 = serializeRecord(record2); + } + + @Override + public void serialize(Blackhole bh) { + byte[] serialized = serializeRecord(testData); + bh.consume(serialized); + } + + @Override + public void deserialize(Blackhole bh) { + DataGenerator.TestRecord deserialized = deserializeRecord(serializedRecord1); + bh.consume(deserialized); + } + + @Override + public void projectAndSerialize(Blackhole bh) { + // Full round trip: deserialize, project to a new object, re-serialize + DataGenerator.TestRecord original = deserializeRecord(serializedRecord1); + + // Simulate projection by creating projected object + DataGenerator.ProjectedRecord projected = new DataGenerator.ProjectedRecord(); + projected.id = original.id; + projected.timestamp = original.timestamp; + projected.tags = original.tags.subList(0, Math.min(5, original.tags.size())); + + byte[] serialized = serializeProjectedRecord(projected); + bh.consume(serialized); + } + + @Override + public void mergeAndSerialize(Blackhole bh) { + // Deserialize both records, merge them, and serialize the result + DataGenerator.TestRecord r1 = deserializeRecord(serializedRecord1); + DataGenerator.TestRecord r2 = testData2; // Use second record directly + + // Create merged record following the pattern from other implementations + DataGenerator.TestRecord merged = new DataGenerator.TestRecord(); + merged.id = r1.id; + merged.timestamp = System.currentTimeMillis(); // new value + merged.flags = r1.flags; + merged.active = false; // new value + merged.value = r1.value; + merged.data = r1.data; + merged.tags = r2.tags; + merged.metadata = r2.metadata; + + byte[] serialized = serializeRecord(merged); + bh.consume(serialized); + } + + @Override + public void accessField(Blackhole bh) { + DataGenerator.TestRecord deserialized = deserializeRecord(serializedRecord1); + long timestamp = deserialized.timestamp; + bh.consume(timestamp); + } + + private byte[] serializeRecord(DataGenerator.TestRecord record) { + Bytes bytes = Bytes.elasticByteBuffer(); + try { + Wire wire = WireType.BINARY.apply(bytes); + + wire.writeDocument(false, w -> { + if (record.id != null) w.write("id").text(record.id); + w.write("timestamp").int64(record.timestamp) + .write("flags").int32(record.flags) + .write("active").bool(record.active) + .write("value").float64(record.value); + + if (record.data != null) { + w.write("data").bytes(record.data); + } + if (record.tags != null) { + w.write("tags").object(record.tags); + } + if (record.metadata != null) { + w.write("metadata").marshallable(m -> { + for (Map.Entry entry : record.metadata.entrySet()) { + m.write(entry.getKey()).text(entry.getValue()); + } + }); + } + }); + + byte[] result = new byte[(int) bytes.readRemaining()]; + bytes.read(result); + return result; + } finally { + bytes.releaseLast(); + } + } + + private byte[] serializeProjectedRecord(DataGenerator.ProjectedRecord record) { + Bytes bytes = Bytes.elasticByteBuffer(); + try { + Wire wire = WireType.BINARY.apply(bytes); + + wire.writeDocument(false, w -> { + if (record.id != null) w.write("id").text(record.id); + w.write("timestamp").int64(record.timestamp); + if (record.tags != null) { + w.write("tags").object(record.tags); + } + }); + + byte[] result = new byte[(int) bytes.readRemaining()]; + bytes.read(result); + return result; + } finally { + bytes.releaseLast(); + } + } + + private DataGenerator.TestRecord deserializeRecord(byte[] data) { + Bytes bytes = Bytes.wrapForRead(data); + try { + Wire wire = new BinaryWire(bytes); + DataGenerator.TestRecord record = new DataGenerator.TestRecord(); + + wire.readDocument(null, w -> { + record.id = w.read("id").text(); + record.timestamp = w.read("timestamp").int64(); + record.flags = w.read("flags").int32(); + record.active = w.read("active").bool(); + record.value = w.read("value").float64(); + record.data = w.read("data").bytes(); + record.tags = (List) w.read("tags").object(); + record.metadata = w.read("metadata").marshallableAsMap(String.class, String.class); + }); + + return record; + } finally { + bytes.releaseLast(); + } + } +} \ No newline at end of file diff --git a/src/jmh/java/com/imprint/benchmark/serializers/ImprintSerializingBenchmark.java b/src/jmh/java/com/imprint/benchmark/serializers/ImprintSerializingBenchmark.java index 490b9d2..bb09890 100644 --- a/src/jmh/java/com/imprint/benchmark/serializers/ImprintSerializingBenchmark.java +++ b/src/jmh/java/com/imprint/benchmark/serializers/ImprintSerializingBenchmark.java @@ -7,12 +7,9 @@ import com.imprint.error.ImprintException; import org.openjdk.jmh.infra.Blackhole; -import java.nio.ByteBuffer; - public class ImprintSerializingBenchmark extends AbstractSerializingBenchmark { private ImprintRecord imprintRecord1; - private ImprintRecordBuilder preBuiltRecord; // Pre-built record for testing private byte[] serializedRecord1; private byte[] serializedRecord2; private static final SchemaId SCHEMA_ID = new SchemaId(1, 1); @@ -26,14 +23,13 @@ public void setup(DataGenerator.TestRecord testRecord, DataGenerator.TestRecord super.setup(testRecord, testRecord2); try { this.imprintRecord1 = buildRecord(testRecord).build(); - this.preBuiltRecord = buildRecord(testRecord); // Pre-built for testing ImprintRecord imprintRecord2 = buildRecord(testRecord2).build(); - ByteBuffer buf1 = this.imprintRecord1.serializeToBuffer(); + var buf1 = this.imprintRecord1.serializeToBuffer(); this.serializedRecord1 = new byte[buf1.remaining()]; buf1.get(this.serializedRecord1); - ByteBuffer buf2 = imprintRecord2.serializeToBuffer(); + var buf2 = imprintRecord2.serializeToBuffer(); this.serializedRecord2 = new byte[buf2.remaining()]; buf2.get(this.serializedRecord2); } catch (ImprintException e) { @@ -56,18 +52,11 @@ private ImprintRecordBuilder buildRecord(DataGenerator.TestRecord pojo) throws I @Override public void serialize(Blackhole bh) { - // Test 3: Just field addition (POJO → Builder) try { - var builder = buildRecord(this.testData); - bh.consume(builder); // Consume builder to prevent dead code elimination - } catch (ImprintException ignored) { + bh.consume(buildRecord(DataGenerator.createTestRecord()).buildToBuffer()); + } catch (ImprintException e) { + throw new RuntimeException(e); } - - // Test 2: Just serialization (Builder → Bytes) - // try{ - // bh.consume(preBuiltRecord.buildToBuffer()); - // } catch (ImprintException ignored) { - // } } @Override @@ -82,8 +71,7 @@ public void deserialize(Blackhole bh) { @Override public void projectAndSerialize(Blackhole bh) { try { - // Should use zero-copy projection directly from existing record - ImprintRecord projected = this.imprintRecord1.project(0, 1, 6); + var projected = this.imprintRecord1.project(0, 1, 6); bh.consume(projected.serializeToBuffer()); } catch (ImprintException e) { throw new RuntimeException(e); diff --git a/src/main/java/com/imprint/core/ImprintFieldObjectMap.java b/src/main/java/com/imprint/core/ImprintFieldObjectMap.java index a6e63de..df4d0ab 100644 --- a/src/main/java/com/imprint/core/ImprintFieldObjectMap.java +++ b/src/main/java/com/imprint/core/ImprintFieldObjectMap.java @@ -1,15 +1,17 @@ package com.imprint.core; +import lombok.Value; + import java.util.Arrays; import java.util.stream.IntStream; /** * Specialized short→object map optimized for ImprintRecordBuilder field IDs. - * Basically a copy of EclipseCollections's primitive map: + * Implementation * - No key-value boxing/unboxing * - Primitive int16 keys * - Open addressing with linear probing - * - Sort values in place and return without allocation (subsequently poisons the map) + * - Sacrifices map to sort values in place and return without allocation/copy */ final class ImprintFieldObjectMap { private static final int DEFAULT_CAPACITY = 64; @@ -20,6 +22,8 @@ final class ImprintFieldObjectMap { private Object[] values; private int size; private int threshold; + //sorting in place and returning the map's internal references means we don't have to allocate or copy to a new array; + //this is definitely a suspicious pattern at best though private boolean poisoned = false; public ImprintFieldObjectMap() { @@ -272,11 +276,12 @@ private void compactEntries() { /** * Sort the first 'count' entries by key using insertion sort (should be fast enough for small arrays). + * //TODO some duplication in here with the sorted values copy */ private void sortEntriesByKey(int count) { for (int i = 1; i < count; i++) { short key = keys[i]; - Object value = values[i]; + var value = values[i]; int j = i - 1; while (j >= 0 && keys[j] > key) { @@ -341,16 +346,11 @@ private static int nextPowerOfTwo(int n) { /** * Result holder for in-place sorted fields - returns both keys and values. */ - public static final class SortedFieldsResult { - public final short[] keys; - public final Object[] values; - public final int count; - - SortedFieldsResult(short[] keys, Object[] values, int count) { - this.keys = keys; - this.values = values; - this.count = count; - } + @Value + public static class SortedFieldsResult { + short[] keys; + Object[] values; + int count; } /** @@ -358,6 +358,7 @@ public static final class SortedFieldsResult { * WARNING: Modifies internal state, and renders map operations unstable and in an illegal state. */ public SortedFieldsResult getSortedFields() { + //It makes more sense to poison the map here for consistency, even though technically it isn't with 0 fields. if (size == 0) { poisoned = true; return new SortedFieldsResult(keys, values, 0); @@ -366,7 +367,6 @@ public SortedFieldsResult getSortedFields() { compactEntries(); sortEntriesByKey(size); poisoned = true; - return new SortedFieldsResult(keys, values, size); } } diff --git a/src/main/java/com/imprint/core/ImprintRecord.java b/src/main/java/com/imprint/core/ImprintRecord.java index a8a745d..35c2941 100644 --- a/src/main/java/com/imprint/core/ImprintRecord.java +++ b/src/main/java/com/imprint/core/ImprintRecord.java @@ -4,57 +4,41 @@ import com.imprint.error.ErrorType; import com.imprint.error.ImprintException; import com.imprint.ops.ImprintOperations; -import com.imprint.types.*; +import com.imprint.types.ImprintDeserializers; +import com.imprint.types.TypeCode; +import com.imprint.util.ImprintBuffer; import com.imprint.util.VarInt; - import lombok.AccessLevel; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.ToString; import lombok.experimental.NonFinal; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.NoSuchElementException; -import java.util.Objects; - -/** - * Imprint Record - *

- * This is the primary way to work with Imprint records, providing: - * - Zero-copy field access via binary search - * - Direct bytes-to-bytes operations (merge, project) - * - Lazy deserializing operations - */ +import java.util.*; + @lombok.Value @EqualsAndHashCode(of = "serializedBytes") @ToString(of = {"header"}) public class ImprintRecord { - ByteBuffer serializedBytes; + ImprintBuffer serializedBytes; @Getter(AccessLevel.PUBLIC) Header header; @Getter(AccessLevel.PACKAGE) // Raw directory bytes (read-only) - ByteBuffer directoryBuffer; + ImprintBuffer directoryBuffer; @Getter(AccessLevel.PACKAGE) // Raw payload bytes - ByteBuffer payload; + ImprintBuffer payload; @NonFinal @Getter(AccessLevel.NONE) //Directory View cache to allow for easier mutable operations needed for lazy initialization Directory.DirectoryView directoryView; - /** - * Package-private constructor for @Value that creates immutable ByteBuffer views. - */ - ImprintRecord(ByteBuffer serializedBytes, Header header, ByteBuffer directoryBuffer, ByteBuffer payload) { + ImprintRecord(ImprintBuffer serializedBytes, Header header, ImprintBuffer directoryBuffer, ImprintBuffer payload) { this.serializedBytes = serializedBytes.asReadOnlyBuffer(); this.header = Objects.requireNonNull(header); this.directoryBuffer = directoryBuffer.asReadOnlyBuffer(); @@ -62,8 +46,6 @@ public class ImprintRecord { this.directoryView = null; } - // ========== STATIC FACTORY METHODS ========== - /** * Create a builder for constructing new ImprintRecord instances. */ @@ -88,27 +70,23 @@ public static ImprintRecordBuilder builder(int fieldspaceId, int schemaHash) { * Deserialize an ImprintRecord from bytes. */ public static ImprintRecord deserialize(byte[] bytes) throws ImprintException { - return fromBytes(ByteBuffer.wrap(bytes)); + return fromBytes(new ImprintBuffer(bytes)); } - public static ImprintRecord deserialize(ByteBuffer buffer) throws ImprintException { + public static ImprintRecord deserialize(ImprintBuffer buffer) throws ImprintException { return fromBytes(buffer); } /** * Create a ImprintRecord from complete serialized bytes. */ - public static ImprintRecord fromBytes(ByteBuffer serializedBytes) throws ImprintException { + public static ImprintRecord fromBytes(ImprintBuffer serializedBytes) throws ImprintException { Objects.requireNonNull(serializedBytes, "Serialized bytes cannot be null"); - - var buffer = serializedBytes.duplicate().order(ByteOrder.LITTLE_ENDIAN); - + var buffer = serializedBytes.duplicate(); // Parse header var header = parseHeader(buffer); - // Extract directory and payload sections var parsedBuffers = parseBuffersFromSerialized(serializedBytes); - return new ImprintRecord(serializedBytes, header, parsedBuffers.directoryBuffer, parsedBuffers.payload); } @@ -120,7 +98,7 @@ public static ImprintRecord fromBytes(ByteBuffer serializedBytes) throws Imprint * Results in a new ImprintRecord without any object creation. */ public ImprintRecord merge(ImprintRecord other) throws ImprintException { - var mergedBytes = ImprintOperations.mergeBytes(this.serializedBytes, other.serializedBytes); + var mergedBytes = ImprintOperations.mergeBytes(this.getSerializedBytes(), other.getSerializedBytes()); return fromBytes(mergedBytes); } @@ -129,7 +107,7 @@ public ImprintRecord merge(ImprintRecord other) throws ImprintException { * Results in a new ImprintRecord without any object creation. */ public ImprintRecord project(int... fieldIds) throws ImprintException { - var projectedBytes = ImprintOperations.projectBytes(this.serializedBytes, fieldIds); + var projectedBytes = ImprintOperations.projectBytes(this.getSerializedBytes(), fieldIds); return fromBytes(projectedBytes); } @@ -145,17 +123,18 @@ public ImprintRecord projectAndMerge(ImprintRecord other, int... projectFields) * Get the raw serialized bytes. * This is the most efficient way to pass the record around. */ - public ByteBuffer getSerializedBytes() { - return serializedBytes.duplicate(); + public ImprintBuffer getSerializedBytes() { + var buffer = serializedBytes.duplicate(); + buffer.position(0); + return buffer; } /** * Get a DirectoryView for straight through directory access. */ public Directory.DirectoryView getDirectoryView() { - if (directoryView == null) { + if (directoryView == null) directoryView = new ImprintDirectoryView(); - } return directoryView; } @@ -169,7 +148,7 @@ public List getDirectory() { /** * Get raw bytes for a field without deserializing. */ - public ByteBuffer getRawBytes(int fieldId) { + public ImprintBuffer getRawBytes(int fieldId) { try { return getFieldBuffer(fieldId); } catch (ImprintException e) { @@ -180,7 +159,7 @@ public ByteBuffer getRawBytes(int fieldId) { /** * Get raw bytes for a field by short ID. */ - public ByteBuffer getRawBytes(short fieldId) { + public ImprintBuffer getRawBytes(short fieldId) { return getRawBytes((int) fieldId); } @@ -212,8 +191,6 @@ public int getFieldCount() { return getDirectoryCount(); } - // ========== TYPED GETTERS ========== - public String getString(int fieldId) throws ImprintException { return (String) getTypedPrimitive(fieldId, com.imprint.types.TypeCode.STRING, "STRING"); } @@ -259,8 +236,10 @@ public ImprintRecord getRow(int fieldId) throws ImprintException { /** * Returns a copy of the bytes. */ - public ByteBuffer serializeToBuffer() { - return serializedBytes.duplicate(); + public ImprintBuffer serializeToBuffer() { + var buffer = serializedBytes.duplicate(); + buffer.position(0); + return buffer; } /** @@ -281,14 +260,14 @@ public int getSerializedSize() { /** * Get and validate a field exists, is not null, and has the expected type. */ - private Object getTypedPrimitive(int fieldId, com.imprint.types.TypeCode expectedTypeCode, String typeName) throws ImprintException { + private Object getTypedPrimitive(int fieldId, TypeCode expectedTypeCode, String typeName) throws ImprintException { var entry = getDirectoryView().findEntry(fieldId); if (entry == null) throw new ImprintException(ErrorType.FIELD_NOT_FOUND, "Field " + fieldId + " not found"); - - if (entry.getTypeCode() == com.imprint.types.TypeCode.NULL) + + if (entry.getTypeCode() == TypeCode.NULL) throw new ImprintException(ErrorType.TYPE_MISMATCH, "Field " + fieldId + " is NULL, cannot retrieve as " + typeName); - + if (entry.getTypeCode() != expectedTypeCode) throw new ImprintException(ErrorType.TYPE_MISMATCH, "Field " + fieldId + " is of type " + entry.getTypeCode() + ", expected " + typeName); @@ -302,8 +281,8 @@ private Object getTypedPrimitive(int fieldId, com.imprint.types.TypeCode expecte /** * Parse buffers from serialized record bytes. */ - private static ParsedBuffers parseBuffersFromSerialized(ByteBuffer serializedRecord) throws ImprintException { - var buffer = serializedRecord.duplicate().order(ByteOrder.LITTLE_ENDIAN); + private static ParsedBuffers parseBuffersFromSerialized(ImprintBuffer serializedRecord) throws ImprintException { + var buffer = serializedRecord.duplicate(); // Parse header and extract sections using shared utility var header = parseHeaderFromBuffer(buffer); @@ -313,10 +292,10 @@ private static ParsedBuffers parseBuffersFromSerialized(ByteBuffer serializedRec } private static class ParsedBuffers { - final ByteBuffer directoryBuffer; - final ByteBuffer payload; + final ImprintBuffer directoryBuffer; + final ImprintBuffer payload; - ParsedBuffers(ByteBuffer directoryBuffer, ByteBuffer payload) { + ParsedBuffers(ImprintBuffer directoryBuffer, ImprintBuffer payload) { this.directoryBuffer = directoryBuffer; this.payload = payload; } @@ -333,7 +312,7 @@ private int getDirectoryCount() { /** * Gets ByteBuffer view of a field's data. */ - private ByteBuffer getFieldBuffer(int fieldId) throws ImprintException { + private ImprintBuffer getFieldBuffer(int fieldId) throws ImprintException { var entry = findDirectoryEntry(fieldId); if (entry == null) return null; @@ -352,7 +331,7 @@ private ByteBuffer getFieldBuffer(int fieldId) throws ImprintException { } private Directory findDirectoryEntry(int fieldId) throws ImprintException { - var searchBuffer = directoryBuffer.duplicate().order(ByteOrder.LITTLE_ENDIAN); + var searchBuffer = directoryBuffer.duplicate(); int count = getDirectoryCount(); if (count == 0) return null; @@ -389,7 +368,7 @@ private Directory findDirectoryEntry(int fieldId) throws ImprintException { } private int findEndOffset(int currentFieldId) throws ImprintException { - var scanBuffer = directoryBuffer.duplicate().order(ByteOrder.LITTLE_ENDIAN); + var scanBuffer = directoryBuffer.duplicate(); int count = getDirectoryCount(); if (count == 0) return payload.limit(); @@ -425,7 +404,7 @@ private int findEndOffset(int currentFieldId) throws ImprintException { return nextOffset; } - private Directory deserializeDirectoryEntry(ByteBuffer buffer) throws ImprintException { + private Directory deserializeDirectoryEntry(ImprintBuffer buffer) throws ImprintException { if (buffer.remaining() < Constants.DIR_ENTRY_BYTES) throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, "Not enough bytes for directory entry"); @@ -479,12 +458,12 @@ public Iterator iterator() { * Iterator that parses directory entries lazily from raw bytes. */ private class ImprintDirectoryIterator implements Iterator { - private final ByteBuffer iterBuffer; + private final ImprintBuffer iterBuffer; private final int totalCount; private int currentIndex; ImprintDirectoryIterator() { - this.iterBuffer = directoryBuffer.duplicate().order(ByteOrder.LITTLE_ENDIAN); + this.iterBuffer = directoryBuffer.duplicate(); this.totalCount = getDirectoryCount(); try { @@ -521,7 +500,7 @@ public Directory next() { * Parse a header from a ByteBuffer without advancing the buffer position. * Utility method shared between {@link ImprintRecord} and {@link ImprintOperations}. */ - public static Header parseHeaderFromBuffer(ByteBuffer buffer) throws ImprintException { + public static Header parseHeaderFromBuffer(ImprintBuffer buffer) throws ImprintException { int startPos = buffer.position(); try { return parseHeader(buffer); @@ -542,11 +521,11 @@ public static int calculateDirectorySize(int entryCount) { * Utility class shared between {@link ImprintRecord} and {@link ImprintOperations}. */ public static class BufferSections { - public final ByteBuffer directoryBuffer; - public final ByteBuffer payloadBuffer; + public final ImprintBuffer directoryBuffer; + public final ImprintBuffer payloadBuffer; public final int directoryCount; - public BufferSections(ByteBuffer directoryBuffer, ByteBuffer payloadBuffer, int directoryCount) { + public BufferSections(ImprintBuffer directoryBuffer, ImprintBuffer payloadBuffer, int directoryCount) { this.directoryBuffer = directoryBuffer; this.payloadBuffer = payloadBuffer; this.directoryCount = directoryCount; @@ -557,7 +536,7 @@ public BufferSections(ByteBuffer directoryBuffer, ByteBuffer payloadBuffer, int * Extract directory and payload sections from a serialized buffer. * Utility method shared between {@link ImprintRecord} and {@link ImprintOperations}. */ - public static BufferSections extractBufferSections(ByteBuffer buffer, Header header) throws ImprintException { + public static BufferSections extractBufferSections(ImprintBuffer buffer, Header header) throws ImprintException { // Skip header buffer.position(buffer.position() + Constants.HEADER_BYTES); @@ -580,7 +559,7 @@ public static BufferSections extractBufferSections(ByteBuffer buffer, Header hea return new BufferSections(directoryBuffer, payloadBuffer, directoryCount); } - private static Header parseHeader(ByteBuffer buffer) throws ImprintException { + private static Header parseHeader(ImprintBuffer buffer) throws ImprintException { if (buffer.remaining() < Constants.HEADER_BYTES) throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, "Not enough bytes for header"); @@ -600,8 +579,8 @@ private static Header parseHeader(ByteBuffer buffer) throws ImprintException { return new Header(flags, new SchemaId(fieldSpaceId, schemaHash), payloadSize); } - private Object deserializePrimitive(com.imprint.types.TypeCode typeCode, ByteBuffer buffer) throws ImprintException { - var valueBuffer = buffer.duplicate().order(ByteOrder.LITTLE_ENDIAN); + private Object deserializePrimitive(com.imprint.types.TypeCode typeCode, ImprintBuffer buffer) throws ImprintException { + var valueBuffer = buffer.duplicate(); switch (typeCode) { case NULL: case BOOL: @@ -623,12 +602,12 @@ private Object deserializePrimitive(com.imprint.types.TypeCode typeCode, ByteBuf } } - private java.util.List deserializePrimitiveArray(ByteBuffer buffer) throws ImprintException { + private List deserializePrimitiveArray(ImprintBuffer buffer) throws ImprintException { VarInt.DecodeResult lengthResult = VarInt.decode(buffer); int length = lengthResult.getValue(); if (length == 0) { - return java.util.Collections.emptyList(); + return Collections.emptyList(); } if (buffer.remaining() < 1) { @@ -654,12 +633,12 @@ private java.util.List deserializePrimitiveArray(ByteBuffer buffer) thro return elements; } - private java.util.Map deserializePrimitiveMap(ByteBuffer buffer) throws ImprintException { + private Map deserializePrimitiveMap(ImprintBuffer buffer) throws ImprintException { VarInt.DecodeResult lengthResult = VarInt.decode(buffer); int length = lengthResult.getValue(); if (length == 0) { - return java.util.Collections.emptyMap(); + return Collections.emptyMap(); } if (buffer.remaining() < 2) { @@ -667,7 +646,7 @@ private java.util.Map deserializePrimitiveMap(ByteBuffer buffer) } var keyType = TypeCode.fromByte(buffer.get()); var valueType = TypeCode.fromByte(buffer.get()); - var map = new java.util.HashMap<>(length); + var map = new HashMap<>(length); for (int i = 0; i < length; i++) { var keyPrimitive = ImprintDeserializers.deserializePrimitive(buffer, keyType); @@ -682,7 +661,6 @@ private java.util.Map deserializePrimitiveMap(ByteBuffer buffer) } else { valuePrimitive = ImprintDeserializers.deserializePrimitive(buffer, valueType); } - map.put(keyPrimitive, valuePrimitive); } diff --git a/src/main/java/com/imprint/core/ImprintRecordBuilder.java b/src/main/java/com/imprint/core/ImprintRecordBuilder.java index 967aac7..3b77809 100644 --- a/src/main/java/com/imprint/core/ImprintRecordBuilder.java +++ b/src/main/java/com/imprint/core/ImprintRecordBuilder.java @@ -6,13 +6,13 @@ import com.imprint.types.ImprintSerializers; import com.imprint.types.MapKey; import com.imprint.types.TypeCode; -import com.imprint.util.VarInt; +import com.imprint.util.ImprintBuffer; import lombok.SneakyThrows; +import lombok.Value; -import java.nio.BufferOverflowException; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.util.*; +import java.util.List; +import java.util.Map; +import java.util.Objects; /** * A fluent builder for creating ImprintRecord instances with type-safe, @@ -41,13 +41,13 @@ public final class ImprintRecordBuilder { private final ImprintFieldObjectMap fields; private int estimatedPayloadSize = 0; - // Direct primitive storage to avoid Value object creation - @lombok.Value + + @Value static class FieldValue { byte typeCode; Object value; - // Fast factory methods for primitives + // Factory methods for primitives static FieldValue ofInt32(int value) { return new FieldValue(TypeCode.INT32.getCode(), value); } static FieldValue ofInt64(long value) { return new FieldValue(TypeCode.INT64.getCode(), value); } static FieldValue ofFloat32(float value) { return new FieldValue(TypeCode.FLOAT32.getCode(), value); } @@ -98,7 +98,7 @@ public ImprintRecordBuilder field(int id, byte[] value) { return addField(id, FieldValue.ofBytes(value)); } - // Collections - store as raw collections, convert during serialization + // Collections - store as raw collections for now, convert during serialization public ImprintRecordBuilder field(int id, List values) { return addField(id, FieldValue.ofArray(values)); } @@ -137,65 +137,33 @@ public ImprintRecordBuilder fields(Map fieldsMap) { return this; } - // Builder utilities - public boolean hasField(int id) { - return fields.containsKey(id); - } - - public int fieldCount() { - return fields.size(); - } - - public Set fieldIds() { - var ids = new HashSet(fields.size()); - var keys = fields.getKeys(); - for (var key : keys) { - ids.add(key); - } - return ids; - } // Build the final record public ImprintRecord build() throws ImprintException { // Build to bytes and then create ImprintRecord from bytes for consistency - var serializedBytes = buildToBuffer(); - return ImprintRecord.fromBytes(serializedBytes); + var buffer = buildToBuffer(); + return ImprintRecord.fromBytes(buffer); } /** - * Builds the record and serializes it directly to a ByteBuffer. + * Builds the record and serializes it directly to a ByteBuffer using growable buffer optimization. * * @return A read-only ByteBuffer containing the fully serialized record. * @throws ImprintException if serialization fails. */ - public ByteBuffer buildToBuffer() throws ImprintException { + public ImprintBuffer buildToBuffer() throws ImprintException { // 1. Calculate conservative size BEFORE sorting (which invalidates the map) int conservativeSize = calculateConservativePayloadSize(); - - // 2. Sort fields by ID for directory ordering (zero allocation) + // 2. Sort fields by ID for directory ordering var sortedFieldsResult = getSortedFieldsResult(); - var sortedValues = sortedFieldsResult.values; - var sortedKeys = sortedFieldsResult.keys; - var fieldCount = sortedFieldsResult.count; - - // 3. Serialize payload and calculate offsets with overflow handling - PayloadSerializationResult result = null; - int bufferSizeMultiplier = 1; - - while (result == null && bufferSizeMultiplier <= 64) { - try { - result = serializePayload(sortedValues, fieldCount, conservativeSize, bufferSizeMultiplier); - } catch (BufferOverflowException e) { - bufferSizeMultiplier *= 2; // Try 2x, 4x, 8x, 16x, 32x, 64x - } - } - - if (result == null) { - throw new ImprintException(ErrorType.SERIALIZATION_ERROR, - "Failed to serialize payload even with 64x buffer size"); - } - - return serializeToBuffer(schemaId, sortedKeys, sortedValues, result.offsets, fieldCount, result.payload); + var sortedValues = sortedFieldsResult.getValues(); + var sortedKeys = sortedFieldsResult.getKeys(); + var fieldCount = sortedFieldsResult.getCount(); + // 3. Calculate directory size + int directorySize = ImprintRecord.calculateDirectorySize(fieldCount); + // 4. Use growable buffer to eliminate size guessing and retry logic + return serializeToBuffer(schemaId, sortedKeys, sortedValues, fieldCount, + conservativeSize, directorySize); } /** @@ -211,7 +179,6 @@ private ImprintRecordBuilder addField(int id, FieldValue fieldValue) { Objects.requireNonNull(fieldValue, "FieldValue cannot be null"); int newSize = estimateFieldSize(fieldValue); var oldEntry = fields.putAndReturnOld(id, fieldValue); - if (oldEntry != null) { int oldSize = estimateFieldSize(oldEntry); estimatedPayloadSize += newSize - oldSize; @@ -226,6 +193,7 @@ private FieldValue convertToFieldValue(Object obj) { if (obj == null) { return FieldValue.ofNull(); } + if (obj instanceof Boolean) { return FieldValue.ofBool((Boolean) obj); } @@ -277,50 +245,80 @@ private MapKey convertToMapKey(Object obj) { throw new IllegalArgumentException("Unsupported map key type: " + obj.getClass().getName()); } + /** + * Fast field size estimation using heuristics for performance. + */ + @SneakyThrows private int estimateFieldSize(FieldValue fieldValue) { - TypeCode typeCode; - try { - typeCode = TypeCode.fromByte(fieldValue.typeCode); - } catch (ImprintException e) { - throw new RuntimeException("Invalid type code in FieldValue: " + fieldValue.typeCode, e); - } + var typeCode = TypeCode.fromByte(fieldValue.typeCode); return ImprintSerializers.estimateSize(typeCode, fieldValue.value); } + /** + * Get current estimated payload size with 25% buffer. + */ private int calculateConservativePayloadSize() { // Add 25% buffer for safety margin return Math.max(estimatedPayloadSize + (estimatedPayloadSize / 4), 4096); } - private static class PayloadSerializationResult { - final int[] offsets; - final ByteBuffer payload; - PayloadSerializationResult(int[] offsets, ByteBuffer payload) { - this.offsets = offsets; - this.payload = payload; - } - } - private PayloadSerializationResult serializePayload(Object[] sortedFields, int fieldCount, int conservativeSize, int sizeMultiplier) throws ImprintException { - var payloadBuffer = ByteBuffer.allocate(conservativeSize * sizeMultiplier); - payloadBuffer.order(ByteOrder.LITTLE_ENDIAN); - return doSerializePayload(sortedFields, fieldCount, payloadBuffer); - } - - private PayloadSerializationResult doSerializePayload(Object[] sortedFields, int fieldCount, ByteBuffer payloadBuffer) throws ImprintException { + /** + * Serialize using growable buffer - eliminates size guessing and retry logic. + * Uses growable buffer that automatically expands as needed during serialization. + * //TODO: we have multiple places where we write header/directory and we should probably consolidate that + */ + private ImprintBuffer serializeToBuffer(SchemaId schemaId, short[] sortedKeys, Object[] sortedValues, + int fieldCount, int conservativePayloadSize, int directorySize) throws ImprintException { + + // Start with conservative estimate, use fixed size buffer first + int initialSize = Constants.HEADER_BYTES + directorySize + conservativePayloadSize; + var buffer = new ImprintBuffer(new byte[initialSize * 2]); // Extra capacity to avoid growth + + // Reserve space for header and directory - write payload first + int headerAndDirSize = Constants.HEADER_BYTES + directorySize; + buffer.position(headerAndDirSize); + + // Serialize payload and collect offsets - buffer will grow automatically int[] offsets = new int[fieldCount]; for (int i = 0; i < fieldCount; i++) { - var fieldValue = (FieldValue) sortedFields[i]; - offsets[i] = payloadBuffer.position(); - serializeFieldValue(fieldValue, payloadBuffer); + var fieldValue = (FieldValue) sortedValues[i]; + offsets[i] = buffer.position() - headerAndDirSize; // Offset relative to payload start + serializeFieldValue(fieldValue, buffer); } - payloadBuffer.flip(); - var payloadView = payloadBuffer.slice().asReadOnlyBuffer(); - return new PayloadSerializationResult(offsets, payloadView); + + int actualPayloadSize = buffer.position() - headerAndDirSize; + + // Now write header and directory at the beginning + buffer.position(0); + + // Write header with actual payload size + buffer.putByte(Constants.MAGIC); + buffer.putByte(Constants.VERSION); + buffer.putByte((byte) 0); // flags + buffer.putInt(schemaId.getFieldSpaceId()); + buffer.putInt(schemaId.getSchemaHash()); + buffer.putInt(actualPayloadSize); + + // Write directory + writeDirectoryToBuffer(sortedKeys, sortedValues, offsets, fieldCount, buffer); + + // Set final limit and prepare for reading + int finalSize = Constants.HEADER_BYTES + directorySize + actualPayloadSize; + + // Ensure minimum buffer size for validation (at least header size) + if (finalSize < Constants.HEADER_BYTES) { + throw new IllegalStateException("Buffer size (" + finalSize + ") is smaller than minimum header size (" + Constants.HEADER_BYTES + ")"); + } + + buffer.position(0); + buffer.limit(finalSize); + + return buffer.asReadOnlyBuffer(); } - private void serializeFieldValue(FieldValue fieldValue, ByteBuffer buffer) throws ImprintException { + private void serializeFieldValue(FieldValue fieldValue, ImprintBuffer buffer) throws ImprintException { var typeCode = TypeCode.fromByte(fieldValue.typeCode); var value = fieldValue.value; switch (typeCode) { @@ -355,30 +353,34 @@ private void serializeFieldValue(FieldValue fieldValue, ByteBuffer buffer) throw serializeMap((Map) value, buffer); break; case ROW: - // Nested records + // Nested record serialization var nestedRecord = (ImprintRecord) value; var serializedRow = nestedRecord.serializeToBuffer(); - buffer.put(serializedRow); + // Copy data from read-only ByteBuffer to byte array first + byte[] rowBytes = new byte[serializedRow.remaining()]; + serializedRow.get(rowBytes); + buffer.putBytes(rowBytes); break; default: throw new ImprintException(ErrorType.SERIALIZATION_ERROR, "Unknown type code: " + typeCode); } } - //TODO arrays and maps need to be handled better - private void serializeArray(List list, ByteBuffer buffer) throws ImprintException { + //TODO kinda hacky here, arrays and maps definitely need some functional updates to the flow + private void serializeArray(List list, ImprintBuffer buffer) throws ImprintException { ImprintSerializers.serializeArray(list, buffer, this::getTypeCodeForObject, this::serializeObjectDirect); } - private void serializeMap(Map map, ByteBuffer buffer) throws ImprintException { + private void serializeMap(Map map, ImprintBuffer buffer) throws ImprintException { ImprintSerializers.serializeMap(map, buffer, this::convertToMapKey, this::getTypeCodeForObject, this::serializeObjectDirect); } - + + // Helper methods for static serializers private TypeCode getTypeCodeForObject(Object obj) { var fieldValue = convertToFieldValue(obj); try { @@ -388,7 +390,7 @@ private TypeCode getTypeCodeForObject(Object obj) { } } - private void serializeObjectDirect(Object obj, ByteBuffer buffer) { + private void serializeObjectDirect(Object obj, ImprintBuffer buffer) { try { var fieldValue = convertToFieldValue(obj); serializeFieldValue(fieldValue, buffer); @@ -405,61 +407,23 @@ private ImprintFieldObjectMap.SortedFieldsResult getSortedFieldsResult() { return fields.getSortedFields(); } - /** - * Serialize components into a single ByteBuffer. - */ - private static ByteBuffer serializeToBuffer(SchemaId schemaId, short[] sortedKeys, Object[] sortedValues, int[] offsets, int fieldCount, ByteBuffer payload) { - var header = new Header(new Flags((byte) 0), schemaId, payload.remaining()); - int directorySize = ImprintRecord.calculateDirectorySize(fieldCount); - - int finalSize = Constants.HEADER_BYTES + directorySize + payload.remaining(); - var finalBuffer = ByteBuffer.allocate(finalSize); - finalBuffer.order(ByteOrder.LITTLE_ENDIAN); - - // Write header - finalBuffer.put(Constants.MAGIC); - finalBuffer.put(Constants.VERSION); - finalBuffer.put(header.getFlags().getValue()); - finalBuffer.putInt(header.getSchemaId().getFieldSpaceId()); - finalBuffer.putInt(header.getSchemaId().getSchemaHash()); - finalBuffer.putInt(header.getPayloadSize()); - - // Write directory with FieldValue type codes - writeDirectoryToBuffer(sortedKeys, sortedValues, offsets, fieldCount, finalBuffer); - - // Write payload - finalBuffer.put(payload); - - finalBuffer.flip(); - return finalBuffer.asReadOnlyBuffer(); - } /** * Write directory entries directly to buffer for FieldValue objects. */ - private static void writeDirectoryToBuffer(short[] sortedKeys, Object[] sortedValues, int[] offsets, int fieldCount, ByteBuffer buffer) { - // Write field count at the beginning of directory - // Optimize VarInt encoding for common case (< 128 fields = single byte) - if (fieldCount < 128) { - buffer.put((byte) fieldCount); - } else { - VarInt.encode(fieldCount, buffer); - } + private static void writeDirectoryToBuffer(short[] sortedKeys, Object[] sortedValues, int[] offsets, int fieldCount, ImprintBuffer buffer) { + // Write field count using putVarInt for consistency with ImprintOperations + buffer.putVarInt(fieldCount); // Early return for empty directory if (fieldCount == 0) return; - - - //hopefully JIT vectorizes this for (int i = 0; i < fieldCount; i++) { var fieldValue = (FieldValue) sortedValues[i]; - int pos = buffer.position(); - buffer.putShort(pos, sortedKeys[i]); // bytes 0-1: field ID - buffer.put(pos + 2, fieldValue.typeCode); // byte 2: type code - buffer.putInt(pos + 3, offsets[i]); // bytes 3-6: offset - // Advance buffer position by 7 bytes - buffer.position(pos + 7); + // Write directory entry: field ID (2 bytes), type code (1 byte), offset (4 bytes) + buffer.putShort(sortedKeys[i]); // bytes 0-1: field ID + buffer.putByte(fieldValue.typeCode); // byte 2: type code + buffer.putInt(offsets[i]); // bytes 3-6: offset } } diff --git a/src/main/java/com/imprint/ops/ImprintOperations.java b/src/main/java/com/imprint/ops/ImprintOperations.java index 54e594a..b13a850 100644 --- a/src/main/java/com/imprint/ops/ImprintOperations.java +++ b/src/main/java/com/imprint/ops/ImprintOperations.java @@ -4,375 +4,393 @@ import com.imprint.core.*; import com.imprint.error.ErrorType; import com.imprint.error.ImprintException; -import com.imprint.util.VarInt; +import com.imprint.util.ImprintBuffer; import lombok.Value; import lombok.experimental.UtilityClass; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; import java.util.*; @UtilityClass public class ImprintOperations { - /** - * Pure bytes-to-bytes merge operation that avoids all object creation. - * Performs merge directly on serialized Imprint record buffers. - * - * @param firstBuffer Complete serialized Imprint record - * @param secondBuffer Complete serialized Imprint record - * @return Merged record as serialized bytes - * @throws ImprintException if merge fails - */ - public static ByteBuffer mergeBytes(ByteBuffer firstBuffer, ByteBuffer secondBuffer) throws ImprintException { - validateImprintBuffer(firstBuffer, "firstBuffer"); - validateImprintBuffer(secondBuffer, "secondBuffer"); - - // TODO possible could work directly on the originals but duplicate makes the mark values and offsets easy to reason about - // duplicates to avoid affecting original positions, we'll need to preserve at least one side - var first = firstBuffer.duplicate().order(ByteOrder.LITTLE_ENDIAN); - var second = secondBuffer.duplicate().order(ByteOrder.LITTLE_ENDIAN); - - // Parse headers - var firstHeader = parseHeaderOnly(first); - var secondHeader = parseHeaderOnly(second); - - // Extract directory and payload sections - var firstSections = extractSections(first, firstHeader); - var secondSections = extractSections(second, secondHeader); - - // Perform raw merge - return mergeRawSections(firstHeader, firstSections, secondSections); + public static ImprintBuffer mergeBytes(ImprintBuffer firstBuffer, ImprintBuffer secondBuffer) throws ImprintException { + return Merge.mergeBytes(firstBuffer, secondBuffer); } - /** - * Parse just the header without advancing buffer past it - */ - private static Header parseHeaderOnly(ByteBuffer buffer) throws ImprintException { - return ImprintRecord.parseHeaderFromBuffer(buffer); - } - - /** - * Extract directory and payload sections from a buffer - */ - private static ImprintRecord.BufferSections extractSections(ByteBuffer buffer, Header header) throws ImprintException { - return ImprintRecord.extractBufferSections(buffer, header); + public static ImprintBuffer projectBytes(ImprintBuffer sourceBuffer, int... fieldIds) throws ImprintException { + return Project.projectBytes(sourceBuffer, fieldIds); } + /** - * Merge raw directory and payload sections without object creation - * Assumes incoming streams are already both sorted from the serialization process + * Shared utilities and data structures used by both Merge and Project operations. */ - private static ByteBuffer mergeRawSections(Header firstHeader, ImprintRecord.BufferSections firstSections, ImprintRecord.BufferSections secondSections) throws ImprintException { - // Prepare directory iterators - var firstDirIter = new RawDirectoryIterator(firstSections.directoryBuffer); - var secondDirIter = new RawDirectoryIterator(secondSections.directoryBuffer); - - // Pre-allocate - worst case is sum of both directory counts - int maxEntries = firstSections.directoryCount + secondSections.directoryCount; - var mergedDirectoryEntries = new ArrayList(maxEntries); - var mergedChunks = new ArrayList(maxEntries); - - int totalMergedPayloadSize = 0; - int currentMergedOffset = 0; - - var firstEntry = firstDirIter.hasNext() ? firstDirIter.next() : null; - var secondEntry = secondDirIter.hasNext() ? secondDirIter.next() : null; - - // Merge directories and collect payload chunks - while (firstEntry != null || secondEntry != null) { - RawDirectoryEntry currentEntry; - ByteBuffer sourcePayload; - - if (firstEntry != null && (secondEntry == null || firstEntry.fieldId <= secondEntry.fieldId)) { - // Take from first - currentEntry = firstEntry; - sourcePayload = getFieldPayload(firstSections.payloadBuffer, firstEntry, firstDirIter); + static class Core { + + static class ImprintBufferSections { + final ImprintBuffer directoryBuffer; + final ImprintBuffer payloadBuffer; + final int directoryCount; + + ImprintBufferSections(ImprintBuffer directoryBuffer, ImprintBuffer payloadBuffer, int directoryCount) { + this.directoryBuffer = directoryBuffer; + this.payloadBuffer = payloadBuffer; + this.directoryCount = directoryCount; + } + } + + @Value + static class RawDirectoryEntry { + short fieldId; + byte typeCode; + int offset; + } + + static class ImprintBufferDirectoryIterator { + private final ImprintBuffer buffer; + private final int totalCount; + private int currentIndex; + + ImprintBufferDirectoryIterator(ImprintBuffer directoryBuffer) { + this.buffer = directoryBuffer; + this.buffer.position(0); + this.totalCount = directoryBuffer.remaining() / Constants.DIR_ENTRY_BYTES; + this.currentIndex = 0; + } + + boolean hasNext() { + return currentIndex < totalCount; + } + + RawDirectoryEntry next() { + if (!hasNext()) { + throw new RuntimeException("No more directory entries"); + } + + short fieldId = buffer.getShort(); + byte typeCode = buffer.get(); + int offset = buffer.getInt(); + currentIndex++; - // Skip duplicate in second if present - if (secondEntry != null && firstEntry.fieldId == secondEntry.fieldId) - secondEntry = secondDirIter.hasNext() ? secondDirIter.next() : null; - - firstEntry = firstDirIter.hasNext() ? firstDirIter.next() : null; - } else { - // Take from second - currentEntry = secondEntry; - sourcePayload = getFieldPayload(secondSections.payloadBuffer, secondEntry, secondDirIter); - secondEntry = secondDirIter.hasNext() ? secondDirIter.next() : null; + return new RawDirectoryEntry(fieldId, typeCode, offset); + } + + int getNextEntryOffset(int fallbackOffset) { + if (currentIndex >= totalCount) { + return fallbackOffset; + } + + int savedPos = buffer.position(); + buffer.position(savedPos + 3); // Skip fieldId and typeCode + int offset = buffer.getInt(); + buffer.position(savedPos); + return offset; } - - // Add to merged directory with adjusted offset - var adjustedEntry = new RawDirectoryEntry(currentEntry.fieldId, currentEntry.typeCode, currentMergedOffset); - mergedDirectoryEntries.add(adjustedEntry); - - // Collect payload chunk - mergedChunks.add(sourcePayload.duplicate()); - currentMergedOffset += sourcePayload.remaining(); - totalMergedPayloadSize += sourcePayload.remaining(); } - - // Build final merged buffer - return buildSerializedBuffer(firstHeader, mergedDirectoryEntries, mergedChunks, totalMergedPayloadSize); - } - - /** - * Get payload bytes for a specific field using iterator state - */ - private static ByteBuffer getFieldPayload(ByteBuffer payload, RawDirectoryEntry entry, RawDirectoryIterator iterator) { - int startOffset = entry.offset; - int endOffset = iterator.getNextEntryOffset(payload.limit()); - - var fieldPayload = payload.duplicate(); - fieldPayload.position(startOffset); - fieldPayload.limit(endOffset); - return fieldPayload.slice(); - } - - /** - * Pure bytes-to-bytes projection operation that avoids all object creation. - * Projects a subset of fields directly from a serialized Imprint record. - * - * @param sourceBuffer Complete serialized Imprint record - * @param fieldIds Array of field IDs to include in projection - * @return Projected record as serialized bytes - * @throws ImprintException if projection fails - */ - public static ByteBuffer projectBytes(ByteBuffer sourceBuffer, int... fieldIds) throws ImprintException { - validateImprintBuffer(sourceBuffer, "sourceBuffer"); - - if (fieldIds == null || fieldIds.length == 0) { - return createEmptyRecordBytes(); + static Header parseHeaderFromImprintBuffer(ImprintBuffer buffer) throws ImprintException { + // Use duplicate to avoid modifying the original buffer + var headerBuffer = buffer.duplicate(); + headerBuffer.position(0); + + // Read header components using ImprintBuffer's optimized operations + byte magic = headerBuffer.get(); + byte version = headerBuffer.get(); + byte flags = headerBuffer.get(); + int fieldSpaceId = headerBuffer.getInt(); + int schemaHash = headerBuffer.getInt(); + int payloadSize = headerBuffer.getInt(); + + if (magic != Constants.MAGIC) { + throw new ImprintException(ErrorType.INVALID_BUFFER, "Invalid magic byte"); + } + if (version != Constants.VERSION) { + throw new ImprintException(ErrorType.INVALID_BUFFER, "Unsupported version: " + version); + } + + return new Header(new Flags(flags), new SchemaId(fieldSpaceId, schemaHash), payloadSize); } - var sortedFieldIds = fieldIds.clone(); - Arrays.sort(sortedFieldIds); - - // Duplicate avoids affecting original position which we'll need later - var source = sourceBuffer.duplicate().order(ByteOrder.LITTLE_ENDIAN); - - // Parse header - var header = parseHeaderOnly(source); - - // Extract sections - var sections = extractSections(source, header); - - // Perform raw projection - return projectRawSections(header, sections, sortedFieldIds); - } + /** + * Extract directory and payload sections directly from ImprintBuffer. + */ + static ImprintBufferSections extractSectionsFromImprintBuffer(ImprintBuffer buffer, Header header) { + // Use duplicate to avoid modifying the original buffer + var workingBuffer = buffer.duplicate(); + workingBuffer.position(Constants.HEADER_BYTES); - /** - * Project raw sections without object creation using optimized merge algorithm. - */ - private static ByteBuffer projectRawSections(Header originalHeader, ImprintRecord.BufferSections sections, int[] sortedRequestedFields) throws ImprintException { - - if (sortedRequestedFields.length == 0) { - return buildSerializedBuffer(originalHeader, new RawDirectoryEntry[0], new ByteBuffer[0]); + // Read directory count using ImprintBuffer's VarInt operations + int directoryCount = workingBuffer.getVarInt(); + int directoryStart = workingBuffer.position(); + int directorySize = directoryCount * Constants.DIR_ENTRY_BYTES; + int payloadStart = directoryStart + directorySize; + + // Create sliced buffers for directory and payload + var directoryBuffer = workingBuffer.slice(); + directoryBuffer.limit(directorySize); + + workingBuffer.position(payloadStart); + var payloadBuffer = workingBuffer.slice(); + payloadBuffer.limit(header.getPayloadSize()); + + return new ImprintBufferSections(directoryBuffer, payloadBuffer, directoryCount); } - - // Use pre-sized ArrayLists to avoid System.arraycopy but still be efficient - var projectedEntries = new ArrayList(sortedRequestedFields.length); - var payloadChunks = new ArrayList(sortedRequestedFields.length); - int totalProjectedPayloadSize = 0; - int currentOffset = 0; - int requestedIndex = 0; - - // Optimize: Cache payload buffer reference to avoid getter calls - var payloadBuffer = sections.payloadBuffer; - - // Merge algorithm: two-pointer approach through sorted sequences - var dirIterator = new RawDirectoryIterator(sections.directoryBuffer); - RawDirectoryEntry currentEntry = dirIterator.hasNext() ? dirIterator.next() : null; - - while (currentEntry != null && requestedIndex < sortedRequestedFields.length) { - int fieldId = currentEntry.fieldId; - int targetFieldId = sortedRequestedFields[requestedIndex]; - - if (fieldId == targetFieldId) { - var fieldPayload = getFieldPayload(payloadBuffer, currentEntry, dirIterator); - - // Add to projection with adjusted offset - projectedEntries.add(new RawDirectoryEntry(currentEntry.fieldId, currentEntry.typeCode, currentOffset)); - - // Collect payload chunk here - fieldPayload should already sliced - payloadChunks.add(fieldPayload); - - int payloadSize = fieldPayload.remaining(); - currentOffset += payloadSize; - totalProjectedPayloadSize += payloadSize; - - // Advance both pointers - handle dupes by advancing to next unique field hopefully - do { - requestedIndex++; - } while (requestedIndex < sortedRequestedFields.length && sortedRequestedFields[requestedIndex] == targetFieldId); - - currentEntry = dirIterator.hasNext() ? dirIterator.next() : null; - } else if (fieldId < targetFieldId) { - // Directory field is smaller, advance directory pointer - currentEntry = dirIterator.hasNext() ? dirIterator.next() : null; - } else { - // fieldId > targetFieldId - implies requested field isn't in the directory so advance requested pointer - requestedIndex++; + + static void validateImprintBuffer(ImprintBuffer buffer, String paramName) throws ImprintException { + if (buffer == null) { + throw new ImprintException(ErrorType.INVALID_BUFFER, paramName + " cannot be null"); + } + + if (buffer.remaining() < Constants.HEADER_BYTES) { + throw new ImprintException(ErrorType.INVALID_BUFFER, + paramName + " too small to contain valid Imprint header (minimum " + Constants.HEADER_BYTES + " bytes)"); + } + + // Check magic and version using ImprintBuffer operations without modifying original buffer + var validationBuffer = buffer.duplicate(); + validationBuffer.position(0); + byte magic = validationBuffer.get(); + byte version = validationBuffer.get(); + + if (magic != Constants.MAGIC) { + throw new ImprintException(ErrorType.INVALID_BUFFER, paramName + " does not contain valid Imprint magic byte"); + } + if (version != Constants.VERSION) { + throw new ImprintException(ErrorType.INVALID_BUFFER, paramName + " contains unsupported Imprint version: " + version); } } - - return buildSerializedBuffer(originalHeader, projectedEntries, payloadChunks, totalProjectedPayloadSize); - } - /** - * Build a serialized Imprint record buffer from header, directory entries, and payload chunks. - */ - private static ByteBuffer buildSerializedBuffer(Header originalHeader, RawDirectoryEntry[] directoryEntries, ByteBuffer[] payloadChunks) { - return buildSerializedBuffer(originalHeader, Arrays.asList(directoryEntries), Arrays.asList(payloadChunks), 0); - } - - private static ByteBuffer buildSerializedBuffer(Header originalHeader, List directoryEntries, List payloadChunks, int totalPayloadSize) { - int directorySize = ImprintRecord.calculateDirectorySize(directoryEntries.size()); - int totalSize = Constants.HEADER_BYTES + directorySize + totalPayloadSize; - var finalBuffer = ByteBuffer.allocate(totalSize); - finalBuffer.order(ByteOrder.LITTLE_ENDIAN); - - // header (preserve original schema) - finalBuffer.put(Constants.MAGIC); - finalBuffer.put(Constants.VERSION); - finalBuffer.put(originalHeader.getFlags().getValue()); - finalBuffer.putInt(originalHeader.getSchemaId().getFieldSpaceId()); - finalBuffer.putInt(originalHeader.getSchemaId().getSchemaHash()); - finalBuffer.putInt(totalPayloadSize); - - // directory - VarInt.encode(directoryEntries.size(), finalBuffer); - for (var entry : directoryEntries) { - finalBuffer.putShort(entry.fieldId); - finalBuffer.put(entry.typeCode); - finalBuffer.putInt(entry.offset); + /** + * Create an empty record as ImprintBuffer. + */ + static ImprintBuffer createEmptyRecordBytes() { + // header + empty directory + empty payload + var buffer = ImprintBuffer.growable(Constants.HEADER_BYTES + 1); // +1 for varint 0 + + // header for empty record + buffer.putUnsafeByte(Constants.MAGIC); + buffer.putUnsafeByte(Constants.VERSION); + buffer.putUnsafeByte((byte) 0x01); + buffer.putUnsafeInt(0); + buffer.putUnsafeInt(0); + buffer.putUnsafeInt(0); + + // empty directory + buffer.putVarInt(0); + + // Set final position and limit + int finalSize = buffer.position(); + buffer.position(0); + buffer.limit(finalSize); + return buffer; + } + + static int getFieldSizeOptimized(ImprintBuffer payload, int startOffset, ImprintBufferDirectoryIterator iterator) { + int endOffset = iterator.getNextEntryOffset(payload.remaining()); + return endOffset - startOffset; + } + + static void copyField(ImprintBuffer output, ImprintBuffer source, int offset, int size) { + if (size <= 0) return; + output.putBytes(source.array(), source.arrayOffset() + offset, size); } - - // payload - for (var chunk : payloadChunks) - finalBuffer.put(chunk); - finalBuffer.flip(); - return finalBuffer.asReadOnlyBuffer(); + static ImprintBuffer buildFinalBuffer(Header header, List entries, + ImprintBuffer output, int estimatedDirectorySize, + int actualPayloadSize, int headerAndDirSize) { + + // Calculate actual directory size + int actualDirectorySize = ImprintRecord.calculateDirectorySize(entries.size()); + int actualTotalSize = Constants.HEADER_BYTES + actualDirectorySize + actualPayloadSize; + + // If directory size changed, move payload using UNSAFE + int payloadStartPos = Constants.HEADER_BYTES + actualDirectorySize; + if (actualDirectorySize != estimatedDirectorySize) { + output.moveMemory(headerAndDirSize, payloadStartPos, actualPayloadSize); + } + + // Write header and directory at the beginning + output.position(0); + output.putUnsafeByte(Constants.MAGIC); + output.putUnsafeByte(Constants.VERSION); + output.putUnsafeByte(header.getFlags().getValue()); + output.putUnsafeInt(header.getSchemaId().getFieldSpaceId()); + output.putUnsafeInt(header.getSchemaId().getSchemaHash()); + output.putUnsafeInt(actualPayloadSize); + + // Write directory + output.putVarInt(entries.size()); + for (var entry : entries) { + output.putUnsafeShort(entry.fieldId); + output.putUnsafeByte(entry.typeCode); + output.putUnsafeInt(entry.offset); + } + + output.position(0); + output.limit(actualTotalSize); + return output; + } } + + // ========== MERGE OPERATIONS ========== - - /** - * Create an empty record as serialized bytes - */ - private static ByteBuffer createEmptyRecordBytes() { - // header + empty directory + empty payload - var buffer = ByteBuffer.allocate(Constants.HEADER_BYTES + 1); // +1 for varint 0 - buffer.order(ByteOrder.LITTLE_ENDIAN); - - // header for empty record - buffer.put(Constants.MAGIC); - buffer.put(Constants.VERSION); - buffer.put((byte) 0x01); - buffer.putInt(0); - buffer.putInt(0); - buffer.putInt(0); - - // empty directory - VarInt.encode(0, buffer); - - buffer.flip(); - return buffer.asReadOnlyBuffer(); - } + public static class Merge { - /** - * Validates that a ByteBuffer contains valid Imprint data by checking magic bytes and basic structure. - * - * @param buffer Buffer to validate - * @param paramName Parameter name for error messages - * @throws ImprintException if buffer is invalid - */ - private static void validateImprintBuffer(ByteBuffer buffer, String paramName) throws ImprintException { - if (buffer == null) { - throw new ImprintException(ErrorType.INVALID_BUFFER, paramName + " cannot be null"); + public static ImprintBuffer mergeBytes(ImprintBuffer firstBuffer, ImprintBuffer secondBuffer) throws ImprintException { + Core.validateImprintBuffer(firstBuffer, "firstBuffer"); + Core.validateImprintBuffer(secondBuffer, "secondBuffer"); + + var firstHeader = Core.parseHeaderFromImprintBuffer(firstBuffer); + var secondHeader = Core.parseHeaderFromImprintBuffer(secondBuffer); + + var firstSections = Core.extractSectionsFromImprintBuffer(firstBuffer, firstHeader); + var secondSections = Core.extractSectionsFromImprintBuffer(secondBuffer, secondHeader); + + return mergeRawSections(firstHeader, firstSections, secondSections); } - - if (buffer.remaining() < Constants.HEADER_BYTES) { - throw new ImprintException(ErrorType.INVALID_BUFFER, - paramName + " too small to contain valid Imprint header (minimum " + Constants.HEADER_BYTES + " bytes)"); + + private static ImprintBuffer mergeRawSections(Header firstHeader, Core.ImprintBufferSections firstSections, Core.ImprintBufferSections secondSections) { + + // Estimate reasonable initial size + int maxFields = firstSections.directoryCount + secondSections.directoryCount; + int estimatedDirectorySize = ImprintRecord.calculateDirectorySize(maxFields); + int estimatedPayloadSize = firstSections.payloadBuffer.remaining() + secondSections.payloadBuffer.remaining(); + int estimatedTotalSize = Constants.HEADER_BYTES + estimatedDirectorySize + estimatedPayloadSize; + + // 1. Create growable buffer + var output = ImprintBuffer.growable(estimatedTotalSize); + // Reserve space for header and directory - write payload first + int headerAndDirSize = Constants.HEADER_BYTES + estimatedDirectorySize; + output.position(headerAndDirSize); + + // 2. Merge fields directly into growable buffer + var mergedEntries = new ArrayList(maxFields); + int actualPayloadSize = mergeFieldsDirectToBuffer(output, firstSections, secondSections, mergedEntries); + + // 3. Build and finalize the buffer + return Core.buildFinalBuffer(firstHeader, mergedEntries, output, + estimatedDirectorySize, actualPayloadSize, headerAndDirSize); } - // Check invariants without advancing buffer position - var duplicate = buffer.duplicate(); - byte magic = duplicate.get(); - byte version = duplicate.get(); - if (magic != Constants.MAGIC) - throw new ImprintException(ErrorType.INVALID_BUFFER, paramName + " does not contain valid Imprint magic byte"); - if (version != Constants.VERSION) - throw new ImprintException(ErrorType.INVALID_BUFFER, paramName + " contains unsupported Imprint version: " + version); - } + private static int mergeFieldsDirectToBuffer(ImprintBuffer output, Core.ImprintBufferSections firstSections, Core.ImprintBufferSections secondSections, List mergedEntries) { - /** - * Directory entry container used for raw byte operations - */ - @Value - private static class RawDirectoryEntry { - short fieldId; - byte typeCode; - int offset; - } + var firstIter = new Core.ImprintBufferDirectoryIterator(firstSections.directoryBuffer); + var secondIter = new Core.ImprintBufferDirectoryIterator(secondSections.directoryBuffer); - /** - * Iterator that parses directory entries directly from raw bytes - */ - private static class RawDirectoryIterator { - private final ByteBuffer buffer; - private final int totalCount; - private final int directoryStartPos; - private int currentIndex; - - RawDirectoryIterator(ByteBuffer directoryBuffer) throws ImprintException { - this.buffer = directoryBuffer.duplicate().order(ByteOrder.LITTLE_ENDIAN); - - // Read count and advance to first entry - var countResult = VarInt.decode(buffer); - this.totalCount = countResult.getValue(); - this.directoryStartPos = buffer.position(); - this.currentIndex = 0; + int currentOffset = 0; + int initialPosition = output.position(); + + var firstEntry = firstIter.hasNext() ? firstIter.next() : null; + var secondEntry = secondIter.hasNext() ? secondIter.next() : null; + + while (firstEntry != null || secondEntry != null) { + Core.RawDirectoryEntry selectedEntry; + ImprintBuffer sourcePayload; + Core.ImprintBufferDirectoryIterator sourceIter; + + if (firstEntry != null && (secondEntry == null || firstEntry.fieldId <= secondEntry.fieldId)) { + selectedEntry = firstEntry; + sourcePayload = firstSections.payloadBuffer; + sourceIter = firstIter; + + if (secondEntry != null && firstEntry.fieldId == secondEntry.fieldId) { + secondEntry = secondIter.hasNext() ? secondIter.next() : null; + } + + firstEntry = firstIter.hasNext() ? firstIter.next() : null; + } else { + selectedEntry = secondEntry; + sourcePayload = secondSections.payloadBuffer; + sourceIter = secondIter; + + secondEntry = secondIter.hasNext() ? secondIter.next() : null; + } + + int fieldSize = Core.getFieldSizeOptimized(sourcePayload, selectedEntry.offset, sourceIter); + Core.copyField(output, sourcePayload, selectedEntry.offset, fieldSize); + + mergedEntries.add(new Core.RawDirectoryEntry(selectedEntry.fieldId, selectedEntry.typeCode, currentOffset)); + currentOffset += fieldSize; + } + + return output.position() - initialPosition; } - - boolean hasNext() { - return currentIndex < totalCount; + } + + public static class Project { + + public static ImprintBuffer projectBytes(ImprintBuffer sourceBuffer, int... fieldIds) throws ImprintException { + Core.validateImprintBuffer(sourceBuffer, "sourceBuffer"); + if (fieldIds == null || fieldIds.length == 0) + return Core.createEmptyRecordBytes(); + + var sortedFieldIds = fieldIds.clone(); + Arrays.sort(sortedFieldIds); + var header = Core.parseHeaderFromImprintBuffer(sourceBuffer); + var sections = Core.extractSectionsFromImprintBuffer(sourceBuffer, header); + return projectRawSections(header, sections, sortedFieldIds); } - - RawDirectoryEntry next() throws ImprintException { - if (!hasNext()) - throw new RuntimeException("No more directory entries"); - if (buffer.remaining() < Constants.DIR_ENTRY_BYTES) - throw new ImprintException(ErrorType.BUFFER_UNDERFLOW, "Not enough bytes for directory entry"); + private static ImprintBuffer projectRawSections(Header originalHeader, Core.ImprintBufferSections sections, int[] sortedRequestedFields) { + if (sortedRequestedFields.length == 0) + return Core.createEmptyRecordBytes(); - short fieldId = buffer.getShort(); - byte typeCode = buffer.get(); - int offset = buffer.getInt(); - - currentIndex++; - return new RawDirectoryEntry(fieldId, typeCode, offset); + // Estimate reasonable initial size + int maxFields = sortedRequestedFields.length; + int estimatedDirectorySize = ImprintRecord.calculateDirectorySize(maxFields); + int estimatedPayloadSize = sections.payloadBuffer.remaining(); // Conservative estimate + int estimatedTotalSize = Constants.HEADER_BYTES + estimatedDirectorySize + estimatedPayloadSize; + + // 1. Create growable buffer + var output = ImprintBuffer.growable(estimatedTotalSize); + // Reserve space for header and directory - write payload first + int headerAndDirSize = Constants.HEADER_BYTES + estimatedDirectorySize; + output.position(headerAndDirSize); + + // 2. Project fields directly into growable buffer + var projectedEntries = new ArrayList(maxFields); + int actualPayloadSize = projectFieldsDirectToBuffer(output, sections, sortedRequestedFields, projectedEntries); + + // 3. Build and finalize the buffer + return Core.buildFinalBuffer(originalHeader, projectedEntries, output, estimatedDirectorySize, actualPayloadSize, headerAndDirSize); } - - /** - * Get the offset of the next entry without state overhead. - * Returns the provided fallback if this is the last entry. - */ - int getNextEntryOffset(int fallbackOffset) { - if (currentIndex >= totalCount) - return fallbackOffset; - // Calculate position of next entry directly - int nextEntryPos = directoryStartPos + (currentIndex * Constants.DIR_ENTRY_BYTES); + private static int projectFieldsDirectToBuffer(ImprintBuffer output, Core.ImprintBufferSections sections, int[] sortedRequestedFields, List projectedEntries) { + var dirIterator = new Core.ImprintBufferDirectoryIterator(sections.directoryBuffer); - // Bounds check - optimized to single comparison - if (nextEntryPos + 7 > buffer.limit()) { // DIR_ENTRY_BYTES = 7 - return fallbackOffset; + int currentOffset = 0; + int initialPosition = output.position(); + int requestedIndex = 0; + + // Merge algorithm: two-pointer approach through sorted sequences + var currentEntry = dirIterator.hasNext() ? dirIterator.next() : null; + + while (currentEntry != null && requestedIndex < sortedRequestedFields.length) { + int fieldId = currentEntry.fieldId; + int targetFieldId = sortedRequestedFields[requestedIndex]; + + if (fieldId == targetFieldId) { + int fieldSize = Core.getFieldSizeOptimized(sections.payloadBuffer, currentEntry.offset, dirIterator); + Core.copyField(output, sections.payloadBuffer, currentEntry.offset, fieldSize); + + projectedEntries.add(new Core.RawDirectoryEntry(currentEntry.fieldId, currentEntry.typeCode, currentOffset)); + currentOffset += fieldSize; + + // Advance both pointers - handle dupes by advancing to next unique field + do { + requestedIndex++; + } while (requestedIndex < sortedRequestedFields.length && sortedRequestedFields[requestedIndex] == targetFieldId); + + currentEntry = dirIterator.hasNext() ? dirIterator.next() : null; + } else if (fieldId < targetFieldId) { + // Directory field is smaller, advance directory pointer + currentEntry = dirIterator.hasNext() ? dirIterator.next() : null; + } else { + // fieldId > targetFieldId - implies requested field isn't in the directory so advance requested pointer + requestedIndex++; + } } - - // Read just the offset field (skip fieldId and typeCode) - return buffer.getInt(nextEntryPos + 3); // 2 bytes fieldId + 1 byte typeCode = 3 offset + return output.position() - initialPosition; } } -} +} \ No newline at end of file diff --git a/src/main/java/com/imprint/types/ImprintDeserializers.java b/src/main/java/com/imprint/types/ImprintDeserializers.java index 18f561d..c69ee6d 100644 --- a/src/main/java/com/imprint/types/ImprintDeserializers.java +++ b/src/main/java/com/imprint/types/ImprintDeserializers.java @@ -2,6 +2,7 @@ import com.imprint.error.ErrorType; import com.imprint.error.ImprintException; +import com.imprint.util.ImprintBuffer; import com.imprint.util.VarInt; import lombok.experimental.UtilityClass; @@ -14,8 +15,8 @@ @UtilityClass public final class ImprintDeserializers { - // Primitive boxed deserializers - public static Object deserializePrimitive(ByteBuffer buffer, TypeCode typeCode) throws ImprintException { + // Primitive deserializers + public static Object deserializePrimitive(ImprintBuffer buffer, TypeCode typeCode) throws ImprintException { switch (typeCode) { case NULL: return null; diff --git a/src/main/java/com/imprint/types/ImprintSerializers.java b/src/main/java/com/imprint/types/ImprintSerializers.java index f76f444..9ff338b 100644 --- a/src/main/java/com/imprint/types/ImprintSerializers.java +++ b/src/main/java/com/imprint/types/ImprintSerializers.java @@ -2,11 +2,14 @@ import com.imprint.error.ErrorType; import com.imprint.error.ImprintException; -import com.imprint.util.VarInt; +import com.imprint.util.ImprintBuffer; import lombok.experimental.UtilityClass; import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Map; +import java.util.function.BiConsumer; +import java.util.function.Function; /** * Static serialization methods for all Imprint types. @@ -15,67 +18,65 @@ @UtilityClass public final class ImprintSerializers { - - // Primitive serializers - public static void serializeBool(boolean value, ByteBuffer buffer) { - buffer.put((byte) (value ? 1 : 0)); + @SuppressWarnings("unused") + public static void serializeNull(ByteBuffer buffer) { + // NULL values have no payload data so this is here really only for consistency + } + + public static void serializeBool(boolean value, ImprintBuffer buffer) { + buffer.putByte((byte) (value ? 1 : 0)); } - public static void serializeInt32(int value, ByteBuffer buffer) { + public static void serializeInt32(int value, ImprintBuffer buffer) { buffer.putInt(value); } - public static void serializeInt64(long value, ByteBuffer buffer) { + public static void serializeInt64(long value, ImprintBuffer buffer) { buffer.putLong(value); } - public static void serializeFloat32(float value, ByteBuffer buffer) { + public static void serializeFloat32(float value, ImprintBuffer buffer) { buffer.putFloat(value); } - public static void serializeFloat64(double value, ByteBuffer buffer) { + public static void serializeFloat64(double value, ImprintBuffer buffer) { buffer.putDouble(value); } - public static void serializeString(String value, ByteBuffer buffer) { - byte[] utf8Bytes = value.getBytes(StandardCharsets.UTF_8); - VarInt.encode(utf8Bytes.length, buffer); - buffer.put(utf8Bytes); + public static void serializeString(String value, ImprintBuffer buffer) { + buffer.putString(value); } - public static void serializeBytes(byte[] value, ByteBuffer buffer) { - VarInt.encode(value.length, buffer); - buffer.put(value); + public static void serializeBytes(byte[] value, ImprintBuffer buffer) { + buffer.putVarInt(value.length); + buffer.putBytes(value); } - public static void serializeArray(java.util.List list, ByteBuffer buffer, - java.util.function.Function typeConverter, - java.util.function.BiConsumer elementSerializer) throws ImprintException { - VarInt.encode(list.size(), buffer); + public static void serializeArray(List list, ImprintBuffer buffer, Function typeConverter, BiConsumer elementSerializer) + throws ImprintException { + buffer.putVarInt(list.size()); - if (list.isEmpty()) return; // Empty arrays don't need type code + if (list.isEmpty()) + return; // Empty arrays technically don't need type code // Convert first element to determine element type - Object firstElement = list.get(0); - TypeCode firstTypeCode = typeConverter.apply(firstElement); - buffer.put(firstTypeCode.getCode()); + var firstElement = list.get(0); + var firstTypeCode = typeConverter.apply(firstElement); + buffer.putByte(firstTypeCode.getCode()); - // Serialize all elements - they must be same type - for (Object element : list) { - TypeCode elementTypeCode = typeConverter.apply(element); + // Serialize all elements (must be homogenous collections) + for (var element : list) { + var elementTypeCode = typeConverter.apply(element); if (elementTypeCode != firstTypeCode) { - throw new ImprintException(ErrorType.SCHEMA_ERROR, - "Array elements must have same type"); + throw new ImprintException(ErrorType.SCHEMA_ERROR, "Array elements must have same type"); } elementSerializer.accept(element, buffer); } } - public static void serializeMap(java.util.Map map, ByteBuffer buffer, - java.util.function.Function keyConverter, - java.util.function.Function typeConverter, - java.util.function.BiConsumer valueSerializer) throws ImprintException { - VarInt.encode(map.size(), buffer); + public static void serializeMap(Map map, ImprintBuffer buffer, Function keyConverter, Function typeConverter, + BiConsumer valueSerializer) throws ImprintException { + buffer.putVarInt(map.size()); if (map.isEmpty()) return; @@ -83,11 +84,11 @@ public static void serializeMap(java.util.Map map, ByteBuffer buffer, var first = iterator.next(); // Convert key and value to determine types - MapKey firstKey = keyConverter.apply(first.getKey()); - TypeCode firstValueType = typeConverter.apply(first.getValue()); + var firstKey = keyConverter.apply(first.getKey()); + var firstValueType = typeConverter.apply(first.getValue()); - buffer.put(firstKey.getTypeCode().getCode()); - buffer.put(firstValueType.getCode()); + buffer.putByte(firstKey.getTypeCode().getCode()); + buffer.putByte(firstValueType.getCode()); // Serialize first pair serializeMapKeyDirect(firstKey, buffer); @@ -96,16 +97,14 @@ public static void serializeMap(java.util.Map map, ByteBuffer buffer, // Serialize remaining pairs while (iterator.hasNext()) { var entry = iterator.next(); - MapKey key = keyConverter.apply(entry.getKey()); - TypeCode valueType = typeConverter.apply(entry.getValue()); + var key = keyConverter.apply(entry.getKey()); + var valueType = typeConverter.apply(entry.getValue()); if (key.getTypeCode() != firstKey.getTypeCode()) { - throw new ImprintException(ErrorType.SCHEMA_ERROR, - "Map keys must have same type"); + throw new ImprintException(ErrorType.SCHEMA_ERROR, "Map keys must have same type"); } if (valueType != firstValueType) { - throw new ImprintException(ErrorType.SCHEMA_ERROR, - "Map values must have same type"); + throw new ImprintException(ErrorType.SCHEMA_ERROR, "Map values must have same type"); } serializeMapKeyDirect(key, buffer); @@ -113,7 +112,7 @@ public static void serializeMap(java.util.Map map, ByteBuffer buffer, } } - private static void serializeMapKeyDirect(MapKey key, ByteBuffer buffer) throws ImprintException { + private static void serializeMapKeyDirect(MapKey key, ImprintBuffer buffer) throws ImprintException { switch (key.getTypeCode()) { case INT32: buffer.putInt(((MapKey.Int32Key) key).getValue()); @@ -123,50 +122,40 @@ private static void serializeMapKeyDirect(MapKey key, ByteBuffer buffer) throws break; case BYTES: byte[] bytes = ((MapKey.BytesKey) key).getValue(); - VarInt.encode(bytes.length, buffer); - buffer.put(bytes); + buffer.putVarInt(bytes.length); + buffer.putBytes(bytes); break; case STRING: String str = ((MapKey.StringKey) key).getValue(); - byte[] stringBytes = str.getBytes(StandardCharsets.UTF_8); - VarInt.encode(stringBytes.length, buffer); - buffer.put(stringBytes); + buffer.putString(str); break; default: - throw new ImprintException(ErrorType.SERIALIZATION_ERROR, - "Invalid map key type: " + key.getTypeCode()); + throw new ImprintException(ErrorType.SERIALIZATION_ERROR, "Invalid map key type: " + key.getTypeCode()); } } @SuppressWarnings("unused") - public static void serializeNull(ByteBuffer buffer) { - // NULL values have no payload data but the method helps intent + public static void serializeNull(ImprintBuffer buffer) { + // NULL values have no payload data } - // Rough size estimate since actual takes time; might be able to accomodate this better with a growable buffer though + public static int estimateSize(TypeCode typeCode, Object value) { - switch (typeCode) { - case NULL: return 0; - case BOOL: return 1; - case INT32: - case FLOAT32: - return 4; - case INT64: - case FLOAT64: - return 8; - case STRING: - String str = (String) value; - return str.length() > 1000 ? 5 + str.length() * 3 : 256; - case BYTES: - byte[] bytes = (byte[]) value; - return bytes.length > 1000 ? 5 + bytes.length : 256; - case ARRAY: - case MAP: - return 512; //just rough estimate/guess for now; - case ROW: - return 1024; - default: - return 64; + byte code = typeCode.getCode(); + if (code == TypeCode.INT32.getCode() || code == TypeCode.FLOAT32.getCode()) return 4; + if (code == TypeCode.INT64.getCode() || code == TypeCode.FLOAT64.getCode()) return 8; + if (code == TypeCode.BOOL.getCode()) return 1; + if (code == TypeCode.NULL.getCode()) return 0; + if (code == TypeCode.STRING.getCode()) { + var str = (String) value; + return str.length() > 1000 ? 5 + str.length() * 3 : 256; + } + if (code == TypeCode.BYTES.getCode()) { + var bytes = (byte[]) value; + return bytes.length > 1000 ? 5 + bytes.length : 256; } + if (code == TypeCode.ARRAY.getCode() || code == TypeCode.MAP.getCode()) return 512; + if (code == TypeCode.ROW.getCode()) return 1024; + throw new IllegalArgumentException("Unknown TypeCode: " + typeCode); } } \ No newline at end of file diff --git a/src/main/java/com/imprint/types/MapKey.java b/src/main/java/com/imprint/types/MapKey.java index 5961f4b..640d26b 100644 --- a/src/main/java/com/imprint/types/MapKey.java +++ b/src/main/java/com/imprint/types/MapKey.java @@ -49,7 +49,8 @@ public static MapKey fromPrimitive(TypeCode typeCode, Object primitiveValue) thr case STRING: return fromString((String) primitiveValue); default: - throw new ImprintException(ErrorType.TYPE_MISMATCH, "Cannot convert " + typeCode + " to MapKey"); + throw new ImprintException(ErrorType.TYPE_MISMATCH, + "Cannot convert " + typeCode + " to MapKey"); } } diff --git a/src/main/java/com/imprint/types/TypeCode.java b/src/main/java/com/imprint/types/TypeCode.java index 7c80d87..3304d11 100644 --- a/src/main/java/com/imprint/types/TypeCode.java +++ b/src/main/java/com/imprint/types/TypeCode.java @@ -7,7 +7,6 @@ /** * Type codes for Imprint values. */ -@Getter public enum TypeCode { NULL(0x0), BOOL(0x1), @@ -21,6 +20,7 @@ public enum TypeCode { MAP(0x9), ROW(0xA); // TODO: implement (basically a placeholder for user-defined type) + @Getter private final byte code; private static final TypeCode[] LOOKUP = new TypeCode[11]; @@ -38,7 +38,8 @@ public enum TypeCode { public static TypeCode fromByte(byte code) throws ImprintException { if (code >= 0 && code < LOOKUP.length) { var type = LOOKUP[code]; - if (type != null) return type; + if (type != null) + return type; } throw new ImprintException(ErrorType.INVALID_TYPE_CODE, "Unknown type code: 0x" + Integer.toHexString(code & 0xFF)); } diff --git a/src/main/java/com/imprint/util/ImprintBuffer.java b/src/main/java/com/imprint/util/ImprintBuffer.java new file mode 100644 index 0000000..a4c7e77 --- /dev/null +++ b/src/main/java/com/imprint/util/ImprintBuffer.java @@ -0,0 +1,605 @@ +package com.imprint.util; + +import com.imprint.error.ImprintException; +import lombok.Getter; +import sun.misc.Unsafe; + +import java.lang.reflect.Field; +import java.nio.charset.StandardCharsets; + +@SuppressWarnings({"UnusedReturnValue", "unused"}) +public final class ImprintBuffer { + + // Bounds checking control + private static final boolean BOUNDS_CHECK = Boolean.parseBoolean( + System.getProperty("imprint.buffer.bounds.check", "false")); + + private byte[] array; + private long baseOffset; + private int capacity; + private int position; + private int limit; + + @Getter + private final boolean growable; + + // Unsafe instance for direct memory access + private static final Unsafe UNSAFE; + private static final long ARRAY_BYTE_BASE_OFFSET; + + static { + try { + Field field = sun.misc.Unsafe.class.getDeclaredField("theUnsafe"); + field.setAccessible(true); + UNSAFE = (Unsafe) field.get(null); + ARRAY_BYTE_BASE_OFFSET = UNSAFE.arrayBaseOffset(byte[].class); + } catch (Exception e) { + throw new RuntimeException("Failed to access Unsafe", e); + } + } + + /** + * Create buffer wrapping a byte array. + */ + public ImprintBuffer(byte[] array) { + this(array, false); + } + + /** + * Create buffer wrapping a byte array with optional growth capability. + */ + public ImprintBuffer(byte[] array, boolean growable) { + // Inline the common case to avoid constructor delegation overhead + if (BOUNDS_CHECK && array == null) { + throw new IllegalArgumentException("Array cannot be null"); + } + + this.array = array; + this.baseOffset = ARRAY_BYTE_BASE_OFFSET; + this.capacity = array.length; + this.position = 0; + this.limit = array.length; + this.growable = growable; + } + + /** + * Create buffer wrapping a portion of byte array. + */ + public ImprintBuffer(byte[] array, int offset, int length) { + this(array, offset, length, false); + } + + /** + * Create buffer wrapping a portion of byte array with optional growth capability. + */ + public ImprintBuffer(byte[] array, int offset, int length, boolean growable) { + if (BOUNDS_CHECK) { + if (offset < 0 || length < 0 || offset + length > array.length) { + throw new IllegalArgumentException("Invalid offset/length"); + } + } + + this.array = array; + this.baseOffset = ARRAY_BYTE_BASE_OFFSET + offset; + this.capacity = length; + this.position = 0; + this.limit = length; + this.growable = growable; + } + + + /** + * Create growable buffer with initial capacity. + */ + public static ImprintBuffer growable(int initialCapacity) { + return new ImprintBuffer(new byte[initialCapacity], true); + } + + /** + * Get current position. + */ + public int position() { + return position; + } + + /** + * Set position. + */ + public ImprintBuffer position(int newPosition) { + if (BOUNDS_CHECK && (newPosition < 0 || newPosition > limit)) { + throw new IllegalArgumentException("Invalid position: " + newPosition); + } + this.position = newPosition; + return this; + } + + /** + * Get limit. + */ + public int limit() { + return limit; + } + + /** + * Set limit. + */ + public ImprintBuffer limit(int newLimit) { + if (BOUNDS_CHECK && (newLimit < 0 || newLimit > capacity)) { + throw new IllegalArgumentException("Invalid limit: " + newLimit); + } + this.limit = newLimit; + return this; + } + + /** + * Get remaining bytes. + */ + public int remaining() { + return limit - position; + } + + /** + * Check if buffer has remaining bytes. + */ + public boolean hasRemaining() { + return position < limit; + } + + /** + * Get capacity. + */ + public int capacity() { + return capacity; + } + + /** + * Ensure buffer has space for additional bytes. + */ + private void ensureCapacity(int additionalBytes) { + if (!growable) return; + + int requiredCapacity = position + additionalBytes; + if (requiredCapacity <= capacity) return; + + // Buffer growth strategy: 1.5x with 64-byte alignment - should help with cache efficiency + int newCapacity = Math.max(requiredCapacity, (capacity * 3) / 2); + newCapacity = (newCapacity + 63) & ~63; // Align to 64-byte boundary + + // Allocate new array and copy existing data + byte[] newArray = new byte[newCapacity]; + int currentOffset = (int) (baseOffset - ARRAY_BYTE_BASE_OFFSET); + + UNSAFE.copyMemory(array, baseOffset, newArray, ARRAY_BYTE_BASE_OFFSET + currentOffset, position); + + // Update buffer state + this.array = newArray; + this.baseOffset = ARRAY_BYTE_BASE_OFFSET + currentOffset; + this.capacity = newCapacity - currentOffset; + this.limit = capacity; + } + + /** + * Flip buffer for reading (set limit to position, position to 0). + */ + public ImprintBuffer flip() { + limit = position; + position = 0; + return this; + } + + /** + * Create a slice of this buffer starting from current position. + */ + public ImprintBuffer slice() { + int offset = (int) (baseOffset - ARRAY_BYTE_BASE_OFFSET) + position; + return new ImprintBuffer(array, offset, remaining()); + } + + /** + * Create a duplicate of this buffer. + */ + public ImprintBuffer duplicate() { + int offset = (int) (baseOffset - ARRAY_BYTE_BASE_OFFSET); + var dup = new ImprintBuffer(array, offset, capacity); + dup.position = this.position; + dup.limit = this.limit; + return dup; + } + + /** + * Create read-only view (returns this since ImprintBuffer is write-focused anyways). + */ + public ImprintBuffer asReadOnlyBuffer() { + return duplicate(); + } + + /** + * Write single byte. + */ + public ImprintBuffer putByte(byte value) { + ensureCapacity(1); + if (BOUNDS_CHECK && !growable && position >= limit) { + throw new RuntimeException("Buffer overflow"); + } + + UNSAFE.putByte(array, baseOffset + position, value); + position++; + return this; + } + + /** + * Write single byte. + */ + public ImprintBuffer putUnsafeByte(byte value) { + UNSAFE.putByte(array, baseOffset + position, value); + position++; + return this; + } + + /** + * Write short in little-endian format. + */ + public ImprintBuffer putShort(short value) { + ensureCapacity(2); + if (BOUNDS_CHECK && !growable && position + 2 > limit) { + throw new RuntimeException("Buffer overflow"); + } + + // Write directly - Unsafe already uses native byte order, which is little-endian on x86 + UNSAFE.putShort(array, baseOffset + position, value); + position += 2; + return this; + } + + public ImprintBuffer putUnsafeShort(short value) { + UNSAFE.putShort(array, baseOffset + position, value); + position += 2; + return this; + } + + /** + * Write int32 in little-endian format. + */ + public ImprintBuffer putInt(int value) { + ensureCapacity(4); + if (BOUNDS_CHECK && !growable && position + 4 > limit) { + throw new RuntimeException("Buffer overflow"); + } + UNSAFE.putInt(array, baseOffset + position, value); + position += 4; + return this; + } + + /** + * Write int32 in little-endian format. + */ + public ImprintBuffer putUnsafeInt(int value) { + UNSAFE.putInt(array, baseOffset + position, value); + position += 4; + return this; + } + + /** + * Write int64 in little-endian format. + */ + public ImprintBuffer putUnsafeLong(long value) { + UNSAFE.putLong(array, baseOffset + position, value); + position += 8; + return this; + } + + /** + * Write int64 in little-endian format. + */ + public ImprintBuffer putLong(long value) { + ensureCapacity(8); + if (BOUNDS_CHECK && !growable && position + 8 > limit) { + throw new RuntimeException("Buffer overflow"); + } + + UNSAFE.putLong(array, baseOffset + position, value); + position += 8; + return this; + } + + /** + * Write float32 in little-endian format. + */ + public ImprintBuffer putFloat(float value) { + return putInt(Float.floatToRawIntBits(value)); + } + + /** + * Write float64 in little-endian format. + */ + public ImprintBuffer putDouble(double value) { + return putLong(Double.doubleToRawLongBits(value)); + } + + /** + * Write byte array. + */ + public ImprintBuffer putBytes(byte[] src) { + return putBytes(src, 0, src.length); + } + + /** + * Write portion of byte array. + */ + public ImprintBuffer putBytes(byte[] src, int srcOffset, int length) { + ensureCapacity(length); + if (BOUNDS_CHECK) { + if (!growable && position + length > limit) { + throw new RuntimeException("Buffer overflow"); + } + if (srcOffset < 0 || length < 0 || srcOffset + length > src.length) { + throw new IllegalArgumentException("Invalid src parameters"); + } + } + + // Bulk copy via Unsafe + UNSAFE.copyMemory(src, ARRAY_BYTE_BASE_OFFSET + srcOffset, + array, baseOffset + position, length); + position += length; + return this; + } + + /** + * Read single byte. + */ + public byte get() { + if (BOUNDS_CHECK && position >= limit) { + throw new RuntimeException("Buffer underflow"); + } + + byte value = UNSAFE.getByte(array, baseOffset + position); + position++; + return value; + } + + /** + * Read short in little-endian format. + */ + public short getShort() { + if (BOUNDS_CHECK && position + 2 > limit) { + throw new RuntimeException("Buffer underflow"); + } + + short value = UNSAFE.getShort(array, baseOffset + position); + position += 2; + return value; + } + + public int getIntUnsafe(int position) { + return UNSAFE.getInt(array, ARRAY_BYTE_BASE_OFFSET + arrayOffset() + position); + } + + public short getShortUnsafe(int position) { + return UNSAFE.getShort(array, ARRAY_BYTE_BASE_OFFSET + arrayOffset() + position); + } + + public byte getByteUnsafe(int position) { + return UNSAFE.getByte(array, ARRAY_BYTE_BASE_OFFSET + arrayOffset() + position); + } + + /** + * Read int32 in little-endian format. + */ + public int getInt() { + if (BOUNDS_CHECK && position + 4 > limit) { + throw new RuntimeException("Buffer underflow"); + } + + int value = UNSAFE.getInt(array, baseOffset + position); + position += 4; + return value; + } + + /** + * Read int64 in little-endian format. + */ + public long getLong() { + if (BOUNDS_CHECK && position + 8 > limit) { + throw new RuntimeException("Buffer underflow"); + } + + long value = UNSAFE.getLong(array, baseOffset + position); + position += 8; + return value; + } + + /** + * Read float32 in little-endian format. + */ + public float getFloat() { + return Float.intBitsToFloat(getInt()); + } + + /** + * Read float64 in little-endian format. + */ + public double getDouble() { + return Double.longBitsToDouble(getLong()); + } + + /** + * Read bytes into array. + */ + public ImprintBuffer get(byte[] dst) { + return get(dst, 0, dst.length); + } + + /** + * Read bytes into portion of array. + */ + public ImprintBuffer get(byte[] dst, int dstOffset, int length) { + if (BOUNDS_CHECK) { + if (position + length > limit) { + throw new RuntimeException("Buffer underflow"); + } + if (dstOffset < 0 || length < 0 || dstOffset + length > dst.length) { + throw new IllegalArgumentException("Invalid dst parameters"); + } + } + + // Bulk copy via Unsafe + UNSAFE.copyMemory(array, baseOffset + position, + dst, ARRAY_BYTE_BASE_OFFSET + dstOffset, length); + position += length; + return this; + } + + + /** + * Write VarInt using the centralized VarInt utility class. + */ + public ImprintBuffer putVarInt(int value) { + VarInt.encode(value, this); + return this; + } + + /** + * Read VarInt using the centralized VarInt utility class. + */ + public int getVarInt() { + try { + return VarInt.decode(this).getValue(); + } catch (ImprintException e) { + throw new RuntimeException("Failed to decode VarInt", e); + } + } + + /** + * Write UTF-8 string with length prefix. + * Direct implementation for performance. + */ + public ImprintBuffer putString(String str) { + byte[] utf8Bytes = str.getBytes(StandardCharsets.UTF_8); + putVarInt(utf8Bytes.length); + return putBytes(utf8Bytes); + } + + /** + * Read UTF-8 string with length prefix. + */ + public String getString() { + int length = getVarInt(); + byte[] utf8Bytes = new byte[length]; + get(utf8Bytes); + return new String(utf8Bytes, StandardCharsets.UTF_8); + } + + // ========== UTILITY METHODS ========== + + /** + * Get backing array (for zero-copy operations). + */ + public byte[] array() { + return array; + } + + /** + * Get array offset. + */ + public int arrayOffset() { + return (int) (baseOffset - ARRAY_BYTE_BASE_OFFSET); + } + + public void moveMemory(int srcPos, int dstPos, int length) { + if (length <= 0 || srcPos == dstPos) return; + + long srcAddress = baseOffset + srcPos; + long dstAddress = baseOffset + dstPos; + + // Use UNSAFE copyMemory which handles overlapping regions optimally + UNSAFE.copyMemory(array, srcAddress, array, dstAddress, length); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (obj == null || getClass() != obj.getClass()) return false; + + ImprintBuffer other = (ImprintBuffer) obj; + + // Quick checks first + if (remaining() != other.remaining()) return false; + + // Save positions + int thisPos = this.position; + int otherPos = other.position; + + try { + // Compare byte by byte from current positions + while (hasRemaining() && other.hasRemaining()) { + if (get() != other.get()) { + return false; + } + } + return true; + } finally { + // Restore positions + this.position = thisPos; + other.position = otherPos; + } + } + + @Override + public int hashCode() { + int h = 1; + int savedPos = position; + try { + while (hasRemaining()) { + h = 31 * h + get(); + } + return h; + } finally { + position = savedPos; + } + } + + // ========== BYTEBUFFER COMPATIBILITY METHODS ========== + + /** + * Clear the buffer (set position to 0, limit to capacity). + */ + public ImprintBuffer clear() { + position = 0; + limit = capacity; + return this; + } + + /** + * Rewind the buffer (set position to 0). + */ + public ImprintBuffer rewind() { + position = 0; + return this; + } + + /** + * Put a single byte (ByteBuffer compatibility). + */ + public ImprintBuffer put(byte b) { + return putByte(b); + } + + /** + * Put a byte array (ByteBuffer compatibility). + */ + public ImprintBuffer put(byte[] src) { + return putBytes(src); + } + + /** + * Put a portion of byte array (ByteBuffer compatibility). + */ + public ImprintBuffer put(byte[] src, int offset, int length) { + return putBytes(src, offset, length); + } + + @Override + public String toString() { + return "ImprintBuffer{position=" + position + ", capacity=" + capacity + "}"; + } +} \ No newline at end of file diff --git a/src/main/java/com/imprint/util/VarInt.java b/src/main/java/com/imprint/util/VarInt.java index 29dd4d3..b553b90 100644 --- a/src/main/java/com/imprint/util/VarInt.java +++ b/src/main/java/com/imprint/util/VarInt.java @@ -1,12 +1,10 @@ package com.imprint.util; -import com.imprint.error.ImprintException; import com.imprint.error.ErrorType; -import lombok.*; +import com.imprint.error.ImprintException; +import lombok.Value; import lombok.experimental.UtilityClass; -import java.nio.ByteBuffer; - /** * Utility class for encoding and decoding variable-length integers (VarInt). * Supports encoding/decoding of 32-bit unsigned integers. @@ -23,7 +21,6 @@ public final class VarInt { private static final int[] ENCODED_LENGTHS = new int[CACHE_SIZE]; static { - // Pre-compute encoded lengths for cached values for (int i = 0; i < CACHE_SIZE; i++) { long val = Integer.toUnsignedLong(i); int length = 1; @@ -40,16 +37,14 @@ public final class VarInt { * @param value the value to encode (treated as unsigned) * @param buffer the buffer to write to */ - public static void encode(int value, ByteBuffer buffer) { - // Convert to unsigned long for proper bit manipulation + public static void encode(int value, ImprintBuffer buffer) { long val = Integer.toUnsignedLong(value); - // Encode at least one byte, then continue while value has more bits do { byte b = (byte) (val & SEGMENT_BITS); val >>>= 7; if (val != 0) b |= CONTINUATION_BIT; - buffer.put(b); + buffer.putByte(b); } while (val != 0); } @@ -59,7 +54,7 @@ public static void encode(int value, ByteBuffer buffer) { * @return a DecodeResult containing the decoded value and number of bytes consumed * @throws ImprintException if the VarInt is malformed */ - public static DecodeResult decode(ByteBuffer buffer) throws ImprintException { + public static DecodeResult decode(ImprintBuffer buffer) throws ImprintException { long result = 0; int shift = 0; int bytesRead = 0; diff --git a/src/test/java/com/imprint/IntegrationTest.java b/src/test/java/com/imprint/IntegrationTest.java index cc70873..2b21f8e 100644 --- a/src/test/java/com/imprint/IntegrationTest.java +++ b/src/test/java/com/imprint/IntegrationTest.java @@ -1,7 +1,6 @@ package com.imprint; import com.imprint.core.*; -import com.imprint.types.*; import com.imprint.error.ErrorType; import com.imprint.error.ImprintException; import org.junit.jupiter.api.Test; diff --git a/src/test/java/com/imprint/ops/ImprintOperationsTest.java b/src/test/java/com/imprint/ops/ImprintOperationsTest.java index 4821125..298dcf8 100644 --- a/src/test/java/com/imprint/ops/ImprintOperationsTest.java +++ b/src/test/java/com/imprint/ops/ImprintOperationsTest.java @@ -4,6 +4,7 @@ import com.imprint.core.ImprintRecord; import com.imprint.core.SchemaId; import com.imprint.error.ImprintException; +import com.imprint.util.ImprintBuffer; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Nested; @@ -86,7 +87,7 @@ void shouldMaintainFieldOrderRegardlessOfInputOrder() throws ImprintException { @DisplayName("should handle single field projection") void shouldHandleSingleFieldProjection() throws ImprintException { // When projecting a single field - ImprintRecord projected = multiFieldRecord.project(3); + var projected = multiFieldRecord.project(3); // Then only that field should be present assertEquals(1, projected.getDirectory().size()); @@ -107,9 +108,9 @@ void shouldPreserveAllFieldsWhenProjectingAll() throws ImprintException { // Then all fields should be present with matching values assertEquals(multiFieldRecord.getDirectory().size(), projected.getDirectory().size()); - for (Directory entry : multiFieldRecord.getDirectory()) { - Object originalValue = multiFieldRecord.getValue(entry.getId()); - Object projectedValue = projected.getValue(entry.getId()); + for (var entry : multiFieldRecord.getDirectory()) { + var originalValue = multiFieldRecord.getValue(entry.getId()); + var projectedValue = projected.getValue(entry.getId()); // Handle byte arrays specially since they don't use content equality if (originalValue instanceof byte[] && projectedValue instanceof byte[]) { @@ -126,7 +127,7 @@ void shouldPreserveAllFieldsWhenProjectingAll() throws ImprintException { @DisplayName("should handle empty projection") void shouldHandleEmptyProjection() throws ImprintException { // When projecting no fields - ImprintRecord projected = multiFieldRecord.project(); + var projected = multiFieldRecord.project(); // Then result should be empty but valid assertEquals(0, projected.getDirectory().size()); @@ -137,7 +138,7 @@ void shouldHandleEmptyProjection() throws ImprintException { @DisplayName("should ignore nonexistent fields") void shouldIgnoreNonexistentFields() throws ImprintException { // When projecting mix of existing and non-existing fields - ImprintRecord projected = multiFieldRecord.project(1, 99, 100); + var projected = multiFieldRecord.project(1, 99, 100); // Then only existing fields should be included assertEquals(1, projected.getDirectory().size()); @@ -150,7 +151,7 @@ void shouldIgnoreNonexistentFields() throws ImprintException { @DisplayName("should deduplicate requested fields") void shouldDeduplicateRequestedFields() throws ImprintException { // When projecting the same field multiple times - ImprintRecord projected = multiFieldRecord.project(1, 1, 1); + var projected = multiFieldRecord.project(1, 1, 1); // Then field should only appear once assertEquals(1, projected.getDirectory().size()); @@ -161,7 +162,7 @@ void shouldDeduplicateRequestedFields() throws ImprintException { @DisplayName("should handle projection from empty record") void shouldHandleProjectionFromEmptyRecord() throws ImprintException { // When projecting any fields from empty record - ImprintRecord projected = emptyRecord.project(1, 2, 3); + var projected = emptyRecord.project(1, 2, 3); // Then result should be empty but valid assertEquals(0, projected.getDirectory().size()); @@ -175,7 +176,7 @@ void shouldPreserveExactByteRepresentation() throws ImprintException { byte[] originalBytes = multiFieldRecord.getBytes(7); // When projecting that field - ImprintRecord projected = multiFieldRecord.project(7); + var projected = multiFieldRecord.project(7); // Then the byte representation should be exactly preserved byte[] projectedBytes = projected.getBytes(7); @@ -187,7 +188,7 @@ void shouldPreserveExactByteRepresentation() throws ImprintException { @DisplayName("should reduce payload size when projecting subset") void shouldReducePayloadSizeWhenProjectingSubset() throws ImprintException { // Given a record with large and small fields - ImprintRecord largeRecord = ImprintRecord.builder(testSchema) + var largeRecord = ImprintRecord.builder(testSchema) .field(1, 42) // 4 bytes .field(2, "x".repeat(1000)) // ~1000+ bytes .field(3, 123L) // 8 bytes @@ -197,7 +198,7 @@ void shouldReducePayloadSizeWhenProjectingSubset() throws ImprintException { int originalPayloadSize = largeRecord.getSerializedSize(); // When projecting only the small fields - ImprintRecord projected = largeRecord.project(1, 3); + var projected = largeRecord.project(1, 3); // Then the payload size should be significantly smaller assertTrue(projected.getSerializedSize() < originalPayloadSize, @@ -217,18 +218,18 @@ class MergeOperations { @DisplayName("should merge records with distinct fields") void shouldMergeRecordsWithDistinctFields() throws ImprintException { // Given two records with different fields - ImprintRecord record1 = ImprintRecord.builder(testSchema) + var record1 = ImprintRecord.builder(testSchema) .field(1, 42) .field(3, "hello") .build(); - ImprintRecord record2 = ImprintRecord.builder(testSchema) + var record2 = ImprintRecord.builder(testSchema) .field(2, true) .field(4, 123L) .build(); // When merging the records - ImprintRecord merged = record1.merge(record2); + var merged = record1.merge(record2); // Then all fields should be present assertEquals(4, merged.getDirectory().size()); @@ -249,18 +250,18 @@ void shouldMergeRecordsWithDistinctFields() throws ImprintException { @DisplayName("should merge records with overlapping fields") void shouldMergeRecordsWithOverlappingFields() throws ImprintException { // Given two records with overlapping fields - ImprintRecord record1 = ImprintRecord.builder(testSchema) + var record1 = ImprintRecord.builder(testSchema) .field(2, "first") .field(3, 42) .build(); - ImprintRecord record2 = ImprintRecord.builder(testSchema) + var record2 = ImprintRecord.builder(testSchema) .field(1, true) .field(2, "second") // Overlapping field .build(); // When merging the records - ImprintRecord merged = record1.merge(record2); + var merged = record1.merge(record2); // Then first record's values should take precedence for duplicates assertEquals(3, merged.getDirectory().size()); @@ -273,19 +274,19 @@ void shouldMergeRecordsWithOverlappingFields() throws ImprintException { @DisplayName("should preserve schema id from first record") void shouldPreserveSchemaIdFromFirstRecord() throws ImprintException { // Given two records with different schema IDs - SchemaId schema1 = new SchemaId(1, 0xdeadbeef); - SchemaId schema2 = new SchemaId(1, 0xcafebabe); + var schema1 = new SchemaId(1, 0xdeadbeef); + var schema2 = new SchemaId(1, 0xcafebabe); - ImprintRecord record1 = ImprintRecord.builder(schema1) + var record1 = ImprintRecord.builder(schema1) .field(1, 42) .build(); - ImprintRecord record2 = ImprintRecord.builder(schema2) + var record2 = ImprintRecord.builder(schema2) .field(2, true) .build(); // When merging the records - ImprintRecord merged = record1.merge(record2); + var merged = record1.merge(record2); // Then schema ID from first record should be preserved assertEquals(schema1, merged.getHeader().getSchemaId()); @@ -295,8 +296,8 @@ void shouldPreserveSchemaIdFromFirstRecord() throws ImprintException { @DisplayName("should handle merge with empty record") void shouldHandleMergeWithEmptyRecord() throws ImprintException { // When merging with empty record - ImprintRecord merged1 = multiFieldRecord.merge(emptyRecord); - ImprintRecord merged2 = emptyRecord.merge(multiFieldRecord); + var merged1 = multiFieldRecord.merge(emptyRecord); + var merged2 = emptyRecord.merge(multiFieldRecord); // Then results should contain all original fields assertEquals(multiFieldRecord.getDirectory().size(), merged1.getDirectory().size()); @@ -304,9 +305,9 @@ void shouldHandleMergeWithEmptyRecord() throws ImprintException { // And values should be preserved for (Directory entry : multiFieldRecord.getDirectory()) { - Object originalValue = multiFieldRecord.getValue(entry.getId()); - Object merged1Value = merged1.getValue(entry.getId()); - Object merged2Value = merged2.getValue(entry.getId()); + var originalValue = multiFieldRecord.getValue(entry.getId()); + var merged1Value = merged1.getValue(entry.getId()); + var merged2Value = merged2.getValue(entry.getId()); // Handle byte arrays specially since they don't use content equality if (originalValue instanceof byte[]) { @@ -327,7 +328,7 @@ void shouldHandleMergeWithEmptyRecord() throws ImprintException { @DisplayName("should handle merge of two empty records") void shouldHandleMergeOfTwoEmptyRecords() throws ImprintException { // When merging two empty records - ImprintRecord merged = emptyRecord.merge(emptyRecord); + var merged = emptyRecord.merge(emptyRecord); // Then result should be empty but valid assertEquals(0, merged.getDirectory().size()); @@ -338,18 +339,18 @@ void shouldHandleMergeOfTwoEmptyRecords() throws ImprintException { @DisplayName("should maintain correct payload offsets after merge") void shouldMaintainCorrectPayloadOffsetsAfterMerge() throws ImprintException { // Given records with different field sizes - ImprintRecord record1 = ImprintRecord.builder(testSchema) + var record1 = ImprintRecord.builder(testSchema) .field(1, 42) // 4 bytes .field(3, "hello") // 5+ bytes .build(); - ImprintRecord record2 = ImprintRecord.builder(testSchema) + var record2 = ImprintRecord.builder(testSchema) .field(2, true) // 1 byte .field(4, new byte[]{1, 2, 3, 4, 5}) // 5+ bytes .build(); // When merging - ImprintRecord merged = record1.merge(record2); + var merged = record1.merge(record2); // Then all fields should be accessible with correct values assertEquals(42, merged.getInt32(1)); @@ -588,9 +589,9 @@ void shouldValidateBufferFormatAndRejectInvalidData() throws ImprintException { var validBuffer = validRecord.serializeToBuffer(); // Test invalid magic byte - var invalidMagic = ByteBuffer.allocate(20); - invalidMagic.put((byte) 0x99); // Invalid magic - invalidMagic.put((byte) 0x01); // Valid version + var invalidMagic = new ImprintBuffer(new byte[20]); + invalidMagic.putByte((byte) 0x99); // Invalid magic + invalidMagic.putByte((byte) 0x01); // Valid version invalidMagic.flip(); assertThrows(ImprintException.class, () -> @@ -599,8 +600,8 @@ void shouldValidateBufferFormatAndRejectInvalidData() throws ImprintException { ImprintOperations.projectBytes(invalidMagic, 1)); // Test buffer too small - var tooSmall = ByteBuffer.allocate(5); - tooSmall.put(new byte[]{1, 2, 3, 4, 5}); + var tooSmall = new ImprintBuffer(new byte[5]); + tooSmall.putBytes(new byte[]{1, 2, 3, 4, 5}); tooSmall.flip(); assertThrows(ImprintException.class, () -> @@ -609,9 +610,9 @@ void shouldValidateBufferFormatAndRejectInvalidData() throws ImprintException { ImprintOperations.projectBytes(tooSmall, 1)); // Test invalid version - var invalidVersion = ByteBuffer.allocate(20); - invalidVersion.put((byte) 0x49); // Valid magic - invalidVersion.put((byte) 0x99); // Invalid version + var invalidVersion = new ImprintBuffer(new byte[20]); + invalidVersion.putByte((byte) 0x49); // Valid magic + invalidVersion.putByte((byte) 0x99); // Invalid version invalidVersion.flip(); assertThrows(ImprintException.class, () -> @@ -694,5 +695,4 @@ var record = ImprintRecord.builder(testSchema) assertEquals("field5", projectedRecord.getString(5)); } } - } diff --git a/src/test/java/com/imprint/profile/ProfilerTest.java b/src/test/java/com/imprint/profile/ProfilerTest.java index 5c38457..e2e9c9d 100644 --- a/src/test/java/com/imprint/profile/ProfilerTest.java +++ b/src/test/java/com/imprint/profile/ProfilerTest.java @@ -22,10 +22,10 @@ void profileMergeOperations() throws Exception { System.out.println("Starting merge profiler test - attach profiler now..."); Thread.sleep(3000); - profileSmallMerges(); + //profileSmallMerges(); profileLargeMerges(); - profileOverlappingMerges(); - profileDisjointMerges(); + //profileOverlappingMerges(); + //profileDisjointMerges(); } /** @@ -66,7 +66,7 @@ private void profileLargeMerges() throws Exception { var record1 = createTestRecord(100); var record2 = createTestRecord(100); - int iterations = 100_000; + int iterations = 1000000; System.out.printf("Beginning large merge profiling (%,d iterations)...%n", iterations); long start = System.nanoTime(); @@ -261,7 +261,7 @@ private void profileSerialization(String testName, int recordSize, int iteration long start = System.nanoTime(); for (int i = 0; i < iterations; i++) { - var builder = ImprintRecord.builder(schemaId); + var builder = ImprintRecord.builder(schemaId, 512); // Add various field types based on recordSize for (int fieldId = 1; fieldId <= recordSize; fieldId++) { diff --git a/src/test/java/com/imprint/util/ImprintBufferTest.java b/src/test/java/com/imprint/util/ImprintBufferTest.java new file mode 100644 index 0000000..725f32c --- /dev/null +++ b/src/test/java/com/imprint/util/ImprintBufferTest.java @@ -0,0 +1,263 @@ +package com.imprint.util; + +import org.junit.jupiter.api.Test; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import static org.junit.jupiter.api.Assertions.*; + +/** + * Test for ImprintBuffer functionality and compatibility with ByteBuffer. + */ +class ImprintBufferTest { + + @Test + void testBasicPrimitiveWrites() { + byte[] array = new byte[64]; + ImprintBuffer buffer = new ImprintBuffer(array); + + // Test all primitive types + buffer.putByte((byte) 0x42) + .putInt(0x12345678) + .putLong(0x123456789ABCDEF0L) + .putFloat(3.14f) + .putDouble(2.718281828); + + // Verify position advanced correctly + assertEquals(1 + 4 + 8 + 4 + 8, buffer.position()); + + + buffer.position(0); + + assertEquals((byte) 0x42, buffer.get()); + assertEquals(0x12345678, buffer.getInt()); + assertEquals(0x123456789ABCDEF0L, buffer.getLong()); + assertEquals(3.14f, buffer.getFloat(), 0.001f); + assertEquals(2.718281828, buffer.getDouble(), 0.000001); + } + + @Test + void testByteArrayWrites() { + byte[] array = new byte[64]; + ImprintBuffer buffer = new ImprintBuffer(array); + + byte[] testData = {1, 2, 3, 4, 5}; + buffer.putBytes(testData); + + assertEquals(5, buffer.position()); + + // Verify data written correctly + for (int i = 0; i < testData.length; i++) { + assertEquals(testData[i], array[i]); + } + } + + @Test + void testVarIntWrites() { + byte[] array = new byte[64]; + ImprintBuffer buffer = new ImprintBuffer(array); + + // Test small value (single byte) + buffer.putVarInt(42); + assertEquals(1, buffer.position()); + assertEquals(42, array[0]); + + // Reset and test larger value + buffer.position(0); + buffer.putVarInt(300); // Requires 2 bytes + assertEquals(2, buffer.position()); + + // Verify VarInt encoding matches our VarInt utility + buffer.position(0); + var compareBuffer = new ImprintBuffer(new byte[64]); + VarInt.encode(300, compareBuffer); + + for (int i = 0; i < compareBuffer.position(); i++) { + assertEquals(compareBuffer.array()[i], array[i], "VarInt encoding mismatch at byte " + i); + } + } + + @Test + void testStringWrites() throws Exception { + byte[] array = new byte[64]; + ImprintBuffer buffer = new ImprintBuffer(array); + + String testString = "Hello"; + buffer.putString(testString); + + // Should write length (VarInt) + UTF-8 bytes + assertTrue(buffer.position() > testString.length()); + + // Verify by reading back with ByteBuffer + buffer.position(0); + + VarInt.DecodeResult lengthResult = VarInt.decode(buffer); + assertEquals(testString.getBytes().length, lengthResult.getValue()); + + byte[] stringBytes = new byte[lengthResult.getValue()]; + buffer.get(stringBytes); + assertEquals(testString, new String(stringBytes)); + } + + @Test + void testVarIntRoundTrip() { + byte[] array = new byte[64]; + ImprintBuffer buffer = new ImprintBuffer(array); + + // Test various VarInt values to ensure putVarInt/getVarInt work correctly after consolidation + int[] testValues = { + 0, // Single byte: 0 + 1, // Single byte: 1 + 127, // Single byte: 127 (max single byte value) + 128, // Two bytes: 128 (min two byte value) + 300, // Two bytes: 300 + 16383, // Two bytes: 16383 (max two byte value) + 16384, // Three bytes: 16384 (min three byte value) + 1048575, // Three bytes: 1048575 (max three byte value) + 16777215, // Four bytes: 16777215 (max four byte value) + 268435455, // Five bytes: 268435455 (max five byte value) + Integer.MAX_VALUE // Five bytes: max int value + }; + + for (int testValue : testValues) { + // Reset buffer for each test + buffer.position(0); + + // Write VarInt using ImprintBuffer method (should delegate to VarInt utility) + buffer.putVarInt(testValue); + int writePosition = buffer.position(); + + // Read back using ImprintBuffer method (should delegate to VarInt utility) + buffer.position(0); + int readValue = buffer.getVarInt(); + int readPosition = buffer.position(); + + // Verify round-trip correctness + assertEquals(testValue, readValue, "VarInt round-trip failed for value: " + testValue); + assertEquals(writePosition, readPosition, "Read/write positions don't match for value: " + testValue); + + // Also verify that our ImprintBuffer methods produce the same result as VarInt utility directly + buffer.position(0); + var directEncodeBuffer = new ImprintBuffer(new byte[64]); + try { + VarInt.encode(testValue, directEncodeBuffer); + + // Compare the encoded bytes + for (int i = 0; i < writePosition; i++) { + assertEquals(directEncodeBuffer.array()[i], array[i], + "Direct VarInt encoding differs from ImprintBuffer encoding at byte " + i + " for value " + testValue); + } + } catch (Exception e) { + fail("VarInt utility encoding failed for value: " + testValue + ", error: " + e.getMessage()); + } + } + } + + @Test + void testBoundsChecking() { + // Since bounds checking is disabled by default for performance, + // this test verifies that operations beyond capacity don't throw exceptions + // when bounds checking is disabled (which is the expected production behavior) + + byte[] array = new byte[4]; + ImprintBuffer buffer = new ImprintBuffer(array); + + // This should work + buffer.putInt(42); + + // With bounds checking disabled (default), this should NOT throw an exception + // The buffer will write beyond capacity but won't check bounds for performance + assertDoesNotThrow(() -> buffer.putByte((byte) 1)); + + // Verify the byte was written beyond the original array bounds + assertEquals(5, buffer.position()); + } + + @Test + void testCompatibilityWithByteBuffer() { + // Test that ImprintBuffer produces same results as ByteBuffer + byte[] imprintArray = new byte[32]; + byte[] byteBufferArray = new byte[32]; + + ImprintBuffer imprintBuffer = new ImprintBuffer(imprintArray); + ByteBuffer byteBuffer = ByteBuffer.wrap(byteBufferArray); + byteBuffer.order(ByteOrder.LITTLE_ENDIAN); + + // Write same data to both + int testInt = 0x12345678; + long testLong = 0x123456789ABCDEF0L; + float testFloat = 3.14f; + + imprintBuffer.putInt(testInt).putLong(testLong).putFloat(testFloat); + byteBuffer.putInt(testInt).putLong(testLong).putFloat(testFloat); + + // Compare results + for (int i = 0; i < byteBuffer.position(); i++) { + assertEquals(byteBufferArray[i], imprintArray[i], + "Mismatch at position " + i); + } + } + + @Test + void testFlipSliceAndLimit() { + byte[] array = new byte[64]; + ImprintBuffer buffer = new ImprintBuffer(array); + + // Write some data + buffer.putInt(42).putInt(100); + assertEquals(8, buffer.position()); + assertEquals(64, buffer.limit()); + + // Test flip + buffer.flip(); + assertEquals(0, buffer.position()); + assertEquals(8, buffer.limit()); + + // Test reading back + assertEquals(42, buffer.getInt()); + assertEquals(100, buffer.getInt()); + + // Test slice + buffer.position(4); + ImprintBuffer slice = buffer.slice(); + assertEquals(0, slice.position()); + assertEquals(4, slice.limit()); // remaining from position 4 to limit 8 + assertEquals(100, slice.getInt()); + } + + @Test + void testMethodChaining() { + byte[] array = new byte[64]; + ImprintBuffer buffer = new ImprintBuffer(array); + + // Test method chaining like ByteBuffer + buffer.position(10).limit(50); + assertEquals(10, buffer.position()); + assertEquals(50, buffer.limit()); + assertEquals(40, buffer.remaining()); + } + + @Test + void testReadOperations() { + byte[] array = new byte[64]; + ImprintBuffer buffer = new ImprintBuffer(array); + + // Write data + buffer.putByte((byte) 0x42) + .putShort((short) 0x1234) + .putInt(0x12345678) + .putLong(0x123456789ABCDEF0L) + .putFloat(3.14f) + .putDouble(2.718281828); + + // Flip for reading + buffer.flip(); + + // Read back and verify + assertEquals((byte) 0x42, buffer.get()); + assertEquals((short) 0x1234, buffer.getShort()); + assertEquals(0x12345678, buffer.getInt()); + assertEquals(0x123456789ABCDEF0L, buffer.getLong()); + assertEquals(3.14f, buffer.getFloat(), 0.001f); + assertEquals(2.718281828, buffer.getDouble(), 0.000001); + } +} \ No newline at end of file diff --git a/src/test/java/com/imprint/util/VarIntTest.java b/src/test/java/com/imprint/util/VarIntTest.java index 677afb7..cc3acd3 100644 --- a/src/test/java/com/imprint/util/VarIntTest.java +++ b/src/test/java/com/imprint/util/VarIntTest.java @@ -16,7 +16,7 @@ void shouldRoundtripCommonValues() throws ImprintException { }; for (int value : testCases) { - ByteBuffer buffer = ByteBuffer.allocate(10); + var buffer = new ImprintBuffer(new byte[10]); VarInt.encode(value, buffer); int encodedLength = buffer.position(); @@ -40,7 +40,7 @@ void shouldEncodeKnownValuesCorrectly() { } private void assertEncodedBytes(int value, int... expectedBytes) { - ByteBuffer buffer = ByteBuffer.allocate(10); + var buffer = new ImprintBuffer(new byte[10]); VarInt.encode(value, buffer); buffer.flip(); @@ -57,7 +57,7 @@ private void assertEncodedBytes(int value, int... expectedBytes) { @Test void shouldWorkWithByteBuffer() throws ImprintException { - ByteBuffer buffer = ByteBuffer.allocate(10); + var buffer = new ImprintBuffer(new byte[10]); VarInt.encode(16384, buffer); buffer.flip(); @@ -79,8 +79,8 @@ void shouldCalculateEncodedLength() { @Test void shouldHandleBufferUnderflow() { - ByteBuffer buffer = ByteBuffer.allocate(1); - buffer.put((byte) 0x80); // incomplete varint + var buffer = new ImprintBuffer(new byte[1]); + buffer.putByte((byte) 0x80); // incomplete varint buffer.flip(); assertThatThrownBy(() -> VarInt.decode(buffer)) @@ -91,8 +91,8 @@ void shouldHandleBufferUnderflow() { @Test void shouldHandleOverlongEncoding() { - ByteBuffer buffer = ByteBuffer.allocate(10); - buffer.put(new byte[]{(byte) 0x80, (byte) 0x80, (byte) 0x80, (byte) 0x80, (byte) 0x80, 0x01}); + var buffer = new ImprintBuffer(new byte[10]); + buffer.putBytes(new byte[]{(byte) 0x80, (byte) 0x80, (byte) 0x80, (byte) 0x80, (byte) 0x80, 0x01}); buffer.flip(); assertThatThrownBy(() -> VarInt.decode(buffer)) @@ -103,8 +103,8 @@ void shouldHandleOverlongEncoding() { @Test void shouldHandleOverflow() { - ByteBuffer buffer = ByteBuffer.allocate(10); - buffer.put(new byte[]{(byte) 0x80, (byte) 0x80, (byte) 0x80, (byte) 0x80, 0x10}); + var buffer = new ImprintBuffer(new byte[10]); + buffer.putBytes(new byte[]{(byte) 0x80, (byte) 0x80, (byte) 0x80, (byte) 0x80, 0x10}); buffer.flip(); assertThatThrownBy(() -> VarInt.decode(buffer))