From 7e1aeafaa3857621cef111ff0d1f1ead2ba175f9 Mon Sep 17 00:00:00 2001 From: panini Date: Sat, 21 Nov 2020 14:47:15 +0900 Subject: [PATCH 01/12] Add Writer & Utf8 --- runtime-lite/src/main/kotlin/Utf8.kt | 167 +++++++++++++++++++++++++ runtime-lite/src/main/kotlin/Writer.kt | 160 +++++++++++++++++++++++ 2 files changed, 327 insertions(+) create mode 100644 runtime-lite/src/main/kotlin/Utf8.kt create mode 100644 runtime-lite/src/main/kotlin/Writer.kt diff --git a/runtime-lite/src/main/kotlin/Utf8.kt b/runtime-lite/src/main/kotlin/Utf8.kt new file mode 100644 index 0000000..cf3507e --- /dev/null +++ b/runtime-lite/src/main/kotlin/Utf8.kt @@ -0,0 +1,167 @@ +internal object Utf8 { + const val MAX_BYTES_PER_CHAR = 3 + + internal class UnpairedSurrogateException(index: Int, length: Int) : + IllegalArgumentException("Unpaired surrogate at index $index of $length") + + fun encode(input: CharSequence, out: ByteArray, offset: Int, length: Int): Int { + val utf16Length = input.length + var j = offset + var i = 0 + val limit = offset + length + // Designed to take advantage of + // https://wikis.oracle.com/display/HotSpotInternals/RangeCheckElimination + run { + var c: Char? = null + while (i < utf16Length && i + j < limit && input[i].also { c = it }.toInt() < 0x80) { + out[j + i] = c!!.toByte() //TODO: can we do this without !! + i++ + } + } + if (i == utf16Length) { + return j + utf16Length + } + j += i + var c: Char + while (i < utf16Length) { + c = input[i] + if (c.toInt() < 0x80 && j < limit) { + out[j++] = c.toByte() + } else if (c.toInt() < 0x800 && j <= limit - 2) { // 11 bits, two UTF-8 bytes + out[j++] = (0xF shl 6 or (c.toInt() ushr 6)).toByte() + out[j++] = (0x80 or (0x3F and c.toInt())).toByte() + } else if ((c < Char.MIN_SURROGATE || Char.MAX_SURROGATE < c) && j <= limit - 3) { + // Maximum single-char code point is 0xFFFF, 16 bits, three UTF-8 bytes + out[j++] = (0xF shl 5 or (c.toInt() ushr 12)).toByte() + out[j++] = (0x80 or (0x3F and (c.toInt() ushr 6))).toByte() + out[j++] = (0x80 or (0x3F and c.toInt())).toByte() + } else if (j <= limit - 4) { + // Minimum code point represented by a surrogate pair is 0x10000, 17 bits, + // four UTF-8 bytes + var low: Char? = null + if (i + 1 == input.length + || !Character.isSurrogatePair(c, input[++i].also { low = it })) { + throw UnpairedSurrogateException(i - 1, utf16Length) + } + val codePoint: Int = Character.toCodePoint(c, low!!) //TODO: can I do this without !! + out[j++] = (0xF shl 4 or (codePoint ushr 18)).toByte() + out[j++] = (0x80 or (0x3F and (codePoint ushr 12))).toByte() + out[j++] = (0x80 or (0x3F and (codePoint ushr 6))).toByte() + out[j++] = (0x80 or (0x3F and codePoint)).toByte() + } else { + // If we are surrogates and we're not a surrogate pair, always throw an + // UnpairedSurrogateException instead of an ArrayOutOfBoundsException. + if (Char.MIN_SURROGATE <= c && c <= Char.MAX_SURROGATE + && (i + 1 == input.length + || !Character.isSurrogatePair(c, input[i + 1]))) { + throw UnpairedSurrogateException(i, utf16Length) + } + throw IndexOutOfBoundsException("Failed writing $c at index $j") + } + i++ + } + return j + } + + /** + * Returns the number of bytes in the UTF-8-encoded form of `sequence`. For a string, + * this method is equivalent to `string.getBytes(UTF_8).length`, but is more efficient in + * both time and space. + * + * @throws IllegalArgumentException if `sequence` contains ill-formed UTF-16 (unpaired + * surrogates) + */ + fun encodedLength(sequence: CharSequence): Int { + // Warning to maintainers: this implementation is highly optimized. + val utf16Length = sequence.length + var utf8Length = utf16Length + var i = 0 + + // This loop optimizes for pure ASCII. + while (i < utf16Length && sequence[i].toInt() < 0x80) { + i++ + } + + // This loop optimizes for chars less than 0x800. + while (i < utf16Length) { + val c = sequence[i] + if (c.toInt() < 0x800) { + utf8Length += 0x7f - c.toInt() ushr 31 // branch free! + } else { + utf8Length += encodedLengthGeneral(sequence, i) + break + } + i++ + } + if (utf8Length < utf16Length) { + // Necessary and sufficient condition for overflow because of maximum 3x expansion + throw IllegalArgumentException("UTF-8 length does not fit in int: " + + (utf8Length + (1L shl 32))) + } + return utf8Length + } + + private fun encodedLengthGeneral(sequence: CharSequence, start: Int): Int { + val utf16Length = sequence.length + var utf8Length = 0 + var i = start + while (i < utf16Length) { + val c = sequence[i] + if (c.toInt() < 0x800) { + utf8Length += 0x7f - c.toInt() ushr 31 // branch free! + } else { + utf8Length += 2 + // jdk7+: if (Character.isSurrogate(c)) { + if (Char.MIN_SURROGATE <= c && c <= Char.MAX_SURROGATE) { + // Check that we have a well-formed surrogate pair. + val cp: Int = Character.codePointAt(sequence, i) + if (cp < Character.MIN_SUPPLEMENTARY_CODE_POINT) { + throw UnpairedSurrogateException(i, utf16Length) + } + i++ + } + } + i++ + } + return utf8Length + } +} + +object Character { + + const val MIN_SUPPLEMENTARY_CODE_POINT = 0x010000 + + fun isSurrogatePair(high: Char, low: Char): Boolean { + return isHighSurrogate(high) && isLowSurrogate(low) + } + + fun isHighSurrogate(ch: Char): Boolean { + return ch >= Char.MIN_HIGH_SURROGATE && ch.toInt() < Char.MAX_HIGH_SURROGATE.toInt() + 1 + } + + fun isLowSurrogate(ch: Char): Boolean { + return ch >= Char.MIN_LOW_SURROGATE && ch.toInt() < Char.MAX_LOW_SURROGATE.toInt() + 1 + } + + fun toCodePoint(high: Char, low: Char): Int { + // Optimized form of: + // return ((high - MIN_HIGH_SURROGATE) << 10) + // + (low - MIN_LOW_SURROGATE) + // + MIN_SUPPLEMENTARY_CODE_POINT; + return (high.toInt() shl 10) + low.toInt() + (MIN_SUPPLEMENTARY_CODE_POINT + - (Char.MIN_HIGH_SURROGATE.toInt() shl 10) + - Char.MIN_LOW_SURROGATE.toInt()) + } + + fun codePointAt(seq: CharSequence, index: Int): Int { + var index = index + val c1 = seq[index] + if (isHighSurrogate(c1) && ++index < seq.length) { + val c2 = seq[index] + if (isLowSurrogate(c2)) { + return toCodePoint(c1, c2) + } + } + return c1.toInt() + } +} \ No newline at end of file diff --git a/runtime-lite/src/main/kotlin/Writer.kt b/runtime-lite/src/main/kotlin/Writer.kt new file mode 100644 index 0000000..9b0ffa3 --- /dev/null +++ b/runtime-lite/src/main/kotlin/Writer.kt @@ -0,0 +1,160 @@ +package jp.co.panpanini + +class Writer(private val byteArray: ByteArray) { + + private var position = 0 + + fun spaceLeft(): Int = byteArray.size - position + + fun writeTag(value: Int) { + writeUInt32(value) + } + + fun writeInt32(value: Int) { + if (value >= 0) { + writeUInt32(value) + } else { + writeUInt64(value.toLong()) + } + } + + fun writeUInt32(value: Int) { + var value = value + while (true) { + if (value and 0x7F.inv() == 0) { + byteArray[position++] = value.toByte() + return + } else { + byteArray[position++] = (value and 0x7F or 0x80).toByte() + value = value ushr 7 + } + } + } + + fun writeInt64(value: Long) { + writeUInt64(value) + } + + fun writeUInt64(value: Long) { + var value = value + while (true) { + if (value and 0x7FL.inv() == 0L) { + byteArray[position++] = value.toByte() + return + } else { + byteArray[position++] = (value.toInt() and 0x7F or 0x80).toByte() + value = value ushr 7 + } + } + } + + fun writeFixed32(value: Int) { + byteArray[position++] = (value and 0xFF).toByte() + byteArray[position++] = (value shr 8 and 0xFF).toByte() + byteArray[position++] = (value shr 16 and 0xFF).toByte() + byteArray[position++] = (value shr 24 and 0xFF).toByte() + } + + fun writeFixed64(value: Long) { + byteArray[position++] = (value and 0xFF).toByte() + byteArray[position++] = (value shr 8 and 0xFF).toByte() + byteArray[position++] = (value shr 16 and 0xFF).toByte() + byteArray[position++] = (value shr 24 and 0xFF).toByte() + byteArray[position++] = ((value shr 32).toInt() and 0xFF).toByte() + byteArray[position++] = ((value shr 40).toInt() and 0xFF).toByte() + byteArray[position++] = ((value shr 48).toInt() and 0xFF).toByte() + byteArray[position++] = ((value shr 56).toInt() and 0xFF).toByte() + } + + fun writeSInt32(value: Int) { + writeUInt32(value.encodeZigZag32()) + } + + fun writeSInt64(value: Long) { + writeUInt64(value.encodeZigZag64()) + } + + fun writeSFixed32(value: Int) { + writeFixed32(value) + } + + fun writeSFixed64(value: Long) { + writeFixed64(value) + } + + fun writeBool(value: Boolean) { + writeByte((if (value) 1 else 0).toByte()) + } + + fun writeDouble(value: Double) { + writeFixed64(value.toRawBits()) + } + + fun writeFloat(value: Float) { + writeFixed32(value.toRawBits()) + } + + fun writeByte(value: Byte) { + byteArray[position++] = value + } + + fun writeBytes(value: ByteArray) { + value.forEach(::writeByte) + } + + fun writeString(value: String) { + val oldPosition = position + // UTF-8 byte length of the string is at least its UTF-16 code unit length (value.length()), + // and at most 3 times of it. We take advantage of this in both branches below. + val maxLength: Int = value.length * Utf8.MAX_BYTES_PER_CHAR + val maxLengthVarIntSize: Int = computeUInt32Size(maxLength) + val minLengthVarIntSize: Int = computeUInt32Size(value.length) + if (minLengthVarIntSize == maxLengthVarIntSize) { + position = oldPosition + minLengthVarIntSize + val newPosition: Int = Utf8.encode(value, byteArray, position, spaceLeft()) + // Since this class is stateful and tracks the position, we rewind and store the state, + // prepend the length, then reset it back to the end of the string. + position = oldPosition + val length = newPosition - oldPosition - minLengthVarIntSize + writeUInt32(length) + position = newPosition + } else { + val length: Int = Utf8.encodedLength(value) + writeUInt32(length) + position = Utf8.encode(value, byteArray, position, spaceLeft()) + } + } + + fun complete(): ByteArray { + return byteArray + } + + companion object { + fun allocate(size: Int) : Writer = Writer(ByteArray(size)) + } +} + +private fun computeUInt32Size(value: Int): Int { + if (value and (0.inv() shl 7) == 0) { + return 1 + } + if (value and (0.inv() shl 14) == 0) { + return 2 + } + if (value and (0.inv() shl 21) == 0) { + return 3 + } + return if (value and (0.inv() shl 28) == 0) { + 4 + } else 5 +} + +fun Int.encodeZigZag32(): Int { + // Note: the right-shift must be arithmetic + return this shl 1 xor (this shr 31) +} + +fun Long.encodeZigZag64(): Long { + // Note: the right-shift must be arithmetic + return this shl 1 xor (this shr 63) +} \ No newline at end of file From dbe59ec556c4014f96a54ae8c431f059895a7f3d Mon Sep 17 00:00:00 2001 From: panini Date: Sat, 21 Nov 2020 14:47:31 +0900 Subject: [PATCH 02/12] Add simple Writer tests --- runtime-lite/src/test/kotlin/WriterTest.kt | 157 +++++++++++++++++++++ 1 file changed, 157 insertions(+) create mode 100644 runtime-lite/src/test/kotlin/WriterTest.kt diff --git a/runtime-lite/src/test/kotlin/WriterTest.kt b/runtime-lite/src/test/kotlin/WriterTest.kt new file mode 100644 index 0000000..2d7e912 --- /dev/null +++ b/runtime-lite/src/test/kotlin/WriterTest.kt @@ -0,0 +1,157 @@ +package jp.co.panpanini + +import com.nhaarman.mockitokotlin2.spy +import com.nhaarman.mockitokotlin2.verify +import org.assertj.core.api.Assertions.assertThat +import org.junit.Before +import org.junit.Test + +class WriterTest { + + private lateinit var byteArray: ByteArray + + private lateinit var target: Writer + + + @Before + fun setup() { + byteArray = ByteArray(100) + target = spy(Writer(byteArray)) + } + + fun setup(size: Int) { + byteArray = ByteArray(size) + target = spy(Writer(byteArray)) + } + + @Test + fun `writeTag should write the UInt32 representation to byteArray`() { + val input = 150 + + target.writeTag(input) + + verify(target).writeUInt32(input) + } + + @Test + fun `writeInt32 should writeUInt32 value when value is greater than 0`() { + val input = 150 + + target.writeInt32(input) + + verify(target).writeUInt32(input) + } + + @Test + fun `writeInt32 should writeUInt64 value when value is less than 0`() { + val input = -150 + target.writeInt32(input) + + verify(target).writeUInt64(input.toLong()) + } + + @Test + fun `writeInt64 should call writeUInt64`() { + val input = 1L + + target.writeInt64(input) + + verify(target).writeUInt64(input) + } + + @Test + fun `writeSInt32 should call writeUInt32 with zigzag encoded value`() { + val input = 1 + + target.writeSInt32(input) + + verify(target).writeUInt32(input.encodeZigZag32()) + } + + @Test + fun `writeSInt64 should call writeUInt64 with zigzag encoded value`() { + val input = 1L + target.writeSInt64(input) + + verify(target).writeUInt64(input.encodeZigZag64()) + } + + @Test + fun `writeSFixed32 should call writeFixed32`() { + val input = 1 + + target.writeSFixed32(input) + + verify(target).writeFixed32(input) + } + + @Test + fun `writeSFixed64 should call writeFixed64`() { + val input = 1L + + target.writeSFixed64(input) + + verify(target).writeFixed64(input) + } + + @Test + fun `writeBool should call writeByte`() { + var input = true + + target.writeBool(input) + + verify(target).writeByte(1.toByte()) + + input = false + + target.writeBool(input) + + verify(target).writeByte(0.toByte()) + } + + @Test + fun `writeDouble should call writeFixed64 with raw bits`() { + val input = 1.0 + + target.writeDouble(input) + + verify(target).writeFixed64(input.toRawBits()) + } + + @Test + fun `writeFloat should call writeFixed32 with raw bits`() { + val input = 1f + + target.writeFloat(input) + + verify(target).writeFixed32(input.toRawBits()) + } + + @Test + fun `writeByte should write the byte to byteArray`() { + setup(1) + val input = 2.toByte() + + target.writeByte(input) + + assertThat(byteArray[0]).isEqualTo(input) + } + + @Test + fun `writeBytes should call writeByte for each value in the byte array`() { + val byte1 = 1.toByte() + val byte2 = 2.toByte() + val input = byteArrayOf(byte1, byte2) + + target.writeBytes(input) + + verify(target).writeByte(byte1) + verify(target).writeByte(byte2) + } + + @Test + fun `complete should return the byteArray`() { + assertThat(target.complete()).isSameAs(byteArray) + } + +} \ No newline at end of file From 852e1d020761db36c73bbf5ea0023042da163719 Mon Sep 17 00:00:00 2001 From: panini Date: Sat, 21 Nov 2020 15:09:05 +0900 Subject: [PATCH 03/12] Add writeUInt32 tests --- runtime-lite/src/test/kotlin/WriterTest.kt | 28 ++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/runtime-lite/src/test/kotlin/WriterTest.kt b/runtime-lite/src/test/kotlin/WriterTest.kt index 2d7e912..1e17d98 100644 --- a/runtime-lite/src/test/kotlin/WriterTest.kt +++ b/runtime-lite/src/test/kotlin/WriterTest.kt @@ -154,4 +154,32 @@ class WriterTest { assertThat(target.complete()).isSameAs(byteArray) } + @Test + fun `writeUInt32 should write 7bit integer in 1 byte`() { + setup(1) + val input = 127 // largest 7-bit integer + val expected = byteArrayOf(input.toByte()) + + target.writeUInt32(input) + + assertThat(byteArray).isEqualTo(expected) + } + + @Test + fun `writeUInt32 should write 32-bit integer in 5 bytes`() { + setup(5) + val input = Integer.MAX_VALUE // largest 32-bit integer + val expected = byteArrayOf( + 0b11111111.toByte(), + 0b11111111.toByte(), + 0b11111111.toByte(), + 0b11111111.toByte(), + 0b00000111.toByte() + ) + + target.writeUInt32(input) + + assertThat(byteArray).isEqualTo(expected) + } + } \ No newline at end of file From 00dc61b098d01f4b71456d56178f859a9c76a9c2 Mon Sep 17 00:00:00 2001 From: panini Date: Sat, 21 Nov 2020 15:14:26 +0900 Subject: [PATCH 04/12] Add writeUInt64 tests --- runtime-lite/src/test/kotlin/WriterTest.kt | 51 +++++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/runtime-lite/src/test/kotlin/WriterTest.kt b/runtime-lite/src/test/kotlin/WriterTest.kt index 1e17d98..c3661db 100644 --- a/runtime-lite/src/test/kotlin/WriterTest.kt +++ b/runtime-lite/src/test/kotlin/WriterTest.kt @@ -158,7 +158,7 @@ class WriterTest { fun `writeUInt32 should write 7bit integer in 1 byte`() { setup(1) val input = 127 // largest 7-bit integer - val expected = byteArrayOf(input.toByte()) + val expected = byteArrayOf(0b01111111.toByte()) target.writeUInt32(input) @@ -182,4 +182,53 @@ class WriterTest { assertThat(byteArray).isEqualTo(expected) } + @Test + fun `writeUInt64 should write 7bit long in 1 byte`() { + setup(1) + val input = 127L // largest 7-bit integer + val expected = byteArrayOf(0b01111111.toByte()) + + target.writeUInt64(input) + + assertThat(byteArray).isEqualTo(expected) + } + + @Test + fun `writeUInt64 should write 32-bit integer in 5 bytes`() { + setup(5) + val input = Integer.MAX_VALUE.toLong() // largest 32-bit integer + val expected = byteArrayOf( + 0b11111111.toByte(), + 0b11111111.toByte(), + 0b11111111.toByte(), + 0b11111111.toByte(), + 0b00000111.toByte() + ) + + target.writeUInt64(input) + + assertThat(byteArray).isEqualTo(expected) + } + + @Test + fun `writeUInt64 should write 64-bit integer in 9 bytes`() { + setup(9) + val input = Long.MAX_VALUE // largest 64-bit integer + val expected = byteArrayOf( + 0b11111111.toByte(), + 0b11111111.toByte(), + 0b11111111.toByte(), + 0b11111111.toByte(), + 0b11111111.toByte(), + 0b11111111.toByte(), + 0b11111111.toByte(), + 0b11111111.toByte(), + 0b01111111.toByte() + ) + + target.writeUInt64(input) + + assertThat(byteArray).isEqualTo(expected) + } + } \ No newline at end of file From d60c9304b77125121073d0e40a47f0274bda402b Mon Sep 17 00:00:00 2001 From: panini Date: Sat, 21 Nov 2020 15:24:47 +0900 Subject: [PATCH 05/12] Add byte order comment to Writer --- runtime-lite/src/main/kotlin/Writer.kt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/runtime-lite/src/main/kotlin/Writer.kt b/runtime-lite/src/main/kotlin/Writer.kt index 9b0ffa3..863fe02 100644 --- a/runtime-lite/src/main/kotlin/Writer.kt +++ b/runtime-lite/src/main/kotlin/Writer.kt @@ -18,6 +18,7 @@ class Writer(private val byteArray: ByteArray) { } } + // bytes are written in smallest to largest order fun writeUInt32(value: Int) { var value = value while (true) { @@ -31,10 +32,12 @@ class Writer(private val byteArray: ByteArray) { } } + // bytes are written in smallest to largest order fun writeInt64(value: Long) { writeUInt64(value) } + // bytes are written in smallest to largest order fun writeUInt64(value: Long) { var value = value while (true) { @@ -48,6 +51,7 @@ class Writer(private val byteArray: ByteArray) { } } + // bytes are written in smallest to largest order fun writeFixed32(value: Int) { byteArray[position++] = (value and 0xFF).toByte() byteArray[position++] = (value shr 8 and 0xFF).toByte() @@ -55,6 +59,7 @@ class Writer(private val byteArray: ByteArray) { byteArray[position++] = (value shr 24 and 0xFF).toByte() } + // bytes are written in smallest to largest order fun writeFixed64(value: Long) { byteArray[position++] = (value and 0xFF).toByte() byteArray[position++] = (value shr 8 and 0xFF).toByte() From 1d4e99f8313c720444c49500ff2f1d4533194ebb Mon Sep 17 00:00:00 2001 From: panini Date: Sat, 21 Nov 2020 15:25:03 +0900 Subject: [PATCH 06/12] Add fixed32 test --- runtime-lite/src/test/kotlin/WriterTest.kt | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/runtime-lite/src/test/kotlin/WriterTest.kt b/runtime-lite/src/test/kotlin/WriterTest.kt index c3661db..6fd9de2 100644 --- a/runtime-lite/src/test/kotlin/WriterTest.kt +++ b/runtime-lite/src/test/kotlin/WriterTest.kt @@ -231,4 +231,20 @@ class WriterTest { assertThat(byteArray).isEqualTo(expected) } + @Test + fun `writeFixed32 should write 4 bytes exactly`() { + setup(4) + val input = 2140483647 + val expected = byteArrayOf( + 0b00111111.toByte(), + 0b00110000.toByte(), + 0b10010101.toByte(), + 0b01111111.toByte() + ) + + target.writeFixed32(input) + + assertThat(byteArray).isEqualTo(expected) + } + } \ No newline at end of file From 8e7dfd72cb6aee02ee0e7582f2a41f6e100815c6 Mon Sep 17 00:00:00 2001 From: panini Date: Sat, 21 Nov 2020 15:29:05 +0900 Subject: [PATCH 07/12] Add writeFixed64 test --- runtime-lite/src/test/kotlin/WriterTest.kt | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/runtime-lite/src/test/kotlin/WriterTest.kt b/runtime-lite/src/test/kotlin/WriterTest.kt index 6fd9de2..ffb28b8 100644 --- a/runtime-lite/src/test/kotlin/WriterTest.kt +++ b/runtime-lite/src/test/kotlin/WriterTest.kt @@ -247,4 +247,24 @@ class WriterTest { assertThat(byteArray).isEqualTo(expected) } + @Test + fun `writeFixed64 should write 8 bytes exactly`() { + setup(8) + val input = 9223302036854775807L + val expected = byteArrayOf( + 0b11111111.toByte(), + 0b10011111.toByte(), + 0b11011101.toByte(), + 0b11011010.toByte(), + 0b01010101.toByte(), + 0b11000000.toByte(), + 0b11111111.toByte(), + 0b01111111.toByte() + ) + + target.writeFixed64(input) + + assertThat(byteArray).isEqualTo(expected) + } + } \ No newline at end of file From 9fe86f4619baaf8536cf7fd2695dde03bffa0e10 Mon Sep 17 00:00:00 2001 From: panini Date: Sat, 21 Nov 2020 15:32:35 +0900 Subject: [PATCH 08/12] Update Marshaller to use Writer instead of CodedInputStream --- runtime-lite/src/main/kotlin/Marshaller.kt | 42 +++++----- .../src/test/kotlin/MarshallerTest.kt | 83 +++++++++---------- 2 files changed, 59 insertions(+), 66 deletions(-) diff --git a/runtime-lite/src/main/kotlin/Marshaller.kt b/runtime-lite/src/main/kotlin/Marshaller.kt index cfe7750..3529591 100644 --- a/runtime-lite/src/main/kotlin/Marshaller.kt +++ b/runtime-lite/src/main/kotlin/Marshaller.kt @@ -1,71 +1,69 @@ package jp.co.panpanini -import com.google.protobuf.CodedOutputStream - -class Marshaller(private val stream: CodedOutputStream, private val bytes: ByteArray? = null) { +class Marshaller(private val stream: Writer) { companion object { - fun allocate(size: Int) = ByteArray(size).let { Marshaller(CodedOutputStream.newInstance(it), it) } + fun allocate(size: Int) = Marshaller(Writer.allocate(size)) } - fun writeTag(tag: Int) = this.apply { stream.writeInt32NoTag(tag) } + fun writeTag(tag: Int) = this.apply { stream.writeTag(tag) } - fun writeTag(fieldNum: Int, wireType: Int) = this.apply { stream.writeInt32NoTag((fieldNum shl 3) or wireType) } + fun writeTag(fieldNum: Int, wireType: Int) = this.apply { stream.writeTag((fieldNum shl 3) or wireType) } fun writeDouble(value: Double) { - stream.writeDoubleNoTag(value) + stream.writeDouble(value) } fun writeFloat(value: Float) { - stream.writeFloatNoTag(value) + stream.writeFloat(value) } fun writeInt32(value: Int) { - stream.writeInt32NoTag(value) + stream.writeInt32(value) } fun writeInt64(value: Long) { - stream.writeInt64NoTag(value) + stream.writeInt64(value) } fun writeUInt32(value: Int) { - stream.writeUInt32NoTag(value) + stream.writeUInt32(value) } fun writeUInt64(value: Long) { - stream.writeUInt64NoTag(value) + stream.writeUInt64(value) } fun writeSInt32(value: Int) { - stream.writeSInt32NoTag(value) + stream.writeSInt32(value) } fun writeSInt64(value: Long) { - stream.writeSInt64NoTag(value) + stream.writeSInt64(value) } fun writeFixed32(value: Int) { - stream.writeFixed32NoTag(value) + stream.writeFixed32(value) } fun writeFixed64(value: Long) { - stream.writeFixed64NoTag(value) + stream.writeFixed64(value) } fun writeSFixed32(value: Int) { - stream.writeSFixed32NoTag(value) + stream.writeSFixed32(value) } fun writeSFixed64(value: Long) { - stream.writeSFixed64NoTag(value) + stream.writeSFixed64(value) } fun writeBool(value: Boolean) { - stream.writeBoolNoTag(value) + stream.writeBool(value) } fun writeString(value: String) { - stream.writeStringNoTag(value) + stream.writeString(value) } fun writeBytes(value: ByteArr) { @@ -73,7 +71,7 @@ class Marshaller(private val stream: CodedOutputStream, private val bytes: ByteA } fun writeBytes(value: ByteArray) { - stream.writeByteArrayNoTag(value) + stream.writeBytes(value) } fun writeUnknownFields(fields: Map) { @@ -100,7 +98,7 @@ class Marshaller(private val stream: CodedOutputStream, private val bytes: ByteA writeInt32(value.value) } - fun complete() = bytes + fun complete() = stream.complete() fun > writeMap( tag: Int, diff --git a/runtime-lite/src/test/kotlin/MarshallerTest.kt b/runtime-lite/src/test/kotlin/MarshallerTest.kt index 67178c8..6eaa9c7 100644 --- a/runtime-lite/src/test/kotlin/MarshallerTest.kt +++ b/runtime-lite/src/test/kotlin/MarshallerTest.kt @@ -1,4 +1,3 @@ -import com.google.protobuf.CodedOutputStream import com.nhaarman.mockitokotlin2.mock import com.nhaarman.mockitokotlin2.spy import com.nhaarman.mockitokotlin2.verify @@ -6,26 +5,23 @@ import com.nhaarman.mockitokotlin2.whenever import jp.co.panpanini.ByteArr import jp.co.panpanini.Marshaller import jp.co.panpanini.Message -import org.assertj.core.api.Assertions.assertThat +import jp.co.panpanini.Writer import org.junit.Test import org.mockito.ArgumentMatchers.anyInt class MarshallerTest { + private var stream: Writer = mock { } - private var stream: CodedOutputStream = mock { } - - private var bytes: ByteArray? = ByteArray(0) - - private var target: Marshaller = Marshaller(stream, bytes) + private var target: Marshaller = Marshaller(stream) @Test - fun `writeTag should call stream#writeInt32NoTag`() { + fun `writeTag should call stream#writeTag`() { val input = 100 target.writeTag(input) - verify(stream).writeInt32NoTag(input) + verify(stream).writeTag(input) } @Test @@ -37,133 +33,133 @@ class MarshallerTest { target.writeTag(fieldNum, wireType) - verify(stream).writeInt32NoTag(fieldNumSHL3 + wireType) + verify(stream).writeTag(fieldNumSHL3 + wireType) } @Test - fun `writeDouble should call stream#writeDoubleNoTag`() { + fun `writeDouble should call stream#writeDouble`() { val input = 1.0 target.writeDouble(input) - verify(stream).writeDoubleNoTag(input) + verify(stream).writeDouble(input) } @Test - fun `writeFloat should call stream#writeFloatNoTag`() { + fun `writeFloat should call stream#writeFloat`() { val input = 1f target.writeFloat(input) - verify(stream).writeFloatNoTag(input) + verify(stream).writeFloat(input) } @Test - fun `writeInt32 should call stream#writeInt32NoTag`() { + fun `writeInt32 should call stream#writeInt32`() { val input = 1 target.writeInt32(input) - verify(stream).writeInt32NoTag(input) + verify(stream).writeInt32(input) } @Test - fun `writeInt64 should call stream#writeInt64NoTag`() { + fun `writeInt64 should call stream#writeInt64`() { val input = 1L target.writeInt64(input) - verify(stream).writeInt64NoTag(input) + verify(stream).writeInt64(input) } @Test - fun `writeUInt32 should call stream#writeUInt32NoTag`() { + fun `writeUInt32 should call stream#writeUInt32`() { val input = 1 target.writeUInt32(input) - verify(stream).writeUInt32NoTag(input) + verify(stream).writeUInt32(input) } @Test - fun `writeUInt64 should call stream#writeUInt64NoTag`() { + fun `writeUInt64 should call stream#writeUInt64`() { val input = 1L target.writeUInt64(input) - verify(stream).writeUInt64NoTag(input) + verify(stream).writeUInt64(input) } @Test - fun `writeSInt32 should call stream#writeSInt32NoTag`() { + fun `writeSInt32 should call stream#writeSInt32`() { val input = 1 target.writeSInt32(input) - verify(stream).writeSInt32NoTag(input) + verify(stream).writeSInt32(input) } @Test - fun `writeSInt64 should call stream#writeSInt64NoTag`() { + fun `writeSInt64 should call stream#writeSInt64`() { val input = 1L target.writeSInt64(input) - verify(stream).writeSInt64NoTag(input) + verify(stream).writeSInt64(input) } @Test - fun `writeFixed32 should call stream#writeFixed32NoTag`() { + fun `writeFixed32 should call stream#writeFixed32`() { val input = 1 target.writeFixed32(input) - verify(stream).writeFixed32NoTag(input) + verify(stream).writeFixed32(input) } @Test - fun `writeFixed64 should call stream#writeFixed64NoTag`() { + fun `writeFixed64 should call stream#writeFixed64`() { val input = 1L target.writeFixed64(input) - verify(stream).writeFixed64NoTag(input) + verify(stream).writeFixed64(input) } @Test - fun `writeSFixed32 should call stream#writeSFixed32NoTag`() { + fun `writeSFixed32 should call stream#writeSFixed32`() { val input = 1 target.writeSFixed32(input) - verify(stream).writeSFixed32NoTag(input) + verify(stream).writeSFixed32(input) } @Test - fun `writeSFixed64 should call stream#writeSFixed64NoTag`() { + fun `writeSFixed64 should call stream#writeSFixed64`() { val input = 1L target.writeSFixed64(input) - verify(stream).writeSFixed64NoTag(input) + verify(stream).writeSFixed64(input) } @Test - fun `writeBool should call stream#writeBoolNoTag`() { + fun `writeBool should call stream#writeBool`() { val input = true target.writeBool(input) - verify(stream).writeBoolNoTag(input) + verify(stream).writeBool(input) } @Test - fun `writeString should call stream#writeStringNoTag`() { + fun `writeString should call stream#writeString`() { val input = "" target.writeString("") - verify(stream).writeStringNoTag(input) + verify(stream).writeString(input) } @Test @@ -180,12 +176,12 @@ class MarshallerTest { } @Test - fun `writeBytes should call stream#writeByteArrayNoTag`() { + fun `writeBytes should call stream#writeByteArray`() { val input = ByteArray(0) target.writeBytes(input) - verify(stream).writeByteArrayNoTag(input) + verify(stream).writeBytes(input) } @Test @@ -214,11 +210,10 @@ class MarshallerTest { } @Test - fun `complete should return the byte array`() { - - val result = target.complete() + fun `complete should call writer#complete`() { + target.complete() - assertThat(result).isEqualTo(bytes) + verify(stream).complete() } @Test From 51784a8f234dad07cf33953b7e6e84b5e0fcc5af Mon Sep 17 00:00:00 2001 From: panini Date: Sat, 21 Nov 2020 15:32:59 +0900 Subject: [PATCH 09/12] rename stream -> writer --- runtime-lite/src/main/kotlin/Marshaller.kt | 38 +++++++++--------- .../src/test/kotlin/MarshallerTest.kt | 40 +++++++++---------- 2 files changed, 39 insertions(+), 39 deletions(-) diff --git a/runtime-lite/src/main/kotlin/Marshaller.kt b/runtime-lite/src/main/kotlin/Marshaller.kt index 3529591..4cacab4 100644 --- a/runtime-lite/src/main/kotlin/Marshaller.kt +++ b/runtime-lite/src/main/kotlin/Marshaller.kt @@ -1,69 +1,69 @@ package jp.co.panpanini -class Marshaller(private val stream: Writer) { +class Marshaller(private val writer: Writer) { companion object { fun allocate(size: Int) = Marshaller(Writer.allocate(size)) } - fun writeTag(tag: Int) = this.apply { stream.writeTag(tag) } + fun writeTag(tag: Int) = this.apply { writer.writeTag(tag) } - fun writeTag(fieldNum: Int, wireType: Int) = this.apply { stream.writeTag((fieldNum shl 3) or wireType) } + fun writeTag(fieldNum: Int, wireType: Int) = this.apply { writer.writeTag((fieldNum shl 3) or wireType) } fun writeDouble(value: Double) { - stream.writeDouble(value) + writer.writeDouble(value) } fun writeFloat(value: Float) { - stream.writeFloat(value) + writer.writeFloat(value) } fun writeInt32(value: Int) { - stream.writeInt32(value) + writer.writeInt32(value) } fun writeInt64(value: Long) { - stream.writeInt64(value) + writer.writeInt64(value) } fun writeUInt32(value: Int) { - stream.writeUInt32(value) + writer.writeUInt32(value) } fun writeUInt64(value: Long) { - stream.writeUInt64(value) + writer.writeUInt64(value) } fun writeSInt32(value: Int) { - stream.writeSInt32(value) + writer.writeSInt32(value) } fun writeSInt64(value: Long) { - stream.writeSInt64(value) + writer.writeSInt64(value) } fun writeFixed32(value: Int) { - stream.writeFixed32(value) + writer.writeFixed32(value) } fun writeFixed64(value: Long) { - stream.writeFixed64(value) + writer.writeFixed64(value) } fun writeSFixed32(value: Int) { - stream.writeSFixed32(value) + writer.writeSFixed32(value) } fun writeSFixed64(value: Long) { - stream.writeSFixed64(value) + writer.writeSFixed64(value) } fun writeBool(value: Boolean) { - stream.writeBool(value) + writer.writeBool(value) } fun writeString(value: String) { - stream.writeString(value) + writer.writeString(value) } fun writeBytes(value: ByteArr) { @@ -71,7 +71,7 @@ class Marshaller(private val stream: Writer) { } fun writeBytes(value: ByteArray) { - stream.writeBytes(value) + writer.writeBytes(value) } fun writeUnknownFields(fields: Map) { @@ -98,7 +98,7 @@ class Marshaller(private val stream: Writer) { writeInt32(value.value) } - fun complete() = stream.complete() + fun complete() = writer.complete() fun > writeMap( tag: Int, diff --git a/runtime-lite/src/test/kotlin/MarshallerTest.kt b/runtime-lite/src/test/kotlin/MarshallerTest.kt index 6eaa9c7..673227a 100644 --- a/runtime-lite/src/test/kotlin/MarshallerTest.kt +++ b/runtime-lite/src/test/kotlin/MarshallerTest.kt @@ -11,9 +11,9 @@ import org.mockito.ArgumentMatchers.anyInt class MarshallerTest { - private var stream: Writer = mock { } + private var writer: Writer = mock { } - private var target: Marshaller = Marshaller(stream) + private var target: Marshaller = Marshaller(writer) @Test fun `writeTag should call stream#writeTag`() { @@ -21,7 +21,7 @@ class MarshallerTest { target.writeTag(input) - verify(stream).writeTag(input) + verify(writer).writeTag(input) } @Test @@ -33,7 +33,7 @@ class MarshallerTest { target.writeTag(fieldNum, wireType) - verify(stream).writeTag(fieldNumSHL3 + wireType) + verify(writer).writeTag(fieldNumSHL3 + wireType) } @Test @@ -42,7 +42,7 @@ class MarshallerTest { target.writeDouble(input) - verify(stream).writeDouble(input) + verify(writer).writeDouble(input) } @Test @@ -51,7 +51,7 @@ class MarshallerTest { target.writeFloat(input) - verify(stream).writeFloat(input) + verify(writer).writeFloat(input) } @Test @@ -60,7 +60,7 @@ class MarshallerTest { target.writeInt32(input) - verify(stream).writeInt32(input) + verify(writer).writeInt32(input) } @Test @@ -69,7 +69,7 @@ class MarshallerTest { target.writeInt64(input) - verify(stream).writeInt64(input) + verify(writer).writeInt64(input) } @Test @@ -78,7 +78,7 @@ class MarshallerTest { target.writeUInt32(input) - verify(stream).writeUInt32(input) + verify(writer).writeUInt32(input) } @Test @@ -87,7 +87,7 @@ class MarshallerTest { target.writeUInt64(input) - verify(stream).writeUInt64(input) + verify(writer).writeUInt64(input) } @Test @@ -96,7 +96,7 @@ class MarshallerTest { target.writeSInt32(input) - verify(stream).writeSInt32(input) + verify(writer).writeSInt32(input) } @Test @@ -105,7 +105,7 @@ class MarshallerTest { target.writeSInt64(input) - verify(stream).writeSInt64(input) + verify(writer).writeSInt64(input) } @Test @@ -114,7 +114,7 @@ class MarshallerTest { target.writeFixed32(input) - verify(stream).writeFixed32(input) + verify(writer).writeFixed32(input) } @Test @@ -123,7 +123,7 @@ class MarshallerTest { target.writeFixed64(input) - verify(stream).writeFixed64(input) + verify(writer).writeFixed64(input) } @Test @@ -132,7 +132,7 @@ class MarshallerTest { target.writeSFixed32(input) - verify(stream).writeSFixed32(input) + verify(writer).writeSFixed32(input) } @Test @@ -141,7 +141,7 @@ class MarshallerTest { target.writeSFixed64(input) - verify(stream).writeSFixed64(input) + verify(writer).writeSFixed64(input) } @Test @@ -150,7 +150,7 @@ class MarshallerTest { target.writeBool(input) - verify(stream).writeBool(input) + verify(writer).writeBool(input) } @Test @@ -159,7 +159,7 @@ class MarshallerTest { target.writeString("") - verify(stream).writeString(input) + verify(writer).writeString(input) } @Test @@ -181,7 +181,7 @@ class MarshallerTest { target.writeBytes(input) - verify(stream).writeBytes(input) + verify(writer).writeBytes(input) } @Test @@ -213,7 +213,7 @@ class MarshallerTest { fun `complete should call writer#complete`() { target.complete() - verify(stream).complete() + verify(writer).complete() } @Test From 69ceca579098f4b3104663e37820b41ecf8e7bdb Mon Sep 17 00:00:00 2001 From: panini Date: Sun, 22 Nov 2020 12:18:49 +0900 Subject: [PATCH 10/12] Make Marshaller constructor internal --- runtime-lite/src/main/kotlin/Marshaller.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime-lite/src/main/kotlin/Marshaller.kt b/runtime-lite/src/main/kotlin/Marshaller.kt index 4cacab4..6e04486 100644 --- a/runtime-lite/src/main/kotlin/Marshaller.kt +++ b/runtime-lite/src/main/kotlin/Marshaller.kt @@ -1,6 +1,6 @@ package jp.co.panpanini -class Marshaller(private val writer: Writer) { +class Marshaller internal constructor(private val writer: Writer) { companion object { fun allocate(size: Int) = Marshaller(Writer.allocate(size)) From 1c4e1af7e474d0b25e86f179521f4623bf3299be Mon Sep 17 00:00:00 2001 From: panini Date: Sun, 22 Nov 2020 12:19:10 +0900 Subject: [PATCH 11/12] Pass Utf8 instance to Writer, make Writer internal to make testing easier --- runtime-lite/src/main/kotlin/Writer.kt | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/runtime-lite/src/main/kotlin/Writer.kt b/runtime-lite/src/main/kotlin/Writer.kt index 863fe02..a08bec3 100644 --- a/runtime-lite/src/main/kotlin/Writer.kt +++ b/runtime-lite/src/main/kotlin/Writer.kt @@ -1,6 +1,8 @@ package jp.co.panpanini -class Writer(private val byteArray: ByteArray) { +import Utf8 + +internal class Writer(private val byteArray: ByteArray, private val utf8: Utf8 = Utf8) { private var position = 0 @@ -111,12 +113,12 @@ class Writer(private val byteArray: ByteArray) { val oldPosition = position // UTF-8 byte length of the string is at least its UTF-16 code unit length (value.length()), // and at most 3 times of it. We take advantage of this in both branches below. - val maxLength: Int = value.length * Utf8.MAX_BYTES_PER_CHAR + val maxLength: Int = value.length * utf8.MAX_BYTES_PER_CHAR val maxLengthVarIntSize: Int = computeUInt32Size(maxLength) val minLengthVarIntSize: Int = computeUInt32Size(value.length) if (minLengthVarIntSize == maxLengthVarIntSize) { position = oldPosition + minLengthVarIntSize - val newPosition: Int = Utf8.encode(value, byteArray, position, spaceLeft()) + val newPosition: Int = utf8.encode(value, byteArray, position, spaceLeft()) // Since this class is stateful and tracks the position, we rewind and store the state, // prepend the length, then reset it back to the end of the string. position = oldPosition @@ -124,9 +126,9 @@ class Writer(private val byteArray: ByteArray) { writeUInt32(length) position = newPosition } else { - val length: Int = Utf8.encodedLength(value) + val length: Int = utf8.encodedLength(value) writeUInt32(length) - position = Utf8.encode(value, byteArray, position, spaceLeft()) + position = utf8.encode(value, byteArray, position, spaceLeft()) } } From 6abf7866eacd02c199a1e51773c770f75d16354a Mon Sep 17 00:00:00 2001 From: panini Date: Sun, 22 Nov 2020 12:31:46 +0900 Subject: [PATCH 12/12] Add writeString tests --- runtime-lite/src/test/kotlin/WriterTest.kt | 33 ++++++++++++++++++++-- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/runtime-lite/src/test/kotlin/WriterTest.kt b/runtime-lite/src/test/kotlin/WriterTest.kt index ffb28b8..293c3ae 100644 --- a/runtime-lite/src/test/kotlin/WriterTest.kt +++ b/runtime-lite/src/test/kotlin/WriterTest.kt @@ -1,13 +1,17 @@ package jp.co.panpanini -import com.nhaarman.mockitokotlin2.spy -import com.nhaarman.mockitokotlin2.verify +import Utf8 +import com.nhaarman.mockitokotlin2.* import org.assertj.core.api.Assertions.assertThat import org.junit.Before import org.junit.Test +import org.mockito.ArgumentMatchers.anyInt +import org.mockito.ArgumentMatchers.anyString class WriterTest { + private val utf8: Utf8 = mock { } + private lateinit var byteArray: ByteArray private lateinit var target: Writer @@ -16,7 +20,7 @@ class WriterTest { @Before fun setup() { byteArray = ByteArray(100) - target = spy(Writer(byteArray)) + target = spy(Writer(byteArray, utf8)) } fun setup(size: Int) { @@ -267,4 +271,27 @@ class WriterTest { assertThat(byteArray).isEqualTo(expected) } + @Test + fun `writeString should call utf8#encode`() { + val input = "Hello, World!" + val computedLength = 13 + + whenever(utf8.encode(anyString(), any(), anyInt(), anyInt())).thenReturn(computedLength) + + target.writeString(input) + + verify(target).writeUInt32(computedLength - 1) // string length int is 1 byte + verify(utf8).encode(input, byteArray, 1, 99) + } + + @Test + fun `writeString should call utf8#encodedLength for a long string`() { + val input = (0..1000).joinToString { "test$it " } + + target.writeString(input) + + verify(utf8).encodedLength(input) // ask utf8 for the right length + verify(utf8).encode(input, byteArray, 1, 99) + } + } \ No newline at end of file