From cc23408645f84aa4dfc3cb03e1005a1b3864bfd3 Mon Sep 17 00:00:00 2001 From: gmpassos Date: Sat, 26 Jul 2025 00:35:14 -0300 Subject: [PATCH 01/13] v1.9.5 - reflection_factory: ^2.5.2 - swiss_knife: ^3.3.3 - test: ^1.26.3 --- CHANGELOG.md | 7 +++++++ lib/src/bones_api_base.dart | 2 +- pubspec.yaml | 8 ++++---- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index baef3d5..02194a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## 1.9.5 + +- reflection_factory: ^2.5.2 +- swiss_knife: ^3.3.3 + +- test: ^1.26.3 + ## 1.9.4 - Main updates (see `v1.9.4-beta.*` for more): diff --git a/lib/src/bones_api_base.dart b/lib/src/bones_api_base.dart index 745daa5..82fc851 100644 --- a/lib/src/bones_api_base.dart +++ b/lib/src/bones_api_base.dart @@ -42,7 +42,7 @@ typedef APILogger = void Function(APIRoot apiRoot, String type, String? message, /// Bones API Library class. class BonesAPI { // ignore: constant_identifier_names - static const String VERSION = '1.9.4'; + static const String VERSION = '1.9.5'; static bool _boot = false; diff --git a/pubspec.yaml b/pubspec.yaml index 0e80b8f..4ce7b10 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -1,6 +1,6 @@ name: bones_api description: Bones_API - A powerful API backend framework for Dart. It comes with a built-in HTTP Server, route handler, entity handler, SQL translator, and DB adapters. -version: 1.9.4 +version: 1.9.5 homepage: https://github.com/Colossus-Services/bones_api environment: @@ -12,9 +12,9 @@ executables: dependencies: async_extension: ^1.2.15 async_events: ^1.3.0 - reflection_factory: ^2.5.1 + reflection_factory: ^2.5.2 statistics: ^1.2.0 - swiss_knife: ^3.3.0 + swiss_knife: ^3.3.3 data_serializer: ^1.2.1 shared_map: ^1.1.9 graph_explorer: ^1.0.2 @@ -55,7 +55,7 @@ dev_dependencies: lints: ^5.1.1 build_runner: ^2.4.15 build_verify: ^3.1.1 - test: ^1.26.2 + test: ^1.26.3 pubspec: ^2.3.0 dependency_validator: ^4.1.3 coverage: ^1.15.0 From 3d826055f8803203540cd2915e7f73586092e1cc Mon Sep 17 00:00:00 2001 From: gmpassos Date: Sat, 26 Jul 2025 00:49:41 -0300 Subject: [PATCH 02/13] - `APIServer`: - `_resolveBodyImpl`: - Catch `OutOfMemoryError` and log. - Log errors while encoding payload to JSON. --- CHANGELOG.md | 5 +++++ lib/src/bones_api_server.dart | 14 ++++++++++---- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 02194a2..2602d55 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ ## 1.9.5 +- `APIServer`: + - `_resolveBodyImpl`: + - Catch `OutOfMemoryError` and log. + - Log errors while encoding payload to JSON. + - reflection_factory: ^2.5.2 - swiss_knife: ^3.3.3 diff --git a/lib/src/bones_api_server.dart b/lib/src/bones_api_server.dart index d1c84c7..65a569c 100644 --- a/lib/src/bones_api_server.dart +++ b/lib/src/bones_api_server.dart @@ -1756,11 +1756,17 @@ class APIServer extends _APIServerBase { apiResponse.payloadMimeType ??= 'application/json'; return s; - } catch (e) { - var s = payload.toString(); + } on OutOfMemoryError catch (e, s) { + var msg = "`OutOfMemoryError` while encoding payload to JSON!"; + _log.severe(msg, e, s); + return apiResponse.asError(error: '** $msg\n$e\n$s'); + } catch (e, s) { + _log.warning("ERROR while encoding payload to JSON!", e, s); + apiResponse.headers['X-Payload-Encoding-Error'] = '$e'; + var p = payload.toString(); apiResponse.payloadMimeType ??= - resolveBestTextMimeType(s, apiResponse.payloadFileExtension); - return s; + resolveBestTextMimeType(p, apiResponse.payloadFileExtension); + return p; } } From 09ef5f9aca2bdd8e28dc6c355ae66c906995a12f Mon Sep 17 00:00:00 2001 From: gmpassos Date: Sat, 26 Jul 2025 00:55:01 -0300 Subject: [PATCH 03/13] - `APIServer`: - `_resolveBodyImpl`: - Log errors while encoding payload to JSON. - Catch `OutOfMemoryError` and log. - Return `apiResponse.asError` on errors. --- CHANGELOG.md | 3 ++- lib/src/bones_api_server.dart | 11 ++++------- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2602d55..33bc2b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,8 +2,9 @@ - `APIServer`: - `_resolveBodyImpl`: - - Catch `OutOfMemoryError` and log. - Log errors while encoding payload to JSON. + - Catch `OutOfMemoryError` and log. + - Return `apiResponse.asError` on errors. - reflection_factory: ^2.5.2 - swiss_knife: ^3.3.3 diff --git a/lib/src/bones_api_server.dart b/lib/src/bones_api_server.dart index 65a569c..ecf86e2 100644 --- a/lib/src/bones_api_server.dart +++ b/lib/src/bones_api_server.dart @@ -1759,14 +1759,11 @@ class APIServer extends _APIServerBase { } on OutOfMemoryError catch (e, s) { var msg = "`OutOfMemoryError` while encoding payload to JSON!"; _log.severe(msg, e, s); - return apiResponse.asError(error: '** $msg\n$e\n$s'); + return apiResponse.asError(error: '** $msg\n$e', stackTrace: s); } catch (e, s) { - _log.warning("ERROR while encoding payload to JSON!", e, s); - apiResponse.headers['X-Payload-Encoding-Error'] = '$e'; - var p = payload.toString(); - apiResponse.payloadMimeType ??= - resolveBestTextMimeType(p, apiResponse.payloadFileExtension); - return p; + var msg = "ERROR while encoding payload to JSON!"; + _log.severe(msg, e, s); + return apiResponse.asError(error: '** $msg\n$e', stackTrace: s); } } From e821cd5dcd36b59599360d32710a44a587c761f4 Mon Sep 17 00:00:00 2001 From: gmpassos Date: Sun, 27 Jul 2025 05:13:47 -0300 Subject: [PATCH 04/13] - Added `AutoGZipSink`, `GZipSink` and `BytesSink` and `BytesBuffer`. --- lib/src/bones_api_utils_sink.dart | 229 ++++++++++++++++++++++++++++++ 1 file changed, 229 insertions(+) create mode 100644 lib/src/bones_api_utils_sink.dart diff --git a/lib/src/bones_api_utils_sink.dart b/lib/src/bones_api_utils_sink.dart new file mode 100644 index 0000000..877eef6 --- /dev/null +++ b/lib/src/bones_api_utils_sink.dart @@ -0,0 +1,229 @@ +import 'dart:convert'; +import 'dart:io'; +import 'dart:typed_data'; + +class AutoGZipSink extends ByteConversionSinkBuffered { + ByteConversionSinkBuffered _sink; + + final int minGZipLength; + + AutoGZipSink({this.minGZipLength = 512, int capacity = 1024}) + : _sink = BytesSink(capacity: capacity); + + @override + int get inputLength => _sink.inputLength; + + bool _gzip = false; + + bool get isGzip => _gzip; + + void switchToGzip() { + if (_gzip) return; + var prevBytes = _sink.toBytes(copy: true); + _sink.reset(); + + _sink = GZipSink(bytesSink: _sink); + _gzip = true; + + _sink.add(prevBytes); + } + + @override + void add(List chunk) { + if (!_gzip && _sink.length + chunk.length >= minGZipLength) { + switchToGzip(); + } + + _sink.add(chunk); + } + + @override + void close() { + _sink.close(); + } + + @override + Uint8List toBytes({bool copy = false}) { + return _sink.toBytes(); + } + + @override + int get capacity => _sink.capacity; + + @override + int get length => _sink.length; + + @override + void reset() { + if (!_gzip) { + _sink.reset(); + } else { + var gzipSink = _sink as GZipSink; + + // Close to avoid memory leak: + gzipSink.close(); + + _sink = gzipSink._bytesSink; + _sink.reset(); + _gzip = false; + } + } +} + +class GZipSink extends ByteConversionSinkBuffered { + final ByteConversionSinkBuffered _bytesSink; + late ByteConversionSink _gzipSink; + final int level; + + GZipSink( + {this.level = 4, + ByteConversionSinkBuffered? bytesSink, + int capacity = 1024 * 4}) + : _bytesSink = bytesSink ?? BytesSink(capacity: capacity) { + _gzipSink = _createGZipEncoder(level); + } + + ByteConversionSink _createGZipEncoder(int level) => ZLibEncoder( + gzip: true, + level: level, + windowBits: ZLibOption.defaultWindowBits, + memLevel: ZLibOption.defaultMemLevel, + strategy: ZLibOption.strategyDefault, + dictionary: null, + raw: false, + ).startChunkedConversion(_bytesSink); + + @override + int get length => _bytesSink.length; + + @override + int get capacity => _bytesSink.capacity; + + int _inputLength = 0; + + @override + int get inputLength => _inputLength; + + @override + void add(List chunk) { + _inputLength += chunk.length; + _gzipSink.add(chunk); + } + + @override + void close() => _gzipSink.close(); + + @override + Uint8List toBytes({bool copy = false}) => _bytesSink.toBytes(); + + @override + void reset() { + _gzipSink.close(); + _bytesSink.reset(); + _gzipSink = _createGZipEncoder(level); + _inputLength = 0; + } +} + +class BytesSink extends BytesBuffer implements ByteConversionSinkBuffered { + BytesSink({super.capacity}); + + @override + int get inputLength => length; + + @override + void close() {} + + @override + void addSlice(List chunk, int start, int end, bool isLast) { + if (start != 0 || end != chunk.length) { + chunk = chunk.sublist(start, end); + } + add(chunk); + if (isLast) close(); + } +} + +abstract class ByteConversionSinkBuffered extends ByteConversionSink { + int get length; + + int get capacity; + + int get inputLength; + + Uint8List toBytes({bool copy = false}); + + void reset(); +} + +class BytesBuffer { + Uint8List _buffer; + int _length = 0; + + BytesBuffer({int capacity = 1204 * 4}) : _buffer = Uint8List(capacity); + + int get length => _length; + + int get capacity => _buffer.length; + + /// Returns the added bytes as a [Uint8List]. See [add] + /// + /// If [copy] is `false` and the buffer is fully used (`_length == _buffer.length`), + /// returns the internal buffer directly to avoid copying. + /// Otherwise, returns a view of the buffer up to the actual written length. + Uint8List toBytes({bool copy = false}) { + if (!copy && _length == _buffer.length) { + return _buffer; + } + return Uint8List.sublistView(_buffer, 0, _length); + } + + /// Add butes to the buffer. + void add(List bytes) { + final chunkLength = bytes.length; + final requiredLength = _length + chunkLength; + + if (requiredLength > _buffer.length) { + _increaseCapacity(requiredLength); + } + + _buffer.setRange(_length, _length + chunkLength, bytes); + _length += chunkLength; + } + + void _increaseCapacity(int requiredLength) { + final newLength = computeNewLength(_buffer.length, requiredLength); + assert(newLength >= requiredLength); + final newBuffer = Uint8List(newLength); + newBuffer.setRange(0, _length, _buffer); + _buffer = newBuffer; + } + + int computeNewLength(int prevLength, int requiredLength) { + assert(prevLength < requiredLength); + + if (requiredLength < 1024 * 1024 * 8) { + var newLength = prevLength * 2; + while (newLength < requiredLength) { + newLength = newLength * 2; + } + return newLength; + } else if (requiredLength < 1024 * 1024 * 32) { + var newLength = (prevLength * 1.5).toInt(); + while (newLength < requiredLength) { + newLength = (newLength * 1.5).toInt(); + } + return newLength; + } else { + var newLength = (prevLength * 1.25).toInt(); + while (newLength < requiredLength) { + newLength = (newLength * 1.25).toInt(); + } + return newLength; + } + } + + void reset() { + _length = 0; + } +} From f5d20e332d174ecf3bd93bfa22758d42a4d17ec8 Mon Sep 17 00:00:00 2001 From: gmpassos Date: Sun, 27 Jul 2025 05:25:29 -0300 Subject: [PATCH 05/13] docs --- lib/src/bones_api_utils_sink.dart | 53 ++++++++++++++----------------- 1 file changed, 24 insertions(+), 29 deletions(-) diff --git a/lib/src/bones_api_utils_sink.dart b/lib/src/bones_api_utils_sink.dart index 877eef6..aca7d14 100644 --- a/lib/src/bones_api_utils_sink.dart +++ b/lib/src/bones_api_utils_sink.dart @@ -2,9 +2,12 @@ import 'dart:convert'; import 'dart:io'; import 'dart:typed_data'; +/// Automatically switches to GZip compression when input exceeds [minGZipLength]. class AutoGZipSink extends ByteConversionSinkBuffered { + /// Inner sink that may switch between plain and gzip modes. ByteConversionSinkBuffered _sink; + /// Minimum input size to trigger GZip compression. final int minGZipLength; AutoGZipSink({this.minGZipLength = 512, int capacity = 1024}) @@ -15,16 +18,16 @@ class AutoGZipSink extends ByteConversionSinkBuffered { bool _gzip = false; + /// Returns `true` if GZip compression is active. bool get isGzip => _gzip; + /// Switches to GZip compression. Keeps previously added data. void switchToGzip() { if (_gzip) return; var prevBytes = _sink.toBytes(copy: true); _sink.reset(); - _sink = GZipSink(bytesSink: _sink); _gzip = true; - _sink.add(prevBytes); } @@ -33,19 +36,14 @@ class AutoGZipSink extends ByteConversionSinkBuffered { if (!_gzip && _sink.length + chunk.length >= minGZipLength) { switchToGzip(); } - _sink.add(chunk); } @override - void close() { - _sink.close(); - } + void close() => _sink.close(); @override - Uint8List toBytes({bool copy = false}) { - return _sink.toBytes(); - } + Uint8List toBytes({bool copy = false}) => _sink.toBytes(copy: copy); @override int get capacity => _sink.capacity; @@ -53,16 +51,14 @@ class AutoGZipSink extends ByteConversionSinkBuffered { @override int get length => _sink.length; + /// Resets internal state and disables GZip if active. @override void reset() { if (!_gzip) { _sink.reset(); } else { var gzipSink = _sink as GZipSink; - - // Close to avoid memory leak: - gzipSink.close(); - + gzipSink.close(); // Avoid memory leak _sink = gzipSink._bytesSink; _sink.reset(); _gzip = false; @@ -70,16 +66,17 @@ class AutoGZipSink extends ByteConversionSinkBuffered { } } +/// Wraps a sink to apply GZip compression using [ZLibEncoder]. class GZipSink extends ByteConversionSinkBuffered { final ByteConversionSinkBuffered _bytesSink; late ByteConversionSink _gzipSink; final int level; - GZipSink( - {this.level = 4, - ByteConversionSinkBuffered? bytesSink, - int capacity = 1024 * 4}) - : _bytesSink = bytesSink ?? BytesSink(capacity: capacity) { + GZipSink({ + this.level = 4, + ByteConversionSinkBuffered? bytesSink, + int capacity = 1024 * 4, + }) : _bytesSink = bytesSink ?? BytesSink(capacity: capacity) { _gzipSink = _createGZipEncoder(level); } @@ -116,6 +113,7 @@ class GZipSink extends ByteConversionSinkBuffered { @override Uint8List toBytes({bool copy = false}) => _bytesSink.toBytes(); + /// Resets the compressor and underlying buffer. @override void reset() { _gzipSink.close(); @@ -125,6 +123,7 @@ class GZipSink extends ByteConversionSinkBuffered { } } +/// Buffer that accumulates bytes and supports reset and conversion to [Uint8List]. class BytesSink extends BytesBuffer implements ByteConversionSinkBuffered { BytesSink({super.capacity}); @@ -144,11 +143,10 @@ class BytesSink extends BytesBuffer implements ByteConversionSinkBuffered { } } +/// Base class for sinks with tracking and reset support. abstract class ByteConversionSinkBuffered extends ByteConversionSink { int get length; - int get capacity; - int get inputLength; Uint8List toBytes({bool copy = false}); @@ -156,6 +154,7 @@ abstract class ByteConversionSinkBuffered extends ByteConversionSink { void reset(); } +/// A growing byte buffer with auto-resize and slice access. class BytesBuffer { Uint8List _buffer; int _length = 0; @@ -166,11 +165,7 @@ class BytesBuffer { int get capacity => _buffer.length; - /// Returns the added bytes as a [Uint8List]. See [add] - /// - /// If [copy] is `false` and the buffer is fully used (`_length == _buffer.length`), - /// returns the internal buffer directly to avoid copying. - /// Otherwise, returns a view of the buffer up to the actual written length. + /// Returns buffer contents. May return internal buffer if fully used and [copy] is false. Uint8List toBytes({bool copy = false}) { if (!copy && _length == _buffer.length) { return _buffer; @@ -178,15 +173,13 @@ class BytesBuffer { return Uint8List.sublistView(_buffer, 0, _length); } - /// Add butes to the buffer. + /// Adds [bytes] to buffer, resizing if needed. void add(List bytes) { final chunkLength = bytes.length; final requiredLength = _length + chunkLength; - if (requiredLength > _buffer.length) { _increaseCapacity(requiredLength); } - _buffer.setRange(_length, _length + chunkLength, bytes); _length += chunkLength; } @@ -199,13 +192,14 @@ class BytesBuffer { _buffer = newBuffer; } + /// Calculates next buffer size based on usage pattern. int computeNewLength(int prevLength, int requiredLength) { assert(prevLength < requiredLength); if (requiredLength < 1024 * 1024 * 8) { var newLength = prevLength * 2; while (newLength < requiredLength) { - newLength = newLength * 2; + newLength *= 2; } return newLength; } else if (requiredLength < 1024 * 1024 * 32) { @@ -223,6 +217,7 @@ class BytesBuffer { } } + /// Clears the buffer content without releasing memory. void reset() { _length = 0; } From 53c0467cfe86c8f598e237fb62712d1846516258 Mon Sep 17 00:00:00 2001 From: gmpassos Date: Sun, 27 Jul 2025 05:25:48 -0300 Subject: [PATCH 06/13] bones_api_utils_sink_test.dart --- test/bones_api_utils_sink_test.dart | 260 ++++++++++++++++++++++++++++ 1 file changed, 260 insertions(+) create mode 100644 test/bones_api_utils_sink_test.dart diff --git a/test/bones_api_utils_sink_test.dart b/test/bones_api_utils_sink_test.dart new file mode 100644 index 0000000..388464c --- /dev/null +++ b/test/bones_api_utils_sink_test.dart @@ -0,0 +1,260 @@ +import 'package:bones_api/src/bones_api_utils_sink.dart'; +import 'package:test/test.dart'; + +void main() { + group('BytesBuffer', () { + test('basic', () { + var bs = BytesBuffer(); + + expect(bs.length, equals(0)); + expect(bs.toBytes(), equals([])); + + bs.add([1, 2, 3]); + expect(bs.length, equals(3)); + expect(bs.toBytes(), equals([1, 2, 3])); + + bs.add([4, 5, 6, 7, 8, 9, 10]); + expect(bs.length, equals(10)); + expect(bs.toBytes(), equals([1, 2, 3, 4, 5, 6, 7, 8, 9, 10])); + + bs.reset(); + expect(bs.length, equals(0)); + expect(bs.toBytes(), equals([])); + + bs.add([10, 20, 30]); + expect(bs.length, equals(3)); + expect(bs.toBytes(), equals([10, 20, 30])); + }); + }); + + group('BytesSink', () { + test('basic', () { + var bs = BytesSink(capacity: 8); + + expect(bs.capacity, equals(8)); + expect(bs.length, equals(0)); + expect(bs.inputLength, equals(0)); + expect(bs.toBytes(), equals([])); + + bs.add([1, 2, 3]); + expect(bs.length, equals(3)); + expect(bs.inputLength, equals(3)); + expect(bs.toBytes(), equals([1, 2, 3])); + + bs.add([4, 5, 6, 7, 8, 9, 10]); + expect(bs.length, equals(10)); + expect(bs.inputLength, equals(10)); + expect(bs.toBytes(), equals([1, 2, 3, 4, 5, 6, 7, 8, 9, 10])); + + bs.addSlice([100, 101, 102], 1, 3, false); + expect(bs.capacity, equals(16)); + expect(bs.length, equals(12)); + expect(bs.inputLength, equals(12)); + expect(bs.toBytes(), equals([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 101, 102])); + + bs.reset(); + expect(bs.length, equals(0)); + expect(bs.inputLength, equals(0)); + expect(bs.toBytes(), equals([])); + + bs.add([10, 20, 30]); + expect(bs.length, equals(3)); + expect(bs.inputLength, equals(3)); + expect(bs.toBytes(), equals([10, 20, 30])); + }); + }); + + group('GZipSink', () { + test('basic', () { + const gzipHeader = [31, 139, 8, 0, 0, 0, 0, 0, 0, 19]; + + var bs = GZipSink(capacity: 8); + + expect(bs.capacity, equals(8)); + expect(bs.length, equals(0)); + expect(bs.inputLength, equals(0)); + expect(bs.toBytes(), equals([])); + + bs.add([1, 2, 3]); + expect(bs.capacity, equals(16)); + expect(bs.length, equals(10)); + expect(bs.inputLength, equals(3)); + expect(bs.toBytes(), equals(gzipHeader)); + + bs.add([4, 5, 6, 7, 8, 9, 10]); + expect(bs.length, equals(10)); + expect(bs.inputLength, equals(10)); + expect(bs.toBytes(), equals(gzipHeader)); + + bs.addSlice([100, 101, 102], 1, 3, false); + expect(bs.capacity, equals(16)); + expect(bs.length, equals(10)); + expect(bs.inputLength, equals(12)); + expect(bs.toBytes(), equals(gzipHeader)); + + bs.close(); + expect(bs.capacity, equals(32)); + expect(bs.length, equals(32)); + expect(bs.inputLength, equals(12)); + print(bs.toBytes()); + expect( + bs.toBytes(), + equals([ + ...gzipHeader, + 99, + 100, + 98, + 102, + 97, + 101, + 99, + 231, + 224, + 228, + 74, + 77, + 3, + 0, + 58, + 8, + 70, + 196, + 12, + 0, + 0, + 0 + ])); + + bs.reset(); + expect(bs.capacity, equals(32)); + expect(bs.length, equals(0)); + expect(bs.inputLength, equals(0)); + expect(bs.toBytes(), equals([])); + + bs.add([1, 2, 3]); + expect(bs.capacity, equals(32)); + expect(bs.length, equals(10)); + expect(bs.inputLength, equals(3)); + expect(bs.toBytes(), equals(gzipHeader)); + + bs.close(); + expect(bs.capacity, equals(32)); + expect(bs.length, equals(23)); + expect(bs.inputLength, equals(3)); + expect( + bs.toBytes(), + equals([ + ...gzipHeader, + 99, + 100, + 98, + 6, + 0, + 29, + 128, + 188, + 85, + 3, + 0, + 0, + 0 + ])); + }); + }); + + group('AutoGZipSink', () { + test('basic', () { + const gzipHeader = [31, 139, 8, 0, 0, 0, 0, 0, 0, 19]; + + var bs = AutoGZipSink(minGZipLength: 10, capacity: 8); + + expect(bs.isGzip, isFalse); + expect(bs.capacity, equals(8)); + expect(bs.length, equals(0)); + expect(bs.inputLength, equals(0)); + expect(bs.toBytes(), equals([])); + + bs.add([1, 2, 3]); + expect(bs.isGzip, isFalse); + expect(bs.capacity, equals(8)); + expect(bs.length, equals(3)); + expect(bs.inputLength, equals(3)); + expect(bs.toBytes(), equals([1, 2, 3])); + + bs.add([4, 5, 6, 7, 8]); + expect(bs.isGzip, isFalse); + expect(bs.capacity, equals(8)); + expect(bs.length, equals(8)); + expect(bs.inputLength, equals(8)); + expect(bs.toBytes(), equals([1, 2, 3, 4, 5, 6, 7, 8])); + + bs.add([9, 10]); + expect(bs.isGzip, isTrue); + expect(bs.length, equals(10)); + expect(bs.inputLength, equals(10)); + expect(bs.toBytes(), equals(gzipHeader)); + + bs.addSlice([100, 101, 102], 1, 3, false); + expect(bs.isGzip, isTrue); + expect(bs.capacity, equals(16)); + expect(bs.length, equals(10)); + expect(bs.inputLength, equals(12)); + expect(bs.toBytes(), equals(gzipHeader)); + + bs.close(); + expect(bs.isGzip, isTrue); + expect(bs.capacity, equals(32)); + expect(bs.length, equals(32)); + expect(bs.inputLength, equals(12)); + print(bs.toBytes()); + expect( + bs.toBytes(), + equals([ + ...gzipHeader, + 99, + 100, + 98, + 102, + 97, + 101, + 99, + 231, + 224, + 228, + 74, + 77, + 3, + 0, + 58, + 8, + 70, + 196, + 12, + 0, + 0, + 0 + ])); + + bs.reset(); + expect(bs.isGzip, isFalse); + expect(bs.capacity, equals(32)); + expect(bs.length, equals(0)); + expect(bs.inputLength, equals(0)); + expect(bs.toBytes(), equals([])); + + bs.add([1, 2, 3]); + expect(bs.isGzip, isFalse); + expect(bs.capacity, equals(32)); + expect(bs.length, equals(3)); + expect(bs.inputLength, equals(3)); + expect(bs.toBytes(), equals([1, 2, 3])); + + bs.close(); + expect(bs.isGzip, isFalse); + expect(bs.capacity, equals(32)); + expect(bs.length, equals(3)); + expect(bs.inputLength, equals(3)); + expect(bs.toBytes(), equals([1, 2, 3])); + }); + }); +} From aa33de60290cc6034c7978068a84c107c753f798 Mon Sep 17 00:00:00 2001 From: gmpassos Date: Sun, 27 Jul 2025 05:26:05 -0300 Subject: [PATCH 07/13] - Added `AutoGZipSink`, `GZipSink` and `BytesSink` and `BytesBuffer`. --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 33bc2b0..a6485ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ - Catch `OutOfMemoryError` and log. - Return `apiResponse.asError` on errors. +- Added `AutoGZipSink`, `GZipSink` and `BytesSink` and `BytesBuffer`. + - reflection_factory: ^2.5.2 - swiss_knife: ^3.3.3 From 121eadb0ddeeae3be94d15badeb2c418a2490e50 Mon Sep 17 00:00:00 2001 From: gmpassos Date: Sun, 27 Jul 2025 05:26:59 -0300 Subject: [PATCH 08/13] - `Json`: - Added `encodeToSink`. --- CHANGELOG.md | 2 ++ lib/src/bones_api_utils_json.dart | 26 ++++++++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a6485ec..3576d39 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ - Log errors while encoding payload to JSON. - Catch `OutOfMemoryError` and log. - Return `apiResponse.asError` on errors. + - `_jsonEncodePayload`: + - Now uses `AutoGZipSink` and `Json.encodeToSink` to stream JSON encoding with automatic GZip compression based on output size. - Added `AutoGZipSink`, `GZipSink` and `BytesSink` and `BytesBuffer`. diff --git a/lib/src/bones_api_utils_json.dart b/lib/src/bones_api_utils_json.dart index 1056803..fa0189e 100644 --- a/lib/src/bones_api_utils_json.dart +++ b/lib/src/bones_api_utils_json.dart @@ -154,6 +154,32 @@ class Json { pretty: pretty, autoResetEntityCache: autoResetEntityCache); } + /// Sames as [encode] but outputs to [sink]. + static void encodeToSink(Object? o, Sink> sink, + {bool pretty = false, + JsonFieldMatcher? maskField, + String maskText = '***', + JsonFieldMatcher? removeField, + bool removeNullFields = false, + ToEncodableJsonProvider? toEncodableProvider, + ToEncodable? toEncodable, + EntityHandlerProvider? entityHandlerProvider, + EntityCache? entityCache, + bool? autoResetEntityCache}) { + var jsonEncoder = _buildJsonEncoder( + maskField, + maskText, + removeField, + removeNullFields, + toEncodableProvider, + toEncodable, + entityHandlerProvider, + entityCache); + + return jsonEncoder.encodeToSink(o, sink, + pretty: pretty, autoResetEntityCache: autoResetEntityCache); + } + static final JsonEncoder defaultEncoder = JsonEncoder( toEncodableProvider: (o) => _jsonEncodableProvider(o, null), entityCache: JsonEntityCacheSimple(), From 86ace4f6a81a3c9e92fc64fcfc9b87e4cc33ea10 Mon Sep 17 00:00:00 2001 From: gmpassos Date: Sun, 27 Jul 2025 05:28:54 -0300 Subject: [PATCH 09/13] - `Json`: - Added `encodeToSink`. --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3576d39..6834722 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,9 @@ - Added `AutoGZipSink`, `GZipSink` and `BytesSink` and `BytesBuffer`. +- `Json`: + - Added `encodeToSink`. + - reflection_factory: ^2.5.2 - swiss_knife: ^3.3.3 From a026c011ba26a971a95deb49eb39cea320e1b5bf Mon Sep 17 00:00:00 2001 From: gmpassos Date: Sun, 27 Jul 2025 05:29:06 -0300 Subject: [PATCH 10/13] - `_jsonEncodePayload`: - Now uses `AutoGZipSink` and `Json.encodeToSink` to stream JSON encoding with automatic GZip compression based on output size. --- lib/src/bones_api_server.dart | 211 ++++++++++++++++++++++++++-------- 1 file changed, 166 insertions(+), 45 deletions(-) diff --git a/lib/src/bones_api_server.dart b/lib/src/bones_api_server.dart index ecf86e2..10c3d32 100644 --- a/lib/src/bones_api_server.dart +++ b/lib/src/bones_api_server.dart @@ -32,6 +32,7 @@ import 'bones_api_utils.dart'; import 'bones_api_utils_httpclient.dart'; import 'bones_api_utils_isolate.dart'; import 'bones_api_utils_json.dart'; +import 'bones_api_utils_sink.dart'; final _log = logging.Logger('APIServer'); @@ -1749,13 +1750,14 @@ class APIServer extends _APIServerBase { final jsonEncodeInit = DateTime.now(); try { - var s = _jsonEncodePayload(apiResponse, payload); + // Note: `jsonBytes` may be compressed with GZip. + var jsonBytes = _jsonEncodePayload(apiResponse, payload); var jsonEncodeTime = DateTime.now().difference(jsonEncodeInit); - apiResponse.setMetric('API-response-json', duration: jsonEncodeTime); + apiResponse.setMetric('API-Response-JSON', duration: jsonEncodeTime); apiResponse.payloadMimeType ??= 'application/json'; - return s; + return jsonBytes; } on OutOfMemoryError catch (e, s) { var msg = "`OutOfMemoryError` while encoding payload to JSON!"; _log.severe(msg, e, s); @@ -1767,24 +1769,62 @@ class APIServer extends _APIServerBase { } } - static String _jsonEncodePayload(APIResponse apiResponse, payload) { + /// Encodes the given [payload] to JSON and writes it to an [AutoGZipSink]. + /// + /// Returns either raw JSON bytes or GZip-compressed JSON bytes. + static Uint8List _jsonEncodePayload( + APIResponse apiResponse, payload) { final apiRequest = apiResponse.apiRequest; + final encodeInit = DateTime.now(); + final bytesSink = AutoGZipSink(); + if (apiRequest != null) { final routeHandler = apiRequest.routeHandler; - if (routeHandler != null) { var accessRules = routeHandler.entityAccessRules; - if (!accessRules.isInnocuous) { - return Json.encode(payload, + Json.encodeToSink(payload, bytesSink, toEncodableProvider: (o) => accessRules.toJsonEncodable( apiRequest, Json.defaultToEncodableJsonProvider(), o)); + + return _resolveJsonEncodePayloadBytes( + apiResponse, bytesSink, encodeInit); } } } - return Json.encode(payload, toEncodable: ReflectionFactory.toJsonEncodable); + Json.encodeToSink(payload, bytesSink, + toEncodable: ReflectionFactory.toJsonEncodable); + + return _resolveJsonEncodePayloadBytes(apiResponse, bytesSink, encodeInit); + } + + /// Finalizes the JSON encoding and returns the resulting bytes. + /// + /// Closes the [bytesSink] and collects the output. + /// If GZip was applied, adds headers to [apiResponse]. + /// See [_defineGZipEncodedHeaders]. + static Uint8List _resolveJsonEncodePayloadBytes( + APIResponse apiResponse, + AutoGZipSink bytesSink, + DateTime encodeInit) { + bytesSink.close(); + + final bytes = bytesSink.toBytes(); + + if (bytesSink.isGzip) { + var inputLength = bytesSink.inputLength; + var compressionCapacity = bytesSink.capacity; + + _defineGZipEncodedHeaders(apiResponse.headers, + bodyLength: inputLength, + compressedBodyLength: bytes.length, + compressionCapacity: compressionCapacity, + gzipInit: encodeInit); + } + + return bytes; } static final RegExp _htmlTag = RegExp(r'<\w+.*?>'); @@ -1941,9 +1981,17 @@ class APIServer extends _APIServerBase { return def; } - static String resolveServerTiming(Map metrics) { + static String resolveServerTiming(Map metrics, + [String? serverTiming]) { var s = StringBuffer(); + if (serverTiming != null) { + serverTiming = serverTiming.trim(); + if (serverTiming.isNotEmpty) { + s.write(serverTiming); + } + } + for (var e in metrics.entries) { var metric = e.value; @@ -2618,6 +2666,49 @@ final class APIServerWorker extends _APIServerBase { return apiResponse.fileResponse; } + var retPayload = APIServer.resolveBody(apiResponse.payload, apiResponse); + + final serverResponseDelay = this.serverResponseDelay; + if (serverResponseDelay != null && !serverResponseDelay.isNegative) { + final retPayloadOrig = retPayload; + + _log.info( + "[DEV] Response #${apiRequest.id} delayed in ${serverResponseDelay.toStringUnit()}: ${apiRequest.requestedUri}"); + + retPayload = Future.delayed(serverResponseDelay, () async { + var payload = await retPayloadOrig; + return payload; + }); + } + + return retPayload.resolveMapped((payload) { + if (payload is APIResponse) { + var apiResponse2 = payload; + + return APIServer.resolveBody(apiResponse2.payload, apiResponse2) + .resolveMapped((payload2) { + var headers = _resolveAPIResponseHeaders(apiResponse, apiRequest); + + var response = _sendAPIResponse( + request, apiRequest, apiResponse2, headers, payload2); + + apiResponse.disposeAsync(); + + return _processResponse(apiRequest, apiResponse2, request, response); + }); + } else { + var headers = _resolveAPIResponseHeaders(apiResponse, apiRequest); + + var response = _sendAPIResponse( + request, apiRequest, apiResponse, headers, payload); + + return _processResponse(apiRequest, apiResponse, request, response); + } + }); + } + + Map _resolveAPIResponseHeaders( + APIResponse apiResponse, APIRequest apiRequest) { var headers = {}; if (!apiResponse.hasCORS) { @@ -2669,41 +2760,7 @@ final class APIServerWorker extends _APIServerBase { // headers['X-APIToken'] = apiRequest.credential?.token ?? '?'; - var retPayload = APIServer.resolveBody(apiResponse.payload, apiResponse); - - final serverResponseDelay = this.serverResponseDelay; - if (serverResponseDelay != null && !serverResponseDelay.isNegative) { - final retPayloadOrig = retPayload; - - _log.info( - "[DEV] Response #${apiRequest.id} delayed in ${serverResponseDelay.toStringUnit()}: ${apiRequest.requestedUri}"); - - retPayload = Future.delayed(serverResponseDelay, () async { - var payload = await retPayloadOrig; - return payload; - }); - } - - return retPayload.resolveMapped((payload) { - if (payload is APIResponse) { - var apiResponse2 = payload; - - return APIServer.resolveBody(apiResponse2.payload, apiResponse2) - .resolveMapped((payload2) { - var response = _sendAPIResponse( - request, apiRequest, apiResponse2, headers, payload2); - - apiResponse.disposeAsync(); - - return _processResponse(apiRequest, apiResponse2, request, response); - }); - } else { - var response = _sendAPIResponse( - request, apiRequest, apiResponse, headers, payload); - - return _processResponse(apiRequest, apiResponse, request, response); - } - }); + return headers; } FutureOr _processResponse(APIRequest apiRequest, @@ -2780,7 +2837,10 @@ final class APIServerWorker extends _APIServerBase { ? CombinedMapView([apiRequest.metrics, apiResponse.metrics]) : apiResponse.metrics; - headers['server-timing'] = APIServer.resolveServerTiming(allMetrics); + headers['server-timing'] = APIServer.resolveServerTiming( + allMetrics, + headers['server-timing']?.toString(), + ); if (cookieless) { headers.remove(HttpHeaders.setCookieHeader); @@ -2920,3 +2980,64 @@ extension _APIRequestExtension on APIRequest { return Request(method.name, requestedUri); } } + +/// Adds GZip-related headers to the given [headers] map. +/// +/// Sets the `Content-Encoding` and `Content-Length` headers to indicate +/// GZip compression and its size. Optionally adds: +/// +/// - `X-Compression-Ratio`: with detailed compression info +/// - `Server-Timing`: duration of the compression step (if [gzipInit] is provided) +/// +/// Parameters: +/// - [bodyLength]: original uncompressed body size in bytes +/// - [compressedBodyLength]: resulting GZip-compressed size +/// - [compressionCapacity]: internal capacity allocated for compression +/// - [addCompressionRatioHeader]: whether to include `X-Compression-Ratio` +/// - [addServerTiming]: whether to append timing info to `Server-Timing` +/// - [gzipInit]: start time of GZip encoding, required for timing +/// - [serverTimingEntryName]: label used in the `Server-Timing` header +void _defineGZipEncodedHeaders(Map headers, + {required int bodyLength, + required int compressedBodyLength, + required int compressionCapacity, + bool addCompressionRatioHeader = true, + bool addServerTiming = true, + DateTime? gzipInit, + String serverTimingEntryName = 'obj->json->gzip'}) { + headers[HttpHeaders.contentEncodingHeader] = 'gzip'; + headers[HttpHeaders.contentLengthHeader] = compressedBodyLength.toString(); + + if (addCompressionRatioHeader) { + var compressionRatio = compressedBodyLength / bodyLength; + + var compressionRatioStr = '$compressionRatio'; + if (compressionRatioStr.length > 6) { + compressionRatioStr = compressionRatio.toStringAsFixed(4); + } + + headers['X-Compression-Ratio'] = + '$compressionRatioStr ($compressedBodyLength/$bodyLength/$compressionCapacity)'; + } + + if (addServerTiming && gzipInit != null) { + const headerServerTiming = 'server-timing'; + + var gzipTime = DateTime.now().difference(gzipInit); + var dur = gzipTime.inMicroseconds / 1000; + + var serverTiming2 = StringBuffer(); + + var serverTiming = headers[headerServerTiming]?.toString(); + if (serverTiming != null && serverTiming.isNotEmpty) { + serverTiming2.write(serverTiming); + serverTiming2.write(','); + } + + serverTiming2.write(serverTimingEntryName); + serverTiming2.write(';dur='); + serverTiming2.write(dur); + + headers[headerServerTiming] = serverTiming2.toString(); + } +} From 938e19c0ea82b56000bf3e3adbfc15dff8755886 Mon Sep 17 00:00:00 2001 From: gmpassos Date: Sun, 27 Jul 2025 05:44:56 -0300 Subject: [PATCH 11/13] reflection --- test/bones_api_test.reflection.g.dart | 4 ++-- test/bones_api_test_entities.reflection.g.dart | 4 ++-- test/bones_api_test_modules.reflection.g.dart | 4 ++-- test/bones_api_test_utils_test.reflection.g.dart | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/test/bones_api_test.reflection.g.dart b/test/bones_api_test.reflection.g.dart index 6ee7c5e..888537b 100644 --- a/test/bones_api_test.reflection.g.dart +++ b/test/bones_api_test.reflection.g.dart @@ -1,6 +1,6 @@ // // GENERATED CODE - DO NOT MODIFY BY HAND! -// BUILDER: reflection_factory/2.5.1 +// BUILDER: reflection_factory/2.5.2 // BUILD COMMAND: dart run build_runner build // @@ -22,7 +22,7 @@ typedef __TI = TypeInfo; typedef __PR = ParameterReflection; mixin __ReflectionMixin { - static final Version _version = Version.parse('2.5.1'); + static final Version _version = Version.parse('2.5.2'); Version get reflectionFactoryVersion => _version; diff --git a/test/bones_api_test_entities.reflection.g.dart b/test/bones_api_test_entities.reflection.g.dart index 8bfd14f..fea2395 100644 --- a/test/bones_api_test_entities.reflection.g.dart +++ b/test/bones_api_test_entities.reflection.g.dart @@ -1,6 +1,6 @@ // // GENERATED CODE - DO NOT MODIFY BY HAND! -// BUILDER: reflection_factory/2.5.1 +// BUILDER: reflection_factory/2.5.2 // BUILD COMMAND: dart run build_runner build // @@ -22,7 +22,7 @@ typedef __TI = TypeInfo; typedef __PR = ParameterReflection; mixin __ReflectionMixin { - static final Version _version = Version.parse('2.5.1'); + static final Version _version = Version.parse('2.5.2'); Version get reflectionFactoryVersion => _version; diff --git a/test/bones_api_test_modules.reflection.g.dart b/test/bones_api_test_modules.reflection.g.dart index 1c3e6f2..cc113d4 100644 --- a/test/bones_api_test_modules.reflection.g.dart +++ b/test/bones_api_test_modules.reflection.g.dart @@ -1,6 +1,6 @@ // // GENERATED CODE - DO NOT MODIFY BY HAND! -// BUILDER: reflection_factory/2.5.1 +// BUILDER: reflection_factory/2.5.2 // BUILD COMMAND: dart run build_runner build // @@ -22,7 +22,7 @@ typedef __TI = TypeInfo; typedef __PR = ParameterReflection; mixin __ReflectionMixin { - static final Version _version = Version.parse('2.5.1'); + static final Version _version = Version.parse('2.5.2'); Version get reflectionFactoryVersion => _version; diff --git a/test/bones_api_test_utils_test.reflection.g.dart b/test/bones_api_test_utils_test.reflection.g.dart index 24a6412..4aeec2d 100644 --- a/test/bones_api_test_utils_test.reflection.g.dart +++ b/test/bones_api_test_utils_test.reflection.g.dart @@ -1,6 +1,6 @@ // // GENERATED CODE - DO NOT MODIFY BY HAND! -// BUILDER: reflection_factory/2.5.1 +// BUILDER: reflection_factory/2.5.2 // BUILD COMMAND: dart run build_runner build // @@ -22,7 +22,7 @@ typedef __TI = TypeInfo; typedef __PR = ParameterReflection; mixin __ReflectionMixin { - static final Version _version = Version.parse('2.5.1'); + static final Version _version = Version.parse('2.5.2'); Version get reflectionFactoryVersion => _version; From 498875f385075ecb9f54027978f9ed7479755f87 Mon Sep 17 00:00:00 2001 From: gmpassos Date: Sun, 27 Jul 2025 05:56:12 -0300 Subject: [PATCH 12/13] bones_api_utils_sink_test.dart --- test/bones_api_utils_sink_test.dart | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/test/bones_api_utils_sink_test.dart b/test/bones_api_utils_sink_test.dart index 388464c..82d2036 100644 --- a/test/bones_api_utils_sink_test.dart +++ b/test/bones_api_utils_sink_test.dart @@ -1,3 +1,5 @@ +import 'dart:io'; + import 'package:bones_api/src/bones_api_utils_sink.dart'; import 'package:test/test.dart'; @@ -66,7 +68,7 @@ void main() { group('GZipSink', () { test('basic', () { - const gzipHeader = [31, 139, 8, 0, 0, 0, 0, 0, 0, 19]; + var gzipHeader = _getGZipHeader(); var bs = GZipSink(capacity: 8); @@ -164,7 +166,7 @@ void main() { group('AutoGZipSink', () { test('basic', () { - const gzipHeader = [31, 139, 8, 0, 0, 0, 0, 0, 0, 19]; + var gzipHeader = _getGZipHeader(); var bs = AutoGZipSink(minGZipLength: 10, capacity: 8); @@ -258,3 +260,9 @@ void main() { }); }); } + +List _getGZipHeader() { + //[31, 139, 8, 0, 0, 0, 0, 0, 0, 19] + var compressed = gzip.encode([1, 2, 3]); + return compressed.sublist(0, 10); +} From b6da8d6fe76fdabc14ab817ecc2a6149e6c54dc3 Mon Sep 17 00:00:00 2001 From: gmpassos Date: Sun, 27 Jul 2025 06:09:33 -0300 Subject: [PATCH 13/13] bones_api_utils_sink_test.dart --- test/bones_api_utils_sink_test.dart | 1 + 1 file changed, 1 insertion(+) diff --git a/test/bones_api_utils_sink_test.dart b/test/bones_api_utils_sink_test.dart index 82d2036..726ac96 100644 --- a/test/bones_api_utils_sink_test.dart +++ b/test/bones_api_utils_sink_test.dart @@ -1,3 +1,4 @@ +@TestOn('vm') import 'dart:io'; import 'package:bones_api/src/bones_api_utils_sink.dart';