From be3d6be87d5f93b5db18ee241fe182dac436dde2 Mon Sep 17 00:00:00 2001 From: Takuma Homma Date: Tue, 3 Mar 2026 00:50:08 +0900 Subject: [PATCH 1/8] Support diffThreshold to handle comparison failures that depend on the image generation environment --- README.md | 1 + lib/alchemist.dart | 1 + lib/src/alchemist_config.dart | 28 +++- lib/src/alchemist_file_comparator.dart | 73 +++++++++ lib/src/golden_test.dart | 1 + lib/src/golden_test_runner.dart | 24 +++ test/src/alchemist_config_test.dart | 116 ++++++++++++++ test/src/alchemist_file_comparator_test.dart | 155 +++++++++++++++++++ test/src/golden_test_runner_test.dart | 118 ++++++++++++++ 9 files changed, 515 insertions(+), 2 deletions(-) create mode 100644 lib/src/alchemist_file_comparator.dart create mode 100644 test/src/alchemist_file_comparator_test.dart diff --git a/README.md b/README.md index 6d62c61..d546a46 100644 --- a/README.md +++ b/README.md @@ -220,6 +220,7 @@ Both the `PlatformGoldensConfig` and `CiGoldensConfig` classes contain a number | `bool enabled` | `true` | Indicates if this type of test should run. If set to `false`, this type of test is never allowed to run. Defaults to `true`. | | `bool obscureText` | `true` for CI, `false` for platform | Indicates if the text in the rendered widget should be obscured by colored rectangles. This is useful for circumventing issues with Flutter's font rendering between host platforms. | | `bool renderShadows` | `false` for CI, `true` for platform | Indicates if shadows should actually be rendered, or if they should be replaced by opaque colors. This is useful because shadow rendering can be inconsistent between test runs. | +| `double diffThreshold` | `0.0` | The maximum allowed pixel difference ratio (between `0.0` inclusive and `1.0` exclusive) before a golden test fails. The value is a fraction of differing pixels — for example, `0.001` means 0.1% and `0.01` means 1%. When the diff is greater than `0` but within this threshold, the test passes and a warning is printed. This is useful for handling minor cross-platform or cross-architecture rendering differences. Defaults to `0.0` (no tolerance). | | `FilePathResolver filePathResolver` | `<_defaultFilePathResolver>` | A function that resolves the path to the golden file, relative to the test that generates it. By default, CI golden test files are placed in `goldens/ci/`, and readable golden test files are placed in `goldens/`. | | `ThemeData? theme` | `null` | The theme to use for this type of test. If `null`, the enclosing `AlchemistConfig`'s `theme` will be used, or `ThemeData.light()` if that is also `null`. _Note that CI tests are always run using the Ahem font family, which is a font that solely renders square characters. This is done to ensure that CI tests are always consistent across platforms._ | diff --git a/lib/alchemist.dart b/lib/alchemist.dart index c9b6b6d..3d16282 100644 --- a/lib/alchemist.dart +++ b/lib/alchemist.dart @@ -1,4 +1,5 @@ export 'src/alchemist_config.dart'; +export 'src/alchemist_file_comparator.dart'; export 'src/blocked_text_image.dart'; export 'src/golden_test.dart'; export 'src/golden_test_group.dart'; diff --git a/lib/src/alchemist_config.dart b/lib/src/alchemist_config.dart index b2c6a03..1f22611 100644 --- a/lib/src/alchemist_config.dart +++ b/lib/src/alchemist_config.dart @@ -289,8 +289,14 @@ abstract class GoldensConfig extends Equatable { required this.renderShadows, FilePathResolver? filePathResolver, ThemeData? theme, - }) : _filePathResolver = filePathResolver, - _theme = theme; + double? diffThreshold, + }) : assert( + diffThreshold == null || (diffThreshold >= 0.0 && diffThreshold < 1.0), + 'diffThreshold must be between 0.0 (inclusive) and 1.0 (exclusive)', + ), + _filePathResolver = filePathResolver, + _theme = theme, + _diffThreshold = diffThreshold; /// Whether or not the golden tests should run. final bool enabled; @@ -345,6 +351,14 @@ abstract class GoldensConfig extends Equatable { ThemeData? get theme => _theme; final ThemeData? _theme; + /// The maximum fraction of differing pixels that is still considered a + /// passing test. Defaults to 0.0 (no threshold). + /// + /// A value of 0.001 means up to 0.1% of pixels may differ. When a diff is + /// within the threshold but greater than 0, a warning is printed. + double get diffThreshold => _diffThreshold ?? 0.0; + final double? _diffThreshold; + /// Creates a copy of this [GoldensConfig] and replaces the given fields. GoldensConfig copyWith({ bool? enabled, @@ -352,6 +366,7 @@ abstract class GoldensConfig extends Equatable { bool? renderShadows, FilePathResolver? filePathResolver, ThemeData? theme, + double? diffThreshold, }); /// Creates a copy and merges this [GoldensConfig] with the given config, @@ -365,6 +380,7 @@ abstract class GoldensConfig extends Equatable { filePathResolver, theme, renderShadows, + diffThreshold, ]; } @@ -396,6 +412,7 @@ class PlatformGoldensConfig extends GoldensConfig { super.renderShadows = true, super.filePathResolver, super.theme, + super.diffThreshold, }) : _platforms = platforms; @override @@ -427,6 +444,7 @@ class PlatformGoldensConfig extends GoldensConfig { bool? renderShadows, FilePathResolver? filePathResolver, ThemeData? theme, + double? diffThreshold, }) { return PlatformGoldensConfig( platforms: platforms ?? this.platforms, @@ -435,6 +453,7 @@ class PlatformGoldensConfig extends GoldensConfig { renderShadows: renderShadows ?? this.renderShadows, filePathResolver: filePathResolver ?? this.filePathResolver, theme: theme ?? this.theme, + diffThreshold: diffThreshold ?? _diffThreshold, ); } @@ -447,6 +466,7 @@ class PlatformGoldensConfig extends GoldensConfig { renderShadows: other?.renderShadows, filePathResolver: other?._filePathResolver, theme: other?._theme, + diffThreshold: other?._diffThreshold, ); } @@ -480,6 +500,7 @@ class CiGoldensConfig extends GoldensConfig { super.renderShadows = false, super.filePathResolver, super.theme, + super.diffThreshold, }); @override @@ -492,6 +513,7 @@ class CiGoldensConfig extends GoldensConfig { bool? renderShadows, FilePathResolver? filePathResolver, ThemeData? theme, + double? diffThreshold, }) { return CiGoldensConfig( enabled: enabled ?? this.enabled, @@ -499,6 +521,7 @@ class CiGoldensConfig extends GoldensConfig { renderShadows: renderShadows ?? this.renderShadows, filePathResolver: filePathResolver ?? this.filePathResolver, theme: theme ?? this.theme, + diffThreshold: diffThreshold ?? _diffThreshold, ); } @@ -510,6 +533,7 @@ class CiGoldensConfig extends GoldensConfig { renderShadows: other?.renderShadows, filePathResolver: other?._filePathResolver, theme: other?._theme, + diffThreshold: other?._diffThreshold, ); } } diff --git a/lib/src/alchemist_file_comparator.dart b/lib/src/alchemist_file_comparator.dart new file mode 100644 index 0000000..d63f9d1 --- /dev/null +++ b/lib/src/alchemist_file_comparator.dart @@ -0,0 +1,73 @@ +import 'package:flutter/foundation.dart'; +import 'package:flutter_test/flutter_test.dart'; + +/// A [LocalFileComparator] that passes golden tests within a configurable +/// pixel-diff diffThreshold. +/// +/// When [diffThreshold] is greater than 0 and the diff percentage is within +/// diffThreshold, the test passes and a warning is printed. This is useful for +/// handling minor cross-platform or cross-architecture rendering differences. +class AlchemistFileComparator extends LocalFileComparator { + /// Creates an [AlchemistFileComparator] with the given [testUri] and + /// [diffThreshold]. + /// + /// The [diffThreshold] must be between 0.0 (inclusive) and 1.0 (exclusive). + AlchemistFileComparator(super.testUri, this.diffThreshold) + : assert( + diffThreshold >= 0.0 && diffThreshold < 1.0, + 'diffThreshold must be between 0.0 (inclusive) and 1.0 (exclusive)', + ); + + /// Creates an [AlchemistFileComparator] from an [existing] + /// [LocalFileComparator], preserving its base directory. + factory AlchemistFileComparator.fromExisting( + LocalFileComparator existing, + double diffThreshold, + ) { + return AlchemistFileComparator(existing.basedir, diffThreshold); + } + + /// The maximum fraction of differing pixels that is still considered a + /// passing test. + /// + /// Must be between 0.0 (inclusive) and 1.0 (exclusive). When the diff + /// percentage exceeds 0 but is within this threshold, the test passes and a + /// warning is printed. A value of 0.0 means no threshold is applied. + final double diffThreshold; + + /// Compares the given [imageBytes] to the [goldenBytes] and returns the + /// [ComparisonResult]. + /// + /// Exposed for testing. In production, [compare] calls this method + /// internally. + @protected + @visibleForTesting + Future compareImageBytes( + Uint8List imageBytes, + Uint8List goldenBytes, + ) { + return GoldenFileComparator.compareLists(imageBytes, goldenBytes); + } + + /// Compares [imageBytes] to the golden file identified by [golden]. + /// + /// Returns `true` if the images match, or if the diff percentage is within + /// [diffThreshold]. Returns `false` and generates failure output otherwise. + @override + Future compare(Uint8List imageBytes, Uri golden) async { + final goldenBytes = await getGoldenBytes(golden); + final result = await compareImageBytes( + imageBytes, + Uint8List.fromList(goldenBytes), + ); + + if (result.passed) return true; + + if (diffThreshold > 0 && result.diffPercent <= diffThreshold) { + return true; + } + + await generateFailureOutput(result, golden, basedir); + return false; + } +} diff --git a/lib/src/golden_test.dart b/lib/src/golden_test.dart index acdacfc..76878e2 100644 --- a/lib/src/golden_test.dart +++ b/lib/src/golden_test.dart @@ -192,6 +192,7 @@ Future goldenTest( pumpBeforeTest: pumpBeforeTest, pumpWidget: pumpWidget, whilePerforming: whilePerforming, + diffThreshold: variantConfig.diffThreshold, ); }, tags: tags, diff --git a/lib/src/golden_test_runner.dart b/lib/src/golden_test_runner.dart index f13ebde..16abfc3 100644 --- a/lib/src/golden_test_runner.dart +++ b/lib/src/golden_test_runner.dart @@ -1,5 +1,6 @@ import 'dart:ui' as ui; +import 'package:alchemist/src/alchemist_file_comparator.dart'; import 'package:alchemist/src/golden_test_adapter.dart'; import 'package:alchemist/src/golden_test_theme.dart'; import 'package:alchemist/src/interactions.dart'; @@ -42,6 +43,7 @@ abstract class GoldenTestRunner { PumpAction pumpBeforeTest = onlyPumpAndSettle, PumpWidget pumpWidget = onlyPumpWidget, Interaction? whilePerforming, + double diffThreshold = 0.0, }); } @@ -69,6 +71,7 @@ class FlutterGoldenTestRunner extends GoldenTestRunner { PumpAction pumpBeforeTest = onlyPumpAndSettle, PumpWidget pumpWidget = onlyPumpWidget, Interaction? whilePerforming, + double diffThreshold = 0.0, }) async { assert( goldenPath is String || goldenPath is Uri, @@ -80,6 +83,24 @@ class FlutterGoldenTestRunner extends GoldenTestRunner { final mementoDebugDisableShadows = debugDisableShadows; debugDisableShadows = !renderShadows; + GoldenFileComparator? originalComparator; + if (diffThreshold > 0) { + final comparator = goldenFileComparator; + if (comparator is LocalFileComparator) { + originalComparator = comparator; + goldenFileComparator = AlchemistFileComparator.fromExisting( + comparator, + diffThreshold, + ); + } else { + throw UnsupportedError( + 'diffThreshold is set to $diffThreshold but the current ' + 'GoldenFileComparator (${comparator.runtimeType}) is not a ' + 'LocalFileComparator. diffThreshold is not supported.', + ); + } + } + Future? imageFuture; try { await goldenTestAdapter.pumpGoldenTest( @@ -125,6 +146,9 @@ class FlutterGoldenTestRunner extends GoldenTestRunner { rethrow; } } finally { + if (originalComparator != null) { + goldenFileComparator = originalComparator; + } debugDisableShadows = mementoDebugDisableShadows; final image = await imageFuture; image?.dispose(); diff --git a/test/src/alchemist_config_test.dart b/test/src/alchemist_config_test.dart index 117cc7c..93f9e56 100644 --- a/test/src/alchemist_config_test.dart +++ b/test/src/alchemist_config_test.dart @@ -241,6 +241,10 @@ void main() { expect(defaultValue.renderShadows, isTrue); }); + test('for diffThreshold', () { + expect(defaultValue.diffThreshold, 0.0); + }); + group('for default filePathResolver', () { test('generates path correctly', () { expect( @@ -251,6 +255,36 @@ void main() { }); }); + group('diffThreshold', () { + test('asserts when negative', () { + expect( + () => PlatformGoldensConfig(diffThreshold: -0.1), + throwsA(isA()), + ); + }); + + test('asserts when exceeds 1.0', () { + expect( + () => PlatformGoldensConfig(diffThreshold: 1.1), + throwsA(isA()), + ); + }); + + test('asserts when diffThreshold is 1.0', () { + expect( + () => PlatformGoldensConfig(diffThreshold: 1.0), + throwsA(isA()), + ); + }); + + test('accepts 0.0 as boundary value', () { + expect( + () => PlatformGoldensConfig(diffThreshold: 0.0), + returnsNormally, + ); + }); + }); + group('copyWith', () { test('does nothing if no arguments are provided', () { expect( @@ -280,6 +314,17 @@ void main() { ), ); }); + + test('copies diffThreshold', () { + expect( + const PlatformGoldensConfig().copyWith(diffThreshold: 0.001), + isA().having( + (c) => c.diffThreshold, + 'diffThreshold', + 0.001, + ), + ); + }); }); group('merge', () { @@ -311,6 +356,19 @@ void main() { ), ); }); + + test('propagates diffThreshold', () { + expect( + const PlatformGoldensConfig().merge( + const PlatformGoldensConfig(diffThreshold: 0.005), + ), + isA().having( + (c) => c.diffThreshold, + 'diffThreshold', + 0.005, + ), + ); + }); }); }); @@ -331,6 +389,10 @@ void main() { expect(defaultValue.renderShadows, isFalse); }); + test('for diffThreshold', () { + expect(defaultValue.diffThreshold, 0.0); + }); + test('for filePathResolver', () { expect( defaultValue.filePathResolver('foo', 'bar'), @@ -343,6 +405,36 @@ void main() { }); }); + group('diffThreshold', () { + test('asserts when negative', () { + expect( + () => CiGoldensConfig(diffThreshold: -0.1), + throwsA(isA()), + ); + }); + + test('asserts when exceeds 1.0', () { + expect( + () => CiGoldensConfig(diffThreshold: 1.1), + throwsA(isA()), + ); + }); + + test('asserts when diffThreshold is 1.0', () { + expect( + () => CiGoldensConfig(diffThreshold: 1.0), + throwsA(isA()), + ); + }); + + test('accepts 0.0 as boundary value', () { + expect( + () => CiGoldensConfig(diffThreshold: 0.0), + returnsNormally, + ); + }); + }); + group('copyWith', () { test('does nothing if no arguments are provided', () { expect( @@ -368,6 +460,17 @@ void main() { isA().having((c) => c.enabled, 'enabled', enabled), ); }); + + test('copies diffThreshold', () { + expect( + const CiGoldensConfig().copyWith(diffThreshold: 0.002), + isA().having( + (c) => c.diffThreshold, + 'diffThreshold', + 0.002, + ), + ); + }); }); group('merge', () { @@ -399,6 +502,19 @@ void main() { ), ); }); + + test('propagates diffThreshold', () { + expect( + const CiGoldensConfig().merge( + const CiGoldensConfig(diffThreshold: 0.003), + ), + isA().having( + (c) => c.diffThreshold, + 'diffThreshold', + 0.003, + ), + ); + }); }); }); } diff --git a/test/src/alchemist_file_comparator_test.dart b/test/src/alchemist_file_comparator_test.dart new file mode 100644 index 0000000..aba2fa5 --- /dev/null +++ b/test/src/alchemist_file_comparator_test.dart @@ -0,0 +1,155 @@ +import 'dart:typed_data'; + +import 'package:alchemist/src/alchemist_file_comparator.dart'; +import 'package:flutter_test/flutter_test.dart'; + +class _TestAlchemistFileComparator extends AlchemistFileComparator { + _TestAlchemistFileComparator({ + required double diffThreshold, + required ComparisonResult result, + }) : _result = result, + super(Uri.parse('file:///test/_alchemist.dart'), diffThreshold); + + final ComparisonResult _result; + + @override + Future compareImageBytes( + Uint8List imageBytes, + Uint8List goldenBytes, + ) async { + return _result; + } + + @override + Future getGoldenBytes(Uri golden) async { + return Uint8List(0); + } + + @override + Future generateFailureOutput( + ComparisonResult result, + Uri golden, + Uri basedir, { + String key = '', + }) async { + return ''; + } +} + +void main() { + group('AlchemistFileComparator', () { + group('constructor', () { + test('asserts when diffThreshold is negative', () { + expect( + () => AlchemistFileComparator( + Uri.parse('file:///test/_alchemist.dart'), + -0.1, + ), + throwsAssertionError, + ); + }); + + test('asserts when diffThreshold exceeds 1.0', () { + expect( + () => AlchemistFileComparator( + Uri.parse('file:///test/_alchemist.dart'), + 1.1, + ), + throwsAssertionError, + ); + }); + + test('asserts when diffThreshold is 1.0', () { + expect( + () => AlchemistFileComparator( + Uri.parse('file:///test/_alchemist.dart'), + 1.0, + ), + throwsAssertionError, + ); + }); + + test('accepts 0.0 as boundary value', () { + expect( + () => AlchemistFileComparator( + Uri.parse('file:///test/_alchemist.dart'), + 0.0, + ), + returnsNormally, + ); + }); + }); + + group('fromExisting', () { + test('sets basedir and diffThreshold from existing comparator', () { + final existing = LocalFileComparator( + Uri.parse('file:///some/path/test.dart'), + ); + final comparator = AlchemistFileComparator.fromExisting( + existing, + 0.001, + ); + expect(comparator.basedir, existing.basedir); + expect(comparator.diffThreshold, 0.001); + }); + }); + + group('compare', () { + test('passes when underlying comparison passes', () async { + final comparator = _TestAlchemistFileComparator( + diffThreshold: 0.0, + result: ComparisonResult(passed: true, diffPercent: 0.0), + ); + + final result = await comparator.compare( + Uint8List(0), + Uri.parse('golden.png'), + ); + + expect(result, isTrue); + }); + + test('passes when diff is within diffThreshold', () async { + final comparator = _TestAlchemistFileComparator( + diffThreshold: 0.01, + result: ComparisonResult(passed: false, diffPercent: 0.005), + ); + + final result = await comparator.compare( + Uint8List(0), + Uri.parse('golden.png'), + ); + + expect(result, isTrue); + }); + + test('fails when diff exceeds diffThreshold', () async { + final comparator = _TestAlchemistFileComparator( + diffThreshold: 0.001, + result: ComparisonResult(passed: false, diffPercent: 0.005), + ); + + final result = await comparator.compare( + Uint8List(0), + Uri.parse('golden.png'), + ); + + expect(result, isFalse); + }); + + test('fails when diffThreshold is 0 and diff > 0', () async { + final comparator = _TestAlchemistFileComparator( + diffThreshold: 0.0, + result: ComparisonResult(passed: false, diffPercent: 0.001), + ); + + final result = await comparator.compare( + Uint8List(0), + Uri.parse('golden.png'), + ); + + expect(result, isFalse); + }); + }); + }); +} diff --git a/test/src/golden_test_runner_test.dart b/test/src/golden_test_runner_test.dart index 1791c32..7ca3dfc 100644 --- a/test/src/golden_test_runner_test.dart +++ b/test/src/golden_test_runner_test.dart @@ -1,6 +1,7 @@ import 'dart:async'; import 'dart:ui' as ui; +import 'package:alchemist/src/alchemist_file_comparator.dart'; import 'package:alchemist/src/golden_test_adapter.dart'; import 'package:alchemist/src/golden_test_runner.dart'; import 'package:flutter/material.dart'; @@ -13,6 +14,8 @@ class MockUiImage extends Mock implements ui.Image {} class MockWidgetTester extends Mock implements WidgetTester {} +class _FakeGoldenFileComparator extends Fake implements GoldenFileComparator {} + void main() { setUpAll(() { registerFallbackValue(MockWidgetTester()); @@ -152,6 +155,121 @@ void main() { expect(debugDisableShadowsDuringTestRun, isFalse); }); + testWidgets( + 'installs AlchemistFileComparator when diffThreshold > 0 and comparator ' + 'is LocalFileComparator', + (tester) async { + final originalComparator = LocalFileComparator( + Uri.parse('file:///test/golden_test.dart'), + ); + goldenFileComparator = originalComparator; + + GoldenFileComparator? comparatorDuringTest; + when( + () => goldenTestAdapter.withForceUpdateGoldenFiles( + callback: any(named: 'callback'), + ), + ).thenAnswer((invocation) async { + comparatorDuringTest = goldenFileComparator; + await (invocation.namedArguments[#callback] + as MatchesGoldenFileInvocation) + .call(); + }); + + await goldenTestRunner.run( + tester: tester, + goldenPath: 'path/to/golden', + widget: const SizedBox(), + diffThreshold: 0.001, + ); + + expect(comparatorDuringTest, isA()); + expect( + (comparatorDuringTest as AlchemistFileComparator).diffThreshold, + 0.001, + ); + expect(goldenFileComparator, same(originalComparator)); + }, + ); + + testWidgets( + 'restores original comparator after test throws', + (tester) async { + final originalComparator = LocalFileComparator( + Uri.parse('file:///test/golden_test.dart'), + ); + goldenFileComparator = originalComparator; + + final givenException = Exception('test error'); + when( + () => goldenTestAdapter.withForceUpdateGoldenFiles( + callback: any(named: 'callback'), + ), + ).thenAnswer((_) async => throw givenException); + + await expectLater( + goldenTestRunner.run( + tester: tester, + goldenPath: 'path/to/golden', + widget: const SizedBox(), + diffThreshold: 0.001, + ), + throwsA(same(givenException)), + ); + + expect(goldenFileComparator, same(originalComparator)); + }, + ); + + testWidgets( + 'does not change comparator when diffThreshold is 0', + (tester) async { + final originalComparator = LocalFileComparator( + Uri.parse('file:///test/golden_test.dart'), + ); + goldenFileComparator = originalComparator; + + GoldenFileComparator? comparatorDuringTest; + when( + () => goldenTestAdapter.withForceUpdateGoldenFiles( + callback: any(named: 'callback'), + ), + ).thenAnswer((invocation) async { + comparatorDuringTest = goldenFileComparator; + await (invocation.namedArguments[#callback] + as MatchesGoldenFileInvocation) + .call(); + }); + + await goldenTestRunner.run( + tester: tester, + goldenPath: 'path/to/golden', + widget: const SizedBox(), + diffThreshold: 0.0, + ); + + expect(comparatorDuringTest, same(originalComparator)); + }, + ); + + testWidgets( + 'throws AssertionError when diffThreshold > 0 and comparator is not ' + 'LocalFileComparator', + (tester) async { + goldenFileComparator = _FakeGoldenFileComparator(); + + await expectLater( + goldenTestRunner.run( + tester: tester, + goldenPath: 'path/to/golden', + widget: const SizedBox(), + diffThreshold: 0.001, + ), + throwsAssertionError, + ); + }, + ); + testWidgets('resets window size after the test has run', (tester) async { late final Size sizeDuringTestRun; final originalSize = tester.view.physicalSize; From f8c10799766e54242f3f4f47c59e8b2749e1c51d Mon Sep 17 00:00:00 2001 From: Takuma Homma Date: Thu, 5 Mar 2026 23:58:52 +0900 Subject: [PATCH 2/8] Run dart format --- test/src/alchemist_config_test.dart | 5 +- test/src/golden_test_runner_test.dart | 102 +++++++++++++------------- 2 files changed, 51 insertions(+), 56 deletions(-) diff --git a/test/src/alchemist_config_test.dart b/test/src/alchemist_config_test.dart index 93f9e56..a8384f2 100644 --- a/test/src/alchemist_config_test.dart +++ b/test/src/alchemist_config_test.dart @@ -428,10 +428,7 @@ void main() { }); test('accepts 0.0 as boundary value', () { - expect( - () => CiGoldensConfig(diffThreshold: 0.0), - returnsNormally, - ); + expect(() => CiGoldensConfig(diffThreshold: 0.0), returnsNormally); }); }); diff --git a/test/src/golden_test_runner_test.dart b/test/src/golden_test_runner_test.dart index 7ca3dfc..54735ba 100644 --- a/test/src/golden_test_runner_test.dart +++ b/test/src/golden_test_runner_test.dart @@ -192,65 +192,63 @@ void main() { }, ); - testWidgets( - 'restores original comparator after test throws', - (tester) async { - final originalComparator = LocalFileComparator( - Uri.parse('file:///test/golden_test.dart'), - ); - goldenFileComparator = originalComparator; + testWidgets('restores original comparator after test throws', ( + tester, + ) async { + final originalComparator = LocalFileComparator( + Uri.parse('file:///test/golden_test.dart'), + ); + goldenFileComparator = originalComparator; - final givenException = Exception('test error'); - when( - () => goldenTestAdapter.withForceUpdateGoldenFiles( - callback: any(named: 'callback'), - ), - ).thenAnswer((_) async => throw givenException); + final givenException = Exception('test error'); + when( + () => goldenTestAdapter.withForceUpdateGoldenFiles( + callback: any(named: 'callback'), + ), + ).thenAnswer((_) async => throw givenException); - await expectLater( - goldenTestRunner.run( - tester: tester, - goldenPath: 'path/to/golden', - widget: const SizedBox(), - diffThreshold: 0.001, - ), - throwsA(same(givenException)), - ); + await expectLater( + goldenTestRunner.run( + tester: tester, + goldenPath: 'path/to/golden', + widget: const SizedBox(), + diffThreshold: 0.001, + ), + throwsA(same(givenException)), + ); - expect(goldenFileComparator, same(originalComparator)); - }, - ); + expect(goldenFileComparator, same(originalComparator)); + }); - testWidgets( - 'does not change comparator when diffThreshold is 0', - (tester) async { - final originalComparator = LocalFileComparator( - Uri.parse('file:///test/golden_test.dart'), - ); - goldenFileComparator = originalComparator; + testWidgets('does not change comparator when diffThreshold is 0', ( + tester, + ) async { + final originalComparator = LocalFileComparator( + Uri.parse('file:///test/golden_test.dart'), + ); + goldenFileComparator = originalComparator; - GoldenFileComparator? comparatorDuringTest; - when( - () => goldenTestAdapter.withForceUpdateGoldenFiles( - callback: any(named: 'callback'), - ), - ).thenAnswer((invocation) async { - comparatorDuringTest = goldenFileComparator; - await (invocation.namedArguments[#callback] - as MatchesGoldenFileInvocation) - .call(); - }); + GoldenFileComparator? comparatorDuringTest; + when( + () => goldenTestAdapter.withForceUpdateGoldenFiles( + callback: any(named: 'callback'), + ), + ).thenAnswer((invocation) async { + comparatorDuringTest = goldenFileComparator; + await (invocation.namedArguments[#callback] + as MatchesGoldenFileInvocation) + .call(); + }); - await goldenTestRunner.run( - tester: tester, - goldenPath: 'path/to/golden', - widget: const SizedBox(), - diffThreshold: 0.0, - ); + await goldenTestRunner.run( + tester: tester, + goldenPath: 'path/to/golden', + widget: const SizedBox(), + diffThreshold: 0.0, + ); - expect(comparatorDuringTest, same(originalComparator)); - }, - ); + expect(comparatorDuringTest, same(originalComparator)); + }); testWidgets( 'throws AssertionError when diffThreshold > 0 and comparator is not ' From 8ec0c5c47a429f9d5973ef476545e2f7283418a0 Mon Sep 17 00:00:00 2001 From: Takuma Homma Date: Thu, 5 Mar 2026 23:59:02 +0900 Subject: [PATCH 3/8] Fix baseDir usage --- lib/src/alchemist_file_comparator.dart | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/src/alchemist_file_comparator.dart b/lib/src/alchemist_file_comparator.dart index d63f9d1..14c4f37 100644 --- a/lib/src/alchemist_file_comparator.dart +++ b/lib/src/alchemist_file_comparator.dart @@ -24,7 +24,10 @@ class AlchemistFileComparator extends LocalFileComparator { LocalFileComparator existing, double diffThreshold, ) { - return AlchemistFileComparator(existing.basedir, diffThreshold); + return AlchemistFileComparator( + existing.basedir.resolve('_alchemist.dart'), + diffThreshold, + ); } /// The maximum fraction of differing pixels that is still considered a From 23572277a0c7d529484dc5a531529d6f4564e7a2 Mon Sep 17 00:00:00 2001 From: Takuma Homma Date: Fri, 6 Mar 2026 01:12:14 +0900 Subject: [PATCH 4/8] fix: correct expected error type in golden test runner test --- test/src/golden_test_runner_test.dart | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/src/golden_test_runner_test.dart b/test/src/golden_test_runner_test.dart index 54735ba..d1e0d97 100644 --- a/test/src/golden_test_runner_test.dart +++ b/test/src/golden_test_runner_test.dart @@ -251,7 +251,7 @@ void main() { }); testWidgets( - 'throws AssertionError when diffThreshold > 0 and comparator is not ' + 'throws UnsupportedError when diffThreshold > 0 and comparator is not ' 'LocalFileComparator', (tester) async { goldenFileComparator = _FakeGoldenFileComparator(); @@ -263,7 +263,7 @@ void main() { widget: const SizedBox(), diffThreshold: 0.001, ), - throwsAssertionError, + throwsA(isA()), ); }, ); From 14547f6c43458d748c3be1f668c7fa2bc8652a2e Mon Sep 17 00:00:00 2001 From: Takuma Homma Date: Fri, 6 Mar 2026 01:33:49 +0900 Subject: [PATCH 5/8] Run dart analyzer issues --- test/src/alchemist_config_test.dart | 8 ++++---- test/src/alchemist_file_comparator_test.dart | 10 +++++----- test/src/golden_test_runner_test.dart | 3 +-- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/test/src/alchemist_config_test.dart b/test/src/alchemist_config_test.dart index a8384f2..ed5c853 100644 --- a/test/src/alchemist_config_test.dart +++ b/test/src/alchemist_config_test.dart @@ -272,14 +272,14 @@ void main() { test('asserts when diffThreshold is 1.0', () { expect( - () => PlatformGoldensConfig(diffThreshold: 1.0), + () => PlatformGoldensConfig(diffThreshold: 1), throwsA(isA()), ); }); test('accepts 0.0 as boundary value', () { expect( - () => PlatformGoldensConfig(diffThreshold: 0.0), + () => const PlatformGoldensConfig(diffThreshold: 0), returnsNormally, ); }); @@ -422,13 +422,13 @@ void main() { test('asserts when diffThreshold is 1.0', () { expect( - () => CiGoldensConfig(diffThreshold: 1.0), + () => CiGoldensConfig(diffThreshold: 1), throwsA(isA()), ); }); test('accepts 0.0 as boundary value', () { - expect(() => CiGoldensConfig(diffThreshold: 0.0), returnsNormally); + expect(() => const CiGoldensConfig(diffThreshold: 0), returnsNormally); }); }); diff --git a/test/src/alchemist_file_comparator_test.dart b/test/src/alchemist_file_comparator_test.dart index aba2fa5..09a4b3f 100644 --- a/test/src/alchemist_file_comparator_test.dart +++ b/test/src/alchemist_file_comparator_test.dart @@ -63,7 +63,7 @@ void main() { expect( () => AlchemistFileComparator( Uri.parse('file:///test/_alchemist.dart'), - 1.0, + 1, ), throwsAssertionError, ); @@ -73,7 +73,7 @@ void main() { expect( () => AlchemistFileComparator( Uri.parse('file:///test/_alchemist.dart'), - 0.0, + 0, ), returnsNormally, ); @@ -97,8 +97,8 @@ void main() { group('compare', () { test('passes when underlying comparison passes', () async { final comparator = _TestAlchemistFileComparator( - diffThreshold: 0.0, - result: ComparisonResult(passed: true, diffPercent: 0.0), + diffThreshold: 0, + result: ComparisonResult(passed: true, diffPercent: 0), ); final result = await comparator.compare( @@ -139,7 +139,7 @@ void main() { test('fails when diffThreshold is 0 and diff > 0', () async { final comparator = _TestAlchemistFileComparator( - diffThreshold: 0.0, + diffThreshold: 0, result: ComparisonResult(passed: false, diffPercent: 0.001), ); diff --git a/test/src/golden_test_runner_test.dart b/test/src/golden_test_runner_test.dart index d1e0d97..85f3846 100644 --- a/test/src/golden_test_runner_test.dart +++ b/test/src/golden_test_runner_test.dart @@ -185,7 +185,7 @@ void main() { expect(comparatorDuringTest, isA()); expect( - (comparatorDuringTest as AlchemistFileComparator).diffThreshold, + (comparatorDuringTest! as AlchemistFileComparator).diffThreshold, 0.001, ); expect(goldenFileComparator, same(originalComparator)); @@ -244,7 +244,6 @@ void main() { tester: tester, goldenPath: 'path/to/golden', widget: const SizedBox(), - diffThreshold: 0.0, ); expect(comparatorDuringTest, same(originalComparator)); From 6f65b0a681a790e8c69b5d1d94e0f634e07df849 Mon Sep 17 00:00:00 2001 From: Takuma Homma Date: Fri, 6 Mar 2026 01:57:08 +0900 Subject: [PATCH 6/8] Add missing alchemist_file_comparator method tests --- test/src/alchemist_file_comparator_test.dart | 22 ++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/test/src/alchemist_file_comparator_test.dart b/test/src/alchemist_file_comparator_test.dart index 09a4b3f..a3f4c10 100644 --- a/test/src/alchemist_file_comparator_test.dart +++ b/test/src/alchemist_file_comparator_test.dart @@ -1,4 +1,5 @@ import 'dart:typed_data'; +import 'dart:ui' as ui; import 'package:alchemist/src/alchemist_file_comparator.dart'; import 'package:flutter_test/flutter_test.dart'; @@ -94,6 +95,27 @@ void main() { }); }); + group('compareImageBytes', () { + test('returns passed result when images are identical', () async { + final comparator = AlchemistFileComparator( + Uri.parse('file:///test/_alchemist.dart'), + 0, + ); + + final recorder = ui.PictureRecorder(); + ui.Canvas( + recorder, + ).drawColor(const ui.Color(0xFFFFFFFF), ui.BlendMode.src); + final picture = recorder.endRecording(); + final image = await picture.toImage(1, 1); + final byteData = await image.toByteData(format: ui.ImageByteFormat.png); + final pngBytes = byteData!.buffer.asUint8List(); + + final result = await comparator.compareImageBytes(pngBytes, pngBytes); + expect(result.passed, isTrue); + }); + }); + group('compare', () { test('passes when underlying comparison passes', () async { final comparator = _TestAlchemistFileComparator( From 6d827bce0024149f4df8d2285c810372d25f0a67 Mon Sep 17 00:00:00 2001 From: Takuma Homma Date: Wed, 11 Mar 2026 23:11:16 +0900 Subject: [PATCH 7/8] fix: move comparator setup into try block and restrict to exact LocalFileComparator type --- lib/src/golden_test_runner.dart | 34 ++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/lib/src/golden_test_runner.dart b/lib/src/golden_test_runner.dart index 16abfc3..69d3caa 100644 --- a/lib/src/golden_test_runner.dart +++ b/lib/src/golden_test_runner.dart @@ -84,25 +84,25 @@ class FlutterGoldenTestRunner extends GoldenTestRunner { debugDisableShadows = !renderShadows; GoldenFileComparator? originalComparator; - if (diffThreshold > 0) { - final comparator = goldenFileComparator; - if (comparator is LocalFileComparator) { - originalComparator = comparator; - goldenFileComparator = AlchemistFileComparator.fromExisting( - comparator, - diffThreshold, - ); - } else { - throw UnsupportedError( - 'diffThreshold is set to $diffThreshold but the current ' - 'GoldenFileComparator (${comparator.runtimeType}) is not a ' - 'LocalFileComparator. diffThreshold is not supported.', - ); - } - } - Future? imageFuture; try { + if (diffThreshold > 0) { + final comparator = goldenFileComparator; + if (comparator is LocalFileComparator && + comparator.runtimeType == LocalFileComparator) { + originalComparator = comparator; + goldenFileComparator = AlchemistFileComparator.fromExisting( + comparator, + diffThreshold, + ); + } else { + throw UnsupportedError( + 'diffThreshold is set to $diffThreshold but the current ' + 'GoldenFileComparator (${comparator.runtimeType}) is not a ' + 'LocalFileComparator. diffThreshold is not supported.', + ); + } + } await goldenTestAdapter.pumpGoldenTest( tester: tester, rootKey: rootKey, From 9ad8504d90a849576d298fbb564368175933dcde Mon Sep 17 00:00:00 2001 From: Takuma Homma Date: Wed, 11 Mar 2026 23:12:45 +0900 Subject: [PATCH 8/8] docs: remove incorrect "warning is printed" from diffThreshold documentation The implementation does not print a warning when the diff passes within the threshold. Update the doc comments in AlchemistFileComparator, GoldensConfig, and README to reflect the actual behavior. --- README.md | 2 +- lib/src/alchemist_config.dart | 2 +- lib/src/alchemist_file_comparator.dart | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index d546a46..18e9665 100644 --- a/README.md +++ b/README.md @@ -220,7 +220,7 @@ Both the `PlatformGoldensConfig` and `CiGoldensConfig` classes contain a number | `bool enabled` | `true` | Indicates if this type of test should run. If set to `false`, this type of test is never allowed to run. Defaults to `true`. | | `bool obscureText` | `true` for CI, `false` for platform | Indicates if the text in the rendered widget should be obscured by colored rectangles. This is useful for circumventing issues with Flutter's font rendering between host platforms. | | `bool renderShadows` | `false` for CI, `true` for platform | Indicates if shadows should actually be rendered, or if they should be replaced by opaque colors. This is useful because shadow rendering can be inconsistent between test runs. | -| `double diffThreshold` | `0.0` | The maximum allowed pixel difference ratio (between `0.0` inclusive and `1.0` exclusive) before a golden test fails. The value is a fraction of differing pixels — for example, `0.001` means 0.1% and `0.01` means 1%. When the diff is greater than `0` but within this threshold, the test passes and a warning is printed. This is useful for handling minor cross-platform or cross-architecture rendering differences. Defaults to `0.0` (no tolerance). | +| `double diffThreshold` | `0.0` | The maximum allowed pixel difference ratio (between `0.0` inclusive and `1.0` exclusive) before a golden test fails. The value is a fraction of differing pixels — for example, `0.001` means 0.1% and `0.01` means 1%. When the diff is greater than `0` but within this threshold, the test passes. This is useful for handling minor cross-platform or cross-architecture rendering differences. Defaults to `0.0` (no tolerance). | | `FilePathResolver filePathResolver` | `<_defaultFilePathResolver>` | A function that resolves the path to the golden file, relative to the test that generates it. By default, CI golden test files are placed in `goldens/ci/`, and readable golden test files are placed in `goldens/`. | | `ThemeData? theme` | `null` | The theme to use for this type of test. If `null`, the enclosing `AlchemistConfig`'s `theme` will be used, or `ThemeData.light()` if that is also `null`. _Note that CI tests are always run using the Ahem font family, which is a font that solely renders square characters. This is done to ensure that CI tests are always consistent across platforms._ | diff --git a/lib/src/alchemist_config.dart b/lib/src/alchemist_config.dart index 1f22611..5b8a502 100644 --- a/lib/src/alchemist_config.dart +++ b/lib/src/alchemist_config.dart @@ -355,7 +355,7 @@ abstract class GoldensConfig extends Equatable { /// passing test. Defaults to 0.0 (no threshold). /// /// A value of 0.001 means up to 0.1% of pixels may differ. When a diff is - /// within the threshold but greater than 0, a warning is printed. + /// within the threshold but greater than 0, the test passes. double get diffThreshold => _diffThreshold ?? 0.0; final double? _diffThreshold; diff --git a/lib/src/alchemist_file_comparator.dart b/lib/src/alchemist_file_comparator.dart index 14c4f37..afcbc20 100644 --- a/lib/src/alchemist_file_comparator.dart +++ b/lib/src/alchemist_file_comparator.dart @@ -5,8 +5,8 @@ import 'package:flutter_test/flutter_test.dart'; /// pixel-diff diffThreshold. /// /// When [diffThreshold] is greater than 0 and the diff percentage is within -/// diffThreshold, the test passes and a warning is printed. This is useful for -/// handling minor cross-platform or cross-architecture rendering differences. +/// diffThreshold, the test passes. This is useful for handling minor +/// cross-platform or cross-architecture rendering differences. class AlchemistFileComparator extends LocalFileComparator { /// Creates an [AlchemistFileComparator] with the given [testUri] and /// [diffThreshold]. @@ -34,8 +34,8 @@ class AlchemistFileComparator extends LocalFileComparator { /// passing test. /// /// Must be between 0.0 (inclusive) and 1.0 (exclusive). When the diff - /// percentage exceeds 0 but is within this threshold, the test passes and a - /// warning is printed. A value of 0.0 means no threshold is applied. + /// percentage exceeds 0 but is within this threshold, the test passes. + /// A value of 0.0 means no threshold is applied. final double diffThreshold; /// Compares the given [imageBytes] to the [goldenBytes] and returns the