feat: Support diffThreshold to handle comparison failures that depend on the image generation environment

README.md

@@ -220,6 +220,7 @@ Both the `PlatformGoldensConfig` and `CiGoldensConfig` classes contain a number
 | `bool enabled`                      | `true`                              | Indicates if this type of test should run. If set to `false`, this type of test is never allowed to run. Defaults to `true`.                                                                                                                                                                                                                                  |
 | `bool obscureText`                  | `true` for CI, `false` for platform | Indicates if the text in the rendered widget should be obscured by colored rectangles. This is useful for circumventing issues with Flutter's font rendering between host platforms.                                                                                                                                                                          |
 | `bool renderShadows`                | `false` for CI, `true` for platform | Indicates if shadows should actually be rendered, or if they should be replaced by opaque colors. This is useful because shadow rendering can be inconsistent between test runs.                                                                                                                                                                              |
+| `double diffThreshold`              | `0.0`                               | The maximum allowed pixel difference ratio (between `0.0` inclusive and `1.0` exclusive) before a golden test fails. The value is a fraction of differing pixels — for example, `0.001` means 0.1% and `0.01` means 1%. When the diff is greater than `0` but within this threshold, the test passes. This is useful for handling minor cross-platform or cross-architecture rendering differences. Defaults to `0.0` (no tolerance).  |
 | `FilePathResolver filePathResolver` | `<_defaultFilePathResolver>`        | A function that resolves the path to the golden file, relative to the test that generates it. By default, CI golden test files are placed in `goldens/ci/`, and readable golden test files are placed in `goldens/`.                                                                                                                                          |
 | `ThemeData? theme`                  | `null`                              | The theme to use for this type of test. If `null`, the enclosing `AlchemistConfig`'s `theme` will be used, or `ThemeData.light()` if that is also `null`. _Note that CI tests are always run using the Ahem font family, which is a font that solely renders square characters. This is done to ensure that CI tests are always consistent across platforms._ |
 

lib/alchemist.dart

@@ -1,4 +1,5 @@
 export 'src/alchemist_config.dart';
+export 'src/alchemist_file_comparator.dart';
 export 'src/blocked_text_image.dart';
 export 'src/golden_test.dart';
 export 'src/golden_test_group.dart';

lib/src/alchemist_config.dart

@@ -289,8 +289,14 @@ abstract class GoldensConfig extends Equatable {
     required this.renderShadows,
     FilePathResolver? filePathResolver,
     ThemeData? theme,
-  }) : _filePathResolver = filePathResolver,
-       _theme = theme;
+    double? diffThreshold,
+  }) : assert(
+         diffThreshold == null || (diffThreshold >= 0.0 && diffThreshold < 1.0),
+         'diffThreshold must be between 0.0 (inclusive) and 1.0 (exclusive)',
+       ),
+       _filePathResolver = filePathResolver,
+       _theme = theme,
+       _diffThreshold = diffThreshold;
 
   /// Whether or not the golden tests should run.
   final bool enabled;
@@ -345,13 +351,22 @@ abstract class GoldensConfig extends Equatable {
   ThemeData? get theme => _theme;
   final ThemeData? _theme;
 
+  /// The maximum fraction of differing pixels that is still considered a
+  /// passing test. Defaults to 0.0 (no threshold).
+  ///
+  /// A value of 0.001 means up to 0.1% of pixels may differ. When a diff is
+  /// within the threshold but greater than 0, the test passes.
+  double get diffThreshold => _diffThreshold ?? 0.0;
+  final double? _diffThreshold;
+
   /// Creates a copy of this [GoldensConfig] and replaces the given fields.
   GoldensConfig copyWith({
     bool? enabled,
     bool? obscureText,
     bool? renderShadows,
     FilePathResolver? filePathResolver,
     ThemeData? theme,
+    double? diffThreshold,
   });
 
   /// Creates a copy and merges this [GoldensConfig] with the given config,
@@ -365,6 +380,7 @@ abstract class GoldensConfig extends Equatable {
     filePathResolver,
     theme,
     renderShadows,
+    diffThreshold,
   ];
 }
 
@@ -396,6 +412,7 @@ class PlatformGoldensConfig extends GoldensConfig {
     super.renderShadows = true,
     super.filePathResolver,
     super.theme,
+    super.diffThreshold,
   }) : _platforms = platforms;
 
   @override
@@ -427,6 +444,7 @@ class PlatformGoldensConfig extends GoldensConfig {
     bool? renderShadows,
     FilePathResolver? filePathResolver,
     ThemeData? theme,
+    double? diffThreshold,
   }) {
     return PlatformGoldensConfig(
       platforms: platforms ?? this.platforms,
@@ -435,6 +453,7 @@ class PlatformGoldensConfig extends GoldensConfig {
       renderShadows: renderShadows ?? this.renderShadows,
       filePathResolver: filePathResolver ?? this.filePathResolver,
       theme: theme ?? this.theme,
+      diffThreshold: diffThreshold ?? _diffThreshold,
     );
   }
 
@@ -447,6 +466,7 @@ class PlatformGoldensConfig extends GoldensConfig {
       renderShadows: other?.renderShadows,
       filePathResolver: other?._filePathResolver,
       theme: other?._theme,
+      diffThreshold: other?._diffThreshold,
     );
   }
 
@@ -480,6 +500,7 @@ class CiGoldensConfig extends GoldensConfig {
     super.renderShadows = false,
     super.filePathResolver,
     super.theme,
+    super.diffThreshold,
   });
 
   @override
@@ -492,13 +513,15 @@ class CiGoldensConfig extends GoldensConfig {
     bool? renderShadows,
     FilePathResolver? filePathResolver,
     ThemeData? theme,
+    double? diffThreshold,
   }) {
     return CiGoldensConfig(
       enabled: enabled ?? this.enabled,
       obscureText: obscureText ?? this.obscureText,
       renderShadows: renderShadows ?? this.renderShadows,
       filePathResolver: filePathResolver ?? this.filePathResolver,
       theme: theme ?? this.theme,
+      diffThreshold: diffThreshold ?? _diffThreshold,
     );
   }
 
@@ -510,6 +533,7 @@ class CiGoldensConfig extends GoldensConfig {
       renderShadows: other?.renderShadows,
       filePathResolver: other?._filePathResolver,
       theme: other?._theme,
+      diffThreshold: other?._diffThreshold,
     );
   }
 }

lib/src/alchemist_file_comparator.dart

@@ -0,0 +1,76 @@
+import 'package:flutter/foundation.dart';
+import 'package:flutter_test/flutter_test.dart';
+
+/// A [LocalFileComparator] that passes golden tests within a configurable
+/// pixel-diff diffThreshold.
+///
+/// When [diffThreshold] is greater than 0 and the diff percentage is within
+/// diffThreshold, the test passes. This is useful for handling minor
+/// cross-platform or cross-architecture rendering differences.
+class AlchemistFileComparator extends LocalFileComparator {
+  /// Creates an [AlchemistFileComparator] with the given [testUri] and
+  /// [diffThreshold].
+  ///
+  /// The [diffThreshold] must be between 0.0 (inclusive) and 1.0 (exclusive).
+  AlchemistFileComparator(super.testUri, this.diffThreshold)
+    : assert(
+        diffThreshold >= 0.0 && diffThreshold < 1.0,
+        'diffThreshold must be between 0.0 (inclusive) and 1.0 (exclusive)',
+      );
+
+  /// Creates an [AlchemistFileComparator] from an [existing]
+  /// [LocalFileComparator], preserving its base directory.
+  factory AlchemistFileComparator.fromExisting(
+    LocalFileComparator existing,
+    double diffThreshold,
+  ) {
+    return AlchemistFileComparator(
+      existing.basedir.resolve('_alchemist.dart'),
+      diffThreshold,
+    );
+  }
+
+  /// The maximum fraction of differing pixels that is still considered a
+  /// passing test.
+  ///
+  /// Must be between 0.0 (inclusive) and 1.0 (exclusive). When the diff
+  /// percentage exceeds 0 but is within this threshold, the test passes.
+  /// A value of 0.0 means no threshold is applied.
+  final double diffThreshold;
+
+  /// Compares the given [imageBytes] to the [goldenBytes] and returns the
+  /// [ComparisonResult].
+  ///
+  /// Exposed for testing. In production, [compare] calls this method
+  /// internally.
+  @protected
+  @visibleForTesting
+  Future<ComparisonResult> compareImageBytes(
+    Uint8List imageBytes,
+    Uint8List goldenBytes,
+  ) {
+    return GoldenFileComparator.compareLists(imageBytes, goldenBytes);
+  }
+
+  /// Compares [imageBytes] to the golden file identified by [golden].
+  ///
+  /// Returns `true` if the images match, or if the diff percentage is within
+  /// [diffThreshold]. Returns `false` and generates failure output otherwise.
+  @override
+  Future<bool> compare(Uint8List imageBytes, Uri golden) async {
+    final goldenBytes = await getGoldenBytes(golden);
+    final result = await compareImageBytes(
+      imageBytes,
+      Uint8List.fromList(goldenBytes),
+    );
+
+    if (result.passed) return true;
+
+    if (diffThreshold > 0 && result.diffPercent <= diffThreshold) {
+      return true;
+    }
+
+    await generateFailureOutput(result, golden, basedir);
+    return false;
+  }
+}

lib/src/golden_test.dart

@@ -192,6 +192,7 @@ Future<void> goldenTest(
         pumpBeforeTest: pumpBeforeTest,
         pumpWidget: pumpWidget,
         whilePerforming: whilePerforming,
+        diffThreshold: variantConfig.diffThreshold,
       );
     },
     tags: tags,

lib/src/golden_test_runner.dart

@@ -1,5 +1,6 @@
 import 'dart:ui' as ui;
 
+import 'package:alchemist/src/alchemist_file_comparator.dart';
 import 'package:alchemist/src/golden_test_adapter.dart';
 import 'package:alchemist/src/golden_test_theme.dart';
 import 'package:alchemist/src/interactions.dart';
@@ -42,6 +43,7 @@ abstract class GoldenTestRunner {
     PumpAction pumpBeforeTest = onlyPumpAndSettle,
     PumpWidget pumpWidget = onlyPumpWidget,
     Interaction? whilePerforming,
+    double diffThreshold = 0.0,
   });
 }
 
@@ -69,6 +71,7 @@ class FlutterGoldenTestRunner extends GoldenTestRunner {
     PumpAction pumpBeforeTest = onlyPumpAndSettle,
     PumpWidget pumpWidget = onlyPumpWidget,
     Interaction? whilePerforming,
+    double diffThreshold = 0.0,
   }) async {
     assert(
       goldenPath is String || goldenPath is Uri,
@@ -80,8 +83,26 @@ class FlutterGoldenTestRunner extends GoldenTestRunner {
     final mementoDebugDisableShadows = debugDisableShadows;
     debugDisableShadows = !renderShadows;
 
+    GoldenFileComparator? originalComparator;
     Future<ui.Image>? imageFuture;
     try {
+      if (diffThreshold > 0) {
+        final comparator = goldenFileComparator;
+        if (comparator is LocalFileComparator &&
+            comparator.runtimeType == LocalFileComparator) {
+          originalComparator = comparator;
+          goldenFileComparator = AlchemistFileComparator.fromExisting(
+            comparator,
+            diffThreshold,
+          );
+        } else {
+          throw UnsupportedError(
+            'diffThreshold is set to $diffThreshold but the current '
+            'GoldenFileComparator (${comparator.runtimeType}) is not a '
+            'LocalFileComparator. diffThreshold is not supported.',
+          );
+        }
+      }
       await goldenTestAdapter.pumpGoldenTest(
         tester: tester,
         rootKey: rootKey,
@@ -125,6 +146,9 @@ class FlutterGoldenTestRunner extends GoldenTestRunner {
         rethrow;
       }
     } finally {
+      if (originalComparator != null) {
+        goldenFileComparator = originalComparator;
+      }
       debugDisableShadows = mementoDebugDisableShadows;
       final image = await imageFuture;
       image?.dispose();