From b487daaa4fc683ea7e79f962fd0200c3a6d621ab Mon Sep 17 00:00:00 2001 From: Simon Schrottner Date: Mon, 30 Mar 2026 15:55:45 +0200 Subject: [PATCH 1/2] test(operators): add follow-up error scenarios for operator null-return behaviour MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Covers the remaining cases from https://github.com/open-feature/flagd/issues/1874 identified in the review of #342: - starts_with / ends_with: non-string first argument → must return null - starts_with / ends_with: wrong argument count → must return null - sem_ver: wrong argument count (missing target version) → must return null - fractional: all-zero bucket weights (no bucket matched) → must return null - fractional: negative bucket weight clamped to zero → "one" gets effective weight 0, "two" gets 100% of the weight All scenarios use a bare operator as the targeting expression (not wrapped in 'if') so a null result selects the defaultVariant directly. Scenarios are tagged @operator-errors and mirrored in both the SDK-level gherkin (gherkin/targeting.feature) and the evaluator-level gherkin files (evaluator/gherkin/{string,semver,fractional}.feature) with matching flag definitions added to both flags/edge-case-flags.json and evaluator/flags/testkit-flags.json. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: Simon Schrottner --- evaluator/flags/testkit-flags.json | 64 ++++++++++++++++++++++++++++ evaluator/gherkin/fractional.feature | 18 ++++++++ evaluator/gherkin/semver.feature | 10 +++++ evaluator/gherkin/string.feature | 27 ++++++++++++ flags/edge-case-flags.json | 64 ++++++++++++++++++++++++++++ gherkin/targeting.feature | 45 +++++++++++++++++++ 6 files changed, 228 insertions(+) diff --git a/evaluator/flags/testkit-flags.json b/evaluator/flags/testkit-flags.json index fc696d4..f9b2a87 100644 --- a/evaluator/flags/testkit-flags.json +++ b/evaluator/flags/testkit-flags.json @@ -583,6 +583,70 @@ "miss" ] } + }, + "starts-with-non-string-flag": { + "state": "ENABLED", + "variants": { "fallback": "fallback" }, + "defaultVariant": "fallback", + "targeting": { + "starts_with": [{"var": "num"}, "abc"] + } + }, + "ends-with-non-string-flag": { + "state": "ENABLED", + "variants": { "fallback": "fallback" }, + "defaultVariant": "fallback", + "targeting": { + "ends_with": [{"var": "num"}, "xyz"] + } + }, + "starts-with-wrong-args-flag": { + "state": "ENABLED", + "variants": { "fallback": "fallback" }, + "defaultVariant": "fallback", + "targeting": { + "starts_with": ["abc"] + } + }, + "ends-with-wrong-args-flag": { + "state": "ENABLED", + "variants": { "fallback": "fallback" }, + "defaultVariant": "fallback", + "targeting": { + "ends_with": ["xyz"] + } + }, + "fractional-zero-weights-flag": { + "state": "ENABLED", + "variants": { "one": "one", "two": "two", "fallback": "fallback" }, + "defaultVariant": "fallback", + "targeting": { + "fractional": [ + {"var": "targetingKey"}, + ["one", 0], + ["two", 0] + ] + } + }, + "fractional-negative-weight-flag": { + "state": "ENABLED", + "variants": { "one": "one", "two": "two", "fallback": "fallback" }, + "defaultVariant": "fallback", + "targeting": { + "fractional": [ + {"var": "targetingKey"}, + ["one", -50], + ["two", 100] + ] + } + }, + "semver-wrong-args-flag": { + "state": "ENABLED", + "variants": { "fallback": "fallback" }, + "defaultVariant": "fallback", + "targeting": { + "sem_ver": [{"var": "version"}, "="] + } } }, "$evaluators": { diff --git a/evaluator/gherkin/fractional.feature b/evaluator/gherkin/fractional.feature index ee24e7d..8e73f27 100644 --- a/evaluator/gherkin/fractional.feature +++ b/evaluator/gherkin/fractional.feature @@ -195,3 +195,21 @@ Feature: Evaluator fractional operator And a String-flag with key "fractional-null-bucket-key-flag" and a fallback value "wrong" When the flag was evaluated with details Then the resolved details value should be "fallback" + # Follow-up error scenarios from https://github.com/open-feature/flagd/issues/1874 + + @operator-errors + Scenario: fractional with all-zero bucket weights falls back to default variant + Given an evaluator + And a String-flag with key "fractional-zero-weights-flag" and a fallback value "wrong" + And a context containing a targeting key with value "any-user" + When the flag was evaluated with details + Then the resolved details value should be "fallback" + + @operator-errors + Scenario: fractional negative bucket weight is clamped to zero + # ["one", -50] is treated as ["one", 0]; "two" gets 100% of the weight + Given an evaluator + And a String-flag with key "fractional-negative-weight-flag" and a fallback value "wrong" + And a context containing a targeting key with value "any-user" + When the flag was evaluated with details + Then the resolved details value should be "two" diff --git a/evaluator/gherkin/semver.feature b/evaluator/gherkin/semver.feature index 8004bee..d220909 100644 --- a/evaluator/gherkin/semver.feature +++ b/evaluator/gherkin/semver.feature @@ -86,3 +86,13 @@ Feature: Evaluator semantic version operator | 1.0.0 | match | | 1.0.0+other | match | | 2.0.0 | no-match | + + # Follow-up error scenarios from https://github.com/open-feature/flagd/issues/1874 + + @operator-errors + Scenario: sem_ver returns null for wrong argument count + Given an evaluator + And a String-flag with key "semver-wrong-args-flag" and a fallback value "wrong" + And a context containing a key "version", with type "String" and with value "1.0.0" + When the flag was evaluated with details + Then the resolved details value should be "fallback" diff --git a/evaluator/gherkin/string.feature b/evaluator/gherkin/string.feature index 14e995f..ff3ec52 100644 --- a/evaluator/gherkin/string.feature +++ b/evaluator/gherkin/string.feature @@ -16,3 +16,30 @@ Feature: Evaluator string comparison operator | uvwxyz | postfix | | abcxyz | prefix | | lmnopq | none | + + # Follow-up error scenarios from https://github.com/open-feature/flagd/issues/1874 + # starts_with and ends_with must return null (not false) on error so that the + # default variant is selected, rather than looking up a non-existent "false" variant. + + @operator-errors + Scenario Outline: starts_with and ends_with return null for non-string input + Given an evaluator + And a String-flag with key "" and a fallback value "wrong" + And a context containing a key "num", with type "Integer" and with value "123" + When the flag was evaluated with details + Then the resolved details value should be "fallback" + Examples: + | key | + | starts-with-non-string-flag | + | ends-with-non-string-flag | + + @operator-errors + Scenario Outline: starts_with and ends_with return null for wrong argument count + Given an evaluator + And a String-flag with key "" and a fallback value "wrong" + When the flag was evaluated with details + Then the resolved details value should be "fallback" + Examples: + | key | + | starts-with-wrong-args-flag | + | ends-with-wrong-args-flag | diff --git a/flags/edge-case-flags.json b/flags/edge-case-flags.json index f7fcf30..451efbe 100644 --- a/flags/edge-case-flags.json +++ b/flags/edge-case-flags.json @@ -92,6 +92,70 @@ ["two", 50] ] } + }, + "starts-with-non-string-flag": { + "state": "ENABLED", + "variants": { "fallback": "fallback" }, + "defaultVariant": "fallback", + "targeting": { + "starts_with": [{"var": "num"}, "abc"] + } + }, + "ends-with-non-string-flag": { + "state": "ENABLED", + "variants": { "fallback": "fallback" }, + "defaultVariant": "fallback", + "targeting": { + "ends_with": [{"var": "num"}, "xyz"] + } + }, + "starts-with-wrong-args-flag": { + "state": "ENABLED", + "variants": { "fallback": "fallback" }, + "defaultVariant": "fallback", + "targeting": { + "starts_with": ["abc"] + } + }, + "ends-with-wrong-args-flag": { + "state": "ENABLED", + "variants": { "fallback": "fallback" }, + "defaultVariant": "fallback", + "targeting": { + "ends_with": ["xyz"] + } + }, + "fractional-zero-weights-flag": { + "state": "ENABLED", + "variants": { "one": "one", "two": "two", "fallback": "fallback" }, + "defaultVariant": "fallback", + "targeting": { + "fractional": [ + {"var": "targetingKey"}, + ["one", 0], + ["two", 0] + ] + } + }, + "fractional-negative-weight-flag": { + "state": "ENABLED", + "variants": { "one": "one", "two": "two", "fallback": "fallback" }, + "defaultVariant": "fallback", + "targeting": { + "fractional": [ + {"var": "targetingKey"}, + ["one", -50], + ["two", 100] + ] + } + }, + "semver-wrong-args-flag": { + "state": "ENABLED", + "variants": { "fallback": "fallback" }, + "defaultVariant": "fallback", + "targeting": { + "sem_ver": [{"var": "version"}, "="] + } } } } diff --git a/gherkin/targeting.feature b/gherkin/targeting.feature index 8a193b9..51fdd52 100644 --- a/gherkin/targeting.feature +++ b/gherkin/targeting.feature @@ -359,3 +359,48 @@ Feature: Targeting rules Given a String-flag with key "fractional-null-bucket-key-flag" and a default value "wrong" When the flag was evaluated with details Then the resolved details value should be "fallback" + # Follow-up error scenarios from https://github.com/open-feature/flagd/issues/1874 + # Operators must return null (not false) on error so the default variant is selected. + + @operator-errors @string + Scenario Outline: starts_with and ends_with return null for non-string input + Given a String-flag with key "" and a default value "wrong" + And a context containing a key "num", with type "Integer" and with value "123" + When the flag was evaluated with details + Then the resolved details value should be "fallback" + Examples: + | key | + | starts-with-non-string-flag | + | ends-with-non-string-flag | + + @operator-errors @string + Scenario Outline: starts_with and ends_with return null for wrong argument count + Given a String-flag with key "" and a default value "wrong" + When the flag was evaluated with details + Then the resolved details value should be "fallback" + Examples: + | key | + | starts-with-wrong-args-flag | + | ends-with-wrong-args-flag | + + @operator-errors @semver + Scenario: sem_ver returns null for wrong argument count + Given a String-flag with key "semver-wrong-args-flag" and a default value "wrong" + And a context containing a key "version", with type "String" and with value "1.0.0" + When the flag was evaluated with details + Then the resolved details value should be "fallback" + + @operator-errors @fractional + Scenario: fractional with all-zero bucket weights falls back to default variant + Given a String-flag with key "fractional-zero-weights-flag" and a default value "wrong" + And a context containing a targeting key with value "any-user" + When the flag was evaluated with details + Then the resolved details value should be "fallback" + + @operator-errors @fractional + Scenario: fractional negative bucket weight is clamped to zero + # ["one", -50] is treated as ["one", 0]; "two" gets 100% of the weight + Given a String-flag with key "fractional-negative-weight-flag" and a default value "wrong" + And a context containing a targeting key with value "any-user" + When the flag was evaluated with details + Then the resolved details value should be "two" From 604df926b3b1c6caf5133a25e27c3bda87b2f90a Mon Sep 17 00:00:00 2001 From: Todd Baert Date: Thu, 9 Apr 2026 16:43:51 -0400 Subject: [PATCH 2/2] fixup! test(operators): add follow-up error scenarios for operator null-return behaviour Signed-off-by: Todd Baert --- evaluator/flags/testkit-flags.json | 75 +++++++++++++------ evaluator/gherkin/fractional.feature | 3 + evaluator/gherkin/semver.feature | 2 + evaluator/gherkin/string.feature | 2 + flags/edge-case-flags.json | 105 ++++++++++++++++++++------- gherkin/targeting.feature | 9 ++- 6 files changed, 144 insertions(+), 52 deletions(-) diff --git a/evaluator/flags/testkit-flags.json b/evaluator/flags/testkit-flags.json index f9b2a87..1db5f14 100644 --- a/evaluator/flags/testkit-flags.json +++ b/evaluator/flags/testkit-flags.json @@ -462,29 +462,38 @@ }, "semver-invalid-version-flag": { "state": "ENABLED", - "variants": { "match": "match", "no-match": "no-match", "fallback": "fallback" }, + "variants": { "true": "true", "false": "false", "fallback": "fallback" }, "defaultVariant": "fallback", "targeting": { - "sem_ver": [{"var": "version"}, "=", "1.0.0"] + "if": [ + {"sem_ver": [{"var": "version"}, "=", "1.0.0"]}, + "true", "false" + ] } }, "semver-invalid-operator-flag": { "state": "ENABLED", - "variants": { "match": "match", "no-match": "no-match", "fallback": "fallback" }, + "variants": { "true": "true", "false": "false", "fallback": "fallback" }, "defaultVariant": "fallback", "targeting": { - "sem_ver": [{"var": "version"}, "===", "1.0.0"] + "if": [ + {"sem_ver": [{"var": "version"}, "===", "1.0.0"]}, + "true", "false" + ] } }, "fractional-null-bucket-key-flag": { "state": "ENABLED", - "variants": { "one": "one", "two": "two", "fallback": "fallback" }, + "variants": { "true": "true", "false": "false", "fallback": "fallback" }, "defaultVariant": "fallback", "targeting": { - "fractional": [ - {"var": "missing_key"}, - ["one", 50], - ["two", 50] + "if": [ + {"fractional": [ + {"var": "missing_key"}, + ["one", 50], + ["two", 50] + ]}, + "true", "false" ] } }, @@ -586,45 +595,60 @@ }, "starts-with-non-string-flag": { "state": "ENABLED", - "variants": { "fallback": "fallback" }, + "variants": { "true": "true", "false": "false", "fallback": "fallback" }, "defaultVariant": "fallback", "targeting": { - "starts_with": [{"var": "num"}, "abc"] + "if": [ + {"starts_with": [{"var": "num"}, "abc"]}, + "true", "false" + ] } }, "ends-with-non-string-flag": { "state": "ENABLED", - "variants": { "fallback": "fallback" }, + "variants": { "true": "true", "false": "false", "fallback": "fallback" }, "defaultVariant": "fallback", "targeting": { - "ends_with": [{"var": "num"}, "xyz"] + "if": [ + {"ends_with": [{"var": "num"}, "xyz"]}, + "true", "false" + ] } }, "starts-with-wrong-args-flag": { "state": "ENABLED", - "variants": { "fallback": "fallback" }, + "variants": { "true": "true", "false": "false", "fallback": "fallback" }, "defaultVariant": "fallback", "targeting": { - "starts_with": ["abc"] + "if": [ + {"starts_with": ["abc"]}, + "true", "false" + ] } }, "ends-with-wrong-args-flag": { "state": "ENABLED", - "variants": { "fallback": "fallback" }, + "variants": { "true": "true", "false": "false", "fallback": "fallback" }, "defaultVariant": "fallback", "targeting": { - "ends_with": ["xyz"] + "if": [ + {"ends_with": ["xyz"]}, + "true", "false" + ] } }, "fractional-zero-weights-flag": { "state": "ENABLED", - "variants": { "one": "one", "two": "two", "fallback": "fallback" }, + "variants": { "true": "true", "false": "false", "fallback": "fallback" }, "defaultVariant": "fallback", "targeting": { - "fractional": [ - {"var": "targetingKey"}, - ["one", 0], - ["two", 0] + "if": [ + {"fractional": [ + {"var": "targetingKey"}, + ["one", 0], + ["two", 0] + ]}, + "true", "false" ] } }, @@ -642,10 +666,13 @@ }, "semver-wrong-args-flag": { "state": "ENABLED", - "variants": { "fallback": "fallback" }, + "variants": { "true": "true", "false": "false", "fallback": "fallback" }, "defaultVariant": "fallback", "targeting": { - "sem_ver": [{"var": "version"}, "="] + "if": [ + {"sem_ver": [{"var": "version"}, "="]}, + "true", "false" + ] } } }, diff --git a/evaluator/gherkin/fractional.feature b/evaluator/gherkin/fractional.feature index 8e73f27..a58fc37 100644 --- a/evaluator/gherkin/fractional.feature +++ b/evaluator/gherkin/fractional.feature @@ -195,6 +195,8 @@ Feature: Evaluator fractional operator And a String-flag with key "fractional-null-bucket-key-flag" and a fallback value "wrong" When the flag was evaluated with details Then the resolved details value should be "fallback" + And the reason should be "DEFAULT" + # Follow-up error scenarios from https://github.com/open-feature/flagd/issues/1874 @operator-errors @@ -204,6 +206,7 @@ Feature: Evaluator fractional operator And a context containing a targeting key with value "any-user" When the flag was evaluated with details Then the resolved details value should be "fallback" + And the reason should be "DEFAULT" @operator-errors Scenario: fractional negative bucket weight is clamped to zero diff --git a/evaluator/gherkin/semver.feature b/evaluator/gherkin/semver.feature index d220909..18c5130 100644 --- a/evaluator/gherkin/semver.feature +++ b/evaluator/gherkin/semver.feature @@ -39,6 +39,7 @@ Feature: Evaluator semantic version operator And a context containing a key "version", with type "String" and with value "" When the flag was evaluated with details Then the resolved details value should be "fallback" + And the reason should be "DEFAULT" Examples: | key | context_value | | semver-invalid-version-flag | not-a-version | @@ -96,3 +97,4 @@ Feature: Evaluator semantic version operator And a context containing a key "version", with type "String" and with value "1.0.0" When the flag was evaluated with details Then the resolved details value should be "fallback" + And the reason should be "DEFAULT" diff --git a/evaluator/gherkin/string.feature b/evaluator/gherkin/string.feature index ff3ec52..7e77176 100644 --- a/evaluator/gherkin/string.feature +++ b/evaluator/gherkin/string.feature @@ -28,6 +28,7 @@ Feature: Evaluator string comparison operator And a context containing a key "num", with type "Integer" and with value "123" When the flag was evaluated with details Then the resolved details value should be "fallback" + And the reason should be "DEFAULT" Examples: | key | | starts-with-non-string-flag | @@ -39,6 +40,7 @@ Feature: Evaluator string comparison operator And a String-flag with key "" and a fallback value "wrong" When the flag was evaluated with details Then the resolved details value should be "fallback" + And the reason should be "DEFAULT" Examples: | key | | starts-with-wrong-args-flag | diff --git a/flags/edge-case-flags.json b/flags/edge-case-flags.json index 451efbe..42d6eac 100644 --- a/flags/edge-case-flags.json +++ b/flags/edge-case-flags.json @@ -56,84 +56,128 @@ "semver-invalid-version-flag": { "state": "ENABLED", "variants": { - "match": "match", - "no-match": "no-match", + "true": "true", + "false": "false", "fallback": "fallback" }, "defaultVariant": "fallback", "targeting": { - "sem_ver": [{"var": "version"}, "=", "1.0.0"] + "if": [ + {"sem_ver": [{"var": "version"}, "=", "1.0.0"]}, + "true", "false" + ] } }, "semver-invalid-operator-flag": { "state": "ENABLED", "variants": { - "match": "match", - "no-match": "no-match", + "true": "true", + "false": "false", "fallback": "fallback" }, "defaultVariant": "fallback", "targeting": { - "sem_ver": [{"var": "version"}, "===", "1.0.0"] + "if": [ + {"sem_ver": [{"var": "version"}, "===", "1.0.0"]}, + "true", "false" + ] } }, "fractional-null-bucket-key-flag": { "state": "ENABLED", "variants": { - "one": "one", - "two": "two", + "true": "true", + "false": "false", "fallback": "fallback" }, "defaultVariant": "fallback", "targeting": { - "fractional": [ - {"var": "missing_key"}, - ["one", 50], - ["two", 50] + "if": [ + {"fractional": [ + {"var": "missing_key"}, + ["one", 50], + ["two", 50] + ]}, + "true", "false" ] } }, "starts-with-non-string-flag": { "state": "ENABLED", - "variants": { "fallback": "fallback" }, + "variants": { + "true": "true", + "false": "false", + "fallback": "fallback" + }, "defaultVariant": "fallback", "targeting": { - "starts_with": [{"var": "num"}, "abc"] + "if": [ + {"starts_with": [{"var": "num"}, "abc"]}, + "true", "false" + ] } }, "ends-with-non-string-flag": { "state": "ENABLED", - "variants": { "fallback": "fallback" }, + "variants": { + "true": "true", + "false": "false", + "fallback": "fallback" + }, "defaultVariant": "fallback", "targeting": { - "ends_with": [{"var": "num"}, "xyz"] + "if": [ + {"ends_with": [{"var": "num"}, "xyz"]}, + "true", "false" + ] } }, "starts-with-wrong-args-flag": { "state": "ENABLED", - "variants": { "fallback": "fallback" }, + "variants": { + "true": "true", + "false": "false", + "fallback": "fallback" + }, "defaultVariant": "fallback", "targeting": { - "starts_with": ["abc"] + "if": [ + {"starts_with": ["abc"]}, + "true", "false" + ] } }, "ends-with-wrong-args-flag": { "state": "ENABLED", - "variants": { "fallback": "fallback" }, + "variants": { + "true": "true", + "false": "false", + "fallback": "fallback" + }, "defaultVariant": "fallback", "targeting": { - "ends_with": ["xyz"] + "if": [ + {"ends_with": ["xyz"]}, + "true", "false" + ] } }, "fractional-zero-weights-flag": { "state": "ENABLED", - "variants": { "one": "one", "two": "two", "fallback": "fallback" }, + "variants": { + "true": "true", + "false": "false", + "fallback": "fallback" + }, "defaultVariant": "fallback", "targeting": { - "fractional": [ - {"var": "targetingKey"}, - ["one", 0], - ["two", 0] + "if": [ + {"fractional": [ + {"var": "targetingKey"}, + ["one", 0], + ["two", 0] + ]}, + "true", "false" ] } }, @@ -151,10 +195,17 @@ }, "semver-wrong-args-flag": { "state": "ENABLED", - "variants": { "fallback": "fallback" }, + "variants": { + "true": "true", + "false": "false", + "fallback": "fallback" + }, "defaultVariant": "fallback", "targeting": { - "sem_ver": [{"var": "version"}, "="] + "if": [ + {"sem_ver": [{"var": "version"}, "="]}, + "true", "false" + ] } } } diff --git a/gherkin/targeting.feature b/gherkin/targeting.feature index 51fdd52..ec90364 100644 --- a/gherkin/targeting.feature +++ b/gherkin/targeting.feature @@ -349,6 +349,7 @@ Feature: Targeting rules And a context containing a key "version", with type "String" and with value "" When the flag was evaluated with details Then the resolved details value should be "fallback" + And the reason should be "DEFAULT" Examples: | key | context_value | | semver-invalid-version-flag | not-a-version | @@ -359,6 +360,8 @@ Feature: Targeting rules Given a String-flag with key "fractional-null-bucket-key-flag" and a default value "wrong" When the flag was evaluated with details Then the resolved details value should be "fallback" + And the reason should be "DEFAULT" + # Follow-up error scenarios from https://github.com/open-feature/flagd/issues/1874 # Operators must return null (not false) on error so the default variant is selected. @@ -368,6 +371,7 @@ Feature: Targeting rules And a context containing a key "num", with type "Integer" and with value "123" When the flag was evaluated with details Then the resolved details value should be "fallback" + And the reason should be "DEFAULT" Examples: | key | | starts-with-non-string-flag | @@ -378,8 +382,9 @@ Feature: Targeting rules Given a String-flag with key "" and a default value "wrong" When the flag was evaluated with details Then the resolved details value should be "fallback" + And the reason should be "DEFAULT" Examples: - | key | + | key | | starts-with-wrong-args-flag | | ends-with-wrong-args-flag | @@ -389,6 +394,7 @@ Feature: Targeting rules And a context containing a key "version", with type "String" and with value "1.0.0" When the flag was evaluated with details Then the resolved details value should be "fallback" + And the reason should be "DEFAULT" @operator-errors @fractional Scenario: fractional with all-zero bucket weights falls back to default variant @@ -396,6 +402,7 @@ Feature: Targeting rules And a context containing a targeting key with value "any-user" When the flag was evaluated with details Then the resolved details value should be "fallback" + And the reason should be "DEFAULT" @operator-errors @fractional Scenario: fractional negative bucket weight is clamped to zero