Enable secret_scopes tests on testserver; add /Groups endpoint (#4222)

Author: denik

acceptance/bin/sort_acls_json.py

@@ -0,0 +1,45 @@
+#!/usr/bin/env python3
+"""
+Sort ACLs in JSON files recursively to ensure consistent ordering.
+
+This script reads JSON from stdin, recursively finds all "acls" arrays,
+sorts them by principal, and outputs the normalized JSON pretty-printed.
+
+Usage:
+    cat file.json | sort_acls_json.py
+    sort_acls_json.py < file.json
+"""
+
+import json
+import sys
+
+
+def sort_acls_recursive(obj):
+    """Recursively traverse the object and sort any 'acls' arrays by principal."""
+    if isinstance(obj, dict):
+        result = {}
+        for key, value in obj.items():
+            if key == "acls" and isinstance(value, list):
+                result[key] = sorted(value, key=repr)
+            else:
+                result[key] = sort_acls_recursive(value)
+        return result
+    elif isinstance(obj, list):
+        return [sort_acls_recursive(item) for item in obj]
+    else:
+        return obj
+
+
+def main():
+    raw = sys.stdin.read()
+    try:
+        data = json.loads(raw)
+    except Exception:
+        print("Not json:\n" + raw, flush=True)
+        raise
+    normalized = sort_acls_recursive(data)
+    print(json.dumps(normalized, indent=2))
+
+
+if __name__ == "__main__":
+    main()

acceptance/bundle/resources/secret_scopes/basic/out.plan2.direct.json

@@ -55,13 +55,13 @@
       "remote_state": {
         "scope_name": "test-scope-[UNIQUE_NAME]-1",
         "acls": [
-          {
-            "permission": "WRITE",
-            "principal": "deco-test-user@databricks.com"
-          },
           {
             "permission": "MANAGE",
             "principal": "[USERNAME]"
+          },
+          {
+            "permission": "WRITE",
+            "principal": "deco-test-user@databricks.com"
           }
         ]
       },

acceptance/bundle/resources/secret_scopes/basic/out.plan_verify_no_drift.direct.json

@@ -22,13 +22,13 @@
       "remote_state": {
         "scope_name": "test-scope-[UNIQUE_NAME]-2",
         "acls": [
-          {
-            "permission": "WRITE",
-            "principal": "deco-test-user@databricks.com"
-          },
           {
             "permission": "MANAGE",
             "principal": "[USERNAME]"
+          },
+          {
+            "permission": "WRITE",
+            "principal": "deco-test-user@databricks.com"
           }
         ]
       }

acceptance/bundle/resources/secret_scopes/basic/out.test.toml

@@ -23,6 +23,10 @@ Deployment complete!
   "value":"bXktc2VjcmV0LXZhbHVl"
 }
 
+>>> [CLI] secrets list-acls test-scope-[UNIQUE_NAME]-1
+{"permission":"MANAGE","principal":"[USERNAME]"}
+{"permission":"WRITE","principal":"deco-test-user@databricks.com"}
+
 >>> print_requests.py //secrets
 {
   "method": "POST",
@@ -77,6 +81,10 @@ Deployment complete!
   "value":"YW5vdGhlci1zZWNyZXQtdmFsdWU="
 }
 
+>>> [CLI] secrets list-acls test-scope-[UNIQUE_NAME]-2
+{"permission":"MANAGE","principal":"[USERNAME]"}
+{"permission":"WRITE","principal":"deco-test-user@databricks.com"}
+
 >>> print_requests.py //secrets
 {
   "method": "POST",

acceptance/bundle/resources/secret_scopes/basic/output.txt

@@ -9,7 +9,7 @@ cleanup() {
 trap cleanup EXIT
 
 title "create the secret scope"
-trace $CLI bundle plan -o json > out.plan1.$DATABRICKS_BUNDLE_ENGINE.json
+trace $CLI bundle plan -o json | sort_acls_json.py > out.plan1.$DATABRICKS_BUNDLE_ENGINE.json
 trace $CLI bundle deploy
 
 scope_name=$($CLI bundle summary --output json | jq -r '.resources.secret_scopes.my_scope.name')
@@ -18,14 +18,15 @@ trace $CLI secrets list-scopes -o json | jq ".[] | select(.name == \"$scope_name
 title "put and get secret in first scope"
 trace $CLI secrets put-secret $scope_name my-key --string-value "my-secret-value"
 trace $CLI secrets get-secret $scope_name my-key
+trace $CLI secrets list-acls $scope_name | jq -c '.[]' | sort
 
 trace print_requests.py //secrets
 
 title "update the name of the scope (should recreate)"
 export SECRET_SCOPE_NAME="test-scope-$UNIQUE_NAME-2"
 envsubst < databricks.yml.tmpl > databricks.yml
 
-trace $CLI bundle plan -o json > out.plan2.$DATABRICKS_BUNDLE_ENGINE.json
+trace $CLI bundle plan -o json | sort_acls_json.py > out.plan2.$DATABRICKS_BUNDLE_ENGINE.json
 trace $CLI bundle deploy
 
 # Capture API requests for verification. Terraform cleans up ACLs before deleting the scope, but direct does not, hence the difference in requests.
@@ -37,8 +38,9 @@ trace $CLI secrets list-scopes -o json | jq ".[] | select(.name == \"$scope_name
 title "put and get secret in recreated scope"
 trace $CLI secrets put-secret $scope_name another-key --string-value "another-secret-value"
 trace $CLI secrets get-secret $scope_name another-key
+trace $CLI secrets list-acls $scope_name | jq -c '.[]' | sort
 
 trace print_requests.py //secrets
 
 title "verify there's no persistent drift"
-trace $CLI bundle plan -o json > out.plan_verify_no_drift.$DATABRICKS_BUNDLE_ENGINE.json
+trace $CLI bundle plan -o json | sort_acls_json.py > out.plan_verify_no_drift.$DATABRICKS_BUNDLE_ENGINE.json

acceptance/bundle/resources/secret_scopes/basic/script

@@ -1,4 +1,4 @@
 Cloud = true
-Local = false
+Local = true
 RecordRequests = true
 IsServicePrincipal = true