Skip to content

Commit d770547

Browse files
chaliyclaude
chaliy
and
claude
authored
test(skills): integration tests for real-world skills.sh scripts (#292)
## Summary - Add 10 real bash scripts from top skills.sh repos as test fixtures - Parse + execute them through bashkit with stubbed external binaries (az, helm, npm, curl, python3) - **10/10 parse, 6/10 execute, 4 ignored** with tracked bugs ## What's tested | Script | Source | Bash features | |--------|--------|--------------| | azure_discover_rank.sh | microsoft/github-copilot-for-azure | `declare -A`, `${!MAP[@]}`, jq pipes, `set -euo pipefail` | | helm_validate_chart.sh | wshobson/agents | functions, `command -v`, `grep -q`, `awk`, echo -e ANSI | | jwt_test_setup.sh | giuseppe-trisciuoglio/developer-kit | `${var: -3}` substring, `local`, `trap EXIT`, `curl -w` | | stitch_fetch.sh | nichochar/stitch-skills | curl wrapper, `$?` check | | stitch_download_asset.sh | nichochar/stitch-skills | `dirname`, `mkdir -p`, `command -v`, `stat` | | find_polluter.sh | nichochar/superpowers | `for`/`$(find)`, `$(( ))`, `wc|tr`, `|| true` | ## Bugs found - #289: backslash line continuation fails in some parser contexts - #290: while/case arg parsing loop hits MaxLoopIterations - #291: `[ -f ]` doesn't see VFS files after cd in script execution ## Test plan - [x] `cargo test --test skills_tests` — 16 pass, 0 fail, 4 ignored - [x] `cargo test --test spec_tests` — existing tests unaffected - [x] `cargo clippy -- -D warnings` clean - [x] `cargo fmt --check` clean --------- Co-authored-by: Claude <noreply@anthropic.com>
1 parent 8e10e34 commit d770547

File tree

11 files changed

+1948
-0
lines changed

11 files changed

+1948
-0
lines changed

‎crates/bashkit/tests/skills_fixtures/azure_discover_rank.sh‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
#!/bin/bash
2+
# discover_and_rank.sh
3+
# Discovers available capacity for an Azure OpenAI model across all regions,
4+
# cross-references with existing projects and subscription quota, and outputs a ranked table.
5+
#
6+
# Usage: ./discover_and_rank.sh <model-name> <model-version> [min-capacity]
7+
# Example: ./discover_and_rank.sh o3-mini 2025-01-31 200
8+
#
9+
# Output: Ranked table of regions with capacity, quota, project counts, and match status
10+
#
11+
# NOTE: Backslash line continuations removed for bashkit parser compatibility.
12+
# Original at: microsoft/github-copilot-for-azure
13+
14+
set -euo pipefail
15+
16+
MODEL_NAME="${1:?Usage: $0 <model-name> <model-version> [min-capacity]}"
17+
MODEL_VERSION="${2:?Usage: $0 <model-name> <model-version> [min-capacity]}"
18+
MIN_CAPACITY="${3:-0}"
19+
20+
SUB_ID=$(az account show --query id -o tsv)
21+
22+
# Query model capacity across all regions (GlobalStandard SKU)
23+
CAPACITY_JSON=$(az rest --method GET --url "https://management.azure.com/subscriptions/${SUB_ID}/providers/Microsoft.CognitiveServices/modelCapacities" --url-parameters api-version=2024-10-01 modelFormat=OpenAI modelName="$MODEL_NAME" modelVersion="$MODEL_VERSION" 2>/dev/null)
24+
25+
# Query all AI Services projects
26+
PROJECTS_JSON=$(az rest --method GET --url "https://management.azure.com/subscriptions/${SUB_ID}/providers/Microsoft.CognitiveServices/accounts" --url-parameters api-version=2024-10-01 --query "value[?kind=='AIServices'].{name:name, location:location}" 2>/dev/null)
27+
28+
# Get unique regions from capacity results for quota checking
29+
REGIONS=$(echo "$CAPACITY_JSON" | jq -r '.value[] | select(.properties.skuName=="GlobalStandard" and .properties.availableCapacity > 0) | .location' | sort -u)
30+
31+
# Build quota map: check subscription quota per region
32+
declare -A QUOTA_MAP
33+
for region in $REGIONS; do
34+
usage_json=$(az cognitiveservices usage list --location "$region" --subscription "$SUB_ID" -o json 2>/dev/null || echo "[]")
35+
quota_avail=$(echo "$usage_json" | jq -r --arg name "OpenAI.GlobalStandard.$MODEL_NAME" '[.[] | select(.name.value == $name)] | if length > 0 then .[0].limit - .[0].currentValue else 0 end')
36+
QUOTA_MAP[$region]="${quota_avail:-0}"
37+
done
38+
39+
# Export quota map as JSON for Python
40+
QUOTA_JSON="{"
41+
first=true
42+
for region in "${!QUOTA_MAP[@]}"; do
43+
if [ "$first" = true ]; then first=false; else QUOTA_JSON+=","; fi
44+
QUOTA_JSON+="\"$region\":${QUOTA_MAP[$region]}"
45+
done
46+
QUOTA_JSON+="}"
47+
48+
# Combine, rank, and output using inline Python (available on all Azure CLI installs)
49+
python3 -c "
50+
import json, sys
51+
52+
capacity = json.loads('''${CAPACITY_JSON}''')
53+
projects = json.loads('''${PROJECTS_JSON}''')
54+
quota = json.loads('''${QUOTA_JSON}''')
55+
min_cap = int('${MIN_CAPACITY}')
56+
57+
# Build capacity map (GlobalStandard only)
58+
cap_map = {}
59+
for item in capacity.get('value', []):
60+
props = item.get('properties', {})
61+
if props.get('skuName') == 'GlobalStandard' and props.get('availableCapacity', 0) > 0:
62+
region = item.get('location', '')
63+
cap_map[region] = max(cap_map.get(region, 0), props['availableCapacity'])
64+
65+
# Build project count map
66+
proj_map = {}
67+
proj_sample = {}
68+
for p in (projects if isinstance(projects, list) else []):
69+
loc = p.get('location', '')
70+
proj_map[loc] = proj_map.get(loc, 0) + 1
71+
if loc not in proj_sample:
72+
proj_sample[loc] = p.get('name', '')
73+
74+
# Combine and rank
75+
results = []
76+
for region, cap in cap_map.items():
77+
meets = cap >= min_cap
78+
q = quota.get(region, 0)
79+
quota_ok = q > 0
80+
results.append({
81+
'region': region,
82+
'available': cap,
83+
'meets': meets,
84+
'projects': proj_map.get(region, 0),
85+
'sample': proj_sample.get(region, '(none)'),
86+
'quota': q,
87+
'quota_ok': quota_ok
88+
})
89+
90+
# Sort: meets target first, then quota available, then by project count, then by capacity
91+
results.sort(key=lambda x: (-x['meets'], -x['quota_ok'], -x['projects'], -x['available']))
92+
93+
# Output
94+
total = len(results)
95+
matching = sum(1 for r in results if r['meets'])
96+
with_quota = sum(1 for r in results if r['meets'] and r['quota_ok'])
97+
with_projects = sum(1 for r in results if r['meets'] and r['projects'] > 0)
98+
99+
print(f'Model: {\"${MODEL_NAME}\"} v{\"${MODEL_VERSION}\"} | SKU: GlobalStandard | Min Capacity: {min_cap}K TPM')
100+
print(f'Regions with capacity: {total} | Meets target: {matching} | With quota: {with_quota} | With projects: {with_projects}')
101+
print()
102+
print(f'{\"Region\":<22} {\"Available\":<12} {\"Meets Target\":<14} {\"Quota\":<12} {\"Projects\":<10} {\"Sample Project\"}')
103+
print('-' * 100)
104+
for r in results:
105+
mark = 'YES' if r['meets'] else 'no'
106+
q_display = f'{r[\"quota\"]}K' if r['quota'] > 0 else '0 (none)'
107+
print(f'{r[\"region\"]:<22} {r[\"available\"]}K{\"\":.<10} {mark:<14} {q_display:<12} {r[\"projects\"]:<10} {r[\"sample\"]}')
108+
"

‎crates/bashkit/tests/skills_fixtures/azure_generate_url.sh‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
#!/bin/bash
2+
# Generate Azure AI Foundry portal URL for a model deployment
3+
# This script creates a direct clickable link to view a deployment in the Azure AI Foundry portal
4+
5+
set -e
6+
7+
# Function to display usage
8+
usage() {
9+
cat << EOF
10+
Usage: $0 --subscription SUBSCRIPTION_ID --resource-group RESOURCE_GROUP \\
11+
--foundry-resource FOUNDRY_RESOURCE --project PROJECT_NAME \\
12+
--deployment DEPLOYMENT_NAME
13+
14+
Generate Azure AI Foundry deployment URL
15+
16+
Required arguments:
17+
--subscription Azure subscription ID (GUID)
18+
--resource-group Resource group name
19+
--foundry-resource Foundry resource (account) name
20+
--project Project name
21+
--deployment Deployment name
22+
23+
Example:
24+
$0 --subscription d5320f9a-73da-4a74-b639-83efebc7bb6f \\
25+
--resource-group bani-host \\
26+
--foundry-resource banide-host-resource \\
27+
--project banide-host \\
28+
--deployment text-embedding-ada-002
29+
EOF
30+
exit 1
31+
}
32+
33+
# Parse command line arguments
34+
while [[ $# -gt 0 ]]; do
35+
case $1 in
36+
--subscription)
37+
SUBSCRIPTION_ID="$2"
38+
shift 2
39+
;;
40+
--resource-group)
41+
RESOURCE_GROUP="$2"
42+
shift 2
43+
;;
44+
--foundry-resource)
45+
FOUNDRY_RESOURCE="$2"
46+
shift 2
47+
;;
48+
--project)
49+
PROJECT_NAME="$2"
50+
shift 2
51+
;;
52+
--deployment)
53+
DEPLOYMENT_NAME="$2"
54+
shift 2
55+
;;
56+
-h|--help)
57+
usage
58+
;;
59+
*)
60+
echo "Unknown option: $1"
61+
usage
62+
;;
63+
esac
64+
done
65+
66+
# Validate required arguments
67+
if [ -z "$SUBSCRIPTION_ID" ] || [ -z "$RESOURCE_GROUP" ] || [ -z "$FOUNDRY_RESOURCE" ] || [ -z "$PROJECT_NAME" ] || [ -z "$DEPLOYMENT_NAME" ]; then
68+
echo "Error: Missing required arguments"
69+
usage
70+
fi
71+
72+
# Convert subscription GUID to bytes (big-endian/string order) and encode as base64url
73+
# Remove hyphens from GUID
74+
GUID_HEX=$(echo "$SUBSCRIPTION_ID" | tr -d '-')
75+
76+
# Convert hex string to bytes and base64 encode
77+
# Using xxd to convert hex to binary, then base64 encode
78+
ENCODED_SUB=$(echo "$GUID_HEX" | xxd -r -p | base64 | tr '+' '-' | tr '/' '_' | tr -d '=')
79+
80+
# Build the encoded resource path
81+
# Format: {encoded-sub-id},{resource-group},,{foundry-resource},{project-name}
82+
# Note: Two commas between resource-group and foundry-resource
83+
ENCODED_PATH="${ENCODED_SUB},${RESOURCE_GROUP},,${FOUNDRY_RESOURCE},${PROJECT_NAME}"
84+
85+
# Build the full URL
86+
BASE_URL="https://ai.azure.com/nextgen/r/"
87+
DEPLOYMENT_PATH="/build/models/deployments/${DEPLOYMENT_NAME}/details"
88+
89+
echo "${BASE_URL}${ENCODED_PATH}${DEPLOYMENT_PATH}"

‎crates/bashkit/tests/skills_fixtures/azure_query_capacity.sh‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
#!/bin/bash
2+
# query_capacity.sh
3+
# Queries available capacity for an Azure OpenAI model.
4+
#
5+
# Usage:
6+
# ./query_capacity.sh <model-name> [model-version] [region] [sku]
7+
# Examples:
8+
# ./query_capacity.sh o3-mini # List versions
9+
# ./query_capacity.sh o3-mini 2025-01-31 # All regions
10+
# ./query_capacity.sh o3-mini 2025-01-31 eastus2 # Specific region
11+
# ./query_capacity.sh o3-mini 2025-01-31 "" Standard # Different SKU
12+
13+
set -euo pipefail
14+
15+
MODEL_NAME="${1:?Usage: $0 <model-name> [model-version] [region] [sku]}"
16+
MODEL_VERSION="${2:-}"
17+
REGION="${3:-}"
18+
SKU="${4:-GlobalStandard}"
19+
20+
SUB_ID=$(az account show --query id -o tsv)
21+
22+
# If no version, list available versions
23+
if [ -z "$MODEL_VERSION" ]; then
24+
LOC="${REGION:-eastus}"
25+
echo "Available versions for $MODEL_NAME:"
26+
az cognitiveservices model list --location "$LOC" --query "[?model.name=='$MODEL_NAME'].{Version:model.version, Format:model.format}" --output table 2>/dev/null
27+
exit 0
28+
fi
29+
30+
# Build URL
31+
if [ -n "$REGION" ]; then
32+
URL="https://management.azure.com/subscriptions/${SUB_ID}/providers/Microsoft.CognitiveServices/locations/${REGION}/modelCapacities"
33+
else
34+
URL="https://management.azure.com/subscriptions/${SUB_ID}/providers/Microsoft.CognitiveServices/modelCapacities"
35+
fi
36+
37+
# Query capacity
38+
CAPACITY_RESULT=$(az rest --method GET --url "$URL" --url-parameters api-version=2024-10-01 modelFormat=OpenAI modelName="$MODEL_NAME" modelVersion="$MODEL_VERSION" 2>/dev/null)
39+
40+
# Get regions with capacity
41+
REGIONS_WITH_CAP=$(echo "$CAPACITY_RESULT" | jq -r ".value[] | select(.properties.skuName==\"$SKU\" and .properties.availableCapacity > 0) | .location" 2>/dev/null | sort -u)
42+
43+
if [ -z "$REGIONS_WITH_CAP" ]; then
44+
echo "No capacity found for $MODEL_NAME v$MODEL_VERSION ($SKU)"
45+
echo "Try a different SKU or version."
46+
exit 0
47+
fi
48+
49+
echo "Capacity: $MODEL_NAME v$MODEL_VERSION ($SKU)"
50+
echo ""
51+
printf "%-22s %-12s %-15s %s\n" "Region" "Available" "Quota" "SKU"
52+
printf -- '-%.0s' {1..60}; echo ""
53+
54+
for region in $REGIONS_WITH_CAP; do
55+
avail=$(echo "$CAPACITY_RESULT" | jq -r ".value[] | select(.location==\"$region\" and .properties.skuName==\"$SKU\") | .properties.availableCapacity" 2>/dev/null | head -1)
56+
57+
# Check subscription quota
58+
usage_json=$(az cognitiveservices usage list --location "$region" --subscription "$SUB_ID" -o json 2>/dev/null || echo "[]")
59+
quota_avail=$(echo "$usage_json" | jq -r --arg name "OpenAI.$SKU.$MODEL_NAME" '[.[] | select(.name.value == $name)] | if length > 0 then .[0].limit - .[0].currentValue else 0 end' 2>/dev/null || echo "?")
60+
61+
if [ "$quota_avail" = "0" ]; then
62+
quota_display="0 (none)"
63+
elif [ "$quota_avail" = "?" ]; then
64+
quota_display="?"
65+
else
66+
quota_display="${quota_avail}K"
67+
fi
68+
69+
printf "%-22s %-12s %-15s %s\n" "$region" "${avail}K TPM" "$quota_display" "$SKU"
70+
done

0 commit comments

Comments
 (0)