From eb780233868b139a8fd98166b355f988df2339eb Mon Sep 17 00:00:00 2001 From: streamer45 Date: Tue, 27 Jan 2026 20:07:34 +0100 Subject: [PATCH] feat(marketplace): append-only registry w/ per-plugin versions - Rename workflow input to snapshot_version (update release caller) - Enforce immutable manifests for existing plugin versions (diff on mismatch) - Preserve streamkit.pub in generated registry output - Add append-only regression test and pass --public-key for isolation --- .github/workflows/marketplace-build.yml | 11 +- .github/workflows/marketplace-release.yml | 16 +- .github/workflows/release.yml | 2 +- AGENTS.md | 2 +- marketplace/PORTABILITY_REVIEW.md | 29 -- marketplace/official-plugins.json | 8 + .../helsinki/{marketplace.yml => plugin.yml} | 1 + .../kokoro/{marketplace.yml => plugin.yml} | 1 + .../matcha/{marketplace.yml => plugin.yml} | 1 + .../nllb/{marketplace.yml => plugin.yml} | 1 + .../piper/{marketplace.yml => plugin.yml} | 1 + .../{marketplace.yml => plugin.yml} | 1 + .../vad/{marketplace.yml => plugin.yml} | 1 + .../whisper/{marketplace.yml => plugin.yml} | 1 + scripts/marketplace/build_registry.py | 364 +++++++++++++++--- .../marketplace/generate_official_plugins.py | 58 ++- scripts/marketplace/test_append_only.py | 262 +++++++++++++ scripts/marketplace/verify_bundles.py | 29 +- 18 files changed, 678 insertions(+), 111 deletions(-) delete mode 100644 marketplace/PORTABILITY_REVIEW.md rename plugins/native/helsinki/{marketplace.yml => plugin.yml} (98%) rename plugins/native/kokoro/{marketplace.yml => plugin.yml} (97%) rename plugins/native/matcha/{marketplace.yml => plugin.yml} (97%) rename plugins/native/nllb/{marketplace.yml => plugin.yml} (97%) rename plugins/native/piper/{marketplace.yml => plugin.yml} (98%) rename plugins/native/sensevoice/{marketplace.yml => plugin.yml} (98%) rename plugins/native/vad/{marketplace.yml => plugin.yml} (97%) rename plugins/native/whisper/{marketplace.yml => plugin.yml} (99%) create mode 100755 scripts/marketplace/test_append_only.py diff --git a/.github/workflows/marketplace-build.yml b/.github/workflows/marketplace-build.yml index 7113858e..7d03d805 100644 --- a/.github/workflows/marketplace-build.yml +++ b/.github/workflows/marketplace-build.yml @@ -7,8 +7,8 @@ name: Marketplace Build (Reusable) on: workflow_call: inputs: - version: - description: "Marketplace version (e.g., 1.2.3)" + snapshot_version: + description: "Marketplace snapshot version (not plugin version; used for release naming)" required: true type: string release_tag: @@ -36,7 +36,7 @@ env: RUST_BACKTRACE: 1 SHERPA_ONNX_VERSION: "1.12.17" MINISIGN_DEB_URL: "http://launchpadlibrarian.net/780165111/minisign_0.12-1_amd64.deb" - MARKETPLACE_VERSION: ${{ inputs.version }} + SNAPSHOT_VERSION: ${{ inputs.snapshot_version }} RELEASE_TAG: ${{ inputs.release_tag }} REGISTRY_BASE_URL: ${{ inputs.registry_base_url || 'https://streamkit.dev/registry' }} @@ -52,7 +52,7 @@ jobs: - name: Install system dependencies run: | sudo apt-get update - sudo apt-get install -y cmake pkg-config libclang-dev wget libopenblas-dev zstd patchelf python3-yaml + sudo apt-get install -y cmake pkg-config libclang-dev wget libopenblas-dev zstd patchelf python3-yaml python3-tomli - name: Install minisign run: | @@ -146,10 +146,9 @@ jobs: - name: Build registry artifacts run: | - VERSION="${MARKETPLACE_VERSION#v}" python3 scripts/marketplace/build_registry.py \ --plugins marketplace/official-plugins.json \ - --version "${VERSION}" \ + --existing-registry docs/public/registry \ --bundle-base-url "https://github.com/${{ github.repository }}/releases/download/${RELEASE_TAG}" \ --registry-base-url "${REGISTRY_BASE_URL}" \ --bundles-out dist/bundles \ diff --git a/.github/workflows/marketplace-release.yml b/.github/workflows/marketplace-release.yml index f99219e1..1a595d8d 100644 --- a/.github/workflows/marketplace-release.yml +++ b/.github/workflows/marketplace-release.yml @@ -7,11 +7,11 @@ name: Marketplace Release on: workflow_dispatch: inputs: - version: - description: "Marketplace version (e.g., 1.2.3)" + snapshot_version: + description: "Marketplace snapshot version (e.g., 1.2.3, 0.2.0-dev)" required: true release_tag: - description: "Release tag (defaults to marketplace-v)" + description: "Release tag (defaults to marketplace-v)" required: false prerelease: description: "Mark release as prerelease" @@ -20,7 +20,7 @@ on: default: false env: - RELEASE_TAG: ${{ inputs.release_tag || format('marketplace-v{0}', inputs.version) }} + RELEASE_TAG: ${{ inputs.release_tag || format('marketplace-v{0}', inputs.snapshot_version) }} jobs: marketplace: @@ -30,8 +30,8 @@ jobs: pull-requests: write secrets: inherit with: - version: ${{ inputs.version }} - release_tag: ${{ inputs.release_tag || format('marketplace-v{0}', inputs.version) }} + snapshot_version: ${{ inputs.snapshot_version }} + release_tag: ${{ inputs.release_tag || format('marketplace-v{0}', inputs.snapshot_version) }} create-release: name: Create Marketplace Release @@ -52,11 +52,11 @@ jobs: with: tag_name: ${{ env.RELEASE_TAG }} target_commitish: ${{ github.sha }} - name: "Marketplace ${{ inputs.version }}" + name: "Marketplace ${{ inputs.snapshot_version }}" files: | artifacts/**/marketplace-bundles/*.tar.zst body: | - Marketplace bundles for version `${{ inputs.version }}`. + Marketplace bundles for snapshot `${{ inputs.snapshot_version }}`. draft: false prerelease: ${{ inputs.prerelease }} env: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1d02a498..ea18e0ae 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -87,7 +87,7 @@ jobs: pull-requests: write secrets: inherit with: - version: ${{ github.ref_name }} + snapshot_version: ${{ github.ref_name }} release_tag: ${{ github.ref_name }} create-release: diff --git a/AGENTS.md b/AGENTS.md index 64794e33..08e1452c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -39,7 +39,7 @@ When making a plugin official and downloadable from the registry, update all of the following: - Plugin source under `plugins/native//` (crate metadata + README). -- Plugin metadata in `plugins/native//marketplace.yml` (id, entrypoint, +- Plugin metadata in `plugins/native//plugin.yml` (id, version, entrypoint, artifact path, models, licenses, homepage/repo). - Generate `marketplace/official-plugins.json` with `scripts/marketplace/generate_official_plugins.py` and commit the result. diff --git a/marketplace/PORTABILITY_REVIEW.md b/marketplace/PORTABILITY_REVIEW.md deleted file mode 100644 index 36f40f8f..00000000 --- a/marketplace/PORTABILITY_REVIEW.md +++ /dev/null @@ -1,29 +0,0 @@ - - -# Marketplace Portability Review - -Collected from local artifacts in `plugins/native/*/target/release/*.so` on 2026-01-26. -ldd checks were run on the local dev environment (not a clean container). - -| Plugin | Non-glibc NEEDED deps | RUNPATH/RPATH | ldd (local) | Recommendation | -| --- | --- | --- | --- | --- | -| `helsinki` | `libgcc_s.so.1` | — | ok | system dependency (accepted) | -| `kokoro` | `libsherpa-onnx-c-api.so`, `libgcc_s.so.1` | `RUNPATH=/usr/local/lib` | ok | must bundle + `$ORIGIN` | -| `matcha` | `libsherpa-onnx-c-api.so`, `libgcc_s.so.1` | `RUNPATH=/usr/local/lib` | ok | must bundle + `$ORIGIN` | -| `nllb` | `libstdc++.so.6`, `libgcc_s.so.1` | — | ok | system dependency (accepted) | -| `piper` | `libsherpa-onnx-c-api.so`, `libgcc_s.so.1` | `RUNPATH=/usr/local/lib` | ok | must bundle + `$ORIGIN` | -| `sensevoice` | `libsherpa-onnx-c-api.so`, `libstdc++.so.6`, `libgcc_s.so.1` | `RUNPATH=/usr/local/lib` | ok | must bundle + `$ORIGIN` | -| `vad` | `libsherpa-onnx-c-api.so`, `libgcc_s.so.1` | `RUNPATH=/usr/local/lib` | ok | must bundle + `$ORIGIN` | -| `whisper` | `libstdc++.so.6`, `libgcc_s.so.1` | — | ok | system dependency (accepted) | - -## Proposed v1 stance (decision checkpoint) - -Decision: option 3 (mix). - -- Bundle sherpa-onnx shared libs with official bundles and set `RUNPATH=$ORIGIN` (or equivalent). -- Rely on system OpenSSL (libssl/libcrypto) and GCC runtime (libstdc++/libgcc_s). -- When present, `pocket-tts` may rely on system OpenSSL 3. diff --git a/marketplace/official-plugins.json b/marketplace/official-plugins.json index 49272d4f..cd99beef 100644 --- a/marketplace/official-plugins.json +++ b/marketplace/official-plugins.json @@ -3,6 +3,7 @@ { "id": "helsinki", "name": "Helsinki", + "version": "0.1.0", "node_kind": "helsinki", "kind": "native", "entrypoint": "libhelsinki.so", @@ -41,6 +42,7 @@ { "id": "kokoro", "name": "Kokoro", + "version": "0.1.0", "node_kind": "kokoro", "kind": "native", "entrypoint": "libkokoro.so", @@ -67,6 +69,7 @@ { "id": "matcha", "name": "Matcha", + "version": "0.1.0", "node_kind": "matcha", "kind": "native", "entrypoint": "libmatcha.so", @@ -93,6 +96,7 @@ { "id": "nllb", "name": "NLLB", + "version": "0.1.0", "node_kind": "nllb", "kind": "native", "entrypoint": "libnllb.so", @@ -118,6 +122,7 @@ { "id": "piper", "name": "Piper", + "version": "0.1.0", "node_kind": "piper", "kind": "native", "entrypoint": "libpiper.so", @@ -158,6 +163,7 @@ { "id": "sensevoice", "name": "SenseVoice", + "version": "0.1.0", "node_kind": "sensevoice", "kind": "native", "entrypoint": "libsensevoice.so", @@ -198,6 +204,7 @@ { "id": "vad", "name": "VAD", + "version": "0.1.0", "node_kind": "vad", "kind": "native", "entrypoint": "libvad.so", @@ -224,6 +231,7 @@ { "id": "whisper", "name": "Whisper", + "version": "0.1.1", "node_kind": "whisper", "kind": "native", "entrypoint": "libwhisper.so", diff --git a/plugins/native/helsinki/marketplace.yml b/plugins/native/helsinki/plugin.yml similarity index 98% rename from plugins/native/helsinki/marketplace.yml rename to plugins/native/helsinki/plugin.yml index 89d0ca9c..c193039c 100644 --- a/plugins/native/helsinki/marketplace.yml +++ b/plugins/native/helsinki/plugin.yml @@ -1,5 +1,6 @@ id: helsinki name: Helsinki +version: 0.1.0 node_kind: helsinki kind: native entrypoint: libhelsinki.so diff --git a/plugins/native/kokoro/marketplace.yml b/plugins/native/kokoro/plugin.yml similarity index 97% rename from plugins/native/kokoro/marketplace.yml rename to plugins/native/kokoro/plugin.yml index 413427ac..e3ecf113 100644 --- a/plugins/native/kokoro/marketplace.yml +++ b/plugins/native/kokoro/plugin.yml @@ -1,5 +1,6 @@ id: kokoro name: Kokoro +version: 0.1.0 node_kind: kokoro kind: native entrypoint: libkokoro.so diff --git a/plugins/native/matcha/marketplace.yml b/plugins/native/matcha/plugin.yml similarity index 97% rename from plugins/native/matcha/marketplace.yml rename to plugins/native/matcha/plugin.yml index fe8089f3..9282dc0b 100644 --- a/plugins/native/matcha/marketplace.yml +++ b/plugins/native/matcha/plugin.yml @@ -1,5 +1,6 @@ id: matcha name: Matcha +version: 0.1.0 node_kind: matcha kind: native entrypoint: libmatcha.so diff --git a/plugins/native/nllb/marketplace.yml b/plugins/native/nllb/plugin.yml similarity index 97% rename from plugins/native/nllb/marketplace.yml rename to plugins/native/nllb/plugin.yml index b5e6aa3c..aff20609 100644 --- a/plugins/native/nllb/marketplace.yml +++ b/plugins/native/nllb/plugin.yml @@ -1,5 +1,6 @@ id: nllb name: NLLB +version: 0.1.0 node_kind: nllb kind: native entrypoint: libnllb.so diff --git a/plugins/native/piper/marketplace.yml b/plugins/native/piper/plugin.yml similarity index 98% rename from plugins/native/piper/marketplace.yml rename to plugins/native/piper/plugin.yml index 1541565a..c20559b7 100644 --- a/plugins/native/piper/marketplace.yml +++ b/plugins/native/piper/plugin.yml @@ -1,5 +1,6 @@ id: piper name: Piper +version: 0.1.0 node_kind: piper kind: native entrypoint: libpiper.so diff --git a/plugins/native/sensevoice/marketplace.yml b/plugins/native/sensevoice/plugin.yml similarity index 98% rename from plugins/native/sensevoice/marketplace.yml rename to plugins/native/sensevoice/plugin.yml index c5d3bcbc..2c4748eb 100644 --- a/plugins/native/sensevoice/marketplace.yml +++ b/plugins/native/sensevoice/plugin.yml @@ -1,5 +1,6 @@ id: sensevoice name: SenseVoice +version: 0.1.0 node_kind: sensevoice kind: native entrypoint: libsensevoice.so diff --git a/plugins/native/vad/marketplace.yml b/plugins/native/vad/plugin.yml similarity index 97% rename from plugins/native/vad/marketplace.yml rename to plugins/native/vad/plugin.yml index 6b1b9f5b..a8ab9637 100644 --- a/plugins/native/vad/marketplace.yml +++ b/plugins/native/vad/plugin.yml @@ -1,5 +1,6 @@ id: vad name: VAD +version: 0.1.0 node_kind: vad kind: native entrypoint: libvad.so diff --git a/plugins/native/whisper/marketplace.yml b/plugins/native/whisper/plugin.yml similarity index 99% rename from plugins/native/whisper/marketplace.yml rename to plugins/native/whisper/plugin.yml index 33bd388a..8cf9186f 100644 --- a/plugins/native/whisper/marketplace.yml +++ b/plugins/native/whisper/plugin.yml @@ -1,5 +1,6 @@ id: whisper name: Whisper +version: 0.1.1 node_kind: whisper kind: native entrypoint: libwhisper.so diff --git a/scripts/marketplace/build_registry.py b/scripts/marketplace/build_registry.py index 0fb48140..6c8d74bc 100644 --- a/scripts/marketplace/build_registry.py +++ b/scripts/marketplace/build_registry.py @@ -4,6 +4,7 @@ import argparse import datetime +import difflib import hashlib import json import os @@ -140,6 +141,37 @@ def strip_none(payload: dict) -> dict: return {key: value for key, value in payload.items() if value is not None} +def dump_manifest_bytes(manifest: dict) -> bytes: + """Produce canonical manifest bytes matching write_json formatting.""" + return (json.dumps(manifest, indent=2, sort_keys=False) + "\n").encode("utf-8") + + +def build_manifest( + plugin: dict, + plugin_version: str, + bundle_block: dict, +) -> dict: + """Build manifest dict from plugin metadata and bundle info.""" + manifest = { + "schema_version": 1, + "id": plugin["id"], + "name": plugin.get("name"), + "version": plugin_version, + "node_kind": plugin["node_kind"], + "kind": plugin["kind"], + "description": plugin.get("description"), + "license": plugin.get("license"), + "license_url": plugin.get("license_url"), + "homepage": plugin.get("homepage"), + "repository": plugin.get("repository"), + "entrypoint": plugin["entrypoint"], + "bundle": bundle_block, + "compatibility": plugin.get("compatibility"), + "models": plugin.get("models", []), + } + return strip_none(manifest) + + def sign_manifest(manifest_path: pathlib.Path, signing_key: pathlib.Path) -> pathlib.Path: signature_path = manifest_path.with_name("manifest.minisig") subprocess.run( @@ -158,10 +190,114 @@ def sign_manifest(manifest_path: pathlib.Path, signing_key: pathlib.Path) -> pat return signature_path +def is_prerelease(version: str) -> bool: + """Check if version has prerelease identifier (before any +build).""" + # Strip build metadata first + if "+" in version: + version = version.split("+", 1)[0] + return "-" in version + + +def parse_semver_key(version: str) -> tuple: + """ + Parse SemVer into a sortable key tuple. + Returns (major, minor, patch, is_stable, prerelease_parts). + + Per SemVer 2.0.0: + - Build metadata (+...) is ignored for precedence + - Prerelease versions have lower precedence than normal versions + - Prerelease identifiers are compared by: + * Numeric identifiers are compared as integers + * Alphanumeric identifiers are compared lexically + * Numeric identifiers have lower precedence than non-numeric + """ + # Strip build metadata (everything after +) + if "+" in version: + version = version.split("+", 1)[0] + + # Split into base version and prerelease + if "-" in version: + base, prerelease = version.split("-", 1) + is_stable = False + else: + base, prerelease = version, "" + is_stable = True + + # Parse base version + parts = base.split(".") + if len(parts) != 3: + raise ValueError(f"Invalid semver base: {version}") + try: + major, minor, patch = map(int, parts) + except ValueError as exc: + raise ValueError(f"Invalid semver numbers in: {version}") from exc + + # Parse prerelease identifiers + prerelease_parts = [] + if prerelease: + for part in prerelease.split("."): + # Try to parse as int, otherwise keep as string + try: + # Numeric identifier + prerelease_parts.append((0, int(part))) + except ValueError: + # Alphanumeric identifier + prerelease_parts.append((1, part)) + + # Return sortable key: + # - (major, minor, patch) compares numerically + # - is_stable=True sorts higher than is_stable=False for same base version + # - prerelease_parts compares element-wise per SemVer rules + return (major, minor, patch, is_stable, prerelease_parts) + + +def load_existing_registry(registry_path: pathlib.Path) -> tuple[dict[tuple[str, str], dict], dict]: + """ + Load existing registry and return: + - Map of (plugin_id, version) -> {manifest, signature_path} + - Map of plugin_id -> {versions: [...], metadata} + """ + existing = {} + plugins_dir = registry_path / "plugins" + if not plugins_dir.exists(): + return existing, {} + + # Load manifests and signatures + for plugin_dir in plugins_dir.iterdir(): + if not plugin_dir.is_dir(): + continue + plugin_id = plugin_dir.name + + for version_dir in plugin_dir.iterdir(): + if not version_dir.is_dir(): + continue + version = version_dir.name + + manifest_path = version_dir / "manifest.json" + signature_path = version_dir / "manifest.minisig" + + if manifest_path.exists() and signature_path.exists(): + manifest = json.loads(manifest_path.read_text()) + existing[(plugin_id, version)] = { + "manifest": manifest, + "manifest_path": manifest_path, + "signature_path": signature_path, + } + + # Load index.json for published_at timestamps and metadata + index_map = {} + index_path = registry_path / "index.json" + if index_path.exists(): + index_data = json.loads(index_path.read_text()) + for plugin in index_data.get("plugins", []): + index_map[plugin["id"]] = plugin + + return existing, index_map + + def main() -> int: parser = argparse.ArgumentParser() parser.add_argument("--plugins", required=True, help="Path to plugin metadata JSON") - parser.add_argument("--version", required=True, help="Release version (e.g., 1.2.3)") parser.add_argument( "--bundle-base-url", required=True, help="Base URL for bundle downloads" ) @@ -171,12 +307,16 @@ def main() -> int: parser.add_argument("--bundles-out", required=True, help="Output directory for bundles") parser.add_argument("--registry-out", required=True, help="Output directory for registry JSON") parser.add_argument("--signing-key", required=True, help="Path to minisign secret key") + parser.add_argument( + "--existing-registry", + help="Path to existing registry directory (for append-only mode)", + ) + parser.add_argument( + "--public-key", + help="Path to minisign public key to include in registry (default: docs/public/registry/streamkit.pub if exists)", + ) args = parser.parse_args() - if not args.version.strip(): - print("Release version must be non-empty", file=sys.stderr) - return 1 - plugins_path = pathlib.Path(args.plugins) bundles_out = pathlib.Path(args.bundles_out) registry_out = pathlib.Path(args.registry_out) @@ -189,6 +329,15 @@ def main() -> int: print(f"Missing minisign key: {signing_key}", file=sys.stderr) return 1 + # Load existing registry if provided + existing_registry = {} + existing_index_map = {} + if args.existing_registry: + existing_registry_path = pathlib.Path(args.existing_registry) + if existing_registry_path.exists(): + existing_registry, existing_index_map = load_existing_registry(existing_registry_path) + print(f"Loaded {len(existing_registry)} existing plugin versions from registry") + metadata = json.loads(plugins_path.read_text()) plugins = metadata.get("plugins", []) if not plugins: @@ -199,65 +348,188 @@ def main() -> int: registry_base_url = normalize_base_url(args.registry_base_url) published_at = datetime.date.today().isoformat() - registry_plugins = [] + # Track all versions per plugin for index.json + plugin_versions_map = {} # plugin_id -> list of version entries + work_root = registry_out / ".work" if work_root.exists(): shutil.rmtree(work_root) for plugin in plugins: - bundle_info = build_bundle(plugin, args.version, bundles_out, work_root) - manifest = { - "schema_version": 1, - "id": plugin["id"], - "name": plugin.get("name"), - "version": args.version, - "node_kind": plugin["node_kind"], - "kind": plugin["kind"], - "description": plugin.get("description"), - "license": plugin.get("license"), - "license_url": plugin.get("license_url"), - "homepage": plugin.get("homepage"), - "repository": plugin.get("repository"), - "entrypoint": plugin["entrypoint"], - "bundle": { + plugin_id = plugin["id"] + plugin_version = plugin.get("version") + if not plugin_version: + print(f"ERROR: Plugin {plugin_id} missing version field", file=sys.stderr) + return 1 + + key = (plugin_id, plugin_version) + + # Check if this version already exists in the registry + if key in existing_registry: + # Verify immutability: check if republishing with same version would change manifest + existing = existing_registry[key] + existing_manifest = existing["manifest"] + + # Build would-be manifest using current plugin fields but existing bundle + would_be_manifest = build_manifest( + plugin, + plugin_version, + existing_manifest["bundle"], + ) + + # Compare parsed JSON objects (robust to formatting differences like trailing newlines) + if existing_manifest != would_be_manifest: + print( + f"ERROR: {plugin_id}@{plugin_version} already exists in registry " + f"but manifest content would change; bump plugin.yml version.", + file=sys.stderr, + ) + print(f"Existing manifest: {existing['manifest_path']}", file=sys.stderr) + # Show diff for debugging + existing_json = json.dumps(existing_manifest, indent=2, sort_keys=False) + would_be_json = json.dumps(would_be_manifest, indent=2, sort_keys=False) + diff = difflib.unified_diff( + existing_json.splitlines(keepends=True), + would_be_json.splitlines(keepends=True), + fromfile="existing", + tofile="would-be", + ) + print("Manifest differences:", file=sys.stderr) + print("".join(diff), file=sys.stderr) + return 1 + + print(f"Reusing existing {plugin_id} v{plugin_version}") + + # Copy forward existing manifest and signature + manifest_dir = registry_out / "plugins" / plugin_id / plugin_version + manifest_path = manifest_dir / "manifest.json" + signature_path = manifest_dir / "manifest.minisig" + + ensure_dir(manifest_dir) + shutil.copy2(existing["manifest_path"], manifest_path) + shutil.copy2(existing["signature_path"], signature_path) + else: + # Build new version + bundle_info = build_bundle(plugin, plugin_version, bundles_out, work_root) + bundle_block = { "url": f"{bundle_base_url}/{bundle_info['bundle_name']}", "sha256": bundle_info["sha256"], "size_bytes": bundle_info["size_bytes"], - }, - "compatibility": plugin.get("compatibility"), - "models": plugin.get("models", []), - } - manifest = strip_none(manifest) + } + manifest = build_manifest(plugin, plugin_version, bundle_block) + + manifest_dir = registry_out / "plugins" / plugin_id / plugin_version + manifest_path = manifest_dir / "manifest.json" + write_json(manifest_path, manifest) + sign_manifest(manifest_path, signing_key) + + print( + f"Built {plugin_id} v{plugin_version} -> {bundle_info['bundle_name']} ({bundle_info['sha256']})" + ) + + # Build index.json by merging all versions (existing + new) + # First, collect all versions from existing registry + for (plugin_id, version), existing in existing_registry.items(): + if plugin_id not in plugin_versions_map: + plugin_versions_map[plugin_id] = [] + + # Get published_at from existing index.json if available + existing_published_at = published_at + if plugin_id in existing_index_map: + for ver_entry in existing_index_map[plugin_id].get("versions", []): + if ver_entry.get("version") == version: + existing_published_at = ver_entry.get("published_at", published_at) + break + + plugin_versions_map[plugin_id].append( + { + "version": version, + "manifest_url": f"{registry_base_url}/plugins/{plugin_id}/{version}/manifest.json", + "signature_url": f"{registry_base_url}/plugins/{plugin_id}/{version}/manifest.minisig", + "published_at": existing_published_at, + } + ) - manifest_dir = registry_out / "plugins" / plugin["id"] / args.version - manifest_path = manifest_dir / "manifest.json" - write_json(manifest_path, manifest) - signature_path = sign_manifest(manifest_path, signing_key) + # Add current plugins (may be new or update existing entries) + for plugin in plugins: + plugin_id = plugin["id"] + plugin_version = plugin["version"] + + if plugin_id not in plugin_versions_map: + plugin_versions_map[plugin_id] = [] + + # Check if this version is already in the list + already_exists = any(v["version"] == plugin_version for v in plugin_versions_map[plugin_id]) + if not already_exists: + plugin_versions_map[plugin_id].append( + { + "version": plugin_version, + "manifest_url": f"{registry_base_url}/plugins/{plugin_id}/{plugin_version}/manifest.json", + "signature_url": f"{registry_base_url}/plugins/{plugin_id}/{plugin_version}/manifest.minisig", + "published_at": published_at, + } + ) + + # Build final index with sorted versions and computed latest + # Include all plugins that have versions in plugin_versions_map + plugin_metadata = {p["id"]: p for p in plugins} + registry_plugins = [] + + for plugin_id in sorted(plugin_versions_map.keys()): + versions = plugin_versions_map[plugin_id] + + # Sort versions by semver (highest precedence first) + versions.sort(key=lambda v: parse_semver_key(v["version"]), reverse=True) + + # Determine latest: prefer max stable version, otherwise max prerelease + stable_versions = [v for v in versions if not is_prerelease(v["version"])] + if stable_versions: + latest = stable_versions[0]["version"] + elif versions: + latest = versions[0]["version"] + else: + # Fallback (shouldn't happen) + latest = versions[0]["version"] if versions else "0.0.0" + + # Get plugin metadata from current plugins or existing registry + plugin_meta = plugin_metadata.get(plugin_id, {}) + if not plugin_meta and existing_registry: + # Try to get metadata from first existing version + for (pid, ver), existing in existing_registry.items(): + if pid == plugin_id: + plugin_meta = existing["manifest"] + break registry_plugins.append( { - "id": plugin["id"], - "name": plugin.get("name"), - "description": plugin.get("description"), - "latest": args.version, - "versions": [ - { - "version": args.version, - "manifest_url": f"{registry_base_url}/plugins/{plugin['id']}/{args.version}/manifest.json", - "signature_url": f"{registry_base_url}/plugins/{plugin['id']}/{args.version}/manifest.minisig", - "published_at": published_at, - } - ], + "id": plugin_id, + "name": plugin_meta.get("name", plugin_id), + "description": plugin_meta.get("description"), + "latest": latest, + "versions": versions, } ) - print( - f"Built bundle for {plugin['id']} -> {bundle_info['bundle_name']} ({bundle_info['sha256']})" - ) - index = {"schema_version": 1, "plugins": registry_plugins} write_json(registry_out / "index.json", index) + # Copy public key if available + public_key_path = None + if args.public_key: + public_key_path = pathlib.Path(args.public_key) + else: + # Try default location + default_key = pathlib.Path("docs/public/registry/streamkit.pub") + if default_key.exists(): + public_key_path = default_key + + if public_key_path and public_key_path.exists(): + dest_key = registry_out / "streamkit.pub" + shutil.copy2(public_key_path, dest_key) + print(f"Copied public key to registry: {dest_key}") + elif args.public_key: + print(f"WARNING: Specified public key not found: {args.public_key}", file=sys.stderr) + if work_root.exists(): shutil.rmtree(work_root) diff --git a/scripts/marketplace/generate_official_plugins.py b/scripts/marketplace/generate_official_plugins.py index 8d05da46..b097b812 100644 --- a/scripts/marketplace/generate_official_plugins.py +++ b/scripts/marketplace/generate_official_plugins.py @@ -17,8 +17,21 @@ def load_yaml(path: pathlib.Path) -> dict: return yaml.safe_load(path.read_text()) +def load_toml(path: pathlib.Path) -> dict: + try: + import tomllib + except ImportError: + try: + import tomli as tomllib + except ImportError as exc: + raise RuntimeError( + "tomli is required for Python < 3.11. Install python3-tomli or pip install tomli." + ) from exc + return tomllib.loads(path.read_text()) + + def validate_plugin(plugin: dict, plugin_dir: pathlib.Path) -> dict: - required = ["id", "name", "node_kind", "kind", "entrypoint", "artifact", "description", "license"] + required = ["id", "name", "node_kind", "kind", "entrypoint", "artifact", "description", "license", "version"] missing = [key for key in required if not plugin.get(key)] if missing: raise ValueError(f"{plugin_dir}: missing required fields: {', '.join(missing)}") @@ -26,9 +39,26 @@ def validate_plugin(plugin: dict, plugin_dir: pathlib.Path) -> dict: raise ValueError( f"{plugin_dir}: id '{plugin['id']}' must match directory name" ) + + # For native plugins, enforce version matches Cargo.toml + if plugin["kind"] == "native": + cargo_toml_path = plugin_dir / "Cargo.toml" + if not cargo_toml_path.exists(): + raise ValueError(f"{plugin_dir}: native plugin missing Cargo.toml") + cargo_data = load_toml(cargo_toml_path) + cargo_version = cargo_data.get("package", {}).get("version") + if not cargo_version: + raise ValueError(f"{plugin_dir}: Cargo.toml missing [package].version") + if plugin["version"] != cargo_version: + raise ValueError( + f"{plugin_dir}: version mismatch; plugin.yml has '{plugin['version']}' " + f"but Cargo.toml has '{cargo_version}'. Update both or keep them aligned." + ) + ordered_keys = [ "id", "name", + "version", "node_kind", "kind", "entrypoint", @@ -62,11 +92,27 @@ def main() -> int: for plugin_dir in sorted(plugins_root.iterdir()): if not plugin_dir.is_dir(): continue - metadata_path = plugin_dir / "marketplace.yml" - if not metadata_path.exists(): - metadata_path = plugin_dir / "marketplace.yaml" - if not metadata_path.exists(): - errors.append(f"Missing {plugin_dir}/marketplace.yml") + # Search order: plugin.yml, plugin.yaml, then deprecated marketplace.yml/yaml + metadata_path = None + is_deprecated = False + for candidate in ["plugin.yml", "plugin.yaml"]: + candidate_path = plugin_dir / candidate + if candidate_path.exists(): + metadata_path = candidate_path + break + if not metadata_path: + for candidate in ["marketplace.yml", "marketplace.yaml"]: + candidate_path = plugin_dir / candidate + if candidate_path.exists(): + metadata_path = candidate_path + is_deprecated = True + print( + f"WARNING: {metadata_path} uses deprecated filename; rename to plugin.yml", + file=sys.stderr, + ) + break + if not metadata_path: + errors.append(f"Missing {plugin_dir}/plugin.yml") continue try: data = load_yaml(metadata_path) or {} diff --git a/scripts/marketplace/test_append_only.py b/scripts/marketplace/test_append_only.py new file mode 100755 index 00000000..10cfde90 --- /dev/null +++ b/scripts/marketplace/test_append_only.py @@ -0,0 +1,262 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: © 2025 StreamKit Contributors +# SPDX-License-Identifier: MPL-2.0 + +""" +Lightweight test for append-only registry behavior. +Tests immutability enforcement and reuse of existing versions. +""" + +import json +import pathlib +import shutil +import subprocess +import sys +import tempfile + + +def setup_test_registry(registry_path: pathlib.Path) -> None: + """Create a minimal existing registry with one plugin version.""" + plugin_id = "test-plugin" + version = "0.1.0" + + # Create directory structure + manifest_dir = registry_path / "plugins" / plugin_id / version + manifest_dir.mkdir(parents=True, exist_ok=True) + + # Create manifest.json + manifest = { + "schema_version": 1, + "id": plugin_id, + "name": "Test Plugin", + "version": version, + "node_kind": "test", + "kind": "native", + "description": "Test plugin for append-only registry", + "license": "MPL-2.0", + "entrypoint": "libtest.so", + "bundle": { + "url": "https://example.com/test-plugin-0.1.0-bundle.tar.zst", + "sha256": "abcd1234" * 8, + "size_bytes": 1024, + }, + "models": [], + } + manifest_path = manifest_dir / "manifest.json" + manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=False) + "\n") + + # Create dummy signature + signature_path = manifest_dir / "manifest.minisig" + signature_path.write_text("dummy signature\n") + + # Create index.json + index = { + "schema_version": 1, + "plugins": [ + { + "id": plugin_id, + "name": "Test Plugin", + "description": "Test plugin for append-only registry", + "latest": version, + "versions": [ + { + "version": version, + "manifest_url": f"https://example.com/registry/plugins/{plugin_id}/{version}/manifest.json", + "signature_url": f"https://example.com/registry/plugins/{plugin_id}/{version}/manifest.minisig", + "published_at": "2025-01-01", + } + ], + } + ], + } + index_path = registry_path / "index.json" + index_path.write_text(json.dumps(index, indent=2, sort_keys=False) + "\n") + + # Create public key + pubkey_path = registry_path / "streamkit.pub" + pubkey_path.write_text("dummy public key\n") + + +def create_test_plugin_metadata( + plugins_path: pathlib.Path, description: str = "Test plugin for append-only registry" +) -> None: + """Create plugin metadata JSON.""" + metadata = { + "plugins": [ + { + "id": "test-plugin", + "name": "Test Plugin", + "version": "0.1.0", + "node_kind": "test", + "kind": "native", + "entrypoint": "libtest.so", + "artifact": "/tmp/nonexistent.so", # Won't be used in reuse scenario + "description": description, + "license": "MPL-2.0", + "models": [], + } + ] + } + plugins_path.write_text(json.dumps(metadata, indent=2) + "\n") + + +def test_identical_reuse(tmp_dir: pathlib.Path) -> bool: + """Test that identical plugin metadata reuses existing version without error.""" + print("\n=== Test 1: Identical metadata should reuse existing version ===") + + existing_registry = tmp_dir / "existing_registry" + setup_test_registry(existing_registry) + + plugins_json = tmp_dir / "plugins.json" + create_test_plugin_metadata(plugins_json) + + output_registry = tmp_dir / "output_registry" + bundles_out = tmp_dir / "bundles" + dummy_key = tmp_dir / "dummy.key" + dummy_key.write_text("dummy signing key\n") + + # Point to the test's public key, not the repo's + public_key = existing_registry / "streamkit.pub" + + # Run build_registry in skip-signing mode (we'll mock this by using a dummy key) + # Since we're reusing, it won't need to sign + result = subprocess.run( + [ + "python3", + "scripts/marketplace/build_registry.py", + "--plugins", + str(plugins_json), + "--existing-registry", + str(existing_registry), + "--bundle-base-url", + "https://example.com/bundles", + "--registry-base-url", + "https://example.com/registry", + "--bundles-out", + str(bundles_out), + "--registry-out", + str(output_registry), + "--signing-key", + str(dummy_key), + "--public-key", + str(public_key), + ], + capture_output=True, + text=True, + ) + + if result.returncode != 0: + print(f"FAIL: Expected success but got exit code {result.returncode}") + print(f"STDOUT: {result.stdout}") + print(f"STDERR: {result.stderr}") + return False + + # Check that no bundles were created + if bundles_out.exists() and list(bundles_out.iterdir()): + print(f"FAIL: Expected no new bundles but found: {list(bundles_out.iterdir())}") + return False + + # Check that manifest was copied forward + manifest_path = output_registry / "plugins" / "test-plugin" / "0.1.0" / "manifest.json" + if not manifest_path.exists(): + print(f"FAIL: Manifest not found at {manifest_path}") + return False + + # Check that streamkit.pub was copied + pubkey_path = output_registry / "streamkit.pub" + if not pubkey_path.exists(): + print(f"FAIL: Public key not found at {pubkey_path}") + return False + + print("PASS: Identical metadata reused existing version correctly") + return True + + +def test_changed_metadata_fails(tmp_dir: pathlib.Path) -> bool: + """Test that changed metadata without version bump fails with immutability error.""" + print("\n=== Test 2: Changed metadata without version bump should fail ===") + + existing_registry = tmp_dir / "existing_registry" + setup_test_registry(existing_registry) + + plugins_json = tmp_dir / "plugins.json" + # Change description without bumping version + create_test_plugin_metadata(plugins_json, description="CHANGED DESCRIPTION") + + output_registry = tmp_dir / "output_registry" + bundles_out = tmp_dir / "bundles" + dummy_key = tmp_dir / "dummy.key" + dummy_key.write_text("dummy signing key\n") + + # Point to the test's public key, not the repo's + public_key = existing_registry / "streamkit.pub" + + result = subprocess.run( + [ + "python3", + "scripts/marketplace/build_registry.py", + "--plugins", + str(plugins_json), + "--existing-registry", + str(existing_registry), + "--bundle-base-url", + "https://example.com/bundles", + "--registry-base-url", + "https://example.com/registry", + "--bundles-out", + str(bundles_out), + "--registry-out", + str(output_registry), + "--signing-key", + str(dummy_key), + "--public-key", + str(public_key), + ], + capture_output=True, + text=True, + ) + + if result.returncode == 0: + print("FAIL: Expected failure but build succeeded") + print(f"STDOUT: {result.stdout}") + return False + + if "already exists in registry but manifest content would change" not in result.stderr: + print("FAIL: Expected immutability error but got different error") + print(f"STDERR: {result.stderr}") + return False + + print("PASS: Immutability check correctly rejected changed metadata") + return True + + +def main() -> int: + """Run all tests.""" + print("Running append-only registry tests...") + + with tempfile.TemporaryDirectory() as tmp_dir_str: + tmp_dir = pathlib.Path(tmp_dir_str) + + test1_dir = tmp_dir / "test1" + test1_dir.mkdir() + test1_passed = test_identical_reuse(test1_dir) + + test2_dir = tmp_dir / "test2" + test2_dir.mkdir() + test2_passed = test_changed_metadata_fails(test2_dir) + + print("\n" + "=" * 60) + if test1_passed and test2_passed: + print("✓ All tests passed!") + return 0 + else: + print("✗ Some tests failed") + if not test1_passed: + print(" - Test 1 (identical reuse) FAILED") + if not test2_passed: + print(" - Test 2 (immutability check) FAILED") + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/marketplace/verify_bundles.py b/scripts/marketplace/verify_bundles.py index 06d5a69c..bb263d58 100644 --- a/scripts/marketplace/verify_bundles.py +++ b/scripts/marketplace/verify_bundles.py @@ -37,18 +37,13 @@ def extract_bundle(bundle_path: pathlib.Path, dest: pathlib.Path) -> None: ) -def find_bundle(bundles_dir: pathlib.Path, plugin_id: str) -> pathlib.Path | None: - suffix = "-bundle.tar.zst" - matches = [ - path - for path in bundles_dir.glob("*.tar.zst") - if path.name.startswith(f"{plugin_id}-") and path.name.endswith(suffix) - ] - if len(matches) == 1: - return matches[0] - if len(matches) > 1: - raise RuntimeError(f"Multiple bundles found for {plugin_id}: {matches}") - return None +def find_bundle( + bundles_dir: pathlib.Path, plugin_id: str, version: str +) -> pathlib.Path | None: + """Find bundle for specific plugin version. Returns None if not found.""" + bundle_name = f"{plugin_id}-{version}-bundle.tar.zst" + bundle_path = bundles_dir / bundle_name + return bundle_path if bundle_path.exists() else None def main() -> int: @@ -77,10 +72,16 @@ def main() -> int: for plugin in plugins: plugin_id = plugin["id"] + plugin_version = plugin.get("version") + if not plugin_version: + errors.append(f"{plugin_id}: missing version field in metadata") + continue + entrypoint = plugin["entrypoint"] - bundle_path = find_bundle(bundles_dir, plugin_id) + bundle_path = find_bundle(bundles_dir, plugin_id, plugin_version) if bundle_path is None: - errors.append(f"Missing bundle for {plugin_id} in {bundles_dir}") + # Bundle not found - assume it was published earlier (append-only mode) + print(f"Skipping {plugin_id} v{plugin_version} (bundle not in {bundles_dir}, likely already published)") continue with tempfile.TemporaryDirectory() as tmp_dir: