dfetch-hub.toml

# dfetch-hub.toml — source definitions for dfetch-hub
#
# This file tells dfetch-hub WHERE to look for reusable components.
# It is plain TOML; one or more [[source]] blocks, each with a different
# discovery strategy.
#
# ── STRATEGY TYPES ────────────────────────────────────────────────────────────
#
#   single        A single git/svn repo treated as one component.
#   subfolders    One repo whose immediate subfolders are each a component
#                 (e.g. vcpkg ports/, conan recipes/).
#   registry      A well-known package registry whose index is scraped to
#                 yield a list of upstream repos.
#   git-regex     Clone (or ls-remote) one repo and collect all sub-repos
#                 found by following URLs that match a regex inside its files.
#   webpage-list  Scrape a web page for git/svn URLs via CSS selector or regex.
#
# ── AUTHENTICATION ─────────────────────────────────────────────────────────────
# Tokens are read from environment variables (see [auth] section below).
# Never put tokens in this file.
#
# ── CANONICAL-URL DEDUPLICATION ───────────────────────────────────────────────
# If two sources resolve to the same upstream repo URL (normalised: lowercase,
# .git suffix stripped, trailing slash stripped) dfetch-hub merges them into
# one Component and records both origins in component.catalog_sources[].
#
# ── TAG FILTERING ─────────────────────────────────────────────────────────────
# Named filters are defined in [filter.<name>] blocks and referenced from
# [[source]] blocks via the `filter` key.  Each filter has optional `include`
# and `exclude` rule lists:
#
#   include  All rules must match for a tag to be kept.
#            An empty list passes every tag through.
#   exclude  Any matching rule removes the tag.
#
# Rule fields:
#   kind   "prefix"  — tag must start with the given prefix (after normalisation)
#          "regex"   — tag must match the regular expression
#          "semver"  — tag must be a valid semantic version (vX.Y.Z or X.Y.Z,
#                      optional pre-release / build-metadata suffix)
#   value  The prefix string or regex pattern.
#          Supports {{component}} which is replaced at runtime with the
#          component name (monorepo subfolder), enabling one shared filter
#          definition to work correctly for every component in a monorepo.
#   case   "smart" (default) — split CamelCase/PascalCase, strip separators
#                              (hyphens, underscores, spaces, periods), lowercase.
#                              LowPassFilter / low-pass-filter / LOW_PASS_FILTER
#                              all compare as "lowpassfilter".
#          "insensitive"     — lowercase only; separators preserved.
#          "sensitive"       — exact byte-level comparison; no transformation.
#          "normalize-lower" — strip separators then lowercase (no CamelCase split).
#          "normalize-upper" — strip separators then uppercase (no CamelCase split).
#
# ══════════════════════════════════════════════════════════════════════════════

# Optional global settings
[settings]
concurrency = 8          # parallel HTTP / git fetches
catalog_path = "dfetch_hub/data/catalog.json"
output_dir   = "site"

# Authentication (values are env-var NAMES, not the tokens themselves)
[auth]
github_token = "DFETCH_HUB_GITHUB_TOKEN"
gitlab_token = "DFETCH_HUB_GITLAB_TOKEN"
svn_username = "DFETCH_HUB_SVN_USER"
svn_password = "DFETCH_HUB_SVN_PASS"

# ── EXAMPLE: single repo ──────────────────────────────────────────────────────
# [[source]]
# name     = "dfetch"
# strategy = "single"
# url      = "https://github.com/dfetch-org/dfetch"
# label    = "github"

# ── EXAMPLE: subfolders — vcpkg ports ─────────────────────────────────────────
# Each subdirectory of `ports/` is treated as a separate component.
# dfetch-hub reads portfile.cmake and vcpkg.json to discover the upstream URL,
# then fetches metadata from that upstream repo.
[[source]]
name        = "vcpkg"
strategy    = "subfolders"
url         = "https://github.com/microsoft/vcpkg"
path        = "ports"          # which subfolder to enumerate (default: repo root)
manifest    = "vcpkg.json"     # which file inside each subfolder holds metadata
label       = "vcpkg"          # tag added to every component found here

# # ── EXAMPLE: subfolders — conan-center-index ──────────────────────────────────
[[source]]
name        = "conan-center"
strategy    = "subfolders"
url         = "https://github.com/conan-io/conan-center-index"
path        = "recipes"
manifest    = "conandata.yml"
label       = "conan"

[[source]]
name     = "clib"
strategy = "catalog-file"
# The GitHub wiki is a regular git repo at <repo>.wiki.git
url      = "https://github.com/clibs/clib.wiki.git"
# Which file in the repo holds the package index (only this file is fetched)
manifest = "Packages.md"
label    = "clib"

# # ── EXAMPLE: registry — ESP-IDF component manager ─────────────────────────────
# # A known registry type; dfetch-hub knows how to talk to its API.
# [[source]]
# name        = "esp-idf-components"
# strategy    = "registry"
# registry    = "idf-component-manager"   # built-in scraper id
# # API endpoint override (optional; dfetch-hub has the default baked in):
# # endpoint  = "https://components.espressif.com/api/..."
# label       = "esp-idf"

# ── EXAMPLE: registry — npm ───────────────────────────────────────────────────
# [[source]]
# name     = "my-npm-scope"
# strategy = "registry"
# registry = "npm"
# # Filter by scope, keyword, or author:
# filter   = "scope:@my-org"
# label    = "npm"

# # ── EXAMPLE: git-regex — find all repos referenced in a config file ───────────
# Clone (shallow) one "index repo" and extract all git URLs from files
# matching `file_glob`. Useful for Awesome-lists, PlatformIO registries, etc.
# [[source]]
# name       = "awesome-embedded"
# strategy   = "git-regex"
# url        = "https://github.com/nhivp/Awesome-Embedded"
# file_glob  = "README.md"
# url_regex  = "https://github\\.com/[\\w.-]+/[\\w.-]+"
# label      = "awesome-embedded"
# # Optionally restrict to repos whose path matches:
# # path_filter = ".*"

# ── EXAMPLE: subfolders — monorepo with readme manifests ───────────────────────
# A monorepo where packages live in a `packages/` subfolder, each with a README.
# The `manifest = "readme"` tells dfetch-hub to scan for README.* files.
[[source]]
name        = "ts-monorepo"
strategy    = "subfolders"
url         = "https://github.com/bakeruk/modern-typescript-monorepo-example"
path        = "packages"
manifest    = "readme"
label       = "ts-monorepo"

# ── EXAMPLE: catalog-file — Zephyr project dependencies ─────────────────────
# Parses the west.yml in the Zephyr RTOS repository and catalogues every
# project listed in manifest.projects (HAL modules, MCUboot, mbedTLS, …).
# Each project becomes a separate catalog entry pointing to its upstream repo.
# Uses strategy='catalog-file' with manifest='west.yml' — only that single file
# is fetched; the manifest field selects the parser
# (west.yml → WestProject; Packages.md → CLibPackage, …).
[[source]]
name     = "zephyr-west"
strategy = "catalog-file"
url      = "https://github.com/zephyrproject-rtos/zephyr"
manifest = "west.yml"
label    = "zephyr"

# ── EXAMPLE: tag filtering — semver-only tags ────────────────────────────────
# Reject non-release tags (latest, main, nightly, …) and keep only tags that
# parse as semantic versions.
#
# [filter.semver-only]
# include = [{kind = "semver"}]
#
# [[source]]
# name     = "my-lib"
# strategy = "subfolders"
# url      = "https://github.com/example/my-lib"
# filter   = "semver-only"

# ── EXAMPLE: tag filtering — monorepo per-component tags ─────────────────────
# A monorepo where each component tags its own releases as "audio/v1.2.3",
# "video/v2.0.0", etc.  The {{component}} placeholder is substituted at runtime
# with the actual subfolder name so a single shared filter works for every
# component without duplication.
#
# [filter.monorepo]
# include = [
#   {kind = "prefix", value = "{{component}}/", case = "sensitive"},
#   {kind = "semver", value = ""},
# ]
# exclude = [{kind = "regex", value = "-(alpha|beta|rc)\\d*$"}]
#
# [[source]]
# name     = "audio-video-sdk"
# strategy = "subfolders"
# url      = "https://github.com/example/av-sdk"
# path     = "components"
# manifest = "readme"
# filter   = "monorepo"

# ── EXAMPLE: tag filtering — exclude pre-release and nightly tags ─────────────
# Keep all semver tags except pre-release and nightly/date-based builds.
#
# [filter.stable-only]
# include = [{kind = "semver"}]
# exclude = [
#   {kind = "regex", value = "-(alpha|beta|rc|nightly|dev)"},
#   {kind = "regex", value = "^\\d{8}"},   # date-stamped builds like 20240101
# ]

# # ── EXAMPLE: SVN — single repo ────────────────────────────────────────────────
# [[source]]
# name     = "my-svn-lib"
# strategy = "single"
# url      = "svn+ssh://svn.example.com/repos/mylib"
# vcs      = "svn"    # explicit; otherwise inferred from URL scheme

# # ── EXAMPLE: SVN — subfolders (classic SVN layout with tags/trunk/branches) ───
# [[source]]
# name     = "svn-monorepo"
# strategy = "subfolders"
# url      = "https://svn.example.com/repos/components"
# vcs      = "svn"
# path     = "trunk/libs"
# label    = "internal-svn"