Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 4 additions & 8 deletions .github/workflows/claude.yml
Original file line number Diff line number Diff line change
Expand Up @@ -69,8 +69,8 @@ jobs:
with:
claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}
github_token: ${{ secrets.GITHUB_TOKEN }}
allowed_non_write_users: Copilot,copilot,jules[bot],jules
allowed_bots: "github-actions[bot],copilot[bot],dependabot[bot],copilot,github-actions,gemini[bot],claude[bot],jules[bot]"
allowed_non_write_users: Copilot
allowed_bots: "github-actions[bot],copilot[bot],dependabot[bot],copilot,github-actions,gemini[bot],claude[bot]"
trigger_phrase: "@claude"
Comment on lines 70 to 74
assignee_trigger: claude[bot]
label_trigger: claude
Expand Down Expand Up @@ -105,8 +105,8 @@ jobs:
with:
claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}
github_token: ${{ secrets.GITHUB_TOKEN }}
allowed_non_write_users: Copilot,copilot,jules[bot],jules
allowed_bots: "github-actions[bot],copilot[bot],dependabot[bot],copilot,github-actions,gemini[bot],claude[bot],jules[bot]"
allowed_non_write_users: Copilot
allowed_bots: "github-actions[bot],copilot[bot],dependabot[bot],copilot,github-actions,gemini[bot],claude[bot]"
trigger_phrase: "@claude"
assignee_trigger: claude
label_trigger: claude
Expand Down Expand Up @@ -142,8 +142,6 @@ jobs:
with:
claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}
github_token: ${{ secrets.GITHUB_TOKEN }}
allowed_non_write_users: Copilot,copilot,jules[bot],jules
allowed_bots: "github-actions[bot],copilot[bot],dependabot[bot],copilot,github-actions,gemini[bot],claude[bot],jules[bot]"
trigger_phrase: "@claude"
assignee_trigger: claude
label_trigger: claude
Expand Down Expand Up @@ -181,8 +179,6 @@ jobs:
with:
claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}
github_token: ${{ secrets.GITHUB_TOKEN }}
allowed_non_write_users: Copilot,copilot,jules[bot],jules
allowed_bots: "github-actions[bot],copilot[bot],dependabot[bot],copilot,github-actions,gemini[bot],claude[bot],jules[bot]"
trigger_phrase: "@claude"
assignee_trigger: claude
label_trigger: claude
Expand Down
4 changes: 2 additions & 2 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -229,5 +229,5 @@ test-results.xml
mise.local.toml
mise.local.env

.exportify/
!.exportify/config.toml
.gemini/
gha-creds-*.json
20 changes: 10 additions & 10 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,7 @@ dependencies = [
# So we pin it to make sure we don't break on minor releases
"pydantic==2.12.5",
# for, you know, platform dirs
"platformdirs>=4.9.4",
"platformdirs>=4.9.2",
# psutil used for resource governance/limiting by engine
"psutil>=7.2.2",
"textcase>=0.4.5",
Expand All @@ -140,20 +140,20 @@ dependencies = [
# for local providers (sentence-transformers, fastembed) to detect CPU/GPU features
"py-cpuinfo>=9.0.0",
# * ================ CLI Dependencies ==================*
"cyclopts>=4.10.0",
"rich>=14.3.3",
"cyclopts>=4.5.1",
"rich>=14.3.0",
# * ================ Provider Clients ==================*
# we must pin these to specific versions to ensure compatibility with our ClientOptions subclasses
"boto3==1.42.19",
"cohere==5.20.7",
"cohere==5.20.1",
"fastembed==0.7.4; python_version < '3.14'",
"google-genai==1.56.0",
# NOTE: We're waiting on pydantic-ai to update to 1.0+ before we can upgrade too
"huggingface-hub>=1.7.1",
"huggingface-hub==0.36.2",
"mistralai==1.10.0",
"openai==2.28.0",
"qdrant-client==1.17.1",
"pydantic-ai-slim>=1.68.0",
"openai==2.17.0",
"qdrant-client==1.16.2",
"pydantic-ai-slim>=1.56.0",
Comment on lines 140 to +156
"sentence-transformers==5.2.0; python_version <= '3.14'",
"voyageai==0.3.7",
# * ================ Indexing and Engine ==================*
Expand All @@ -171,12 +171,12 @@ dependencies = [
# fastmcp is the core MCP server framework
"fastmcp>=2.14.5",
# just used for types but we need them at runtime for Pydantic models
"mcp>=1.23.3",
"mcp>=1.19.0",
# Runs the core admin/management server
"uvicorn[standard]>=0.40.0",
# * ================ Configuration and Settings ==================*
# pydantic-settings with toml and yaml support for config files
"pydantic-settings[toml,yaml]>=2.13.1", # Pulls: tomli>=2.0.1, pyyaml>=6.0.1
"pydantic-settings[toml,yaml]>=2.12.0", # Pulls: tomli>=2.0.1, pyyaml>=6.0.1
# For writing toml config files
"tomli-w>=1.2.0",
# * ================ Telemetry and Observability ==================*
Expand Down
2 changes: 1 addition & 1 deletion scripts/build/generate-mcp-server-json.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,9 +25,9 @@
ConfigLanguage,
EnvFormat,
Provider,
ProviderCategory,
ProviderEnvVarInfo,
ProviderEnvVars,
ProviderCategory,
SemanticSearchLanguage,
)

Expand Down
5 changes: 1 addition & 4 deletions scripts/model_data/hf-models.json
100644 → 100755
Original file line number Diff line number Diff line change
Expand Up @@ -213,7 +213,6 @@
]
}
},
"models": {},
"models": {
"Alibaba-NLP/gte-modernbert-base": {
"adapted_from": null,
Expand Down Expand Up @@ -3925,6 +3924,4 @@
"opensearch-project/opensearch-neural-sparse-encoding-doc-v2-mini"
]
}
{
"models": {}
}
}
Empty file modified scripts/model_data/hf-models.json.license
100644 → 100755
Empty file.
31 changes: 17 additions & 14 deletions scripts/model_data/mteb_to_codeweaver.py
100644 → 100755
Original file line number Diff line number Diff line change
Expand Up @@ -47,11 +47,10 @@


# make sure codeweaver is importable
sys.path.insert(0, str(Path(__file__).parent.parent.parent / "src"))
sys.path.insert(0, str(Path(__file__).parent.parent))

from codeweaver.providers.provider import Provider

from codeweaver.providers.embedding.capabilities.types import PartialCapabilities
from codeweaver.core import Provider
from codeweaver.providers import PartialCapabilities

Comment on lines 49 to 54

# TODO: Finish refactor to use these inline constants and eliminate the hf-models.json
Expand Down Expand Up @@ -100,13 +99,6 @@
Note: FastEmbed also has some aliases, but we handle those dynamically below.
"""

KNOWN_ALIASES: dict[str, dict[ModelName, ModelName]] = {"ollama": OLLAMA_ALIASES}
"""A mapping of provider names to their HF name → provider alias mappings.

Keys are provider name strings (e.g. "ollama") and values are dicts mapping HF model names
to the provider-specific alias. FastEmbed aliases are handled dynamically via get_fastembed_aliases().
"""

KNOWN_SPARSE_MODELS = {
Provider.FASTEMBED: [
"Qdrant/bm25",
Expand Down Expand Up @@ -390,7 +382,13 @@

type DataMap = dict[ModelName, SimplifiedModelMeta]

type ModelMap = dict[ModelMaker, dict[ModelName, tuple[HFModelProviders, ...]]]
type ModelMap = dict[
ModelMaker,
dict[
ModelName,
tuple[Annotated[HFModelProviders, BeforeValidator(lambda v: Provider.from_string(v))], ...],
],
]
"""A mapping of model makers to their models and the providers that support each model."""


Expand Down Expand Up @@ -501,7 +499,7 @@
def aliases(self) -> AliasMap:
"""A computed field for the alias map."""
alias_map: AliasMap = defaultdict(dict)
for provider, aliases in KNOWN_ALIASES.items():

Check failure on line 502 in scripts/model_data/mteb_to_codeweaver.py

View workflow job for this annotation

GitHub Actions / Lint / Lint and Format

ruff (F821)

scripts/model_data/mteb_to_codeweaver.py:502:34: F821 Undefined name `KNOWN_ALIASES`
for hf_name, alias in aliases.items():
alias_map[Provider.from_string(provider)][hf_name] = alias
return alias_map | {Provider.FASTEMBED: get_fastembed_aliases()}
Expand All @@ -522,24 +520,29 @@
return cls.model_validate_json(cls._json_path.read_text())


if JSON_CACHE.exists():
"""
_ROOT = RootJson.load()
DATA = _ROOT.models
MODEL_MAP_DATA = _ROOT.model_map
ALIAS_MAP_DATA = _ROOT.aliases
Comment on lines +523 to 527
SPARSE_MODELS = _ROOT.sparse_models

FLATTENED_ALIASES = _ROOT.flattened_aliases
else:
_ROOT = RootJson(models={})
DATA = {}
MODEL_MAP_DATA = {}
ALIAS_MAP_DATA = {}
SPARSE_MODELS = {}
FLATTENED_ALIASES = {}
"""


def mteb_to_capabilities(model: SimplifiedModelMeta) -> PartialCapabilities:
"""
Convert an MTEB model metadata dictionary to a PartialCapabilities object.
"""
loader = getattr(model, "loader", {})
loader = loader if isinstance(loader, dict) else {}
caps = {
"name": model["name"],
"default_dimension": model.get("embed_dim"),
Expand Down Expand Up @@ -589,12 +592,12 @@
caps["output_dimensions"] = (
(1024, 512, 256, 128, 64, 32) if "-v3" in name else (2048, 1024, 512, 256, 128)
)
aliases = [v for val in ALIAS_MAP_DATA.values() for v in val.values()]

Check failure on line 595 in scripts/model_data/mteb_to_codeweaver.py

View workflow job for this annotation

GitHub Actions / Lint / Lint and Format

ruff (F821)

scripts/model_data/mteb_to_codeweaver.py:595:29: F821 Undefined name `ALIAS_MAP_DATA`
if aliased := next((v for v in aliases if caps["name"] == v), None):
caps["hf_name"] = next((k for k, v in FLATTENED_ALIASES.items() if v == aliased), None)

Check failure on line 597 in scripts/model_data/mteb_to_codeweaver.py

View workflow job for this annotation

GitHub Actions / Lint / Lint and Format

ruff (F821)

scripts/model_data/mteb_to_codeweaver.py:597:47: F821 Undefined name `FLATTENED_ALIASES`
caps["name"] = aliased
elif caps["name"] in FLATTENED_ALIASES and not caps.get("hf_name"):

Check failure on line 599 in scripts/model_data/mteb_to_codeweaver.py

View workflow job for this annotation

GitHub Actions / Lint / Lint and Format

ruff (F821)

scripts/model_data/mteb_to_codeweaver.py:599:26: F821 Undefined name `FLATTENED_ALIASES`
caps["hf_name"] = next((k for k in FLATTENED_ALIASES if k == caps["name"]), None)

Check failure on line 600 in scripts/model_data/mteb_to_codeweaver.py

View workflow job for this annotation

GitHub Actions / Lint / Lint and Format

ruff (F821)

scripts/model_data/mteb_to_codeweaver.py:600:44: F821 Undefined name `FLATTENED_ALIASES`
if "Qwen3" in model["name"]:
caps["other"] = cast(dict[str, Any], caps.get("other", {})) | {
"model": {
Expand Down Expand Up @@ -664,15 +667,15 @@
if not isinstance(cap_partial["output_dtypes"], tuple):
cap_partial["output_dtypes"] = (cap_partial["output_dtypes"],)
capabilities = []
cap_map = MODEL_MAP_DATA.get(model_maker, {})

Check failure on line 670 in scripts/model_data/mteb_to_codeweaver.py

View workflow job for this annotation

GitHub Actions / Lint / Lint and Format

ruff (F821)

scripts/model_data/mteb_to_codeweaver.py:670:15: F821 Undefined name `MODEL_MAP_DATA`
maker_title = to_camel(model_maker)
cap_type = f"type {maker_title}Provider = Literal[{', '.join(sorted({f'{format_python_value(v)}' if isinstance(val, tuple) else format_python_value(val) for val in cap_map.values() for v in val}))}]"
printed_map = f"CAP_MAP: dict[Literal[{', '.join(f'"{k!s}"' for k in cap_map)}], tuple[{maker_title}Provider, ...]] = {format_python_dict(cast(dict[str, Any], cap_map), indent_level=0)}\n\n"
sanitized_names = [
sanitize_name(
cast(str, caps["name"])
if caps["name"] not in FLATTENED_ALIASES

Check failure on line 677 in scripts/model_data/mteb_to_codeweaver.py

View workflow job for this annotation

GitHub Actions / Lint / Lint and Format

ruff (F821)

scripts/model_data/mteb_to_codeweaver.py:677:36: F821 Undefined name `FLATTENED_ALIASES`
else FLATTENED_ALIASES[caps["name"]]

Check failure on line 678 in scripts/model_data/mteb_to_codeweaver.py

View workflow job for this annotation

GitHub Actions / Lint / Lint and Format

ruff (F821)

scripts/model_data/mteb_to_codeweaver.py:678:18: F821 Undefined name `FLATTENED_ALIASES`
)
for caps in cap_partials
]
Expand Down Expand Up @@ -756,8 +759,8 @@

def sanitize_name(name: str) -> str:
"""Convert model name to valid Python identifier."""
if name.lower() in FLATTENED_ALIASES:

Check failure on line 762 in scripts/model_data/mteb_to_codeweaver.py

View workflow job for this annotation

GitHub Actions / Lint / Lint and Format

ruff (F821)

scripts/model_data/mteb_to_codeweaver.py:762:24: F821 Undefined name `FLATTENED_ALIASES`
name = FLATTENED_ALIASES[cast(str, name.lower())]

Check failure on line 763 in scripts/model_data/mteb_to_codeweaver.py

View workflow job for this annotation

GitHub Actions / Lint / Lint and Format

ruff (F821)

scripts/model_data/mteb_to_codeweaver.py:763:16: F821 Undefined name `FLATTENED_ALIASES`
return name.replace("-", "_").replace(".", "_").replace("/", "_").replace(":", "_").upper()


Expand Down
Empty file modified scripts/model_data/secondary_providers.json
100644 → 100755
Empty file.
Empty file modified scripts/model_data/secondary_providers.json.license
100644 → 100755
Empty file.
2 changes: 0 additions & 2 deletions src/codeweaver/core/di/container.py
Original file line number Diff line number Diff line change
Expand Up @@ -562,8 +562,6 @@ async def resolve(
if self._is_union_type(interface):
instance = await self._resolve_union_interface(interface, cache_key, _resolution_stack)
return cast(T, instance)
elif interface is type(None):
return cast(T, None)

# 1. Check overrides first
# We check overrides before tags and singletons because overrides
Expand Down
5 changes: 1 addition & 4 deletions src/codeweaver/core/statistics.py
Original file line number Diff line number Diff line change
Expand Up @@ -84,10 +84,7 @@ async def _check_profile(container: Container) -> bool | None:
):
from codeweaver.providers.config.profiles import ProviderProfile

return (
profile is ProviderProfile.RECOMMENDED_CLOUD
or profile is ProviderProfile.RECOMMENDED
)
return profile in [ProviderProfile.RECOMMENDED_CLOUD, ProviderProfile.RECOMMENDED]
return None


Expand Down
30 changes: 6 additions & 24 deletions src/codeweaver/core/utils/generation.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,21 +19,9 @@


if sys.version_info < (3, 14):
try:
from uuid_extensions import uuid7 as uuid7_gen
except ImportError:
def uuid7_gen(*args, **kwargs) -> UUID7:
from pydantic import UUID7
from uuid import uuid4
return cast(UUID7, uuid4())
from uuid_extensions import uuid7 as uuid7_gen
else:
try:
from uuid import uuid7 as uuid7_gen
except ImportError:
def uuid7_gen(*args, **kwargs) -> UUID7:
from pydantic import UUID7
from uuid import uuid4
return cast(UUID7, uuid4())
from uuid import uuid7 as uuid7_gen


def uuid7() -> UUID7:
Expand All @@ -56,16 +44,10 @@ def uuid7_as_timestamp(
) -> int | datetime.datetime | None:
"""Utility to extract the timestamp from a UUID7, optionally as a datetime."""
if sys.version_info < (3, 14):
try:
from uuid_extensions import time_ns, uuid_to_datetime

return uuid_to_datetime(uuid) if as_datetime else time_ns(uuid)
except ImportError:
return datetime.datetime.now(datetime.UTC) if as_datetime else int(datetime.datetime.now(datetime.UTC).timestamp() * 1e9)
try:
from uuid import uuid7
except ImportError:
return datetime.datetime.now(datetime.UTC) if as_datetime else int(datetime.datetime.now(datetime.UTC).timestamp() * 1e9)
from uuid_extensions import time_ns, uuid_to_datetime

return uuid_to_datetime(uuid) if as_datetime else time_ns(uuid)
from uuid import uuid7

uuid = uuid7(uuid) if isinstance(uuid, str | int) else uuid
return (
Expand Down
4 changes: 1 addition & 3 deletions src/codeweaver/engine/chunker/delimiters/custom.py
Original file line number Diff line number Diff line change
Expand Up @@ -357,14 +357,12 @@ def generate_rst_character_ranges(character: str) -> list[str]:
nestable=False,
)

HTML_BLOCK_TAGS = frozenset({"html", "body", "main", "section", "article"})

HTML_TAGS_PATTERNS = [
DelimiterPattern(
starts=[f"<{tag}"],
ends=[f"</{tag}>"],
kind=DelimiterKind.BLOCK
if tag in HTML_BLOCK_TAGS
if tag in ["html", "body", "main", "section", "article"]
else DelimiterKind.PARAGRAPH,
inclusive=True,
take_whole_lines=True,
Expand Down
35 changes: 13 additions & 22 deletions src/codeweaver/providers/config/clients/multi.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,20 +49,14 @@
GoogleCredentials = Any

if has_package("fastembed") is not None or has_package("fastembed_gpu") is not None:
try:
from fastembed.common.types import OnnxProvider
except ImportError:
OnnxProvider = Any # type: ignore[assignment, misc]
from fastembed.common.types import OnnxProvider
else:
OnnxProvider = Any # type: ignore[assignment, misc]
OnnxProvider = object

Comment on lines 51 to 55
if has_package("torch") is not None:
try:
from torch.nn import Module
except ImportError:
Module = Any # type: ignore[assignment, misc]
from torch.nn import Module
else:
Module = Any # type: ignore[assignment, misc]
Module = object
if has_package("sentence_transformers") is not None:
# SentenceTransformerModelCardData contains these forward references:
# - eval_results_dict: dict[SentenceEvaluator, dict[str, Any]] | None
Expand Down Expand Up @@ -240,10 +234,7 @@ class FastEmbedClientOptions(ClientOptions):
model_name: str
cache_dir: str | None = None
threads: int | None = None
onnx_providers: Annotated[
Sequence[OnnxProvider] | None,
Field(alias="providers", serialization_alias="providers"),
] = None
providers: Sequence[OnnxProvider] | None = None
cuda: bool | None = None
device_ids: list[int] | None = None
lazy_load: bool = True
Expand All @@ -254,10 +245,10 @@ def _resolve_device_settings(self) -> Self:
from codeweaver.core import effective_cpu_count

cpu_count = effective_cpu_count()
updates: dict[str, Any] = {"threads": self.threads or cpu_count}
object.__setattr__(self, "threads", self.threads or cpu_count)
if self.cuda is False:
updates["device_ids"] = []
return self.model_copy(update=updates)
object.__setattr__(self, "device_ids", [])
return self
from codeweaver.providers.optimize import decide_fastembed_runtime

decision = decide_fastembed_runtime(
Expand All @@ -272,11 +263,11 @@ def _resolve_device_settings(self) -> Self:
else:
cuda = False
device_ids = []
updates["cuda"] = cuda
updates["device_ids"] = device_ids
if cuda and (not self.onnx_providers or ONNX_CUDA_PROVIDER not in self.onnx_providers):
updates["onnx_providers"] = [ONNX_CUDA_PROVIDER, *(self.onnx_providers or [])]
return self.model_copy(update=updates)
object.__setattr__(self, "cuda", cuda)
object.__setattr__(self, "device_ids", device_ids)
if cuda and (not self.providers or ONNX_CUDA_PROVIDER not in self.providers):
object.__setattr__(self, "providers", [ONNX_CUDA_PROVIDER, *(self.providers or [])])
return self

def _telemetry_keys(self) -> dict[FilteredKeyT, AnonymityConversion]:
return {FilteredKey("cache_dir"): AnonymityConversion.HASH}
Expand Down
4 changes: 1 addition & 3 deletions src/codeweaver/providers/config/profiles.py
Original file line number Diff line number Diff line change
Expand Up @@ -268,9 +268,7 @@ def _recommended_default(
),
data=(TavilyProviderSettings(provider=Provider.TAVILY),)
if Provider.TAVILY.has_env_auth and has_package("tavily")
else (DuckDuckGoProviderSettings(provider=Provider.DUCKDUCKGO),)
if has_package("ddgs")
else (),
else (DuckDuckGoProviderSettings(provider=Provider.DUCKDUCKGO),),
vector_store=(
Comment on lines 269 to 272
QdrantVectorStoreProviderSettings(
provider=Provider.QDRANT,
Expand Down
2 changes: 1 addition & 1 deletion src/codeweaver/providers/reranking/providers/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -462,7 +462,7 @@ def _process_results(
Note: This sync method is only called from async contexts (from the rerank method).
"""
# voyage and cohere return token count, others do not
if self.provider not in (Provider.VOYAGE, Provider.COHERE):
if self.provider not in [Provider.VOYAGE, Provider.COHERE]:
# We're always called from async context (rerank method), so we can safely get the loop
try:
loop = loop or asyncio.get_running_loop()
Expand Down
9 changes: 2 additions & 7 deletions src/codeweaver/server/agent_api/search/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -177,12 +177,7 @@ async def _resolve_indexer_from_container() -> IndexingService | None:

container = get_container()
return await container.resolve(IndexingService)
except Exception as e:
logger.warning(
"Failed to resolve IndexingService from container: %s",
e,
exc_info=True,
)
except Exception:
return None


Expand All @@ -205,7 +200,7 @@ async def _ensure_index_ready(
# We enable reconciliation by default to fix any partial indexes
await indexer.index_project(add_dense=True, add_sparse=True)
except Exception as e:
logger.warning("Auto-indexing failed: %s", e, exc_info=True)
logger.warning("Auto-indexing failed: %s", e)


async def _build_search_package(package: SearchPackageDep) -> SearchPackage:
Expand Down
Loading
Loading