pyproject.toml

[build-system]
requires = ["setuptools>=45", "wheel", "cython>=3.0.0"]
build-backend = "setuptools.build_meta"


[project]
name = "chonkie"
version = "1.2.1"
description = "🦛 CHONK your texts with Chonkie ✨ - The no-nonsense chunking library"
readme = "README.md"
requires-python = ">=3.9"
license = { file = "LICENSE" }
keywords = [
    "chunking",
    "rag",
    "retrieval-augmented-generation",
    "nlp",
    "natural-language-processing",
    "text-processing",
    "text-analysis",
    "text-chunking",
    "artificial-intelligence",
    "machine-learning",
]
authors = [
    { name = "Bhavnick Minhas", email = "bhavnick@chonkie.ai" },
    { name = "Shreyash Nigam", email = "shreyash@chonkie.ai" },
]
maintainers = [
    { name = "Bhavnick Minhas", email = "bhavnick@chonkie.ai" },
    { name = "Shreyash Nigam", email = "shreyash@chonkie.ai" },
]

classifiers = [
    "Programming Language :: Python",
    "Programming Language :: Python :: 3",
    "Programming Language :: Python :: 3.9",
    "Programming Language :: Python :: 3.10",
    "Programming Language :: Python :: 3.11",
    "Programming Language :: Python :: 3.12",
    "Programming Language :: Python :: 3.13",
    "Intended Audience :: Developers",
    "Intended Audience :: Science/Research",
    "Intended Audience :: Information Technology",
    "Topic :: Scientific/Engineering :: Artificial Intelligence",
    "Topic :: Scientific/Engineering :: Information Analysis",
    "Topic :: Text Processing :: Linguistic",
]
dependencies = ["tqdm>=4.64.0"]

[project.urls]
Homepage = "https://github.com/chonkie-inc/chonkie"
Documentation = "https://docs.chonkie.ai"
"Bug Tracker" = "https://github.com/chonkie-inc/chonkie/issues"

[project.optional-dependencies]
# Optional dependencies for the utils
hub = ["huggingface-hub>=0.24.0", "jsonschema>=4.23.0"]
viz = ["rich>=13.0.0"]

# Optional dependencies for tokenizers
tokenizers = ["tokenizers>=0.16.0"]
tiktoken = ["tiktoken>=0.5.0"]

# Optional dependencies for the chunkers
code = [
    "tree-sitter>=0.20.0",
    "tree-sitter-language-pack>=0.7.0",
    "magika>=0.6.0, <0.7.0",
]
neural = ["transformers>=4.0.0", "torch>=2.0.0, <3.0"]

# Optional dependencies for the embeddings
model2vec = ["tokenizers>=0.16.0", "model2vec>=0.3.0", "numpy>=2.0.0, <3.0"]
st = [
    "tokenizers>=0.16.0",
    "sentence-transformers>=3.0.0",
    "numpy>=2.0.0, <3.0",
    "accelerate>=1.6.0",
]
openai = [
    "tiktoken>=0.5.0",
    "openai>=1.0.0",
    "numpy>=2.0.0, <3.0",
    "pydantic>=2.0.0",
]
semantic = ["tokenizers>=0.16.0", "model2vec>=0.3.0", "numpy>=2.0.0, <3.0"]
gemini = ["pydantic>=2.0.0", "google-genai>=1.0.0"]
azure-openai = [
    "azure-identity>=1.23.0",
    "tiktoken>=0.5.0",
    "openai>=1.0.0",
    "numpy>=2.0.0, <3.0",
    "pydantic>=2.0.0",
]
voyageai = ["tokenizers>=0.16.0", "voyageai>=0.3.2", "numpy>=2.0.0, <3.0"]
cohere = ["tokenizers>=0.16.0", "cohere>=5.13.0", "numpy>=2.0.0, <3.0"]
jina = ["tokenizers>=0.16.0", "numpy>=2.0.0, <3.0"]

# Optional dependencies for the friends
chroma = ["chromadb>=1.0.0"]
qdrant = ["qdrant-client>=1.0.0"]
tpuf = ["turbopuffer[fast]>=0.2.0"]
pgvector = ["vecs>=0.4.0"]
weaviate = ["weaviate-client>=4.16.7"]
datasets = ["datasets>=4.0.0"]
pinecone = ["pinecone"]
mongodb = ["pymongo"]

# Optional dependencies for the genie
genie = ["pydantic>=2.0.0", "google-genai>=1.0.0"]

# All dependencies
all = [
    "tokenizers>=0.16.0",
    "tiktoken>=0.5.0",
    "rich>=13.0.0",
    "tree-sitter>=0.20.0",
    "tree-sitter-language-pack>=0.7.0",
    "magika>=0.6.0, <0.7.0",
    "sentence-transformers>=3.0.0",
    "numpy>=2.0.0, <3.0",
    "openai>=1.0.0",
    "model2vec>=0.3.0",
    "cohere>=5.13.0",
    "accelerate>=1.6.0",
    "huggingface-hub>=0.24.0",
    "jsonschema>=4.23.0",
    "pydantic>=2.0.0",
    "google-genai>=1.0.0",
    "transformers>=4.0.0",
    "torch>=2.0.0, <3.0",
    "chromadb>=1.0.0",
    "qdrant-client>=1.0.0",
    "turbopuffer[fast]>=0.2.0",
    "weaviate-client>=4.16.7",
    "azure-identity>=1.23.0",
    "pinecone",
    "pymongo",
    "voyageai>=0.3.2",
]
dev = [
    "datasets>=1.14.0",
    "transformers>=4.0.0",
    "pytest>=6.2.0",
    "pytest-cov>=4.0.0",
    "pytest-xdist>=2.5.0",
    "pytest-asyncio>=0.26.0",
    "coverage",
    "ruff>=0.0.265",
    "mypy>=1.11.0",
    "cython>=3.0.0",
]

[tool.pytest.ini_options]
pythonpath = "src"
asyncio_mode = "strict"
asyncio_default_fixture_loop_scope = "function"

[tool.setuptools]
package-dir = { "" = "src" }
packages = [
    "chonkie",
    "chonkie.chunker",
    "chonkie.types",
    "chonkie.embeddings",
    "chonkie.utils",
    "chonkie.refinery",
    "chonkie.genie",
    "chonkie.friends",
    "chonkie.friends.porters",
    "chonkie.friends.handshakes",
    "chonkie.cloud",
    "chonkie.cloud.auth",
    "chonkie.cloud.chunker",
    "chonkie.cloud.refineries",
    "chonkie.experimental",
    "chonkie.fetcher",
    "chonkie.chef",
]

[tool.setuptools.package-data]
"chonkie.chunker.c_extensions" = ["*.pyx", "*.pxd", "*.c", "*.so"]

[[tool.setuptools.ext-modules]]
name = "chonkie.chunker.c_extensions.split"
sources = ["src/chonkie/chunker/c_extensions/split.pyx"]

[[tool.setuptools.ext-modules]]
name = "chonkie.chunker.c_extensions.merge"
sources = ["src/chonkie/chunker/c_extensions/merge.pyx"]

[[tool.setuptools.ext-modules]]
name = "chonkie.chunker.c_extensions.savgol"
sources = ["src/chonkie/chunker/c_extensions/savgol.pyx"]

[tool.ruff]
exclude = ["*.ipynb"]
lint.select = ["F", "I", "D"]
lint.ignore = ["D211", "D213", "D203"]

[tool.ruff.format]
preview = true
indent-style = "space"

[tool.mypy]
disallow_untyped_defs = true
ignore_missing_imports = true
no_implicit_optional = true
check_untyped_defs = true
show_error_codes = true