kitoken/packages/python/pyproject.toml at main · Systemcluster/kitoken · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
[project]

name = "kitoken"
description = "Fast and versatile tokenizer for language models, supporting BPE, Unigram and WordPiece tokenization"
requires-python = ">=3.10"
license = { text = "BSD-2-Clause" }
version = "0.10.1"
classifiers = [
    "Programming Language :: Rust",
    "Programming Language :: Python :: Implementation :: CPython",
    "Programming Language :: Python :: Implementation :: PyPy",
    "Operating System :: Microsoft :: Windows",
    "Operating System :: POSIX :: Linux",
    "Operating System :: MacOS :: MacOS X",
    "Environment :: Console",
    "Topic :: File Formats",
    "Topic :: Text Processing :: Linguistic",
    "Topic :: Scientific/Engineering :: Artificial Intelligence",
    "Topic :: Software Development :: Libraries :: Python Modules",
    "Intended Audience :: Developers",
    "Intended Audience :: Science/Research",
    "Intended Audience :: End Users/Desktop",
    "License :: OSI Approved :: BSD License",
    "Typing :: Typed",
]
keywords = ["tokenizer", "nlp", "bpe", "unigram", "wordpiece"]
authors = [{ name = "Christian Sdunek", email = "me@systemcluster.me" }]
readme = "README.md"


[project.urls]

Homepage = "https://kitoken.dev"
Repository = "https://github.com/Systemcluster/kitoken"


[build-system]

requires = [
    "maturin==1.12.7",
]
build-backend = "maturin"


[dependency-groups]

dev = [
    "maturin==1.12.7",
    "pip>=26.0.1",
]


[tool.uv.pip]

universal = true


[tool.maturin]

profile = "performance"
bindings = "pyo3"
manifest-path = "Cargo.toml"
strip = true
include-debuginfo = false
include = [
    "kitoken.pyi",
]
exclude = [
    "test.py",
]