ArcticTraining/pyproject.toml at main · vx120/ArcticTraining · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"

[tool.hatch.metadata]
# this is needed when we want to point to a github branch in `dependencies` - leave it enabled
allow-direct-references = true
license-files = [
  "LICENSE",
]

[tool.hatch.build]
include = [
  "arctic_training_cli.py",
  "arctic_training",
]

[project.urls]
Homepage = "https://github.com/snowflakedb/ArcticTraining"
Issues = "https://github.com/snowflakedb/ArcticTraining/issues"

[project]
name = "arctic_training"
version = "0.7.1.dev0"
description = "Snowflake LLM training library"
authors = [
    {author = "Michael Wyatt", email = "michael.wyatt@snowflake.com"},
    {author = "Stas Bekman", email = "stas.bekman@snowflake.com"},
    {author = "Jeff Rasley", email = "jeff.rasley@snowflake.com"},
    {author = "Zhewei Yao", email = "zhewei.yao@snowflake.com"},
    {author = "Samyam Rajbhandari", email = "samyam.rajbhandari@snowflake.com"},
    {author = "Aurick Qiao", email = "aurick.qiao@snowflake.com"},
    {author = "Canwen Xu", email = "canwen.xu@snowflake.com"}
]
license = "Apache-2.0"
readme = "README.md"
homepage = "https://github.com/snowflakedb/ArcticTraining"
repository = "https://github.com/snowflakedb/ArcticTraining"
keywords = ["llm", "training"]

classifiers = [
  # How mature is this project?  4 - Beta / 5 - Production/Stable
  "Development Status :: 4 - Beta",

  "Programming Language :: Python :: 3",
  "Programming Language :: Python :: 3.10",
  "Programming Language :: Python :: 3.11",
  "Programming Language :: Python :: 3.12",
]

requires-python = ">= 3.8"

# Dependencies for the project
dependencies = [
    "datasets",
    "deepspeed>=0.18.2",  # UlyssesSPAttentionHF.register_with_transformers api change
    "devtools",
    "jsonlines",
    "loguru",
    "openai>=1.48.0",
    "peft",
    "psutil",
    "pydantic>=2.10",
    "nvidia-ml-py",
    "tabulate",
    "torch",
    "tqdm",
    "transformers>=4.56,<5.0.0", # torch_dtype deprecation, 5.0 is unstable at the moment
    "wandb",
    "yq",
]

# Optional development dependencies
[project.optional-dependencies]
all = [
    "arctic_training[dev]",
    "arctic_training[docs]",
    "arctic_training[testing]",
    "arctic_training[vllm]",
    "arctic_training[cortex]",
]

dev = [
    "arctic_training[docs]",
    "arctic_training[formatting]",
    "arctic_training[testing]",
]

testing = [
    "pytest",
    "pytest-instafail",
    "parameterized",
    "pytest-xdist",
]

formatting = [
    "pre-commit",
    "autoflake",
]

docs = [
    "autodoc_pydantic>=2.0.0",
    "pydantic>=2.0",
    "sphinx>=8.1.3",
    "sphinx-autodoc-typehints",
    "sphinx-copybutton",
    "sphinx-rtd-theme",
]

vllm = [
    "vllm==0.6.2",
]

cortex = [
    "snowflake-connector-python==3.12.3",
]

[project.scripts]
arctic_training = "arctic_training_cli:main"
arctic_training_run = "arctic_training.entrypoint:launch"
arctic_synth = "arctic_training.synth:main"