Skip to content

Commit 5e1f465

Browse files
kevssimkevssim
committed
Merge branch 'dev' of https://github.com/modelscope/twinkle into dev
2 parents 615047b + dad0119 commit 5e1f465

File tree

13 files changed

+61
-75
lines changed

13 files changed

+61
-75
lines changed

‎.dev_scripts/build_docs.sh‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
pip install -r requirements/docs.txt
21
cd docs
32
rm -rf build
43

‎.dev_scripts/ci_container_test.sh‎

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
if [ "$MODELSCOPE_SDK_DEBUG" == "True" ]; then
22
# pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple
3-
pip install -r requirements/tests.txt -i https://mirrors.aliyun.com/pypi/simple/
43
git config --global --add safe.directory /twinkle
54
git config --global user.email tmp
65
git config --global user.name tmp.com
@@ -12,31 +11,26 @@ if [ "$MODELSCOPE_SDK_DEBUG" == "True" ]; then
1211
if [ $? -ne 0 ]; then
1312
echo "linter test failed, please run 'pre-commit run --all-files' to check"
1413
echo "From the repository folder"
15-
echo "Run 'pip install -r requirements/tests.txt' install test dependencies."
1614
echo "Run 'pre-commit install' install pre-commit hooks."
1715
echo "Finally run linter with command: 'pre-commit run --all-files' to check."
1816
echo "Ensure there is no failure!!!!!!!!"
1917
exit -1
2018
fi
2119
fi
2220

23-
pip install -r requirements/framework.txt -U -i https://mirrors.aliyun.com/pypi/simple/
2421
pip install decord einops -U -i https://mirrors.aliyun.com/pypi/simple/
2522
pip uninstall autoawq -y
2623
pip install optimum
27-
pip install diffusers
28-
# pip install autoawq -U --no-deps
2924

3025
# test with install
3126
pip install .
32-
pip install auto_gptq bitsandbytes deepspeed -U -i https://mirrors.aliyun.com/pypi/simple/
3327
else
3428
echo "Running case in release image, run case directly!"
3529
fi
3630
# remove torch_extensions folder to avoid ci hang.
3731
rm -rf ~/.cache/torch_extensions
3832
if [ $# -eq 0 ]; then
39-
ci_command="python tests/run.py --subprocess"
33+
ci_command="pytest tests"
4034
else
4135
ci_command="$@"
4236
fi

‎.dev_scripts/dockerci.sh‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,20 @@
22
MODELSCOPE_CACHE_DIR_IN_CONTAINER=/modelscope_cache
33
CODE_DIR=$PWD
44
CODE_DIR_IN_CONTAINER=/twinkle
5+
mkdir -p ~/.cache
6+
MODELSCOPE_CACHE=~/.cache
7+
IMAGE_NAME=modelscope-registry.us-west-1.cr.aliyuncs.com/modelscope-repo/modelscope
8+
IMAGE_VERSION=ci_image
9+
MODELSCOPE_HOME_CACHE=~/.cache
10+
CI_TEST=True
11+
MODELSCOPE_SDK_DEBUG=True
12+
CI_COMMAND='bash .dev_scripts/ci_container_test.sh pytest tests'
513
MODELSCOPE_SDK_DEBUG=True
614
echo "$USER"
715
gpus='0,1 2,3'
816
cpu_sets='0-15 16-31'
917
cpu_sets_arr=($cpu_sets)
1018
is_get_file_lock=false
11-
CI_COMMAND=${CI_COMMAND:-bash .dev_scripts/ci_container_test.sh python tests/run.py --parallel 2 --run_config tests/run_config.yaml}
1219
echo "ci command: $CI_COMMAND"
1320
PR_CHANGED_FILES="${PR_CHANGED_FILES:-}"
1421
echo "PR modified files: $PR_CHANGED_FILES"

‎.dev_scripts/dockerci_npu.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ MODELSCOPE_SDK_DEBUG=True
55
echo "$USER"
66
gpus='0,1 2,3'
77
is_get_file_lock=false
8-
CI_COMMAND=${CI_COMMAND:-bash .dev_scripts/ci_container_test.sh python tests/run.py --parallel 2 --run_config tests/run_config.yaml}
8+
CI_COMMAND=${CI_COMMAND:-bash .dev_scripts/ci_container_test.sh pytest tests}
99
echo "ci command: $CI_COMMAND"
1010
PR_CHANGED_FILES="${PR_CHANGED_FILES:-}"
1111
echo "PR modified files: $PR_CHANGED_FILES"

‎.github/workflows/citest.yaml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,5 +73,4 @@ jobs:
7373
shell: bash
7474
run: |
7575
set -e
76-
source /mnt/modelscope/ci_env.sh
7776
bash .dev_scripts/dockerci.sh

‎.github/workflows/citest_npu.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -71,5 +71,5 @@ jobs:
7171
export TEST_LEVEL=0
7272
mkdir -p ~/.cache
7373
export MODELSCOPE_CACHE=~/.cache
74-
export CI_COMMAND='bash .dev_scripts/ci_container_test.sh python tests/run.py --parallel 2 --subprocess --run_config tests/run_config.yaml'
74+
export CI_COMMAND='bash .dev_scripts/ci_container_test.sh pytest tests'
7575
bash .dev_scripts/dockerci_npu.sh

‎.pre-commit-config.yaml‎

Lines changed: 10 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,11 @@ repos:
55
- id: flake8
66
exclude: |
77
(?x)^(
8-
thirdparty/|
98
examples/|
10-
tests/run.py
9+
cookbook/|
10+
client_tools/|
11+
src/twinkle_client|
12+
1113
)$
1214
- repo: https://github.com/PyCQA/isort.git
1315
rev: 7.0.0
@@ -16,33 +18,21 @@ repos:
1618
exclude: |
1719
(?x)^(
1820
examples/|
19-
tests/run.py|
21+
cookbook/|
22+
client_tools/|
23+
src/twinkle_client|
2024
)$
2125
- repo: https://github.com/google/yapf
2226
rev: v0.43.0
2327
hooks:
2428
- id: yapf
2529
exclude: |
2630
(?x)^(
27-
thirdparty/|
2831
examples/|
29-
tests/run.py
32+
cookbook/|
33+
client_tools/|
34+
src/twinkle_client|
3035
)$
3136
- repo: https://github.com/pre-commit/pre-commit-hooks.git
3237
rev: v6.0.0
3338
hooks:
34-
- id: trailing-whitespace
35-
exclude: thirdparty/|tests/run.py
36-
- id: check-yaml
37-
exclude: thirdparty/|tests/run.py
38-
- id: end-of-file-fixer
39-
exclude: thirdparty/|tests/run.py
40-
- id: requirements-txt-fixer
41-
exclude: thirdparty/|tests/run.py
42-
- id: double-quote-string-fixer
43-
exclude: thirdparty/|tests/run.py
44-
- id: check-merge-conflict
45-
exclude: thirdparty/|tests/run.py
46-
- id: mixed-line-ending
47-
exclude: thirdparty/|tests/run.py
48-
args: ["--fix=lf"]

‎.pre-commit-config_local.yaml‎

Lines changed: 9 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,10 @@ repos:
55
- id: flake8
66
exclude: |
77
(?x)^(
8-
thirdparty/|
98
examples/|
10-
tests/run.py
9+
cookbook/|
10+
client_tools/|
11+
src/twinkle_client|
1112
)$
1213
- repo: /home/admin/pre-commit/isort
1314
rev: 4.3.21
@@ -16,36 +17,21 @@ repos:
1617
exclude: |
1718
(?x)^(
1819
examples/|
19-
tests/run.py|
20+
cookbook/|
21+
client_tools/|
22+
src/twinkle_client|
2023
)$
2124
- repo: /home/admin/pre-commit/mirrors-yapf
2225
rev: v0.30.0
2326
hooks:
2427
- id: yapf
2528
exclude: |
2629
(?x)^(
27-
thirdparty/|
2830
examples/|
29-
tests/run.py
31+
cookbook/|
32+
client_tools/|
33+
src/twinkle_client|
3034
)$
3135
- repo: /home/admin/pre-commit/pre-commit-hooks
3236
rev: v3.1.0
3337
hooks:
34-
- id: trailing-whitespace
35-
exclude: thirdparty/|tests/run.py
36-
- id: check-yaml
37-
exclude: thirdparty/|tests/run.py
38-
- id: end-of-file-fixer
39-
exclude: thirdparty/
40-
- id: requirements-txt-fixer
41-
exclude: thirdparty/|tests/run.py
42-
- id: double-quote-string-fixer
43-
exclude: thirdparty/|tests/run.py
44-
- id: check-merge-conflict
45-
exclude: thirdparty/|tests/run.py
46-
- id: fix-encoding-pragma
47-
exclude: thirdparty/|tests/run.py
48-
args: ["--remove"]
49-
- id: mixed-line-ending
50-
exclude: thirdparty/|tests/run.py
51-
args: ["--fix=lf"]

‎cookbook/legacy/grpo/gsm8k.py‎

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,6 @@
6262
# SwanLab experiment tracking
6363
USE_SWANLAB = bool(int(os.environ.get('USE_SWANLAB', '1')))
6464
if USE_SWANLAB:
65-
os.environ['SWANLAB_API_KEY'] = '3hVJrk0veNB2NCm72UdJg'
6665
import swanlab
6766
swanlab.login(api_key=os.environ['SWANLAB_API_KEY'], save=True)
6867
swanlab.init(project="twinkle-gsm8k", config={
@@ -285,13 +284,6 @@ def main():
285284
remote_group='model',
286285
)
287286

288-
# NOTE: Since the training loop externally manages micro-batch splitting
289-
# and only calls clip_grad_and_step once per optimizer step, we set
290-
# gradient_accumulation_steps=1 here. The internal do_grad_sync check
291-
# in OptimizerGroup uses cur_step to decide when to perform optimizer
292-
# updates; with external micro-batch management, setting gas>1 would
293-
# cause cur_step to never align with the internal trigger condition,
294-
# effectively preventing optimizer.step() from ever executing.
295287
model.add_adapter_to_model(
296288
ADAPTER_NAME,
297289
lora_config,
@@ -334,6 +326,7 @@ def main():
334326
'enforce_eager': True,
335327
'enable_sleep_mode': False,
336328
'enable_lora': True,
329+
"logprobs_mode": "processed_logprobs",
337330
},
338331
device_mesh=sampler_mesh,
339332
remote_group='sampler',

‎cookbook/legacy/grpo/lora_backup.py‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
from twinkle.dataset import Dataset, DatasetMeta
1616
from twinkle.model import TransformersModel
1717
from twinkle.processor import InputProcessor
18-
from twinkle.sampler import VLLMSampler
18+
from twinkle.sampler import vLLMSampler
1919
from twinkle.template import Template
2020
from twinkle.metric import CompletionRewardMetric
2121

@@ -44,7 +44,6 @@
4444

4545
# SwanLab is optional - only used if SWANLAB_API_KEY is set
4646
USE_SWANLAB = True
47-
os.environ['SWANLAB_API_KEY'] = '3hVJrk0veNB2NCm72UdJg'
4847
if USE_SWANLAB:
4948
import swanlab
5049
if USE_SWANLAB:
@@ -134,7 +133,7 @@ def main():
134133
model.set_processor(InputProcessor, adapter_name=ADAPTER_NAME)
135134
model.set_template('Template', model_id=MODEL_ID, adapter_name=ADAPTER_NAME)
136135

137-
sampler = VLLMSampler(
136+
sampler = vLLMSampler(
138137
model_id=MODEL_ID,
139138
engine_args={
140139
'load_format': 'dummy',

0 commit comments

Comments
 (0)