Skip to content

Commit ae5dcff

Browse files
hjh0119hjh0119
committed
merge dev
2 parents 5b7202b + 2af73f2 commit ae5dcff

File tree

199 files changed

+8229
-2775
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

199 files changed

+8229
-2775
lines changed

‎.dev_scripts/build_docs.sh‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
pip install -r requirements/docs.txt
2+
cd docs
3+
rm -rf build
4+
5+
# update api rst
6+
#rm -rf source/api/
7+
#sphinx-apidoc --module-first -o source/api/ ../modelscope/
8+
make html

‎.dev_scripts/ci_container_test.sh‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
if [ "$MODELSCOPE_SDK_DEBUG" == "True" ]; then
2+
# pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple
3+
pip install -r requirements/tests.txt -i https://mirrors.aliyun.com/pypi/simple/
4+
git config --global --add safe.directory /twinkle
5+
git config --global user.email tmp
6+
git config --global user.name tmp.com
7+
8+
# linter test
9+
# use internal project for pre-commit due to the network problem
10+
if [ `git remote -v | grep alibaba | wc -l` -gt 1 ]; then
11+
pre-commit run -c .pre-commit-config_local.yaml --all-files
12+
if [ $? -ne 0 ]; then
13+
echo "linter test failed, please run 'pre-commit run --all-files' to check"
14+
echo "From the repository folder"
15+
echo "Run 'pip install -r requirements/tests.txt' install test dependencies."
16+
echo "Run 'pre-commit install' install pre-commit hooks."
17+
echo "Finally run linter with command: 'pre-commit run --all-files' to check."
18+
echo "Ensure there is no failure!!!!!!!!"
19+
exit -1
20+
fi
21+
fi
22+
23+
pip install -r requirements/framework.txt -U -i https://mirrors.aliyun.com/pypi/simple/
24+
pip install decord einops -U -i https://mirrors.aliyun.com/pypi/simple/
25+
pip uninstall autoawq -y
26+
pip install optimum
27+
pip install diffusers
28+
# pip install autoawq -U --no-deps
29+
30+
# test with install
31+
pip install .
32+
pip install auto_gptq bitsandbytes deepspeed -U -i https://mirrors.aliyun.com/pypi/simple/
33+
else
34+
echo "Running case in release image, run case directly!"
35+
fi
36+
# remove torch_extensions folder to avoid ci hang.
37+
rm -rf ~/.cache/torch_extensions
38+
if [ $# -eq 0 ]; then
39+
ci_command="python tests/run.py --subprocess"
40+
else
41+
ci_command="$@"
42+
fi
43+
echo "Running case with command: $ci_command"
44+
$ci_command

‎.dev_scripts/dockerci.sh‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
#!/bin/bash
2+
MODELSCOPE_CACHE_DIR_IN_CONTAINER=/modelscope_cache
3+
CODE_DIR=$PWD
4+
CODE_DIR_IN_CONTAINER=/twinkle
5+
MODELSCOPE_SDK_DEBUG=True
6+
echo "$USER"
7+
gpus='0,1 2,3'
8+
cpu_sets='0-15 16-31'
9+
cpu_sets_arr=($cpu_sets)
10+
is_get_file_lock=false
11+
CI_COMMAND=${CI_COMMAND:-bash .dev_scripts/ci_container_test.sh python tests/run.py --parallel 2 --run_config tests/run_config.yaml}
12+
echo "ci command: $CI_COMMAND"
13+
PR_CHANGED_FILES="${PR_CHANGED_FILES:-}"
14+
echo "PR modified files: $PR_CHANGED_FILES"
15+
PR_CHANGED_FILES=${PR_CHANGED_FILES//[ ]/#}
16+
echo "PR_CHANGED_FILES: $PR_CHANGED_FILES"
17+
idx=0
18+
for gpu in $gpus
19+
do
20+
exec {lock_fd}>"/tmp/gpu$gpu" || exit 1
21+
flock -n "$lock_fd" || { echo "WARN: gpu $gpu is in use!" >&2; idx=$((idx+1)); continue; }
22+
echo "get gpu lock $gpu"
23+
24+
CONTAINER_NAME="twinkle-ci-$idx"
25+
let is_get_file_lock=true
26+
27+
# pull image if there are update
28+
docker pull ${IMAGE_NAME}:${IMAGE_VERSION}
29+
if [ "$MODELSCOPE_SDK_DEBUG" == "True" ]; then
30+
echo 'debugging'
31+
docker run --rm --name $CONTAINER_NAME --shm-size=16gb \
32+
--cpuset-cpus=${cpu_sets_arr[$idx]} \
33+
--gpus='"'"device=$gpu"'"' \
34+
-v $CODE_DIR:$CODE_DIR_IN_CONTAINER \
35+
-v $MODELSCOPE_CACHE:$MODELSCOPE_CACHE_DIR_IN_CONTAINER \
36+
-v $MODELSCOPE_HOME_CACHE/$idx:/root \
37+
-v /home/admin/pre-commit:/home/admin/pre-commit \
38+
-e CI_TEST=True \
39+
-e TEST_LEVEL=$TEST_LEVEL \
40+
-e MODELSCOPE_CACHE=$MODELSCOPE_CACHE_DIR_IN_CONTAINER \
41+
-e MODELSCOPE_DOMAIN=$MODELSCOPE_DOMAIN \
42+
-e MODELSCOPE_SDK_DEBUG=True \
43+
-e HUB_DATASET_ENDPOINT=$HUB_DATASET_ENDPOINT \
44+
-e TEST_ACCESS_TOKEN_CITEST=$TEST_ACCESS_TOKEN_CITEST \
45+
-e TEST_ACCESS_TOKEN_SDKDEV=$TEST_ACCESS_TOKEN_SDKDEV \
46+
-e TEST_LEVEL=$TEST_LEVEL \
47+
-e MODELSCOPE_ENVIRONMENT='ci' \
48+
-e TEST_UPLOAD_MS_TOKEN=$TEST_UPLOAD_MS_TOKEN \
49+
-e MODEL_TAG_URL=$MODEL_TAG_URL \
50+
-e MODELSCOPE_API_TOKEN=$MODELSCOPE_API_TOKEN \
51+
-e PR_CHANGED_FILES=$PR_CHANGED_FILES \
52+
--workdir=$CODE_DIR_IN_CONTAINER \
53+
${IMAGE_NAME}:${IMAGE_VERSION} \
54+
$CI_COMMAND
55+
else
56+
docker run --rm --name $CONTAINER_NAME --shm-size=16gb \
57+
--cpuset-cpus=${cpu_sets_arr[$idx]} \
58+
--gpus='"'"device=$gpu"'"' \
59+
-v $CODE_DIR:$CODE_DIR_IN_CONTAINER \
60+
-v $MODELSCOPE_CACHE:$MODELSCOPE_CACHE_DIR_IN_CONTAINER \
61+
-v $MODELSCOPE_HOME_CACHE/$idx:/root \
62+
-v /home/admin/pre-commit:/home/admin/pre-commit \
63+
-e CI_TEST=True \
64+
-e TEST_LEVEL=$TEST_LEVEL \
65+
-e MODELSCOPE_CACHE=$MODELSCOPE_CACHE_DIR_IN_CONTAINER \
66+
-e MODELSCOPE_DOMAIN=$MODELSCOPE_DOMAIN \
67+
-e HUB_DATASET_ENDPOINT=$HUB_DATASET_ENDPOINT \
68+
-e TEST_ACCESS_TOKEN_CITEST=$TEST_ACCESS_TOKEN_CITEST \
69+
-e TEST_ACCESS_TOKEN_SDKDEV=$TEST_ACCESS_TOKEN_SDKDEV \
70+
-e TEST_LEVEL=$TEST_LEVEL \
71+
-e MODELSCOPE_ENVIRONMENT='ci' \
72+
-e TEST_UPLOAD_MS_TOKEN=$TEST_UPLOAD_MS_TOKEN \
73+
-e MODEL_TAG_URL=$MODEL_TAG_URL \
74+
-e MODELSCOPE_API_TOKEN=$MODELSCOPE_API_TOKEN \
75+
-e PR_CHANGED_FILES=$PR_CHANGED_FILES \
76+
--workdir=$CODE_DIR_IN_CONTAINER \
77+
${IMAGE_NAME}:${IMAGE_VERSION} \
78+
$CI_COMMAND
79+
fi
80+
if [ $? -ne 0 ]; then
81+
echo "Running test case failed, please check the log!"
82+
exit -1
83+
fi
84+
break
85+
done
86+
if [ "$is_get_file_lock" = false ] ; then
87+
echo 'No free GPU!'
88+
exit 1
89+
fi

‎.dev_scripts/dockerci_npu.sh‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
#!/bin/bash
2+
MODELSCOPE_CACHE_DIR=/modelscope_cache
3+
CODE_DIR=$PWD
4+
MODELSCOPE_SDK_DEBUG=True
5+
echo "$USER"
6+
gpus='0,1 2,3'
7+
is_get_file_lock=false
8+
CI_COMMAND=${CI_COMMAND:-bash .dev_scripts/ci_container_test.sh python tests/run.py --parallel 2 --run_config tests/run_config.yaml}
9+
echo "ci command: $CI_COMMAND"
10+
PR_CHANGED_FILES="${PR_CHANGED_FILES:-}"
11+
echo "PR modified files: $PR_CHANGED_FILES"
12+
PR_CHANGED_FILES=${PR_CHANGED_FILES//[ ]/#}
13+
echo "PR_CHANGED_FILES: $PR_CHANGED_FILES"
14+
idx=0
15+
for gpu in $gpus
16+
do
17+
exec {lock_fd}>"/tmp/gpu$gpu" || exit 1
18+
flock -n "$lock_fd" || { echo "WARN: gpu $gpu is in use!" >&2; idx=$((idx+1)); continue; }
19+
echo "get gpu lock $gpu"
20+
21+
let is_get_file_lock=true
22+
23+
# 设置环境变量
24+
export CI_TEST=True
25+
export TEST_LEVEL=$TEST_LEVEL
26+
export MODELSCOPE_CACHE=${MODELSCOPE_CACHE:-$MODELSCOPE_CACHE_DIR}
27+
export MODELSCOPE_DOMAIN=$MODELSCOPE_DOMAIN
28+
export HUB_DATASET_ENDPOINT=$HUB_DATASET_ENDPOINT
29+
export TEST_ACCESS_TOKEN_CITEST=$TEST_ACCESS_TOKEN_CITEST
30+
export TEST_ACCESS_TOKEN_SDKDEV=$TEST_ACCESS_TOKEN_SDKDEV
31+
export MODELSCOPE_ENVIRONMENT='ci'
32+
export TEST_UPLOAD_MS_TOKEN=$TEST_UPLOAD_MS_TOKEN
33+
export MODEL_TAG_URL=$MODEL_TAG_URL
34+
export MODELSCOPE_API_TOKEN=$MODELSCOPE_API_TOKEN
35+
export PR_CHANGED_FILES=$PR_CHANGED_FILES
36+
export CUDA_VISIBLE_DEVICES=$gpu
37+
38+
if [ "$MODELSCOPE_SDK_DEBUG" == "True" ]; then
39+
export MODELSCOPE_SDK_DEBUG=True
40+
echo 'debugging'
41+
fi
42+
43+
# 切换到代码目录并执行命令
44+
cd $CODE_DIR
45+
eval $CI_COMMAND
46+
47+
if [ $? -ne 0 ]; then
48+
echo "Running test case failed, please check the log!"
49+
exit -1
50+
fi
51+
break
52+
done
53+
54+
if [ "$is_get_file_lock" = false ] ; then
55+
echo 'No free GPU!'
56+
exit 1
57+
fi

‎.github/ISSUE_TEMPLATE/1-bug-report.yml‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
name: "🐛 Bug Report"
2+
description: Create a bug report to help us improve twinkle
3+
labels: ["bug"]
4+
5+
body:
6+
- type: markdown
7+
attributes:
8+
value: |
9+
Thank you for supporting twinkle and taking the time to submit this issue.
10+
感谢你对 twinkle 的支持和抽出时间提交相关 issue。
11+
12+
- type: checkboxes
13+
id: checklist
14+
attributes:
15+
label: Checklist / 检查清单
16+
options:
17+
- label: I have searched existing issues, and this is a new bug report. / 我已经搜索过现有的 issues，确认这是一个新的 bug report。
18+
required: true
19+
20+
21+
- type: textarea
22+
id: bug-description
23+
validations:
24+
required: true
25+
attributes:
26+
label: Bug Description / Bug 描述
27+
description: |
28+
Please describe the issue you encountered. It's better to include error screenshots or stack trace information.
29+
请详细描述你遇到的问题，最好包含报错截图或报错栈信息。
30+
31+
32+
- type: textarea
33+
id: reproduction-steps
34+
validations:
35+
required: true
36+
attributes:
37+
label: How to Reproduce / 如何复现
38+
description: |
39+
Please provide steps to reproduce the issue, including twinkle version, runtime environment, and detailed reproduction steps.
40+
请提供复现问题的步骤，包括 twinkle 的版本、运行环境、详细的复现步骤等。
41+
42+
43+
- type: textarea
44+
id: additional-information
45+
attributes:
46+
label: Additional Information / 补充信息
47+
description: |
48+
Please provide any additional information here.
49+
在这里补充其他相关信息。

‎.github/ISSUE_TEMPLATE/2-feature-request.yml‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
name: "🚀 Feature Request"
2+
description: Submit a request for a new feature
3+
labels: ["enhancement"]
4+
5+
body:
6+
- type: markdown
7+
attributes:
8+
value: |
9+
Thank you for supporting twinkle and taking the time to submit this issue.
10+
感谢你对 twinkle 的支持和抽出时间提交相关 issue。
11+
12+
- type: checkboxes
13+
id: checklist
14+
attributes:
15+
label: Checklist / 检查清单
16+
options:
17+
- label: I have searched existing issues, and this is a new feature request. / 我已经搜索过现有的 issues，确认这是一个新的 Feature Request。
18+
required: true
19+
20+
- type: textarea
21+
id: feature-request-description
22+
validations:
23+
required: true
24+
attributes:
25+
label: Feature Request Description / Feature Request 描述
26+
description: |
27+
Please provide a detailed description of the new feature you would like to see added.
28+
请详细描述您希望添加的新功能特性。
29+
30+
31+
- type: textarea
32+
id: pull-request
33+
attributes:
34+
label: Pull Request / Pull Request 信息
35+
description: |
36+
Have you already submitted or plan to submit a Pull Request? Please share your plans.
37+
你是否已经提交或即将提交 Pull Request？请说明你的计划。

‎.github/ISSUE_TEMPLATE/3-question-discussion.yml‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
name: "🤔 Question & Discussion"
2+
description: Create an issue for questions and discussions
3+
labels: ["question"]
4+
5+
body:
6+
- type: markdown
7+
attributes:
8+
value: |
9+
Thank you for supporting twinkle and taking the time to submit this issue.
10+
感谢你对 twinkle 的支持和抽出时间提交相关 issue。
11+
12+
- type: checkboxes
13+
id: checklist
14+
attributes:
15+
label: Checklist / 检查清单
16+
options:
17+
- label: I have searched existing issues, and this is a new question or discussion topic. / 我已经搜索过现有的 issues，确认这是一个新的问题与讨论。
18+
required: true
19+
20+
- type: textarea
21+
id: question-description
22+
validations:
23+
required: true
24+
attributes:
25+
label: Question Description / 问题描述
26+
description: |
27+
Please describe the question or topic you would like to discuss.
28+
请描述你想要讨论的问题或话题。

‎.github/ISSUE_TEMPLATE/config.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
blank_issues_enabled: false

‎.github/PULL_REQUEST_TEMPLATE.md‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
# PR type
2+
- [ ] Bug Fix
3+
- [ ] New Feature
4+
- [ ] Document Updates
5+
- [ ] More Models or Datasets Support
6+
7+
# PR information
8+
9+
Write the detail information belongs to this PR.
10+
11+
## Experiment results
12+
13+
Paste your experiment result here(if needed).

‎.github/SECURITY.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
# Reporting Security Issues
2+
3+
Usually security issues of a deep learning project come from non-standard 3rd packages or continuous running services. If you are suffering from security issues from our project, please consider reporting to us. We appreciate your efforts to responsibly disclose your findings, and will make every effort to acknowledge your contributions.

0 commit comments

Comments
 (0)