From 8a4931664742d2488d3d1c70848a3c5f706eeff2 Mon Sep 17 00:00:00 2001 From: 0x45f Date: Thu, 4 Sep 2025 14:33:59 +0800 Subject: [PATCH 01/14] Add ci yaml --- .github/workflows/ci-temp-model-test.yaml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 .github/workflows/ci-temp-model-test.yaml diff --git a/.github/workflows/ci-temp-model-test.yaml b/.github/workflows/ci-temp-model-test.yaml new file mode 100644 index 0000000000..2be388aa75 --- /dev/null +++ b/.github/workflows/ci-temp-model-test.yaml @@ -0,0 +1,23 @@ +name: ci-temp-model-test + +on: + push: + branches: [ "master" ] + pull_request: + branches: [ "master" ] + +jobs: + ci-temp-model-test: + runs-on: ci-temp-model-test + concurrency: + group: ci-temp-model-test-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + steps: + - name: Checkout flaggems + uses: actions/checkout@v4 + + - name: Checkout benchmark + uses: actions/checkout@v4 + with: + repository: pytorch/benchmark + path: benchmark From b3d633fed6e39ed698f462fc5e8cf647491735c3 Mon Sep 17 00:00:00 2001 From: 0x45f Date: Fri, 5 Sep 2025 11:27:40 +0800 Subject: [PATCH 02/14] Fix --- .github/workflows/ci-temp-model-test.yaml | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci-temp-model-test.yaml b/.github/workflows/ci-temp-model-test.yaml index 2be388aa75..460f07e80d 100644 --- a/.github/workflows/ci-temp-model-test.yaml +++ b/.github/workflows/ci-temp-model-test.yaml @@ -1,4 +1,4 @@ -name: ci-temp-model-test +name: ci-temp-test-model on: push: @@ -6,11 +6,14 @@ on: pull_request: branches: [ "master" ] +env: + CUDA_VISIBLE_DEVICES: 7 + jobs: - ci-temp-model-test: - runs-on: ci-temp-model-test + ci-temp-test-model: + runs-on: ci-temp-test-model concurrency: - group: ci-temp-model-test-${{ github.event.pull_request.number || github.ref }} + group: ci-temp-test-model-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true steps: - name: Checkout flaggems @@ -21,3 +24,14 @@ jobs: with: repository: pytorch/benchmark path: benchmark + + - name: Install and run benchmark + shell: bash + run: | + source tools/run_command.sh + run_command conda activate torchbenchmark + run_command pip install -e . + run_command cd benchmark + run_command python install.py + run_command python run_benchmark.py test_bench --accuracy --device cuda --test eval --output output.json --models resnet50 + run_command cd .. From 451eb5c015dd61f112ad04307e73547c610579e2 Mon Sep 17 00:00:00 2001 From: 0x45f Date: Fri, 5 Sep 2025 13:48:26 +0800 Subject: [PATCH 03/14] Fix --- .github/workflows/ci-temp-model-test.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci-temp-model-test.yaml b/.github/workflows/ci-temp-model-test.yaml index 460f07e80d..52d73cef71 100644 --- a/.github/workflows/ci-temp-model-test.yaml +++ b/.github/workflows/ci-temp-model-test.yaml @@ -29,6 +29,7 @@ jobs: shell: bash run: | source tools/run_command.sh + run_command conda init run_command conda activate torchbenchmark run_command pip install -e . run_command cd benchmark From 263efb757a7b2b77ce71cef932371958cb9c05e1 Mon Sep 17 00:00:00 2001 From: 0x45f Date: Fri, 5 Sep 2025 13:54:57 +0800 Subject: [PATCH 04/14] FIx --- .github/workflows/ci-temp-model-test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-temp-model-test.yaml b/.github/workflows/ci-temp-model-test.yaml index 52d73cef71..555df0890a 100644 --- a/.github/workflows/ci-temp-model-test.yaml +++ b/.github/workflows/ci-temp-model-test.yaml @@ -29,7 +29,7 @@ jobs: shell: bash run: | source tools/run_command.sh - run_command conda init + run_command zsh run_command conda activate torchbenchmark run_command pip install -e . run_command cd benchmark From 5a261c8dbab98ad0c768a24a6db732bd1c18d875 Mon Sep 17 00:00:00 2001 From: 0x45f Date: Fri, 5 Sep 2025 13:56:03 +0800 Subject: [PATCH 05/14] Fix --- .github/workflows/ci-temp-model-test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-temp-model-test.yaml b/.github/workflows/ci-temp-model-test.yaml index 555df0890a..9f1a109bdc 100644 --- a/.github/workflows/ci-temp-model-test.yaml +++ b/.github/workflows/ci-temp-model-test.yaml @@ -29,7 +29,7 @@ jobs: shell: bash run: | source tools/run_command.sh - run_command zsh + run_command source activate base run_command conda activate torchbenchmark run_command pip install -e . run_command cd benchmark From 9d386da6208939c50cbe36aa2b2b9e16b110cbaf Mon Sep 17 00:00:00 2001 From: 0x45f Date: Fri, 5 Sep 2025 14:02:10 +0800 Subject: [PATCH 06/14] Fix --- .github/workflows/ci-temp-model-test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-temp-model-test.yaml b/.github/workflows/ci-temp-model-test.yaml index 9f1a109bdc..f3c489578c 100644 --- a/.github/workflows/ci-temp-model-test.yaml +++ b/.github/workflows/ci-temp-model-test.yaml @@ -26,7 +26,7 @@ jobs: path: benchmark - name: Install and run benchmark - shell: bash + shell: zsh run: | source tools/run_command.sh run_command source activate base From 616bcbc879d77aea688e3ba1f0084699bb6512ab Mon Sep 17 00:00:00 2001 From: 0x45f Date: Fri, 5 Sep 2025 14:04:00 +0800 Subject: [PATCH 07/14] Fix --- .github/workflows/ci-temp-model-test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-temp-model-test.yaml b/.github/workflows/ci-temp-model-test.yaml index f3c489578c..9f1a109bdc 100644 --- a/.github/workflows/ci-temp-model-test.yaml +++ b/.github/workflows/ci-temp-model-test.yaml @@ -26,7 +26,7 @@ jobs: path: benchmark - name: Install and run benchmark - shell: zsh + shell: bash run: | source tools/run_command.sh run_command source activate base From 8e98cd5c655728b84cbd247f781bbe42bdcbaae0 Mon Sep 17 00:00:00 2001 From: 0x45f Date: Fri, 5 Sep 2025 14:04:51 +0800 Subject: [PATCH 08/14] Fix --- .github/workflows/ci-temp-model-test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-temp-model-test.yaml b/.github/workflows/ci-temp-model-test.yaml index 9f1a109bdc..52d73cef71 100644 --- a/.github/workflows/ci-temp-model-test.yaml +++ b/.github/workflows/ci-temp-model-test.yaml @@ -29,7 +29,7 @@ jobs: shell: bash run: | source tools/run_command.sh - run_command source activate base + run_command conda init run_command conda activate torchbenchmark run_command pip install -e . run_command cd benchmark From 8219efcc1f4409fb9fa22f8e0b2f692ff50a1ea2 Mon Sep 17 00:00:00 2001 From: 0x45f Date: Fri, 5 Sep 2025 14:09:08 +0800 Subject: [PATCH 09/14] Fix --- .github/workflows/ci-temp-model-test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-temp-model-test.yaml b/.github/workflows/ci-temp-model-test.yaml index 52d73cef71..52a831f206 100644 --- a/.github/workflows/ci-temp-model-test.yaml +++ b/.github/workflows/ci-temp-model-test.yaml @@ -29,7 +29,7 @@ jobs: shell: bash run: | source tools/run_command.sh - run_command conda init + run_command bash ~/.bashrc run_command conda activate torchbenchmark run_command pip install -e . run_command cd benchmark From 8eb68bc12da495d3d85742519a49cc4ef3699d81 Mon Sep 17 00:00:00 2001 From: 0x45f Date: Fri, 5 Sep 2025 14:10:27 +0800 Subject: [PATCH 10/14] Fix --- .github/workflows/ci-temp-model-test.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci-temp-model-test.yaml b/.github/workflows/ci-temp-model-test.yaml index 52a831f206..371cc44e5d 100644 --- a/.github/workflows/ci-temp-model-test.yaml +++ b/.github/workflows/ci-temp-model-test.yaml @@ -30,6 +30,7 @@ jobs: run: | source tools/run_command.sh run_command bash ~/.bashrc + run_command conda init bash run_command conda activate torchbenchmark run_command pip install -e . run_command cd benchmark From d6b7344cbef9df3bad91c7ef232adcecdaff6efb Mon Sep 17 00:00:00 2001 From: 0x45f Date: Fri, 5 Sep 2025 14:16:27 +0800 Subject: [PATCH 11/14] Fix --- .github/workflows/ci-temp-model-test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-temp-model-test.yaml b/.github/workflows/ci-temp-model-test.yaml index 371cc44e5d..e1744b30e2 100644 --- a/.github/workflows/ci-temp-model-test.yaml +++ b/.github/workflows/ci-temp-model-test.yaml @@ -29,7 +29,7 @@ jobs: shell: bash run: | source tools/run_command.sh - run_command bash ~/.bashrc + run_command source "/root/anaconda3/etc/profile.d/conda.sh" run_command conda init bash run_command conda activate torchbenchmark run_command pip install -e . From e7919015ba7796d40456c029b5a3c78a09ca8175 Mon Sep 17 00:00:00 2001 From: 0x45f Date: Fri, 5 Sep 2025 16:58:32 +0800 Subject: [PATCH 12/14] Fix --- .github/workflows/ci-temp-model-test.yaml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci-temp-model-test.yaml b/.github/workflows/ci-temp-model-test.yaml index e1744b30e2..e5c70d9194 100644 --- a/.github/workflows/ci-temp-model-test.yaml +++ b/.github/workflows/ci-temp-model-test.yaml @@ -31,9 +31,7 @@ jobs: source tools/run_command.sh run_command source "/root/anaconda3/etc/profile.d/conda.sh" run_command conda init bash - run_command conda activate torchbenchmark + run_command conda activate wz-torch-bench run_command pip install -e . - run_command cd benchmark - run_command python install.py - run_command python run_benchmark.py test_bench --accuracy --device cuda --test eval --output output.json --models resnet50 - run_command cd .. + run_command python benchmark/install.py + run_command python benchmark/run_benchmark.py test_bench --accuracy --device cuda --test eval --output output.json --models resnet50 From 3670ccd477fed4a61bfece3d150431c103ab98d0 Mon Sep 17 00:00:00 2001 From: 0x45f Date: Fri, 5 Sep 2025 18:04:00 +0800 Subject: [PATCH 13/14] Fix --- .github/workflows/ci-temp-model-test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-temp-model-test.yaml b/.github/workflows/ci-temp-model-test.yaml index e5c70d9194..4d8216d26e 100644 --- a/.github/workflows/ci-temp-model-test.yaml +++ b/.github/workflows/ci-temp-model-test.yaml @@ -33,5 +33,5 @@ jobs: run_command conda init bash run_command conda activate wz-torch-bench run_command pip install -e . - run_command python benchmark/install.py + run_command python benchmark/install.py models resnet50 run_command python benchmark/run_benchmark.py test_bench --accuracy --device cuda --test eval --output output.json --models resnet50 From c1d91c3f5358dbc03351dda5344119020fba053b Mon Sep 17 00:00:00 2001 From: 0x45f Date: Mon, 15 Sep 2025 15:42:28 +0800 Subject: [PATCH 14/14] Fix --- .github/workflows/ci-temp-model-test.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/ci-temp-model-test.yaml b/.github/workflows/ci-temp-model-test.yaml index 4d8216d26e..24678fc038 100644 --- a/.github/workflows/ci-temp-model-test.yaml +++ b/.github/workflows/ci-temp-model-test.yaml @@ -33,5 +33,10 @@ jobs: run_command conda init bash run_command conda activate wz-torch-bench run_command pip install -e . + run_command sed -i '/self\.worker\.run("import torch")/a\ self.worker.run(\ + """\ + import flag_gems\ + flag_gems.enable(record=False, once=True, path='"'"'benchmark/oplist.log'"'"')""")' benchmark/torchbenchmark/__init__.py run_command python benchmark/install.py models resnet50 run_command python benchmark/run_benchmark.py test_bench --accuracy --device cuda --test eval --output output.json --models resnet50 + run_command sed -i '/self\.worker\.run($/,/^[[:space:]]*flag_gems\.enable.*oplist\.log.*""")/d' benchmark/torchbenchmark/__init__.py