preparing for the blog

.github/workflows/cla.yml

@@ -4,10 +4,11 @@ on:
   issue_comment:
     types: [created]
   pull_request_target:
-    types: [opened,closed,synchronize]
+    types: [opened,closed,synchronize]  
 
 jobs:
   cla-check:
+    if: github.repository_owner == 'mlcommons'
     runs-on: ubuntu-latest
     steps:
       - name: "MLCommons CLA bot check"
@@ -22,7 +23,7 @@ jobs:
           path-to-signatures: 'cla-bot/v1/cla.json'
           # branch should not be protected
           branch: 'main'
-          allowlist: user1,bot*
+          allowlist: user1,mlcommons-bot,bot*
           remote-organization-name: mlcommons
           remote-repository-name: systems
 

.github/workflows/test-cmlcr-mlperf-inference-rgat.yml

@@ -0,0 +1,55 @@
+name: MLPerf inference R-GAT via CMLCR
+
+on:
+  pull_request_target:
+    branches: [ "master", "main", "dev" ]
+    paths:
+      - '.github/workflows/test-mlperf-inference-rgat.yml'
+      - '**'
+      - '!**.md'
+
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest]
+        python-version: [ "3.12" ]
+        backend: [ "pytorch" ]
+        implementation: [ "python" ]
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v3
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install CMX
+      run: |
+        pip install cmind
+    - name: Install mlcflow
+      run: |
+        pip install mlcflow
+        pip install tabulate
+    - name: Pull MLOps repository
+      run: |
+        cmlc pull repo mlcommons@mlperf-automations --branch=main
+    - name: Test MLPerf Inference R-GAT using ${{ matrix.backend }} on ${{ matrix.os }}
+      run: |
+        cmlcr run,mlperf,inference,generate-run-cmds,_submission,_short --adr.inference-src.tags=_branch.dev --pull_changes=yes --pull_inference_changes=yes  --submitter="MLC" --hw_name=gh_${{ matrix.os }}_x86 --model=rgat --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --adr.compiler.tags=gcc --category=datacenter --quiet  -v --target_qps=1
+
+    - name: Push Test MLPerf Results (only for cTuning dev branches)
+      if: github.repository_owner == 'ctuning'
+      env:
+          USER: "GitHub Action"
+          EMAIL: "admin@cTuning.org"
+          GITHUB_TOKEN: ${{ secrets.TEST_RESULTS_GITHUB_TOKEN }}
+      run: |
+        git config --global user.name "${{ env.USER }}"
+        git config --global user.email "${{ env.EMAIL }}"
+        git config --global credential.https://github.com.helper ""
+        git config --global credential.https://github.com.helper "!gh auth git-credential"
+        git config --global credential.https://gist.github.com.helper ""
+        git config --global credential.https://gist.github.com.helper "!gh auth git-credential"
+        cmlcr push,github,mlperf,inference,submission --repo_url="https://github.com/ctuning/test_mlperf_inference_submissions" --repo_branch=main --commit_message="Results from R-Gat GH action on ${{ matrix.os }}" --quiet

.github/workflows/test-cmx-image-classification-onnx.yml

@@ -1,7 +1,7 @@
 # This workflow will install Python dependencies, run tests and lint with a variety of Python versions
 # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
 
-name: 'CMX: image classification with ONNX'
+name: 'Image classification with ONNX via CMX'
 
 on:
   pull_request:
@@ -29,7 +29,7 @@ jobs:
     - name: Install dependencies
       run: |
         python3 -m pip install cmind
-        cmx pull repo --url=https://github.com/mlcommons/cm4mlops
+        cmx pull repo mlcommons@ck --dir=cm4mlops/cm4mlops
         cmx test core
     - name: Test image classification with ONNX
       run: |

.github/workflows/test-cmx-mlperf-inference-bert-deepsparse-tf-onnxruntime-pytorch.yml

@@ -0,0 +1,67 @@
+# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+name: MLPerf inference bert (deepsparse, tf, onnxruntime, pytorch) via CMX
+
+on:
+  pull_request_target:
+    branches: [ "main", "master", "dev"]
+    paths:
+      - '.github/workflows/test-mlperf-inference-bert-deepsparse-tf-onnxruntime-pytorch.yml'
+      - '**'
+      - '!**.md'
+
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        # 3.12 didn't work on 20240305 - need to check
+        python-version: [ "3.11" ]
+        backend: [ "deepsparse", "tf", "onnxruntime", "pytorch" ]
+        precision: [ "int8", "fp32" ]
+        os: [ubuntu-latest, windows-latest, macos-latest]
+        exclude:
+          - backend: tf
+          - backend: pytorch
+          - backend: onnxruntime
+          - precision: fp32
+          - os: windows-latest
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v3
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Configure git longpaths (Windows)
+      if: matrix.os == 'windows-latest'
+      run: |
+        git config --system core.longpaths true
+    - name: Install dependencies
+      run: |
+        python3 -m pip install cmind
+        cmx pull repo mlcommons@ck --dir=cm4mlops/cm4mlops
+    - name: Test MLPerf Inference Bert ${{ matrix.backend }} on ${{ matrix.os }}
+      if: matrix.os == 'windows-latest'
+      run: |
+        cmx run script --tags=run,mlperf,inference,generate-run-cmds,_submission,_short --submitter="MLCommons" --hw_name=gh_${{ matrix.os }} --model=bert-99 --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=5 --adr.loadgen.tags=_from-pip --pip_loadgen=yes --precision=${{ matrix.precision }} --target_qps=1 --v --quiet
+    - name: Test MLPerf Inference Bert ${{ matrix.backend }} on ${{ matrix.os }}
+      if: matrix.os != 'windows-latest'
+      run: |
+        cmx run script --tags=run,mlperf,inference,generate-run-cmds,_submission,_short --submitter="MLCommons" --hw_name=gh_${{ matrix.os }}_x86 --model=bert-99 --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=5 --precision=${{ matrix.precision }} --target_qps=1 --v --quiet
+    - name: Push Test MLPerf Results (only for cTuning dev branches)
+      if: github.repository_owner == 'ctuning'
+      env:
+          USER: "GitHub Action"
+          EMAIL: "admin@cTuning.org"
+          GITHUB_TOKEN: ${{ secrets.TEST_RESULTS_GITHUB_TOKEN }}
+      run: |
+        git config --global user.name "${{ env.USER }}"
+        git config --global user.email "${{ env.EMAIL }}"
+        git config --global credential.https://github.com.helper ""
+        git config --global credential.https://github.com.helper "!gh auth git-credential"
+        git config --global credential.https://gist.github.com.helper ""
+        git config --global credential.https://gist.github.com.helper "!gh auth git-credential"
+        cmx run script --tags=push,github,mlperf,inference,submission  --repo_url="https://github.com/ctuning/test_mlperf_inference_submissions"  --repo_branch=main --commit_message="Results from R50 GH action on ${{ matrix.os }}" --quiet

.github/workflows/test-cmx-mlperf-inference-resnet50.yml

@@ -0,0 +1,67 @@
+# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+name: MLPerf inference ResNet50 via CMX
+
+on:
+  pull_request_target:
+    branches: [ "main", "master", "dev"]
+    paths:
+      - '.github/workflows/test-cmx-mlperf-inference-resnet50.yml'
+      - '**'
+      - '!**.md'
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    env:
+      CM_INDEX: "on"
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+        python-version: [ "3.12" ]
+        backend: [ "onnxruntime", "tf" ]
+        implementation: [ "python", "cpp" ]
+        exclude:
+          - backend: tf
+            implementation: cpp
+          - os: macos-latest
+            backend: tf
+          - os: windows-latest
+            implementation: cpp
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v3
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Configure git longpaths (Windows)
+      if: matrix.os == 'windows-latest'
+      run: |
+        git config --system core.longpaths true
+    - name: Install dependencies
+      run: |
+        python3 -m pip install cmind
+        cmx pull repo mlcommons@ck --dir=cm4mlops/cm4mlops
+    - name: Test MLPerf Inference ResNet50 (Windows)
+      if: matrix.os == 'windows-latest'
+      run: |
+        cmx run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --hw_name=gh_${{ matrix.os }}_x86 --model=resnet50 --adr.loadgen.tags=_from-pip --pip_loadgen=yes --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --target_qps=1  --quiet 
+    - name: Test MLPerf Inference ResNet50 (Linux/macOS)
+      if: matrix.os != 'windows-latest'
+      run: |
+        cmx run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --hw_name=gh_${{ matrix.os }}_x86 --model=resnet50 --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --target_qps=1 --v --quiet 
+    - name: Push Test MLPerf Results (only for cTuning dev branches)
+      if: github.repository_owner == 'ctuning'
+      env:
+          USER: "GitHub Action"
+          EMAIL: "admin@cTuning.org"
+          GITHUB_TOKEN: ${{ secrets.TEST_RESULTS_GITHUB_TOKEN }}
+      run: |
+        git config --global user.name "${{ env.USER }}"
+        git config --global user.email "${{ env.EMAIL }}"
+        git config --global credential.https://github.com.helper ""
+        git config --global credential.https://github.com.helper "!gh auth git-credential"
+        git config --global credential.https://gist.github.com.helper ""
+        git config --global credential.https://gist.github.com.helper "!gh auth git-credential"
+        cmx run script --tags=push,github,mlperf,inference,submission  --repo_url="https://github.com/ctuning/test_mlperf_inference_submissions"  --repo_branch=main --commit_message="Results from R50 GH action on ${{ matrix.os }}" --quiet

.github/workflows/test-cmx-mlperf-inference-rgat.yml

@@ -0,0 +1,52 @@
+name: MLPerf inference R-GAT via CMX
+
+on:
+  pull_request:
+    branches: [ "main", "master", "dev" ]
+    paths:
+      - '.github/workflows/test-mlperf-inference-rgat.yml'
+      - '**'
+      - '!**.md'
+
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest]
+        python-version: [ "3.12" ]
+        backend: [ "pytorch" ]
+        implementation: [ "python" ]
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v3
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Configure git longpaths (Windows)
+      if: matrix.os == 'windows-latest'
+      run: |
+        git config --system core.longpaths true
+    - name: Install dependencies
+      run: |
+        python3 -m pip install cmind
+        cmx pull repo mlcommons@ck --dir=cm4mlops/cm4mlops
+    - name: Test MLPerf Inference R-GAT using ${{ matrix.backend }} on ${{ matrix.os }}
+      run: |
+        cmx run script --tags=run,mlperf,inference,generate-run-cmds,_submission,_short --submitter="MLCommons" --adr.inference-src.tags=_branch.dev --pull_changes=yes --pull_inference_changes=yes  --submitter="MLCommons" --hw_name=gh_${{ matrix.os }}_x86 --model=rgat --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --adr.compiler.tags=gcc --category=datacenter --quiet  --v --target_qps=1
+    - name: Push Test MLPerf Results (only for cTuning dev branches)
+      if: github.repository_owner == 'ctuning'
+      env:
+          USER: "GitHub Action"
+          EMAIL: "admin@cTuning.org"
+          GITHUB_TOKEN: ${{ secrets.TEST_RESULTS_GITHUB_TOKEN }}
+      run: |
+        git config --global user.name "${{ env.USER }}"
+        git config --global user.email "${{ env.EMAIL }}"
+        git config --global credential.https://github.com.helper ""
+        git config --global credential.https://github.com.helper "!gh auth git-credential"
+        git config --global credential.https://gist.github.com.helper ""
+        git config --global credential.https://gist.github.com.helper "!gh auth git-credential"
+        cmx run script --tags=push,github,mlperf,inference,submission  --repo_url="https://github.com/ctuning/test_mlperf_inference_submissions"  --repo_branch=main --commit_message="Results from R50 GH action on ${{ matrix.os }}" --quiet

.github/workflows/test-cmx.yml

@@ -0,0 +1,51 @@
+# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+name: CMX test
+
+on:
+  pull_request:
+    branches: [ "master", "main", "dev" ]
+    paths:
+      - '.github/workflows/test-cm.yml'
+      - 'cm/**'
+      - '!cm/**.md'
+
+jobs:
+  build:
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+        on: [ubuntu-latest, windows-latest, macos-latest]
+        exclude:
+          - python-version: "3.7"
+            on: "macos-latest"
+    runs-on: "${{ matrix.on }}"
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        ref: ${{ github.event.pull_request.head.sha }}
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v3
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies and test cm pull repo
+      run: |
+        python3 -m pip install --upgrade pip
+        python3 -m pip install flake8 pytest
+        python3 -m pip install -r requirements.txt
+        python3 -m pip install --ignore-installed --verbose pip setuptools
+        python3 -m pip install cmind
+        python3 -m cmind
+        cmx pull repo mlcommons@ck --dir2=cm4mlops
+    - name: Test
+      run: |
+        python tests/test_cm.py
+        cd && mkdir tmp_cm_repo
+        cd tmp_cm_repo && cm init repo
+        cmx add script tmp_cm_repo:my-test-script --tags=test,script
+        cmx add script .:my-test-script2 --tags=test2,script
+        cd && cmx add repo my-test-repo
+        cmx add script my-test-repo:my-test-script --tags=test,script
+        cd $HOME/CM/repos/my-test-repo && cmx add script .:my-test-script2 --tags=test2,script