diff --git a/.github/workflows/docker_images.yml b/.github/workflows/docker_images.yml index 47503a0b..bf58bc9b 100644 --- a/.github/workflows/docker_images.yml +++ b/.github/workflows/docker_images.yml @@ -32,8 +32,8 @@ jobs: - docker/lightgbm-v3.2.1/linux_cuda_build.dockerfile - docker/lightgbm-v3.2.1/linux_gpu_build.dockerfile - docker/lightgbm-v3.2.1/linux_gpu_pip.dockerfile - - docker/lightgbm-v3.3.0/linux_cpu_mpi_build.dockerfile - - docker/lightgbm-v3.3.0/linux_cpu_mpi_pip.dockerfile + - docker/lightgbm-v3.3.3/linux_cpu_mpi_build.dockerfile + - docker/lightgbm-v3.3.3/linux_cpu_mpi_pip.dockerfile - docker/lightgbm-custom/v321_patch_cpu_mpi_build.dockerfile - src/scripts/training/lightgbm_python/default.dockerfile - src/scripts/inferencing/lightgbm_python/default.dockerfile @@ -73,7 +73,7 @@ jobs: matrix: dockerfile: - docker/lightgbm-v3.2.1/windows_cpu_pip.dockerfile - - docker/lightgbm-v3.3.0/windows_cpu_pip.dockerfile + - docker/lightgbm-v3.3.3/windows_cpu_pip.dockerfile steps: - uses: actions/checkout@v2 diff --git a/conf/experiments/benchmarks/lightgbm-inferencing.yaml b/conf/experiments/benchmarks/lightgbm-inferencing.yaml index 188b7b0c..4ee672c1 100644 --- a/conf/experiments/benchmarks/lightgbm-inferencing.yaml +++ b/conf/experiments/benchmarks/lightgbm-inferencing.yaml @@ -71,10 +71,10 @@ lightgbm_inferencing_config: # list all inferencing frameworks and their builds variants: - - framework: lightgbm_python # v3.3.0 via pypi - - framework: lightgbm_c_api # v3.3.0 with C API prediction - - framework: lightgbm_c_api # v3.3.0 with C API prediction - build: docker/lightgbm-custom/v330_patch_cpu_mpi_build.dockerfile + - framework: lightgbm_python # v3.3.3 via pypi + - framework: lightgbm_c_api # v3.3.3 with C API prediction + - framework: lightgbm_c_api # v3.3.3 with C API prediction + build: docker/lightgbm-custom/v333_patch_cpu_mpi_build.dockerfile - framework: lightgbm_c_api # v3.2.1 with C API prediction build: docker/lightgbm-v3.2.1/linux_cpu_mpi_build.dockerfile - framework: lightgbm_c_api # v3.2.1 with C API prediction diff --git a/conf/experiments/lightgbm-inferencing.yaml b/conf/experiments/lightgbm-inferencing.yaml index 2e68c4d3..c14be28b 100644 --- a/conf/experiments/lightgbm-inferencing.yaml +++ b/conf/experiments/lightgbm-inferencing.yaml @@ -27,10 +27,10 @@ lightgbm_inferencing_config: # list all inferencing frameworks and their builds variants: - - framework: lightgbm_python # v3.3.0 via pypi - - framework: lightgbm_c_api # v3.3.0 with C API prediction - - framework: lightgbm_c_api # v3.3.0 with C API prediction - build: docker/lightgbm-custom/v330_patch_cpu_mpi_build.dockerfile + - framework: lightgbm_python # v3.3.3 via pypi + - framework: lightgbm_c_api # v3.3.3 with C API prediction + - framework: lightgbm_c_api # v3.3.3 with C API prediction + build: docker/lightgbm-custom/v333_patch_cpu_mpi_build.dockerfile - framework: lightgbm_c_api # v3.2.1 with C API prediction build: docker/lightgbm-v3.2.1/linux_cpu_mpi_build.dockerfile - framework: lightgbm_c_api # v3.2.1 with C API prediction diff --git a/conf/experiments/lightgbm_training/cpu-custom.yaml b/conf/experiments/lightgbm_training/cpu-custom.yaml index bd22666f..39c087e8 100644 --- a/conf/experiments/lightgbm_training/cpu-custom.yaml +++ b/conf/experiments/lightgbm_training/cpu-custom.yaml @@ -63,7 +63,7 @@ lightgbm_training_config: #target: null # optional: force target for this training job nodes: 1 processes: 1 - build: "docker/lightgbm-custom/v330_patch_cpu_mpi_build.dockerfile" # relative to lightgbm_python folder + build: "docker/lightgbm-custom/v333_patch_cpu_mpi_build.dockerfile" # relative to lightgbm_python folder # model registration # naming convention: "{register_model_prefix}-{task_key}-{num_iterations}trees-{num_leaves}leaves-{register_model_suffix}" diff --git a/conf/experiments/lightgbm_training/gpu.yaml b/conf/experiments/lightgbm_training/gpu.yaml index 692f36cc..66866bf3 100644 --- a/conf/experiments/lightgbm_training/gpu.yaml +++ b/conf/experiments/lightgbm_training/gpu.yaml @@ -63,7 +63,7 @@ lightgbm_training_config: #target: null # optional: force target for this training job nodes: 1 processes: 1 - build: "docker/lightgbm-v3.3.0/linux_gpu_pip.dockerfile" # relative to lightgbm_python folder + build: "docker/lightgbm-v3.3.3/linux_gpu_pip.dockerfile" # relative to lightgbm_python folder # model registration # naming convention: "{register_model_prefix}-{task_key}-{num_iterations}trees-{num_leaves}leaves-{register_model_suffix}" diff --git a/conf/experiments/lightgbm_training/mpi.yaml b/conf/experiments/lightgbm_training/mpi.yaml index 23ce128a..96198d59 100644 --- a/conf/experiments/lightgbm_training/mpi.yaml +++ b/conf/experiments/lightgbm_training/mpi.yaml @@ -62,7 +62,7 @@ lightgbm_training_config: # compute parameters runtime: #target: null # optional: force target for this training job - build: "docker/lightgbm-v3.3.0/linux_cpu_mpi_pip.dockerfile" + build: "docker/lightgbm-v3.3.3/linux_cpu_mpi_pip.dockerfile" nodes: 2 processes: 1 diff --git a/docker/lightgbm-custom/v330_patch_cpu_mpi_build.dockerfile b/docker/lightgbm-custom/v333_patch_cpu_mpi_build.dockerfile similarity index 84% rename from docker/lightgbm-custom/v330_patch_cpu_mpi_build.dockerfile rename to docker/lightgbm-custom/v333_patch_cpu_mpi_build.dockerfile index 7041380f..8174a84f 100644 --- a/docker/lightgbm-custom/v330_patch_cpu_mpi_build.dockerfile +++ b/docker/lightgbm-custom/v333_patch_cpu_mpi_build.dockerfile @@ -1,10 +1,10 @@ FROM mcr.microsoft.com/azureml/openmpi3.1.2-ubuntu18.04:20210615.v1 -LABEL lightgbmbenchmark.linux.cpu.mpi.build.version="3.3.0-patch/20211109.1" +LABEL lightgbmbenchmark.linux.cpu.mpi.build.version="3.3.3-patch/20221020.1" # Those arguments will NOT be used by AzureML # they are here just to allow for lightgbm-benchmark build to actually check # dockerfiles in a PR against their actual branch -ARG lightgbm_version="3.3.0" +ARG lightgbm_version="3.3.3" ARG lightgbm_benchmark_branch=main RUN apt-get update && \ @@ -62,15 +62,15 @@ ENV PATH $AZUREML_CONDA_ENVIRONMENT_PATH/bin:$PATH # Install pip dependencies RUN HOROVOD_WITH_TENSORFLOW=1 \ pip install 'pandas>=1.1,<1.2' \ - 'numpy>=1.10,<1.20' \ - 'matplotlib==3.4.3' \ - 'scipy~=1.5.0' \ - 'scikit-learn~=0.24.1' \ - 'azureml-core==1.35.0' \ - 'azureml-defaults==1.35.0' \ - 'azureml-mlflow==1.35.0' \ - 'azureml-telemetry==1.35.0' \ - 'mpi4py==3.1.1' + 'numpy>=1.10,<1.20' \ + 'matplotlib==3.4.3' \ + 'scipy~=1.5.0' \ + 'scikit-learn~=0.24.1' \ + 'azureml-core==1.35.0' \ + 'azureml-defaults==1.35.0' \ + 'azureml-mlflow==1.35.0' \ + 'azureml-telemetry==1.35.0' \ + 'mpi4py==3.1.1' RUN pip install --upgrade pip setuptools wheel && \ pip install 'cmake==3.21.0' diff --git a/docker/lightgbm-v3.3.0/linux_cpu_mpi_build.dockerfile b/docker/lightgbm-v3.3.3/linux_cpu_mpi_build.dockerfile similarity index 82% rename from docker/lightgbm-v3.3.0/linux_cpu_mpi_build.dockerfile rename to docker/lightgbm-v3.3.3/linux_cpu_mpi_build.dockerfile index 4f66f5a0..31d7d2e3 100644 --- a/docker/lightgbm-v3.3.0/linux_cpu_mpi_build.dockerfile +++ b/docker/lightgbm-v3.3.3/linux_cpu_mpi_build.dockerfile @@ -1,10 +1,10 @@ FROM mcr.microsoft.com/azureml/openmpi3.1.2-ubuntu18.04:20211012.v1 -LABEL lightgbmbenchmark.linux.cpu.mpi.build.version="3.3.0/20211115.1" +LABEL lightgbmbenchmark.linux.cpu.mpi.build.version="3.3.3/20221020.1" # Those arguments will NOT be used by AzureML # they are here just to allow for lightgbm-benchmark build to actually check # dockerfiles in a PR against their actual branch -ARG lightgbm_version="3.3.0" +ARG lightgbm_version="3.3.3" ARG lightgbm_benchmark_branch=main RUN apt-get update && \ @@ -57,15 +57,15 @@ ENV PATH $AZUREML_CONDA_ENVIRONMENT_PATH/bin:$PATH # Install pip dependencies RUN HOROVOD_WITH_TENSORFLOW=1 \ pip install 'pandas>=1.1,<1.2' \ - 'numpy>=1.10,<1.20' \ - 'matplotlib==3.4.3' \ - 'scipy~=1.5.0' \ - 'scikit-learn~=0.24.1' \ - 'azureml-core==1.35.0' \ - 'azureml-defaults==1.35.0' \ - 'azureml-mlflow==1.35.0' \ - 'azureml-telemetry==1.35.0' \ - 'mpi4py==3.1.1' + 'numpy>=1.10,<1.20' \ + 'matplotlib==3.4.3' \ + 'scipy~=1.5.0' \ + 'scikit-learn~=0.24.1' \ + 'azureml-core==1.35.0' \ + 'azureml-defaults==1.35.0' \ + 'azureml-mlflow==1.35.0' \ + 'azureml-telemetry==1.35.0' \ + 'mpi4py==3.1.1' RUN pip install --upgrade pip setuptools wheel && \ pip install 'cmake==3.21.0' diff --git a/docker/lightgbm-v3.3.0/linux_cpu_mpi_pip.dockerfile b/docker/lightgbm-v3.3.3/linux_cpu_mpi_pip.dockerfile similarity index 67% rename from docker/lightgbm-v3.3.0/linux_cpu_mpi_pip.dockerfile rename to docker/lightgbm-v3.3.3/linux_cpu_mpi_pip.dockerfile index 59ca3f32..82324ff0 100644 --- a/docker/lightgbm-v3.3.0/linux_cpu_mpi_pip.dockerfile +++ b/docker/lightgbm-v3.3.3/linux_cpu_mpi_pip.dockerfile @@ -1,10 +1,10 @@ FROM mcr.microsoft.com/azureml/openmpi4.1.0-ubuntu20.04:latest -LABEL lightgbmbenchmark.linux.cpu.mpi.pip.version="3.3.0/20220114.1" +LABEL lightgbmbenchmark.linux.cpu.mpi.pip.version="3.3.3/20221020.1" # Those arguments will NOT be used by AzureML when building the image # they are here just to allow for lightgbm-benchmark build to actually check # dockerfiles in a PR against their actual branch -ARG lightgbm_version="3.3.0" +ARG lightgbm_version="3.3.3" ENV AZUREML_CONDA_ENVIRONMENT_PATH /azureml-envs/lightgbm @@ -18,15 +18,15 @@ ENV PATH $AZUREML_CONDA_ENVIRONMENT_PATH/bin:$PATH # Install pip dependencies RUN HOROVOD_WITH_TENSORFLOW=1 \ pip install 'pandas>=1.1,<1.2' \ - 'numpy>=1.10,<1.20' \ - 'matplotlib==3.4.3' \ - 'scipy~=1.5.0' \ - 'scikit-learn~=0.24.1' \ - 'azureml-core==1.35.0' \ - 'azureml-defaults==1.35.0' \ - 'azureml-mlflow==1.35.0' \ - 'azureml-telemetry==1.35.0' \ - 'mpi4py==3.1.1' + 'numpy>=1.10,<1.20' \ + 'matplotlib==3.4.3' \ + 'scipy~=1.5.0' \ + 'scikit-learn~=0.24.1' \ + 'azureml-core==1.35.0' \ + 'azureml-defaults==1.35.0' \ + 'azureml-mlflow==1.35.0' \ + 'azureml-telemetry==1.35.0' \ + 'mpi4py==3.1.1' # install lightgbm with mpi RUN pip install --upgrade pip setuptools wheel && \ diff --git a/docker/lightgbm-v3.3.0/windows_cpu_pip.dockerfile b/docker/lightgbm-v3.3.3/windows_cpu_pip.dockerfile similarity index 56% rename from docker/lightgbm-v3.3.0/windows_cpu_pip.dockerfile rename to docker/lightgbm-v3.3.3/windows_cpu_pip.dockerfile index f40508e0..24d46cdd 100644 --- a/docker/lightgbm-v3.3.0/windows_cpu_pip.dockerfile +++ b/docker/lightgbm-v3.3.3/windows_cpu_pip.dockerfile @@ -1,10 +1,10 @@ FROM mcr.microsoft.com/azureml/windows-servercore-1809:latest -LABEL lightgbmbenchmark.windows.cpu.mpi.pip.version="3.3.0/20211108.1" +LABEL lightgbmbenchmark.windows.cpu.mpi.pip.version="3.3.3/20221020.1" # Those arguments will NOT be used by AzureML # they are here just to allow for lightgbm-benchmark build to actually check # dockerfiles in a PR against their actual branch -ARG lightgbm_version="3.3.0" +ARG lightgbm_version="3.3.3" ENV AZUREML_CONDA_ENVIRONMENT_PATH /azureml-envs/lightgbm @@ -17,11 +17,11 @@ RUN conda create -p $AZUREML_CONDA_ENVIRONMENT_PATH \ # Install pip dependencies RUN pip install 'pandas>=1.1,<1.2' \ - 'numpy>=1.10,<1.20' \ - 'scipy~=1.5.0' \ - 'scikit-learn~=0.24.1' \ - 'azureml-core==1.35.0' \ - 'azureml-defaults==1.35.0' \ - 'azureml-mlflow==1.35.0' \ - 'azureml-telemetry==1.35.0' \ - lightgbm==$($Env:lightgbm_version) + 'numpy>=1.10,<1.20' \ + 'scipy~=1.5.0' \ + 'scikit-learn~=0.24.1' \ + 'azureml-core==1.35.0' \ + 'azureml-defaults==1.35.0' \ + 'azureml-mlflow==1.35.0' \ + 'azureml-telemetry==1.35.0' \ + lightgbm==$($Env:lightgbm_version) diff --git a/docs/run/azureml/benchmark-inferencing.md b/docs/run/azureml/benchmark-inferencing.md index 6f612c98..7deae1ce 100644 --- a/docs/run/azureml/benchmark-inferencing.md +++ b/docs/run/azureml/benchmark-inferencing.md @@ -34,10 +34,10 @@ lightgbm_inferencing_config: # list all inferencing frameworks and their builds variants: - - framework: lightgbm_python # v3.3.0 via pypi - - framework: lightgbm_c_api # v3.3.0 with C API prediction - - framework: lightgbm_c_api # v3.3.0 with C API prediction - build: docker/lightgbm-custom/v330_patch_cpu_mpi_build.dockerfile + - framework: lightgbm_python # v3.3.3 via pypi + - framework: lightgbm_c_api # v3.3.3 with C API prediction + - framework: lightgbm_c_api # v3.3.3 with C API prediction + build: docker/lightgbm-custom/v333_patch_cpu_mpi_build.dockerfile - framework: lightgbm_c_api # v3.2.1 with C API prediction build: docker/lightgbm-v3.2.1/linux_cpu_mpi_build.dockerfile - framework: lightgbm_c_api # v3.2.1 with C API prediction @@ -88,10 +88,10 @@ For each task, the pipeline will run inferencing "variants", which are different ```yaml # list all inferencing frameworks and their builds variants: - - framework: lightgbm_python # v3.3.0 via pypi - - framework: lightgbm_c_api # v3.3.0 with C API prediction - - framework: lightgbm_c_api # v3.3.0 with C API prediction - build: docker/lightgbm-custom/v330_patch_cpu_mpi_build.dockerfile + - framework: lightgbm_python # v3.3.3 via pypi + - framework: lightgbm_c_api # v3.3.3 with C API prediction + - framework: lightgbm_c_api # v3.3.3 with C API prediction + build: docker/lightgbm-custom/v333_patch_cpu_mpi_build.dockerfile - framework: lightgbm_c_api # v3.2.1 with C API prediction build: docker/lightgbm-v3.2.1/linux_cpu_mpi_build.dockerfile - framework: lightgbm_c_api # v3.2.1 with C API prediction diff --git a/docs/run/azureml/train-on-your-data.md b/docs/run/azureml/train-on-your-data.md index af2e2abb..d9c24f21 100644 --- a/docs/run/azureml/train-on-your-data.md +++ b/docs/run/azureml/train-on-your-data.md @@ -101,7 +101,7 @@ lightgbm_training_config: training: device_type: "gpu" runtime: - build: "docker/lightgbm-v3.3.0/linux_gpu_pip.dockerfile" + build: "docker/lightgbm-v3.3.3/linux_gpu_pip.dockerfile" ``` When running the pipeline, it will automatically run on the gpu cluster you've named in your `compute/custom.yaml` file. diff --git a/src/scripts/inferencing/lightgbm_c_api/default.dockerfile b/src/scripts/inferencing/lightgbm_c_api/default.dockerfile index f50022c5..e954f316 100644 --- a/src/scripts/inferencing/lightgbm_c_api/default.dockerfile +++ b/src/scripts/inferencing/lightgbm_c_api/default.dockerfile @@ -1,10 +1,10 @@ FROM mcr.microsoft.com/azureml/openmpi3.1.2-ubuntu18.04:20211012.v1 -LABEL lightgbmbenchmark.linux.cpu.mpi.build.version="3.3.0/20211115.1" +LABEL lightgbmbenchmark.linux.cpu.mpi.build.version="3.3.3/20221020.1" # Those arguments will NOT be used by AzureML # they are here just to allow for lightgbm-benchmark build to actually check # dockerfiles in a PR against their actual branch -ARG lightgbm_version="3.3.0" +ARG lightgbm_version="3.3.3" ARG lightgbm_benchmark_branch=main RUN apt-get update && \ @@ -57,15 +57,15 @@ ENV PATH $AZUREML_CONDA_ENVIRONMENT_PATH/bin:$PATH # Install pip dependencies RUN HOROVOD_WITH_TENSORFLOW=1 \ pip install 'pandas>=1.1,<1.2' \ - 'numpy>=1.10,<1.20' \ - 'matplotlib==3.4.3' \ - 'scipy~=1.5.0' \ - 'scikit-learn~=0.24.1' \ - 'azureml-core==1.35.0' \ - 'azureml-defaults==1.35.0' \ - 'azureml-mlflow==1.35.0' \ - 'azureml-telemetry==1.35.0' \ - 'mpi4py==3.1.1' + 'numpy>=1.10,<1.20' \ + 'matplotlib==3.4.3' \ + 'scipy~=1.5.0' \ + 'scikit-learn~=0.24.1' \ + 'azureml-core==1.35.0' \ + 'azureml-defaults==1.35.0' \ + 'azureml-mlflow==1.35.0' \ + 'azureml-telemetry==1.35.0' \ + 'mpi4py==3.1.1' RUN pip install --upgrade pip setuptools wheel && \ pip install 'cmake==3.21.0' diff --git a/src/scripts/inferencing/lightgbm_ort/default.dockerfile b/src/scripts/inferencing/lightgbm_ort/default.dockerfile new file mode 100644 index 00000000..6384b1f4 --- /dev/null +++ b/src/scripts/inferencing/lightgbm_ort/default.dockerfile @@ -0,0 +1,32 @@ +FROM mcr.microsoft.com/azureml/openmpi4.1.0-ubuntu20.04:latest +LABEL lightgbmbenchmark.linux.cpu.ray.version="0.1.2/20220111.1" + +ENV AZUREML_CONDA_ENVIRONMENT_PATH /azureml-envs/lightgbm + +ARG lightgbm_version="3.3.3" + +# Create conda environment +RUN conda create -p $AZUREML_CONDA_ENVIRONMENT_PATH \ + python=3.8 pip=20.2.4 + +# Prepend path to AzureML conda environment +ENV PATH $AZUREML_CONDA_ENVIRONMENT_PATH/bin:$PATH + +# Install pip dependencies +RUN pip install 'pandas>=1.1,<1.2' \ + 'numpy>=1.10,<1.20' \ + 'matplotlib==3.4.3' \ + 'scipy~=1.5.0' \ + 'scikit-learn~=0.24.1' \ + 'azureml-core==1.35.0' \ + 'azureml-defaults==1.35.0' \ + 'azureml-mlflow==1.35.0' \ + 'azureml-telemetry==1.35.0' \ + 'mpi4py==3.1.1' \ + 'onnxruntime==1.12.1' \ + 'onnxmltools==1.11.1' \ + 'onnxconverter-common==1.12.2' + +# install lightgbm with mpi +RUN pip install lightgbm==${lightgbm_version} \ + pip install 'protobuf==3.20' \ No newline at end of file diff --git a/src/scripts/inferencing/lightgbm_python/default.dockerfile b/src/scripts/inferencing/lightgbm_python/default.dockerfile index 419a5444..60aaeb30 100644 --- a/src/scripts/inferencing/lightgbm_python/default.dockerfile +++ b/src/scripts/inferencing/lightgbm_python/default.dockerfile @@ -1,10 +1,10 @@ FROM mcr.microsoft.com/azureml/openmpi3.1.2-ubuntu18.04:20210615.v1 -LABEL lightgbmbenchmark.linux.cpu.mpi.pip.version="3.3.0/20211210.1" +LABEL lightgbmbenchmark.linux.cpu.mpi.pip.version="3.3.3/20221020.1" # Those arguments will NOT be used by AzureML # they are here just to allow for lightgbm-benchmark build to actually check # dockerfiles in a PR against their actual branch -ARG lightgbm_version="3.3.0" +ARG lightgbm_version="3.3.3" ENV AZUREML_CONDA_ENVIRONMENT_PATH /azureml-envs/lightgbm @@ -18,15 +18,15 @@ ENV PATH $AZUREML_CONDA_ENVIRONMENT_PATH/bin:$PATH # Install pip dependencies RUN HOROVOD_WITH_TENSORFLOW=1 \ pip install 'pandas>=1.1,<1.2' \ - 'numpy>=1.10,<1.20' \ - 'matplotlib==3.4.3' \ - 'scipy~=1.5.0' \ - 'scikit-learn~=0.24.1' \ - 'azureml-core==1.35.0' \ - 'azureml-defaults==1.35.0' \ - 'azureml-mlflow==1.35.0' \ - 'azureml-telemetry==1.35.0' \ - 'mpi4py==3.1.1' + 'numpy>=1.10,<1.20' \ + 'matplotlib==3.4.3' \ + 'scipy~=1.5.0' \ + 'scikit-learn~=0.24.1' \ + 'azureml-core==1.35.0' \ + 'azureml-defaults==1.35.0' \ + 'azureml-mlflow==1.35.0' \ + 'azureml-telemetry==1.35.0' \ + 'mpi4py==3.1.1' # install lightgbm with mpi RUN pip install --upgrade pip setuptools wheel && \ diff --git a/src/scripts/training/lightgbm_python/default.dockerfile b/src/scripts/training/lightgbm_python/default.dockerfile index 6848faf6..e4df1ae7 100644 --- a/src/scripts/training/lightgbm_python/default.dockerfile +++ b/src/scripts/training/lightgbm_python/default.dockerfile @@ -1,5 +1,5 @@ FROM mcr.microsoft.com/azureml/openmpi4.1.0-ubuntu20.04:20220516.v1 -LABEL lightgbmbenchmark.linux.cpu.mpi.pip.version="3.3.1/20211210.1" +LABEL lightgbmbenchmark.linux.cpu.mpi.pip.version="3.3.3/20221020.1" # https://github.com/microsoft/lightgbm-transform/blob/main/docs/Installation-Guide.rst # Install CMake, gcc, g++, boost. @@ -24,8 +24,8 @@ RUN mkdir build && cd build && cmake -DBOND_ENABLE_GRPC=FALSE .. && make -j4 && # Those arguments will NOT be used by AzureML # they are here just to allow for lightgbm-benchmark build to actually check # dockerfiles in a PR against their actual branch -ARG lightgbm_version="3.3.1" -ARG lightgbm_transform_version="3.3.1.post1" +ARG lightgbm_version="3.3.3" +ARG lightgbm_transform_version="3.3.2.post1" ENV AZUREML_CONDA_ENVIRONMENT_PATH /azureml-envs/lightgbm @@ -39,16 +39,16 @@ ENV PATH $AZUREML_CONDA_ENVIRONMENT_PATH/bin:$PATH # Install pip dependencies RUN HOROVOD_WITH_TENSORFLOW=1 \ pip install 'pandas>=1.1,<1.2' \ - 'numpy>=1.10,<1.20' \ - 'matplotlib==3.4.3' \ - 'scipy~=1.5.0' \ - 'scikit-learn~=0.24.1' \ - 'azureml-core==1.35.0' \ - 'azureml-defaults==1.35.0' \ - 'azureml-mlflow==1.35.0' \ - 'azureml-telemetry==1.35.0' \ - 'mpi4py==3.1.1' \ - 'omegaconf' + 'numpy>=1.10,<1.20' \ + 'matplotlib==3.4.3' \ + 'scipy~=1.5.0' \ + 'scikit-learn~=0.24.1' \ + 'azureml-core==1.35.0' \ + 'azureml-defaults==1.35.0' \ + 'azureml-mlflow==1.35.0' \ + 'azureml-telemetry==1.35.0' \ + 'mpi4py==3.1.1' \ + 'omegaconf' # install lightgbm with mpi RUN pip install --upgrade pip setuptools wheel && \ diff --git a/src/scripts/training/lightgbm_python/spec.yaml b/src/scripts/training/lightgbm_python/spec.yaml index a569edd4..1ae43770 100644 --- a/src/scripts/training/lightgbm_python/spec.yaml +++ b/src/scripts/training/lightgbm_python/spec.yaml @@ -18,7 +18,7 @@ tags: git: https://github.com/microsoft/lightgbm-benchmark docs: https://microsoft.github.io/lightgbm-benchmark framework: lightgbm - framework_version: 3.3.0-basic + framework_version: 3.3.3-basic inputs: # Inputs diff --git a/src/scripts/training/ray_tune/default.dockerfile b/src/scripts/training/ray_tune/default.dockerfile index fcf659e6..92911fb6 100644 --- a/src/scripts/training/ray_tune/default.dockerfile +++ b/src/scripts/training/ray_tune/default.dockerfile @@ -1,7 +1,7 @@ FROM mcr.microsoft.com/azureml/openmpi4.1.0-ubuntu20.04:latest LABEL lightgbmbenchmark.linux.cpu.ray.version="0.1.2/20220111.1" -ARG lightgbm_version="3.3.0" +ARG lightgbm_version="3.3.3" ENV AZUREML_CONDA_ENVIRONMENT_PATH /azureml-envs/lightgbm @@ -15,19 +15,19 @@ ENV PATH $AZUREML_CONDA_ENVIRONMENT_PATH/bin:$PATH # Install pip dependencies RUN HOROVOD_WITH_TENSORFLOW=1 \ pip install 'pandas>=1.1,<1.2' \ - 'numpy>=1.10,<1.20' \ - 'matplotlib==3.4.3' \ - 'scipy~=1.5.0' \ - 'scikit-learn~=0.24.1' \ - 'azureml-core==1.35.0' \ - 'azureml-defaults==1.35.0' \ - 'azureml-mlflow==1.35.0' \ - 'azureml-telemetry==1.35.0' \ - 'ray==1.9.2' \ - 'flaml==0.9.6' \ - 'mpi4py==3.1.1' \ - 'hpbandster==0.7.4' \ - 'ConfigSpace==0.5.0' \ - 'optuna==2.8.0' \ - 'protobuf==3.20.1' \ - lightgbm==${lightgbm_version} \ No newline at end of file + 'numpy>=1.10,<1.20' \ + 'matplotlib==3.4.3' \ + 'scipy~=1.5.0' \ + 'scikit-learn~=0.24.1' \ + 'azureml-core==1.35.0' \ + 'azureml-defaults==1.35.0' \ + 'azureml-mlflow==1.35.0' \ + 'azureml-telemetry==1.35.0' \ + 'ray==1.9.2' \ + 'flaml==0.9.6' \ + 'mpi4py==3.1.1' \ + 'hpbandster==0.7.4' \ + 'ConfigSpace==0.5.0' \ + 'optuna==2.8.0' \ + 'protobuf==3.20.1' \ + lightgbm==${lightgbm_version} \ No newline at end of file diff --git a/src/scripts/training/ray_tune_distributed/default.dockerfile b/src/scripts/training/ray_tune_distributed/default.dockerfile index 5bb04a04..aed0cfde 100644 --- a/src/scripts/training/ray_tune_distributed/default.dockerfile +++ b/src/scripts/training/ray_tune_distributed/default.dockerfile @@ -1,7 +1,7 @@ FROM mcr.microsoft.com/azureml/openmpi4.1.0-ubuntu20.04:latest LABEL lightgbmbenchmark.linux.cpu.ray.version="0.1.2/20220111.1" -ARG lightgbm_version="3.3.0" +ARG lightgbm_version="3.3.3" ENV AZUREML_CONDA_ENVIRONMENT_PATH /azureml-envs/lightgbm @@ -15,16 +15,16 @@ ENV PATH $AZUREML_CONDA_ENVIRONMENT_PATH/bin:$PATH # Install pip dependencies RUN HOROVOD_WITH_TENSORFLOW=1 \ pip install 'pandas>=1.1,<1.2' \ - 'numpy>=1.10,<1.20' \ - 'matplotlib==3.4.3' \ - 'scipy~=1.5.0' \ - 'scikit-learn~=0.24.1' \ - 'azureml-core==1.35.0' \ - 'azureml-defaults==1.35.0' \ - 'azureml-mlflow==1.35.0' \ - 'azureml-telemetry==1.35.0' \ - 'ray==1.9.2' \ - 'flaml==0.9.6' \ - 'lightgbm-ray==0.1.4' \ - 'mpi4py==3.1.1' \ - 'protobuf==3.20.1' \ No newline at end of file + 'numpy>=1.10,<1.20' \ + 'matplotlib==3.4.3' \ + 'scipy~=1.5.0' \ + 'scikit-learn~=0.24.1' \ + 'azureml-core==1.35.0' \ + 'azureml-defaults==1.35.0' \ + 'azureml-mlflow==1.35.0' \ + 'azureml-telemetry==1.35.0' \ + 'ray==1.9.2' \ + 'flaml==0.9.6' \ + 'lightgbm-ray==0.1.4' \ + 'mpi4py==3.1.1' \ + 'protobuf==3.20.1' \ No newline at end of file