From d8e75e34d0a4315acf9a643deb56240186e53222 Mon Sep 17 00:00:00 2001 From: Aaron Davidson Date: Mon, 13 Apr 2020 14:20:52 -0700 Subject: [PATCH 01/23] Create pythonapp.yml --- .github/workflows/pythonapp.yml | 37 +++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 .github/workflows/pythonapp.yml diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml new file mode 100644 index 0000000000000..25a32ef60ff69 --- /dev/null +++ b/.github/workflows/pythonapp.yml @@ -0,0 +1,37 @@ +# This workflow will install Python dependencies, run tests and lint with a single version of Python +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: Python application + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Set up Python 3.8 + uses: actions/setup-python@v1 + with: + python-version: 3.8 + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + - name: Lint with flake8 + run: | + pip install flake8 + # stop the build if there are Python syntax errors or undefined names + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + - name: Test with pytest + run: | + pip install pytest + pytest From b2a2c3f45be8c4dad363a9aeec0aa40e62490c88 Mon Sep 17 00:00:00 2001 From: Aaron Davidson Date: Mon, 13 Apr 2020 14:30:22 -0700 Subject: [PATCH 02/23] Update pythonapp --- .github/workflows/pythonapp.yml | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index 25a32ef60ff69..3187c76ec62f4 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -16,21 +16,17 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Set up Python 3.8 + - name: Set up Python 3.6 uses: actions/setup-python@v1 with: - python-version: 3.8 + python-version: 3.6 - name: Install dependencies run: | - python -m pip install --upgrade pip - pip install -r requirements.txt + INSTALL_SMALL_PYTHON_DEPS=true INSTALL_LARGE_PYTHON_DEPS=true source ./travis/install-common-deps.sh + pip install -r ./travis/lint-requirements.txt - name: Lint with flake8 run: | - pip install flake8 - # stop the build if there are Python syntax errors or undefined names - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + ./lint.sh - name: Test with pytest run: | pip install pytest From 74a7d03454025d370610bee2a7e1ed6f11ce39f0 Mon Sep 17 00:00:00 2001 From: Aaron Davidson Date: Mon, 13 Apr 2020 14:33:45 -0700 Subject: [PATCH 03/23] Hi! From 558971a25b23c0d461caf041bcdb30f3d70dbd6b Mon Sep 17 00:00:00 2001 From: Aaron Davidson Date: Mon, 13 Apr 2020 14:35:50 -0700 Subject: [PATCH 04/23] No chown --- travis/install-common-deps.sh | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/travis/install-common-deps.sh b/travis/install-common-deps.sh index 3d2b059d84214..509e080973d59 100755 --- a/travis/install-common-deps.sh +++ b/travis/install-common-deps.sh @@ -2,15 +2,11 @@ set -ex sudo mkdir -p /travis-install -sudo chown travis /travis-install +#sudo chown travis /travis-install # (The conda installation steps below are taken from http://conda.pydata.org/docs/travis.html) # We do this conditionally because it saves us some downloading if the # version is the same. -if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then - wget https://repo.anaconda.com/miniconda/Miniconda2-latest-Linux-x86_64.sh -O /travis-install/miniconda.sh; -else - wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O /travis-install/miniconda.sh; -fi +wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O /travis-install/miniconda.sh; bash /travis-install/miniconda.sh -b -p $HOME/miniconda export PATH="$HOME/miniconda/bin:$PATH" From a0e8de2fc1aabebbbfd54f09649ce501131ab2f1 Mon Sep 17 00:00:00 2001 From: Aaron Davidson Date: Mon, 13 Apr 2020 14:36:00 -0700 Subject: [PATCH 05/23] Different workflows --- .github/workflows/jira.pr.yml | 20 -------------------- 1 file changed, 20 deletions(-) delete mode 100644 .github/workflows/jira.pr.yml diff --git a/.github/workflows/jira.pr.yml b/.github/workflows/jira.pr.yml deleted file mode 100644 index 45d1a3e03aaae..0000000000000 --- a/.github/workflows/jira.pr.yml +++ /dev/null @@ -1,20 +0,0 @@ -on: - pull_request: - branches: - - master - -name: Jira Ticket Creation for PRs - -jobs: - create_ticket: - name: Create Jira ticket over API - runs-on: ubuntu-latest - steps: - - name: Call REST API - shell: bash - run: > - curl - -u ${{ secrets.JIRA_USER_EMAIL }}:${{ secrets.JIRA_API_TOKEN }} - -H 'Content-Type: application/json' - --data-raw '{"fields":{"project":{"key":"ML"},"summary":"Review \"${{ github.event.pull_request.title }}\" by ${{ github.event.pull_request.user.login }} [${{ github.repository }} #${{ github.event.number }}] ","description":"${{ github.event.pull_request._links.html.href }}","customfield_10008":"ML-9504","assignee":{"name":"eng-ml-platform-team"},"issuetype":{"name":"Story"},"components":[{"name":"ML Platform"}]}}' - ${{ secrets.JIRA_BASE_URL }}/rest/api/2/issue/ From 6781fb3fec19e7808a03e84c11bb92808846f4fd Mon Sep 17 00:00:00 2001 From: Aaron Davidson Date: Mon, 13 Apr 2020 14:37:08 -0700 Subject: [PATCH 06/23] Testing --- travis/install-common-deps.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/travis/install-common-deps.sh b/travis/install-common-deps.sh index 509e080973d59..54ff1c9b936ed 100755 --- a/travis/install-common-deps.sh +++ b/travis/install-common-deps.sh @@ -8,6 +8,8 @@ sudo mkdir -p /travis-install # version is the same. wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O /travis-install/miniconda.sh; +chmod +x /travis-install/miniconda.sh + bash /travis-install/miniconda.sh -b -p $HOME/miniconda export PATH="$HOME/miniconda/bin:$PATH" hash -r From 98c6d89bc3cbcec573b965e78c37002e67f27bae Mon Sep 17 00:00:00 2001 From: Aaron Davidson Date: Mon, 13 Apr 2020 14:37:49 -0700 Subject: [PATCH 07/23] Testing --- travis/install-common-deps.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/travis/install-common-deps.sh b/travis/install-common-deps.sh index 54ff1c9b936ed..a745e8ea880e7 100755 --- a/travis/install-common-deps.sh +++ b/travis/install-common-deps.sh @@ -6,9 +6,7 @@ sudo mkdir -p /travis-install # (The conda installation steps below are taken from http://conda.pydata.org/docs/travis.html) # We do this conditionally because it saves us some downloading if the # version is the same. -wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O /travis-install/miniconda.sh; - -chmod +x /travis-install/miniconda.sh +wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh bash /travis-install/miniconda.sh -b -p $HOME/miniconda export PATH="$HOME/miniconda/bin:$PATH" From 4bff97686959a2d2bec1ccd9c06cd4382e8f9001 Mon Sep 17 00:00:00 2001 From: Aaron Davidson Date: Mon, 13 Apr 2020 14:39:10 -0700 Subject: [PATCH 08/23] Testing --- travis/install-common-deps.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/travis/install-common-deps.sh b/travis/install-common-deps.sh index a745e8ea880e7..4909b87685a8f 100755 --- a/travis/install-common-deps.sh +++ b/travis/install-common-deps.sh @@ -8,7 +8,7 @@ sudo mkdir -p /travis-install # version is the same. wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh -bash /travis-install/miniconda.sh -b -p $HOME/miniconda +bash miniconda.sh -b -p $HOME/miniconda export PATH="$HOME/miniconda/bin:$PATH" hash -r conda config --set always_yes yes --set changeps1 no From 6df4afbb5e4be69b4cc4306617a6a6aec2be173e Mon Sep 17 00:00:00 2001 From: Aaron Davidson Date: Mon, 13 Apr 2020 14:47:32 -0700 Subject: [PATCH 09/23] Testing --- travis/install-common-deps.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/travis/install-common-deps.sh b/travis/install-common-deps.sh index 4909b87685a8f..b794c20dfd5f1 100755 --- a/travis/install-common-deps.sh +++ b/travis/install-common-deps.sh @@ -6,7 +6,7 @@ sudo mkdir -p /travis-install # (The conda installation steps below are taken from http://conda.pydata.org/docs/travis.html) # We do this conditionally because it saves us some downloading if the # version is the same. -wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh +wget -q https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh bash miniconda.sh -b -p $HOME/miniconda export PATH="$HOME/miniconda/bin:$PATH" @@ -31,14 +31,14 @@ if [[ "$INSTALL_LARGE_PYTHON_DEPS" == "true" ]]; then # Hack: make sure all spark-* scripts are executable. # Conda installs 2 version spark-* scripts and makes the ones spark # uses not executable. This is a temporary fix to unblock the tests. - ls -lha `find /home/travis/miniconda/envs/test-environment/ -path "*bin/spark-*"` - chmod 777 `find /home/travis/miniconda/envs/test-environment/ -path "*bin/spark-*"` - ls -lha `find /home/travis/miniconda/envs/test-environment/ -path "*bin/spark-*"` + ls -lha `find $HOME/miniconda/envs/test-environment/ -path "*bin/spark-*"` + chmod 777 `find $HOE/miniconda/envs/test-environment/ -path "*bin/spark-*"` + ls -lha `find $HOME/miniconda/envs/test-environment/ -path "*bin/spark-*"` fi pip install . export MLFLOW_HOME=$(pwd) # Remove boto config present in Travis VMs (https://github.com/travis-ci/travis-ci/issues/7940) -sudo rm -f /etc/boto.cfg +#sudo rm -f /etc/boto.cfg # Print current environment info pip list which mlflow From 614af1e2fdaa0cc2e6bb8c2a4f542e03840450ac Mon Sep 17 00:00:00 2001 From: Aaron Davidson Date: Mon, 13 Apr 2020 14:57:39 -0700 Subject: [PATCH 10/23] Testing --- .github/workflows/pythonapp.yml | 4 ++-- travis/install-common-deps.sh | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index 3187c76ec62f4..f73aff2c2f456 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -24,10 +24,10 @@ jobs: run: | INSTALL_SMALL_PYTHON_DEPS=true INSTALL_LARGE_PYTHON_DEPS=true source ./travis/install-common-deps.sh pip install -r ./travis/lint-requirements.txt - - name: Lint with flake8 + - name: Run lint run: | ./lint.sh - - name: Test with pytest + - name: Run tests run: | pip install pytest pytest diff --git a/travis/install-common-deps.sh b/travis/install-common-deps.sh index b794c20dfd5f1..63a3a32faff63 100755 --- a/travis/install-common-deps.sh +++ b/travis/install-common-deps.sh @@ -24,16 +24,16 @@ python --version pip install --upgrade pip==19.3.1 # Install Python test dependencies only if we're running Python tests if [[ "$INSTALL_SMALL_PYTHON_DEPS" == "true" ]]; then - pip install -r ./travis/small-requirements.txt + pip install --quiet -r ./travis/small-requirements.txt fi if [[ "$INSTALL_LARGE_PYTHON_DEPS" == "true" ]]; then - pip install -r ./travis/large-requirements.txt + pip install --quiet -r ./travis/large-requirements.txt # Hack: make sure all spark-* scripts are executable. # Conda installs 2 version spark-* scripts and makes the ones spark # uses not executable. This is a temporary fix to unblock the tests. - ls -lha `find $HOME/miniconda/envs/test-environment/ -path "*bin/spark-*"` - chmod 777 `find $HOE/miniconda/envs/test-environment/ -path "*bin/spark-*"` - ls -lha `find $HOME/miniconda/envs/test-environment/ -path "*bin/spark-*"` + ls -lha $(find $HOME/miniconda/envs/test-environment/ -path "*bin/spark-*") + chmod 777 $(find $HOE/miniconda/envs/test-environment/ -path "*bin/spark-*") + ls -lha $(find $HOME/miniconda/envs/test-environment/ -path "*bin/spark-*") fi pip install . export MLFLOW_HOME=$(pwd) From baf6dbdc8c70865e53f5fc93e2c7913b0b16dca0 Mon Sep 17 00:00:00 2001 From: Aaron Davidson Date: Mon, 13 Apr 2020 15:02:26 -0700 Subject: [PATCH 11/23] Testing --- .github/workflows/pythonapp.yml | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index f73aff2c2f456..6f180bfc82500 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -10,8 +10,7 @@ on: branches: [ master ] jobs: - build: - + lint: runs-on: ubuntu-latest steps: @@ -22,12 +21,24 @@ jobs: python-version: 3.6 - name: Install dependencies run: | - INSTALL_SMALL_PYTHON_DEPS=true INSTALL_LARGE_PYTHON_DEPS=true source ./travis/install-common-deps.sh pip install -r ./travis/lint-requirements.txt - name: Run lint run: | ./lint.sh + + + python-small: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Python 3.6 + uses: actions/setup-python@v1 + with: + python-version: 3.6 + - name: Install dependencies + run: | + INSTALL_SMALL_PYTHON_DEPS=true source ./travis/install-common-deps.sh - name: Run tests run: | - pip install pytest - pytest + ./travis/stage-python3.sh +~ From 2a98a5ebbebda23f465255ed19795b9e1a195910 Mon Sep 17 00:00:00 2001 From: Aaron Davidson Date: Mon, 13 Apr 2020 15:04:17 -0700 Subject: [PATCH 12/23] Testing --- .github/workflows/pythonapp.yml | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index 6f180bfc82500..2f0a8788dc678 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -40,5 +40,23 @@ jobs: INSTALL_SMALL_PYTHON_DEPS=true source ./travis/install-common-deps.sh - name: Run tests run: | - ./travis/stage-python3.sh -~ + ./travis/run-small-python-tests.sh + ./test-generate-protos.sh + + + python-large: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Python 3.6 + uses: actions/setup-python@v1 + with: + python-version: 3.6 + - name: Install dependencies + run: | + INSTALL_LARGE_PYTHON_DEPS=true source ./travis/install-common-deps.sh + - name: Run tests + run: | + ./travis/run-large-python-tests.sh + ./travis/test-anaconda-compatibility.sh "anaconda3:2020.02" + ./travis/test-anaconda-compatibility.sh "anaconda3:2019.03" From e604e34deb677014ef72abc488e7ba046a74fe8f Mon Sep 17 00:00:00 2001 From: Aaron Davidson Date: Mon, 13 Apr 2020 15:10:01 -0700 Subject: [PATCH 13/23] Testing --- .github/workflows/pythonapp.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index 2f0a8788dc678..283599a651dab 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -22,6 +22,7 @@ jobs: - name: Install dependencies run: | pip install -r ./travis/lint-requirements.txt + INSTALL_SMALL_PYTHON_DEPS=true source ./travis/install-common-deps.sh - name: Run lint run: | ./lint.sh @@ -40,6 +41,7 @@ jobs: INSTALL_SMALL_PYTHON_DEPS=true source ./travis/install-common-deps.sh - name: Run tests run: | + pip install pytest ./travis/run-small-python-tests.sh ./test-generate-protos.sh From b5713921623897625243f0ee3be2bff4446d8a0f Mon Sep 17 00:00:00 2001 From: Aaron Davidson Date: Mon, 13 Apr 2020 15:14:54 -0700 Subject: [PATCH 14/23] Testing --- .github/workflows/pythonapp.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index 283599a651dab..0f6e67372a7b7 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -25,6 +25,7 @@ jobs: INSTALL_SMALL_PYTHON_DEPS=true source ./travis/install-common-deps.sh - name: Run lint run: | + source activate test-environment ./lint.sh @@ -41,7 +42,7 @@ jobs: INSTALL_SMALL_PYTHON_DEPS=true source ./travis/install-common-deps.sh - name: Run tests run: | - pip install pytest + source activate test-environment ./travis/run-small-python-tests.sh ./test-generate-protos.sh From e949693557ee6896e9db7d71418278e0930c0e04 Mon Sep 17 00:00:00 2001 From: Aaron Davidson Date: Mon, 13 Apr 2020 16:00:41 -0700 Subject: [PATCH 15/23] Testing --- .github/workflows/pythonapp.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index 0f6e67372a7b7..c93d92fa6fe51 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -21,8 +21,8 @@ jobs: python-version: 3.6 - name: Install dependencies run: | + INSTALL_LARGE_PYTHON_DEPS=true INSTALL_SMALL_PYTHON_DEPS=true source ./travis/install-common-deps.sh pip install -r ./travis/lint-requirements.txt - INSTALL_SMALL_PYTHON_DEPS=true source ./travis/install-common-deps.sh - name: Run lint run: | source activate test-environment From 50c81113c8a1aba7e2faafae8911955acb0d888c Mon Sep 17 00:00:00 2001 From: Aaron Davidson Date: Tue, 14 Apr 2020 09:05:36 -0700 Subject: [PATCH 16/23] Miniconda activation --- .github/workflows/pythonapp.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index c93d92fa6fe51..f88a0f5ab223d 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -25,6 +25,7 @@ jobs: pip install -r ./travis/lint-requirements.txt - name: Run lint run: | + export PATH="$HOME/miniconda/bin:$PATH" source activate test-environment ./lint.sh @@ -42,6 +43,7 @@ jobs: INSTALL_SMALL_PYTHON_DEPS=true source ./travis/install-common-deps.sh - name: Run tests run: | + export PATH="$HOME/miniconda/bin:$PATH" source activate test-environment ./travis/run-small-python-tests.sh ./test-generate-protos.sh @@ -60,6 +62,7 @@ jobs: INSTALL_LARGE_PYTHON_DEPS=true source ./travis/install-common-deps.sh - name: Run tests run: | + export PATH="$HOME/miniconda/bin:$PATH" ./travis/run-large-python-tests.sh ./travis/test-anaconda-compatibility.sh "anaconda3:2020.02" ./travis/test-anaconda-compatibility.sh "anaconda3:2019.03" From 80d7dca64eb300a5644fb24ca9117ca534f1044a Mon Sep 17 00:00:00 2001 From: Aaron Davidson Date: Tue, 14 Apr 2020 09:15:17 -0700 Subject: [PATCH 17/23] Testing --- travis/install-common-deps.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/travis/install-common-deps.sh b/travis/install-common-deps.sh index 63a3a32faff63..57203deadb4df 100755 --- a/travis/install-common-deps.sh +++ b/travis/install-common-deps.sh @@ -32,7 +32,7 @@ if [[ "$INSTALL_LARGE_PYTHON_DEPS" == "true" ]]; then # Conda installs 2 version spark-* scripts and makes the ones spark # uses not executable. This is a temporary fix to unblock the tests. ls -lha $(find $HOME/miniconda/envs/test-environment/ -path "*bin/spark-*") - chmod 777 $(find $HOE/miniconda/envs/test-environment/ -path "*bin/spark-*") + chmod 777 $(find $HOME/miniconda/envs/test-environment/ -path "*bin/spark-*") ls -lha $(find $HOME/miniconda/envs/test-environment/ -path "*bin/spark-*") fi pip install . From 53213416030dec5ab66614555e8e65f4a27ee5eb Mon Sep 17 00:00:00 2001 From: Aaron Davidson Date: Tue, 14 Apr 2020 09:27:01 -0700 Subject: [PATCH 18/23] Testing --- tests/projects/test_projects.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/projects/test_projects.py b/tests/projects/test_projects.py index 6297a31345eb7..d69cb56ef37a9 100644 --- a/tests/projects/test_projects.py +++ b/tests/projects/test_projects.py @@ -189,6 +189,7 @@ def test_invalid_run_mode(): mlflow.projects.run(uri=TEST_PROJECT_DIR, backend="some unsupported mode") +@pytest.mark.large def test_use_conda(): """ Verify that we correctly handle the `use_conda` argument.""" # Verify we throw an exception when conda is unavailable @@ -207,6 +208,7 @@ def test_use_conda(): os.environ["CONDA_EXE"] = conda_exe_path +@pytest.mark.large def test_expected_tags_logged_when_using_conda(): with mock.patch.object(mlflow.tracking.MlflowClient, "set_tag") as tag_mock: try: From 46c6052eaa98172d3c75928a1b6526ddcb73ac8d Mon Sep 17 00:00:00 2001 From: Aaron Davidson Date: Tue, 14 Apr 2020 09:30:35 -0700 Subject: [PATCH 19/23] Testing --- .github/workflows/pythonapp.yml | 2 ++ travis/install-common-deps.sh | 8 ++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index f88a0f5ab223d..00654b7682a97 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -40,9 +40,11 @@ jobs: python-version: 3.6 - name: Install dependencies run: | + export GITHUB_WORKFLOW=1 INSTALL_SMALL_PYTHON_DEPS=true source ./travis/install-common-deps.sh - name: Run tests run: | + export GITHUB_WORKFLOW=1 export PATH="$HOME/miniconda/bin:$PATH" source activate test-environment ./travis/run-small-python-tests.sh diff --git a/travis/install-common-deps.sh b/travis/install-common-deps.sh index 57203deadb4df..c08314f9d7e5d 100755 --- a/travis/install-common-deps.sh +++ b/travis/install-common-deps.sh @@ -2,7 +2,9 @@ set -ex sudo mkdir -p /travis-install -#sudo chown travis /travis-install +if [[ -z $GITHUB_WORKFLOW ]]; then + sudo chown travis /travis-install +fi # (The conda installation steps below are taken from http://conda.pydata.org/docs/travis.html) # We do this conditionally because it saves us some downloading if the # version is the same. @@ -38,7 +40,9 @@ fi pip install . export MLFLOW_HOME=$(pwd) # Remove boto config present in Travis VMs (https://github.com/travis-ci/travis-ci/issues/7940) -#sudo rm -f /etc/boto.cfg +if [[ -z $GITHUB_WORKFLOW ]]; then + sudo rm -f /etc/boto.cfg +fi # Print current environment info pip list which mlflow From 18fa71ebc4125a5e6965ef7ddc1f75b77df570a0 Mon Sep 17 00:00:00 2001 From: Aaron Davidson Date: Tue, 14 Apr 2020 09:31:18 -0700 Subject: [PATCH 20/23] Testing --- .github/workflows/pythonapp.yml | 44 +-------------------------------- 1 file changed, 1 insertion(+), 43 deletions(-) diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index 00654b7682a97..1d346232e332a 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -1,7 +1,4 @@ -# This workflow will install Python dependencies, run tests and lint with a single version of Python -# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions - -name: Python application +name: MLflow tests on: push: @@ -10,26 +7,6 @@ on: branches: [ master ] jobs: - lint: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 - - name: Set up Python 3.6 - uses: actions/setup-python@v1 - with: - python-version: 3.6 - - name: Install dependencies - run: | - INSTALL_LARGE_PYTHON_DEPS=true INSTALL_SMALL_PYTHON_DEPS=true source ./travis/install-common-deps.sh - pip install -r ./travis/lint-requirements.txt - - name: Run lint - run: | - export PATH="$HOME/miniconda/bin:$PATH" - source activate test-environment - ./lint.sh - - python-small: runs-on: ubuntu-latest steps: @@ -49,22 +26,3 @@ jobs: source activate test-environment ./travis/run-small-python-tests.sh ./test-generate-protos.sh - - - python-large: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Set up Python 3.6 - uses: actions/setup-python@v1 - with: - python-version: 3.6 - - name: Install dependencies - run: | - INSTALL_LARGE_PYTHON_DEPS=true source ./travis/install-common-deps.sh - - name: Run tests - run: | - export PATH="$HOME/miniconda/bin:$PATH" - ./travis/run-large-python-tests.sh - ./travis/test-anaconda-compatibility.sh "anaconda3:2020.02" - ./travis/test-anaconda-compatibility.sh "anaconda3:2019.03" From 2967f488699c8404126b7cb1ef3d1a50dacdf3c7 Mon Sep 17 00:00:00 2001 From: Aaron Davidson Date: Tue, 14 Apr 2020 10:01:25 -0700 Subject: [PATCH 21/23] Testing --- .github/workflows/pythonapp.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index 1d346232e332a..4c58c4746c748 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -25,4 +25,3 @@ jobs: export PATH="$HOME/miniconda/bin:$PATH" source activate test-environment ./travis/run-small-python-tests.sh - ./test-generate-protos.sh From 00a8ec3ee29f0a38bbd6f3ddb6321f13d3c948fb Mon Sep 17 00:00:00 2001 From: Aaron Davidson Date: Tue, 14 Apr 2020 10:11:56 -0700 Subject: [PATCH 22/23] Testing --- tests/models/test_cli.py | 4 ++++ tests/projects/test_docker_projects.py | 1 + 2 files changed, 5 insertions(+) diff --git a/tests/models/test_cli.py b/tests/models/test_cli.py index 550e0921644a8..ca6175cdf428a 100644 --- a/tests/models/test_cli.py +++ b/tests/models/test_cli.py @@ -127,6 +127,7 @@ def test_model_with_no_deployable_flavors_fails_pollitely(): assert "No suitable flavor backend was found for the model." in stderr +@pytest.mark.large def test_serve_gunicorn_opts(iris_data, sk_model): if sys.platform == "win32": pytest.skip("This test requires gunicorn which is not available on windows.") @@ -159,6 +160,7 @@ def test_serve_gunicorn_opts(iris_data, sk_model): assert expected_command_pattern.search(stdout) is not None +@pytest.mark.large def test_predict(iris_data, sk_model): with TempDir(chdr=True) as tmp: with mlflow.start_run() as active_run: @@ -245,6 +247,7 @@ def test_predict(iris_data, sk_model): assert all(expected == actual) +@pytest.mark.large def test_prepare_env_passes(sk_model): if no_conda: pytest.skip("This test requires conda.") @@ -270,6 +273,7 @@ def test_prepare_env_passes(sk_model): assert p.wait() == 0 +@pytest.mark.large def test_prepare_env_fails(sk_model): if no_conda: pytest.skip("This test requires conda.") diff --git a/tests/projects/test_docker_projects.py b/tests/projects/test_docker_projects.py index 1e558759da979..c95f0f3fde629 100644 --- a/tests/projects/test_docker_projects.py +++ b/tests/projects/test_docker_projects.py @@ -26,6 +26,7 @@ def _build_uri(base_uri, subdirectory): @pytest.mark.parametrize("use_start_run", map(str, [0, 1])) +@pytest.mark.large def test_docker_project_execution( use_start_run, tmpdir, docker_example_base_image): # pylint: disable=unused-argument From 0d15b223d6d75a384870501ef9864f1dcf51f268 Mon Sep 17 00:00:00 2001 From: Aaron Davidson Date: Tue, 14 Apr 2020 10:19:58 -0700 Subject: [PATCH 23/23] Testing --- tests/projects/test_docker_projects.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/projects/test_docker_projects.py b/tests/projects/test_docker_projects.py index c95f0f3fde629..24368b248aad3 100644 --- a/tests/projects/test_docker_projects.py +++ b/tests/projects/test_docker_projects.py @@ -69,6 +69,7 @@ def test_docker_project_execution( ("databricks://some-profile", "-e MLFLOW_TRACKING_URI=databricks ") ]) @mock.patch('databricks_cli.configure.provider.ProfileConfigProvider') +@pytest.mark.large def test_docker_project_tracking_uri_propagation( ProfileConfigProvider, tmpdir, tracking_uri, expected_command_segment, docker_example_base_image): # pylint: disable=unused-argument