From 1d1c77ad6ce15912b4661e1745f1bcd3413439d5 Mon Sep 17 00:00:00 2001 From: Patryk Date: Wed, 6 Oct 2021 17:30:12 +0200 Subject: [PATCH 01/15] Bump version for typeguard and pyspark --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 190f3d0..aa63cc5 100644 --- a/setup.py +++ b/setup.py @@ -50,7 +50,7 @@ def read_content(filepath): read_content("README.rst") + read_content(os.path.join("docs/source", "CHANGELOG.rst"))) -requires = ['setuptools', 'typeguard==2.5.0', 'pyspark==3.0.1', 'findspark'] +requires = ['setuptools', 'typeguard==2.9.1', 'pyspark==3.1.2', 'findspark'] extras_require = { 'reST': ['Sphinx'], From fe42d373e3f6dfcbaedb8ffbdb7eb238933167bc Mon Sep 17 00:00:00 2001 From: Patryk Date: Wed, 6 Oct 2021 22:20:19 +0200 Subject: [PATCH 02/15] Change pyspark version in requirements.txt --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f8241b2..205ef23 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -pyspark==3.0.1 +pyspark==3.1.2 findspark setuptools wheel From 4cca69defea7ccbda56d102284955085f279de05 Mon Sep 17 00:00:00 2001 From: Patryk Date: Thu, 14 Oct 2021 19:47:53 +0200 Subject: [PATCH 03/15] Change pip snapshot repo --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9129f46..75dd50b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -21,7 +21,7 @@ variables: DOC_PORT_PREFIX: 11 HUGO_VERSION: 0.76.3-ext-ubuntu HUGO_ENV: production - PIP_SNAPSHOT_REPO: https://zsibio.ii.pw.edu.pl/nexus/repository/bdg-pip/simple + PIP_SNAPSHOT_REPO: https://zsibio.ii.pw.edu.pl/nexus/repository/maven-snapshots PAGE_URL: pysequila.biodatageeks.org SEQUILA_VERSION: 0.6.5 SPARK_IMAGE: biodatageeks/ds-notebook:spark-pysequila-3.0.1-0.1.8-g5db832d From 2228e37d8b0823bc75fb3c8898daf6bd0ae01820 Mon Sep 17 00:00:00 2001 From: Patryk Date: Wed, 27 Oct 2021 19:49:48 +0200 Subject: [PATCH 04/15] Twine upload without ssl check --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 75dd50b..ac8e8ed 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -135,7 +135,7 @@ build-python37: - source $VERSION_FILE - pip install -U -r requirements.txt script: - - twine upload --repository=$PYPI_REPO dist/* --config-file=/root/pypi/.pypirc + - export CURL_CA_BUNDLE="" && twine upload --repository=$PYPI_REPO dist/* --config-file=/root/pypi/.pypirc tags: [ docker ] From 636fc18a80bf4ca780548816b4f37cecf814958e Mon Sep 17 00:00:00 2001 From: Patryk Date: Wed, 27 Oct 2021 20:18:34 +0200 Subject: [PATCH 05/15] Temp commmit --- build_image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_image.sh b/build_image.sh index f5c50e5..ac71f72 100755 --- a/build_image.sh +++ b/build_image.sh @@ -14,4 +14,4 @@ else docker build --no-cache -t biodatageeks/$CI_PROJECT_NAME-$CONTAINER_SUFIX . if [ $(docker ps | grep $CI_PROJECT_NAME-$CONTAINER_SUFIX | wc -l) -gt 0 ]; then docker stop $CI_PROJECT_NAME-$CONTAINER_SUFIX && docker rm $CI_PROJECT_NAME-$CONTAINER_SUFIX; fi docker run --restart unless-stopped -d -p ${DOC_PORT_PREFIX}$PORT_PROD:80 --name $CI_PROJECT_NAME-$CONTAINER_SUFIX biodatageeks/$CI_PROJECT_NAME-$CONTAINER_SUFIX -fi \ No newline at end of file +fi From eeb03bac2ecc0b519d6626bdc0128386cb9682f5 Mon Sep 17 00:00:00 2001 From: Patryk Date: Tue, 9 Nov 2021 19:35:21 +0100 Subject: [PATCH 06/15] Temp commmit2 --- build_image.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/build_image.sh b/build_image.sh index ac71f72..a8724be 100755 --- a/build_image.sh +++ b/build_image.sh @@ -15,3 +15,4 @@ else if [ $(docker ps | grep $CI_PROJECT_NAME-$CONTAINER_SUFIX | wc -l) -gt 0 ]; then docker stop $CI_PROJECT_NAME-$CONTAINER_SUFIX && docker rm $CI_PROJECT_NAME-$CONTAINER_SUFIX; fi docker run --restart unless-stopped -d -p ${DOC_PORT_PREFIX}$PORT_PROD:80 --name $CI_PROJECT_NAME-$CONTAINER_SUFIX biodatageeks/$CI_PROJECT_NAME-$CONTAINER_SUFIX fi + From 85001935a960e47e4510c187421b15c591caf42b Mon Sep 17 00:00:00 2001 From: Patryk Date: Tue, 9 Nov 2021 22:21:03 +0100 Subject: [PATCH 07/15] Doc-build set pysequila version manually --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ac8e8ed..27cdd59 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -172,7 +172,7 @@ doc-build: - source /opt/conda/etc/profile.d/conda.sh - conda activate /tmp/jovyan/venv/$VENV_NAME/ - source "/tmp/jovyan/.sdkman/bin/sdkman-init.sh" - - pip install -U sphinx==$SPHINX_VERSION nbsphinx sphinx_rtd_theme rst2pdf sphinxcontrib-github_ribbon "sphinxcontrib-bibtex<2.0.0" pysequila==$VERSION --extra-index-url=$PIP_SNAPSHOT_REPO + - pip install -U sphinx==$SPHINX_VERSION nbsphinx sphinx_rtd_theme rst2pdf sphinxcontrib-github_ribbon "sphinxcontrib-bibtex<2.0.0" pysequila==0.2.1+geeb03ba.snapshot --extra-index-url=$PIP_SNAPSHOT_REPO - export PYSPARK_SUBMIT_ARGS="--packages org.biodatageeks:sequila_${SCALA_VERSION}:${SEQUILA_VERSION} --driver-memory 4g pyspark-shell" script: - cd docs && ./docs.sh html From 241c318ba44ab7a592deec33ab42f782f1db2923 Mon Sep 17 00:00:00 2001 From: Patryk Date: Tue, 9 Nov 2021 22:55:40 +0100 Subject: [PATCH 08/15] Change pip_snapshot_repo to bdg-pip --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 27cdd59..8358af5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -21,7 +21,7 @@ variables: DOC_PORT_PREFIX: 11 HUGO_VERSION: 0.76.3-ext-ubuntu HUGO_ENV: production - PIP_SNAPSHOT_REPO: https://zsibio.ii.pw.edu.pl/nexus/repository/maven-snapshots + PIP_SNAPSHOT_REPO: https://zsibio.ii.pw.edu.pl/nexus/repository/bdg-pip PAGE_URL: pysequila.biodatageeks.org SEQUILA_VERSION: 0.6.5 SPARK_IMAGE: biodatageeks/ds-notebook:spark-pysequila-3.0.1-0.1.8-g5db832d From d0f1c52ffb9b15a76c0def6ea04822d21d9c3a8c Mon Sep 17 00:00:00 2001 From: Patryk Date: Wed, 17 Nov 2021 20:11:46 +0100 Subject: [PATCH 09/15] Doc-build set use bdg-pip/simple --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8358af5..8d5c80e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -21,7 +21,7 @@ variables: DOC_PORT_PREFIX: 11 HUGO_VERSION: 0.76.3-ext-ubuntu HUGO_ENV: production - PIP_SNAPSHOT_REPO: https://zsibio.ii.pw.edu.pl/nexus/repository/bdg-pip + PIP_SNAPSHOT_REPO: https://zsibio.ii.pw.edu.pl/nexus/repository/bdg-pip/simple PAGE_URL: pysequila.biodatageeks.org SEQUILA_VERSION: 0.6.5 SPARK_IMAGE: biodatageeks/ds-notebook:spark-pysequila-3.0.1-0.1.8-g5db832d From 0f65e025b2556b0ac1e627f46c33a5ec8516966a Mon Sep 17 00:00:00 2001 From: Patryk Date: Wed, 17 Nov 2021 22:55:45 +0100 Subject: [PATCH 10/15] Explicitly set pyspark sequila version --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8d5c80e..4c0cd00 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -173,7 +173,7 @@ doc-build: - conda activate /tmp/jovyan/venv/$VENV_NAME/ - source "/tmp/jovyan/.sdkman/bin/sdkman-init.sh" - pip install -U sphinx==$SPHINX_VERSION nbsphinx sphinx_rtd_theme rst2pdf sphinxcontrib-github_ribbon "sphinxcontrib-bibtex<2.0.0" pysequila==0.2.1+geeb03ba.snapshot --extra-index-url=$PIP_SNAPSHOT_REPO - - export PYSPARK_SUBMIT_ARGS="--packages org.biodatageeks:sequila_${SCALA_VERSION}:${SEQUILA_VERSION} --driver-memory 4g pyspark-shell" + - export PYSPARK_SUBMIT_ARGS="--packages org.biodatageeks:sequila_${SCALA_VERSION}:0.2.1+geeb03ba.snapshot --driver-memory 4g pyspark-shell" script: - cd docs && ./docs.sh html artifacts: From 9b720427921360a6ed32460add7cd07ceaf59eea Mon Sep 17 00:00:00 2001 From: Patryk Date: Thu, 18 Nov 2021 19:01:04 +0100 Subject: [PATCH 11/15] Allow error in nbsphinx --- docs/source/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 2bf001c..3274782 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -69,6 +69,6 @@ # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ['_static'] - +nbsphinx_allow_errors = True # -- Extension configuration ------------------------------------------------- From cb5bc0f5e65d3769235a45e07d2f1079a3a520ae Mon Sep 17 00:00:00 2001 From: Patryk Date: Thu, 18 Nov 2021 19:38:38 +0100 Subject: [PATCH 12/15] use wget no-check-certificate --- docs/source/notebooks | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/notebooks b/docs/source/notebooks index cbc4ab2..58964bc 160000 --- a/docs/source/notebooks +++ b/docs/source/notebooks @@ -1 +1 @@ -Subproject commit cbc4ab2342a6e42a55b7a631530d46d0b7c20611 +Subproject commit 58964bcc85f904f17b686f55c5270adf9eec8643 From 321097163c5a5248c133f627d7c3a8c3fa90e64f Mon Sep 17 00:00:00 2001 From: Patryk Date: Thu, 18 Nov 2021 19:52:50 +0100 Subject: [PATCH 13/15] Remove nbsphinx_allow_errors --- docs/source/conf.py | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 3274782..3ef1755 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -69,6 +69,5 @@ # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ['_static'] -nbsphinx_allow_errors = True # -- Extension configuration ------------------------------------------------- From 92c8b45447caea146abb52b332a6cac1ebefc5f1 Mon Sep 17 00:00:00 2001 From: Patryk Date: Thu, 18 Nov 2021 19:59:31 +0100 Subject: [PATCH 14/15] use wget no-check-certificate --- docs/source/notebooks | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/notebooks b/docs/source/notebooks index 58964bc..48934cd 160000 --- a/docs/source/notebooks +++ b/docs/source/notebooks @@ -1 +1 @@ -Subproject commit 58964bcc85f904f17b686f55c5270adf9eec8643 +Subproject commit 48934cd7a23dfb126703fe6e32361804699d3fe0 From 4589d9ff87e892231ec83f8bef5e5d9328dea605 Mon Sep 17 00:00:00 2001 From: Patryk Date: Sat, 20 Nov 2021 19:29:47 +0100 Subject: [PATCH 15/15] notebooks sub-repository indicator --- docs/source/notebooks | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/notebooks b/docs/source/notebooks index 58964bc..48934cd 160000 --- a/docs/source/notebooks +++ b/docs/source/notebooks @@ -1 +1 @@ -Subproject commit 58964bcc85f904f17b686f55c5270adf9eec8643 +Subproject commit 48934cd7a23dfb126703fe6e32361804699d3fe0