From 6d20e5d4a7bd65c4cad2562060b14dc2f9807972 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Suszy=C5=84ski?= Date: Tue, 5 Nov 2024 12:32:26 +0100 Subject: [PATCH] publishing to PyPI --- .github/workflows/publish.yml | 30 ++++ .gitignore | 4 +- .gitlab-ci.yml | 285 ---------------------------------- README.rst | 27 +++- requirements.txt | 30 +++- requirements_dev.txt | 13 -- setup.cfg | 2 +- setup.py | 10 +- tox.ini | 39 +++-- 9 files changed, 113 insertions(+), 327 deletions(-) create mode 100644 .github/workflows/publish.yml delete mode 100644 .gitlab-ci.yml delete mode 100644 requirements_dev.txt diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 00000000..21a3c525 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,30 @@ +name: Upload Python Package to PyPI when a Release is Created + +on: + release: + types: [created] + +jobs: + pypi-publish: + name: Publish release to PyPI + runs-on: bdg-runners + environment: + name: pypi + url: https://pypi.org/p/pysequila + permissions: + id-token: write + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.9" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install setuptools wheel + - name: Build package + run: | + python setup.py sdist bdist_wheel # Could also be python -m build + - name: Publish package distributions to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/.gitignore b/.gitignore index 23c7a0f2..08788652 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ *.egg *.egg/ *.egg-info/ +.eggs/ *.pyc .tox/ _build/ @@ -15,4 +16,5 @@ page/config.toml page/docs/* docs/source_v/* page/resources -.DS_Store \ No newline at end of file +.DS_Store +.venv/ diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index bfdeac8e..00000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,285 +0,0 @@ -stages: - - version - - tox - - behave-test - - build - - publish - - doc-build - - page-build - - doc-serve - - page-serve - -variables: - GIT_SUBMODULE_STRATEGY: recursive - VERSION_FILE: version.sh - GIT_DEPTH: 500 - KIND_VERSION: 0.9.0 - KUBERNETES_VERSION: 1.19.1 - DOCKER_VERSION: 19.03.12 - HELM_VERSION: 3.3.1 - JUPYTER_VERSION: 0.9.1 - SPHINX_VERSION: 5.1.1 - DOC_PORT_PREFIX: 11 - HUGO_VERSION: 0.76.3-ext-ubuntu - HUGO_ENV: production - NEXUS_HOST: zsibio.ii.pw.edu.pl - PIP_SNAPSHOT_REPO: http://${NEXUS_HOST}/nexus/repository/bdg-pip/simple - MAVEN_SNAPSHOT_REPO: https://zsibio.ii.pw.edu.pl/nexus/repository/maven-snapshots/ - PAGE_URL: pysequila.biodatageeks.org - SEQUILA_VERSION: 1.0.1+g68cb8220-SNAPSHOT - SPARK_IMAGE: jupyter/pyspark-notebook:spark-3.2.1 - SCALA_VERSION: "2.12" - VENV_NAME: pysequila - -.python37Vars: &python37Vars - PYTHON_VERSION: "3.7" - -.python38Vars: &python38Vars - PYTHON_VERSION: "3.8" - -.python39Vars: &python39Vars - PYTHON_VERSION: "3.9" - -.publishSnapshotVar: &publishSnapshotVar - PYPI_REPO: "zsibio" - DOC_URL_PREFIX: "$PAGE_URL/snapshot" - DOC_URL: "https://$PAGE_URL/doc/snapshot/" - VERSIONED: "true" - -.publishReleaseVar: &publishReleaseVar - PYPI_REPO: "pypi" - DOC_URL_PREFIX: "$PAGE_URL" - DOC_URL: "https://$PAGE_URL/doc/" - VERSIONED: "true" - -.publishMasterVar: &publishMasterVar - PYPI_REPO: "pypi" - DOC_URL_PREFIX: "$PAGE_URL" - DOC_URL: "https://$PAGE_URL/doc/" - VERSIONED: "false" - -# --------------------------------- STAGE: version --------------------------------------------------------------------- -version-branch: - stage: version - image: - name: mdomke/git-semver:v4.0.1 - entrypoint: [""] - script: - - cd /builds/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME && git-semver --format x.y.z+m | sed -e 's/$/-SNAPSHOT/g' | xargs -I {} echo "export VERSION={}" > $VERSION_FILE - - cat $VERSION_FILE - artifacts: - paths: - - $VERSION_FILE - tags: [ docker ] - except: [ master, tags ] - -version-tag: - stage: version - image: - name: mdomke/git-semver:v4.0.1 - entrypoint: [""] - script: - - cd /builds/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME && git-semver --no-hash --no-pre | xargs -I {} echo "export VERSION={}" > $VERSION_FILE - - cat $VERSION_FILE - artifacts: - paths: - - $VERSION_FILE - tags: [ docker ] - except: - - branches - -tox: - variables: - <<: *python37Vars - stage: tox - image: python:$PYTHON_VERSION - cache: - key: $CI_PROJECT_PATH - paths: - - "/root/.cache/" - before_script: - - pip install -U -r requirements.txt - script: - - tox - artifacts: - paths: - - htmlcov/ - tags: [ docker ] - - -behave-test: - stage: behave-test - variables: - <<: *python39Vars - image: - name: $SPARK_IMAGE - before_script: - - pip install -U -e . - - pip install behave==1.2.6 inflect==5.3.0 coverage - - export PYSPARK_SUBMIT_ARGS="--packages org.biodatageeks:sequila_${SCALA_VERSION}:${SEQUILA_VERSION} --repositories $MAVEN_SNAPSHOT_REPO --driver-memory 4g pyspark-shell" - script: - - coverage run $(which behave) - - coverage report - artifacts: - paths: - - /builds/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/docs/build/ - tags: [ docker ] - -.build: &build - image: python:$PYTHON_VERSION - cache: - key: $CI_PROJECT_PATH - paths: - - "/root/.cache/" - before_script: - - source $VERSION_FILE - - pip install -U -r requirements.txt - script: - - python setup.py sdist bdist_wheel - - twine check dist/* - artifacts: - paths: - - /builds/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/dist/ - tags: [ docker ] - - -build-python37: - stage: build - variables: - <<: *python37Vars - <<: *build - except: [ master ] - -.publish: &publish - image: python:$PYTHON_VERSION - cache: - key: $CI_PROJECT_PATH - paths: - - "/root/.cache/" - before_script: - - source $VERSION_FILE - - pip install -U -r requirements.txt - script: - - twine upload --repository=$PYPI_REPO dist/* --config-file=/root/pypi/.pypirc - tags: [ docker ] - - -publishSnapshotpython37: - stage: publish - variables: - <<: *python37Vars - <<: *publishSnapshotVar - <<: *publish - except: [ master, tags ] - -publishReleasetpython37: - stage: publish - variables: - <<: *python37Vars - <<: *publishReleaseVar - <<: *publish - only: - - tags - except: - - branches - - -.doc-build: &doc-build - stage: doc-build - variables: - <<: *python39Vars - image: - name: $SPARK_IMAGE - before_script: - - source $VERSION_FILE - - pip install -U sphinx==$SPHINX_VERSION nbsphinx sphinx_rtd_theme rst2pdf sphinxcontrib-github_ribbon sphinxcontrib-bibtex pysequila==$VERSION --extra-index-url=$PIP_SNAPSHOT_REPO --trusted-host $NEXUS_HOST - - export PYSPARK_SUBMIT_ARGS="--packages org.biodatageeks:sequila_${SCALA_VERSION}:${SEQUILA_VERSION} --repositories $MAVEN_SNAPSHOT_REPO --driver-memory 4g pyspark-shell" - script: - - cd docs && ./docs.sh html - artifacts: - paths: - - /builds/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/docs/build/ - tags: [ docker ] - - - -.page-build: &page-build - stage: page-build - image: - name: klakegg/hugo:$HUGO_VERSION - entrypoint: [""] - before_script: - - apt update && apt-get install -y gettext - script: - - envsubst < page/config-template.toml > page/config.toml - - cd page && hugo --enableGitInfo=false - artifacts: - paths: - - /builds/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/page/docs/ - tags: [ docker ] - -doc-buildSnapshot: - <<: *doc-build - except: [ master, tags ] - -doc-buildRelease: - <<: *doc-build - needs: - - publishReleasetpython37 - only: - - tags - - -page-buildSnapshot: - variables: - <<: *publishSnapshotVar - <<: *page-build - except: [ master, tags ] - -page-buildRelease: - variables: - <<: *publishReleaseVar - <<: *page-build - only: - - tags - -page-buildMaster: - variables: - <<: *publishMasterVar - <<: *page-build - only: - - master - -doc-serve: - stage: doc-serve - image: - name: docker:$DOCKER_VERSION - before_script: - - source $VERSION_FILE - script: - - cd /builds/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/docs - - ../build_image.sh "doc" 81 80 - tags: [ docker ] - except: [ master] - -page-serve: - stage: page-serve - image: - name: docker:$DOCKER_VERSION - before_script: - - source $VERSION_FILE - script: - - cd /builds/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/page - - ../build_image.sh "page" 85 84 - tags: [ docker ] - except: [ master] - -page-serve-master: - stage: page-serve - image: - name: docker:$DOCKER_VERSION - script: - - cd /builds/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/page - - ../build_image.sh "page" 85 84 - tags: [ docker ] - only: [ master ] \ No newline at end of file diff --git a/README.rst b/README.rst index 312cfad9..485b76b3 100644 --- a/README.rst +++ b/README.rst @@ -51,7 +51,17 @@ Features * other utility functions * support for both SQL and Dataframe/Dataset API -Setup +Building +===== + +:: + + $ python3.9 -m venv .venv + $ source .venv/bin/activate + $ pip install -r requirements.txt + $ python setup.py sdist bdist_wheel + +Install ===== :: @@ -66,19 +76,20 @@ Usage :: $ python + >>> import os >>> from pysequila import SequilaSession + >>> curr_dir = os.getcwd() >>> ss = SequilaSession \ .builder \ - .config("spark.jars.packages", "org.biodatageeks:sequila_2.12:1.1.0") \ + .config("spark.jars.packages", "org.biodatageeks:sequila_2.12:1.3.6") \ .config("spark.driver.memory", "2g") \ .getOrCreate() - >>> ss.sql( - f""" + >>> ss.sql(f""" CREATE TABLE IF NOT EXISTS reads USING org.biodatageeks.sequila.datasources.BAM.BAMDataSource - OPTIONS(path "/features/data/NA12878.multichrom.md.bam") - """ - >>> ss.sql ("SELECT * FROM coverage('reads', 'NA12878','/features/data/Homo_sapiens_assembly18_chr1_chrM.small.fasta") + OPTIONS(path "{curr_dir}/features/data/NA12878.multichrom.md.bam") + """) + >>> df = ss.sql (f"SELECT * FROM coverage('reads', 'NA12878','{curr_dir}/features/data/Homo_sapiens_assembly18_chr1_chrM.small.fasta')") >>> # or using DataFrame/DataSet API - >>> ss.coverage("/features/data/NA12878.multichrom.md.bam", "/features/data/Homo_sapiens_assembly18_chr1_chrM.small.fasta") + >>> df = ss.coverage(f"{curr_dir}/features/data/NA12878.multichrom.md.bam", "{curr_dir}/features/data/Homo_sapiens_assembly18_chr1_chrM.small.fasta") diff --git a/requirements.txt b/requirements.txt index aed59a8c..c05f3386 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,26 @@ +coverage==7.6.4 +exceptiongroup==1.2.2 +findspark==2.0.1 +iniconfig==2.0.0 +numpy==2.0.2 +packaging==24.1 +pandas==2.2.3 +pip==24.2 +pluggy==1.5.0 +py4j==0.10.9.5 +pycodestyle==2.12.1 pyspark==3.2.2 -findspark -setuptools -wheel -twine -tox \ No newline at end of file +pytest==8.3.3 +pytest-cov==6.0.0 +pytest-random==0.2 +pytest-remove-stale-bytecode==6.0 +python-dateutil==2.9.0.post0 +pytz==2024.2 +six==1.16.0 +tomli==2.0.2 +typeguard==2.9.1 +tzdata==2024.2 +tox==4.23.2 +setuptools==75.2.0 +wheel==0.44.0 +twine==5.1.1 \ No newline at end of file diff --git a/requirements_dev.txt b/requirements_dev.txt deleted file mode 100644 index 5187259e..00000000 --- a/requirements_dev.txt +++ /dev/null @@ -1,13 +0,0 @@ -pyspark==3.0.1 -findspark -setuptools -wheel -twine -tox -nbsphinx -sphinx==3.2.1 -sphinx_rtd_theme -rst2pdf -sphinxcontrib-github_ribbon -sphinxcontrib-bibtex -pandoc \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index a627cd26..72be8807 100644 --- a/setup.cfg +++ b/setup.cfg @@ -17,4 +17,4 @@ upload-dir = docs/build/html formats = gztar [bdist_wheel] -universal = 1 +universal = 0 diff --git a/setup.py b/setup.py index 2c3b98b7..18220bb0 100644 --- a/setup.py +++ b/setup.py @@ -49,10 +49,14 @@ def read_content(filepath): "Programming Language :: Python :: Implementation :: PyPy", ] - long_description = read_content("README.rst") + read_content(os.path.join("docs/source", "CHANGELOG.rst")) -requires = ["setuptools", "typeguard==2.9.1", "pyspark==3.2.2", "findspark", "pandas"] +requires = [ + "typeguard==2.9.1", + "pyspark==3.2.2", + "findspark==2.0.1", + "pandas==2.2.3" +] extras_require = { "reST": ["Sphinx"], @@ -62,7 +66,7 @@ def read_content(filepath): setup( name="pysequila", - version=os.getenv("VERSION", "0.1.0"), + version=os.getenv("VERSION", "0.4.1"), description="An SQL-based solution for large-scale genomic analysis", long_description=long_description, long_description_content_type="text/x-rst", diff --git a/tox.ini b/tox.ini index 39445866..e1efc09a 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] envlist = - py37, + py39, pycodestyle pydocstyle @@ -13,22 +13,39 @@ disable=W0212 [py] deps= - pytest-cov - pytest-random - pytest-remove-stale-bytecode - pyspark==3.1.2 - findspark + coverage==7.6.4 + exceptiongroup==1.2.2 + findspark==2.0.1 + iniconfig==2.0.0 + numpy==2.0.2 + packaging==24.1 + pandas==2.2.3 + pluggy==1.5.0 + py4j==0.10.9.5 + pycodestyle==2.12.1 + pyspark==3.2.2 + pytest==8.3.3 + pytest-cov==6.0.0 + pytest-random==0.2 + pytest-remove-stale-bytecode==6.0 + python-dateutil==2.9.0.post0 + pytz==2024.2 + setuptools==75.2.0 + six==1.16.0 + tomli==2.0.2 typeguard==2.9.1 + tzdata==2024.2 + wheel==0.44.0 [pycodestyle] show-source=True statistics=True -exclude=venv,.svn,CVS,.bzr,.hg,.git,__pycache__,.tox,setup.py,docs, features +exclude=venv,.venv,.eggs,.svn,CVS,.bzr,.hg,.git,__pycache__,.tox,setup.py,docs, features -[testenv:py37] +[testenv:py39] deps= {[py]deps} -basepython = python3.7 +basepython = python3.9 [testenv:pypy3] @@ -40,11 +57,11 @@ basepython = pypy3 deps= {[py]deps} pycodestyle -basepython = python3.7 +basepython = python3.9 commands = pycodestyle --max-line-length=128 --first [testenv:pydocstyle] deps= pydocstyle commands = pydocstyle pysequila -basepython = python3.7 +basepython = python3.9