diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml new file mode 100644 index 000000000..46d13d83c --- /dev/null +++ b/.github/workflows/create-release.yml @@ -0,0 +1,21 @@ +name: Create GitHub release + +on: + push: + tags: + - 'v*' + +permissions: + contents: write + +jobs: + create-release: + runs-on: ubuntu-latest + steps: + - name: Create release with generated notes + uses: softprops/action-gh-release@v2 + with: + tag_name: ${{ github.ref_name }} + generate_release_notes: true + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/publish-release.yml b/.github/workflows/publish-release.yml new file mode 100644 index 000000000..5d89c4edc --- /dev/null +++ b/.github/workflows/publish-release.yml @@ -0,0 +1,120 @@ +name: Publish release artifacts + +on: + release: + types: [published] + workflow_dispatch: + inputs: + tag: + description: 'Git ref (tag or branch) to publish or test' + required: true + type: string + mode: + description: 'Where to publish artifacts' + required: true + default: release + type: choice + options: + - release + - test + +permissions: + contents: read + +jobs: + publish-backend: + if: ${{ github.event_name != 'release' || !github.event.release.prerelease }} + runs-on: ubuntu-latest + env: + RELEASE_MODE: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.mode || 'release' }} + RELEASE_TAG: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.tag || github.event.release.tag_name }} + steps: + - name: Check out release tag + uses: actions/checkout@v4 + with: + ref: ${{ env.RELEASE_TAG }} + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install build tooling + run: | + python -m pip install --upgrade pip + python -m pip install build + + - name: Build backend distribution + working-directory: backend + run: python -m build + + - name: Publish backend to PyPI + if: env.RELEASE_MODE == 'release' + uses: pypa/gh-action-pypi-publish@release/v1 + with: + packages_dir: backend/dist + password: ${{ secrets.PYPI_API_TOKEN }} + + - name: Publish backend to TestPyPI + if: env.RELEASE_MODE == 'test' + uses: pypa/gh-action-pypi-publish@release/v1 + with: + packages_dir: backend/dist + repository-url: https://test.pypi.org/legacy/ + password: ${{ secrets.TEST_PYPI_API_TOKEN }} + + publish-labextension: + if: ${{ github.event_name != 'release' || !github.event.release.prerelease }} + needs: publish-backend + runs-on: ubuntu-latest + env: + RELEASE_MODE: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.mode || 'release' }} + RELEASE_TAG: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.tag || github.event.release.tag_name }} + NODE_AUTH_TOKEN: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.mode == 'test' && secrets.TEST_NPM_TOKEN || secrets.NPM_TOKEN }} + steps: + - name: Check out release tag + uses: actions/checkout@v4 + with: + ref: ${{ env.RELEASE_TAG }} + fetch-depth: 0 + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + registry-url: 'https://registry.npmjs.org' + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install JupyterLab build tooling + working-directory: labextension + run: | + python -m pip install --upgrade pip + python -m pip install 'jupyterlab>=4.2,<5.0' + + - name: Install Node dependencies + working-directory: labextension + run: | + corepack enable + jlpm install --frozen-lockfile + + - name: Build labextension assets + working-directory: labextension + run: jlpm build:prod + + - name: Publish labextension to npm + if: env.RELEASE_MODE == 'release' + working-directory: labextension + run: npm publish + + - name: Exercise npm publish workflow (dry run) + if: env.RELEASE_MODE == 'test' + working-directory: labextension + run: | + npm publish --dry-run + mkdir -p dist + npm pack --pack-destination dist diff --git a/CONTRIBUTE.md b/CONTRIBUTE.md new file mode 100644 index 000000000..f7e7f3541 --- /dev/null +++ b/CONTRIBUTE.md @@ -0,0 +1,126 @@ +# Contributing to the Kale Backend + +This guide walks through setting up a local development environment for the Kale +backend, explains how the package version flows through the system, and shows how +to iterate safely using a temporary Python package index. + +> The backend lives under `backend/`. All commands below assume you run them from +> the repository root unless noted otherwise. + +## 1. Prerequisites + +- Python 3.10 (managed via `pyenv`, `conda`, or your system package manager). +- Recent `pip` and `virtualenv`/`venv`. +- `node`/`yarn` are only required when working on the labextension (not covered here). +- Optional but recommended: `devpi-server` and `devpi-client` for local package + publishing (see “Local package index”). + +## 2. Create a virtual environment + +```bash +python -m venv .venv +source .venv/bin/activate +python -m pip install --upgrade pip +``` + +## 3. Install the backend in editable mode + +```bash +cd backend +pip install -e .[dev] +``` + +The `pip install -e .[dev]` step relies on `backend/pyproject.toml`: + +- Dependencies are declared in `[project] dependencies`. +- `setuptools_scm` infers the package version from git tags, so the version number + evolves automatically (`1.2.3` → `1.2.3.devN+gHASH` for untagged commits). You do + *not* have to edit any file to bump versions during development. + +## 4. Understanding version detection in code generation + +Kale exposes its version at runtime via `kale.__version__`. When a notebook is +compiled into a Kubeflow Pipelines (KFP) component: + +- `kale.compiler.Compiler._get_package_list_from_imports` always includes a Kale + dependency. If a concrete version is available (`__version__ != "0+unknown"`), + the component pins to `kubeflow-kale==`. Otherwise it falls back to an + unpinned `kubeflow-kale`. +- `kale.common.utils.compute_pip_index_urls` determines which PyPI indexes are + baked into the generated component. It honours environment variables in the + following order: + 1. `KALE_PIP_INDEX_URLS` – comma separated list, highest priority. + 2. `KALE_DEV_MODE` + optional `KALE_DEVPI_SIMPLE_URL` – enables your local devpi. + 3. Default fallback – `https://pypi.org/simple`. + +Because compiled workflows freeze both the Kale version and the index list, +updating your local package copy requires re‑publishing the wheel/sdist that the +pipeline will install. That is why a disposable package index (devpi) is useful – +you can iterate without uploading experimental builds to the public PyPI. + +## 5. Working with the local devpi index + +The `backend/scripts/` directory provides helpers for spinning up a local devpi +registry and publishing the freshly built Kale package: + +| Script | Purpose | Typical usage | +| ------ | ------- | ------------- | +| `devpi-up.sh` | Start (or reuse) a devpi-server instance on `HOST:PORT` and create a volatile `root/` derived from the public PyPI mirror. Prints the environment variables needed for pip. | `./scripts/devpi-up.sh` | +| `devpi-publish.sh` | Build Kale (`python -m build`), infer the newest artifact version from `dist/`, remove any existing release with the same version from the target devpi index, then upload the artifacts. | `./scripts/devpi-publish.sh` | +| `devpi-main.sh` | Convenience wrapper that calls the two scripts above and exports `KALE_DEV_MODE=1` + `KALE_DEVPI_SIMPLE_URL` in the current shell. | `source ./scripts/devpi-main.sh` | + +Recommended loop when iterating on compiled pipelines: + +```bash +source .venv/bin/activate +cd backend + +# Start devpi and export KALE_DEV_MODE / KALE_DEVPI_SIMPLE_URL / pip overrides. +source scripts/devpi-main.sh + +# Make your code changes… +# Rebuild and publish to the local index (removes the previous dev build). +./scripts/devpi-publish.sh + +# Re-run notebook compilation so the generated component pulls from devpi. +kale --help # or your usual CLI invocation pointing at the notebook +# For example you can run this quick test to validate the the generated DSL points +# for the correct Kale version +# kale --nb ./examples/base/candies_sharing.ipynb --dev +``` + +> Note: the scripts rely on `devpi-server`, `devpi-client`, and `python -m build`. +> Install them once in your virtualenv: `pip install devpi-server devpi-client build`. + +## 6. Running tests + +```bash +cd backend +python -m pytest backend/kale/tests/unit_tests +``` + +Integration tests under `backend/kale/tests/e2e/` perform notebook → DSL +compilation and compare against golden files in `backend/kale/tests/assets/`. +If you alter template output, update the generated files accordingly. + +## 7. Formatting and linting + +- Python formatting is enforced via the existing code style (PEP 8). Many generated + files run through `autopep8`. +- The labextension has independent tooling (`jlpm lint`, `jlpm prettier`). Run + those only if you modify the frontend package. + +## 8. Typical development checklist + +1. Activate your virtual environment. +2. `pip install -e .[dev]`. +3. Optionally start `devpi` via `source scripts/devpi-main.sh`. +4. Implement and re-publish using `scripts/devpi-publish.sh` when you need a new + dev build. +5. Update fixtures (`backend/kale/tests/assets/kfp_dsl/*.py`) whenever template + output changes. +6. Run unit and (when relevant) e2e tests. +7. Before committing, ensure `git status` shows only intentional changes. + +Happy hacking! If anything in this guide is unclear, open an issue or PR with +improvements. Contributions are welcome.*** diff --git a/backend/README.md b/backend/README.md new file mode 100644 index 000000000..3b5223b20 --- /dev/null +++ b/backend/README.md @@ -0,0 +1,5 @@ +# Kubeflow Kale Backend + +This package contains the Kubeflow Kale Python backend. For general +project documentation and usage instructions, please refer to the +repository root [README](../README.md). diff --git a/backend/kale/__init__.py b/backend/kale/__init__.py index 400498437..c9be56be9 100644 --- a/backend/kale/__init__.py +++ b/backend/kale/__init__.py @@ -12,6 +12,27 @@ # See the License for the specific language governing permissions and # limitations under the License. +# --- Detect package version at runtime --- +# Use: +# from kale import __version__ as KALE_VERSION +# use KALE_VERSION wherever a display/log/version check is needed +try: + from importlib.metadata import version as _pkg_version, PackageNotFoundError +except Exception: # Py<3.8 fallback if needed + from importlib_metadata import ( # type: ignore + version as _pkg_version, + PackageNotFoundError, + ) + +try: + __version__ = _pkg_version("kubeflow-kale") +except PackageNotFoundError: + # this might happen when a developer tried to test Kale locally from source + # without installing it first. + __version__ = "0+unknown" + +# ----------------------------------------- + from typing import NamedTuple, Any @@ -34,6 +55,18 @@ class Artifact(NamedTuple): from .processors import NotebookProcessor, NotebookConfig, PythonProcessor from kale.common import logutils -__all__ = ["PipelineParam", "Artifact",'NotebookProcessor', 'Step', 'StepConfig', 'Pipeline', 'PipelineConfig', 'VolumeConfig', 'Compiler', 'marshal'] +__all__ = [ + "PipelineParam", + "Artifact", + "NotebookProcessor", + "Step", + "StepConfig", + "Pipeline", + "PipelineConfig", + "VolumeConfig", + "Compiler", + "marshal", +] + logutils.get_or_create_logger(module=__name__, name="kale") del logutils diff --git a/backend/kale/cli.py b/backend/kale/cli.py index 4c443701a..76fb2209d 100644 --- a/backend/kale/cli.py +++ b/backend/kale/cli.py @@ -12,9 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os import argparse -# import os -# import warnings from argparse import RawTextHelpFormatter from kale.processors import NotebookProcessor @@ -60,6 +59,26 @@ def main(): general_group.add_argument('--run_pipeline', action='store_const', const=True) general_group.add_argument('--debug', action='store_true') + general_group.add_argument( + '--dev', + action='store_true', + help='Bake local dev index (devpi) into generated components.', + ) + general_group.add_argument( + '--pip-index-urls', + type=str, + help=('Comma-separated PEP 503 simple indexes to bake into components.' + 'Overrides --dev/KALE_DEV_MODE. Example: ' + '"http://127.0.0.1:3141/root/dev/+simple/,' + 'https://pypi.org/simple"'), + ) + general_group.add_argument( + '--devpi-simple-url', + type=str, + default=None, + help=('Devpi simple URL to use when --dev is set. ' + 'Default: http://127.0.0.1:3141/root/dev/+simple/'), + ) metadata_group = parser.add_argument_group('Notebook Metadata Overrides', METADATA_GROUP_DESC) @@ -84,6 +103,13 @@ def main(): help='The access mode for the created volumes') args = parser.parse_args() + if args.pip_index_urls: + os.environ["KALE_PIP_INDEX_URLS"] = args.pip_index_urls + elif args.dev: + os.environ["KALE_DEV_MODE"] = "1" + if args.devpi_simple_url: + os.environ["KALE_DEVPI_SIMPLE_URL"] = args.devpi_simple_url + # get the notebook metadata args group mt_overrides_group = next( filter(lambda x: x.title == 'Notebook Metadata Overrides', diff --git a/backend/kale/common/utils.py b/backend/kale/common/utils.py index 064335a8e..3d678511e 100644 --- a/backend/kale/common/utils.py +++ b/backend/kale/common/utils.py @@ -195,3 +195,64 @@ def dedent(text: str): if len(matches) < len(text.splitlines()): return text return re.sub(r"(?m)^.{%d}" % min(map(len, matches)), "", text) + + +def compute_pip_index_urls() -> list[str]: + """Compute the list of pip simple index URLs for generated KFP components. + + Using a local PyPI index is useful when itering on local + developement with an unpublished version of Kale. + + Precedence: + 1. If `KALE_PIP_INDEX_URLS` is set, split its comma-separated value and + return that list (order preserved). + 2. Else, if `KALE_DEV_MODE` is truthy (`1`, `true`, `yes`, or `on`), + return a list with the devpi simple URL (`KALE_DEVPI_SIMPLE_URL`) or + its default value. + 3. Otherwise, return the production default: + ["https://pypi.org/simple"]. + + Environment variables: + KALE_PIP_INDEX_URLS: + Comma-separated list of PEP 503 “simple” index URLs. Highest + priority. + KALE_DEV_MODE: + Boolean-like flag enabling dev mode (interprets 1/true/yes/on). + KALE_DEVPI_SIMPLE_URL: + Devpi “simple” index URL used when dev mode is enabled. + + Returns: + list[str]: Index URLs suitable for the `pip_index_urls` parameter in + `@kfp_dsl.component`. + """ + pypi_prod_url = "https://pypi.org/simple" + urls: list[str] + + # explicit override wins + env_override = os.getenv("KALE_PIP_INDEX_URLS") + if env_override: + urls = [u.strip() for u in env_override.split(",") if u.strip()] + else: + urls = [] + dev_mode = os.getenv("KALE_DEV_MODE", "").lower() in { + "1", + "true", + "yes", + "on", + } + if dev_mode: + urls.append( + os.getenv( + "KALE_DEVPI_SIMPLE_URL", + "http://127.0.0.1:3141/root/dev/+simple/", + ) + ) + + # important to keep the prod url at the end to preserve package + # resolution order. + urls.append(pypi_prod_url) + + # remove duplicates while keeping order + if pypi_prod_url not in urls: + urls.append(pypi_prod_url) + return urls diff --git a/backend/kale/compiler.py b/backend/kale/compiler.py index 85b63fc03..ea87fb522 100644 --- a/backend/kale/compiler.py +++ b/backend/kale/compiler.py @@ -20,8 +20,9 @@ from typing import NamedTuple from jinja2 import Environment, PackageLoader, FileSystemLoader +from kale import __version__ as KALE_VERSION from kale.pipeline import Pipeline, Step, PipelineParam -from kale.common import kfputils +from kale.common import kfputils, utils log = logging.getLogger(__name__) @@ -204,6 +205,7 @@ def _encode_source(s): packages_list = self._get_package_list_from_imports() fn_code = template.render( + pip_index_urls=utils.compute_pip_index_urls(), step=step, component_signature_args=component_signature_args, pipeline_params=pipeline_params, @@ -269,9 +271,10 @@ def _get_package_list_from_imports(self): A list of unique top-level package names. """ package_names = set() - # Ensure 'kale' is always included - package_names.add("kubeflow-kale==1.0.0.dev13") - # Ensure 'kfp' is always included + if KALE_VERSION != "0+unknown": + package_names.add(f"kubeflow-kale=={KALE_VERSION}") + else: + package_names.add("kubeflow-kale") package_names.add("kfp>=2.0.0") lines = self.imports_and_functions.strip().split('\n') diff --git a/backend/kale/templates/new_nb_function_template.jinja2 b/backend/kale/templates/new_nb_function_template.jinja2 index 943b6fe00..04aa318b8 100644 --- a/backend/kale/templates/new_nb_function_template.jinja2 +++ b/backend/kale/templates/new_nb_function_template.jinja2 @@ -1,7 +1,7 @@ @kfp_dsl.component( base_image='python:3.10', packages_to_install={{ packages_list}}, - pip_index_urls = [ 'https://test.pypi.org/simple', 'https://pypi.org/simple' ], + pip_index_urls = {{ pip_index_urls }}, ) def {{ step.name }}_step({{ component_signature_args }}): _kale_pipeline_parameters_block = ''' diff --git a/backend/kale/tests/assets/kfp_dsl/iris.py b/backend/kale/tests/assets/kfp_dsl/iris.py index 9429619be..c62248781 100644 --- a/backend/kale/tests/assets/kfp_dsl/iris.py +++ b/backend/kale/tests/assets/kfp_dsl/iris.py @@ -6,8 +6,8 @@ @kfp_dsl.component( base_image='python:3.10', packages_to_install=['kfp>=2.0.0', - 'kubeflow-kale==1.0.0.dev13', 'numpy', 'scikit-learn'], - pip_index_urls=['https://test.pypi.org/simple', 'https://pypi.org/simple'], + 'kubeflow-kale', 'numpy', 'scikit-learn'], + pip_index_urls=['https://pypi.org/simple'], ) def load_transform_data_step(load_transform_data_html_report: Output[HTML], x_trn_artifact: Output[Dataset], x_tst_artifact: Output[Dataset], y_trn_artifact: Output[Dataset], y_tst_artifact: Output[Dataset], n_estimators_param: int = 500, max_depth_param: int = 2): _kale_pipeline_parameters_block = ''' @@ -79,8 +79,8 @@ def load_transform_data_step(load_transform_data_html_report: Output[HTML], x_tr @kfp_dsl.component( base_image='python:3.10', packages_to_install=['kfp>=2.0.0', - 'kubeflow-kale==1.0.0.dev13', 'numpy', 'scikit-learn'], - pip_index_urls=['https://test.pypi.org/simple', 'https://pypi.org/simple'], + 'kubeflow-kale', 'numpy', 'scikit-learn'], + pip_index_urls=['https://pypi.org/simple'], ) def train_model_step(train_model_html_report: Output[HTML], x_trn_artifact: Input[Dataset], y_trn_artifact: Input[Dataset], model_artifact: Output[Model], n_estimators_param: int = 500, max_depth_param: int = 2): _kale_pipeline_parameters_block = ''' @@ -151,8 +151,8 @@ def train_model_step(train_model_html_report: Output[HTML], x_trn_artifact: Inpu @kfp_dsl.component( base_image='python:3.10', packages_to_install=['kfp>=2.0.0', - 'kubeflow-kale==1.0.0.dev13', 'numpy', 'scikit-learn'], - pip_index_urls=['https://test.pypi.org/simple', 'https://pypi.org/simple'], + 'kubeflow-kale', 'numpy', 'scikit-learn'], + pip_index_urls=['https://pypi.org/simple'], ) def evaluate_model_step(evaluate_model_html_report: Output[HTML], model_artifact: Input[Model], x_tst_artifact: Input[Dataset], y_tst_artifact: Input[Dataset], n_estimators_param: int = 500, max_depth_param: int = 2): _kale_pipeline_parameters_block = ''' diff --git a/backend/kale/tests/assets/kfp_dsl/pipeline_parameters_and_metrics.py b/backend/kale/tests/assets/kfp_dsl/pipeline_parameters_and_metrics.py index 552b27f91..a0106c3f6 100644 --- a/backend/kale/tests/assets/kfp_dsl/pipeline_parameters_and_metrics.py +++ b/backend/kale/tests/assets/kfp_dsl/pipeline_parameters_and_metrics.py @@ -5,8 +5,8 @@ @kfp_dsl.component( base_image='python:3.10', - packages_to_install=['kfp>=2.0.0', 'kubeflow-kale==1.0.0.dev13', 'numpy'], - pip_index_urls=['https://test.pypi.org/simple', 'https://pypi.org/simple'], + packages_to_install=['kfp>=2.0.0', 'kubeflow-kale', 'numpy'], + pip_index_urls=['https://pypi.org/simple'], ) def create_matrix_step(create_matrix_html_report: Output[HTML], rnd_matrix_artifact: Output[Dataset], d1: int = 5, d2: int = 6, booltest: bool = True, strtest: str = 'test'): _kale_pipeline_parameters_block = ''' @@ -72,8 +72,8 @@ def create_matrix_step(create_matrix_html_report: Output[HTML], rnd_matrix_artif @kfp_dsl.component( base_image='python:3.10', - packages_to_install=['kfp>=2.0.0', 'kubeflow-kale==1.0.0.dev13', 'numpy'], - pip_index_urls=['https://test.pypi.org/simple', 'https://pypi.org/simple'], + packages_to_install=['kfp>=2.0.0', 'kubeflow-kale', 'numpy'], + pip_index_urls=['https://pypi.org/simple'], ) def sum_matrix_step(sum_matrix_html_report: Output[HTML], rnd_matrix_artifact: Input[Dataset], d1: int = 5, d2: int = 6, booltest: bool = True, strtest: str = 'test'): _kale_pipeline_parameters_block = ''' diff --git a/backend/kale/tests/e2e/test_e2e.py b/backend/kale/tests/e2e/test_e2e.py index b0afce35d..38d832024 100644 --- a/backend/kale/tests/e2e/test_e2e.py +++ b/backend/kale/tests/e2e/test_e2e.py @@ -31,6 +31,7 @@ os.path.join(THIS_DIR, "../assets/kfp_dsl/", "pipeline_parameters_and_metrics.py")), ]) +@mock.patch("kale.compiler.KALE_VERSION", new="0+unknown") @mock.patch("kale.common.utils.random_string") def test_notebook_to_dsl(random_string, notebook_path, dsl_path): """Test code generation end to end from notebook to DSL.""" diff --git a/backend/kale/tests/unit_tests/test_utils.py b/backend/kale/tests/unit_tests/test_utils.py index 2ff7feb1f..b4dedcaf0 100644 --- a/backend/kale/tests/unit_tests/test_utils.py +++ b/backend/kale/tests/unit_tests/test_utils.py @@ -64,3 +64,74 @@ def test_dedent(): ) assert utils.dedent(text) == target + + +def _clear_env(monkeypatch): + """Ensure Kale-specific env vars are unset for predictable tests.""" + for key in ( + "KALE_PIP_INDEX_URLS", + "KALE_DEV_MODE", + "KALE_DEVPI_SIMPLE_URL", + ): + monkeypatch.delenv(key, raising=False) + + +def test_compute_pip_index_urls_default(monkeypatch): + """When no env overrides are present, fall back to PyPI only.""" + _clear_env(monkeypatch) + + assert utils.compute_pip_index_urls() == ["https://pypi.org/simple"] + + +def test_compute_pip_index_urls_dev_mode_default_url(monkeypatch): + """Dev mode without a custom index uses the default devpi URL + PyPI.""" + _clear_env(monkeypatch) + monkeypatch.setenv("KALE_DEV_MODE", "true") + + assert utils.compute_pip_index_urls() == [ + "http://127.0.0.1:3141/root/dev/+simple/", + "https://pypi.org/simple", + ] + + +def test_compute_pip_index_urls_dev_mode_custom_url(monkeypatch): + """Dev mode honors KALE_DEVPI_SIMPLE_URL before appending PyPI.""" + _clear_env(monkeypatch) + monkeypatch.setenv("KALE_DEV_MODE", "1") + monkeypatch.setenv("KALE_DEVPI_SIMPLE_URL", + "https://devpi.example/simple/") + + assert utils.compute_pip_index_urls() == [ + "https://devpi.example/simple/", + "https://pypi.org/simple", + ] + + +def test_compute_pip_index_urls_override(monkeypatch): + """Explicit overrides take precedence and keep their order.""" + _clear_env(monkeypatch) + monkeypatch.setenv( + "KALE_PIP_INDEX_URLS", + "https://mirror.one/simple, https://mirror.two/simple", + ) + + assert utils.compute_pip_index_urls() == [ + "https://mirror.one/simple", + "https://mirror.two/simple", + "https://pypi.org/simple", + ] + + +def test_compute_pip_index_urls_override_beats_dev_mode(monkeypatch): + """An explicit override should win even when dev mode is enabled.""" + _clear_env(monkeypatch) + monkeypatch.setenv("KALE_DEV_MODE", "true") + monkeypatch.setenv( + "KALE_PIP_INDEX_URLS", + "https://mirror.only/simple", + ) + + assert utils.compute_pip_index_urls() == [ + "https://mirror.only/simple", + "https://pypi.org/simple", + ] diff --git a/backend/pyproject.toml b/backend/pyproject.toml new file mode 100644 index 000000000..c7a8967ec --- /dev/null +++ b/backend/pyproject.toml @@ -0,0 +1,66 @@ +[build-system] +requires = ["setuptools>=64", "setuptools_scm>=8", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "kubeflow-kale" +# setuptools_scm handles the version automatically +dynamic = ["version"] +description = "Convert JupyterNotebooks to Kubeflow Pipelines deployments" +readme = "README.md" +requires-python = ">=3.10.0" +license = { text = "Apache License Version 2.0" } + +dependencies = [ + "kfp>=2.0.0", + "autopep8>=2.0.0", + "astor>=0.8.1", + "networkx>=3.0.0", + "jinja2>=3.0.0", + "pyflakes>=3.0.0", + "dill>=0.3.8", + "IPython>=8.30.0", + "jupyter-client>=8.6.3", + "jupyter-core>=5.8.1", + "nbconvert>=7.16.0", + "nbformat>=5.10.4", + "ipykernel>=6.29.5", + "notebook>=7.4.4", + "packaging>=25.0", + "progress>=1.5", + "kubernetes>=30.0.0", +] + +authors = [ + { name = "Stefano Fioravanzo", email = "stefano.fioravanzo@gmail.com" } +] + +[project.scripts] +kale = "kale.cli:main" +kale_server = "kale.cli:server" +kale-volumes = "kale.cli:kale_volumes" + +[project.urls] +Homepage = "https://github.com/kubeflow-kale/kale" + +[project.optional-dependencies] +dev = [ + "pytest", + "pytest-clarity", + "testfixtures", + "pytest-cov", + "flake8", + "flake8-docstrings", +] + +# autodiscover packages +[tool.setuptools.packages.find] +where = ["."] +include = ["kale*"] + +# setuptools_scm configuration (git tags versions) +[tool.setuptools_scm] +# Tag form: v1.0.0 -> 1.0.0 ; commit after tag -> 1.0.0.devN+gHASH +local_scheme = "node-and-date" +fallback_version = "0+unknown" +root = ".." diff --git a/backend/scripts/devpi-main.sh b/backend/scripts/devpi-main.sh new file mode 100755 index 000000000..0c453ffa4 --- /dev/null +++ b/backend/scripts/devpi-main.sh @@ -0,0 +1,11 @@ +# 1) Start devpi and export env for dev mode +./scripts/devpi-up.sh +export KALE_DEV_MODE=1 +export KALE_DEVPI_SIMPLE_URL=http://127.0.0.1:3141/root/dev/+simple/ + +# 2) Build and publish to local index (auto removes same dev version) +./scripts/devpi-publish.sh + +# 3) Run your KFP pipeline generation/execution +# The template now uses the local index while KALE_DEV_MODE=1 is set. +# kale compile ... \ No newline at end of file diff --git a/backend/scripts/devpi-publish.sh b/backend/scripts/devpi-publish.sh new file mode 100755 index 000000000..c38c80f22 --- /dev/null +++ b/backend/scripts/devpi-publish.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +set -euo pipefail + +INDEX_URL="${INDEX_URL:-${KALE_DEVPI_INDEX_URL:-http://127.0.0.1:3141/root/dev}}" + +python -m build + +# Ensure we’re pointed at the right index +devpi use "$INDEX_URL" +devpi login root --password '' || true + +# Remove exact version if re-uploading same devN during a quick iteration +PKG="kubeflow-kale" +VERSION="$(python - <<'PY' +from pathlib import Path +import sys + +dist_dir = Path("dist") +if not dist_dir.exists(): + sys.stderr.write("No dist/ directory found. Run `python -m build` first.\\n") + sys.exit(1) + +wheel_candidates = sorted( + dist_dir.glob("kubeflow_kale-*.whl"), + key=lambda p: p.stat().st_mtime, + reverse=True, +) +sdist_candidates = sorted( + dist_dir.glob("kubeflow_kale-*.tar.gz"), + key=lambda p: p.stat().st_mtime, + reverse=True, +) + +artifact = next(iter(wheel_candidates or sdist_candidates), None) +if artifact is None: + sys.stderr.write( + "Could not detect a freshly built kubeflow-kale artifact in dist/.\\n" + ) + sys.exit(1) + +name = artifact.name +prefix = "kubeflow_kale-" +version_part = name[len(prefix):] +if name.endswith(".whl"): + version = version_part.split("-", 1)[0] +elif name.endswith(".tar.gz"): + version = version_part[: -len(".tar.gz")] +else: + sys.stderr.write(f"Unrecognized artifact format: {name}\\n") + sys.exit(1) + +print(version) +PY +)" +echo "Publishing $PKG==$VERSION to $INDEX_URL ..." +devpi remove "${PKG}==${VERSION}" --yes || true + +devpi upload diff --git a/backend/scripts/devpi-up.sh b/backend/scripts/devpi-up.sh new file mode 100755 index 000000000..890563ae3 --- /dev/null +++ b/backend/scripts/devpi-up.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash +set -euo pipefail + +HOST="${HOST:-127.0.0.1}" +PORT="${PORT:-3141}" +INDEX_NAME="${INDEX_NAME:-dev}" +SERVER_DIR="${DEVPI_SERVERDIR:-${HOME}/.devpi/server}" + +if ! devpi-server --help >/dev/null 2>&1; then + echo "devpi-server not found; install with: pip install devpi-server devpi-client" + exit 1 +fi +if ! devpi --help >/dev/null 2>&1; then + echo "devpi-client not found; install with: pip install devpi-client" + exit 1 +fi +if ! devpi-init --help >/dev/null 2>&1; then + echo "devpi-init not found; install with: pip install devpi-server" + exit 1 +fi + +mkdir -p "${SERVER_DIR}" + +# Initialize the server directory on first use. +if [ ! -f "${SERVER_DIR}/.serverversion" ]; then + echo "Initializing devpi-server data directory at ${SERVER_DIR} ..." + devpi-init --serverdir "${SERVER_DIR}" +fi + +echo "Starting devpi-server on http://${HOST}:${PORT} (serverdir=${SERVER_DIR}) ..." +if pgrep -f "devpi-server --serverdir ${SERVER_DIR}" >/dev/null 2>&1; then + echo "devpi-server already running." +else + devpi-server \ + --serverdir "${SERVER_DIR}" \ + --host "${HOST}" \ + --port "${PORT}" & + DEVPI_PID=$! + echo "devpi-server started with PID ${DEVPI_PID}" +fi + +sleep 2 + +# Wait until the server is reachable. +echo -n "Waiting for devpi-server to become ready" +ready=false +for attempt in {1..30}; do + if devpi use "http://${HOST}:${PORT}" >/dev/null 2>&1; then + ready=true + break + fi + echo -n "." + sleep 1 +done + +if [ "${ready}" != "true" ]; then + echo + echo "Failed to contact devpi-server at http://${HOST}:${PORT}." + exit 1 +fi +echo " ... ready." + +devpi use "http://${HOST}:${PORT}" + +# Set no password for root on local dev +devpi user -c root password='' || true +devpi login root --password '' || true + +# Create or reuse an index based on root/pypi as cache mirror +if devpi index -l | grep -q "root/${INDEX_NAME}"; then + echo "Index root/${INDEX_NAME} exists, ensuring volatile=True ..." + devpi index "root/${INDEX_NAME}" volatile=True || true +else + devpi index -c "${INDEX_NAME}" bases=/root/pypi volatile=True +fi + +devpi use "root/${INDEX_NAME}" + +echo +echo "Export these for pip in dev mode:" +echo " export KALE_DEV_MODE=1" +echo " export KALE_DEVPI_SIMPLE_URL=http://${HOST}:${PORT}/root/${INDEX_NAME}/+simple/" +echo " export PIP_INDEX_URL=\$KALE_DEVPI_SIMPLE_URL" +echo " export PIP_EXTRA_INDEX_URL=https://pypi.org/simple" diff --git a/backend/setup.py b/backend/setup.py index 16280813e..12dcd9eef 100644 --- a/backend/setup.py +++ b/backend/setup.py @@ -14,60 +14,5 @@ from setuptools import setup - -setup( - name='kubeflow-kale', - version='1.0.0.dev13', - description='Convert JupyterNotebooks to Kubeflow Pipelines deployments', - url='https://github.com/kubeflow-kale/kale', - author='Stefano Fioravanzo', - author_email='stefano.fioravanzo@gmail.com', - license='Apache License Version 2.0', - packages=['kale', - 'kale.common', - 'kale.config', - 'kale.marshal', - 'kale.processors', - 'kale.rpc', - 'kale.kfserving', - 'kale.sdk', - ], - install_requires=[ - 'kfp>=2.0.0', - 'autopep8 >=2.0.0', - 'astor >= 0.8.1', - 'networkx >=3.0.0', - 'jinja2 >=3.0.0', - 'pyflakes >=3.0.0', - 'dill >=0.3.8', - 'IPython >= 8.30.0', - 'jupyter-client >=8.6.3', - 'jupyter-core >= 5.8.1', - 'nbconvert >=7.16.0', - 'nbformat >=5.10.4', - 'ipykernel >= 6.29.5', - 'notebook >= 7.4.4', - 'packaging >= 25.0', - # 'ml_metadata >= 1.14.0', - 'progress >= 1.5', - # 'kfserving >= 0.4.0, < 0.5.0', - 'kubernetes >= 30.0.0', - ], - extras_require={ - 'dev': [ - 'pytest', - 'pytest-clarity', - 'testfixtures', - 'pytest-cov', - 'flake8', - 'flake8-docstrings' - ] - }, - entry_points={'console_scripts': - ['kale=kale.cli:main', - 'kale_server=kale.cli:server', - 'kale-volumes=kale.cli:kale_volumes']}, - python_requires='>=3.10.0', - include_package_data=True, - zip_safe=False -) +if __name__ == "__main__": + setup() diff --git a/docs/release-testing.md b/docs/release-testing.md new file mode 100644 index 000000000..6cdad645f --- /dev/null +++ b/docs/release-testing.md @@ -0,0 +1,48 @@ +# Exercising the release workflow in test mode + +The `Publish release artifacts` workflow supports two entry points: + +- **Tag-driven releases**: publish to PyPI and npm automatically when a `v*` release + is published on GitHub. +- **Manual runs**: run the same workflow on demand from the Actions tab in either + real release mode or a non-destructive test mode. + +## Running a manual rehearsal + +1. Open **Actions → Publish release artifacts → Run workflow**. +2. Enter the git ref to validate in the **tag** field. This can be any + existing tag or branch – the workflow checks out that ref before building. +3. Choose **test** for the **mode** input so the jobs target staging services. + +In test mode the workflow will: + +- Build the backend wheels and upload them to TestPyPI using the + `TEST_PYPI_API_TOKEN` secret. No artefacts touch the production index. +- Build the labextension, run `npm publish --dry-run`, and produce a tarball via + `npm pack`. This exercises the packaging logic end-to-end without creating a + release on npm. You can optionally provide a `TEST_NPM_TOKEN` secret if you want + to validate authentication as well, but it is not required for the dry run. + +After the rehearsal, re-run the workflow with **mode** set +to `release` (or create/publish a GitHub release tagged `v*`) to publish to the +production registries using the existing `PYPI_API_TOKEN` and `NPM_TOKEN` +secrets. + +## Required secrets + +| Secret name | Used in mode | Purpose | +| ---------------------- | ------------ | ------------------------------------------------- | +| `PYPI_API_TOKEN` | release | Publish backend artefacts to PyPI. | +| `NPM_TOKEN` | release | Publish the labextension to npm. | +| `TEST_PYPI_API_TOKEN` | test | Push backend artefacts to https://test.pypi.org. | +| `TEST_NPM_TOKEN` (opt) | test | Auth token for registry checks during dry runs. | + +## Troubleshooting + +- **Missing TestPyPI token**: the TestPyPI upload step fails fast if the token is + absent. Provide `TEST_PYPI_API_TOKEN` in the repository secrets to enable test + runs. +- **Dry-run validation**: `npm publish --dry-run` performs the same validations as + a real publish (including checking for missing files and metadata) but stops + short of creating the release. Inspect the generated tarball in the workflow + artefacts to review the package contents.