diff --git a/README.md b/README.md index 92c901b1..13fd1e20 100644 --- a/README.md +++ b/README.md @@ -114,13 +114,8 @@ limitations imposed by the Kale data marshalling model. ## Resources - Kale introduction [blog post](https://medium.com/kubeflow/automating-jupyter-notebook-deployments-to-kubeflow-pipelines-with-kale-a4ede38bea1f) -- Codelabs showcasing Kale working in MiniKF with Arrikto's [Rok](https://www.arrikto.com/): - - [From Notebook to Kubeflow Pipelines](https://codelabs.developers.google.com/codelabs/cloud-kubeflow-minikf-kale/#0) - - [From Notebook to Kubeflow Pipelines with HP Tuning](https://arrik.to/demowfhp) - KubeCon NA Tutorial 2019: [From Notebook to Kubeflow Pipelines: An End-to-End Data Science Workflow](https://kccncna19.sched.com/event/Uaeq/tutorial-from-notebook-to-kubeflow-pipelines-an-end-to-end-data-science-workflow-michelle-casbon-google-stefano-fioravanzo-fondazione-bruno-kessler-ilias-katsakioris-arrikto?iframe=no&w=100%&sidebar=yes&bg=no) / [video](http://youtube.com/watch?v=C9rJzTzVzvQ) -- CNCF Webinar 2020: [From Notebook to Kubeflow Pipelines with MiniKF & Kale](https://www.cncf.io/webinars/from-notebook-to-kubeflow-pipelines-with-minikf-kale/) - / [video](https://www.youtube.com/watch?v=1fX9ZFWkvvs) - KubeCon EU Tutorial 2020: [From Notebook to Kubeflow Pipelines with HP Tuning: A Data Science Journey](https://kccnceu20.sched.com/event/ZerG/tutorial-from-notebook-to-kubeflow-pipelines-with-hp-tuning-a-data-science-journey-stefano-fioravanzo-ilias-katsakioris-arrikto) / [video](https://www.youtube.com/watch?v=QK0NxhyADpM) diff --git a/backend/kale/compiler.py b/backend/kale/compiler.py index a965feb1..0cf4903d 100644 --- a/backend/kale/compiler.py +++ b/backend/kale/compiler.py @@ -28,8 +28,8 @@ log = logging.getLogger(__name__) PY_FN_TEMPLATE = "py_function_template.jinja2" -NB_FN_TEMPLATE = "new_nb_function_template.jinja2" -PIPELINE_TEMPLATE = "new_pipeline_template.jinja2" +NB_FN_TEMPLATE = "nb_function_template.jinja2" +PIPELINE_TEMPLATE = "pipeline_template.jinja2" PIPELINE_ORIGIN = {"nb": NB_FN_TEMPLATE, "py": PY_FN_TEMPLATE} KFP_DSL_ARTIFACT_IMPORTS = [ diff --git a/backend/kale/templates/new_nb_function_template.jinja2 b/backend/kale/templates/nb_function_template.jinja2 similarity index 100% rename from backend/kale/templates/new_nb_function_template.jinja2 rename to backend/kale/templates/nb_function_template.jinja2 diff --git a/backend/kale/templates/new_pipeline_template.jinja2 b/backend/kale/templates/new_pipeline_template.jinja2 deleted file mode 100644 index ecdf758d..00000000 --- a/backend/kale/templates/new_pipeline_template.jinja2 +++ /dev/null @@ -1,62 +0,0 @@ -import json -import kfp.dsl as kfp_dsl -from kfp.dsl import Input, Output, Dataset, HTML, Metrics, ClassificationMetrics, Artifact, Model - -{{ lightweight_components | join('\n\n') }} - -@kfp_dsl.pipeline( - name='{{ pipeline_name }}', - description='{{ pipeline_description }}' -) -def auto_generated_pipeline( -{%- for param_name, param_info in pipeline_param_info.items() %} - {{ param_name.lower() }}: {{ param_info.type }} = {% if param_info.type == 'bool' %}{{ param_info.default }}{% else %}{{ param_info.default | tojson }}{% endif %}{% if not loop.last %},{% endif %} -{%- endfor %} -): - """Auto-generated pipeline function.""" - -{% set steps_list = pipeline.steps | list %} -{% for step in steps_list %} - {{ step.name }}_task = {{ step.name }}_step( - {%- if step_inputs.get(step.name) %} - {%- for input_var in step_inputs[step.name] %} - {{ input_var }}_input_artifact={{ step_inputs_sources[step.name][input_var] }}_task.outputs["{{ input_var }}_output_artifact"]{% if not loop.last or pipeline_param_info %},{% endif %} - {%- endfor %} - {%- endif %} - {%- if pipeline_param_info %} - {%- for param_name, param_info in pipeline_param_info.items() %} - {{ param_info.clean_name }}={{ param_name.lower() }}{% if not loop.last %},{% endif %} - {%- endfor %} - {%- endif %} - ) - - {% if loop.index0 > 0 %} - {%- if step_inputs.get(step.name) %} - {%- for input_var in step_inputs[step.name] %} - {{ step.name }}_task.after({{ step_inputs_sources[step.name][input_var] }}_task) - {%- endfor %} - {%- else %} - {{ step.name }}_task.after({{ steps_list[loop.index0 - 1].name }}_task) - {%- endif %} - {% endif %} - - {{ step.name }}_task.set_display_name("{{ component_names[step.name] }}-step") - - {%- if step.config.limits %} - {%- for limit_key, limit_value in step.config.limits.items() %} - {%- if limit_key in ['nvidia.com/gpu', 'amd.com/gpu'] %} - {{ step.name }}_task.set_accelerator_type("{{ limit_key }}").set_accelerator_limit({{ limit_value }}) - {%- endif %} - {%- endfor %} - {%- endif %} - -{% endfor %} - -if __name__ == "__main__": - from kfp import compiler - - pipeline_filename = auto_generated_pipeline.__name__ + '.yaml' - compiler.Compiler().compile(auto_generated_pipeline, pipeline_filename) - - print(f"Pipeline compiled to {pipeline_filename}") - print("To run, upload this YAML to your KFP v2 instance or use kfp.Client().create_run_from_pipeline_func.") diff --git a/backend/kale/templates/pipeline_template.jinja2 b/backend/kale/templates/pipeline_template.jinja2 index 00568b9b..ecdf758d 100644 --- a/backend/kale/templates/pipeline_template.jinja2 +++ b/backend/kale/templates/pipeline_template.jinja2 @@ -1,206 +1,62 @@ import json +import kfp.dsl as kfp_dsl +from kfp.dsl import Input, Output, Dataset, HTML, Metrics, ClassificationMetrics, Artifact, Model -import kfp.dsl as _kfp_dsl -import kfp.components as _kfp_components +{{ lightweight_components | join('\n\n') }} -from collections import OrderedDict -from kubernetes import client as k8s_client - -{# PIPELINE LIGHTWEIGHT COMPONENTS #} -{% for func in lightweight_components -%} -{{func}} -{% endfor -%} - -{# DEFINE PIPELINE TASKS FROM FUNCTIONS #} -{%- for name in pipeline.steps_names -%} -{% if base_image != '' %} -_kale_{{ name }}_op = _kfp_components.func_to_container_op({{ name }}, base_image='{{ base_image }}') -{% else %} -_kale_{{ name }}_op = _kfp_components.func_to_container_op({{ name }}) -{% endif %} -{% endfor -%} - -{# DECLARE PIPELINE #} -@_kfp_dsl.pipeline( - name='{{ pipeline_name }}', - description='{{ pipeline_description }}' +@kfp_dsl.pipeline( + name='{{ pipeline_name }}', + description='{{ pipeline_description }}' ) -def auto_generated_pipeline({%- for arg in pipeline.pps_names -%} - {{ arg }}='{{ (pipeline.pps_values)[loop.index-1] }}' - {%- if loop.index < pipeline.pps_values|length -%}, - {%- endif -%} - {%- endfor -%}): - _kale_pvolumes_dict = OrderedDict() - _kale_volume_step_names = [] - _kale_volume_name_parameters = [] - - {% if timeout %} - _kfp_dsl.get_pipeline_conf().set_timeout({{ timeout }}) - {% endif %} - - {% for vol in volumes -%} - {% set name = vol['name'] %} - {% set mountpoint = vol['mount_point'] %} - {% set pvc_size = vol['size']|string|default ('') + vol['size_type']|default ('') %} - {% set annotations = vol['annotations']|default({}) %} - {% set storage_class_name = vol['storage_class_name'] %} - _kale_annotations = {{ annotations }} - - {% if vol['type'] == 'pv' %} - - _kale_pvc{{ loop.index }} = k8s_client.V1PersistentVolumeClaim( - api_version="v1", - kind="PersistentVolumeClaim", - metadata=k8s_client.V1ObjectMeta( - name="{{ name }}-claim-{{ pipeline_name }}" - ), - spec=k8s_client.V1PersistentVolumeClaimSpec( - volume_name="{{ name }}", - access_modes={{ vol['volume_access_mode'] }}, - {%- if storage_class_name %} - storage_class_name="{{ storage_class_name }}", - {%- endif %} - resources=k8s_client.V1ResourceRequirements( - requests={"storage": "{{ pvc_size }}"} - ) - ) - ) - - _kale_vop{{ loop.index }} = _kfp_dsl.VolumeOp( - name="pvc-data{{ loop.index }}", - annotations=_kale_annotations, - k8s_resource=_kale_pvc{{ loop.index }} - ) - _kale_volume = _kale_vop{{ loop.index }}.volume - _kale_volume_step_names.append(_kale_vop{{ loop.index }}.name) - _kale_volume_name_parameters.append(_kale_vop{{ loop.index }}.outputs["name"].full_name) - - {% elif vol['type'] == 'pvc' %} - - _kale_volume = _kfp_dsl.PipelineVolume(pvc='{{ name }}') - - {% elif vol['type'] == 'new_pvc' %} - - _kale_vop{{ loop.index }} = _kfp_dsl.VolumeOp( - name='create-volume-{{ loop.index }}', - resource_name='{{ name }}', - {%- if annotations %} - annotations=_kale_annotations, - {% endif -%} - modes={{ vol['volume_access_mode'] }}, - {%- if storage_class_name %} - storage_class="{{ storage_class_name }}", - {%- endif %} - size='{{ pvc_size }}' +def auto_generated_pipeline( +{%- for param_name, param_info in pipeline_param_info.items() %} + {{ param_name.lower() }}: {{ param_info.type }} = {% if param_info.type == 'bool' %}{{ param_info.default }}{% else %}{{ param_info.default | tojson }}{% endif %}{% if not loop.last %},{% endif %} +{%- endfor %} +): + """Auto-generated pipeline function.""" + +{% set steps_list = pipeline.steps | list %} +{% for step in steps_list %} + {{ step.name }}_task = {{ step.name }}_step( + {%- if step_inputs.get(step.name) %} + {%- for input_var in step_inputs[step.name] %} + {{ input_var }}_input_artifact={{ step_inputs_sources[step.name][input_var] }}_task.outputs["{{ input_var }}_output_artifact"]{% if not loop.last or pipeline_param_info %},{% endif %} + {%- endfor %} + {%- endif %} + {%- if pipeline_param_info %} + {%- for param_name, param_info in pipeline_param_info.items() %} + {{ param_info.clean_name }}={{ param_name.lower() }}{% if not loop.last %},{% endif %} + {%- endfor %} + {%- endif %} ) - _kale_volume = _kale_vop{{ loop.index }}.volume - _kale_volume_step_names.append(_kale_vop{{ loop.index }}.name) - _kale_volume_name_parameters.append(_kale_vop{{ loop.index }}.outputs["name"].full_name) - - {% endif %} - - _kale_pvolumes_dict['{{ mountpoint }}'] = _kale_volume - {% endfor %} - - {% if marshal_volume %} - _kale_marshal_vop = _kfp_dsl.VolumeOp( - name="kale-marshal-volume", - resource_name="kale-marshal-pvc", - modes={{ pipeline.config.volume_access_mode }}, - {%- if pipeline.config.storage_class_name %} - storage_class="{{ pipeline.config.storage_class_name }}", - {%- endif %} - size="1Gi" - ) - _kale_volume_step_names.append(_kale_marshal_vop.name) - _kale_volume_name_parameters.append(_kale_marshal_vop.outputs["name"].full_name) - _kale_pvolumes_dict['{{ marshal_path }}'] = _kale_marshal_vop.volume + {% if loop.index0 > 0 %} + {%- if step_inputs.get(step.name) %} + {%- for input_var in step_inputs[step.name] %} + {{ step.name }}_task.after({{ step_inputs_sources[step.name][input_var] }}_task) + {%- endfor %} + {%- else %} + {{ step.name }}_task.after({{ steps_list[loop.index0 - 1].name }}_task) + {%- endif %} {% endif %} - _kale_volume_step_names.sort() - _kale_volume_name_parameters.sort() + {{ step.name }}_task.set_display_name("{{ component_names[step.name] }}-step") - {% for step in pipeline.steps %} - _kale_{{ step.name }}_task = _kale_{{ step.name }}_op({{ pipeline.all_steps_parameters[step.name]|join(', ') }})\ - .add_pvolumes(_kale_pvolumes_dict)\ - .after({{ pipeline.pipeline_dependencies_tasks[ step.name ]|map('add_prefix', '_kale_')|map('add_suffix', '_task')|join(', ') }}) - {%- if step.config.annotations %} - _kale_step_annotations = {{ step.config.annotations }} - for _kale_k, _kale_v in _kale_step_annotations.items(): - _kale_{{ step.name }}_task.add_pod_annotation(_kale_k, _kale_v) - {%- endif %} - {%- if step.config.labels %} - _kale_step_labels = {{ step.config.labels }} - for _kale_k, _kale_v in _kale_step_labels.items(): - _kale_{{ step.name }}_task.add_pod_label(_kale_k, _kale_v) - {%- endif %} {%- if step.config.limits %} - _kale_step_limits = {{ step.config.limits }} - for _kale_k, _kale_v in _kale_step_limits.items(): - _kale_{{ step.name }}_task.container.add_resource_limit(_kale_k, _kale_v) - {%- endif %} - {%- if step.config.retry_count %} - _kale_{{ step.name }}_task.set_retry_strategy( - num_retries={{ step.config.retry_count }}, - retry_policy="Always", - backoff_duration={{ step.config.retry_interval|quote_if_not_none }}, - backoff_factor={{ step.config.retry_factor }}, - backoff_max_duration={{ step.config.retry_max_interval|quote_if_not_none }}) - {%- endif %} - _kale_{{ step.name }}_task.container.working_dir = "{{ abs_working_dir }}" - _kale_{{ step.name }}_task.container.set_security_context(k8s_client.V1SecurityContext(run_as_user=0)) - _kale_output_artifacts = {} - {%- if step.metrics %} - _kale_output_artifacts.update({'mlpipeline-metrics': '/tmp/mlpipeline-metrics.json'}) + {%- for limit_key, limit_value in step.config.limits.items() %} + {%- if limit_key in ['nvidia.com/gpu', 'amd.com/gpu'] %} + {{ step.name }}_task.set_accelerator_type("{{ limit_key }}").set_accelerator_limit({{ limit_value }}) {%- endif %} - {%- if pipeline.processor.id == "nb" and step.name != "pipeline_metrics" %} - _kale_output_artifacts.update({'mlpipeline-ui-metadata': '/tmp/mlpipeline-ui-metadata.json'}) - _kale_output_artifacts.update({'{{ step.name }}': '/{{ step.name }}.html'}) - {%- endif %} - {%- if pipeline.processor.id == "py" and step.artifacts and step.name != "pipeline_metrics" %} - _kale_output_artifacts.update({'mlpipeline-ui-metadata': '/tmp/mlpipeline-ui-metadata.json'}) - {%- for artifact in step.artifacts %} - _kale_output_artifacts.update({'{{ artifact["name"] }}': '{{ artifact["path"] }}'}) {%- endfor %} {%- endif %} - _kale_{{ step.name }}_task.output_artifact_paths.update(_kale_output_artifacts) - _kale_{{ step.name }}_task.add_pod_label("pipelines.kubeflow.org/metadata_written", "true") - _kale_dep_names = (_kale_{{ step.name }}_task.dependent_names + - _kale_volume_step_names) - _kale_{{ step.name }}_task.add_pod_annotation( - "kubeflow-kale.org/dependent-templates", json.dumps(_kale_dep_names)) - if _kale_volume_name_parameters: - _kale_{{ step.name }}_task.add_pod_annotation( - "kubeflow-kale.org/volume-name-parameters", - json.dumps(_kale_volume_name_parameters)) - {% endfor %} - - {# Snaphosts #} - {% for vol in volumes -%} - {% if vol['snapshot'] %} - _kale_snapshot{{ loop.index }} = _kfp_dsl.VolumeSnapshotOp( - name='snapshot-volume-{{ loop.index }}', - resource_name='{{ vol['snapshot_name'] }}', - volume=_kale_vop{{ loop.index }}.volume.after({{ pipeline.get_leaf_nodes()|map('add_prefix', '_kale_')|map('add_suffix', '_task')|join(', ') }}) - ) - {% endif %} - {% endfor %} +{% endfor %} -{# The script will deploy the pipeline if run manually #} if __name__ == "__main__": - pipeline_func = auto_generated_pipeline - pipeline_filename = pipeline_func.__name__ + '.pipeline.tar.gz' - import kfp.compiler as compiler - compiler.Compiler().compile(pipeline_func, pipeline_filename) + from kfp import compiler - # Get or create an experiment and submit a pipeline run - import kfp - client = kfp.Client() - experiment = client.create_experiment('{{ experiment_name }}') + pipeline_filename = auto_generated_pipeline.__name__ + '.yaml' + compiler.Compiler().compile(auto_generated_pipeline, pipeline_filename) - # Submit a pipeline run - from kale.common import kfputils - pipeline_id, version_id = kfputils.upload_pipeline(pipeline_filename, "{{ pipeline_name }}") - run_result = kfputils.run_pipeline(experiment_name=experiment.name, pipeline_id=pipeline_id, version_id=version_id) + print(f"Pipeline compiled to {pipeline_filename}") + print("To run, upload this YAML to your KFP v2 instance or use kfp.Client().create_run_from_pipeline_func.") diff --git a/docker/jupyterlab/Dockerfile.rok b/docker/jupyterlab/Dockerfile.rok deleted file mode 100644 index e0ebb169..00000000 --- a/docker/jupyterlab/Dockerfile.rok +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright 2025 The Kubeflow Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Use tensorflow-1.14.0 based image with Rok as a base image -FROM gcr.io/arrikto-public/tensorflow-1.14.0-notebook-cpu:kubecon-workshop - -USER root - -RUN echo 'alias grep="grep --color=auto' >> /etc/bash.bashrc && \ - echo 'alias ls="ls --color=auto' >> /etc/bash.bashrc - -# Install latest KFP SDK & Kale & JupyterLab Extension -RUN pip3 install --upgrade pip && \ - # XXX: Install enum34==1.1.8 because other versions lead to errors during - # KFP installation - pip3 install --upgrade "enum34==1.1.8" && \ - pip3 install --upgrade "jupyterlab>=2.0.0,<3.0.0" && \ - pip3 install --upgrade kubeflow-kale && \ - jupyter labextension install kubeflow-kale-labextension - -RUN echo "jovyan ALL=(ALL:ALL) NOPASSWD:ALL" > /etc/sudoers.d/jovyan -WORKDIR /home/jovyan -USER jovyan - -CMD ["sh", "-c", \ - "jupyter lab --notebook-dir=/home/jovyan --ip=0.0.0.0 --no-browser \ - --allow-root --port=8888 --LabApp.token='' --LabApp.password='' \ - --LabApp.allow_origin='*' --LabApp.base_url=${NB_PREFIX}"] diff --git a/docker/jupyterlab/Dockerfile.rok.dev b/docker/jupyterlab/Dockerfile.rok.dev deleted file mode 100644 index fe72bbd8..00000000 --- a/docker/jupyterlab/Dockerfile.rok.dev +++ /dev/null @@ -1,53 +0,0 @@ -# Copyright 2025 The Kubeflow Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Use tensorflow-1.14.0 based image with Rok as a base image -FROM gcr.io/arrikto-public/tensorflow-1.14.0-notebook-cpu:kubecon-workshop - -USER root - -RUN echo 'alias grep="grep --color=auto' >> /etc/bash.bashrc && \ - echo 'alias ls="ls --color=auto' >> /etc/bash.bashrc - -# Install latest KFP SDK -RUN pip3 install --upgrade pip && \ - # XXX: Install enum34==1.1.8 because other versions lead to errors during - # KFP installation - pip3 install --upgrade "enum34==1.1.8" && \ - pip3 install --upgrade "jupyterlab>=2.0.0,<3.0.0" - -# Install Kale from KALE_BRANCH (defaults to "master") -ARG KALE_BRANCH="master" -WORKDIR / -RUN rm -rf /kale && \ - git clone -b ${KALE_BRANCH} https://github.com/kubeflow-kale/kale - -WORKDIR /kale/backend -RUN pip3 install --upgrade . - -WORKDIR /kale/labextension -RUN jlpm install && \ - jlpm run build && \ - jupyter labextension install . - -RUN jupyter lab build --dev-build=False - -RUN echo "jovyan ALL=(ALL:ALL) NOPASSWD:ALL" > /etc/sudoers.d/jovyan -WORKDIR /home/jovyan -USER jovyan - -CMD ["sh", "-c", \ - "jupyter lab --notebook-dir=/home/jovyan --ip=0.0.0.0 --no-browser \ - --allow-root --port=8888 --LabApp.token='' --LabApp.password='' \ - --LabApp.allow_origin='*' --LabApp.base_url=${NB_PREFIX}"] diff --git a/examples/sdk/compression-pipeline.py b/examples/sdk/compression-pipeline.py deleted file mode 100644 index 98abf855..00000000 --- a/examples/sdk/compression-pipeline.py +++ /dev/null @@ -1,140 +0,0 @@ -# Copyright 2026 The Kubeflow Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -This file showcases how you can run pipelines that call external utilities -or software. Python becomes the orchestrator of your pipeline and the -entry point to any CLI tool installed in your environment. - -In order to run external commands from Python, you can user the `cmdutils` -module in the `rok_common` package. - -Running a command is as simple as: - -``` -> from rok_common.cmdutils import run -> run("ls -la") -``` - -`run` will wait for the command to complete its execution, before retuning the -result. Once the command is done, `run` returns and object of type -`ExtCommand` that can be used to inspect what happened. - -Some useful properties of `ExtCommand`: - -- `out`: the stdout stream produced by the command -- `err`: the stderr stream produced by the command -- `pid`: the PID of the process created by the command -- `returncode`: the return code of the process - -In case the command fails, `run` will raise a `CommandExecutionError` -exception. - -Note: if you want to read the `out` and `err` properties for analysis, after -the command has finished running, do the following: - -``` -> from rok_common.cmdutils import run, STORE -> ext = run("ls -la", stdout=STORE, stderr=STORE) -> print(ext.out) -> print(ext.err) -``` - -If you want to read more about the various arguments of `run` and how you can -customize the execution: - -``` -> from rok_common.cmdutils import run, ExtCommand -> help(run) -> help(ExtCommand) # Read docstrings form the class definition and `__init__` -``` - -or, if you are inside a Jupyter Notebook, use the Jupyter helper `?` to print -the docstrings nicely: - -``` -[cell1]> from rok_common.cmdutils import run, ExtCommand -[cell2]> run? -[cell3]> ExtCommand? -``` - -""" - -from kale.sdk import pipeline, step - - -@step(name="download") -def download(src_bucket): - import os - from rok_common.cmdutils import run, STORE - - path = os.getenv("GOOGLE_APPLICATION_CREDENTIALS") - cmd = ("gcloud auth activate-service-account " - "--key-file %s" % path) - run(cmd) - - # download data - cmd = "gsutil cp -R %s ./images" % src_bucket - # will raise an exception with error message in case - # of failure. - cmd = run(cmd, stdout=STORE, stderr=STORE) - return "./images" - - -@step(name="compression") -def compress(data_path): - from rok_common.cmdutils import run, STORE - cmd = "tar -czvf images.tar.gz %s" % data_path - try: - run(cmd) - except Exception as e: - print("Compression command failed") - print(e) - return "Error" - return "images.tar.gz" - - -@step(name="upload") -def upload(compress_res, destination_bucket): - import os - from rok_common.cmdutils import run, STORE - - if compress_res == "Error": - # Perform some clean up - run("rm -rf images") - return - - path = os.getenv("GOOGLE_APPLICATION_CREDENTIALS") - cmd = ("gcloud auth activate-service-account " - "--key-file %s" % path) - run(cmd) - - cmd = "gsutil cp -R %s %s" % (compress_res, destination_bucket) - cmd = run(cmd, stdout=STORE, stderr=STORE) - - -@pipeline(name="compression-pipeline", - experiment="compression-pipeline", - autosnapshot=False) -def compression_pipeline(source_bucket="gs://default/", - destination_bucket="gs://default/"): - path = download(source_bucket) - compress_res = compress(path) - upload(compress_res, destination_bucket) - - -if __name__ == "__main__": - compression_pipeline( - source_bucket="gs://shell-demo/images", - destination_bucket="gs://shell-demo") diff --git a/examples/sdk/skeleton.py b/examples/sdk/skeleton.py index 631dfedc..3e282289 100644 --- a/examples/sdk/skeleton.py +++ b/examples/sdk/skeleton.py @@ -32,17 +32,9 @@ Just make sure to `import` all your modules *inside* the function definition, as the code that will run in the Kubeflow pipeline won't have the entire context of the current script. - -If you are using Rok to take snapshots (see below on how this works with the -Kale SDK) and reproduce the current environment in the pipeline step, you -can use relative imports to reference local scripts. """ @step(name="my_step") def foo(a): - # Using a relative import to another local script will work as long as - # you are using rok to snapshot the current environment and mount a clone - # of the volume in the pipeline step: - # from .script import bar import sys sys.stdout.write(a) # return multiple values. These could be used by different subsequent @@ -148,9 +140,8 @@ def my_pipeline(parameter="input"): - Kale validates the current code, to make sure that it can be converted to a pipeline -- Rok takes a snapshot of your mounted volumes -- Kale creates a new KFP pipelines, using the same docker image as your current - environment as base image for the step and seeding clones of your volumes. +- Kale creates a new KFP pipeline, using the specified docker image as the + base image for the pipeline steps - Kale creates (if necessary) a new KFP experiment, based on the provided name - Kale uploads a new pipeline definition - Kale starts a new pipeline run diff --git a/examples/serving/sklearn/iris.ipynb b/examples/serving/sklearn/iris.ipynb index 279451ef..412c8328 100644 --- a/examples/serving/sklearn/iris.ipynb +++ b/examples/serving/sklearn/iris.ipynb @@ -427,8 +427,7 @@ "pipeline_name": "iris-pipeline", "snapshot_volumes": true, "steps_defaults": [ - "label:access-ml-pipeline:true", - "label:access-rok:true" + "label:access-ml-pipeline:true" ], "volumes": [ { @@ -469,4 +468,4 @@ }, "nbformat": 4, "nbformat_minor": 4 -} +} \ No newline at end of file diff --git a/examples/serving/xgboost/iris.ipynb b/examples/serving/xgboost/iris.ipynb index 2a5e38c8..76ca8dfa 100644 --- a/examples/serving/xgboost/iris.ipynb +++ b/examples/serving/xgboost/iris.ipynb @@ -428,8 +428,7 @@ "pipeline_name": "iris-pipeline", "snapshot_volumes": true, "steps_defaults": [ - "label:access-ml-pipeline:true", - "label:access-rok:true" + "label:access-ml-pipeline:true" ], "volumes": [ { @@ -470,4 +469,4 @@ }, "nbformat": 4, "nbformat_minor": 4 -} +} \ No newline at end of file diff --git a/labextension/src/components/AnnotationInput.tsx b/labextension/src/components/AnnotationInput.tsx index 5a91e727..4efe1280 100644 --- a/labextension/src/components/AnnotationInput.tsx +++ b/labextension/src/components/AnnotationInput.tsx @@ -27,7 +27,6 @@ interface IAnnotationInputProps { label: string; volumeIdx: number; annotationIdx: number; - rokAvailable?: boolean; cannotBeDeleted?: boolean; annotation: { key: string; value: string }; deleteValue: (idx: number, annotationIdx: number) => void; diff --git a/labextension/src/icons/BrowseInRokBlue.tsx b/labextension/src/icons/BrowseInRokBlue.tsx deleted file mode 100644 index 2e320374..00000000 --- a/labextension/src/icons/BrowseInRokBlue.tsx +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright 2026 The Kubeflow Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -import * as React from 'react'; - -export default class BrowseInRokBLue extends React.Component<{ - style?: React.CSSProperties; -}> { - public render(): JSX.Element { - return ( - - - - - - - - - - - - - - - - - - ); - } -} diff --git a/labextension/src/widget.tsx b/labextension/src/widget.tsx index 91028b52..414e6608 100644 --- a/labextension/src/widget.tsx +++ b/labextension/src/widget.tsx @@ -93,7 +93,6 @@ async function activate( // TODO: backend can become an Enum that indicates the type of // env we are in (like Local Laptop, MiniKF, GCP, UI without Kale, ...) const backend = await getBackend(kernel); - // let rokError: IRPCError = null; if (backend) { try { await executeRpc(kernel, 'log.setup_logging');