From 65ee31439788fbb161f2f88055eb0825a0e52031 Mon Sep 17 00:00:00 2001 From: Henri Casanova Date: Mon, 4 Aug 2025 15:38:05 -1000 Subject: [PATCH 01/38] Added Dask translator coverage --- .github/workflows/build.yml | 4 + tests/translators/{ => dask}/Dockerfile_Dask | 1 + .../dask/build_docker_dask_docker_image.sh | 5 + .../translators/dask/test_dask_translator.py | 111 ++++++++++++++++++ 4 files changed, 121 insertions(+) rename tests/translators/{ => dask}/Dockerfile_Dask (95%) create mode 100755 tests/translators/dask/build_docker_dask_docker_image.sh create mode 100644 tests/translators/dask/test_dask_translator.py diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3bdf29a1..cb63840e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -22,6 +22,10 @@ jobs: pip install sphinx_rtd_theme recommonmark pytest pytest-cov pip install --upgrade setuptools + - name: Install utilities for testing + run: | + pip install docker + - name: Check package install run: | pip install -e . diff --git a/tests/translators/Dockerfile_Dask b/tests/translators/dask/Dockerfile_Dask similarity index 95% rename from tests/translators/Dockerfile_Dask rename to tests/translators/dask/Dockerfile_Dask index 66e35640..48df02a5 100644 --- a/tests/translators/Dockerfile_Dask +++ b/tests/translators/dask/Dockerfile_Dask @@ -29,6 +29,7 @@ RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1 RUN python3 -m pip install --break-system-packages pathos pandas filelock RUN python3 -m pip install --break-system-packages networkx scipy matplotlib RUN python3 -m pip install --break-system-packages pyyaml jsonschema requests +RUN python3 -m pip install --break-system-packages --upgrade setuptools # Stress-ng RUN apt-get -y install stress-ng diff --git a/tests/translators/dask/build_docker_dask_docker_image.sh b/tests/translators/dask/build_docker_dask_docker_image.sh new file mode 100755 index 00000000..cd2b96e5 --- /dev/null +++ b/tests/translators/dask/build_docker_dask_docker_image.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +set -e + +docker build --platform linux/amd64 -t wfcommons/wfcommons-testing-dask -f Dockerfile_Dask . diff --git a/tests/translators/dask/test_dask_translator.py b/tests/translators/dask/test_dask_translator.py new file mode 100644 index 00000000..8741823c --- /dev/null +++ b/tests/translators/dask/test_dask_translator.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) 2025 The WfCommons Team. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +import pathlib +import pytest +import shutil +import docker +import io +import tarfile +import os +import sys +import json + +from wfcommons import BlastRecipe +from wfcommons.wfbench import WorkflowBenchmark, DaskTranslator + +class TestDaskTranslator: + + def make_tarfile_of_wfcommons(self): + source_dir = os.getcwd() # This assumes the testing is run from the root + tar_stream = io.BytesIO() + with tarfile.open(fileobj=tar_stream, mode='w') as tar: + tar.add(source_dir, arcname=os.path.basename(source_dir)) + tar_stream.seek(0) + return tar_stream + + + @pytest.mark.unit + def test_dask_translator(self) -> None: + + # Create a workflow benchmark object to generate specifications based on a recipe (in /tmp/, whatever) + desired_num_tasks = 45 + benchmark_full_path = "/tmp/blast-benchmark-{desired_num_tasks}.json" + shutil.rmtree(benchmark_full_path, ignore_errors=True) + benchmark = WorkflowBenchmark(recipe=BlastRecipe, num_tasks=desired_num_tasks) + benchmark.create_benchmark(pathlib.Path("/tmp/"), cpu_work=100, data=10, percent_cpu=0.6) + with open(f"/tmp/blast-benchmark-{desired_num_tasks}.json", "r") as f: + generated_json = json.load(f) + num_tasks = len(generated_json["workflow"]["specification"]["tasks"]) + + # Make sure the target dir does not already exist + str_dirpath = "/tmp/dask_translated_workflow/" + dirpath = pathlib.Path(str_dirpath) + if dirpath.exists(): + shutil.rmtree(dirpath) + + # Perform the translation + sys.stderr.write("Translating workflow...\n") + translator = DaskTranslator(benchmark.workflow) + translator.translate(output_folder=dirpath) + + # Pulling the Docker image + client = docker.from_env() + sys.stderr.write("Pulling Docker image...\n") + image_name = "wfcommons/wfcommons-testing-dask" + client.images.pull(image_name) + + # Launch the docker container to actually run the translated workflow + sys.stderr.write("Starting Docker container...\n") + container = client.containers.run( + image_name, + "sleep infinity", + volumes={str_dirpath: {'bind': str_dirpath, 'mode': 'rw'}}, + working_dir=str_dirpath, + tty=True, + detach=True + ) + + sys.stderr.write("Installing WfCommons on the container...\n") + # Copy the WfCommons code to it (removing stuff that should be removed) + target_path = '/tmp/' # inside container + tar_data = self.make_tarfile_of_wfcommons() + container.put_archive(target_path, tar_data) + exit_code, output = container.exec_run("sudo /bin/rm -rf /tmp/WfCommons/build/", stdout=True, stderr=True) + exit_code, output = container.exec_run("sudo /bin/rm -rf /tmp/WfCommons/*.egg-info/", stdout=True, stderr=True) + exit_code, output = container.exec_run("sudo /bin/rm -rf /tmp/WfCommons/bin/cpu-benchmark.o", stdout=True, stderr=True) + exit_code, output = container.exec_run("sudo /bin/rm -rf /tmp/WfCommons/bin/cpu-benchmark", stdout=True, stderr=True) + + # Install WfCommons on the container (to install wfbench and cpu-benchmark really) + exit_code, output = container.exec_run("sudo python3 -m pip install -e . --break-system-packages", workdir="/tmp/WfCommons", stdout=True, stderr=True) + + # Copy over the wfbench and cpu-benchmark executables + exit_code, output = container.exec_run("sudo cp -f /tmp/WfCommons/bin/wfbench " + str_dirpath + "bin/", stdout=True, stderr=True) + exit_code, output = container.exec_run("sudo cp -f /tmp/WfCommons/bin/cpu-benchmark " + str_dirpath + "bin/", stdout=True, stderr=True) + + # Run the dask workflow! + sys.stderr.write("Running the Dask workflow on the container...\n") + exit_code, output = container.exec_run("python ./dask_workflow.py", stdout=True, stderr=True) + num_completed_tasks = output.decode().count("completed!") # TODO: This is pretty lame + + # Kill the container + container.remove(force=True) + + # Do sanity checks + sys.stderr.write("Checking sanity...\n") + assert(exit_code == 0) + assert(num_completed_tasks == num_tasks) + # TODO: Look at the (I think) generated run.json file on the container + + + + + + From fe22c084a65672e19356eeb91489adb719528994 Mon Sep 17 00:00:00 2001 From: Henri Casanova Date: Mon, 4 Aug 2025 16:26:12 -1000 Subject: [PATCH 02/38] Re-org of translator tests --- .../{dask/Dockerfile_Dask => Dockerfile.dask} | 0 ...ockerfile_Nextflow => Dockerfile.nextflow} | 1 + .../{Dockerfile_Parsl => Dockerfile.parsl} | 1 + .../translators/build_docker_docker_images.sh | 8 ++ .../dask/build_docker_dask_docker_image.sh | 5 - .../parsl/test_parsl_translator.py | 111 ++++++++++++++++++ 6 files changed, 121 insertions(+), 5 deletions(-) rename tests/translators/{dask/Dockerfile_Dask => Dockerfile.dask} (100%) rename tests/translators/{Dockerfile_Nextflow => Dockerfile.nextflow} (95%) rename tests/translators/{Dockerfile_Parsl => Dockerfile.parsl} (95%) create mode 100755 tests/translators/build_docker_docker_images.sh delete mode 100755 tests/translators/dask/build_docker_dask_docker_image.sh create mode 100644 tests/translators/parsl/test_parsl_translator.py diff --git a/tests/translators/dask/Dockerfile_Dask b/tests/translators/Dockerfile.dask similarity index 100% rename from tests/translators/dask/Dockerfile_Dask rename to tests/translators/Dockerfile.dask diff --git a/tests/translators/Dockerfile_Nextflow b/tests/translators/Dockerfile.nextflow similarity index 95% rename from tests/translators/Dockerfile_Nextflow rename to tests/translators/Dockerfile.nextflow index 5f24725f..14259766 100644 --- a/tests/translators/Dockerfile_Nextflow +++ b/tests/translators/Dockerfile.nextflow @@ -30,6 +30,7 @@ RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1 RUN pip install --break-system-packages pathos pandas filelock RUN pip install --break-system-packages networkx scipy matplotlib RUN pip install --break-system-packages pyyaml jsonschema requests +RUN python3 -m pip install --break-system-packages --upgrade setuptools # Stress-ng RUN apt-get -y install stress-ng diff --git a/tests/translators/Dockerfile_Parsl b/tests/translators/Dockerfile.parsl similarity index 95% rename from tests/translators/Dockerfile_Parsl rename to tests/translators/Dockerfile.parsl index 56af99b7..dd92d5a7 100644 --- a/tests/translators/Dockerfile_Parsl +++ b/tests/translators/Dockerfile.parsl @@ -29,6 +29,7 @@ RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1 RUN python3 -m pip install --break-system-packages pathos pandas filelock RUN python3 -m pip install --break-system-packages networkx scipy matplotlib RUN python3 -m pip install --break-system-packages pyyaml jsonschema requests +RUN python3 -m pip install --break-system-packages --upgrade setuptools # Stress-ng RUN apt-get -y install stress-ng diff --git a/tests/translators/build_docker_docker_images.sh b/tests/translators/build_docker_docker_images.sh new file mode 100755 index 00000000..1ef71088 --- /dev/null +++ b/tests/translators/build_docker_docker_images.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +set -e + +for backend in "dask" "parsl" "nextflow"; do + echo "Building $backend Docker image..." + docker build --platform linux/amd64 -t wfcommons/wfcommons-testing-$backend -f Dockerfile.$backend . +done diff --git a/tests/translators/dask/build_docker_dask_docker_image.sh b/tests/translators/dask/build_docker_dask_docker_image.sh deleted file mode 100755 index cd2b96e5..00000000 --- a/tests/translators/dask/build_docker_dask_docker_image.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -set -e - -docker build --platform linux/amd64 -t wfcommons/wfcommons-testing-dask -f Dockerfile_Dask . diff --git a/tests/translators/parsl/test_parsl_translator.py b/tests/translators/parsl/test_parsl_translator.py new file mode 100644 index 00000000..abd96ec5 --- /dev/null +++ b/tests/translators/parsl/test_parsl_translator.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) 2025 The WfCommons Team. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +import pathlib +import pytest +import shutil +import docker +import io +import tarfile +import os +import sys +import json + +from wfcommons import BlastRecipe +from wfcommons.wfbench import WorkflowBenchmark, DaskTranslator + +class TestParslTranslator: + + def make_tarfile_of_wfcommons(self): + source_dir = os.getcwd() # This assumes the testing is run from the root + tar_stream = io.BytesIO() + with tarfile.open(fileobj=tar_stream, mode='w') as tar: + tar.add(source_dir, arcname=os.path.basename(source_dir)) + tar_stream.seek(0) + return tar_stream + + + @pytest.mark.unit + def test_parsl_translator(self) -> None: + + # Create a workflow benchmark object to generate specifications based on a recipe (in /tmp/, whatever) + desired_num_tasks = 45 + benchmark_full_path = "/tmp/blast-benchmark-{desired_num_tasks}.json" + shutil.rmtree(benchmark_full_path, ignore_errors=True) + benchmark = WorkflowBenchmark(recipe=BlastRecipe, num_tasks=desired_num_tasks) + benchmark.create_benchmark(pathlib.Path("/tmp/"), cpu_work=100, data=10, percent_cpu=0.6) + with open(f"/tmp/blast-benchmark-{desired_num_tasks}.json", "r") as f: + generated_json = json.load(f) + num_tasks = len(generated_json["workflow"]["specification"]["tasks"]) + + # Make sure the target dir does not already exist + str_dirpath = "/tmp/dask_translated_workflow/" + dirpath = pathlib.Path(str_dirpath) + if dirpath.exists(): + shutil.rmtree(dirpath) + + # Perform the translation + sys.stderr.write("Translating workflow...\n") + translator = DaskTranslator(benchmark.workflow) + translator.translate(output_folder=dirpath) + + # Pulling the Docker image + client = docker.from_env() + sys.stderr.write("Pulling Docker image...\n") + image_name = "wfcommons/wfcommons-testing-dask" + client.images.pull(image_name) + + # Launch the docker container to actually run the translated workflow + sys.stderr.write("Starting Docker container...\n") + container = client.containers.run( + image_name, + "sleep infinity", + volumes={str_dirpath: {'bind': str_dirpath, 'mode': 'rw'}}, + working_dir=str_dirpath, + tty=True, + detach=True + ) + + sys.stderr.write("Installing WfCommons on the container...\n") + # Copy the WfCommons code to it (removing stuff that should be removed) + target_path = '/tmp/' # inside container + tar_data = self.make_tarfile_of_wfcommons() + container.put_archive(target_path, tar_data) + exit_code, output = container.exec_run("sudo /bin/rm -rf /tmp/WfCommons/build/", stdout=True, stderr=True) + exit_code, output = container.exec_run("sudo /bin/rm -rf /tmp/WfCommons/*.egg-info/", stdout=True, stderr=True) + exit_code, output = container.exec_run("sudo /bin/rm -rf /tmp/WfCommons/bin/cpu-benchmark.o", stdout=True, stderr=True) + exit_code, output = container.exec_run("sudo /bin/rm -rf /tmp/WfCommons/bin/cpu-benchmark", stdout=True, stderr=True) + + # Install WfCommons on the container (to install wfbench and cpu-benchmark really) + exit_code, output = container.exec_run("sudo python3 -m pip install -e . --break-system-packages", workdir="/tmp/WfCommons", stdout=True, stderr=True) + + # Copy over the wfbench and cpu-benchmark executables + exit_code, output = container.exec_run("sudo cp -f /tmp/WfCommons/bin/wfbench " + str_dirpath + "bin/", stdout=True, stderr=True) + exit_code, output = container.exec_run("sudo cp -f /tmp/WfCommons/bin/cpu-benchmark " + str_dirpath + "bin/", stdout=True, stderr=True) + + # Run the dask workflow! + sys.stderr.write("Running the Dask workflow on the container...\n") + exit_code, output = container.exec_run("python ./dask_workflow.py", stdout=True, stderr=True) + num_completed_tasks = output.decode().count("completed!") # TODO: This is pretty lame + + # Kill the container + container.remove(force=True) + + # Do sanity checks + sys.stderr.write("Checking sanity...\n") + assert(exit_code == 0) + assert(num_completed_tasks == num_tasks) + # TODO: Look at the (I think) generated run.json file on the container + + + + + + From 059372bfdb57be5d7fdda302e2038b2afa4cf7d0 Mon Sep 17 00:00:00 2001 From: Henri Casanova Date: Mon, 4 Aug 2025 17:14:55 -1000 Subject: [PATCH 03/38] Added a Parsl translator test --- tests/translators/README | 1 - .../translators/dask/test_dask_translator.py | 111 ----------- .../parsl/test_parsl_translator.py | 111 ----------- tests/translators/test_translators.py | 176 ++++++++++++++++++ 4 files changed, 176 insertions(+), 223 deletions(-) delete mode 100644 tests/translators/README delete mode 100644 tests/translators/dask/test_dask_translator.py delete mode 100644 tests/translators/parsl/test_parsl_translator.py create mode 100644 tests/translators/test_translators.py diff --git a/tests/translators/README b/tests/translators/README deleted file mode 100644 index ee8377ad..00000000 --- a/tests/translators/README +++ /dev/null @@ -1 +0,0 @@ -For now these are individual Dockerfiles for each workflow system, which can be used for local testing/development. The idea is to eventually merge them all (if possible), and create a Github action for testing all translators. diff --git a/tests/translators/dask/test_dask_translator.py b/tests/translators/dask/test_dask_translator.py deleted file mode 100644 index 8741823c..00000000 --- a/tests/translators/dask/test_dask_translator.py +++ /dev/null @@ -1,111 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# -# Copyright (c) 2025 The WfCommons Team. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. - -import pathlib -import pytest -import shutil -import docker -import io -import tarfile -import os -import sys -import json - -from wfcommons import BlastRecipe -from wfcommons.wfbench import WorkflowBenchmark, DaskTranslator - -class TestDaskTranslator: - - def make_tarfile_of_wfcommons(self): - source_dir = os.getcwd() # This assumes the testing is run from the root - tar_stream = io.BytesIO() - with tarfile.open(fileobj=tar_stream, mode='w') as tar: - tar.add(source_dir, arcname=os.path.basename(source_dir)) - tar_stream.seek(0) - return tar_stream - - - @pytest.mark.unit - def test_dask_translator(self) -> None: - - # Create a workflow benchmark object to generate specifications based on a recipe (in /tmp/, whatever) - desired_num_tasks = 45 - benchmark_full_path = "/tmp/blast-benchmark-{desired_num_tasks}.json" - shutil.rmtree(benchmark_full_path, ignore_errors=True) - benchmark = WorkflowBenchmark(recipe=BlastRecipe, num_tasks=desired_num_tasks) - benchmark.create_benchmark(pathlib.Path("/tmp/"), cpu_work=100, data=10, percent_cpu=0.6) - with open(f"/tmp/blast-benchmark-{desired_num_tasks}.json", "r") as f: - generated_json = json.load(f) - num_tasks = len(generated_json["workflow"]["specification"]["tasks"]) - - # Make sure the target dir does not already exist - str_dirpath = "/tmp/dask_translated_workflow/" - dirpath = pathlib.Path(str_dirpath) - if dirpath.exists(): - shutil.rmtree(dirpath) - - # Perform the translation - sys.stderr.write("Translating workflow...\n") - translator = DaskTranslator(benchmark.workflow) - translator.translate(output_folder=dirpath) - - # Pulling the Docker image - client = docker.from_env() - sys.stderr.write("Pulling Docker image...\n") - image_name = "wfcommons/wfcommons-testing-dask" - client.images.pull(image_name) - - # Launch the docker container to actually run the translated workflow - sys.stderr.write("Starting Docker container...\n") - container = client.containers.run( - image_name, - "sleep infinity", - volumes={str_dirpath: {'bind': str_dirpath, 'mode': 'rw'}}, - working_dir=str_dirpath, - tty=True, - detach=True - ) - - sys.stderr.write("Installing WfCommons on the container...\n") - # Copy the WfCommons code to it (removing stuff that should be removed) - target_path = '/tmp/' # inside container - tar_data = self.make_tarfile_of_wfcommons() - container.put_archive(target_path, tar_data) - exit_code, output = container.exec_run("sudo /bin/rm -rf /tmp/WfCommons/build/", stdout=True, stderr=True) - exit_code, output = container.exec_run("sudo /bin/rm -rf /tmp/WfCommons/*.egg-info/", stdout=True, stderr=True) - exit_code, output = container.exec_run("sudo /bin/rm -rf /tmp/WfCommons/bin/cpu-benchmark.o", stdout=True, stderr=True) - exit_code, output = container.exec_run("sudo /bin/rm -rf /tmp/WfCommons/bin/cpu-benchmark", stdout=True, stderr=True) - - # Install WfCommons on the container (to install wfbench and cpu-benchmark really) - exit_code, output = container.exec_run("sudo python3 -m pip install -e . --break-system-packages", workdir="/tmp/WfCommons", stdout=True, stderr=True) - - # Copy over the wfbench and cpu-benchmark executables - exit_code, output = container.exec_run("sudo cp -f /tmp/WfCommons/bin/wfbench " + str_dirpath + "bin/", stdout=True, stderr=True) - exit_code, output = container.exec_run("sudo cp -f /tmp/WfCommons/bin/cpu-benchmark " + str_dirpath + "bin/", stdout=True, stderr=True) - - # Run the dask workflow! - sys.stderr.write("Running the Dask workflow on the container...\n") - exit_code, output = container.exec_run("python ./dask_workflow.py", stdout=True, stderr=True) - num_completed_tasks = output.decode().count("completed!") # TODO: This is pretty lame - - # Kill the container - container.remove(force=True) - - # Do sanity checks - sys.stderr.write("Checking sanity...\n") - assert(exit_code == 0) - assert(num_completed_tasks == num_tasks) - # TODO: Look at the (I think) generated run.json file on the container - - - - - - diff --git a/tests/translators/parsl/test_parsl_translator.py b/tests/translators/parsl/test_parsl_translator.py deleted file mode 100644 index abd96ec5..00000000 --- a/tests/translators/parsl/test_parsl_translator.py +++ /dev/null @@ -1,111 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# -# Copyright (c) 2025 The WfCommons Team. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. - -import pathlib -import pytest -import shutil -import docker -import io -import tarfile -import os -import sys -import json - -from wfcommons import BlastRecipe -from wfcommons.wfbench import WorkflowBenchmark, DaskTranslator - -class TestParslTranslator: - - def make_tarfile_of_wfcommons(self): - source_dir = os.getcwd() # This assumes the testing is run from the root - tar_stream = io.BytesIO() - with tarfile.open(fileobj=tar_stream, mode='w') as tar: - tar.add(source_dir, arcname=os.path.basename(source_dir)) - tar_stream.seek(0) - return tar_stream - - - @pytest.mark.unit - def test_parsl_translator(self) -> None: - - # Create a workflow benchmark object to generate specifications based on a recipe (in /tmp/, whatever) - desired_num_tasks = 45 - benchmark_full_path = "/tmp/blast-benchmark-{desired_num_tasks}.json" - shutil.rmtree(benchmark_full_path, ignore_errors=True) - benchmark = WorkflowBenchmark(recipe=BlastRecipe, num_tasks=desired_num_tasks) - benchmark.create_benchmark(pathlib.Path("/tmp/"), cpu_work=100, data=10, percent_cpu=0.6) - with open(f"/tmp/blast-benchmark-{desired_num_tasks}.json", "r") as f: - generated_json = json.load(f) - num_tasks = len(generated_json["workflow"]["specification"]["tasks"]) - - # Make sure the target dir does not already exist - str_dirpath = "/tmp/dask_translated_workflow/" - dirpath = pathlib.Path(str_dirpath) - if dirpath.exists(): - shutil.rmtree(dirpath) - - # Perform the translation - sys.stderr.write("Translating workflow...\n") - translator = DaskTranslator(benchmark.workflow) - translator.translate(output_folder=dirpath) - - # Pulling the Docker image - client = docker.from_env() - sys.stderr.write("Pulling Docker image...\n") - image_name = "wfcommons/wfcommons-testing-dask" - client.images.pull(image_name) - - # Launch the docker container to actually run the translated workflow - sys.stderr.write("Starting Docker container...\n") - container = client.containers.run( - image_name, - "sleep infinity", - volumes={str_dirpath: {'bind': str_dirpath, 'mode': 'rw'}}, - working_dir=str_dirpath, - tty=True, - detach=True - ) - - sys.stderr.write("Installing WfCommons on the container...\n") - # Copy the WfCommons code to it (removing stuff that should be removed) - target_path = '/tmp/' # inside container - tar_data = self.make_tarfile_of_wfcommons() - container.put_archive(target_path, tar_data) - exit_code, output = container.exec_run("sudo /bin/rm -rf /tmp/WfCommons/build/", stdout=True, stderr=True) - exit_code, output = container.exec_run("sudo /bin/rm -rf /tmp/WfCommons/*.egg-info/", stdout=True, stderr=True) - exit_code, output = container.exec_run("sudo /bin/rm -rf /tmp/WfCommons/bin/cpu-benchmark.o", stdout=True, stderr=True) - exit_code, output = container.exec_run("sudo /bin/rm -rf /tmp/WfCommons/bin/cpu-benchmark", stdout=True, stderr=True) - - # Install WfCommons on the container (to install wfbench and cpu-benchmark really) - exit_code, output = container.exec_run("sudo python3 -m pip install -e . --break-system-packages", workdir="/tmp/WfCommons", stdout=True, stderr=True) - - # Copy over the wfbench and cpu-benchmark executables - exit_code, output = container.exec_run("sudo cp -f /tmp/WfCommons/bin/wfbench " + str_dirpath + "bin/", stdout=True, stderr=True) - exit_code, output = container.exec_run("sudo cp -f /tmp/WfCommons/bin/cpu-benchmark " + str_dirpath + "bin/", stdout=True, stderr=True) - - # Run the dask workflow! - sys.stderr.write("Running the Dask workflow on the container...\n") - exit_code, output = container.exec_run("python ./dask_workflow.py", stdout=True, stderr=True) - num_completed_tasks = output.decode().count("completed!") # TODO: This is pretty lame - - # Kill the container - container.remove(force=True) - - # Do sanity checks - sys.stderr.write("Checking sanity...\n") - assert(exit_code == 0) - assert(num_completed_tasks == num_tasks) - # TODO: Look at the (I think) generated run.json file on the container - - - - - - diff --git a/tests/translators/test_translators.py b/tests/translators/test_translators.py new file mode 100644 index 00000000..4fb09736 --- /dev/null +++ b/tests/translators/test_translators.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) 2025 The WfCommons Team. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +import pathlib +import pytest +import shutil +import docker +import io +import tarfile +import os +import sys +import json + +from wfcommons import BlastRecipe +from wfcommons.wfbench import WorkflowBenchmark, DaskTranslator, ParslTranslator + + +def start_docker_container(backend, working_dir): + # Pulling the Docker image + client = docker.from_env() + sys.stderr.write("Pulling Docker image...\n") + image_name = f"wfcommons/wfcommons-testing-{backend}" + + try: + image = client.images.get(image_name) + sys.stderr.write(f"Image '{image_name}' is available locally\n") + except ImageNotFound: + sys.stderr.write(f"Pulling image '{image_name}'\n") + client.images.pull(image_name) + + # Launch the docker container to actually run the translated workflow + sys.stderr.write("Starting Docker container...\n") + container = client.containers.run( + image_name, + "sleep infinity", + volumes={working_dir: {'bind': working_dir, 'mode': 'rw'}}, + working_dir=working_dir, + tty=True, + detach=True + ) + return container + +def make_tarfile_of_wfcommons(): + source_dir = os.getcwd() # This assumes the testing is run from the root + tar_stream = io.BytesIO() + with tarfile.open(fileobj=tar_stream, mode='w') as tar: + tar.add(source_dir, arcname=os.path.basename(source_dir)) + tar_stream.seek(0) + return tar_stream + +def install_WfCommons_on_container(container): + sys.stderr.write("Installing WfCommons on the container...\n") + # Copy the WfCommons code to it (removing stuff that should be removed) + target_path = '/tmp/' # inside container + tar_data = make_tarfile_of_wfcommons() + container.put_archive(target_path, tar_data) + exit_code, output = container.exec_run("sudo /bin/rm -rf /tmp/WfCommons/build/", stdout=True, stderr=True) + exit_code, output = container.exec_run("sudo /bin/rm -rf /tmp/WfCommons/*.egg-info/", stdout=True, stderr=True) + exit_code, output = container.exec_run("sudo /bin/rm -rf /tmp/WfCommons/bin/cpu-benchmark.o", stdout=True, + stderr=True) + exit_code, output = container.exec_run("sudo /bin/rm -rf /tmp/WfCommons/bin/cpu-benchmark", stdout=True, + stderr=True) + + # Install WfCommons on the container (to install wfbench and cpu-benchmark really) + exit_code, output = container.exec_run("sudo python3 -m pip install -e . --break-system-packages", + workdir="/tmp/WfCommons", stdout=True, stderr=True) + +def create_workflow_benchmark(): + # Create a workflow benchmark object to generate specifications based on a recipe (in /tmp/, whatever) + desired_num_tasks = 45 + benchmark_full_path = "/tmp/blast-benchmark-{desired_num_tasks}.json" + shutil.rmtree(benchmark_full_path, ignore_errors=True) + benchmark = WorkflowBenchmark(recipe=BlastRecipe, num_tasks=desired_num_tasks) + benchmark.create_benchmark(pathlib.Path("/tmp/"), cpu_work=100, data=10, percent_cpu=0.6) + with open(f"/tmp/blast-benchmark-{desired_num_tasks}.json", "r") as f: + generated_json = json.load(f) + num_tasks = len(generated_json["workflow"]["specification"]["tasks"]) + return benchmark, num_tasks + +class TestDaskTranslator: + + @pytest.mark.unit + def test_dask_translator(self) -> None: + + # Create workflow benchmark + benchmark, num_tasks = create_workflow_benchmark() + + # Create a local translation directory + str_dirpath = "/tmp/dask_translated_workflow/" + dirpath = pathlib.Path(str_dirpath) + if dirpath.exists(): + shutil.rmtree(dirpath) + + # Perform the translation + sys.stderr.write("Translating workflow...\n") + translator = DaskTranslator(benchmark.workflow) + translator.translate(output_folder=dirpath) + + # Pulling the Docker image + container = start_docker_container("dask", str_dirpath) + + # Installing WfCommons on container + install_WfCommons_on_container(container) + + # Copy over the wfbench and cpu-benchmark executables to where they should go + exit_code, output = container.exec_run("sudo cp -f /tmp/WfCommons/bin/wfbench " + str_dirpath + "bin/", stdout=True, stderr=True) + exit_code, output = container.exec_run("sudo cp -f /tmp/WfCommons/bin/cpu-benchmark " + str_dirpath + "bin/", stdout=True, stderr=True) + + # Run the workflow! + sys.stderr.write("Running the Dask workflow on the container...\n") + exit_code, output = container.exec_run("python ./dask_workflow.py", stdout=True, stderr=True) + num_completed_tasks = output.decode().count("completed!") # TODO: This is pretty lame + + # Kill the container + container.remove(force=True) + + # Do sanity checks + sys.stderr.write("Checking sanity...\n") + assert(exit_code == 0) + assert(num_completed_tasks == num_tasks) + # TODO: Look at the (I think) generated run.json file on the container + + + @pytest.mark.unit + def test_parsl_translator(self) -> None: + + # Create workflow benchmark + benchmark, num_tasks = create_workflow_benchmark() + + # Create a local translation directory + str_dirpath = "/tmp/parsl_translated_workflow/" + dirpath = pathlib.Path(str_dirpath) + if dirpath.exists(): + shutil.rmtree(dirpath) + + # Perform the translation + sys.stderr.write("Translating workflow...\n") + translator = ParslTranslator(benchmark.workflow) + translator.translate(output_folder=dirpath) + + # Pulling the Docker image + container = start_docker_container("parsl", str_dirpath) + + # Installing WfCommons on container + install_WfCommons_on_container(container) + + # Copy over the wfbench and cpu-benchmark executables to where they should go + exit_code, output = container.exec_run("sudo cp -f /tmp/WfCommons/bin/wfbench " + str_dirpath + "bin/", stdout=True, stderr=True) + exit_code, output = container.exec_run("sudo cp -f /tmp/WfCommons/bin/cpu-benchmark " + str_dirpath + "bin/", stdout=True, stderr=True) + + # Run the workflow! + sys.stderr.write("Running the Parsl workflow on the container...\n") + exit_code, output = container.exec_run("python ./parsl_workflow.py", stdout=True, stderr=True) + + exit_code, output = container.exec_run(f"cat {str_dirpath}/runinfo/000/parsl.log", stdout=True, stderr=True) + num_completed_tasks = output.decode().count("_complete_task") + + # Kill the container + container.remove(force=True) + + # Do sanity checks + sys.stderr.write("Checking sanity...\n") + assert(exit_code == 0) + assert("completed" in output.decode()) + assert(num_completed_tasks == num_tasks) + + + + From fd81786f54bf3f114d58190d670d6d43c5f8da8e Mon Sep 17 00:00:00 2001 From: Henri Casanova Date: Mon, 4 Aug 2025 17:19:20 -1000 Subject: [PATCH 04/38] Re-enabling skipped tests for now --- tests/recipes/test_recipes.py | 68 +++++++++++++++++------------------ tests/wfbench/test_wfbench.py | 4 +-- 2 files changed, 36 insertions(+), 36 deletions(-) diff --git a/tests/recipes/test_recipes.py b/tests/recipes/test_recipes.py index 484d2518..0d6851b0 100644 --- a/tests/recipes/test_recipes.py +++ b/tests/recipes/test_recipes.py @@ -39,37 +39,37 @@ class TestRecipes: SrasearchRecipe, ] - # @pytest.mark.unit - # @pytest.mark.parametrize( - # "recipe_class", - # recipe_class_list - # ) - # def test_recipes(self, recipe_class) -> None: - # - # recipe = recipe_class.from_num_tasks(num_tasks=200, runtime_factor=1.1, input_file_size_factor=1.5, - # output_file_size_factor=0.8) - # workflow = WorkflowGenerator(recipe).build_workflow() - # - # - # @pytest.mark.unit - # @pytest.mark.parametrize( - # "recipe_class", - # recipe_class_list - # ) - # def test_recipes_errors(self, recipe_class) -> None: - # # Not enough tasks - # recipe = recipe_class.from_num_tasks(num_tasks=2, runtime_factor=1.1, input_file_size_factor=1.5, - # output_file_size_factor=0.8) - # with pytest.raises(ValueError): - # WorkflowGenerator(recipe).build_workflow() - # - # # Bogus parameters - # with pytest.raises(ValueError): - # recipe_class.from_num_tasks(num_tasks=2, runtime_factor=-1.1, input_file_size_factor=1.5, - # output_file_size_factor=0.8) - # with pytest.raises(ValueError): - # recipe_class.from_num_tasks(num_tasks=2, runtime_factor=1.1, input_file_size_factor=-1.5, - # output_file_size_factor=0.8) - # with pytest.raises(ValueError): - # recipe_class.from_num_tasks(num_tasks=2, runtime_factor=1.1, input_file_size_factor=-1.5, - # output_file_size_factor=-0.8) + @pytest.mark.unit + @pytest.mark.parametrize( + "recipe_class", + recipe_class_list + ) + def test_recipes(self, recipe_class) -> None: + + recipe = recipe_class.from_num_tasks(num_tasks=200, runtime_factor=1.1, input_file_size_factor=1.5, + output_file_size_factor=0.8) + workflow = WorkflowGenerator(recipe).build_workflow() + + + @pytest.mark.unit + @pytest.mark.parametrize( + "recipe_class", + recipe_class_list + ) + def test_recipes_errors(self, recipe_class) -> None: + # Not enough tasks + recipe = recipe_class.from_num_tasks(num_tasks=2, runtime_factor=1.1, input_file_size_factor=1.5, + output_file_size_factor=0.8) + with pytest.raises(ValueError): + WorkflowGenerator(recipe).build_workflow() + + # Bogus parameters + with pytest.raises(ValueError): + recipe_class.from_num_tasks(num_tasks=2, runtime_factor=-1.1, input_file_size_factor=1.5, + output_file_size_factor=0.8) + with pytest.raises(ValueError): + recipe_class.from_num_tasks(num_tasks=2, runtime_factor=1.1, input_file_size_factor=-1.5, + output_file_size_factor=0.8) + with pytest.raises(ValueError): + recipe_class.from_num_tasks(num_tasks=2, runtime_factor=1.1, input_file_size_factor=-1.5, + output_file_size_factor=-0.8) diff --git a/tests/wfbench/test_wfbench.py b/tests/wfbench/test_wfbench.py index a4e09773..5e2ea4fa 100644 --- a/tests/wfbench/test_wfbench.py +++ b/tests/wfbench/test_wfbench.py @@ -22,7 +22,7 @@ class TestWfBench: # @pytest.mark.unit - @pytest.mark.skip(reason="Temporarily disabled due to strange module 'scipy.stats' has no attribute 'trapz' error") + # @pytest.mark.skip(reason="Temporarily disabled due to strange module 'scipy.stats' has no attribute 'trapz' error") def test_create_from_recipe(self) -> None: """ Very minimal testing here for creating from recipe @@ -114,7 +114,7 @@ def _to_create_file_as_expected(dirpath: pathlib.Path, # @pytest.mark.unit - @pytest.mark.skip(reason="Temporarily disabled due to strange module 'scipy.stats' has no attribute 'trapz' error") + # @pytest.mark.skip(reason="Temporarily disabled due to strange module 'scipy.stats' has no attribute 'trapz' error") def test_create_from_instance(self) -> None: workflow = BlastRecipe.from_num_tasks(500).build_workflow() benchmark = WorkflowBenchmark(recipe=BlastRecipe, num_tasks=500) From 84c55dd5d35c0c0ae92c9029a2ad6a70e083f786 Mon Sep 17 00:00:00 2001 From: Henri Casanova Date: Mon, 4 Aug 2025 20:43:32 -1000 Subject: [PATCH 05/38] Added Nextflow translator test --- tests/translators/test_translators.py | 46 ++++++++++++++++++++++-- tests/wfchef/test_wfchef.py | 1 - wfcommons/wfbench/translator/nextflow.py | 6 ++-- 3 files changed, 46 insertions(+), 7 deletions(-) diff --git a/tests/translators/test_translators.py b/tests/translators/test_translators.py index 4fb09736..91e1ecd0 100644 --- a/tests/translators/test_translators.py +++ b/tests/translators/test_translators.py @@ -19,7 +19,7 @@ import json from wfcommons import BlastRecipe -from wfcommons.wfbench import WorkflowBenchmark, DaskTranslator, ParslTranslator +from wfcommons.wfbench import WorkflowBenchmark, DaskTranslator, ParslTranslator, NextflowTranslator def start_docker_container(backend, working_dir): @@ -78,13 +78,13 @@ def create_workflow_benchmark(): benchmark_full_path = "/tmp/blast-benchmark-{desired_num_tasks}.json" shutil.rmtree(benchmark_full_path, ignore_errors=True) benchmark = WorkflowBenchmark(recipe=BlastRecipe, num_tasks=desired_num_tasks) - benchmark.create_benchmark(pathlib.Path("/tmp/"), cpu_work=100, data=10, percent_cpu=0.6) + benchmark.create_benchmark(pathlib.Path("/tmp/"), cpu_work=10, data=10, percent_cpu=0.6) with open(f"/tmp/blast-benchmark-{desired_num_tasks}.json", "r") as f: generated_json = json.load(f) num_tasks = len(generated_json["workflow"]["specification"]["tasks"]) return benchmark, num_tasks -class TestDaskTranslator: +class TestTranslators: @pytest.mark.unit def test_dask_translator(self) -> None: @@ -171,6 +171,46 @@ def test_parsl_translator(self) -> None: assert("completed" in output.decode()) assert(num_completed_tasks == num_tasks) + @pytest.mark.unit + def test_nextflow_translator(self) -> None: + + # Create workflow benchmark + benchmark, num_tasks = create_workflow_benchmark() + + # Create a local translation directory + str_dirpath = "/tmp/nextflow_translated_workflow/" + dirpath = pathlib.Path(str_dirpath) + if dirpath.exists(): + shutil.rmtree(dirpath) + # Perform the translation + sys.stderr.write("Translating workflow...\n") + translator = NextflowTranslator(benchmark.workflow) + translator.translate(output_folder=dirpath) + + # Pulling the Docker image + container = start_docker_container("nextflow", str_dirpath) + + # Installing WfCommons on container + install_WfCommons_on_container(container) + + # Copy over the wfbench and cpu-benchmark executables to where they should go + exit_code, output = container.exec_run("sudo cp -f /tmp/WfCommons/bin/wfbench " + str_dirpath + "bin/", + stdout=True, stderr=True) + exit_code, output = container.exec_run("sudo cp -f /tmp/WfCommons/bin/cpu-benchmark " + str_dirpath + "bin/", + stdout=True, stderr=True) + + # Run the workflow! + sys.stderr.write("Running the Nextflow workflow on the container...\n") + exit_code, output = container.exec_run(f"nextflow run ./workflow.nf --pwd .", stdout=True, stderr=True) + ignored, task_exit_codes = container.exec_run("find . -name .exitcode -exec cat {} \;", stdout=True, stderr=True) + + # Kill the container + container.remove(force=True) + + # Do sanity checks + sys.stderr.write("Checking sanity...\n") + assert (exit_code == 0) + assert (task_exit_codes.decode() == num_tasks * "0") diff --git a/tests/wfchef/test_wfchef.py b/tests/wfchef/test_wfchef.py index fe770f7a..2fc1668d 100644 --- a/tests/wfchef/test_wfchef.py +++ b/tests/wfchef/test_wfchef.py @@ -23,7 +23,6 @@ class TestWfChef: @pytest.mark.unit - #@pytest.mark.skip(reason="Temporarily disabled due to strange ModuleNotFoundError: No module named 'pkg_resources' error") def test_create_recipe(self) -> None: """ Just calling the create_recipe function from chef.py directly (i.e., bypassing main()) diff --git a/wfcommons/wfbench/translator/nextflow.py b/wfcommons/wfbench/translator/nextflow.py index 2300de25..3864cf06 100644 --- a/wfcommons/wfbench/translator/nextflow.py +++ b/wfcommons/wfbench/translator/nextflow.py @@ -163,16 +163,16 @@ def _create_task_script(self, task: Task): # Generate input spec input_spec = "'\\[" for f in task.input_files: - input_spec += f"\"{self.output_folder.resolve()}/data/{f.file_id}\"," + input_spec += f"\"{self.output_folder}/data/{f.file_id}\"," input_spec = input_spec[:-1] + "\\]'" # Generate output spec output_spec = "'\\{" for f in task.output_files: - output_spec += f"\"{self.output_folder.resolve()}/data/{f.file_id}\":{str(f.size)}," + output_spec += f"\"{self.output_folder}/data/{f.file_id}\":{str(f.size)}," output_spec = output_spec[:-1] + "\\}'" - code += f"{self.output_folder.resolve()}/bin/{task.program} " + code += f"{self.output_folder}/bin/{task.program} " for a in task.args: if "--output-files" in a: From 6e048f6257d8e007bf2d8e0ed4a1d2a3ccd5e079 Mon Sep 17 00:00:00 2001 From: Henri Casanova Date: Tue, 5 Aug 2025 11:11:17 -1000 Subject: [PATCH 06/38] Testing++ Removed Python 3.9/3.10 Upgraded scipy requirement to 16.1.1 --- .github/workflows/build.yml | 3 +- README.md | 2 +- docs/source/quickstart_installation.rst | 2 +- pyproject.toml | 5 +- setup.py | 2 +- .../Dockerfile_Airflow => Dockerfile.airflow} | 0 .../{airflow/README => README.airflow} | 10 ++-- .../translators/build_docker_docker_images.sh | 2 +- tests/translators/test_translators.py | 50 ++++++++++++++++++- wfcommons/utils.py | 2 +- wfcommons/wfbench/translator/airflow.py | 23 +++++++++ .../wfchef/recipes/blast/task_type_stats.json | 2 +- .../wfchef/recipes/bwa/task_type_stats.json | 2 +- .../recipes/cycles/task_type_stats.json | 2 +- .../recipes/epigenomics/task_type_stats.json | 12 ++--- .../recipes/genome/task_type_stats.json | 2 +- .../recipes/montage/task_type_stats.json | 16 +++--- .../recipes/seismology/task_type_stats.json | 2 +- .../wfchef/recipes/soykb/task_type_stats.json | 2 +- .../recipes/srasearch/task_type_stats.json | 2 +- 20 files changed, 104 insertions(+), 39 deletions(-) rename tests/translators/{airflow/Dockerfile_Airflow => Dockerfile.airflow} (100%) rename tests/translators/{airflow/README => README.airflow} (91%) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index fd59ce75..a1459068 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -7,8 +7,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [ "3.9", "3.10", "3.11", "3.12", "3.13" ] -# python-version: [ "3.9", "3.10" ] + python-version: [ "3.11", "3.12", "3.13" ] steps: - uses: actions/checkout@v4 diff --git a/README.md b/README.md index cd43f157..76e95e01 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ This Python package provides a collection of tools for: ## Installation WfCommons is available on [PyPI](https://pypi.org/project/wfcommons). -WfCommons requires Python3.9+ and has been tested on Linux and macOS. +WfCommons requires Python3.11+ and has been tested on Linux and MacOS. ### Installation using pip diff --git a/docs/source/quickstart_installation.rst b/docs/source/quickstart_installation.rst index 43367f3e..a8fc8b4a 100644 --- a/docs/source/quickstart_installation.rst +++ b/docs/source/quickstart_installation.rst @@ -2,7 +2,7 @@ Installation ============ WfCommons is available on `PyPI `_. -WfCommons requires Python3.9+ and has been tested on Linux and macOS. +WfCommons requires Python3.11+ and has been tested on Linux and macOS. Installation using pip ---------------------- diff --git a/pyproject.toml b/pyproject.toml index cf60a08e..bd864791 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,9 +11,6 @@ requires-python = ">=3.9" classifiers = [ "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", "Operating System :: OS Independent", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", @@ -33,7 +30,7 @@ dependencies = [ "pandas", "python-dateutil", "requests", - "scipy", + "scipy>=1.16.1", "pyyaml", "pandas", "shortuuid", diff --git a/setup.py b/setup.py index e6c3f078..410d9ed6 100644 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ class Build(build_ext): def run(self): protoc_command = ["make"] if subprocess.call(protoc_command) != 0: - print("Error: 'make' is not istnalled. Please install 'make' and try again.") + print("Error: 'make' is not installed. Please install 'make' and try again.") sys.exit(-1) super().run() diff --git a/tests/translators/airflow/Dockerfile_Airflow b/tests/translators/Dockerfile.airflow similarity index 100% rename from tests/translators/airflow/Dockerfile_Airflow rename to tests/translators/Dockerfile.airflow diff --git a/tests/translators/airflow/README b/tests/translators/README.airflow similarity index 91% rename from tests/translators/airflow/README rename to tests/translators/README.airflow index d18deeb5..a0291a88 100644 --- a/tests/translators/airflow/README +++ b/tests/translators/README.airflow @@ -1,5 +1,4 @@ -This README file describes steps to install/run Airflow, and then run a -translated workflow. +This README file describes steps to install/run Airflow, and then run a translated workflow. There are three sections: - Installing Airflow on bare-metal @@ -52,7 +51,7 @@ A much simpler alternative is to use Docker. 1. Build the docker image - docker build -t wfcommons-dev -f Dockerfile_Airflow . + docker build -t wfcommons-dev -f Dockerfile.airflow . (if building on a Mac, add the `--platform linux/amd64` argument after build above) @@ -85,8 +84,7 @@ translator.translate(output_folder=pathlib.Path("/tmp/translated_workflow/")) ``` The above will create a JSON worfklow file in /tmp/blast-benchmark-45.json. -In that file, the workflow name (this is used below) is set to -"Blast-Benchmark". +In that file, the workflow name (this is used below) is set to "Blast-Benchmark". The above will also create the translated workflow the /tmp/translated_workflow/ directory. Some directories and files need to be copied/moved as follows: @@ -96,7 +94,7 @@ The above will also create the translated workflow the Finally, run the workflow as: - airflow dags test Blast-Benchmark (not the "Blast-Benchmark" workflow name from above) + airflow dags test Blast-Benchmark (note the "Blast-Benchmark" workflow name from above) diff --git a/tests/translators/build_docker_docker_images.sh b/tests/translators/build_docker_docker_images.sh index 1ef71088..5621f890 100755 --- a/tests/translators/build_docker_docker_images.sh +++ b/tests/translators/build_docker_docker_images.sh @@ -2,7 +2,7 @@ set -e -for backend in "dask" "parsl" "nextflow"; do +for backend in "dask" "parsl" "nextflow" "airflow"; do echo "Building $backend Docker image..." docker build --platform linux/amd64 -t wfcommons/wfcommons-testing-$backend -f Dockerfile.$backend . done diff --git a/tests/translators/test_translators.py b/tests/translators/test_translators.py index 91e1ecd0..7af482ff 100644 --- a/tests/translators/test_translators.py +++ b/tests/translators/test_translators.py @@ -19,7 +19,11 @@ import json from wfcommons import BlastRecipe -from wfcommons.wfbench import WorkflowBenchmark, DaskTranslator, ParslTranslator, NextflowTranslator +from wfcommons.wfbench import WorkflowBenchmark +from wfcommons.wfbench import DaskTranslator +from wfcommons.wfbench import ParslTranslator +from wfcommons.wfbench import NextflowTranslator +from wfcommons.wfbench import AirflowTranslator def start_docker_container(backend, working_dir): @@ -87,6 +91,7 @@ def create_workflow_benchmark(): class TestTranslators: @pytest.mark.unit + # @pytest.mark.skip(reason="tmp") def test_dask_translator(self) -> None: # Create workflow benchmark @@ -129,6 +134,7 @@ def test_dask_translator(self) -> None: @pytest.mark.unit + # @pytest.mark.skip(reason="tmp") def test_parsl_translator(self) -> None: # Create workflow benchmark @@ -172,6 +178,7 @@ def test_parsl_translator(self) -> None: assert(num_completed_tasks == num_tasks) @pytest.mark.unit + # @pytest.mark.skip(reason="tmp") def test_nextflow_translator(self) -> None: # Create workflow benchmark @@ -214,3 +221,44 @@ def test_nextflow_translator(self) -> None: assert (task_exit_codes.decode() == num_tasks * "0") + @pytest.mark.unit + def test_airflow_translator(self) -> None: + + # Create workflow benchmark + benchmark, num_tasks = create_workflow_benchmark() + + # Create a local translation directory + str_dirpath = "/tmp/airflow_translated_workflow/" + dirpath = pathlib.Path(str_dirpath) + if dirpath.exists(): + shutil.rmtree(dirpath) + + # Perform the translation + sys.stderr.write("Translating workflow...\n") + translator = AirflowTranslator(benchmark.workflow) + translator.translate(output_folder=dirpath) + + # Pulling the Docker image + container = start_docker_container("airflow", str_dirpath) + + # Installing WfCommons on container + install_WfCommons_on_container(container) + + # Copy over the wfbench and cpu-benchmark executables to where they should go + # exit_code, output = container.exec_run("sudo cp -f /tmp/WfCommons/bin/wfbench " + str_dirpath + "bin/", + # stdout=True, stderr=True) + # exit_code, output = container.exec_run("sudo cp -f /tmp/WfCommons/bin/cpu-benchmark " + str_dirpath + "bin/", + # stdout=True, stderr=True) + # + # # Run the workflow! + # sys.stderr.write("Running the Airflow workflow on the container...\n") + # exit_code, output = container.exec_run(f"nextflow run ./workflow.nf --pwd .", stdout=True, stderr=True) + # ignored, task_exit_codes = container.exec_run("find . -name .exitcode -exec cat {} \;", stdout=True, stderr=True) + # + # # Kill the container + # container.remove(force=True) + # + # # Do sanity checks + # sys.stderr.write("Checking sanity...\n") + # assert (exit_code == 0) + # assert (task_exit_codes.decode() == num_tasks * "0") diff --git a/wfcommons/utils.py b/wfcommons/utils.py index f9f94556..b65c784e 100644 --- a/wfcommons/utils.py +++ b/wfcommons/utils.py @@ -78,7 +78,7 @@ def best_fit_distribution(data: List[float], logger: Optional[Logger] = None) -> distribution_names: List[str] = [ 'alpha', 'arcsine', 'argus', 'beta', 'chi', 'chi2', 'dgamma', 'dweibull', 'expon', 'fisk', 'gamma', 'gausshyper', 'levy', 'norm', 'pareto', 'rayleigh', - 'rdist', 'skewnorm', 'trapz', 'triang', 'uniform', 'wald' + 'rdist', 'skewnorm', 'trapezoid', 'triang', 'uniform', 'wald' ] for dist_name in distribution_names: diff --git a/wfcommons/wfbench/translator/airflow.py b/wfcommons/wfbench/translator/airflow.py index c6309425..2906115c 100644 --- a/wfcommons/wfbench/translator/airflow.py +++ b/wfcommons/wfbench/translator/airflow.py @@ -88,6 +88,9 @@ def translate(self, output_folder: pathlib.Path) -> None: self._copy_binary_files(output_folder) self._generate_input_files(output_folder) + # Create the README file + self._write_readme_file(output_folder) + def _prep_commands(self, output_folder: pathlib.Path) -> None: """ Prepares the bash_command strings for the BashOperators. @@ -127,3 +130,23 @@ def _prep_commands(self, output_folder: pathlib.Path) -> None: self.task_commands[task.task_id] = command_str + def _write_readme_file(self, output_folder: pathlib.Path) -> None: + """ + Write the README file. + + :param output_folder: The path of the output folder. + :type output_folder: pathlib.Path + """ + readme_file_path = output_folder.joinpath("README") + with open(readme_file_path, "w") as out: + out.write(f"""Assuming that the translated workflow is in the /tmp/translated_workflow/ directory, before +running the workflow some directories and files need to be copied/moved as follows: + + cp -r /tmp/translated_workflow/ $AIRFLOW_HOME/dags/ + mv $AIRFLOW_HOME/dags/translated_workflow/workflow.py $AIRFLOW_HOME/dags/ + +Now, the workflow can be executed as: + + airflow dags test Workflow-Name (where "Workflow-Name" is the workflow name in the WfCommons-generated + benchmark JSON file, e.g., "Blast-Benchmark") +""") \ No newline at end of file diff --git a/wfcommons/wfchef/recipes/blast/task_type_stats.json b/wfcommons/wfchef/recipes/blast/task_type_stats.json index 7498ab78..1346ea33 100644 --- a/wfcommons/wfchef/recipes/blast/task_type_stats.json +++ b/wfcommons/wfchef/recipes/blast/task_type_stats.json @@ -1 +1 @@ -{"split_fasta": {"runtime": {"min": 0.051992, "max": 3.160018, "distribution": {"name": "arcsine", "params": [-0.2258070520586602, 1.2258070520586604]}}, "input": {".fasta": {"distribution": {"name": "arcsine", "params": [-0.2258070520586602, 1.2258070520586604]}, "min": 203, "max": 201389}, "split_fasta": {"distribution": null, "min": 1, "max": 1}}, "output": {".fasta": {"distribution": {"name": "trapz", "params": [0.9999999999999999, 1.0, -0.10500000000000001, 1.1999999999999997]}, "min": 6, "max": 2015}}}, "blastall": {"runtime": {"min": 8.116334, "max": 1799.556624, "distribution": {"name": "trapz", "params": [1.0, 1.0, -0.10500000000000001, 1.2]}}, "input": {"blastall": {"distribution": null, "min": 7688, "max": 7688}, ".fasta": {"distribution": {"name": "trapz", "params": [0.9999999999999999, 1.0, -0.10500000000000001, 1.1999999999999997]}, "min": 6, "max": 2015}, "nt": {"distribution": {"name": "trapz", "params": [0.9999999999999999, 1.0, -0.10500000000000001, 1.1999999999999997]}, "min": 0, "max": 5117704493}}, "output": {".out": {"distribution": {"name": "argus", "params": [2.465535551931572e-05, -0.7452662890705088, 1.7477663092998088]}, "min": 5, "max": 17952}, ".err": {"distribution": null, "min": 0, "max": 0}}}, "cat_blast": {"runtime": {"min": 0.034811, "max": 16.689957, "distribution": {"name": "arcsine", "params": [-0.2258070520586602, 1.2258070520586604]}}, "input": {"cat_blast": {"distribution": null, "min": 1, "max": 1}, ".out": {"distribution": {"name": "argus", "params": [2.465535551931572e-05, -0.7452662890705088, 1.7477663092998088]}, "min": 5, "max": 17952}}, "output": {"None": {"distribution": {"name": "arcsine", "params": [-0.2258070520586602, 1.2258070520586604]}, "min": 454, "max": 565948}}}, "cat": {"runtime": {"min": 0.009596, "max": 0.021895, "distribution": {"name": "arcsine", "params": [-0.2258070520586602, 1.2258070520586604]}}, "input": {".err": {"distribution": null, "min": 0, "max": 0}}, "output": {".err": {"distribution": null, "min": 0, "max": 0}}}} \ No newline at end of file +{"split_fasta": {"runtime": {"min": 0.051992, "max": 3.160018, "distribution": {"name": "arcsine", "params": [-0.2258070520586602, 1.2258070520586604]}}, "input": {".fasta": {"distribution": {"name": "arcsine", "params": [-0.2258070520586602, 1.2258070520586604]}, "min": 203, "max": 201389}, "split_fasta": {"distribution": null, "min": 1, "max": 1}}, "output": {".fasta": {"distribution": {"name": "trapezoid", "params": [0.9999999999999999, 1.0, -0.10500000000000001, 1.1999999999999997]}, "min": 6, "max": 2015}}}, "blastall": {"runtime": {"min": 8.116334, "max": 1799.556624, "distribution": {"name": "trapezoid", "params": [1.0, 1.0, -0.10500000000000001, 1.2]}}, "input": {"blastall": {"distribution": null, "min": 7688, "max": 7688}, ".fasta": {"distribution": {"name": "trapezoid", "params": [0.9999999999999999, 1.0, -0.10500000000000001, 1.1999999999999997]}, "min": 6, "max": 2015}, "nt": {"distribution": {"name": "trapezoid", "params": [0.9999999999999999, 1.0, -0.10500000000000001, 1.1999999999999997]}, "min": 0, "max": 5117704493}}, "output": {".out": {"distribution": {"name": "argus", "params": [2.465535551931572e-05, -0.7452662890705088, 1.7477663092998088]}, "min": 5, "max": 17952}, ".err": {"distribution": null, "min": 0, "max": 0}}}, "cat_blast": {"runtime": {"min": 0.034811, "max": 16.689957, "distribution": {"name": "arcsine", "params": [-0.2258070520586602, 1.2258070520586604]}}, "input": {"cat_blast": {"distribution": null, "min": 1, "max": 1}, ".out": {"distribution": {"name": "argus", "params": [2.465535551931572e-05, -0.7452662890705088, 1.7477663092998088]}, "min": 5, "max": 17952}}, "output": {"None": {"distribution": {"name": "arcsine", "params": [-0.2258070520586602, 1.2258070520586604]}, "min": 454, "max": 565948}}}, "cat": {"runtime": {"min": 0.009596, "max": 0.021895, "distribution": {"name": "arcsine", "params": [-0.2258070520586602, 1.2258070520586604]}}, "input": {".err": {"distribution": null, "min": 0, "max": 0}}, "output": {".err": {"distribution": null, "min": 0, "max": 0}}}} \ No newline at end of file diff --git a/wfcommons/wfchef/recipes/bwa/task_type_stats.json b/wfcommons/wfchef/recipes/bwa/task_type_stats.json index abbc722f..537ccf33 100644 --- a/wfcommons/wfchef/recipes/bwa/task_type_stats.json +++ b/wfcommons/wfchef/recipes/bwa/task_type_stats.json @@ -1 +1 @@ -{"fastq_reduce": {"runtime": {"min": 0.045811, "max": 4.724154, "distribution": {"name": "argus", "params": [1.599540430972508, -0.5837001726177105, 1.7033104585411558]}}, "input": {"fastq_reduce": {"distribution": null, "min": 2, "max": 2}, ".fastq": {"distribution": {"name": "argus", "params": [1.599540430972508, -0.5837001726177105, 1.7033104585411558]}, "min": 2438, "max": 247788}}, "output": {".fastq": {"distribution": {"name": "rdist", "params": [1.750426650564894, 0.9999999999999998, 2.3624681987884834e-16]}, "min": 24, "max": 249}}}, "bwa_index": {"runtime": {"min": 80.652465, "max": 1158.975678, "distribution": {"name": "arcsine", "params": [-0.2258070520586602, 1.2258070520586604]}}, "input": {"bwa": {"distribution": null, "min": 1445, "max": 1445}, ".fastq": {"distribution": {"name": "arcsine", "params": [-0.2258070520586602, 1.2258070520586604]}, "min": 200438, "max": 2047778}}, "output": {".bwt": {"distribution": {"name": "arcsine", "params": [-0.2258070520586602, 1.2258070520586604]}, "min": 100001, "max": 1000001}, ".pac": {"distribution": {"name": "arcsine", "params": [-0.2258070520586602, 1.2258070520586604]}, "min": 25001, "max": 250001}, ".amb": {"distribution": null, "min": 1, "max": 1}, ".ann": {"distribution": {"name": "arcsine", "params": [-0.2258070520586602, 1.2258070520586604]}, "min": 577, "max": 61667}, ".sa": {"distribution": {"name": "arcsine", "params": [-0.2258070520586602, 1.2258070520586604]}, "min": 50001, "max": 500001}}}, "bwa": {"runtime": {"min": 0.18409, "max": 34.912105, "distribution": {"name": "argus", "params": [1.787473923386625e-05, -0.7146525478639822, 1.7383310890430437]}}, "input": {"bwa": {"distribution": null, "min": 1445, "max": 1445}, ".fastq": {"distribution": {"name": "trapz", "params": [1.0, 1.0, -0.10500000000000001, 1.2]}, "min": 24, "max": 2047778}, ".bwt": {"distribution": {"name": "trapz", "params": [1.0, 1.0, -0.10500000000000001, 1.2]}, "min": 100001, "max": 1000001}, ".pac": {"distribution": {"name": "trapz", "params": [1.0, 1.0, -0.10500000000000001, 1.2]}, "min": 25001, "max": 250001}, ".amb": {"distribution": null, "min": 1, "max": 1}, ".ann": {"distribution": {"name": "rdist", "params": [1.750426650564894, 0.9999999999999998, 2.3624681987884834e-16]}, "min": 577, "max": 61667}, ".sa": {"distribution": {"name": "trapz", "params": [1.0, 1.0, -0.10500000000000001, 1.2]}, "min": 50001, "max": 500001}}, "output": {".sam": {"distribution": {"name": "trapz", "params": [1.0, 1.0, -0.10500000000000001, 1.2]}, "min": 516, "max": 52088}, ".err": {"distribution": null, "min": 1, "max": 1}}}, "cat_bwa": {"runtime": {"min": 0.540879, "max": 519.460108, "distribution": {"name": "arcsine", "params": [-0.2258070520586602, 1.2258070520586604]}}, "input": {"cat_bwa": {"distribution": null, "min": 2, "max": 2}, ".sam": {"distribution": {"name": "trapz", "params": [1.0, 1.0, -0.10500000000000001, 1.2]}, "min": 516, "max": 52088}}, "output": {".sam": {"distribution": {"name": "argus", "params": [1.599540430972508, -0.5837001726177105, 1.7033104585411558]}, "min": 3437, "max": 353943}}}, "cat": {"runtime": {"min": 0.014307, "max": 0.073623, "distribution": {"name": "argus", "params": [1.599540430972508, -0.5837001726177105, 1.7033104585411558]}}, "input": {".err": {"distribution": null, "min": 1, "max": 1}}, "output": {".err": {"distribution": {"name": "argus", "params": [1.599540430972508, -0.5837001726177105, 1.7033104585411558]}, "min": 14, "max": 133}}}} \ No newline at end of file +{"fastq_reduce": {"runtime": {"min": 0.045811, "max": 4.724154, "distribution": {"name": "argus", "params": [1.599540430972508, -0.5837001726177105, 1.7033104585411558]}}, "input": {"fastq_reduce": {"distribution": null, "min": 2, "max": 2}, ".fastq": {"distribution": {"name": "argus", "params": [1.599540430972508, -0.5837001726177105, 1.7033104585411558]}, "min": 2438, "max": 247788}}, "output": {".fastq": {"distribution": {"name": "rdist", "params": [1.750426650564894, 0.9999999999999998, 2.3624681987884834e-16]}, "min": 24, "max": 249}}}, "bwa_index": {"runtime": {"min": 80.652465, "max": 1158.975678, "distribution": {"name": "arcsine", "params": [-0.2258070520586602, 1.2258070520586604]}}, "input": {"bwa": {"distribution": null, "min": 1445, "max": 1445}, ".fastq": {"distribution": {"name": "arcsine", "params": [-0.2258070520586602, 1.2258070520586604]}, "min": 200438, "max": 2047778}}, "output": {".bwt": {"distribution": {"name": "arcsine", "params": [-0.2258070520586602, 1.2258070520586604]}, "min": 100001, "max": 1000001}, ".pac": {"distribution": {"name": "arcsine", "params": [-0.2258070520586602, 1.2258070520586604]}, "min": 25001, "max": 250001}, ".amb": {"distribution": null, "min": 1, "max": 1}, ".ann": {"distribution": {"name": "arcsine", "params": [-0.2258070520586602, 1.2258070520586604]}, "min": 577, "max": 61667}, ".sa": {"distribution": {"name": "arcsine", "params": [-0.2258070520586602, 1.2258070520586604]}, "min": 50001, "max": 500001}}}, "bwa": {"runtime": {"min": 0.18409, "max": 34.912105, "distribution": {"name": "argus", "params": [1.787473923386625e-05, -0.7146525478639822, 1.7383310890430437]}}, "input": {"bwa": {"distribution": null, "min": 1445, "max": 1445}, ".fastq": {"distribution": {"name": "trapezoid", "params": [1.0, 1.0, -0.10500000000000001, 1.2]}, "min": 24, "max": 2047778}, ".bwt": {"distribution": {"name": "trapezoid", "params": [1.0, 1.0, -0.10500000000000001, 1.2]}, "min": 100001, "max": 1000001}, ".pac": {"distribution": {"name": "trapezoid", "params": [1.0, 1.0, -0.10500000000000001, 1.2]}, "min": 25001, "max": 250001}, ".amb": {"distribution": null, "min": 1, "max": 1}, ".ann": {"distribution": {"name": "rdist", "params": [1.750426650564894, 0.9999999999999998, 2.3624681987884834e-16]}, "min": 577, "max": 61667}, ".sa": {"distribution": {"name": "trapezoid", "params": [1.0, 1.0, -0.10500000000000001, 1.2]}, "min": 50001, "max": 500001}}, "output": {".sam": {"distribution": {"name": "trapezoid", "params": [1.0, 1.0, -0.10500000000000001, 1.2]}, "min": 516, "max": 52088}, ".err": {"distribution": null, "min": 1, "max": 1}}}, "cat_bwa": {"runtime": {"min": 0.540879, "max": 519.460108, "distribution": {"name": "arcsine", "params": [-0.2258070520586602, 1.2258070520586604]}}, "input": {"cat_bwa": {"distribution": null, "min": 2, "max": 2}, ".sam": {"distribution": {"name": "trapezoid", "params": [1.0, 1.0, -0.10500000000000001, 1.2]}, "min": 516, "max": 52088}}, "output": {".sam": {"distribution": {"name": "argus", "params": [1.599540430972508, -0.5837001726177105, 1.7033104585411558]}, "min": 3437, "max": 353943}}}, "cat": {"runtime": {"min": 0.014307, "max": 0.073623, "distribution": {"name": "argus", "params": [1.599540430972508, -0.5837001726177105, 1.7033104585411558]}}, "input": {".err": {"distribution": null, "min": 1, "max": 1}}, "output": {".err": {"distribution": {"name": "argus", "params": [1.599540430972508, -0.5837001726177105, 1.7033104585411558]}, "min": 14, "max": 133}}}} \ No newline at end of file diff --git a/wfcommons/wfchef/recipes/cycles/task_type_stats.json b/wfcommons/wfchef/recipes/cycles/task_type_stats.json index cdfdbc30..4048f66e 100644 --- a/wfcommons/wfchef/recipes/cycles/task_type_stats.json +++ b/wfcommons/wfchef/recipes/cycles/task_type_stats.json @@ -1 +1 @@ -{"baseline_cycles": {"runtime": {"min": 1.597, "max": 110.598, "distribution": {"name": "argus", "params": [5.734652014997823e-05, -0.6805728508716575, 1.6811901314620572]}}, "input": {".soil": {"distribution": null, "min": 739, "max": 739}, ".weather": {"distribution": {"name": "trapz", "params": [0.9999999999999999, 1.0, -0.10500000000000001, 1.1999999999999997]}, "min": 480096, "max": 480097}, ".operation": {"distribution": {"name": "rdist", "params": [1.750426650564894, 0.9999999999999998, 2.3624681987884834e-16]}, "min": 673, "max": 1806}, ".ctrl": {"distribution": null, "min": 766, "max": 766}, "cycles_exe": {"distribution": null, "min": 694648, "max": 694648}, ".crop": {"distribution": null, "min": 14434, "max": 14434}}, "output": {".dat": {"distribution": {"name": "rdist", "params": [1.750426650564894, 0.9999999999999998, 2.3624681987884834e-16]}, "min": 1056, "max": 1808675}, ".csv": {"distribution": {"name": "trapz", "params": [1.0, 1.0, -0.10500000000000001, 1.2]}, "min": 275, "max": 281}, ".zip": {"distribution": {"name": "argus", "params": [4.427079012898356e-05, -0.7120439234477358, 1.712710243275252]}, "min": 2267492, "max": 4458415}}}, "cycles": {"runtime": {"min": 1.277, "max": 101.596, "distribution": {"name": "argus", "params": [2.4601472008810825e-05, -0.760671189673141, 1.761325389492184]}}, "input": {".soil": {"distribution": null, "min": 739, "max": 739}, ".weather": {"distribution": {"name": "trapz", "params": [0.9999999999999999, 1.0, -0.10500000000000001, 1.1999999999999997]}, "min": 480096, "max": 480097}, ".operation": {"distribution": {"name": "rdist", "params": [1.750426650564894, 0.9999999999999998, 2.3624681987884834e-16]}, "min": 673, "max": 1806}, ".dat": {"distribution": null, "min": 37728, "max": 37728}, ".ctrl": {"distribution": null, "min": 766, "max": 766}, "cycles_exe": {"distribution": null, "min": 694648, "max": 694648}, ".crop": {"distribution": null, "min": 14434, "max": 14434}}, "output": {".zip": {"distribution": {"name": "argus", "params": [3.76005481127054e-05, -0.6638469786758867, 1.6644768181414857]}, "min": 2928100, "max": 4476265}, ".dat": {"distribution": {"name": "rdist", "params": [1.750426650564894, 0.9999999999999998, 2.3624681987884834e-16]}, "min": 1056, "max": 1808675}, ".csv": {"distribution": {"name": "trapz", "params": [1.0, 1.0, -0.10500000000000001, 1.2]}, "min": 275, "max": 281}}}, "fertilizer_increase_cycles": {"runtime": {"min": 1.258, "max": 101.229, "distribution": {"name": "argus", "params": [0.00024090845979553212, -0.7571715014318916, 1.757837187249918]}}, "input": {".soil": {"distribution": null, "min": 739, "max": 739}, ".weather": {"distribution": {"name": "trapz", "params": [0.9999999999999999, 1.0, -0.10500000000000001, 1.1999999999999997]}, "min": 480096, "max": 480097}, ".operation": {"distribution": {"name": "rdist", "params": [1.750426650564894, 0.9999999999999998, 2.3624681987884834e-16]}, "min": 673, "max": 1806}, ".dat": {"distribution": null, "min": 37728, "max": 37728}, ".ctrl": {"distribution": null, "min": 766, "max": 766}, "cycles_exe": {"distribution": null, "min": 694648, "max": 694648}, ".crop": {"distribution": null, "min": 14434, "max": 14434}}, "output": {".csv": {"distribution": {"name": "trapz", "params": [1.0, 1.0, -0.10500000000000001, 1.2]}, "min": 274, "max": 282}, ".dat": {"distribution": {"name": "rdist", "params": [1.750426650564894, 0.9999999999999998, 2.3624681987884834e-16]}, "min": 1056, "max": 1808675}, ".zip": {"distribution": {"name": "argus", "params": [5.012766959293754e-05, -0.6684439199051191, 1.6690337988564448]}, "min": 2928677, "max": 4486752}}}, "cycles_fertilizer_increase_output_parser": {"runtime": {"min": 0.035, "max": 1.235, "distribution": {"name": "trapz", "params": [1.0, 1.0, -0.10500000000000001, 1.2]}}, "input": {".dat": {"distribution": null, "min": 5980, "max": 5980}, ".csv": {"distribution": {"name": "trapz", "params": [1.0, 1.0, -0.10500000000000001, 1.2]}, "min": 274, "max": 282}}, "output": {".csv": {"distribution": {"name": "argus", "params": [0.0002218467867105605, -0.7480684549751317, 1.7487201180830585]}, "min": 8224, "max": 8604}}}, "cycles_output_summary": {"runtime": {"min": 0.048, "max": 0.39, "distribution": {"name": "norm", "params": [0.08688656476267097, 0.2572832376513094]}}, "input": {".csv": {"distribution": {"name": "trapz", "params": [1.0, 1.0, -0.10500000000000001, 1.2]}, "min": 275, "max": 281}, ".dat": {"distribution": null, "min": 5980, "max": 5980}}, "output": {".csv": {"distribution": {"name": "rdist", "params": [1.7504259613885509, 1.0, 4.589985375327013e-23]}, "min": 74996, "max": 257366}}}, "cycles_fertilizer_increase_output_summary": {"runtime": {"min": 0.045, "max": 0.538, "distribution": {"name": "argus", "params": [0.0001699057370632583, -0.703883277161877, 1.7272982844402591]}}, "input": {".csv": {"distribution": {"name": "argus", "params": [0.0002218467867105605, -0.7480684549751317, 1.7487201180830585]}, "min": 8224, "max": 8604}}, "output": {".csv": {"distribution": {"name": "rdist", "params": [1.7504259613885509, 1.0, 4.589985375327013e-23]}, "min": 122964, "max": 422954}}}, "cycles_plots": {"runtime": {"min": 123.38, "max": 514.566, "distribution": {"name": "beta", "params": [0.11165228211118816, 0.32785735039410685, -0.09952333240693918, 1.0995233324069393]}}, "input": {".csv": {"distribution": {"name": "rdist", "params": [1.7504259613885509, 1.0, 4.589985375327013e-23]}, "min": 74996, "max": 257366}}, "output": {".gif": {"distribution": {"name": "norm", "params": [0.35, 0.36784847423905404]}, "min": 2126304, "max": 6383929}}}} \ No newline at end of file +{"baseline_cycles": {"runtime": {"min": 1.597, "max": 110.598, "distribution": {"name": "argus", "params": [5.734652014997823e-05, -0.6805728508716575, 1.6811901314620572]}}, "input": {".soil": {"distribution": null, "min": 739, "max": 739}, ".weather": {"distribution": {"name": "trapezoid", "params": [0.9999999999999999, 1.0, -0.10500000000000001, 1.1999999999999997]}, "min": 480096, "max": 480097}, ".operation": {"distribution": {"name": "rdist", "params": [1.750426650564894, 0.9999999999999998, 2.3624681987884834e-16]}, "min": 673, "max": 1806}, ".ctrl": {"distribution": null, "min": 766, "max": 766}, "cycles_exe": {"distribution": null, "min": 694648, "max": 694648}, ".crop": {"distribution": null, "min": 14434, "max": 14434}}, "output": {".dat": {"distribution": {"name": "rdist", "params": [1.750426650564894, 0.9999999999999998, 2.3624681987884834e-16]}, "min": 1056, "max": 1808675}, ".csv": {"distribution": {"name": "trapezoid", "params": [1.0, 1.0, -0.10500000000000001, 1.2]}, "min": 275, "max": 281}, ".zip": {"distribution": {"name": "argus", "params": [4.427079012898356e-05, -0.7120439234477358, 1.712710243275252]}, "min": 2267492, "max": 4458415}}}, "cycles": {"runtime": {"min": 1.277, "max": 101.596, "distribution": {"name": "argus", "params": [2.4601472008810825e-05, -0.760671189673141, 1.761325389492184]}}, "input": {".soil": {"distribution": null, "min": 739, "max": 739}, ".weather": {"distribution": {"name": "trapezoid", "params": [0.9999999999999999, 1.0, -0.10500000000000001, 1.1999999999999997]}, "min": 480096, "max": 480097}, ".operation": {"distribution": {"name": "rdist", "params": [1.750426650564894, 0.9999999999999998, 2.3624681987884834e-16]}, "min": 673, "max": 1806}, ".dat": {"distribution": null, "min": 37728, "max": 37728}, ".ctrl": {"distribution": null, "min": 766, "max": 766}, "cycles_exe": {"distribution": null, "min": 694648, "max": 694648}, ".crop": {"distribution": null, "min": 14434, "max": 14434}}, "output": {".zip": {"distribution": {"name": "argus", "params": [3.76005481127054e-05, -0.6638469786758867, 1.6644768181414857]}, "min": 2928100, "max": 4476265}, ".dat": {"distribution": {"name": "rdist", "params": [1.750426650564894, 0.9999999999999998, 2.3624681987884834e-16]}, "min": 1056, "max": 1808675}, ".csv": {"distribution": {"name": "trapezoid", "params": [1.0, 1.0, -0.10500000000000001, 1.2]}, "min": 275, "max": 281}}}, "fertilizer_increase_cycles": {"runtime": {"min": 1.258, "max": 101.229, "distribution": {"name": "argus", "params": [0.00024090845979553212, -0.7571715014318916, 1.757837187249918]}}, "input": {".soil": {"distribution": null, "min": 739, "max": 739}, ".weather": {"distribution": {"name": "trapezoid", "params": [0.9999999999999999, 1.0, -0.10500000000000001, 1.1999999999999997]}, "min": 480096, "max": 480097}, ".operation": {"distribution": {"name": "rdist", "params": [1.750426650564894, 0.9999999999999998, 2.3624681987884834e-16]}, "min": 673, "max": 1806}, ".dat": {"distribution": null, "min": 37728, "max": 37728}, ".ctrl": {"distribution": null, "min": 766, "max": 766}, "cycles_exe": {"distribution": null, "min": 694648, "max": 694648}, ".crop": {"distribution": null, "min": 14434, "max": 14434}}, "output": {".csv": {"distribution": {"name": "trapezoid", "params": [1.0, 1.0, -0.10500000000000001, 1.2]}, "min": 274, "max": 282}, ".dat": {"distribution": {"name": "rdist", "params": [1.750426650564894, 0.9999999999999998, 2.3624681987884834e-16]}, "min": 1056, "max": 1808675}, ".zip": {"distribution": {"name": "argus", "params": [5.012766959293754e-05, -0.6684439199051191, 1.6690337988564448]}, "min": 2928677, "max": 4486752}}}, "cycles_fertilizer_increase_output_parser": {"runtime": {"min": 0.035, "max": 1.235, "distribution": {"name": "trapezoid", "params": [1.0, 1.0, -0.10500000000000001, 1.2]}}, "input": {".dat": {"distribution": null, "min": 5980, "max": 5980}, ".csv": {"distribution": {"name": "trapezoid", "params": [1.0, 1.0, -0.10500000000000001, 1.2]}, "min": 274, "max": 282}}, "output": {".csv": {"distribution": {"name": "argus", "params": [0.0002218467867105605, -0.7480684549751317, 1.7487201180830585]}, "min": 8224, "max": 8604}}}, "cycles_output_summary": {"runtime": {"min": 0.048, "max": 0.39, "distribution": {"name": "norm", "params": [0.08688656476267097, 0.2572832376513094]}}, "input": {".csv": {"distribution": {"name": "trapezoid", "params": [1.0, 1.0, -0.10500000000000001, 1.2]}, "min": 275, "max": 281}, ".dat": {"distribution": null, "min": 5980, "max": 5980}}, "output": {".csv": {"distribution": {"name": "rdist", "params": [1.7504259613885509, 1.0, 4.589985375327013e-23]}, "min": 74996, "max": 257366}}}, "cycles_fertilizer_increase_output_summary": {"runtime": {"min": 0.045, "max": 0.538, "distribution": {"name": "argus", "params": [0.0001699057370632583, -0.703883277161877, 1.7272982844402591]}}, "input": {".csv": {"distribution": {"name": "argus", "params": [0.0002218467867105605, -0.7480684549751317, 1.7487201180830585]}, "min": 8224, "max": 8604}}, "output": {".csv": {"distribution": {"name": "rdist", "params": [1.7504259613885509, 1.0, 4.589985375327013e-23]}, "min": 122964, "max": 422954}}}, "cycles_plots": {"runtime": {"min": 123.38, "max": 514.566, "distribution": {"name": "beta", "params": [0.11165228211118816, 0.32785735039410685, -0.09952333240693918, 1.0995233324069393]}}, "input": {".csv": {"distribution": {"name": "rdist", "params": [1.7504259613885509, 1.0, 4.589985375327013e-23]}, "min": 74996, "max": 257366}}, "output": {".gif": {"distribution": {"name": "norm", "params": [0.35, 0.36784847423905404]}, "min": 2126304, "max": 6383929}}}} \ No newline at end of file diff --git a/wfcommons/wfchef/recipes/epigenomics/task_type_stats.json b/wfcommons/wfchef/recipes/epigenomics/task_type_stats.json index 7d65df7b..13a7fc2a 100644 --- a/wfcommons/wfchef/recipes/epigenomics/task_type_stats.json +++ b/wfcommons/wfchef/recipes/epigenomics/task_type_stats.json @@ -74,7 +74,7 @@ }, ".fq": { "distribution": { - "name": "trapz", + "name": "trapezoid", "params": [ 1.0, 1.0, @@ -132,7 +132,7 @@ "output": { ".sfq": { "distribution": { - "name": "trapz", + "name": "trapezoid", "params": [ 1.0, 1.0, @@ -161,7 +161,7 @@ "input": { ".sfq": { "distribution": { - "name": "trapz", + "name": "trapezoid", "params": [ 1.0, 1.0, @@ -213,7 +213,7 @@ }, ".map": { "distribution": { - "name": "trapz", + "name": "trapezoid", "params": [ 1.0, 1.0, @@ -285,7 +285,7 @@ "output": { ".map": { "distribution": { - "name": "trapz", + "name": "trapezoid", "params": [ 1.0, 1.0, @@ -393,7 +393,7 @@ "output": { ".fq": { "distribution": { - "name": "trapz", + "name": "trapezoid", "params": [ 1.0, 1.0, diff --git a/wfcommons/wfchef/recipes/genome/task_type_stats.json b/wfcommons/wfchef/recipes/genome/task_type_stats.json index 961412ba..35f40454 100644 --- a/wfcommons/wfchef/recipes/genome/task_type_stats.json +++ b/wfcommons/wfchef/recipes/genome/task_type_stats.json @@ -1 +1 @@ -{"individuals": {"runtime": {"min": 48.846, "max": 192.232, "distribution": {"name": "argus", "params": [5.195366960167635e-05, -0.6923240599242786, 1.6948569025371873]}}, "input": {".txt": {"distribution": null, "min": 20078, "max": 20078}, ".vcf": {"distribution": {"name": "trapz", "params": [0.9999999999999999, 1.0, -0.10500000000000001, 1.1999999999999997]}, "min": 1014213207, "max": 2540066220}}, "output": {".gz": {"distribution": {"name": "argus", "params": [0.0024394427876728523, -0.411542131610374, 1.4125981096726774]}, "min": 27275, "max": 29311}}}, "individuals_merge": {"runtime": {"min": 34.471, "max": 157.346, "distribution": {"name": "rdist", "params": [1.0381380525174126, 0.8856734445741212, 0.6098113756086041]}}, "input": {".gz": {"distribution": {"name": "argus", "params": [0.0024394427876728523, -0.411542131610374, 1.4125981096726774]}, "min": 27275, "max": 29311}}, "output": {".gz": {"distribution": {"name": "norm", "params": [0.1810699588477366, 0.26861113455666213]}, "min": 24180, "max": 27336}}}, "sifting": {"runtime": {"min": 0.293, "max": 22.686, "distribution": {"name": "norm", "params": [0.20370370370370366, 0.2753559549391278]}}, "input": {".vcf": {"distribution": {"name": "trapz", "params": [1.0, 1.0, -0.10500000000000001, 1.2]}, "min": 267806263, "max": 1708156254}}, "output": {".txt": {"distribution": {"name": "dweibull", "params": [1.727468520735635, 0.1982279794020313, 0.29839665578862673]}, "min": 231958, "max": 2126612}}}, "mutation_overlap": {"runtime": {"min": 1.61, "max": 92.154, "distribution": {"name": "argus", "params": [0.00026192730594807505, -0.7075548744414277, 1.7102271901418185]}}, "input": {".gz": {"distribution": {"name": "argus", "params": [0.0016194733607993123, -0.6111579247282823, 1.613914274425801]}, "min": 24180, "max": 27336}, ".txt": {"distribution": {"name": "rdist", "params": [1.750426650564894, 0.9999999999999998, 2.3624681987884834e-16]}, "min": 20078, "max": 2126612}, "AFR": {"distribution": null, "min": 8088, "max": 8088}, "GBR": {"distribution": null, "min": 856, "max": 856}, "ALL": {"distribution": null, "min": 28000, "max": 28000}, "SAS": {"distribution": null, "min": 5248, "max": 5248}, "EAS": {"distribution": null, "min": 4896, "max": 4896}, "AMR": {"distribution": null, "min": 4248, "max": 4248}, "EUR": {"distribution": null, "min": 5312, "max": 5312}}, "output": {".gz": {"distribution": {"name": "trapz", "params": [1.0, 1.0, -0.10500000000000001, 1.2]}, "min": 136896, "max": 172605}}}, "frequency": {"runtime": {"min": 76.023, "max": 238.26, "distribution": {"name": "dweibull", "params": [1.3710340206922134, 0.1939059617308234, 0.23184013931258757]}}, "input": {".gz": {"distribution": {"name": "argus", "params": [0.0016194733607993123, -0.6111579247282823, 1.613914274425801]}, "min": 24180, "max": 27336}, ".txt": {"distribution": {"name": "rdist", "params": [1.750426650564894, 0.9999999999999998, 2.3624681987884834e-16]}, "min": 20078, "max": 2126612}, "AFR": {"distribution": null, "min": 8088, "max": 8088}, "GBR": {"distribution": null, "min": 856, "max": 856}, "ALL": {"distribution": null, "min": 28000, "max": 28000}, "SAS": {"distribution": null, "min": 5248, "max": 5248}, "EAS": {"distribution": null, "min": 4896, "max": 4896}, "AMR": {"distribution": null, "min": 4248, "max": 4248}, "EUR": {"distribution": null, "min": 5312, "max": 5312}}, "output": {".gz": {"distribution": {"name": "argus", "params": [0.0013405464361447438, -0.5587070759363937, 1.561653731136413]}, "min": 217409, "max": 298487}}}} \ No newline at end of file +{"individuals": {"runtime": {"min": 48.846, "max": 192.232, "distribution": {"name": "argus", "params": [5.195366960167635e-05, -0.6923240599242786, 1.6948569025371873]}}, "input": {".txt": {"distribution": null, "min": 20078, "max": 20078}, ".vcf": {"distribution": {"name": "trapezoid", "params": [0.9999999999999999, 1.0, -0.10500000000000001, 1.1999999999999997]}, "min": 1014213207, "max": 2540066220}}, "output": {".gz": {"distribution": {"name": "argus", "params": [0.0024394427876728523, -0.411542131610374, 1.4125981096726774]}, "min": 27275, "max": 29311}}}, "individuals_merge": {"runtime": {"min": 34.471, "max": 157.346, "distribution": {"name": "rdist", "params": [1.0381380525174126, 0.8856734445741212, 0.6098113756086041]}}, "input": {".gz": {"distribution": {"name": "argus", "params": [0.0024394427876728523, -0.411542131610374, 1.4125981096726774]}, "min": 27275, "max": 29311}}, "output": {".gz": {"distribution": {"name": "norm", "params": [0.1810699588477366, 0.26861113455666213]}, "min": 24180, "max": 27336}}}, "sifting": {"runtime": {"min": 0.293, "max": 22.686, "distribution": {"name": "norm", "params": [0.20370370370370366, 0.2753559549391278]}}, "input": {".vcf": {"distribution": {"name": "trapezoid", "params": [1.0, 1.0, -0.10500000000000001, 1.2]}, "min": 267806263, "max": 1708156254}}, "output": {".txt": {"distribution": {"name": "dweibull", "params": [1.727468520735635, 0.1982279794020313, 0.29839665578862673]}, "min": 231958, "max": 2126612}}}, "mutation_overlap": {"runtime": {"min": 1.61, "max": 92.154, "distribution": {"name": "argus", "params": [0.00026192730594807505, -0.7075548744414277, 1.7102271901418185]}}, "input": {".gz": {"distribution": {"name": "argus", "params": [0.0016194733607993123, -0.6111579247282823, 1.613914274425801]}, "min": 24180, "max": 27336}, ".txt": {"distribution": {"name": "rdist", "params": [1.750426650564894, 0.9999999999999998, 2.3624681987884834e-16]}, "min": 20078, "max": 2126612}, "AFR": {"distribution": null, "min": 8088, "max": 8088}, "GBR": {"distribution": null, "min": 856, "max": 856}, "ALL": {"distribution": null, "min": 28000, "max": 28000}, "SAS": {"distribution": null, "min": 5248, "max": 5248}, "EAS": {"distribution": null, "min": 4896, "max": 4896}, "AMR": {"distribution": null, "min": 4248, "max": 4248}, "EUR": {"distribution": null, "min": 5312, "max": 5312}}, "output": {".gz": {"distribution": {"name": "trapezoid", "params": [1.0, 1.0, -0.10500000000000001, 1.2]}, "min": 136896, "max": 172605}}}, "frequency": {"runtime": {"min": 76.023, "max": 238.26, "distribution": {"name": "dweibull", "params": [1.3710340206922134, 0.1939059617308234, 0.23184013931258757]}}, "input": {".gz": {"distribution": {"name": "argus", "params": [0.0016194733607993123, -0.6111579247282823, 1.613914274425801]}, "min": 24180, "max": 27336}, ".txt": {"distribution": {"name": "rdist", "params": [1.750426650564894, 0.9999999999999998, 2.3624681987884834e-16]}, "min": 20078, "max": 2126612}, "AFR": {"distribution": null, "min": 8088, "max": 8088}, "GBR": {"distribution": null, "min": 856, "max": 856}, "ALL": {"distribution": null, "min": 28000, "max": 28000}, "SAS": {"distribution": null, "min": 5248, "max": 5248}, "EAS": {"distribution": null, "min": 4896, "max": 4896}, "AMR": {"distribution": null, "min": 4248, "max": 4248}, "EUR": {"distribution": null, "min": 5312, "max": 5312}}, "output": {".gz": {"distribution": {"name": "argus", "params": [0.0013405464361447438, -0.5587070759363937, 1.561653731136413]}, "min": 217409, "max": 298487}}}} \ No newline at end of file diff --git a/wfcommons/wfchef/recipes/montage/task_type_stats.json b/wfcommons/wfchef/recipes/montage/task_type_stats.json index 8d38ac27..eb708d6a 100644 --- a/wfcommons/wfchef/recipes/montage/task_type_stats.json +++ b/wfcommons/wfchef/recipes/montage/task_type_stats.json @@ -15,7 +15,7 @@ "input": { ".fits": { "distribution": { - "name": "trapz", + "name": "trapezoid", "params": [ 1.0, 1.0, @@ -28,7 +28,7 @@ }, ".hdr": { "distribution": { - "name": "trapz", + "name": "trapezoid", "params": [ 1.0, 1.0, @@ -43,7 +43,7 @@ "output": { ".fits": { "distribution": { - "name": "trapz", + "name": "trapezoid", "params": [ 1.0, 1.0, @@ -72,7 +72,7 @@ "input": { ".fits": { "distribution": { - "name": "trapz", + "name": "trapezoid", "params": [ 0.9999999999999999, 1.0, @@ -85,7 +85,7 @@ }, ".hdr": { "distribution": { - "name": "trapz", + "name": "trapezoid", "params": [ 1.0, 1.0, @@ -225,7 +225,7 @@ "input": { ".tbl": { "distribution": { - "name": "trapz", + "name": "trapezoid", "params": [ 1.0, 1.0, @@ -238,7 +238,7 @@ }, ".fits": { "distribution": { - "name": "trapz", + "name": "trapezoid", "params": [ 1.0, 1.0, @@ -349,7 +349,7 @@ }, ".hdr": { "distribution": { - "name": "trapz", + "name": "trapezoid", "params": [ 1.0, 1.0, diff --git a/wfcommons/wfchef/recipes/seismology/task_type_stats.json b/wfcommons/wfchef/recipes/seismology/task_type_stats.json index f974592e..66722eed 100644 --- a/wfcommons/wfchef/recipes/seismology/task_type_stats.json +++ b/wfcommons/wfchef/recipes/seismology/task_type_stats.json @@ -1 +1 @@ -{"sG1IterDecon": {"runtime": {"min": 0.087, "max": 5.615, "distribution": {"name": "argus", "params": [1.7064100735232294e-05, -0.7177573513504376, 1.7186300350031756]}}, "input": {".lht": {"distribution": {"name": "argus", "params": [0.04752143141481561, -0.6877501751085423, 1.6881901722981807]}, "min": 1024, "max": 16012}}, "output": {".stf": {"distribution": {"name": "trapz", "params": [1.0, 1.0, -0.10500000000000001, 1.2]}, "min": 1144, "max": 17016}}}, "wrapper_siftSTFByMisfit": {"runtime": {"min": 0.089, "max": 1.351, "distribution": {"name": "rdist", "params": [1.5504806356651624, 0.0013236200991527764, 0.0013236200991527767]}}, "input": {".stf": {"distribution": {"name": "trapz", "params": [1.0, 1.0, -0.10500000000000001, 1.2]}, "min": 1144, "max": 17016}, ".py": {"distribution": null, "min": 0, "max": 0}, "siftSTFByMisfit": {"distribution": null, "min": 1386, "max": 1386}}, "output": {".gz": {"distribution": {"name": "rdist", "params": [1.5504806356651624, 0.0013236200991527764, 0.0013236200991527767]}, "min": 63471, "max": 687098}}}} \ No newline at end of file +{"sG1IterDecon": {"runtime": {"min": 0.087, "max": 5.615, "distribution": {"name": "argus", "params": [1.7064100735232294e-05, -0.7177573513504376, 1.7186300350031756]}}, "input": {".lht": {"distribution": {"name": "argus", "params": [0.04752143141481561, -0.6877501751085423, 1.6881901722981807]}, "min": 1024, "max": 16012}}, "output": {".stf": {"distribution": {"name": "trapezoid", "params": [1.0, 1.0, -0.10500000000000001, 1.2]}, "min": 1144, "max": 17016}}}, "wrapper_siftSTFByMisfit": {"runtime": {"min": 0.089, "max": 1.351, "distribution": {"name": "rdist", "params": [1.5504806356651624, 0.0013236200991527764, 0.0013236200991527767]}}, "input": {".stf": {"distribution": {"name": "trapezoid", "params": [1.0, 1.0, -0.10500000000000001, 1.2]}, "min": 1144, "max": 17016}, ".py": {"distribution": null, "min": 0, "max": 0}, "siftSTFByMisfit": {"distribution": null, "min": 1386, "max": 1386}}, "output": {".gz": {"distribution": {"name": "rdist", "params": [1.5504806356651624, 0.0013236200991527764, 0.0013236200991527767]}, "min": 63471, "max": 687098}}}} \ No newline at end of file diff --git a/wfcommons/wfchef/recipes/soykb/task_type_stats.json b/wfcommons/wfchef/recipes/soykb/task_type_stats.json index 7e36ca2c..6c046429 100644 --- a/wfcommons/wfchef/recipes/soykb/task_type_stats.json +++ b/wfcommons/wfchef/recipes/soykb/task_type_stats.json @@ -1 +1 @@ -{"alignment_to_reference": {"runtime": {"min": 3.305, "max": 133.207, "distribution": {"name": "rdist", "params": [1.0497506699055814, 0.8751735665957343, 0.5778762692984372]}}, "input": {".dict": {"distribution": null, "min": 142561, "max": 142561}, ".pac": {"distribution": null, "min": 244623820, "max": 244623820}, ".fai": {"distribution": null, "min": 41379, "max": 41379}, ".fastq": {"distribution": null, "min": 23845, "max": 23845}, ".sa": {"distribution": null, "min": 489247688, "max": 489247688}, ".gz": {"distribution": null, "min": 108988506, "max": 108988506}, ".amb": {"distribution": null, "min": 259156, "max": 259156}, ".fa": {"distribution": null, "min": 990744229, "max": 990744229}, ".ann": {"distribution": null, "min": 47448, "max": 47448}, ".bwt": {"distribution": null, "min": 978495356, "max": 978495356}, ".txt": {"distribution": {"name": "rdist", "params": [1.7477959990252214, 1.0, 3.9893803094996296e-22]}, "min": 60, "max": 15976}}, "output": {".sam": {"distribution": null, "min": 112561, "max": 112561}}}, "sort_sam": {"runtime": {"min": 2.082, "max": 89.242, "distribution": {"name": "arcsine", "params": [-0.09850603366048286, 1.098506033660483]}}, "input": {".sam": {"distribution": null, "min": 112561, "max": 112561}, ".gz": {"distribution": null, "min": 108988506, "max": 108988506}}, "output": {".bai": {"distribution": null, "min": 127424, "max": 127424}, ".bam": {"distribution": null, "min": 36576, "max": 36576}}}, "dedup": {"runtime": {"min": 3.169, "max": 152.024, "distribution": {"name": "rdist", "params": [1.056503552352268, 0.8880507079714015, 0.5907534106741043]}}, "input": {".gz": {"distribution": null, "min": 108988506, "max": 108988506}, ".bai": {"distribution": null, "min": 127424, "max": 127424}, ".bam": {"distribution": null, "min": 36576, "max": 36576}}, "output": {".bai": {"distribution": null, "min": 127424, "max": 127424}, ".bam": {"distribution": {"name": "rdist", "params": [1.7504259269384672, 0.9999999999938114, 6.18860928332036e-12]}, "min": 37467, "max": 37470}}}, "add_replace": {"runtime": {"min": 3.382, "max": 139.801, "distribution": {"name": "arcsine", "params": [-0.09850603366048286, 1.098506033660483]}}, "input": {".gz": {"distribution": null, "min": 108988506, "max": 108988506}, ".bai": {"distribution": null, "min": 127424, "max": 127424}, ".bam": {"distribution": {"name": "rdist", "params": [1.7504259269384672, 0.9999999999938114, 6.18860928332036e-12]}, "min": 37467, "max": 37470}}, "output": {".bai": {"distribution": null, "min": 127424, "max": 127424}, ".bam": {"distribution": {"name": "beta", "params": [0.028007929445023696, 0.16801064212397998, -0.08122143521923508, 1.0812214352192353]}, "min": 37953, "max": 37956}}}, "realign_target_creator": {"runtime": {"min": 78.698, "max": 474.546, "distribution": {"name": "arcsine", "params": [-0.08648046099196244, 1.0864804609919627]}}, "input": {".bai": {"distribution": null, "min": 127424, "max": 127424}, ".dict": {"distribution": null, "min": 142561, "max": 142561}, ".pac": {"distribution": null, "min": 244623820, "max": 244623820}, ".fai": {"distribution": null, "min": 41379, "max": 41379}, ".sa": {"distribution": null, "min": 489247688, "max": 489247688}, ".gz": {"distribution": null, "min": 108988506, "max": 108988506}, ".amb": {"distribution": null, "min": 259156, "max": 259156}, ".fa": {"distribution": null, "min": 990744229, "max": 990744229}, ".ann": {"distribution": null, "min": 47448, "max": 47448}, ".bam": {"distribution": {"name": "beta", "params": [0.028007929445023696, 0.16801064212397998, -0.08122143521923508, 1.0812214352192353]}, "min": 37953, "max": 37956}, ".bwt": {"distribution": null, "min": 978495356, "max": 978495356}, ".txt": {"distribution": {"name": "rdist", "params": [1.7477959990252214, 1.0, 3.9893803094996296e-22]}, "min": 60, "max": 15976}}, "output": {".list": {"distribution": null, "min": 40, "max": 40}}}, "indel_realign": {"runtime": {"min": 5.589, "max": 154.005, "distribution": {"name": "rdist", "params": [1.054143443775926, 0.8779716509043156, 0.6087408816735466]}}, "input": {".bai": {"distribution": null, "min": 127424, "max": 127424}, ".dict": {"distribution": null, "min": 142561, "max": 142561}, ".pac": {"distribution": null, "min": 244623820, "max": 244623820}, ".fai": {"distribution": null, "min": 41379, "max": 41379}, ".list": {"distribution": null, "min": 40, "max": 40}, ".sa": {"distribution": null, "min": 489247688, "max": 489247688}, ".gz": {"distribution": null, "min": 108988506, "max": 108988506}, ".amb": {"distribution": null, "min": 259156, "max": 259156}, ".fa": {"distribution": null, "min": 990744229, "max": 990744229}, ".ann": {"distribution": null, "min": 47448, "max": 47448}, ".bam": {"distribution": {"name": "beta", "params": [0.028007929445023696, 0.16801064212397998, -0.08122143521923508, 1.0812214352192353]}, "min": 37953, "max": 37956}, ".bwt": {"distribution": null, "min": 978495356, "max": 978495356}, ".txt": {"distribution": {"name": "rdist", "params": [1.7477959990252214, 1.0, 3.9893803094996296e-22]}, "min": 60, "max": 15976}}, "output": {".bam": {"distribution": {"name": "rdist", "params": [1.0354552049234393, 0.9230820838539932, 0.5764154171873266]}, "min": 39550, "max": 39554}, ".bai": {"distribution": null, "min": 127424, "max": 127424}}}, "haplotype_caller": {"runtime": {"min": 30.011, "max": 709.504, "distribution": {"name": "argus", "params": [0.0008690793369530592, -0.6114566529512571, 1.6139824159282967]}}, "input": {".dict": {"distribution": null, "min": 142561, "max": 142561}, ".bam": {"distribution": {"name": "trapz", "params": [0.9999999999999999, 1.0, -0.10500000000000001, 1.1999999999999997]}, "min": 39550, "max": 39554}, ".pac": {"distribution": null, "min": 244623820, "max": 244623820}, ".bai": {"distribution": null, "min": 127424, "max": 127424}, ".fai": {"distribution": null, "min": 41379, "max": 41379}, ".sa": {"distribution": null, "min": 489247688, "max": 489247688}, ".gz": {"distribution": null, "min": 108988506, "max": 108988506}, ".amb": {"distribution": null, "min": 259156, "max": 259156}, ".fa": {"distribution": null, "min": 990744229, "max": 990744229}, ".ann": {"distribution": null, "min": 47448, "max": 47448}, ".bwt": {"distribution": null, "min": 978495356, "max": 978495356}, ".txt": {"distribution": {"name": "trapz", "params": [0.9999999999999999, 1.0, -0.10500000000000001, 1.1999999999999997]}, "min": 60, "max": 15976}}, "output": {".vcf": {"distribution": {"name": "rdist", "params": [1.750426650564894, 0.9999999999999998, 2.3624681987884834e-16]}, "min": 59037, "max": 75728}, ".idx": {"distribution": {"name": "trapz", "params": [0.9999999999999999, 1.0, -0.10500000000000001, 1.1999999999999997]}, "min": 28447, "max": 28449}}}, "merge_gcvf": {"runtime": {"min": 2562.381, "max": 37861.356, "distribution": {"name": "rdist", "params": [1.5504806356651624, 0.0013236200991527764, 0.0013236200991527767]}}, "input": {".vcf": {"distribution": {"name": "rdist", "params": [1.750426650564894, 0.9999999999999998, 2.3624681987884834e-16]}, "min": 59037, "max": 75728}, ".idx": {"distribution": {"name": "trapz", "params": [0.9999999999999999, 1.0, -0.10500000000000001, 1.1999999999999997]}, "min": 28447, "max": 28449}, ".gz": {"distribution": null, "min": 108988506, "max": 108988506}, ".bwt": {"distribution": null, "min": 978495356, "max": 978495356}, ".txt": {"distribution": {"name": "rdist", "params": [1.5504806356651624, 0.0013236200991527764, 0.0013236200991527767]}, "min": 60, "max": 15976}, ".list": {"distribution": {"name": "rdist", "params": [1.5504806356651624, 0.0013236200991527764, 0.0013236200991527767]}, "min": 1700, "max": 17000}, ".amb": {"distribution": null, "min": 259156, "max": 259156}, ".fai": {"distribution": null, "min": 41379, "max": 41379}, ".pac": {"distribution": null, "min": 244623820, "max": 244623820}, ".fa": {"distribution": null, "min": 990744229, "max": 990744229}, ".dict": {"distribution": null, "min": 142561, "max": 142561}, ".ann": {"distribution": null, "min": 47448, "max": 47448}, ".sa": {"distribution": null, "min": 489247688, "max": 489247688}}, "output": {".idx": {"distribution": {"name": "rdist", "params": [1.5504806356651624, 0.0013236200991527764, 0.0013236200991527767]}, "min": 28886, "max": 29187}, ".vcf": {"distribution": {"name": "rdist", "params": [1.5504806356651624, 0.0013236200991527764, 0.0013236200991527767]}, "min": 104389, "max": 250300}}}, "genotype_gvcfs": {"runtime": {"min": 95.618, "max": 899.232, "distribution": {"name": "arcsine", "params": [-0.09850603366048286, 1.098506033660483]}}, "input": {".vcf": {"distribution": {"name": "rdist", "params": [1.750426650564894, 0.9999999999999998, 2.3624681987884834e-16]}, "min": 59037, "max": 75728}, ".idx": {"distribution": {"name": "trapz", "params": [0.9999999999999999, 1.0, -0.10500000000000001, 1.1999999999999997]}, "min": 28447, "max": 28449}, ".txt": {"distribution": {"name": "trapz", "params": [0.9999999999999999, 1.0, -0.10500000000000001, 1.1999999999999997]}, "min": 60, "max": 15976}, ".dict": {"distribution": null, "min": 142561, "max": 142561}, ".amb": {"distribution": null, "min": 259156, "max": 259156}, ".ann": {"distribution": null, "min": 47448, "max": 47448}, ".fai": {"distribution": null, "min": 41379, "max": 41379}, ".pac": {"distribution": null, "min": 244623820, "max": 244623820}, ".sa": {"distribution": null, "min": 489247688, "max": 489247688}, ".gz": {"distribution": null, "min": 108988506, "max": 108988506}, ".fa": {"distribution": null, "min": 990744229, "max": 990744229}, ".bwt": {"distribution": null, "min": 978495356, "max": 978495356}}, "output": {".vcf": {"distribution": {"name": "rdist", "params": [1.7504259269384672, 0.9999999999938114, 6.18860928332036e-12]}, "min": 59101, "max": 83149}, ".idx": {"distribution": {"name": "rdist", "params": [1.7504259269384672, 0.9999999999938114, 6.18860928332036e-12]}, "min": 28496, "max": 28584}}}, "combine_variants": {"runtime": {"min": 4.156, "max": 163.914, "distribution": {"name": "rdist", "params": [1.5504806356651624, 0.0013236200991527764, 0.0013236200991527767]}}, "input": {".idx": {"distribution": {"name": "rdist", "params": [1.7504259269384672, 0.9999999999938114, 6.18860928332036e-12]}, "min": 28496, "max": 28584}, ".vcf": {"distribution": {"name": "rdist", "params": [1.7504259269384672, 0.9999999999938114, 6.18860928332036e-12]}, "min": 59101, "max": 83149}, ".txt": {"distribution": {"name": "rdist", "params": [1.5504806356651624, 0.0013236200991527764, 0.0013236200991527767]}, "min": 60, "max": 15976}, ".dict": {"distribution": null, "min": 142561, "max": 142561}, ".gz": {"distribution": null, "min": 108988506, "max": 108988506}, ".amb": {"distribution": null, "min": 259156, "max": 259156}, ".ann": {"distribution": null, "min": 47448, "max": 47448}, ".fai": {"distribution": null, "min": 41379, "max": 41379}, ".pac": {"distribution": null, "min": 244623820, "max": 244623820}, ".sa": {"distribution": null, "min": 489247688, "max": 489247688}, ".fa": {"distribution": null, "min": 990744229, "max": 990744229}, ".bwt": {"distribution": null, "min": 978495356, "max": 978495356}}, "output": {".idx": {"distribution": {"name": "rdist", "params": [1.5504806356651624, 0.0013236200991527764, 0.0013236200991527767]}, "min": 28576, "max": 28577}, ".vcf": {"distribution": {"name": "rdist", "params": [1.5504806356651624, 0.0013236200991527764, 0.0013236200991527767]}, "min": 70314, "max": 87692}}}, "select_variants_snp": {"runtime": {"min": 50.953, "max": 197.487, "distribution": {"name": "rdist", "params": [1.5504806356651624, 0.0013236200991527764, 0.0013236200991527767]}}, "input": {".vcf": {"distribution": {"name": "rdist", "params": [1.5504806356651624, 0.0013236200991527764, 0.0013236200991527767]}, "min": 70314, "max": 87692}, ".dict": {"distribution": null, "min": 142561, "max": 142561}, ".pac": {"distribution": null, "min": 244623820, "max": 244623820}, ".fai": {"distribution": null, "min": 41379, "max": 41379}, ".sa": {"distribution": null, "min": 489247688, "max": 489247688}, ".gz": {"distribution": null, "min": 108988506, "max": 108988506}, ".amb": {"distribution": null, "min": 259156, "max": 259156}, ".fa": {"distribution": null, "min": 990744229, "max": 990744229}, ".ann": {"distribution": null, "min": 47448, "max": 47448}, ".bwt": {"distribution": null, "min": 978495356, "max": 978495356}, ".txt": {"distribution": {"name": "rdist", "params": [1.5504806356651624, 0.0013236200991527764, 0.0013236200991527767]}, "min": 60, "max": 15976}}, "output": {".vcf": {"distribution": {"name": "rdist", "params": [1.5504806356651624, 0.0013236200991527764, 0.0013236200991527767]}, "min": 71866, "max": 86556}}}, "filtering_snp": {"runtime": {"min": 4.264, "max": 67.509, "distribution": {"name": "rdist", "params": [1.5504806356651624, 0.0013236200991527764, 0.0013236200991527767]}}, "input": {".dict": {"distribution": null, "min": 142561, "max": 142561}, ".pac": {"distribution": null, "min": 244623820, "max": 244623820}, ".fai": {"distribution": null, "min": 41379, "max": 41379}, ".sa": {"distribution": null, "min": 489247688, "max": 489247688}, ".gz": {"distribution": null, "min": 108988506, "max": 108988506}, ".amb": {"distribution": null, "min": 259156, "max": 259156}, ".fa": {"distribution": null, "min": 990744229, "max": 990744229}, ".ann": {"distribution": null, "min": 47448, "max": 47448}, ".bwt": {"distribution": null, "min": 978495356, "max": 978495356}, ".vcf": {"distribution": {"name": "rdist", "params": [1.5504806356651624, 0.0013236200991527764, 0.0013236200991527767]}, "min": 71866, "max": 86556}, ".txt": {"distribution": {"name": "rdist", "params": [1.5504806356651624, 0.0013236200991527764, 0.0013236200991527767]}, "min": 60, "max": 15976}}, "output": {".idx": {"distribution": {"name": "rdist", "params": [1.5504806356651624, 0.0013236200991527764, 0.0013236200991527767]}, "min": 28545, "max": 28546}, ".vcf": {"distribution": {"name": "rdist", "params": [1.5504806356651624, 0.0013236200991527764, 0.0013236200991527767]}, "min": 74686, "max": 89376}}}, "select_variants_indel": {"runtime": {"min": 48.59, "max": 207.667, "distribution": {"name": "rdist", "params": [1.5504806356651624, 0.0013236200991527764, 0.0013236200991527767]}}, "input": {".vcf": {"distribution": {"name": "rdist", "params": [1.5504806356651624, 0.0013236200991527764, 0.0013236200991527767]}, "min": 70314, "max": 87692}, ".dict": {"distribution": null, "min": 142561, "max": 142561}, ".pac": {"distribution": null, "min": 244623820, "max": 244623820}, ".fai": {"distribution": null, "min": 41379, "max": 41379}, ".sa": {"distribution": null, "min": 489247688, "max": 489247688}, ".gz": {"distribution": null, "min": 108988506, "max": 108988506}, ".amb": {"distribution": null, "min": 259156, "max": 259156}, ".fa": {"distribution": null, "min": 990744229, "max": 990744229}, ".ann": {"distribution": null, "min": 47448, "max": 47448}, ".bwt": {"distribution": null, "min": 978495356, "max": 978495356}, ".txt": {"distribution": {"name": "rdist", "params": [1.5504806356651624, 0.0013236200991527764, 0.0013236200991527767]}, "min": 60, "max": 15976}}, "output": {".vcf": {"distribution": {"name": "rdist", "params": [1.5504806356651624, 0.0013236200991527764, 0.0013236200991527767]}, "min": 67140, "max": 72804}}}, "filtering_indel": {"runtime": {"min": 4.028, "max": 338.588, "distribution": {"name": "rdist", "params": [1.5504806356651624, 0.0013236200991527764, 0.0013236200991527767]}}, "input": {".dict": {"distribution": null, "min": 142561, "max": 142561}, ".pac": {"distribution": null, "min": 244623820, "max": 244623820}, ".fai": {"distribution": null, "min": 41379, "max": 41379}, ".vcf": {"distribution": {"name": "rdist", "params": [1.5504806356651624, 0.0013236200991527764, 0.0013236200991527767]}, "min": 67140, "max": 72804}, ".sa": {"distribution": null, "min": 489247688, "max": 489247688}, ".gz": {"distribution": null, "min": 108988506, "max": 108988506}, ".amb": {"distribution": null, "min": 259156, "max": 259156}, ".fa": {"distribution": null, "min": 990744229, "max": 990744229}, ".ann": {"distribution": null, "min": 47448, "max": 47448}, ".bwt": {"distribution": null, "min": 978495356, "max": 978495356}, ".txt": {"distribution": {"name": "rdist", "params": [1.5504806356651624, 0.0013236200991527764, 0.0013236200991527767]}, "min": 60, "max": 15976}}, "output": {".idx": {"distribution": {"name": "rdist", "params": [1.5504806356651624, 0.0013236200991527764, 0.0013236200991527767]}, "min": 28575, "max": 28576}, ".vcf": {"distribution": {"name": "rdist", "params": [1.5504806356651624, 0.0013236200991527764, 0.0013236200991527767]}, "min": 69861, "max": 75525}}}} \ No newline at end of file +{"alignment_to_reference": {"runtime": {"min": 3.305, "max": 133.207, "distribution": {"name": "rdist", "params": [1.0497506699055814, 0.8751735665957343, 0.5778762692984372]}}, "input": {".dict": {"distribution": null, "min": 142561, "max": 142561}, ".pac": {"distribution": null, "min": 244623820, "max": 244623820}, ".fai": {"distribution": null, "min": 41379, "max": 41379}, ".fastq": {"distribution": null, "min": 23845, "max": 23845}, ".sa": {"distribution": null, "min": 489247688, "max": 489247688}, ".gz": {"distribution": null, "min": 108988506, "max": 108988506}, ".amb": {"distribution": null, "min": 259156, "max": 259156}, ".fa": {"distribution": null, "min": 990744229, "max": 990744229}, ".ann": {"distribution": null, "min": 47448, "max": 47448}, ".bwt": {"distribution": null, "min": 978495356, "max": 978495356}, ".txt": {"distribution": {"name": "rdist", "params": [1.7477959990252214, 1.0, 3.9893803094996296e-22]}, "min": 60, "max": 15976}}, "output": {".sam": {"distribution": null, "min": 112561, "max": 112561}}}, "sort_sam": {"runtime": {"min": 2.082, "max": 89.242, "distribution": {"name": "arcsine", "params": [-0.09850603366048286, 1.098506033660483]}}, "input": {".sam": {"distribution": null, "min": 112561, "max": 112561}, ".gz": {"distribution": null, "min": 108988506, "max": 108988506}}, "output": {".bai": {"distribution": null, "min": 127424, "max": 127424}, ".bam": {"distribution": null, "min": 36576, "max": 36576}}}, "dedup": {"runtime": {"min": 3.169, "max": 152.024, "distribution": {"name": "rdist", "params": [1.056503552352268, 0.8880507079714015, 0.5907534106741043]}}, "input": {".gz": {"distribution": null, "min": 108988506, "max": 108988506}, ".bai": {"distribution": null, "min": 127424, "max": 127424}, ".bam": {"distribution": null, "min": 36576, "max": 36576}}, "output": {".bai": {"distribution": null, "min": 127424, "max": 127424}, ".bam": {"distribution": {"name": "rdist", "params": [1.7504259269384672, 0.9999999999938114, 6.18860928332036e-12]}, "min": 37467, "max": 37470}}}, "add_replace": {"runtime": {"min": 3.382, "max": 139.801, "distribution": {"name": "arcsine", "params": [-0.09850603366048286, 1.098506033660483]}}, "input": {".gz": {"distribution": null, "min": 108988506, "max": 108988506}, ".bai": {"distribution": null, "min": 127424, "max": 127424}, ".bam": {"distribution": {"name": "rdist", "params": [1.7504259269384672, 0.9999999999938114, 6.18860928332036e-12]}, "min": 37467, "max": 37470}}, "output": {".bai": {"distribution": null, "min": 127424, "max": 127424}, ".bam": {"distribution": {"name": "beta", "params": [0.028007929445023696, 0.16801064212397998, -0.08122143521923508, 1.0812214352192353]}, "min": 37953, "max": 37956}}}, "realign_target_creator": {"runtime": {"min": 78.698, "max": 474.546, "distribution": {"name": "arcsine", "params": [-0.08648046099196244, 1.0864804609919627]}}, "input": {".bai": {"distribution": null, "min": 127424, "max": 127424}, ".dict": {"distribution": null, "min": 142561, "max": 142561}, ".pac": {"distribution": null, "min": 244623820, "max": 244623820}, ".fai": {"distribution": null, "min": 41379, "max": 41379}, ".sa": {"distribution": null, "min": 489247688, "max": 489247688}, ".gz": {"distribution": null, "min": 108988506, "max": 108988506}, ".amb": {"distribution": null, "min": 259156, "max": 259156}, ".fa": {"distribution": null, "min": 990744229, "max": 990744229}, ".ann": {"distribution": null, "min": 47448, "max": 47448}, ".bam": {"distribution": {"name": "beta", "params": [0.028007929445023696, 0.16801064212397998, -0.08122143521923508, 1.0812214352192353]}, "min": 37953, "max": 37956}, ".bwt": {"distribution": null, "min": 978495356, "max": 978495356}, ".txt": {"distribution": {"name": "rdist", "params": [1.7477959990252214, 1.0, 3.9893803094996296e-22]}, "min": 60, "max": 15976}}, "output": {".list": {"distribution": null, "min": 40, "max": 40}}}, "indel_realign": {"runtime": {"min": 5.589, "max": 154.005, "distribution": {"name": "rdist", "params": [1.054143443775926, 0.8779716509043156, 0.6087408816735466]}}, "input": {".bai": {"distribution": null, "min": 127424, "max": 127424}, ".dict": {"distribution": null, "min": 142561, "max": 142561}, ".pac": {"distribution": null, "min": 244623820, "max": 244623820}, ".fai": {"distribution": null, "min": 41379, "max": 41379}, ".list": {"distribution": null, "min": 40, "max": 40}, ".sa": {"distribution": null, "min": 489247688, "max": 489247688}, ".gz": {"distribution": null, "min": 108988506, "max": 108988506}, ".amb": {"distribution": null, "min": 259156, "max": 259156}, ".fa": {"distribution": null, "min": 990744229, "max": 990744229}, ".ann": {"distribution": null, "min": 47448, "max": 47448}, ".bam": {"distribution": {"name": "beta", "params": [0.028007929445023696, 0.16801064212397998, -0.08122143521923508, 1.0812214352192353]}, "min": 37953, "max": 37956}, ".bwt": {"distribution": null, "min": 978495356, "max": 978495356}, ".txt": {"distribution": {"name": "rdist", "params": [1.7477959990252214, 1.0, 3.9893803094996296e-22]}, "min": 60, "max": 15976}}, "output": {".bam": {"distribution": {"name": "rdist", "params": [1.0354552049234393, 0.9230820838539932, 0.5764154171873266]}, "min": 39550, "max": 39554}, ".bai": {"distribution": null, "min": 127424, "max": 127424}}}, "haplotype_caller": {"runtime": {"min": 30.011, "max": 709.504, "distribution": {"name": "argus", "params": [0.0008690793369530592, -0.6114566529512571, 1.6139824159282967]}}, "input": {".dict": {"distribution": null, "min": 142561, "max": 142561}, ".bam": {"distribution": {"name": "trapezoid", "params": [0.9999999999999999, 1.0, -0.10500000000000001, 1.1999999999999997]}, "min": 39550, "max": 39554}, ".pac": {"distribution": null, "min": 244623820, "max": 244623820}, ".bai": {"distribution": null, "min": 127424, "max": 127424}, ".fai": {"distribution": null, "min": 41379, "max": 41379}, ".sa": {"distribution": null, "min": 489247688, "max": 489247688}, ".gz": {"distribution": null, "min": 108988506, "max": 108988506}, ".amb": {"distribution": null, "min": 259156, "max": 259156}, ".fa": {"distribution": null, "min": 990744229, "max": 990744229}, ".ann": {"distribution": null, "min": 47448, "max": 47448}, ".bwt": {"distribution": null, "min": 978495356, "max": 978495356}, ".txt": {"distribution": {"name": "trapezoid", "params": [0.9999999999999999, 1.0, -0.10500000000000001, 1.1999999999999997]}, "min": 60, "max": 15976}}, "output": {".vcf": {"distribution": {"name": "rdist", "params": [1.750426650564894, 0.9999999999999998, 2.3624681987884834e-16]}, "min": 59037, "max": 75728}, ".idx": {"distribution": {"name": "trapezoid", "params": [0.9999999999999999, 1.0, -0.10500000000000001, 1.1999999999999997]}, "min": 28447, "max": 28449}}}, "merge_gcvf": {"runtime": {"min": 2562.381, "max": 37861.356, "distribution": {"name": "rdist", "params": [1.5504806356651624, 0.0013236200991527764, 0.0013236200991527767]}}, "input": {".vcf": {"distribution": {"name": "rdist", "params": [1.750426650564894, 0.9999999999999998, 2.3624681987884834e-16]}, "min": 59037, "max": 75728}, ".idx": {"distribution": {"name": "trapezoid", "params": [0.9999999999999999, 1.0, -0.10500000000000001, 1.1999999999999997]}, "min": 28447, "max": 28449}, ".gz": {"distribution": null, "min": 108988506, "max": 108988506}, ".bwt": {"distribution": null, "min": 978495356, "max": 978495356}, ".txt": {"distribution": {"name": "rdist", "params": [1.5504806356651624, 0.0013236200991527764, 0.0013236200991527767]}, "min": 60, "max": 15976}, ".list": {"distribution": {"name": "rdist", "params": [1.5504806356651624, 0.0013236200991527764, 0.0013236200991527767]}, "min": 1700, "max": 17000}, ".amb": {"distribution": null, "min": 259156, "max": 259156}, ".fai": {"distribution": null, "min": 41379, "max": 41379}, ".pac": {"distribution": null, "min": 244623820, "max": 244623820}, ".fa": {"distribution": null, "min": 990744229, "max": 990744229}, ".dict": {"distribution": null, "min": 142561, "max": 142561}, ".ann": {"distribution": null, "min": 47448, "max": 47448}, ".sa": {"distribution": null, "min": 489247688, "max": 489247688}}, "output": {".idx": {"distribution": {"name": "rdist", "params": [1.5504806356651624, 0.0013236200991527764, 0.0013236200991527767]}, "min": 28886, "max": 29187}, ".vcf": {"distribution": {"name": "rdist", "params": [1.5504806356651624, 0.0013236200991527764, 0.0013236200991527767]}, "min": 104389, "max": 250300}}}, "genotype_gvcfs": {"runtime": {"min": 95.618, "max": 899.232, "distribution": {"name": "arcsine", "params": [-0.09850603366048286, 1.098506033660483]}}, "input": {".vcf": {"distribution": {"name": "rdist", "params": [1.750426650564894, 0.9999999999999998, 2.3624681987884834e-16]}, "min": 59037, "max": 75728}, ".idx": {"distribution": {"name": "trapezoid", "params": [0.9999999999999999, 1.0, -0.10500000000000001, 1.1999999999999997]}, "min": 28447, "max": 28449}, ".txt": {"distribution": {"name": "trapezoid", "params": [0.9999999999999999, 1.0, -0.10500000000000001, 1.1999999999999997]}, "min": 60, "max": 15976}, ".dict": {"distribution": null, "min": 142561, "max": 142561}, ".amb": {"distribution": null, "min": 259156, "max": 259156}, ".ann": {"distribution": null, "min": 47448, "max": 47448}, ".fai": {"distribution": null, "min": 41379, "max": 41379}, ".pac": {"distribution": null, "min": 244623820, "max": 244623820}, ".sa": {"distribution": null, "min": 489247688, "max": 489247688}, ".gz": {"distribution": null, "min": 108988506, "max": 108988506}, ".fa": {"distribution": null, "min": 990744229, "max": 990744229}, ".bwt": {"distribution": null, "min": 978495356, "max": 978495356}}, "output": {".vcf": {"distribution": {"name": "rdist", "params": [1.7504259269384672, 0.9999999999938114, 6.18860928332036e-12]}, "min": 59101, "max": 83149}, ".idx": {"distribution": {"name": "rdist", "params": [1.7504259269384672, 0.9999999999938114, 6.18860928332036e-12]}, "min": 28496, "max": 28584}}}, "combine_variants": {"runtime": {"min": 4.156, "max": 163.914, "distribution": {"name": "rdist", "params": [1.5504806356651624, 0.0013236200991527764, 0.0013236200991527767]}}, "input": {".idx": {"distribution": {"name": "rdist", "params": [1.7504259269384672, 0.9999999999938114, 6.18860928332036e-12]}, "min": 28496, "max": 28584}, ".vcf": {"distribution": {"name": "rdist", "params": [1.7504259269384672, 0.9999999999938114, 6.18860928332036e-12]}, "min": 59101, "max": 83149}, ".txt": {"distribution": {"name": "rdist", "params": [1.5504806356651624, 0.0013236200991527764, 0.0013236200991527767]}, "min": 60, "max": 15976}, ".dict": {"distribution": null, "min": 142561, "max": 142561}, ".gz": {"distribution": null, "min": 108988506, "max": 108988506}, ".amb": {"distribution": null, "min": 259156, "max": 259156}, ".ann": {"distribution": null, "min": 47448, "max": 47448}, ".fai": {"distribution": null, "min": 41379, "max": 41379}, ".pac": {"distribution": null, "min": 244623820, "max": 244623820}, ".sa": {"distribution": null, "min": 489247688, "max": 489247688}, ".fa": {"distribution": null, "min": 990744229, "max": 990744229}, ".bwt": {"distribution": null, "min": 978495356, "max": 978495356}}, "output": {".idx": {"distribution": {"name": "rdist", "params": [1.5504806356651624, 0.0013236200991527764, 0.0013236200991527767]}, "min": 28576, "max": 28577}, ".vcf": {"distribution": {"name": "rdist", "params": [1.5504806356651624, 0.0013236200991527764, 0.0013236200991527767]}, "min": 70314, "max": 87692}}}, "select_variants_snp": {"runtime": {"min": 50.953, "max": 197.487, "distribution": {"name": "rdist", "params": [1.5504806356651624, 0.0013236200991527764, 0.0013236200991527767]}}, "input": {".vcf": {"distribution": {"name": "rdist", "params": [1.5504806356651624, 0.0013236200991527764, 0.0013236200991527767]}, "min": 70314, "max": 87692}, ".dict": {"distribution": null, "min": 142561, "max": 142561}, ".pac": {"distribution": null, "min": 244623820, "max": 244623820}, ".fai": {"distribution": null, "min": 41379, "max": 41379}, ".sa": {"distribution": null, "min": 489247688, "max": 489247688}, ".gz": {"distribution": null, "min": 108988506, "max": 108988506}, ".amb": {"distribution": null, "min": 259156, "max": 259156}, ".fa": {"distribution": null, "min": 990744229, "max": 990744229}, ".ann": {"distribution": null, "min": 47448, "max": 47448}, ".bwt": {"distribution": null, "min": 978495356, "max": 978495356}, ".txt": {"distribution": {"name": "rdist", "params": [1.5504806356651624, 0.0013236200991527764, 0.0013236200991527767]}, "min": 60, "max": 15976}}, "output": {".vcf": {"distribution": {"name": "rdist", "params": [1.5504806356651624, 0.0013236200991527764, 0.0013236200991527767]}, "min": 71866, "max": 86556}}}, "filtering_snp": {"runtime": {"min": 4.264, "max": 67.509, "distribution": {"name": "rdist", "params": [1.5504806356651624, 0.0013236200991527764, 0.0013236200991527767]}}, "input": {".dict": {"distribution": null, "min": 142561, "max": 142561}, ".pac": {"distribution": null, "min": 244623820, "max": 244623820}, ".fai": {"distribution": null, "min": 41379, "max": 41379}, ".sa": {"distribution": null, "min": 489247688, "max": 489247688}, ".gz": {"distribution": null, "min": 108988506, "max": 108988506}, ".amb": {"distribution": null, "min": 259156, "max": 259156}, ".fa": {"distribution": null, "min": 990744229, "max": 990744229}, ".ann": {"distribution": null, "min": 47448, "max": 47448}, ".bwt": {"distribution": null, "min": 978495356, "max": 978495356}, ".vcf": {"distribution": {"name": "rdist", "params": [1.5504806356651624, 0.0013236200991527764, 0.0013236200991527767]}, "min": 71866, "max": 86556}, ".txt": {"distribution": {"name": "rdist", "params": [1.5504806356651624, 0.0013236200991527764, 0.0013236200991527767]}, "min": 60, "max": 15976}}, "output": {".idx": {"distribution": {"name": "rdist", "params": [1.5504806356651624, 0.0013236200991527764, 0.0013236200991527767]}, "min": 28545, "max": 28546}, ".vcf": {"distribution": {"name": "rdist", "params": [1.5504806356651624, 0.0013236200991527764, 0.0013236200991527767]}, "min": 74686, "max": 89376}}}, "select_variants_indel": {"runtime": {"min": 48.59, "max": 207.667, "distribution": {"name": "rdist", "params": [1.5504806356651624, 0.0013236200991527764, 0.0013236200991527767]}}, "input": {".vcf": {"distribution": {"name": "rdist", "params": [1.5504806356651624, 0.0013236200991527764, 0.0013236200991527767]}, "min": 70314, "max": 87692}, ".dict": {"distribution": null, "min": 142561, "max": 142561}, ".pac": {"distribution": null, "min": 244623820, "max": 244623820}, ".fai": {"distribution": null, "min": 41379, "max": 41379}, ".sa": {"distribution": null, "min": 489247688, "max": 489247688}, ".gz": {"distribution": null, "min": 108988506, "max": 108988506}, ".amb": {"distribution": null, "min": 259156, "max": 259156}, ".fa": {"distribution": null, "min": 990744229, "max": 990744229}, ".ann": {"distribution": null, "min": 47448, "max": 47448}, ".bwt": {"distribution": null, "min": 978495356, "max": 978495356}, ".txt": {"distribution": {"name": "rdist", "params": [1.5504806356651624, 0.0013236200991527764, 0.0013236200991527767]}, "min": 60, "max": 15976}}, "output": {".vcf": {"distribution": {"name": "rdist", "params": [1.5504806356651624, 0.0013236200991527764, 0.0013236200991527767]}, "min": 67140, "max": 72804}}}, "filtering_indel": {"runtime": {"min": 4.028, "max": 338.588, "distribution": {"name": "rdist", "params": [1.5504806356651624, 0.0013236200991527764, 0.0013236200991527767]}}, "input": {".dict": {"distribution": null, "min": 142561, "max": 142561}, ".pac": {"distribution": null, "min": 244623820, "max": 244623820}, ".fai": {"distribution": null, "min": 41379, "max": 41379}, ".vcf": {"distribution": {"name": "rdist", "params": [1.5504806356651624, 0.0013236200991527764, 0.0013236200991527767]}, "min": 67140, "max": 72804}, ".sa": {"distribution": null, "min": 489247688, "max": 489247688}, ".gz": {"distribution": null, "min": 108988506, "max": 108988506}, ".amb": {"distribution": null, "min": 259156, "max": 259156}, ".fa": {"distribution": null, "min": 990744229, "max": 990744229}, ".ann": {"distribution": null, "min": 47448, "max": 47448}, ".bwt": {"distribution": null, "min": 978495356, "max": 978495356}, ".txt": {"distribution": {"name": "rdist", "params": [1.5504806356651624, 0.0013236200991527764, 0.0013236200991527767]}, "min": 60, "max": 15976}}, "output": {".idx": {"distribution": {"name": "rdist", "params": [1.5504806356651624, 0.0013236200991527764, 0.0013236200991527767]}, "min": 28575, "max": 28576}, ".vcf": {"distribution": {"name": "rdist", "params": [1.5504806356651624, 0.0013236200991527764, 0.0013236200991527767]}, "min": 69861, "max": 75525}}}} \ No newline at end of file diff --git a/wfcommons/wfchef/recipes/srasearch/task_type_stats.json b/wfcommons/wfchef/recipes/srasearch/task_type_stats.json index 0d19fcf1..f31a36a9 100644 --- a/wfcommons/wfchef/recipes/srasearch/task_type_stats.json +++ b/wfcommons/wfchef/recipes/srasearch/task_type_stats.json @@ -1 +1 @@ -{"bowtie2-build": {"runtime": {"min": 0.316, "max": 33.288, "distribution": {"name": "wald", "params": [-0.17799345401850686, 0.7891642364923879]}}, "input": {".fna": {"distribution": null, "min": 98721, "max": 98721}}, "output": {".bt2": {"distribution": {"name": "norm", "params": [0.1, 0.27080128015453203]}, "min": 17, "max": 4226858}}}, "fasterq-dump": {"runtime": {"min": 1.173, "max": 4343.699, "distribution": {"name": "fisk", "params": [3.9199026441968337, -0.30421363341992647, 0.6309055633022219]}}, "input": {}, "output": {".fastq": {"distribution": {"name": "argus", "params": [0.00015637238065969544, -0.2508819176275605, 1.2531514622818194]}, "min": 26168, "max": 1981081890}}}, "bowtie2": {"runtime": {"min": 2.377, "max": 121.024, "distribution": {"name": "arcsine", "params": [-0.07358918218072019, 1.0735891821807204]}}, "input": {".bt2": {"distribution": {"name": "trapz", "params": [0.9999999999999999, 1.0, -0.10500000000000001, 1.1999999999999997]}, "min": 17, "max": 4226858}, ".fastq": {"distribution": {"name": "argus", "params": [0.00015637238065969544, -0.2508819176275605, 1.2531514622818194]}, "min": 26168, "max": 1981081890}}, "output": {".bai": {"distribution": null, "min": 24, "max": 24}, ".bam": {"distribution": {"name": "rdist", "params": [1.750425928578598, 1.0, 1.3654380389187343e-20]}, "min": 314, "max": 317}}}, "merge": {"runtime": {"min": 0.039, "max": 0.305, "distribution": {"name": "beta", "params": [0.12845511317891758, 0.19967338970784937, -0.09952333240693918, 1.0995233324069393]}}, "input": {".bai": {"distribution": null, "min": 24, "max": 24}, ".bam": {"distribution": {"name": "rdist", "params": [1.750425928578598, 1.0, 1.3654380389187343e-20]}, "min": 314, "max": 317}, ".gz": {"distribution": {"name": "trapz", "params": [1.0, 1.0, -0.10500000000000001, 1.2]}, "min": 1540, "max": 6175}}, "output": {".gz": {"distribution": {"name": "dgamma", "params": [0.25734524051070895, 0.2142857142857143, 0.3299110686952953]}, "min": 1540, "max": 10427}}}} \ No newline at end of file +{"bowtie2-build": {"runtime": {"min": 0.316, "max": 33.288, "distribution": {"name": "wald", "params": [-0.17799345401850686, 0.7891642364923879]}}, "input": {".fna": {"distribution": null, "min": 98721, "max": 98721}}, "output": {".bt2": {"distribution": {"name": "norm", "params": [0.1, 0.27080128015453203]}, "min": 17, "max": 4226858}}}, "fasterq-dump": {"runtime": {"min": 1.173, "max": 4343.699, "distribution": {"name": "fisk", "params": [3.9199026441968337, -0.30421363341992647, 0.6309055633022219]}}, "input": {}, "output": {".fastq": {"distribution": {"name": "argus", "params": [0.00015637238065969544, -0.2508819176275605, 1.2531514622818194]}, "min": 26168, "max": 1981081890}}}, "bowtie2": {"runtime": {"min": 2.377, "max": 121.024, "distribution": {"name": "arcsine", "params": [-0.07358918218072019, 1.0735891821807204]}}, "input": {".bt2": {"distribution": {"name": "trapezoid", "params": [0.9999999999999999, 1.0, -0.10500000000000001, 1.1999999999999997]}, "min": 17, "max": 4226858}, ".fastq": {"distribution": {"name": "argus", "params": [0.00015637238065969544, -0.2508819176275605, 1.2531514622818194]}, "min": 26168, "max": 1981081890}}, "output": {".bai": {"distribution": null, "min": 24, "max": 24}, ".bam": {"distribution": {"name": "rdist", "params": [1.750425928578598, 1.0, 1.3654380389187343e-20]}, "min": 314, "max": 317}}}, "merge": {"runtime": {"min": 0.039, "max": 0.305, "distribution": {"name": "beta", "params": [0.12845511317891758, 0.19967338970784937, -0.09952333240693918, 1.0995233324069393]}}, "input": {".bai": {"distribution": null, "min": 24, "max": 24}, ".bam": {"distribution": {"name": "rdist", "params": [1.750425928578598, 1.0, 1.3654380389187343e-20]}, "min": 314, "max": 317}, ".gz": {"distribution": {"name": "trapezoid", "params": [1.0, 1.0, -0.10500000000000001, 1.2]}, "min": 1540, "max": 6175}}, "output": {".gz": {"distribution": {"name": "dgamma", "params": [0.25734524051070895, 0.2142857142857143, 0.3299110686952953]}, "min": 1540, "max": 10427}}}} \ No newline at end of file From ffae854e250d84a51b75c44e7b4077f7b3a73060 Mon Sep 17 00:00:00 2001 From: Henri Casanova Date: Tue, 5 Aug 2025 11:18:02 -1000 Subject: [PATCH 07/38] disabling Airflow test --- tests/translators/test_translators.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/translators/test_translators.py b/tests/translators/test_translators.py index 7af482ff..4eaed75a 100644 --- a/tests/translators/test_translators.py +++ b/tests/translators/test_translators.py @@ -222,6 +222,7 @@ def test_nextflow_translator(self) -> None: @pytest.mark.unit + @pytest.mark.skip(reason="tmp") def test_airflow_translator(self) -> None: # Create workflow benchmark From d91c037b2e8dd0ca914046d418b24d4727701aab Mon Sep 17 00:00:00 2001 From: Henri Casanova Date: Tue, 5 Aug 2025 11:20:43 -1000 Subject: [PATCH 08/38] disabling Nextflow test --- tests/translators/test_translators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/translators/test_translators.py b/tests/translators/test_translators.py index 4eaed75a..a9e36ca5 100644 --- a/tests/translators/test_translators.py +++ b/tests/translators/test_translators.py @@ -178,7 +178,7 @@ def test_parsl_translator(self) -> None: assert(num_completed_tasks == num_tasks) @pytest.mark.unit - # @pytest.mark.skip(reason="tmp") + @pytest.mark.skip(reason="tmp") def test_nextflow_translator(self) -> None: # Create workflow benchmark From ca84fc87b630987081b053441e08097f75096b60 Mon Sep 17 00:00:00 2001 From: Henri Casanova Date: Tue, 5 Aug 2025 11:26:22 -1000 Subject: [PATCH 09/38] Disabling all translator tests --- tests/translators/test_translators.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/translators/test_translators.py b/tests/translators/test_translators.py index a9e36ca5..c9efe0c7 100644 --- a/tests/translators/test_translators.py +++ b/tests/translators/test_translators.py @@ -91,7 +91,7 @@ def create_workflow_benchmark(): class TestTranslators: @pytest.mark.unit - # @pytest.mark.skip(reason="tmp") + @pytest.mark.skip(reason="tmp") def test_dask_translator(self) -> None: # Create workflow benchmark @@ -134,7 +134,7 @@ def test_dask_translator(self) -> None: @pytest.mark.unit - # @pytest.mark.skip(reason="tmp") + @pytest.mark.skip(reason="tmp") def test_parsl_translator(self) -> None: # Create workflow benchmark From 1cd259d861b2b60508dcf0c74ee18c1628e1d69c Mon Sep 17 00:00:00 2001 From: Henri Casanova Date: Tue, 5 Aug 2025 12:23:45 -1000 Subject: [PATCH 10/38] Replaced the use of shutil.which() with getting directly the pythong executable dir path --- wfcommons/wfbench/translator/abstract_translator.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/wfcommons/wfbench/translator/abstract_translator.py b/wfcommons/wfbench/translator/abstract_translator.py index 2f3d1c7d..18480400 100644 --- a/wfcommons/wfbench/translator/abstract_translator.py +++ b/wfcommons/wfbench/translator/abstract_translator.py @@ -10,6 +10,7 @@ import logging import os +import sys import pathlib import shutil import textwrap @@ -90,8 +91,12 @@ def _copy_binary_files(self, output_folder: pathlib.Path) -> None: """ bin_folder = output_folder.joinpath("bin") bin_folder.mkdir(exist_ok=True) - shutil.copy(shutil.which("wfbench"), bin_folder) - shutil.copy(shutil.which("cpu-benchmark"), bin_folder) + # Get the python executable dir + python_executable_dir = os.path.dirname(sys.executable) + # shutil.copy(shutil.which("wfbench"), bin_folder) + shutil.copy(python_executable_dir + "/wfbench", bin_folder) + # shutil.copy(shutil.which("cpu-benchmark"), bin_folder) + shutil.copy(python_executable_dir + "/cpu-benchmark", bin_folder) def _generate_input_files(self, output_folder: pathlib.Path) -> None: """ From a757be122cb51d76bf70a47efc769716d0782ec6 Mon Sep 17 00:00:00 2001 From: Henri Casanova Date: Tue, 5 Aug 2025 17:48:47 -1000 Subject: [PATCH 11/38] Airflow translator testing Dockerfile updates setup.py fix --- setup.py | 23 +++++- tests/translators/Dockerfile.airflow | 19 +++-- tests/translators/README.airflow | 9 ++- tests/translators/test_translators.py | 74 +++++++++++-------- .../wfbench/translator/abstract_translator.py | 4 +- 5 files changed, 85 insertions(+), 44 deletions(-) diff --git a/setup.py b/setup.py index 410d9ed6..e6ec20aa 100644 --- a/setup.py +++ b/setup.py @@ -9,6 +9,11 @@ # (at your option) any later version. import sys +import os +import stat +import shutil +import site +import sysconfig import subprocess from setuptools import setup, find_packages @@ -25,6 +30,17 @@ def run(self): sys.exit(-1) super().run() + # Do a by-hand copy of cpu-benchmark to the bin directory, as this is so + # hard to do using data_file in the setup() declaration + scripts_dir = os.path.join(site.USER_BASE, "bin") + source_path = os.path.join("bin", "cpu-benchmark") + target_path = os.path.join(scripts_dir, "cpu-benchmark") + # Ensure it's executable (just in case) + st = os.stat(source_path) + os.chmod(source_path, st.st_mode | stat.S_IEXEC) + # Copy to scripts directory + shutil.copy2(source_path, target_path) + setup( packages=find_packages(), include_package_data=True, @@ -32,9 +48,10 @@ def run(self): cmdclass={ 'build_ext': Build, }, - data_files=[ - ('bin', ['bin/cpu-benchmark', 'bin/wfbench']) - ], + #data_files=[ + # (user_bin_dir, ['bin/cpu-benchmark']) + #], + scripts=['bin/wfbench'], entry_points={ 'console_scripts': [ 'wfchef=wfcommons.wfchef.chef:main' diff --git a/tests/translators/Dockerfile.airflow b/tests/translators/Dockerfile.airflow index 48394454..d9911c49 100644 --- a/tests/translators/Dockerfile.airflow +++ b/tests/translators/Dockerfile.airflow @@ -34,7 +34,7 @@ RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1 RUN apt-get -y install stress-ng # WfCommons -RUN python3 -m pip install --break-system-packages wfcommons +#RUN python3 -m pip install --break-system-packages wfcommons # Install Airflow RUN python3 -m pip install --break-system-packages apache-airflow==2.10.2 --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.10.2/constraints-3.12.txt" @@ -48,8 +48,11 @@ RUN python3 -m pip install --break-system-packages mysqlclient # Setup directory RUN mkdir /home/wfcommons -# Create an entrypoint script to start mysqld in the background -# and setup the Airflow DB +# Create an entrypoint script to start mysqld in the background and setup the Airflow DB +# This script is then used as the entry points of this Container +# Create ALSO a script to run a benchmark (only the last line differ), which is used for testing, +# because using docker-py, things get complicated with different bash sessions! + RUN echo '#!/bin/bash' > /entrypoint.sh && \ echo 'mysqld --explicit-defaults-for-timestamp &' >> /entrypoint.sh && \ echo 'until mysqladmin ping -h localhost --silent; do' >> /entrypoint.sh && \ @@ -66,10 +69,16 @@ RUN echo '#!/bin/bash' > /entrypoint.sh && \ echo "sed -i ./airflow/airflow.cfg -e 's/sqlite:.*/mysql+mysqldb:\/\/airflow_user:airflow_pass@localhost:3306\/airflow_db/'" >> /entrypoint.sh && \ echo 'airflow db migrate' >> /entrypoint.sh && \ echo 'echo "Airflow database setup!"' >> /entrypoint.sh && \ - echo 'mkdir ./airflow/dags' >> /entrypoint.sh && \ - echo 'exec bash' >> /entrypoint.sh && \ + echo 'mkdir ./airflow/dags' >> /entrypoint.sh + +RUN cp /entrypoint.sh /run_a_workflow.sh + +RUN echo 'exec bash' >> /entrypoint.sh && \ chmod +x /entrypoint.sh +RUN echo 'airflow dags test $1' >> /run_a_workflow.sh && \ + chmod +x /run_a_workflow.sh + WORKDIR /home/wfcommons ENTRYPOINT ["/entrypoint.sh"] diff --git a/tests/translators/README.airflow b/tests/translators/README.airflow index a0291a88..64b25cd1 100644 --- a/tests/translators/README.airflow +++ b/tests/translators/README.airflow @@ -58,7 +58,7 @@ A much simpler alternative is to use Docker. 2. Run the docker container in the directory to contains the translated workflow (see last section below) - docker run -it --rm -v .:/home/wfcommons/mount wfcommons-dev /bin/bash + docker run -it --rm -v .:/home/wfcommons/translated_workflow wfcommons-dev /bin/bash Running a translated workflow with Airflow @@ -83,11 +83,12 @@ translator = AirflowTranslator(benchmark.workflow) translator.translate(output_folder=pathlib.Path("/tmp/translated_workflow/")) ``` -The above will create a JSON worfklow file in /tmp/blast-benchmark-45.json. +The above will create a JSON workflow file in /tmp/blast-benchmark-45.json. In that file, the workflow name (this is used below) is set to "Blast-Benchmark". The above will also create the translated workflow the -/tmp/translated_workflow/ directory. Some directories and files need to be copied/moved as follows: +/tmp/translated_workflow/ directory. Some directories and files need to be copied/moved as follows. +As cp -r /tmp/translated_workflow/ $AIRFLOW_HOME/dags/ mv $AIRFLOW_HOME/dags/translated_workflow/workflow.py $AIRFLOW_HOME/dags/ @@ -97,4 +98,6 @@ Finally, run the workflow as: airflow dags test Blast-Benchmark (note the "Blast-Benchmark" workflow name from above) +Note that if you're using docker, the above two commands need to be done on the container, +with the right paths (i.e., /home/wfcommons/translated_workflow instead of /tmp/translated_workflow/) diff --git a/tests/translators/test_translators.py b/tests/translators/test_translators.py index c9efe0c7..befd9421 100644 --- a/tests/translators/test_translators.py +++ b/tests/translators/test_translators.py @@ -26,7 +26,7 @@ from wfcommons.wfbench import AirflowTranslator -def start_docker_container(backend, working_dir): +def start_docker_container(backend, mounted_dir, working_dir, command = ["sleep", "infinity"]): # Pulling the Docker image client = docker.from_env() sys.stderr.write("Pulling Docker image...\n") @@ -43,8 +43,8 @@ def start_docker_container(backend, working_dir): sys.stderr.write("Starting Docker container...\n") container = client.containers.run( image_name, - "sleep infinity", - volumes={working_dir: {'bind': working_dir, 'mode': 'rw'}}, + command=command, + volumes={mounted_dir: {'bind': mounted_dir, 'mode': 'rw'}}, working_dir=working_dir, tty=True, detach=True @@ -91,7 +91,7 @@ def create_workflow_benchmark(): class TestTranslators: @pytest.mark.unit - @pytest.mark.skip(reason="tmp") + # @pytest.mark.skip(reason="tmp") def test_dask_translator(self) -> None: # Create workflow benchmark @@ -109,12 +109,12 @@ def test_dask_translator(self) -> None: translator.translate(output_folder=dirpath) # Pulling the Docker image - container = start_docker_container("dask", str_dirpath) + container = start_docker_container("dask", str_dirpath, str_dirpath) # Installing WfCommons on container install_WfCommons_on_container(container) - # Copy over the wfbench and cpu-benchmark executables to where they should go + # Copy over the wfbench and cpu-benchmark executables to where they should go on the container exit_code, output = container.exec_run("sudo cp -f /tmp/WfCommons/bin/wfbench " + str_dirpath + "bin/", stdout=True, stderr=True) exit_code, output = container.exec_run("sudo cp -f /tmp/WfCommons/bin/cpu-benchmark " + str_dirpath + "bin/", stdout=True, stderr=True) @@ -134,7 +134,7 @@ def test_dask_translator(self) -> None: @pytest.mark.unit - @pytest.mark.skip(reason="tmp") + # @pytest.mark.skip(reason="tmp") def test_parsl_translator(self) -> None: # Create workflow benchmark @@ -151,8 +151,8 @@ def test_parsl_translator(self) -> None: translator = ParslTranslator(benchmark.workflow) translator.translate(output_folder=dirpath) - # Pulling the Docker image - container = start_docker_container("parsl", str_dirpath) + # Starting the Docker container + container = start_docker_container("parsl", str_dirpath, str_dirpath) # Installing WfCommons on container install_WfCommons_on_container(container) @@ -178,7 +178,7 @@ def test_parsl_translator(self) -> None: assert(num_completed_tasks == num_tasks) @pytest.mark.unit - @pytest.mark.skip(reason="tmp") + # @pytest.mark.skip(reason="tmp") def test_nextflow_translator(self) -> None: # Create workflow benchmark @@ -195,8 +195,8 @@ def test_nextflow_translator(self) -> None: translator = NextflowTranslator(benchmark.workflow) translator.translate(output_folder=dirpath) - # Pulling the Docker image - container = start_docker_container("nextflow", str_dirpath) + # Starting the Docker container + container = start_docker_container("nextflow", str_dirpath, str_dirpath) # Installing WfCommons on container install_WfCommons_on_container(container) @@ -222,7 +222,7 @@ def test_nextflow_translator(self) -> None: @pytest.mark.unit - @pytest.mark.skip(reason="tmp") + # @pytest.mark.skip(reason="tmp") def test_airflow_translator(self) -> None: # Create workflow benchmark @@ -239,27 +239,37 @@ def test_airflow_translator(self) -> None: translator = AirflowTranslator(benchmark.workflow) translator.translate(output_folder=dirpath) - # Pulling the Docker image - container = start_docker_container("airflow", str_dirpath) + # Starting the Docker container + container = start_docker_container("airflow", str_dirpath, "/home/wfcommons/", command=None) + # container = start_docker_container("airflow", str_dirpath, "/home/wfcommons/") # Installing WfCommons on container install_WfCommons_on_container(container) + # Copy over the wfbench and cpu-benchmark executables to where they should go - # exit_code, output = container.exec_run("sudo cp -f /tmp/WfCommons/bin/wfbench " + str_dirpath + "bin/", - # stdout=True, stderr=True) - # exit_code, output = container.exec_run("sudo cp -f /tmp/WfCommons/bin/cpu-benchmark " + str_dirpath + "bin/", - # stdout=True, stderr=True) - # - # # Run the workflow! - # sys.stderr.write("Running the Airflow workflow on the container...\n") - # exit_code, output = container.exec_run(f"nextflow run ./workflow.nf --pwd .", stdout=True, stderr=True) - # ignored, task_exit_codes = container.exec_run("find . -name .exitcode -exec cat {} \;", stdout=True, stderr=True) - # - # # Kill the container - # container.remove(force=True) - # - # # Do sanity checks - # sys.stderr.write("Checking sanity...\n") - # assert (exit_code == 0) - # assert (task_exit_codes.decode() == num_tasks * "0") + exit_code, output = container.exec_run("sudo cp -f /tmp/WfCommons/bin/wfbench /usr/local/bin/", + stdout=True, stderr=True) + exit_code, output = container.exec_run("sudo cp -f /tmp/WfCommons/bin/cpu-benchmark /usr/local/bin/", + stdout=True, stderr=True) + + # Do the necessary copies (some ugly hardcoded stuff here) + airflow_home = "/home/wfcommons/airflow/" # per the Dockerfile + exit_code, output = container.exec_run(f"cp -r /tmp/airflow_translated_workflow {airflow_home}/dags/",stdout=True, stderr=True) + exit_code, output = container.exec_run(f"mv {airflow_home}/dags/airflow_translated_workflow/workflow.py {airflow_home}/dags/",stdout=True, stderr=True) + + # # Run the entry point script by hand (such a hack!) + # exit_code, output = container.exec_run(f"/bin/bash /entrypoint.sh",stdout=True, stderr=True) + # print(output) + + # Run the workflow! + sys.stderr.write("Running the Airflow workflow on the container...\n") + exit_code, output = container.exec_run(cmd="/bin/bash /run_a_workflow.sh Blast-Benchmark", stdout=True, stderr=True) + + # Kill the container + container.remove(force=True) + + # Do sanity checks + sys.stderr.write("Checking sanity...\n") + assert (exit_code == 0) + assert (output.decode().count("completed") == num_tasks * 2) diff --git a/wfcommons/wfbench/translator/abstract_translator.py b/wfcommons/wfbench/translator/abstract_translator.py index 18480400..346d91a8 100644 --- a/wfcommons/wfbench/translator/abstract_translator.py +++ b/wfcommons/wfbench/translator/abstract_translator.py @@ -11,6 +11,7 @@ import logging import os import sys +import site import pathlib import shutil import textwrap @@ -92,7 +93,8 @@ def _copy_binary_files(self, output_folder: pathlib.Path) -> None: bin_folder = output_folder.joinpath("bin") bin_folder.mkdir(exist_ok=True) # Get the python executable dir - python_executable_dir = os.path.dirname(sys.executable) + # python_executable_dir = os.path.dirname(sys.executable) + python_executable_dir = os.path.join(site.getuserbase(), 'bin') # shutil.copy(shutil.which("wfbench"), bin_folder) shutil.copy(python_executable_dir + "/wfbench", bin_folder) # shutil.copy(shutil.which("cpu-benchmark"), bin_folder) From 4365e4612360b3233f77ffd8f6217dd4a1f6e44a Mon Sep 17 00:00:00 2001 From: Henri Casanova Date: Tue, 5 Aug 2025 18:00:35 -1000 Subject: [PATCH 12/38] Fixing toml warning --- pyproject.toml | 4 ++-- setup.cfg | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index bd864791..0cdc827f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ description = "A Framework for Enabling Scientific Workflow Research and Educati readme = "README.md" requires-python = ">=3.9" classifiers = [ - "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", + "License :: OSI Approved :: GPL-3.0-or-later", "Operating System :: OS Independent", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", @@ -21,7 +21,7 @@ classifiers = [ "Topic :: Documentation :: Sphinx", "Topic :: System :: Distributed Computing" ] -license = {text = "GNU General Public License v3 (GPLv3)"} +license = {text = "GPL-3.0-or-later"} dependencies = [ "jsonschema", "matplotlib", diff --git a/setup.cfg b/setup.cfg index b88034e4..08aedd7e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,2 +1,2 @@ [metadata] -description-file = README.md +description_file = README.md From 726df346ef1f8c6cec6bcf813c9df9deda05acd5 Mon Sep 17 00:00:00 2001 From: Henri Casanova Date: Tue, 5 Aug 2025 18:05:28 -1000 Subject: [PATCH 13/38] setup.py fix --- pyproject.toml | 6 +++--- setup.py | 2 ++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 0cdc827f..f6041e52 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,9 +7,9 @@ name = "wfcommons" authors = [{name = "WfCommons team", email = "support@wfcommons.org"}] description = "A Framework for Enabling Scientific Workflow Research and Education" readme = "README.md" -requires-python = ">=3.9" +requires-python = ">=3.11" classifiers = [ - "License :: OSI Approved :: GPL-3.0-or-later", + "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)", "Operating System :: OS Independent", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", @@ -21,7 +21,7 @@ classifiers = [ "Topic :: Documentation :: Sphinx", "Topic :: System :: Distributed Computing" ] -license = {text = "GPL-3.0-or-later"} +license = {expression = "GPL-3.0-or-later"} dependencies = [ "jsonschema", "matplotlib", diff --git a/setup.py b/setup.py index e6ec20aa..dd4b0e6d 100644 --- a/setup.py +++ b/setup.py @@ -33,6 +33,8 @@ def run(self): # Do a by-hand copy of cpu-benchmark to the bin directory, as this is so # hard to do using data_file in the setup() declaration scripts_dir = os.path.join(site.USER_BASE, "bin") + if not pathlib.Path(scripts_dir).is_dir(): + scripts_dir = "/opt/local/bin" source_path = os.path.join("bin", "cpu-benchmark") target_path = os.path.join(scripts_dir, "cpu-benchmark") # Ensure it's executable (just in case) From 5299a5884fb61e5f92c0d924ea00059d27f64820 Mon Sep 17 00:00:00 2001 From: Henri Casanova Date: Tue, 5 Aug 2025 18:09:20 -1000 Subject: [PATCH 14/38] toml fix --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index f6041e52..9e2e74c1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,7 @@ classifiers = [ "Topic :: Documentation :: Sphinx", "Topic :: System :: Distributed Computing" ] -license = {expression = "GPL-3.0-or-later"} +license = "GPL-3.0-or-later" dependencies = [ "jsonschema", "matplotlib", From e554b154b3a6d88b856f9efb7140aa1a830945bc Mon Sep 17 00:00:00 2001 From: Henri Casanova Date: Tue, 5 Aug 2025 18:13:24 -1000 Subject: [PATCH 15/38] toml fix --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 9e2e74c1..e4330656 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ description = "A Framework for Enabling Scientific Workflow Research and Educati readme = "README.md" requires-python = ">=3.11" classifiers = [ - "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)", + "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", "Operating System :: OS Independent", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", @@ -21,7 +21,7 @@ classifiers = [ "Topic :: Documentation :: Sphinx", "Topic :: System :: Distributed Computing" ] -license = "GPL-3.0-or-later" +license = {text = "GNU General Public License v3 (GPLv3)"} dependencies = [ "jsonschema", "matplotlib", From f69363be563c05a96eeaeb08eaf3337594eb4b7b Mon Sep 17 00:00:00 2001 From: Henri Casanova Date: Tue, 5 Aug 2025 18:15:19 -1000 Subject: [PATCH 16/38] missing import --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index dd4b0e6d..b15fe5a4 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,7 @@ import stat import shutil import site -import sysconfig +import pathlib import subprocess from setuptools import setup, find_packages From 53ce2e20a15b5c54d7198ab37148cf305b943244 Mon Sep 17 00:00:00 2001 From: Henri Casanova Date: Tue, 5 Aug 2025 18:18:51 -1000 Subject: [PATCH 17/38] path fix --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b15fe5a4..2fcffff9 100644 --- a/setup.py +++ b/setup.py @@ -34,7 +34,7 @@ def run(self): # hard to do using data_file in the setup() declaration scripts_dir = os.path.join(site.USER_BASE, "bin") if not pathlib.Path(scripts_dir).is_dir(): - scripts_dir = "/opt/local/bin" + scripts_dir = "/usr/local/bin" source_path = os.path.join("bin", "cpu-benchmark") target_path = os.path.join(scripts_dir, "cpu-benchmark") # Ensure it's executable (just in case) From e00bb0a22324463f700d220b3c374f75b9126976 Mon Sep 17 00:00:00 2001 From: Henri Casanova Date: Tue, 5 Aug 2025 18:22:06 -1000 Subject: [PATCH 18/38] Another path fix --- wfcommons/wfbench/translator/abstract_translator.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/wfcommons/wfbench/translator/abstract_translator.py b/wfcommons/wfbench/translator/abstract_translator.py index 346d91a8..0cf294eb 100644 --- a/wfcommons/wfbench/translator/abstract_translator.py +++ b/wfcommons/wfbench/translator/abstract_translator.py @@ -95,6 +95,8 @@ def _copy_binary_files(self, output_folder: pathlib.Path) -> None: # Get the python executable dir # python_executable_dir = os.path.dirname(sys.executable) python_executable_dir = os.path.join(site.getuserbase(), 'bin') + if not pathlib.Path(python_executable_dir).exists(): + python_executable_dir = "/usr/local/bin" # shutil.copy(shutil.which("wfbench"), bin_folder) shutil.copy(python_executable_dir + "/wfbench", bin_folder) # shutil.copy(shutil.which("cpu-benchmark"), bin_folder) From 098fe0fa3a097b05bc18d4fdd6b98b2ac31997ea Mon Sep 17 00:00:00 2001 From: Henri Casanova Date: Tue, 5 Aug 2025 18:27:04 -1000 Subject: [PATCH 19/38] Yet another path fix (for wfbench this time) --- setup.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/setup.py b/setup.py index 2fcffff9..3af656fd 100644 --- a/setup.py +++ b/setup.py @@ -34,14 +34,16 @@ def run(self): # hard to do using data_file in the setup() declaration scripts_dir = os.path.join(site.USER_BASE, "bin") if not pathlib.Path(scripts_dir).is_dir(): - scripts_dir = "/usr/local/bin" - source_path = os.path.join("bin", "cpu-benchmark") - target_path = os.path.join(scripts_dir, "cpu-benchmark") - # Ensure it's executable (just in case) - st = os.stat(source_path) - os.chmod(source_path, st.st_mode | stat.S_IEXEC) - # Copy to scripts directory - shutil.copy2(source_path, target_path) + scripts_dir = "/usr/local/bin/" + + for executable in ["cpu-benchmark", "wfbench"]: + source_path = os.path.join("bin", executable) + target_path = os.path.join(scripts_dir, executable) + # Ensure it's executable (just in case) + st = os.stat(source_path) + os.chmod(source_path, st.st_mode | stat.S_IEXEC) + # Copy to scripts directory + shutil.copy2(source_path, target_path) setup( packages=find_packages(), @@ -53,7 +55,7 @@ def run(self): #data_files=[ # (user_bin_dir, ['bin/cpu-benchmark']) #], - scripts=['bin/wfbench'], + # scripts=['bin/wfbench'], entry_points={ 'console_scripts': [ 'wfchef=wfcommons.wfchef.chef:main' From 2f6ebd8d0cd8ce72a6647480dea53dc7e13d74b5 Mon Sep 17 00:00:00 2001 From: Henri Casanova Date: Tue, 5 Aug 2025 18:30:50 -1000 Subject: [PATCH 20/38] Fixing import --- tests/translators/test_translators.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/translators/test_translators.py b/tests/translators/test_translators.py index befd9421..5187e8cd 100644 --- a/tests/translators/test_translators.py +++ b/tests/translators/test_translators.py @@ -12,6 +12,7 @@ import pytest import shutil import docker +from docker.errors import ImageNotFound import io import tarfile import os From add824b3847d942c22eb73f1ad852581a087092b Mon Sep 17 00:00:00 2001 From: Henri Casanova Date: Tue, 5 Aug 2025 18:38:20 -1000 Subject: [PATCH 21/38] Disabling some tests for easier CI debugging --- tests/translators/test_translators.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/translators/test_translators.py b/tests/translators/test_translators.py index 5187e8cd..b00b03eb 100644 --- a/tests/translators/test_translators.py +++ b/tests/translators/test_translators.py @@ -135,7 +135,7 @@ def test_dask_translator(self) -> None: @pytest.mark.unit - # @pytest.mark.skip(reason="tmp") + @pytest.mark.skip(reason="tmp") def test_parsl_translator(self) -> None: # Create workflow benchmark @@ -179,7 +179,7 @@ def test_parsl_translator(self) -> None: assert(num_completed_tasks == num_tasks) @pytest.mark.unit - # @pytest.mark.skip(reason="tmp") + @pytest.mark.skip(reason="tmp") def test_nextflow_translator(self) -> None: # Create workflow benchmark @@ -223,7 +223,7 @@ def test_nextflow_translator(self) -> None: @pytest.mark.unit - # @pytest.mark.skip(reason="tmp") + @pytest.mark.skip(reason="tmp") def test_airflow_translator(self) -> None: # Create workflow benchmark From 59a6c36534331b22cc3dd1da2c84ba25ec2514b4 Mon Sep 17 00:00:00 2001 From: Henri Casanova Date: Tue, 5 Aug 2025 18:43:03 -1000 Subject: [PATCH 22/38] Re-enabling more tests --- setup.py | 1 + tests/translators/test_translators.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 3af656fd..0d66cc24 100644 --- a/setup.py +++ b/setup.py @@ -44,6 +44,7 @@ def run(self): os.chmod(source_path, st.st_mode | stat.S_IEXEC) # Copy to scripts directory shutil.copy2(source_path, target_path) + sys.stderr.write(f"Copied {source_path} to {target_path}\n") setup( packages=find_packages(), diff --git a/tests/translators/test_translators.py b/tests/translators/test_translators.py index b00b03eb..7b250f6d 100644 --- a/tests/translators/test_translators.py +++ b/tests/translators/test_translators.py @@ -135,7 +135,7 @@ def test_dask_translator(self) -> None: @pytest.mark.unit - @pytest.mark.skip(reason="tmp") + # @pytest.mark.skip(reason="tmp") def test_parsl_translator(self) -> None: # Create workflow benchmark @@ -179,7 +179,7 @@ def test_parsl_translator(self) -> None: assert(num_completed_tasks == num_tasks) @pytest.mark.unit - @pytest.mark.skip(reason="tmp") + # @pytest.mark.skip(reason="tmp") def test_nextflow_translator(self) -> None: # Create workflow benchmark From 58d8034f1eba151696e695dceb417f9d411d6109 Mon Sep 17 00:00:00 2001 From: Henri Casanova Date: Tue, 5 Aug 2025 18:45:03 -1000 Subject: [PATCH 23/38] Re-enabling Airflow test --- tests/translators/test_translators.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/translators/test_translators.py b/tests/translators/test_translators.py index 7b250f6d..48910980 100644 --- a/tests/translators/test_translators.py +++ b/tests/translators/test_translators.py @@ -223,7 +223,7 @@ def test_nextflow_translator(self) -> None: @pytest.mark.unit - @pytest.mark.skip(reason="tmp") + # @pytest.mark.skip(reason="tmp") def test_airflow_translator(self) -> None: # Create workflow benchmark @@ -241,7 +241,7 @@ def test_airflow_translator(self) -> None: translator.translate(output_folder=dirpath) # Starting the Docker container - container = start_docker_container("airflow", str_dirpath, "/home/wfcommons/", command=None) + container = start_docker_container("airflow", str_dirpath, "/home/wfcommons/") # container = start_docker_container("airflow", str_dirpath, "/home/wfcommons/") # Installing WfCommons on container From 7b6edf13983773363d52d1d5dfd16aa8e6da5ac1 Mon Sep 17 00:00:00 2001 From: Henri Casanova Date: Tue, 5 Aug 2025 18:53:25 -1000 Subject: [PATCH 24/38] Adding debugging output to container for CI debugging --- tests/translators/test_translators.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/translators/test_translators.py b/tests/translators/test_translators.py index 48910980..af13b206 100644 --- a/tests/translators/test_translators.py +++ b/tests/translators/test_translators.py @@ -242,6 +242,12 @@ def test_airflow_translator(self) -> None: # Starting the Docker container container = start_docker_container("airflow", str_dirpath, "/home/wfcommons/") + sys.stderr.write(f"Container status: {container.status}\n") + container.reload() + sys.stderr.write(f"Container updated status: {container.status}\n") + logs = container.logs(stdout=True, stderr=True).decode("utf-8") + sys.stderr.write(f"Container logs:\n{logs}\n") + # container = start_docker_container("airflow", str_dirpath, "/home/wfcommons/") # Installing WfCommons on container From 734ff74c2bafa9aaa8d991916bfb3b8d362fdfbd Mon Sep 17 00:00:00 2001 From: Henri Casanova Date: Tue, 5 Aug 2025 18:58:28 -1000 Subject: [PATCH 25/38] adding debug to test --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a1459068..dd776553 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -31,7 +31,7 @@ jobs: pip install -e . - name: Run tests - run: pytest -m unit --cov=wfcommons tests/ + run: pytest -s -v -m unit --cov=wfcommons tests/ - name: Upload coverage if: github.ref == 'refs/heads/main' From 6dcfc37241a55d19f334fdecfaf10a7d629be145 Mon Sep 17 00:00:00 2001 From: Henri Casanova Date: Tue, 5 Aug 2025 21:27:45 -1000 Subject: [PATCH 26/38] Trying a different approach for airflow translator testing --- tests/translators/Dockerfile.airflow | 54 ++++++++++++--------------- tests/translators/README.airflow | 7 +++- tests/translators/test_translators.py | 11 ++---- 3 files changed, 31 insertions(+), 41 deletions(-) diff --git a/tests/translators/Dockerfile.airflow b/tests/translators/Dockerfile.airflow index d9911c49..9cc15057 100644 --- a/tests/translators/Dockerfile.airflow +++ b/tests/translators/Dockerfile.airflow @@ -48,37 +48,29 @@ RUN python3 -m pip install --break-system-packages mysqlclient # Setup directory RUN mkdir /home/wfcommons -# Create an entrypoint script to start mysqld in the background and setup the Airflow DB -# This script is then used as the entry points of this Container -# Create ALSO a script to run a benchmark (only the last line differ), which is used for testing, -# because using docker-py, things get complicated with different bash sessions! - -RUN echo '#!/bin/bash' > /entrypoint.sh && \ - echo 'mysqld --explicit-defaults-for-timestamp &' >> /entrypoint.sh && \ - echo 'until mysqladmin ping -h localhost --silent; do' >> /entrypoint.sh && \ - echo ' echo "Waiting for MySQL to be ready..."' >> /entrypoint.sh && \ - echo ' sleep 2' >> /entrypoint.sh && \ - echo 'done' >> /entrypoint.sh && \ - echo 'echo "MySQL is ready!"' >> /entrypoint.sh && \ - echo 'echo "Setting up database for Airflow..."' >> /entrypoint.sh && \ - echo 'mysql -u root -e "CREATE DATABASE airflow_db CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci";' >> /entrypoint.sh && \ - echo "mysql -u root -e \"CREATE USER 'airflow_user'@'%' IDENTIFIED BY 'airflow_pass';\"" >> /entrypoint.sh && \ - echo "mysql -u root -e \"GRANT ALL PRIVILEGES ON airflow_db.* TO 'airflow_user';\"" >> /entrypoint.sh && \ - echo 'export AIRFLOW_HOME="$(pwd)"/airflow/' >> /entrypoint.sh && \ - echo "airflow config list 1> /dev/null" >> /entrypoint.sh && \ - echo "sed -i ./airflow/airflow.cfg -e 's/sqlite:.*/mysql+mysqldb:\/\/airflow_user:airflow_pass@localhost:3306\/airflow_db/'" >> /entrypoint.sh && \ - echo 'airflow db migrate' >> /entrypoint.sh && \ - echo 'echo "Airflow database setup!"' >> /entrypoint.sh && \ - echo 'mkdir ./airflow/dags' >> /entrypoint.sh - -RUN cp /entrypoint.sh /run_a_workflow.sh - -RUN echo 'exec bash' >> /entrypoint.sh && \ - chmod +x /entrypoint.sh - -RUN echo 'airflow dags test $1' >> /run_a_workflow.sh && \ +# Create an run_a_workflow script to start mysqld in the background and setup the Airflow DB, +# copy things around, and the run the workflow + +RUN echo '#!/bin/bash' > /run_a_workflow.sh && \ + echo 'mysqld --explicit-defaults-for-timestamp &' >> /run_a_workflow.sh && \ + echo 'until mysqladmin ping -h localhost --silent; do' >> /run_a_workflow.sh && \ + echo ' echo "Waiting for MySQL to be ready..."' >> /run_a_workflow.sh && \ + echo ' sleep 2' >> /run_a_workflow.sh && \ + echo 'done' >> /run_a_workflow.sh && \ + echo 'echo "MySQL is ready!"' >> /run_a_workflow.sh && \ + echo 'echo "Setting up database for Airflow..."' >> /run_a_workflow.sh && \ + echo 'mysql -u root -e "CREATE DATABASE airflow_db CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci";' >> /run_a_workflow.sh && \ + echo "mysql -u root -e \"CREATE USER 'airflow_user'@'%' IDENTIFIED BY 'airflow_pass';\"" >> /run_a_workflow.sh && \ + echo "mysql -u root -e \"GRANT ALL PRIVILEGES ON airflow_db.* TO 'airflow_user';\"" >> /run_a_workflow.sh && \ + echo 'export AIRFLOW_HOME="$(pwd)"/airflow/' >> /run_a_workflow.sh && \ + echo "airflow config list 1> /dev/null" >> /run_a_workflow.sh && \ + echo "sed -i ./airflow/airflow.cfg -e 's/sqlite:.*/mysql+mysqldb:\/\/airflow_user:airflow_pass@localhost:3306\/airflow_db/'" >> /run_a_workflow.sh && \ + echo 'airflow db migrate' >> /run_a_workflow.sh && \ + echo 'echo "Airflow database setup!"' >> /run_a_workflow.sh && \ + echo 'mkdir ./airflow/dags' >> /run_a_workflow.sh && \ + echo 'cp -r /tmp/airflow_translated_workflow ./airflow/dags/' >> /run_a_workflow.sh && \ + echo 'mv ./airflow/dags/airflow_translated_workflow/workflow.py ./airflow/dags/workflow.py' >> /run_a_workflow.sh && \ + echo 'airflow dags test $1' >> /run_a_workflow.sh && \ chmod +x /run_a_workflow.sh WORKDIR /home/wfcommons - -ENTRYPOINT ["/entrypoint.sh"] diff --git a/tests/translators/README.airflow b/tests/translators/README.airflow index 64b25cd1..386454db 100644 --- a/tests/translators/README.airflow +++ b/tests/translators/README.airflow @@ -51,14 +51,17 @@ A much simpler alternative is to use Docker. 1. Build the docker image - docker build -t wfcommons-dev -f Dockerfile.airflow . + docker build -t wfcommons/wfcommons-testing-airflow -f Dockerfile.airflow . (if building on a Mac, add the `--platform linux/amd64` argument after build above) + This dockerfile is meant for testing purposes (so it has some testing specific things in it) + + 2. Run the docker container in the directory to contains the translated workflow (see last section below) - docker run -it --rm -v .:/home/wfcommons/translated_workflow wfcommons-dev /bin/bash + docker run -it --rm -v .:/home/wfcommons/translated_workflow wfcommons/wfcommons-testing-airflow /bin/bash Running a translated workflow with Airflow diff --git a/tests/translators/test_translators.py b/tests/translators/test_translators.py index af13b206..23885771 100644 --- a/tests/translators/test_translators.py +++ b/tests/translators/test_translators.py @@ -92,7 +92,7 @@ def create_workflow_benchmark(): class TestTranslators: @pytest.mark.unit - # @pytest.mark.skip(reason="tmp") + @pytest.mark.skip(reason="tmp") def test_dask_translator(self) -> None: # Create workflow benchmark @@ -135,7 +135,7 @@ def test_dask_translator(self) -> None: @pytest.mark.unit - # @pytest.mark.skip(reason="tmp") + @pytest.mark.skip(reason="tmp") def test_parsl_translator(self) -> None: # Create workflow benchmark @@ -179,7 +179,7 @@ def test_parsl_translator(self) -> None: assert(num_completed_tasks == num_tasks) @pytest.mark.unit - # @pytest.mark.skip(reason="tmp") + @pytest.mark.skip(reason="tmp") def test_nextflow_translator(self) -> None: # Create workflow benchmark @@ -262,12 +262,7 @@ def test_airflow_translator(self) -> None: # Do the necessary copies (some ugly hardcoded stuff here) airflow_home = "/home/wfcommons/airflow/" # per the Dockerfile - exit_code, output = container.exec_run(f"cp -r /tmp/airflow_translated_workflow {airflow_home}/dags/",stdout=True, stderr=True) - exit_code, output = container.exec_run(f"mv {airflow_home}/dags/airflow_translated_workflow/workflow.py {airflow_home}/dags/",stdout=True, stderr=True) - # # Run the entry point script by hand (such a hack!) - # exit_code, output = container.exec_run(f"/bin/bash /entrypoint.sh",stdout=True, stderr=True) - # print(output) # Run the workflow! sys.stderr.write("Running the Airflow workflow on the container...\n") From 8b7f183f34ae8ca3d24fadb94ed2883b5f71fb95 Mon Sep 17 00:00:00 2001 From: Henri Casanova Date: Tue, 5 Aug 2025 21:46:12 -1000 Subject: [PATCH 27/38] Making the airflow docker container run as non-root --- tests/translators/Dockerfile.airflow | 8 +++++++- tests/translators/Dockerfile.nextflow | 4 +--- tests/translators/test_translators.py | 1 + 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/tests/translators/Dockerfile.airflow b/tests/translators/Dockerfile.airflow index 9cc15057..7c43bd44 100644 --- a/tests/translators/Dockerfile.airflow +++ b/tests/translators/Dockerfile.airflow @@ -73,4 +73,10 @@ RUN echo '#!/bin/bash' > /run_a_workflow.sh && \ echo 'airflow dags test $1' >> /run_a_workflow.sh && \ chmod +x /run_a_workflow.sh -WORKDIR /home/wfcommons +# Create wfcommons user +RUN useradd -ms /bin/bash wfcommons +RUN adduser wfcommons sudo +RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers +ENV PATH="$PATH:/home/wfcommons/.local/bin/" +USER wfcommons +WORKDIR /home/wfcommons \ No newline at end of file diff --git a/tests/translators/Dockerfile.nextflow b/tests/translators/Dockerfile.nextflow index 14259766..90997c53 100644 --- a/tests/translators/Dockerfile.nextflow +++ b/tests/translators/Dockerfile.nextflow @@ -38,7 +38,7 @@ RUN apt-get -y install stress-ng # Install Java RUN apt-get -y install openjdk-21-jdk -# Create wfcomomns user +# Create wfcommons user RUN useradd -ms /bin/bash wfcommons RUN adduser wfcommons sudo RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers @@ -49,5 +49,3 @@ WORKDIR /home/wfcommons # Install Nextflow and dependencies RUN curl -s https://get.nextflow.io | bash && sudo mv ./nextflow /usr/local/bin/ -# Add wfcommons user - diff --git a/tests/translators/test_translators.py b/tests/translators/test_translators.py index 23885771..136f7f2b 100644 --- a/tests/translators/test_translators.py +++ b/tests/translators/test_translators.py @@ -267,6 +267,7 @@ def test_airflow_translator(self) -> None: # Run the workflow! sys.stderr.write("Running the Airflow workflow on the container...\n") exit_code, output = container.exec_run(cmd="/bin/bash /run_a_workflow.sh Blast-Benchmark", stdout=True, stderr=True) + # print(output) # Kill the container container.remove(force=True) From b518a5a1215901645feef9705c053c153245d199 Mon Sep 17 00:00:00 2001 From: Henri Casanova Date: Tue, 5 Aug 2025 22:01:02 -1000 Subject: [PATCH 28/38] airflow translator debugging --- tests/translators/Dockerfile.airflow | 6 ++---- tests/translators/Dockerfile.dask | 2 +- tests/translators/Dockerfile.nextflow | 2 +- tests/translators/Dockerfile.parsl | 2 +- tests/translators/test_translators.py | 11 ++--------- 5 files changed, 7 insertions(+), 16 deletions(-) diff --git a/tests/translators/Dockerfile.airflow b/tests/translators/Dockerfile.airflow index 7c43bd44..f0f56ad7 100644 --- a/tests/translators/Dockerfile.airflow +++ b/tests/translators/Dockerfile.airflow @@ -1,4 +1,4 @@ -# docker build -t wfcommons-dev -f Dockerfile_Airflow . +# docker build -t wfcommons-dev -f Dockerfile.airflow . # docker run -it --rm -v .:/home/wfcommons/mount wfcommons-dev /bin/bash FROM ubuntu:noble @@ -45,9 +45,6 @@ RUN apt-get install -y python3-dev build-essential RUN apt-get install -y default-libmysqlclient-dev RUN python3 -m pip install --break-system-packages mysqlclient -# Setup directory -RUN mkdir /home/wfcommons - # Create an run_a_workflow script to start mysqld in the background and setup the Airflow DB, # copy things around, and the run the workflow @@ -78,5 +75,6 @@ RUN useradd -ms /bin/bash wfcommons RUN adduser wfcommons sudo RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers ENV PATH="$PATH:/home/wfcommons/.local/bin/" + USER wfcommons WORKDIR /home/wfcommons \ No newline at end of file diff --git a/tests/translators/Dockerfile.dask b/tests/translators/Dockerfile.dask index 48df02a5..4d5b79a4 100644 --- a/tests/translators/Dockerfile.dask +++ b/tests/translators/Dockerfile.dask @@ -1,4 +1,4 @@ -# docker build --platform amd64 -t wfcommons-dev-dask -f Dockerfile_Dask . +# docker build --platform amd64 -t wfcommons-dev-dask -f Dockerfile.dask . # docker run -it --rm -v `pwd`:/home/wfcommons wfcommons-dev-dask /bin/bash FROM amd64/ubuntu:noble diff --git a/tests/translators/Dockerfile.nextflow b/tests/translators/Dockerfile.nextflow index 90997c53..c083391d 100644 --- a/tests/translators/Dockerfile.nextflow +++ b/tests/translators/Dockerfile.nextflow @@ -1,4 +1,4 @@ -# docker build --platform amd64 -t wfcommons-dev-nextflow -f Dockerfile_Nextflow . +# docker build --platform amd64 -t wfcommons-dev-nextflow -f Dockerfile.nextflow . # docker run -it --rm -v `pwd`:/home/wfcommons wfcommons-dev-nextflow /bin/bash FROM amd64/ubuntu:noble diff --git a/tests/translators/Dockerfile.parsl b/tests/translators/Dockerfile.parsl index dd92d5a7..0d7e932c 100644 --- a/tests/translators/Dockerfile.parsl +++ b/tests/translators/Dockerfile.parsl @@ -1,4 +1,4 @@ -# docker build --platform amd64 -t wfcommons-dev -f Dockerfile_Parsl . +# docker build --platform amd64 -t wfcommons-dev -f Dockerfile.parsl . # docker run -it --rm -v `pwd`:/home/wfcommons wfcommons-dev /bin/bash FROM amd64/ubuntu:noble diff --git a/tests/translators/test_translators.py b/tests/translators/test_translators.py index 136f7f2b..c7743ad7 100644 --- a/tests/translators/test_translators.py +++ b/tests/translators/test_translators.py @@ -241,32 +241,25 @@ def test_airflow_translator(self) -> None: translator.translate(output_folder=dirpath) # Starting the Docker container - container = start_docker_container("airflow", str_dirpath, "/home/wfcommons/") + container = start_docker_container("airflow", str_dirpath, "/home/wfcommons/", command=["tail", "-f", "/dev/null"]) sys.stderr.write(f"Container status: {container.status}\n") container.reload() sys.stderr.write(f"Container updated status: {container.status}\n") logs = container.logs(stdout=True, stderr=True).decode("utf-8") sys.stderr.write(f"Container logs:\n{logs}\n") - # container = start_docker_container("airflow", str_dirpath, "/home/wfcommons/") - # Installing WfCommons on container install_WfCommons_on_container(container) - # Copy over the wfbench and cpu-benchmark executables to where they should go exit_code, output = container.exec_run("sudo cp -f /tmp/WfCommons/bin/wfbench /usr/local/bin/", stdout=True, stderr=True) exit_code, output = container.exec_run("sudo cp -f /tmp/WfCommons/bin/cpu-benchmark /usr/local/bin/", stdout=True, stderr=True) - # Do the necessary copies (some ugly hardcoded stuff here) - airflow_home = "/home/wfcommons/airflow/" # per the Dockerfile - - # Run the workflow! sys.stderr.write("Running the Airflow workflow on the container...\n") - exit_code, output = container.exec_run(cmd="/bin/bash /run_a_workflow.sh Blast-Benchmark", stdout=True, stderr=True) + exit_code, output = container.exec_run(cmd="sudo /bin/bash /run_a_workflow.sh Blast-Benchmark", stdout=True, stderr=True) # print(output) # Kill the container From 4773f02b8145a3ffb4fc4f1ab309e53f8d14b055 Mon Sep 17 00:00:00 2001 From: Henri Casanova Date: Tue, 5 Aug 2025 22:12:16 -1000 Subject: [PATCH 29/38] Fixed airflow Docker image --- tests/translators/Dockerfile.airflow | 2 +- tests/translators/test_translators.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/translators/Dockerfile.airflow b/tests/translators/Dockerfile.airflow index f0f56ad7..6f7c7498 100644 --- a/tests/translators/Dockerfile.airflow +++ b/tests/translators/Dockerfile.airflow @@ -1,7 +1,7 @@ # docker build -t wfcommons-dev -f Dockerfile.airflow . # docker run -it --rm -v .:/home/wfcommons/mount wfcommons-dev /bin/bash -FROM ubuntu:noble +FROM amd64/ubuntu:noble LABEL org.containers.image.authors="sukaryor@hawaii.edu" diff --git a/tests/translators/test_translators.py b/tests/translators/test_translators.py index c7743ad7..7c87cdb1 100644 --- a/tests/translators/test_translators.py +++ b/tests/translators/test_translators.py @@ -241,7 +241,7 @@ def test_airflow_translator(self) -> None: translator.translate(output_folder=dirpath) # Starting the Docker container - container = start_docker_container("airflow", str_dirpath, "/home/wfcommons/", command=["tail", "-f", "/dev/null"]) + container = start_docker_container("airflow", str_dirpath, "/home/wfcommons/") sys.stderr.write(f"Container status: {container.status}\n") container.reload() sys.stderr.write(f"Container updated status: {container.status}\n") From fa97b05b7bc6e36f452d60ce22e7c937c25aaa0b Mon Sep 17 00:00:00 2001 From: Henri Casanova Date: Tue, 5 Aug 2025 22:18:16 -1000 Subject: [PATCH 30/38] Re-establishing some tests --- tests/translators/test_translators.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/translators/test_translators.py b/tests/translators/test_translators.py index 7c87cdb1..db9e33f5 100644 --- a/tests/translators/test_translators.py +++ b/tests/translators/test_translators.py @@ -92,7 +92,7 @@ def create_workflow_benchmark(): class TestTranslators: @pytest.mark.unit - @pytest.mark.skip(reason="tmp") + # @pytest.mark.skip(reason="tmp") def test_dask_translator(self) -> None: # Create workflow benchmark @@ -135,7 +135,7 @@ def test_dask_translator(self) -> None: @pytest.mark.unit - @pytest.mark.skip(reason="tmp") + # @pytest.mark.skip(reason="tmp") def test_parsl_translator(self) -> None: # Create workflow benchmark @@ -179,7 +179,7 @@ def test_parsl_translator(self) -> None: assert(num_completed_tasks == num_tasks) @pytest.mark.unit - @pytest.mark.skip(reason="tmp") + # @pytest.mark.skip(reason="tmp") def test_nextflow_translator(self) -> None: # Create workflow benchmark @@ -242,11 +242,11 @@ def test_airflow_translator(self) -> None: # Starting the Docker container container = start_docker_container("airflow", str_dirpath, "/home/wfcommons/") - sys.stderr.write(f"Container status: {container.status}\n") - container.reload() - sys.stderr.write(f"Container updated status: {container.status}\n") - logs = container.logs(stdout=True, stderr=True).decode("utf-8") - sys.stderr.write(f"Container logs:\n{logs}\n") + # sys.stderr.write(f"Container status: {container.status}\n") + # container.reload() + # sys.stderr.write(f"Container updated status: {container.status}\n") + # logs = container.logs(stdout=True, stderr=True).decode("utf-8") + # sys.stderr.write(f"Container logs:\n{logs}\n") # Installing WfCommons on container install_WfCommons_on_container(container) From dbaf9adeb5948d90d6b02bbccddcba1d639205f1 Mon Sep 17 00:00:00 2001 From: Henri Casanova Date: Tue, 5 Aug 2025 22:22:10 -1000 Subject: [PATCH 31/38] gitignore update --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index 00cf563f..98a466e4 100644 --- a/.gitignore +++ b/.gitignore @@ -55,4 +55,8 @@ bin/gpu_benchmark /wfcommons/wfc-1.3_to_dask/out /wfcommons/wfc-1.3_to_dask/htmlcov +# Coverage stuff +.coverage +coverage.xml + test/ From b711f5ca8bda939845ed9e4f9ea65b67e3ca6ff3 Mon Sep 17 00:00:00 2001 From: Henri Casanova Date: Tue, 5 Aug 2025 22:53:52 -1000 Subject: [PATCH 32/38] setup.py improvements --- setup.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/setup.py b/setup.py index 0d66cc24..8c9ac89a 100644 --- a/setup.py +++ b/setup.py @@ -20,6 +20,16 @@ from setuptools.command.build_ext import build_ext +def is_writable(path): + try: + test_file = os.path.join(path, '.write_test_tmp') + with open(test_file, 'w') as f: + f.write('test') + os.remove(test_file) + return True + except Exception: + return False + class Build(build_ext): """Customized setuptools build command - builds protos on build.""" @@ -36,6 +46,11 @@ def run(self): if not pathlib.Path(scripts_dir).is_dir(): scripts_dir = "/usr/local/bin/" + if not pathlib.Path(scripts_dir).is_dir() or not is_writable(scripts_dir): + sys.stderr.write(f"Error: {scripts_dir} is not writable. Please install as root or use --user.\n") + sys.exit(1) + + for executable in ["cpu-benchmark", "wfbench"]: source_path = os.path.join("bin", executable) target_path = os.path.join(scripts_dir, executable) From 215d93e3ea45892bf42711e56dd16744d612c7fb Mon Sep 17 00:00:00 2001 From: Henri Casanova Date: Wed, 6 Aug 2025 12:45:55 -1000 Subject: [PATCH 33/38] Revamped build process --- MANIFEST.in | 4 +-- Makefile | 4 +-- setup.py | 43 +++++++++++++----------- {bin => wfcommons/bin}/cpu-benchmark.cpp | 0 {bin => wfcommons/bin}/gpu_benchmark.cu | 0 {bin => wfcommons/bin}/gpu_benchmark.h | 0 {bin => wfcommons/bin}/kernels.cu | 0 {bin => wfcommons/bin}/kernels.cuh | 0 {bin => wfcommons/bin}/wfbench | 0 wfcommons/cpu_benchmark_cli.py | 7 ++++ 10 files changed, 34 insertions(+), 24 deletions(-) rename {bin => wfcommons/bin}/cpu-benchmark.cpp (100%) rename {bin => wfcommons/bin}/gpu_benchmark.cu (100%) rename {bin => wfcommons/bin}/gpu_benchmark.h (100%) rename {bin => wfcommons/bin}/kernels.cu (100%) rename {bin => wfcommons/bin}/kernels.cuh (100%) rename {bin => wfcommons/bin}/wfbench (100%) create mode 100644 wfcommons/cpu_benchmark_cli.py diff --git a/MANIFEST.in b/MANIFEST.in index dd6f4c00..ad94581e 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -45,5 +45,5 @@ graft wfcommons/wfchef/recipes/bwa/microstructures graft wfcommons/wfchef/recipes/bwa graft wfcommons/wfbench/translator/templates include Makefile -include bin/cpu-benchmark.cpp -include bin/wfbench +include wfcommons/bin/cpu-benchmark.cpp +include wfcommons/bin/wfbench diff --git a/Makefile b/Makefile index c8505c7a..b6e81904 100644 --- a/Makefile +++ b/Makefile @@ -1,9 +1,9 @@ # Makefile -objects = bin/cpu-benchmark.o +objects = wfcommons/bin/cpu-benchmark.o CXX= g++ CPPFLAGS= -std=c++11 -execname = bin/cpu-benchmark +execname = wfcommons/bin/cpu-benchmark # compile $(execname): $(objects) diff --git a/setup.py b/setup.py index 8c9ac89a..b0fccbba 100644 --- a/setup.py +++ b/setup.py @@ -42,24 +42,24 @@ def run(self): # Do a by-hand copy of cpu-benchmark to the bin directory, as this is so # hard to do using data_file in the setup() declaration - scripts_dir = os.path.join(site.USER_BASE, "bin") - if not pathlib.Path(scripts_dir).is_dir(): - scripts_dir = "/usr/local/bin/" - - if not pathlib.Path(scripts_dir).is_dir() or not is_writable(scripts_dir): - sys.stderr.write(f"Error: {scripts_dir} is not writable. Please install as root or use --user.\n") - sys.exit(1) - - - for executable in ["cpu-benchmark", "wfbench"]: - source_path = os.path.join("bin", executable) - target_path = os.path.join(scripts_dir, executable) - # Ensure it's executable (just in case) - st = os.stat(source_path) - os.chmod(source_path, st.st_mode | stat.S_IEXEC) - # Copy to scripts directory - shutil.copy2(source_path, target_path) - sys.stderr.write(f"Copied {source_path} to {target_path}\n") + #scripts_dir = os.path.join(site.USER_BASE, "bin") + #if not pathlib.Path(scripts_dir).is_dir(): + # scripts_dir = "/usr/local/bin/" +# +# if not pathlib.Path(scripts_dir).is_dir() or not is_writable(scripts_dir): +# sys.stderr.write(f"Error: {scripts_dir} is not writable. Please install as root or use --user.\n") +# sys.exit(1) +# +# +# for executable in ["cpu-benchmark", "wfbench"]: +# source_path = os.path.join("bin", executable) +# target_path = os.path.join(scripts_dir, executable) +# # Ensure it's executable (just in case) +# st = os.stat(source_path) +# os.chmod(source_path, st.st_mode | stat.S_IEXEC) +# # Copy to scripts directory +# shutil.copy2(source_path, target_path) +# sys.stderr.write(f"Copied {source_path} to {target_path}\n") setup( packages=find_packages(), @@ -71,10 +71,13 @@ def run(self): #data_files=[ # (user_bin_dir, ['bin/cpu-benchmark']) #], - # scripts=['bin/wfbench'], + scripts=['wfcommons/bin/wfbench'], + #package_data={"wfcommons": ["bin/cpu-benchmark", "bin/wfbench"] + #}, entry_points={ 'console_scripts': [ - 'wfchef=wfcommons.wfchef.chef:main' + 'wfchef=wfcommons.wfchef.chef:main', + 'cpu-benchmark=wfcommons.cpu_benchmark_cli:main' ], 'workflow_recipes': [ 'epigenomics_recipe = wfcommons.wfchef.recipes:EpigenomicsRecipe', diff --git a/bin/cpu-benchmark.cpp b/wfcommons/bin/cpu-benchmark.cpp similarity index 100% rename from bin/cpu-benchmark.cpp rename to wfcommons/bin/cpu-benchmark.cpp diff --git a/bin/gpu_benchmark.cu b/wfcommons/bin/gpu_benchmark.cu similarity index 100% rename from bin/gpu_benchmark.cu rename to wfcommons/bin/gpu_benchmark.cu diff --git a/bin/gpu_benchmark.h b/wfcommons/bin/gpu_benchmark.h similarity index 100% rename from bin/gpu_benchmark.h rename to wfcommons/bin/gpu_benchmark.h diff --git a/bin/kernels.cu b/wfcommons/bin/kernels.cu similarity index 100% rename from bin/kernels.cu rename to wfcommons/bin/kernels.cu diff --git a/bin/kernels.cuh b/wfcommons/bin/kernels.cuh similarity index 100% rename from bin/kernels.cuh rename to wfcommons/bin/kernels.cuh diff --git a/bin/wfbench b/wfcommons/bin/wfbench similarity index 100% rename from bin/wfbench rename to wfcommons/bin/wfbench diff --git a/wfcommons/cpu_benchmark_cli.py b/wfcommons/cpu_benchmark_cli.py new file mode 100644 index 00000000..f06d1659 --- /dev/null +++ b/wfcommons/cpu_benchmark_cli.py @@ -0,0 +1,7 @@ +import subprocess +from pathlib import Path + +def main(): + bin_path = Path(__file__).parent / "bin" / "cpu-benchmark" + subprocess.run([str(bin_path)] + sys.argv[1:]) + From e85b318178e659f52f3403f681b5989f6b9f7a54 Mon Sep 17 00:00:00 2001 From: Henri Casanova Date: Wed, 6 Aug 2025 13:18:52 -1000 Subject: [PATCH 34/38] Removing the -e flag from the pip install in the workflow (otherwise data_files are NOT installed!!!) --- .github/workflows/build.yml | 2 +- setup.py | 11 ++++++----- tests/translators/test_translators.py | 19 ++++++++++--------- wfcommons/cpu_benchmark_cli.py | 7 ------- .../wfbench/translator/abstract_translator.py | 16 +++++++++------- 5 files changed, 26 insertions(+), 29 deletions(-) delete mode 100644 wfcommons/cpu_benchmark_cli.py diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index dd776553..df11c926 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -28,7 +28,7 @@ jobs: - name: Check package install run: | - pip install -e . + pip install . - name: Run tests run: pytest -s -v -m unit --cov=wfcommons tests/ diff --git a/setup.py b/setup.py index b0fccbba..ac45c460 100644 --- a/setup.py +++ b/setup.py @@ -68,16 +68,17 @@ def run(self): cmdclass={ 'build_ext': Build, }, - #data_files=[ - # (user_bin_dir, ['bin/cpu-benchmark']) - #], - scripts=['wfcommons/bin/wfbench'], + data_files=[ + ('bin', ['wfcommons/bin/cpu-benchmark']) + ], + scripts=[ + 'wfcommons/bin/wfbench' + ], #package_data={"wfcommons": ["bin/cpu-benchmark", "bin/wfbench"] #}, entry_points={ 'console_scripts': [ 'wfchef=wfcommons.wfchef.chef:main', - 'cpu-benchmark=wfcommons.cpu_benchmark_cli:main' ], 'workflow_recipes': [ 'epigenomics_recipe = wfcommons.wfchef.recipes:EpigenomicsRecipe', diff --git a/tests/translators/test_translators.py b/tests/translators/test_translators.py index db9e33f5..db5c0506 100644 --- a/tests/translators/test_translators.py +++ b/tests/translators/test_translators.py @@ -116,26 +116,27 @@ def test_dask_translator(self) -> None: install_WfCommons_on_container(container) # Copy over the wfbench and cpu-benchmark executables to where they should go on the container - exit_code, output = container.exec_run("sudo cp -f /tmp/WfCommons/bin/wfbench " + str_dirpath + "bin/", stdout=True, stderr=True) - exit_code, output = container.exec_run("sudo cp -f /tmp/WfCommons/bin/cpu-benchmark " + str_dirpath + "bin/", stdout=True, stderr=True) + exit_code, output = container.exec_run("sudo cp -f /tmp/WfCommons/wfcommons/bin/wfbench " + str_dirpath + "bin/", stdout=True, stderr=True) + exit_code, output = container.exec_run("sudo cp -f /tmp/WfCommons/wfcommons/bin/cpu-benchmark " + str_dirpath + "bin/", stdout=True, stderr=True) # Run the workflow! sys.stderr.write("Running the Dask workflow on the container...\n") exit_code, output = container.exec_run("python ./dask_workflow.py", stdout=True, stderr=True) + print(output) num_completed_tasks = output.decode().count("completed!") # TODO: This is pretty lame # Kill the container - container.remove(force=True) + # container.remove(force=True) # Do sanity checks - sys.stderr.write("Checking sanity...\n") - assert(exit_code == 0) - assert(num_completed_tasks == num_tasks) + # sys.stderr.write("Checking sanity...\n") + # assert(exit_code == 0) + # assert(num_completed_tasks == num_tasks) # TODO: Look at the (I think) generated run.json file on the container @pytest.mark.unit - # @pytest.mark.skip(reason="tmp") + @pytest.mark.skip(reason="tmp") def test_parsl_translator(self) -> None: # Create workflow benchmark @@ -179,7 +180,7 @@ def test_parsl_translator(self) -> None: assert(num_completed_tasks == num_tasks) @pytest.mark.unit - # @pytest.mark.skip(reason="tmp") + @pytest.mark.skip(reason="tmp") def test_nextflow_translator(self) -> None: # Create workflow benchmark @@ -223,7 +224,7 @@ def test_nextflow_translator(self) -> None: @pytest.mark.unit - # @pytest.mark.skip(reason="tmp") + @pytest.mark.skip(reason="tmp") def test_airflow_translator(self) -> None: # Create workflow benchmark diff --git a/wfcommons/cpu_benchmark_cli.py b/wfcommons/cpu_benchmark_cli.py deleted file mode 100644 index f06d1659..00000000 --- a/wfcommons/cpu_benchmark_cli.py +++ /dev/null @@ -1,7 +0,0 @@ -import subprocess -from pathlib import Path - -def main(): - bin_path = Path(__file__).parent / "bin" / "cpu-benchmark" - subprocess.run([str(bin_path)] + sys.argv[1:]) - diff --git a/wfcommons/wfbench/translator/abstract_translator.py b/wfcommons/wfbench/translator/abstract_translator.py index 0cf294eb..30c00cf1 100644 --- a/wfcommons/wfbench/translator/abstract_translator.py +++ b/wfcommons/wfbench/translator/abstract_translator.py @@ -94,13 +94,15 @@ def _copy_binary_files(self, output_folder: pathlib.Path) -> None: bin_folder.mkdir(exist_ok=True) # Get the python executable dir # python_executable_dir = os.path.dirname(sys.executable) - python_executable_dir = os.path.join(site.getuserbase(), 'bin') - if not pathlib.Path(python_executable_dir).exists(): - python_executable_dir = "/usr/local/bin" - # shutil.copy(shutil.which("wfbench"), bin_folder) - shutil.copy(python_executable_dir + "/wfbench", bin_folder) - # shutil.copy(shutil.which("cpu-benchmark"), bin_folder) - shutil.copy(python_executable_dir + "/cpu-benchmark", bin_folder) + # python_executable_dir = os.path.join(site.getuserbase(), 'bin') + # if not pathlib.Path(python_executable_dir).exists(): + # python_executable_dir = "/usr/local/bin" + print("WHICH: ", shutil.which("wfbench")) + print("WHICH: ", shutil.which("cpu-benchmark")) + shutil.copy(shutil.which("wfbench"), bin_folder) + # shutil.copy(python_executable_dir + "/wfbench", bin_folder) + shutil.copy(shutil.which("cpu-benchmark"), bin_folder) + # shutil.copy(python_executable_dir + "/cpu-benchmark", bin_folder) def _generate_input_files(self, output_folder: pathlib.Path) -> None: """ From 72e58b3db28ba65515d0a596ba32ecdc2a94c634 Mon Sep 17 00:00:00 2001 From: Henri Casanova Date: Wed, 6 Aug 2025 13:52:49 -1000 Subject: [PATCH 35/38] Removing "-e" from pip install, so that data_files are installed!! --- .github/workflows/build.yml | 2 +- tests/translators/test_translators.py | 44 ++++++++++--------- .../wfbench/translator/abstract_translator.py | 11 +++-- 3 files changed, 31 insertions(+), 26 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index df11c926..dc92a9f3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -31,7 +31,7 @@ jobs: pip install . - name: Run tests - run: pytest -s -v -m unit --cov=wfcommons tests/ + run: python3 -m pytest -s -v -m unit --cov=wfcommons tests/ - name: Upload coverage if: github.ref == 'refs/heads/main' diff --git a/tests/translators/test_translators.py b/tests/translators/test_translators.py index db5c0506..7d167252 100644 --- a/tests/translators/test_translators.py +++ b/tests/translators/test_translators.py @@ -68,13 +68,13 @@ def install_WfCommons_on_container(container): container.put_archive(target_path, tar_data) exit_code, output = container.exec_run("sudo /bin/rm -rf /tmp/WfCommons/build/", stdout=True, stderr=True) exit_code, output = container.exec_run("sudo /bin/rm -rf /tmp/WfCommons/*.egg-info/", stdout=True, stderr=True) - exit_code, output = container.exec_run("sudo /bin/rm -rf /tmp/WfCommons/bin/cpu-benchmark.o", stdout=True, + exit_code, output = container.exec_run("sudo /bin/rm -rf /tmp/WfCommons/wfcommons/bin/cpu-benchmark.o", stdout=True, stderr=True) - exit_code, output = container.exec_run("sudo /bin/rm -rf /tmp/WfCommons/bin/cpu-benchmark", stdout=True, + exit_code, output = container.exec_run("sudo /bin/rm -rf /tmp/WfCommons/wfcommons/bin/cpu-benchmark", stdout=True, stderr=True) # Install WfCommons on the container (to install wfbench and cpu-benchmark really) - exit_code, output = container.exec_run("sudo python3 -m pip install -e . --break-system-packages", + exit_code, output = container.exec_run("sudo python3 -m pip install . --break-system-packages", workdir="/tmp/WfCommons", stdout=True, stderr=True) def create_workflow_benchmark(): @@ -116,27 +116,27 @@ def test_dask_translator(self) -> None: install_WfCommons_on_container(container) # Copy over the wfbench and cpu-benchmark executables to where they should go on the container - exit_code, output = container.exec_run("sudo cp -f /tmp/WfCommons/wfcommons/bin/wfbench " + str_dirpath + "bin/", stdout=True, stderr=True) - exit_code, output = container.exec_run("sudo cp -f /tmp/WfCommons/wfcommons/bin/cpu-benchmark " + str_dirpath + "bin/", stdout=True, stderr=True) + exit_code, output = container.exec_run(["sh", "-c", "sudo cp -f `which wfbench` " + str_dirpath + "bin/"], stdout=True, stderr=True) + exit_code, output = container.exec_run(["sh", "-c", "sudo cp -f `which cpu-benchmark` " + str_dirpath + "bin/"], stdout=True, stderr=True) # Run the workflow! sys.stderr.write("Running the Dask workflow on the container...\n") exit_code, output = container.exec_run("python ./dask_workflow.py", stdout=True, stderr=True) - print(output) + # print(output) num_completed_tasks = output.decode().count("completed!") # TODO: This is pretty lame # Kill the container - # container.remove(force=True) + container.remove(force=True) # Do sanity checks - # sys.stderr.write("Checking sanity...\n") - # assert(exit_code == 0) - # assert(num_completed_tasks == num_tasks) + sys.stderr.write("Checking sanity...\n") + assert(exit_code == 0) + assert(num_completed_tasks == num_tasks) # TODO: Look at the (I think) generated run.json file on the container @pytest.mark.unit - @pytest.mark.skip(reason="tmp") + # @pytest.mark.skip(reason="tmp") def test_parsl_translator(self) -> None: # Create workflow benchmark @@ -160,8 +160,10 @@ def test_parsl_translator(self) -> None: install_WfCommons_on_container(container) # Copy over the wfbench and cpu-benchmark executables to where they should go - exit_code, output = container.exec_run("sudo cp -f /tmp/WfCommons/bin/wfbench " + str_dirpath + "bin/", stdout=True, stderr=True) - exit_code, output = container.exec_run("sudo cp -f /tmp/WfCommons/bin/cpu-benchmark " + str_dirpath + "bin/", stdout=True, stderr=True) + exit_code, output = container.exec_run(["sh", "-c", "sudo cp -f `which wfbench` " + str_dirpath + "bin/"], + stdout=True, stderr=True) + exit_code, output = container.exec_run(["sh", "-c", "sudo cp -f `which cpu-benchmark` " + str_dirpath + "bin/"], + stdout=True, stderr=True) # Run the workflow! sys.stderr.write("Running the Parsl workflow on the container...\n") @@ -180,7 +182,7 @@ def test_parsl_translator(self) -> None: assert(num_completed_tasks == num_tasks) @pytest.mark.unit - @pytest.mark.skip(reason="tmp") + # @pytest.mark.skip(reason="tmp") def test_nextflow_translator(self) -> None: # Create workflow benchmark @@ -204,9 +206,9 @@ def test_nextflow_translator(self) -> None: install_WfCommons_on_container(container) # Copy over the wfbench and cpu-benchmark executables to where they should go - exit_code, output = container.exec_run("sudo cp -f /tmp/WfCommons/bin/wfbench " + str_dirpath + "bin/", + exit_code, output = container.exec_run(["sh", "-c", "sudo cp -f `which wfbench` " + str_dirpath + "bin/"], stdout=True, stderr=True) - exit_code, output = container.exec_run("sudo cp -f /tmp/WfCommons/bin/cpu-benchmark " + str_dirpath + "bin/", + exit_code, output = container.exec_run(["sh", "-c", "sudo cp -f `which cpu-benchmark` " + str_dirpath + "bin/"], stdout=True, stderr=True) # Run the workflow! @@ -224,7 +226,7 @@ def test_nextflow_translator(self) -> None: @pytest.mark.unit - @pytest.mark.skip(reason="tmp") + # @pytest.mark.skip(reason="tmp") def test_airflow_translator(self) -> None: # Create workflow benchmark @@ -253,10 +255,10 @@ def test_airflow_translator(self) -> None: install_WfCommons_on_container(container) # Copy over the wfbench and cpu-benchmark executables to where they should go - exit_code, output = container.exec_run("sudo cp -f /tmp/WfCommons/bin/wfbench /usr/local/bin/", - stdout=True, stderr=True) - exit_code, output = container.exec_run("sudo cp -f /tmp/WfCommons/bin/cpu-benchmark /usr/local/bin/", - stdout=True, stderr=True) + # exit_code, output = container.exec_run("sudo cp -f /tmp/WfCommons/bin/wfbench /usr/local/bin/", + # stdout=True, stderr=True) + # exit_code, output = container.exec_run("sudo cp -f /tmp/WfCommons/bin/cpu-benchmark /usr/local/bin/", + # stdout=True, stderr=True) # Run the workflow! sys.stderr.write("Running the Airflow workflow on the container...\n") diff --git a/wfcommons/wfbench/translator/abstract_translator.py b/wfcommons/wfbench/translator/abstract_translator.py index 30c00cf1..2f1f2de2 100644 --- a/wfcommons/wfbench/translator/abstract_translator.py +++ b/wfcommons/wfbench/translator/abstract_translator.py @@ -92,18 +92,21 @@ def _copy_binary_files(self, output_folder: pathlib.Path) -> None: """ bin_folder = output_folder.joinpath("bin") bin_folder.mkdir(exist_ok=True) + + # Previously attempted hack: # Get the python executable dir # python_executable_dir = os.path.dirname(sys.executable) # python_executable_dir = os.path.join(site.getuserbase(), 'bin') # if not pathlib.Path(python_executable_dir).exists(): # python_executable_dir = "/usr/local/bin" - print("WHICH: ", shutil.which("wfbench")) - print("WHICH: ", shutil.which("cpu-benchmark")) - shutil.copy(shutil.which("wfbench"), bin_folder) # shutil.copy(python_executable_dir + "/wfbench", bin_folder) - shutil.copy(shutil.which("cpu-benchmark"), bin_folder) # shutil.copy(python_executable_dir + "/cpu-benchmark", bin_folder) + # print("WHICH: ", shutil.which("wfbench")) + # print("WHICH: ", shutil.which("cpu-benchmark")) + shutil.copy(shutil.which("wfbench"), bin_folder) + shutil.copy(shutil.which("cpu-benchmark"), bin_folder) + def _generate_input_files(self, output_folder: pathlib.Path) -> None: """ Generate workflow input files into workflow benchmark's data folder. From 73ab2dfc9312d43e5e183acf1bd5dad8eedb5e11 Mon Sep 17 00:00:00 2001 From: Henri Casanova Date: Wed, 6 Aug 2025 14:00:20 -1000 Subject: [PATCH 36/38] Dead code / commented out code removal --- setup.py | 41 +------------------ .../wfbench/translator/abstract_translator.py | 11 ----- 2 files changed, 1 insertion(+), 51 deletions(-) diff --git a/setup.py b/setup.py index ac45c460..ff657679 100644 --- a/setup.py +++ b/setup.py @@ -9,58 +9,21 @@ # (at your option) any later version. import sys -import os -import stat -import shutil -import site -import pathlib import subprocess from setuptools import setup, find_packages from setuptools.command.build_ext import build_ext - -def is_writable(path): - try: - test_file = os.path.join(path, '.write_test_tmp') - with open(test_file, 'w') as f: - f.write('test') - os.remove(test_file) - return True - except Exception: - return False - class Build(build_ext): """Customized setuptools build command - builds protos on build.""" def run(self): protoc_command = ["make"] if subprocess.call(protoc_command) != 0: - print("Error: 'make' is not installed. Please install 'make' and try again.") + sys.stderr.write("Error: 'make' is not installed. Please install 'make' and try again.\n") sys.exit(-1) super().run() - # Do a by-hand copy of cpu-benchmark to the bin directory, as this is so - # hard to do using data_file in the setup() declaration - #scripts_dir = os.path.join(site.USER_BASE, "bin") - #if not pathlib.Path(scripts_dir).is_dir(): - # scripts_dir = "/usr/local/bin/" -# -# if not pathlib.Path(scripts_dir).is_dir() or not is_writable(scripts_dir): -# sys.stderr.write(f"Error: {scripts_dir} is not writable. Please install as root or use --user.\n") -# sys.exit(1) -# -# -# for executable in ["cpu-benchmark", "wfbench"]: -# source_path = os.path.join("bin", executable) -# target_path = os.path.join(scripts_dir, executable) -# # Ensure it's executable (just in case) -# st = os.stat(source_path) -# os.chmod(source_path, st.st_mode | stat.S_IEXEC) -# # Copy to scripts directory -# shutil.copy2(source_path, target_path) -# sys.stderr.write(f"Copied {source_path} to {target_path}\n") - setup( packages=find_packages(), include_package_data=True, @@ -74,8 +37,6 @@ def run(self): scripts=[ 'wfcommons/bin/wfbench' ], - #package_data={"wfcommons": ["bin/cpu-benchmark", "bin/wfbench"] - #}, entry_points={ 'console_scripts': [ 'wfchef=wfcommons.wfchef.chef:main', diff --git a/wfcommons/wfbench/translator/abstract_translator.py b/wfcommons/wfbench/translator/abstract_translator.py index 2f1f2de2..1c491e06 100644 --- a/wfcommons/wfbench/translator/abstract_translator.py +++ b/wfcommons/wfbench/translator/abstract_translator.py @@ -93,17 +93,6 @@ def _copy_binary_files(self, output_folder: pathlib.Path) -> None: bin_folder = output_folder.joinpath("bin") bin_folder.mkdir(exist_ok=True) - # Previously attempted hack: - # Get the python executable dir - # python_executable_dir = os.path.dirname(sys.executable) - # python_executable_dir = os.path.join(site.getuserbase(), 'bin') - # if not pathlib.Path(python_executable_dir).exists(): - # python_executable_dir = "/usr/local/bin" - # shutil.copy(python_executable_dir + "/wfbench", bin_folder) - # shutil.copy(python_executable_dir + "/cpu-benchmark", bin_folder) - - # print("WHICH: ", shutil.which("wfbench")) - # print("WHICH: ", shutil.which("cpu-benchmark")) shutil.copy(shutil.which("wfbench"), bin_folder) shutil.copy(shutil.which("cpu-benchmark"), bin_folder) From 0572c915ee2b7b035d8bd6734ea3c83a5e1d999e Mon Sep 17 00:00:00 2001 From: Henri Casanova Date: Wed, 6 Aug 2025 14:51:51 -1000 Subject: [PATCH 37/38] Put bin directory back in the root directory (for now) --- MANIFEST.in | 4 ++-- Makefile | 4 ++-- {wfcommons/bin => bin}/cpu-benchmark.cpp | 0 {wfcommons/bin => bin}/gpu_benchmark.cu | 0 {wfcommons/bin => bin}/gpu_benchmark.h | 0 {wfcommons/bin => bin}/kernels.cu | 0 {wfcommons/bin => bin}/kernels.cuh | 0 {wfcommons/bin => bin}/wfbench | 0 setup.py | 4 ++-- tests/translators/test_translators.py | 4 ++-- 10 files changed, 8 insertions(+), 8 deletions(-) rename {wfcommons/bin => bin}/cpu-benchmark.cpp (100%) rename {wfcommons/bin => bin}/gpu_benchmark.cu (100%) rename {wfcommons/bin => bin}/gpu_benchmark.h (100%) rename {wfcommons/bin => bin}/kernels.cu (100%) rename {wfcommons/bin => bin}/kernels.cuh (100%) rename {wfcommons/bin => bin}/wfbench (100%) diff --git a/MANIFEST.in b/MANIFEST.in index ad94581e..dd6f4c00 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -45,5 +45,5 @@ graft wfcommons/wfchef/recipes/bwa/microstructures graft wfcommons/wfchef/recipes/bwa graft wfcommons/wfbench/translator/templates include Makefile -include wfcommons/bin/cpu-benchmark.cpp -include wfcommons/bin/wfbench +include bin/cpu-benchmark.cpp +include bin/wfbench diff --git a/Makefile b/Makefile index b6e81904..6994cc63 100644 --- a/Makefile +++ b/Makefile @@ -1,9 +1,9 @@ # Makefile -objects = wfcommons/bin/cpu-benchmark.o +objects = bin/cpu-benchmark.o CXX= g++ CPPFLAGS= -std=c++11 -execname = wfcommons/bin/cpu-benchmark +execname =bin/cpu-benchmark # compile $(execname): $(objects) diff --git a/wfcommons/bin/cpu-benchmark.cpp b/bin/cpu-benchmark.cpp similarity index 100% rename from wfcommons/bin/cpu-benchmark.cpp rename to bin/cpu-benchmark.cpp diff --git a/wfcommons/bin/gpu_benchmark.cu b/bin/gpu_benchmark.cu similarity index 100% rename from wfcommons/bin/gpu_benchmark.cu rename to bin/gpu_benchmark.cu diff --git a/wfcommons/bin/gpu_benchmark.h b/bin/gpu_benchmark.h similarity index 100% rename from wfcommons/bin/gpu_benchmark.h rename to bin/gpu_benchmark.h diff --git a/wfcommons/bin/kernels.cu b/bin/kernels.cu similarity index 100% rename from wfcommons/bin/kernels.cu rename to bin/kernels.cu diff --git a/wfcommons/bin/kernels.cuh b/bin/kernels.cuh similarity index 100% rename from wfcommons/bin/kernels.cuh rename to bin/kernels.cuh diff --git a/wfcommons/bin/wfbench b/bin/wfbench similarity index 100% rename from wfcommons/bin/wfbench rename to bin/wfbench diff --git a/setup.py b/setup.py index ff657679..317773a0 100644 --- a/setup.py +++ b/setup.py @@ -32,10 +32,10 @@ def run(self): 'build_ext': Build, }, data_files=[ - ('bin', ['wfcommons/bin/cpu-benchmark']) + ('bin', ['bin/cpu-benchmark']) ], scripts=[ - 'wfcommons/bin/wfbench' + 'bin/wfbench' ], entry_points={ 'console_scripts': [ diff --git a/tests/translators/test_translators.py b/tests/translators/test_translators.py index 7d167252..1481df90 100644 --- a/tests/translators/test_translators.py +++ b/tests/translators/test_translators.py @@ -68,9 +68,9 @@ def install_WfCommons_on_container(container): container.put_archive(target_path, tar_data) exit_code, output = container.exec_run("sudo /bin/rm -rf /tmp/WfCommons/build/", stdout=True, stderr=True) exit_code, output = container.exec_run("sudo /bin/rm -rf /tmp/WfCommons/*.egg-info/", stdout=True, stderr=True) - exit_code, output = container.exec_run("sudo /bin/rm -rf /tmp/WfCommons/wfcommons/bin/cpu-benchmark.o", stdout=True, + exit_code, output = container.exec_run("sudo /bin/rm -rf /tmp/WfCommons/bin/cpu-benchmark.o", stdout=True, stderr=True) - exit_code, output = container.exec_run("sudo /bin/rm -rf /tmp/WfCommons/wfcommons/bin/cpu-benchmark", stdout=True, + exit_code, output = container.exec_run("sudo /bin/rm -rf /tmp/WfCommons/bin/cpu-benchmark", stdout=True, stderr=True) # Install WfCommons on the container (to install wfbench and cpu-benchmark really) From 40107a01ba718b76fe05ff57e0a12001c533925d Mon Sep 17 00:00:00 2001 From: Henri Casanova Date: Wed, 6 Aug 2025 14:54:10 -1000 Subject: [PATCH 38/38] whitespace++ --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 6994cc63..c8505c7a 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ objects = bin/cpu-benchmark.o CXX= g++ CPPFLAGS= -std=c++11 -execname =bin/cpu-benchmark +execname = bin/cpu-benchmark # compile $(execname): $(objects)