Added translator tests

.github/workflows/build.yml

@@ -7,8 +7,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [ "3.9", "3.10", "3.11", "3.12", "3.13" ]
-#        python-version: [ "3.9", "3.10" ]
+        python-version: [ "3.11", "3.12", "3.13" ]
     steps:
       - uses: actions/checkout@v4
 
@@ -23,12 +22,16 @@ jobs:
           pip install sphinx_rtd_theme recommonmark pytest pytest-cov
           pip install --upgrade setuptools
 
+      - name: Install utilities for testing
+        run: |
+          pip install docker
+
       - name: Check package install
         run: |
-          pip install -e .
+          pip install .
 
       - name: Run tests
-        run: pytest -m unit --cov=wfcommons tests/
+        run: python3 -m pytest -s -v -m unit --cov=wfcommons tests/
 
       - name: Upload coverage
         if: github.ref == 'refs/heads/main'

.gitignore

@@ -55,4 +55,8 @@ bin/gpu_benchmark
 /wfcommons/wfc-1.3_to_dask/out
 /wfcommons/wfc-1.3_to_dask/htmlcov
 
+# Coverage stuff
+.coverage
+coverage.xml
+
 test/

README.md

@@ -21,7 +21,7 @@ This Python package provides a collection of tools for:
 ## Installation
 
 WfCommons is available on [PyPI](https://pypi.org/project/wfcommons).
-WfCommons requires Python3.9+ and has been tested on Linux and macOS.
+WfCommons requires Python3.11+ and has been tested on Linux and MacOS.
 
 ### Installation using pip
 

docs/source/quickstart_installation.rst

@@ -2,7 +2,7 @@ Installation
 ============
 
 WfCommons is available on `PyPI <https://pypi.org/project/wfcommons>`_.
-WfCommons requires Python3.9+ and has been tested on Linux and macOS.
+WfCommons requires Python3.11+ and has been tested on Linux and macOS.
 
 Installation using pip
 ----------------------

pyproject.toml

@@ -7,13 +7,10 @@ name = "wfcommons"
 authors = [{name = "WfCommons team", email = "support@wfcommons.org"}]
 description = "A Framework for Enabling Scientific Workflow Research and Education"
 readme = "README.md"
-requires-python = ">=3.9"
+requires-python = ">=3.11"
 classifiers = [
     "License :: OSI Approved :: GNU General Public License v3 (GPLv3)",
     "Operating System :: OS Independent",
-    "Programming Language :: Python :: 3",
-    "Programming Language :: Python :: 3.9",
-    "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
     "Programming Language :: Python :: 3.13",
@@ -33,7 +30,7 @@ dependencies = [
     "pandas",
     "python-dateutil",
     "requests",
-    "scipy",
+    "scipy>=1.16.1",
     "pyyaml",
     "pandas",
     "shortuuid",

setup.cfg

@@ -1,2 +1,2 @@
 [metadata]
-description-file = README.md
+description_file = README.md

setup.py

@@ -14,14 +14,13 @@
 from setuptools import setup, find_packages
 from setuptools.command.build_ext import build_ext
 
-
 class Build(build_ext):
     """Customized setuptools build command - builds protos on build."""
 
     def run(self):
         protoc_command = ["make"]
         if subprocess.call(protoc_command) != 0:
-            print("Error: 'make' is not istnalled. Please install 'make' and try again.")
+            sys.stderr.write("Error: 'make' is not installed. Please install 'make' and try again.\n")
             sys.exit(-1) 
         super().run()
 
@@ -33,11 +32,14 @@ def run(self):
         'build_ext': Build,
     },
     data_files=[
-        ('bin', ['bin/cpu-benchmark', 'bin/wfbench'])
+        ('bin', ['bin/cpu-benchmark'])
+    ],
+    scripts=[
+        'bin/wfbench'
     ],
     entry_points={
         'console_scripts': [
-            'wfchef=wfcommons.wfchef.chef:main'
+            'wfchef=wfcommons.wfchef.chef:main',
         ],
         'workflow_recipes': [
             'epigenomics_recipe = wfcommons.wfchef.recipes:EpigenomicsRecipe',

tests/recipes/test_recipes.py

@@ -39,37 +39,37 @@ class TestRecipes:
             SrasearchRecipe,
         ]
 
-    # @pytest.mark.unit
-    # @pytest.mark.parametrize(
-    #     "recipe_class",
-    #     recipe_class_list
-    # )
-    # def test_recipes(self, recipe_class) -> None:
-    #
-    #     recipe = recipe_class.from_num_tasks(num_tasks=200, runtime_factor=1.1, input_file_size_factor=1.5,
-    #                                              output_file_size_factor=0.8)
-    #     workflow = WorkflowGenerator(recipe).build_workflow()
-    #
-    #
-    # @pytest.mark.unit
-    # @pytest.mark.parametrize(
-    #     "recipe_class",
-    #     recipe_class_list
-    # )
-    # def test_recipes_errors(self, recipe_class) -> None:
-    #     # Not enough tasks
-    #     recipe = recipe_class.from_num_tasks(num_tasks=2, runtime_factor=1.1, input_file_size_factor=1.5,
-    #                                          output_file_size_factor=0.8)
-    #     with pytest.raises(ValueError):
-    #         WorkflowGenerator(recipe).build_workflow()
-    #
-    #     # Bogus parameters
-    #     with pytest.raises(ValueError):
-    #         recipe_class.from_num_tasks(num_tasks=2, runtime_factor=-1.1, input_file_size_factor=1.5,
-    #                                     output_file_size_factor=0.8)
-    #     with pytest.raises(ValueError):
-    #         recipe_class.from_num_tasks(num_tasks=2, runtime_factor=1.1, input_file_size_factor=-1.5,
-    #                                     output_file_size_factor=0.8)
-    #     with pytest.raises(ValueError):
-    #         recipe_class.from_num_tasks(num_tasks=2, runtime_factor=1.1, input_file_size_factor=-1.5,
-    #                                     output_file_size_factor=-0.8)
+    @pytest.mark.unit
+    @pytest.mark.parametrize(
+        "recipe_class",
+        recipe_class_list
+    )
+    def test_recipes(self, recipe_class) -> None:
+
+        recipe = recipe_class.from_num_tasks(num_tasks=200, runtime_factor=1.1, input_file_size_factor=1.5,
+                                                 output_file_size_factor=0.8)
+        workflow = WorkflowGenerator(recipe).build_workflow()
+
+
+    @pytest.mark.unit
+    @pytest.mark.parametrize(
+        "recipe_class",
+        recipe_class_list
+    )
+    def test_recipes_errors(self, recipe_class) -> None:
+        # Not enough tasks
+        recipe = recipe_class.from_num_tasks(num_tasks=2, runtime_factor=1.1, input_file_size_factor=1.5,
+                                             output_file_size_factor=0.8)
+        with pytest.raises(ValueError):
+            WorkflowGenerator(recipe).build_workflow()
+
+        # Bogus parameters
+        with pytest.raises(ValueError):
+            recipe_class.from_num_tasks(num_tasks=2, runtime_factor=-1.1, input_file_size_factor=1.5,
+                                        output_file_size_factor=0.8)
+        with pytest.raises(ValueError):
+            recipe_class.from_num_tasks(num_tasks=2, runtime_factor=1.1, input_file_size_factor=-1.5,
+                                        output_file_size_factor=0.8)
+        with pytest.raises(ValueError):
+            recipe_class.from_num_tasks(num_tasks=2, runtime_factor=1.1, input_file_size_factor=-1.5,
+                                        output_file_size_factor=-0.8)

tests/translators/Dockerfile.airflow

@@ -0,0 +1,80 @@
+# docker build -t wfcommons-dev -f Dockerfile.airflow .
+# docker run -it --rm -v .:/home/wfcommons/mount wfcommons-dev /bin/bash
+
+FROM amd64/ubuntu:noble
+
+LABEL org.containers.image.authors="sukaryor@hawaii.edu"
+
+# update repositories
+RUN apt-get update
+
+# set timezone
+RUN echo "America/Los_Angeles" > /etc/timezone && export DEBIAN_FRONTEND=noninteractive && apt-get install -y tzdata
+
+# install useful stuff
+RUN apt-get -y install pkg-config 
+RUN apt-get -y install git
+RUN apt-get -y install wget 
+RUN apt-get -y install make 
+RUN apt-get -y install cmake 
+RUN apt-get -y install cmake-data 
+RUN apt-get -y install sudo 
+RUN apt-get -y install vim --fix-missing
+RUN apt-get -y install gcc 
+#RUN apt-get -y install gcc-multilib
+
+# Python stuff
+RUN apt-get -y install python3 python3-pip
+RUN python3 -m pip install --break-system-packages pathos pandas filelock
+RUN python3 -m pip install --break-system-packages networkx scipy matplotlib
+RUN python3 -m pip install --break-system-packages pyyaml jsonschema requests
+RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1
+
+# Stress-ng
+RUN apt-get -y install stress-ng
+
+# WfCommons
+#RUN python3 -m pip install --break-system-packages wfcommons
+
+# Install Airflow
+RUN python3 -m pip install --break-system-packages apache-airflow==2.10.2 --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.10.2/constraints-3.12.txt"
+
+# Install MySQL/MyClient
+RUN apt-get install -y mysql-server
+RUN apt-get install -y python3-dev build-essential
+RUN apt-get install -y default-libmysqlclient-dev
+RUN python3 -m pip install --break-system-packages mysqlclient
+
+# Create an run_a_workflow script to start mysqld in the background and setup the Airflow DB,
+# copy things around, and the run the workflow
+
+RUN echo '#!/bin/bash' > /run_a_workflow.sh && \
+    echo 'mysqld --explicit-defaults-for-timestamp &' >> /run_a_workflow.sh && \
+    echo 'until mysqladmin ping -h localhost --silent; do' >> /run_a_workflow.sh && \
+    echo '    echo "Waiting for MySQL to be ready..."' >> /run_a_workflow.sh && \
+    echo '    sleep 2' >> /run_a_workflow.sh && \
+    echo 'done' >> /run_a_workflow.sh && \
+    echo 'echo "MySQL is ready!"' >> /run_a_workflow.sh && \
+    echo 'echo "Setting up database for Airflow..."' >> /run_a_workflow.sh && \
+    echo 'mysql -u root -e "CREATE DATABASE airflow_db CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci";' >> /run_a_workflow.sh && \
+    echo "mysql -u root -e \"CREATE USER 'airflow_user'@'%' IDENTIFIED BY 'airflow_pass';\"" >> /run_a_workflow.sh && \
+    echo "mysql -u root -e \"GRANT ALL PRIVILEGES ON airflow_db.* TO 'airflow_user';\"" >> /run_a_workflow.sh && \
+    echo 'export AIRFLOW_HOME="$(pwd)"/airflow/' >> /run_a_workflow.sh && \
+    echo "airflow config list 1> /dev/null" >> /run_a_workflow.sh && \
+    echo "sed -i ./airflow/airflow.cfg -e 's/sqlite:.*/mysql+mysqldb:\/\/airflow_user:airflow_pass@localhost:3306\/airflow_db/'" >> /run_a_workflow.sh && \
+    echo 'airflow db migrate' >> /run_a_workflow.sh && \
+    echo 'echo "Airflow database setup!"' >> /run_a_workflow.sh && \
+    echo 'mkdir ./airflow/dags' >> /run_a_workflow.sh && \
+    echo 'cp -r /tmp/airflow_translated_workflow ./airflow/dags/' >> /run_a_workflow.sh && \
+    echo 'mv ./airflow/dags/airflow_translated_workflow/workflow.py ./airflow/dags/workflow.py' >> /run_a_workflow.sh && \
+    echo 'airflow dags test $1' >> /run_a_workflow.sh && \
+    chmod +x /run_a_workflow.sh
+
+# Create wfcommons user
+RUN useradd -ms /bin/bash wfcommons
+RUN adduser wfcommons sudo
+RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+ENV PATH="$PATH:/home/wfcommons/.local/bin/"
+
+USER wfcommons
+WORKDIR /home/wfcommons

tests/translators/Dockerfile_Dask → tests/translators/Dockerfile.dask

@@ -1,4 +1,4 @@
-# docker build --platform amd64 -t wfcommons-dev-dask -f Dockerfile_Dask .
+# docker build --platform amd64 -t wfcommons-dev-dask -f Dockerfile.dask .
 # docker run -it --rm -v `pwd`:/home/wfcommons wfcommons-dev-dask /bin/bash
 
 FROM amd64/ubuntu:noble
@@ -29,6 +29,7 @@ RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1
 RUN python3 -m pip install --break-system-packages pathos pandas filelock
 RUN python3 -m pip install --break-system-packages networkx scipy matplotlib
 RUN python3 -m pip install --break-system-packages pyyaml jsonschema requests
+RUN python3 -m pip install --break-system-packages --upgrade setuptools
 
 # Stress-ng
 RUN apt-get -y install stress-ng

tests/translators/Dockerfile_Nextflow → tests/translators/Dockerfile.nextflow

@@ -1,4 +1,4 @@
-# docker build --platform amd64 -t wfcommons-dev-nextflow -f Dockerfile_Nextflow .
+# docker build --platform amd64 -t wfcommons-dev-nextflow -f Dockerfile.nextflow .
 # docker run -it --rm -v `pwd`:/home/wfcommons wfcommons-dev-nextflow /bin/bash
 
 FROM amd64/ubuntu:noble
@@ -30,14 +30,15 @@ RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1
 RUN pip install --break-system-packages pathos pandas filelock 
 RUN pip install --break-system-packages networkx scipy matplotlib
 RUN pip install --break-system-packages pyyaml jsonschema requests
+RUN python3 -m pip install --break-system-packages --upgrade setuptools
 
 # Stress-ng
 RUN apt-get -y install stress-ng
 
 # Install Java
 RUN apt-get -y install openjdk-21-jdk
 
-# Create wfcomomns user
+# Create wfcommons user
 RUN useradd -ms /bin/bash wfcommons
 RUN adduser wfcommons sudo
 RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
@@ -48,5 +49,3 @@ WORKDIR /home/wfcommons
 # Install Nextflow and dependencies
 RUN curl -s https://get.nextflow.io | bash  && sudo mv ./nextflow /usr/local/bin/
 
-# Add wfcommons user
-

tests/translators/Dockerfile_Parsl → tests/translators/Dockerfile.parsl

@@ -1,4 +1,4 @@
-# docker build --platform amd64 -t wfcommons-dev -f Dockerfile_Parsl .
+# docker build --platform amd64 -t wfcommons-dev -f Dockerfile.parsl .
 # docker run -it --rm -v `pwd`:/home/wfcommons wfcommons-dev /bin/bash
 
 FROM amd64/ubuntu:noble
@@ -29,6 +29,7 @@ RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1
 RUN python3 -m pip install --break-system-packages pathos pandas filelock
 RUN python3 -m pip install --break-system-packages networkx scipy matplotlib
 RUN python3 -m pip install --break-system-packages pyyaml jsonschema requests
+RUN python3 -m pip install --break-system-packages --upgrade setuptools
 
 # Stress-ng
 RUN apt-get -y install stress-ng

tests/translators/airflow/README → tests/translators/README.airflow

@@ -1,5 +1,4 @@
-This README file describes steps to install/run Airflow, and then run a
-translated workflow.
+This README file describes steps to install/run Airflow, and then run a translated workflow.
 
 There are three sections:
   - Installing Airflow on bare-metal
@@ -52,14 +51,17 @@ A much simpler alternative is to use Docker.
 
 1. Build the docker image
 
-   docker build -t wfcommons-dev -f Dockerfile_Airflow .
+   docker build -t wfcommons/wfcommons-testing-airflow -f Dockerfile.airflow .
 
    (if building on a Mac, add the `--platform linux/amd64` argument after build above)
 
+   This dockerfile is meant for testing purposes (so it has some testing specific things in it)
+
+
 2. Run the docker container in the directory to contains the translated
    workflow (see last section below)
 
-   docker run -it --rm -v .:/home/wfcommons/mount wfcommons-dev /bin/bash
+   docker run -it --rm -v .:/home/wfcommons/translated_workflow wfcommons/wfcommons-testing-airflow /bin/bash
 
 
 Running a translated workflow with Airflow
@@ -84,19 +86,21 @@ translator = AirflowTranslator(benchmark.workflow)
 translator.translate(output_folder=pathlib.Path("/tmp/translated_workflow/"))
 ```
 
-The above will create a JSON worfklow file in /tmp/blast-benchmark-45.json.
-In that file, the workflow name (this is used below) is set to
-"Blast-Benchmark".
+The above will create a JSON workflow file in /tmp/blast-benchmark-45.json.
+In that file, the workflow name (this is used below) is set to "Blast-Benchmark".
 
 The above will also create the translated workflow the
-/tmp/translated_workflow/ directory. Some directories and files need to be copied/moved as follows:
+/tmp/translated_workflow/ directory. Some directories and files need to be copied/moved as follows.
+As
 
   cp -r /tmp/translated_workflow/ $AIRFLOW_HOME/dags/
   mv $AIRFLOW_HOME/dags/translated_workflow/workflow.py $AIRFLOW_HOME/dags/
 
 Finally, run the workflow as: 
 
-  airflow dags test Blast-Benchmark    (not the "Blast-Benchmark" workflow name from above)
+  airflow dags test Blast-Benchmark    (note the "Blast-Benchmark" workflow name from above)
 
 
+Note that if you're using docker, the above two commands need to be done on the container,
+with the right paths (i.e., /home/wfcommons/translated_workflow instead of /tmp/translated_workflow/)