From 618ce439afb4008f142c61a03a32479354da2f4a Mon Sep 17 00:00:00 2001 From: leahh Date: Mon, 7 Jul 2025 17:13:17 -0600 Subject: [PATCH 01/12] super rough commit to save my work --- beeflow/client/bee_client.py | 10 ++- beeflow/common/gdb/generate_graph.py | 22 +++++++ beeflow/wf_manager/resources/wf_utils.py | 8 +++ test_generate.py | 83 ++++++++++++++++++++++++ 4 files changed, 122 insertions(+), 1 deletion(-) create mode 100644 test_generate.py diff --git a/beeflow/client/bee_client.py b/beeflow/client/bee_client.py index 75f6250de..b8a9484ad 100644 --- a/beeflow/client/bee_client.py +++ b/beeflow/client/bee_client.py @@ -719,7 +719,9 @@ def dag(wf_id: str = typer.Argument(..., callback=match_short_id), output_dir: pathlib.Path = typer.Argument(..., help='Path to the where the dag output will be'), no_dag_dir: bool = typer.Option(False, '--no-dag-dir', - help='do not make a subdirectory within ouput_dir for the dags')): + help='do not make a subdirectory within ouput_dir for the dags'), + graphmls_dir: str = typer.Option(None, '--graphmls_dir', + help='Graphmls directory to convert into a DAG')): """Export a DAG of the workflow to a GraphML file.""" output_dir = output_dir.resolve() # Make sure output_dir is an absolute path and exists @@ -730,6 +732,12 @@ def dag(wf_id: str = typer.Argument(..., callback=match_short_id), # output_dir must be a string output_dir = str(output_dir) + + # Convert existing graphmls to DAGs if graphmls_dir was given + if graphmls_dir: + wf_utils.convert_to_dag(wf_id, output_dir, graphmls_dir, no_dag_dir) + typer.secho(f"DAG for workflow {_short_id(wf_id)} has been exported successfully.", + fg=typer.colors.GREEN) # Check if the workflow is archived wf_status = get_wf_status(wf_id) if wf_status == 'Archived': diff --git a/beeflow/common/gdb/generate_graph.py b/beeflow/common/gdb/generate_graph.py index fc26e59c9..f9ca9f728 100644 --- a/beeflow/common/gdb/generate_graph.py +++ b/beeflow/common/gdb/generate_graph.py @@ -43,6 +43,28 @@ def generate_viz(wf_id, output_dir, graphmls_dir, no_dag_dir, workflow_dir=None) save_png(archive_dag_path, png_data) +def generate_all_viz(wf_id, output_dir, graphmls_dir, no_dag_dir): + "Create DAGs from an exisiting collection of GraphMLs." + short_id = wf_id[:6] + if no_dag_dir: + dags_dir = output_dir + else: + dags_dir = output_dir + "/" + short_id + "-dags" + os.makedirs(dags_dir, exist_ok=True) + + for filename in os.listdir(graphmls_dir): + if filename.endswith('.graphml'): + name_without_ext = os.path.splitext(filename)[0] + output_path = dags_dir + "/" + name_without_ext + ".png" + + graph = nx.read_graphml(filename) + dot = graphviz.Digraph(comment='Hierarchical Graph') + add_nodes_to_dot(graph, dot) + add_edges_to_dot(graph, dot) + png_data = dot.pipe(format='png') + save_png(output_path, png_data) + + def backup_dag(path, dags_dir, short_id): """Backup DAGs.""" if os.path.exists(path): diff --git a/beeflow/wf_manager/resources/wf_utils.py b/beeflow/wf_manager/resources/wf_utils.py index a52ead439..b45c06a19 100644 --- a/beeflow/wf_manager/resources/wf_utils.py +++ b/beeflow/wf_manager/resources/wf_utils.py @@ -307,6 +307,14 @@ def export_dag(wf_id, output_dir, graphmls_dir, no_dag_dir, workflow_dir=None): generate_viz(wf_id, output_dir, graphmls_dir, no_dag_dir, workflow_dir) +def convert_to_dag(wf_id, output_dir, graphmls_dir, no_dag_dir): + dot_avail = bool(shutil.which("dot")) + if dot_avail: + generate_all_viz(wf_id, output_dir, graphmls_dir, no_dag_dir) + else: + log.error('Unable to convert graphmls to DAGs. Graphviz is not available.') + + def start_workflow(wf_id): """Attempt to start the workflow, returning True if successful.""" db = connect_db(wfm_db, get_db_path()) diff --git a/test_generate.py b/test_generate.py new file mode 100644 index 000000000..5f8b4205d --- /dev/null +++ b/test_generate.py @@ -0,0 +1,83 @@ +import os +import networkx as nx +import graphviz +def generate_all_viz(wf_id, output_dir, graphmls_dir, no_dag_dir): + "Create DAGs from an exisiting collection of GraphMLs." + short_id = wf_id[:6] + if no_dag_dir: + dags_dir = output_dir + else: + dags_dir = output_dir + "/" + short_id + "-dags" + os.makedirs(dags_dir, exist_ok=True) + + for filename in os.listdir(graphmls_dir): + if filename.endswith('.graphml'): + name_without_ext = os.path.splitext(filename)[0] + output_path = dags_dir + "/" + name_without_ext + ".png" + graphml_path = os.path.join(graphmls_dir, filename) + + graph = nx.read_graphml(graphml_path) + dot = graphviz.Digraph(comment='Hierarchical Graph') + add_nodes_to_dot(graph, dot) + add_edges_to_dot(graph, dot) + png_data = dot.pipe(format='png') + save_png(output_path, png_data) + + +def add_nodes_to_dot(graph, dot): + """Add nodes from the graph to the Graphviz object with labels and colors.""" + label_to_color = { + ":Workflow": 'steelblue', + ":Output": 'mediumseagreen', + ":Metadata": 'skyblue', + ":Task": 'lightcoral', + ":Input": 'sandybrown', + ":Hint": 'plum', + ":Requirement": 'lightpink1' + } + + for node_id, attributes in graph.nodes(data=True): + label = attributes.get('labels', node_id) + node_label, color = get_node_label_and_color(label, attributes, label_to_color) + dot.node(node_id, label=node_label, style='filled', fillcolor=color) + + +def get_node_label_and_color(label, attributes, label_to_color): + """Return the appropriate node label and color based on node type.""" + label_to_attribute = { + ":Workflow": "Workflow", + ":Output": attributes.get('glob', label), + ":Metadata": attributes.get('state', label), + ":Task": attributes.get('name', label), + ":Input": attributes.get('source', label), + ":Hint": attributes.get('class', label), + ":Requirement": attributes.get('class', label) + } + + # Check if the label is in the predefined labels + if label in label_to_attribute: + return label_to_attribute[label], label_to_color.get(label, 'gray') + + # Default case if no match + return label, 'gray' + + +def add_edges_to_dot(graph, dot): + """Add edges from the graph to the Graphviz object with appropriate labels.""" + for source, target, attributes in graph.edges(data=True): + edge_label = attributes.get('label', '') + if edge_label in ('INPUT_OF', 'DESCRIBES', 'HINT_OF', 'REQUIREMENT_OF'): + dot.edge(source, target, label=edge_label, fontsize="10") + elif edge_label in ('DEPENDS_ON', 'RESTARTED_FROM'): + dot.edge(target, source, label=edge_label, penwidth="3", + fontsize="10", fontname="times-bold") + else: + dot.edge(target, source, label=edge_label, fontsize="10") + + +def save_png(output_path, png_data): + """Save png data.""" + with open(output_path, "wb") as png_file: + png_file.write(png_data) + +generate_all_viz("fdbd95", "../workdir", "../workdir/graphmls", False) From 7d96fc9f86777519718dd1d26c5828da9c925f94 Mon Sep 17 00:00:00 2001 From: leahh Date: Tue, 8 Jul 2025 09:35:43 -0600 Subject: [PATCH 02/12] add the fixes from my tester file and remove the file --- beeflow/common/gdb/generate_graph.py | 3 +- test_generate.py | 83 ---------------------------- 2 files changed, 2 insertions(+), 84 deletions(-) delete mode 100644 test_generate.py diff --git a/beeflow/common/gdb/generate_graph.py b/beeflow/common/gdb/generate_graph.py index f9ca9f728..2c499ddbb 100644 --- a/beeflow/common/gdb/generate_graph.py +++ b/beeflow/common/gdb/generate_graph.py @@ -56,8 +56,9 @@ def generate_all_viz(wf_id, output_dir, graphmls_dir, no_dag_dir): if filename.endswith('.graphml'): name_without_ext = os.path.splitext(filename)[0] output_path = dags_dir + "/" + name_without_ext + ".png" + graphml_path = os.path.join(graphmls_dir, filename) - graph = nx.read_graphml(filename) + graph = nx.read_graphml(graphml_path) dot = graphviz.Digraph(comment='Hierarchical Graph') add_nodes_to_dot(graph, dot) add_edges_to_dot(graph, dot) diff --git a/test_generate.py b/test_generate.py deleted file mode 100644 index 5f8b4205d..000000000 --- a/test_generate.py +++ /dev/null @@ -1,83 +0,0 @@ -import os -import networkx as nx -import graphviz -def generate_all_viz(wf_id, output_dir, graphmls_dir, no_dag_dir): - "Create DAGs from an exisiting collection of GraphMLs." - short_id = wf_id[:6] - if no_dag_dir: - dags_dir = output_dir - else: - dags_dir = output_dir + "/" + short_id + "-dags" - os.makedirs(dags_dir, exist_ok=True) - - for filename in os.listdir(graphmls_dir): - if filename.endswith('.graphml'): - name_without_ext = os.path.splitext(filename)[0] - output_path = dags_dir + "/" + name_without_ext + ".png" - graphml_path = os.path.join(graphmls_dir, filename) - - graph = nx.read_graphml(graphml_path) - dot = graphviz.Digraph(comment='Hierarchical Graph') - add_nodes_to_dot(graph, dot) - add_edges_to_dot(graph, dot) - png_data = dot.pipe(format='png') - save_png(output_path, png_data) - - -def add_nodes_to_dot(graph, dot): - """Add nodes from the graph to the Graphviz object with labels and colors.""" - label_to_color = { - ":Workflow": 'steelblue', - ":Output": 'mediumseagreen', - ":Metadata": 'skyblue', - ":Task": 'lightcoral', - ":Input": 'sandybrown', - ":Hint": 'plum', - ":Requirement": 'lightpink1' - } - - for node_id, attributes in graph.nodes(data=True): - label = attributes.get('labels', node_id) - node_label, color = get_node_label_and_color(label, attributes, label_to_color) - dot.node(node_id, label=node_label, style='filled', fillcolor=color) - - -def get_node_label_and_color(label, attributes, label_to_color): - """Return the appropriate node label and color based on node type.""" - label_to_attribute = { - ":Workflow": "Workflow", - ":Output": attributes.get('glob', label), - ":Metadata": attributes.get('state', label), - ":Task": attributes.get('name', label), - ":Input": attributes.get('source', label), - ":Hint": attributes.get('class', label), - ":Requirement": attributes.get('class', label) - } - - # Check if the label is in the predefined labels - if label in label_to_attribute: - return label_to_attribute[label], label_to_color.get(label, 'gray') - - # Default case if no match - return label, 'gray' - - -def add_edges_to_dot(graph, dot): - """Add edges from the graph to the Graphviz object with appropriate labels.""" - for source, target, attributes in graph.edges(data=True): - edge_label = attributes.get('label', '') - if edge_label in ('INPUT_OF', 'DESCRIBES', 'HINT_OF', 'REQUIREMENT_OF'): - dot.edge(source, target, label=edge_label, fontsize="10") - elif edge_label in ('DEPENDS_ON', 'RESTARTED_FROM'): - dot.edge(target, source, label=edge_label, penwidth="3", - fontsize="10", fontname="times-bold") - else: - dot.edge(target, source, label=edge_label, fontsize="10") - - -def save_png(output_path, png_data): - """Save png data.""" - with open(output_path, "wb") as png_file: - png_file.write(png_data) - -generate_all_viz("fdbd95", "../workdir", "../workdir/graphmls", False) From 4a1be9a9bde7021607d3a80980a3ad173ef386da Mon Sep 17 00:00:00 2001 From: leahh Date: Tue, 8 Jul 2025 09:44:57 -0600 Subject: [PATCH 03/12] add render png data function --- beeflow/common/gdb/generate_graph.py | 36 +++++++++++++++------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/beeflow/common/gdb/generate_graph.py b/beeflow/common/gdb/generate_graph.py index 2c499ddbb..ab08377c9 100644 --- a/beeflow/common/gdb/generate_graph.py +++ b/beeflow/common/gdb/generate_graph.py @@ -20,18 +20,7 @@ def generate_viz(wf_id, output_dir, graphmls_dir, no_dag_dir, workflow_dir=None) output_path = dags_dir + "/" + short_id + ".png" backup_dag(output_path, dags_dir, short_id) - # Load the GraphML file using NetworkX - graph = nx.read_graphml(graphml_path) - - # Initialize Graphviz graph - dot = graphviz.Digraph(comment='Hierarchical Graph') - - # Add nodes and edges using helper functions - add_nodes_to_dot(graph, dot) - add_edges_to_dot(graph, dot) - - # Render the graph and save as PNG - png_data = dot.pipe(format='png') + png_data = render_png_data(graphml_path) save_png(output_path, png_data) if workflow_dir: @@ -58,11 +47,7 @@ def generate_all_viz(wf_id, output_dir, graphmls_dir, no_dag_dir): output_path = dags_dir + "/" + name_without_ext + ".png" graphml_path = os.path.join(graphmls_dir, filename) - graph = nx.read_graphml(graphml_path) - dot = graphviz.Digraph(comment='Hierarchical Graph') - add_nodes_to_dot(graph, dot) - add_edges_to_dot(graph, dot) - png_data = dot.pipe(format='png') + png_data = render_png_data(graphml_path) save_png(output_path, png_data) @@ -77,6 +62,23 @@ def backup_dag(path, dags_dir, short_id): shutil.copy(path, backup_path) +def render_png_data(graphml_path): + """Read a GraphML file, build the Graphviz Digraph, and return PNG bytes.""" + # Load the GraphML file using NetworkX + graph = nx.read_graphml(graphml_path) + + # Initialize Graphviz graph + dot = graphviz.Digraph(comment='Hierarchical Graph') + + # Add nodes and edges using helper functions + add_nodes_to_dot(graph, dot) + add_edges_to_dot(graph, dot) + + # Render the graph + png_data = dot.pipe(format='png') + return png_data + + def add_nodes_to_dot(graph, dot): """Add nodes from the graph to the Graphviz object with labels and colors.""" label_to_color = { From ead73db2f2f8b5165d226fcd568cbd8b68cf4079 Mon Sep 17 00:00:00 2001 From: leahh Date: Tue, 8 Jul 2025 09:52:17 -0600 Subject: [PATCH 04/12] add return statement to skip the rest of dag code --- beeflow/client/bee_client.py | 1 + 1 file changed, 1 insertion(+) diff --git a/beeflow/client/bee_client.py b/beeflow/client/bee_client.py index 27d1df39a..4549e90f8 100644 --- a/beeflow/client/bee_client.py +++ b/beeflow/client/bee_client.py @@ -738,6 +738,7 @@ def dag(wf_id: str = typer.Argument(..., callback=match_short_id), wf_utils.convert_to_dag(wf_id, output_dir, graphmls_dir, no_dag_dir) typer.secho(f"DAG for workflow {_short_id(wf_id)} has been exported successfully.", fg=typer.colors.GREEN) + return # Check if the workflow is archived wf_status = get_wf_status(wf_id) if wf_status == 'Archived': From 77c3492fb9f65c65b0ec5e392777abeb209468ec Mon Sep 17 00:00:00 2001 From: leahh Date: Tue, 8 Jul 2025 09:58:16 -0600 Subject: [PATCH 05/12] import generate all viz function --- beeflow/wf_manager/resources/wf_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beeflow/wf_manager/resources/wf_utils.py b/beeflow/wf_manager/resources/wf_utils.py index 224f53046..c1af43889 100644 --- a/beeflow/wf_manager/resources/wf_utils.py +++ b/beeflow/wf_manager/resources/wf_utils.py @@ -10,7 +10,7 @@ from beeflow.common import log as bee_logging from beeflow.common.config_driver import BeeConfig as bc from beeflow.common.gdb import neo4j_driver -from beeflow.common.gdb.generate_graph import generate_viz +from beeflow.common.gdb.generate_graph import generate_viz, generate_all_viz from beeflow.common.gdb.graphml_key_updater import update_graphml from beeflow.common.wf_interface import WorkflowInterface from beeflow.common.connection import Connection From 69574a5d03a78580c4e70784144e29fcfbc57978 Mon Sep 17 00:00:00 2001 From: leahh Date: Tue, 8 Jul 2025 12:42:56 -0600 Subject: [PATCH 06/12] add error messages --- beeflow/client/bee_client.py | 11 +++++++---- beeflow/wf_manager/resources/wf_utils.py | 23 +++++++++++++++++++---- 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/beeflow/client/bee_client.py b/beeflow/client/bee_client.py index 4549e90f8..b6592893c 100644 --- a/beeflow/client/bee_client.py +++ b/beeflow/client/bee_client.py @@ -735,10 +735,13 @@ def dag(wf_id: str = typer.Argument(..., callback=match_short_id), # Convert existing graphmls to DAGs if graphmls_dir was given if graphmls_dir: - wf_utils.convert_to_dag(wf_id, output_dir, graphmls_dir, no_dag_dir) - typer.secho(f"DAG for workflow {_short_id(wf_id)} has been exported successfully.", - fg=typer.colors.GREEN) - return + resp = wf_utils.convert_to_dag(wf_id, output_dir, graphmls_dir, no_dag_dir) + if resp: + error_exit(resp) + else: + typer.secho(f"DAG for workflow {_short_id(wf_id)} has been exported successfully.", + fg=typer.colors.GREEN) + return # Check if the workflow is archived wf_status = get_wf_status(wf_id) if wf_status == 'Archived': diff --git a/beeflow/wf_manager/resources/wf_utils.py b/beeflow/wf_manager/resources/wf_utils.py index c1af43889..56d261758 100644 --- a/beeflow/wf_manager/resources/wf_utils.py +++ b/beeflow/wf_manager/resources/wf_utils.py @@ -313,11 +313,26 @@ def export_dag(wf_id, output_dir, graphmls_dir, no_dag_dir, workflow_dir=None): def convert_to_dag(wf_id, output_dir, graphmls_dir, no_dag_dir): - dot_avail = bool(shutil.which("dot")) - if dot_avail: + """ + Convert a directory of graphmls into DAGs. + + This function is used to turn graphmls that were generated + on a system without graphviz into DAGs on a system with graphviz. + + Returns: + str or None: Returns an error message if Graphviz is not available or an + exception occurs; otherwise returns None on success. + """ + if not shutil.which("dot"): + err_msg = 'Unable to convert graphmls to DAGs. Graphviz is not available.' + return err_msg + + try: generate_all_viz(wf_id, output_dir, graphmls_dir, no_dag_dir) - else: - log.error('Unable to convert graphmls to DAGs. Graphviz is not available.') + return None + except Exception as exc: + err_msg = f'Error while generating visualizations: {exc}' + return err_msg def start_workflow(wf_id): From c59e61fac5e3d76d0c406143273f42b41863cbe2 Mon Sep 17 00:00:00 2001 From: leahh Date: Tue, 8 Jul 2025 15:23:48 -0600 Subject: [PATCH 07/12] capture DAG exceptions in a list --- beeflow/common/gdb/generate_graph.py | 11 +++++++++-- beeflow/wf_manager/resources/wf_utils.py | 13 ++++--------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/beeflow/common/gdb/generate_graph.py b/beeflow/common/gdb/generate_graph.py index ab08377c9..ece51db4e 100644 --- a/beeflow/common/gdb/generate_graph.py +++ b/beeflow/common/gdb/generate_graph.py @@ -41,14 +41,21 @@ def generate_all_viz(wf_id, output_dir, graphmls_dir, no_dag_dir): dags_dir = output_dir + "/" + short_id + "-dags" os.makedirs(dags_dir, exist_ok=True) + msgs = [] for filename in os.listdir(graphmls_dir): if filename.endswith('.graphml'): name_without_ext = os.path.splitext(filename)[0] output_path = dags_dir + "/" + name_without_ext + ".png" graphml_path = os.path.join(graphmls_dir, filename) - png_data = render_png_data(graphml_path) - save_png(output_path, png_data) + try: + png_data = render_png_data(graphml_path) + save_png(output_path, png_data) + except Exception as exc: + err_msg = f'Error while generating visualization for {graphml_path}: {exc}' + msgs.append(err_msg) + + return "\n".join(map(str, msgs)) def backup_dag(path, dags_dir, short_id): diff --git a/beeflow/wf_manager/resources/wf_utils.py b/beeflow/wf_manager/resources/wf_utils.py index 56d261758..c137a4e3b 100644 --- a/beeflow/wf_manager/resources/wf_utils.py +++ b/beeflow/wf_manager/resources/wf_utils.py @@ -324,15 +324,10 @@ def convert_to_dag(wf_id, output_dir, graphmls_dir, no_dag_dir): exception occurs; otherwise returns None on success. """ if not shutil.which("dot"): - err_msg = 'Unable to convert graphmls to DAGs. Graphviz is not available.' - return err_msg - - try: - generate_all_viz(wf_id, output_dir, graphmls_dir, no_dag_dir) - return None - except Exception as exc: - err_msg = f'Error while generating visualizations: {exc}' - return err_msg + msg = 'Unable to convert graphmls to DAGs. Graphviz is not available.' + else: + msg = generate_all_viz(wf_id, output_dir, graphmls_dir, no_dag_dir) + return msg def start_workflow(wf_id): From 8174e90dbaac9360e0288d5ba2401ccf11dd49c5 Mon Sep 17 00:00:00 2001 From: leahh Date: Tue, 8 Jul 2025 15:41:10 -0600 Subject: [PATCH 08/12] make dag dir after png data has been rendered --- beeflow/common/gdb/generate_graph.py | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/beeflow/common/gdb/generate_graph.py b/beeflow/common/gdb/generate_graph.py index ece51db4e..75b4d9935 100644 --- a/beeflow/common/gdb/generate_graph.py +++ b/beeflow/common/gdb/generate_graph.py @@ -10,17 +10,14 @@ def generate_viz(wf_id, output_dir, graphmls_dir, no_dag_dir, workflow_dir=None) """Generate a PNG of a workflow graph from a GraphML file.""" short_id = wf_id[:6] graphml_path = graphmls_dir + "/" + short_id + ".graphml" + # Render png data + png_data = render_png_data(graphml_path) - if no_dag_dir: - dags_dir = output_dir - else: - dags_dir = output_dir + "/" + short_id + "-dags" - os.makedirs(dags_dir, exist_ok=True) - + # Back up DAG and save + dags_dir = output_dir if no_dag_dir else os.path.join(output_dir, f"{short_id}-dags") + os.makedirs(dags_dir, exist_ok=True) output_path = dags_dir + "/" + short_id + ".png" backup_dag(output_path, dags_dir, short_id) - - png_data = render_png_data(graphml_path) save_png(output_path, png_data) if workflow_dir: @@ -35,13 +32,10 @@ def generate_viz(wf_id, output_dir, graphmls_dir, no_dag_dir, workflow_dir=None) def generate_all_viz(wf_id, output_dir, graphmls_dir, no_dag_dir): "Create DAGs from an exisiting collection of GraphMLs." short_id = wf_id[:6] - if no_dag_dir: - dags_dir = output_dir - else: - dags_dir = output_dir + "/" + short_id + "-dags" - os.makedirs(dags_dir, exist_ok=True) + dags_dir = output_dir if no_dag_dir else os.path.join(output_dir, f"{short_id}-dags") msgs = [] + first = True for filename in os.listdir(graphmls_dir): if filename.endswith('.graphml'): name_without_ext = os.path.splitext(filename)[0] @@ -50,6 +44,9 @@ def generate_all_viz(wf_id, output_dir, graphmls_dir, no_dag_dir): try: png_data = render_png_data(graphml_path) + if first: + os.makedirs(dags_dir, exist_ok=True) + first = False save_png(output_path, png_data) except Exception as exc: err_msg = f'Error while generating visualization for {graphml_path}: {exc}' From 36370d73d4f7e09fae816b5c6d5a558a70fcdb9f Mon Sep 17 00:00:00 2001 From: leahh Date: Tue, 8 Jul 2025 16:52:02 -0600 Subject: [PATCH 09/12] add more specific error handeling --- beeflow/common/gdb/generate_graph.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/beeflow/common/gdb/generate_graph.py b/beeflow/common/gdb/generate_graph.py index 75b4d9935..ff605d357 100644 --- a/beeflow/common/gdb/generate_graph.py +++ b/beeflow/common/gdb/generate_graph.py @@ -4,6 +4,8 @@ import shutil import networkx as nx import graphviz +import subprocess +import xml.etree.ElementTree def generate_viz(wf_id, output_dir, graphmls_dir, no_dag_dir, workflow_dir=None): @@ -48,7 +50,12 @@ def generate_all_viz(wf_id, output_dir, graphmls_dir, no_dag_dir): os.makedirs(dags_dir, exist_ok=True) first = False save_png(output_path, png_data) - except Exception as exc: + + except( + nx.NetworkXError, + xml.etree.ElementTree.ParseError, + subprocess.CalledProcessError + ) as exc: err_msg = f'Error while generating visualization for {graphml_path}: {exc}' msgs.append(err_msg) From 26b24ae62bf7de96c65208d98675895787f368f5 Mon Sep 17 00:00:00 2001 From: leahh Date: Mon, 14 Jul 2025 11:24:24 -0600 Subject: [PATCH 10/12] add imports --- beeflow/common/gdb/generate_graph.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/beeflow/common/gdb/generate_graph.py b/beeflow/common/gdb/generate_graph.py index ff605d357..fc353aabb 100644 --- a/beeflow/common/gdb/generate_graph.py +++ b/beeflow/common/gdb/generate_graph.py @@ -2,10 +2,10 @@ import os import shutil -import networkx as nx -import graphviz import subprocess import xml.etree.ElementTree +import networkx as nx +import graphviz def generate_viz(wf_id, output_dir, graphmls_dir, no_dag_dir, workflow_dir=None): From bc497fff14e6821831121a63ff8bc705284a6a70 Mon Sep 17 00:00:00 2001 From: leahh Date: Wed, 23 Jul 2025 14:34:25 -0600 Subject: [PATCH 11/12] add check if graphml dir exists --- beeflow/client/bee_client.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/beeflow/client/bee_client.py b/beeflow/client/bee_client.py index b6592893c..0d3c3cb9b 100644 --- a/beeflow/client/bee_client.py +++ b/beeflow/client/bee_client.py @@ -720,7 +720,7 @@ def dag(wf_id: str = typer.Argument(..., callback=match_short_id), help='Path to the where the dag output will be'), no_dag_dir: bool = typer.Option(False, '--no-dag-dir', help='do not make a subdirectory within ouput_dir for the dags'), - graphmls_dir: str = typer.Option(None, '--graphmls_dir', + graphmls_dir: pathlib.Path = typer.Option(None, '--graphmls_dir', help='Graphmls directory to convert into a DAG')): """Export a DAG of the workflow to a GraphML file.""" output_dir = output_dir.resolve() @@ -735,6 +735,14 @@ def dag(wf_id: str = typer.Argument(..., callback=match_short_id), # Convert existing graphmls to DAGs if graphmls_dir was given if graphmls_dir: + # Make sure graphmls_dir is an absolute path and exists + graphmls_dir = os.path.expanduser(graphmls_dir) + graphmls_dir = os.path.abspath(graphmls_dir) + if not os.path.exists(graphmls_dir): + error_exit(f'Path for graphmls directory "{graphmls_dir}" doesn\'t exist') + graphmls_dir = str(graphmls_dir) + + # Convert resp = wf_utils.convert_to_dag(wf_id, output_dir, graphmls_dir, no_dag_dir) if resp: error_exit(resp) From 23a53d7d3579e6cb8be6073913da8d1bf46c9bee Mon Sep 17 00:00:00 2001 From: leahh Date: Wed, 23 Jul 2025 14:50:56 -0600 Subject: [PATCH 12/12] add graphml conversion documentation --- beeflow/client/bee_client.py | 2 +- docs/sphinx/commands.rst | 3 ++- docs/sphinx/visualization.rst | 4 ++++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/beeflow/client/bee_client.py b/beeflow/client/bee_client.py index 0d3c3cb9b..643bd4977 100644 --- a/beeflow/client/bee_client.py +++ b/beeflow/client/bee_client.py @@ -721,7 +721,7 @@ def dag(wf_id: str = typer.Argument(..., callback=match_short_id), no_dag_dir: bool = typer.Option(False, '--no-dag-dir', help='do not make a subdirectory within ouput_dir for the dags'), graphmls_dir: pathlib.Path = typer.Option(None, '--graphmls_dir', - help='Graphmls directory to convert into a DAG')): + help='Directory of Graphmls to convert into DAGs.')): """Export a DAG of the workflow to a GraphML file.""" output_dir = output_dir.resolve() # Make sure output_dir is an absolute path and exists diff --git a/docs/sphinx/commands.rst b/docs/sphinx/commands.rst index 4b90e0e28..3258ba7eb 100644 --- a/docs/sphinx/commands.rst +++ b/docs/sphinx/commands.rst @@ -109,7 +109,7 @@ Arguments: Arguments: WF_ID [required] -``beeflow dag``: Export a directed acyclic graph (DAG) of a submitted workflow. This command can be run at any point of the workflow. To see the DAG of a workflow before it runs, submit the workflow with the ``--no-start`` flag and then use the dag command. The DAGs are exported to $OUTPUT_DIR/$WD_ID-dags by default. If the ``no-dag-dir`` flag is specified when the dag command is run, the DAG will be exported to $OUTPUT_DIR. The dag command makes multiple versions of the DAGs. The most recent version is $WF_ID.png and the others are $WD_ID_v1.png, $WF_ID_v2.png ... where v1 is the oldest. See :ref:`workflow-visualization` for more information. +``beeflow dag``: Export a directed acyclic graph (DAG) of a submitted workflow. This command can be run at any point of the workflow. To see the DAG of a workflow before it runs, submit the workflow with the ``--no-start`` flag and then use the dag command. The DAGs are exported to $OUTPUT_DIR/$WD_ID-dags by default. If the ``no-dag-dir`` flag is specified when the dag command is run, the DAG will be exported to $OUTPUT_DIR. The dag command makes multiple versions of the DAGs. The most recent version is $WF_ID.png and the others are $WD_ID_v1.png, $WF_ID_v2.png ... where v1 is the oldest. If the ``graphmls_dir`` flag is specified when the dag command is run, BEE will convert the Graphmls in the specified directory into DAGs. See :ref:`workflow-visualization` for more information. Arguments: - WF_ID [required] @@ -117,6 +117,7 @@ Arguments: Options: ``no-dag-dir``: Do not make a subdirectory within the output_dir for the DAGs. + ``graphmls_dir``: Directory of Graphmls to convert into DAGs. Generating and Managing Configuration Files =========================================== diff --git a/docs/sphinx/visualization.rst b/docs/sphinx/visualization.rst index f4c63e913..96c2f5dd0 100644 --- a/docs/sphinx/visualization.rst +++ b/docs/sphinx/visualization.rst @@ -20,6 +20,10 @@ most recent version is $WF_ID.png and the others are $WD_ID_v1.png, $WF_ID_v2.png ... where v1 is the oldest. The graphmls used to make the DAGs are saved in the workflow archive and are saved with their version number. These graphmls can be useful for debugging when there are errors creating the DAGs. +If the --graphmls_dir flag is specified when the dag command is run, BEE will convert +the GraphMLs in the specified directory into DAGs. This is especially helpful +when transferring workflows from a system without Graphviz (BEE will only generate +GraphMLs in that case) to a system with Graphviz. Example DAG ===========