From 968bd09f3d7b8d63be21c168d895b424c0adc094 Mon Sep 17 00:00:00 2001 From: ntalluri Date: Thu, 14 Aug 2025 15:21:21 -0500 Subject: [PATCH 01/15] precommit --- Snakefile | 4 ++-- spras/evaluation.py | 30 +++++++++++++++++++++++++++++- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/Snakefile b/Snakefile index 1daa8dee9..518e7509e 100644 --- a/Snakefile +++ b/Snakefile @@ -525,7 +525,7 @@ rule evaluation_ensemble_pr_curve: run: node_table = Evaluation.from_file(input.gold_standard_file).node_table node_ensemble_dict = Evaluation.edge_frequency_node_ensemble(node_table, input.ensemble_file, input.dataset_file) - Evaluation.precision_recall_curve_node_ensemble(node_ensemble_dict, node_table, output.pr_curve_png, output.pr_curve_file) + Evaluation.precision_recall_curve_node_ensemble(node_ensemble_dict, node_table, input.dataset_file,output.pr_curve_png, output.pr_curve_file) # Returns list of algorithm specific ensemble files per dataset def collect_ensemble_per_algo_per_dataset(wildcards): @@ -544,7 +544,7 @@ rule evaluation_per_algo_ensemble_pr_curve: run: node_table = Evaluation.from_file(input.gold_standard_file).node_table node_ensembles_dict = Evaluation.edge_frequency_node_ensemble(node_table, input.ensemble_files, input.dataset_file) - Evaluation.precision_recall_curve_node_ensemble(node_ensembles_dict, node_table, output.pr_curve_png, output.pr_curve_file, include_aggregate_algo_eval) + Evaluation.precision_recall_curve_node_ensemble(node_ensembles_dict, node_table, input.dataset_file,output.pr_curve_png, output.pr_curve_file, include_aggregate_algo_eval) # Remove the output directory diff --git a/spras/evaluation.py b/spras/evaluation.py index 732e58576..0b99308c3 100644 --- a/spras/evaluation.py +++ b/spras/evaluation.py @@ -354,7 +354,7 @@ def edge_frequency_node_ensemble(node_table: pd.DataFrame, ensemble_files: list[ return node_ensembles_dict @staticmethod - def precision_recall_curve_node_ensemble(node_ensembles: dict, node_table: pd.DataFrame, output_png: str | PathLike, + def precision_recall_curve_node_ensemble(node_ensembles: dict, node_table: pd.DataFrame, dataset_file: str, output_png: str | PathLike, output_file: str | PathLike, aggregate_per_algorithm: bool = False): """ Plots precision-recall (PR) curves for a set of node ensembles evaluated against a gold standard. @@ -387,6 +387,34 @@ def precision_recall_curve_node_ensemble(node_ensembles: dict, node_table: pd.Da if not node_ensemble.empty: y_true = [1 if node in gold_standard_nodes else 0 for node in node_ensemble['Node']] y_scores = node_ensemble['Frequency'].tolist() + + # TODO: add a new one here for y_scores_baseline where the sources and targets frequency are set to 1 + # set the scores for nodes in node_ensemble that are in both the gold_standard_nodes and sources/targets/prizes to 1.0 + # then make that into a new node_ensemble with altered values that are then plotted. + + pickle = Evaluation.from_file(dataset_file) + prizes_df = pickle.get_node_columns(["sources", "targets", "prize"]) + prizes = set(prizes_df['NODEID']) + prize_gold_intersection = prizes & gold_standard_nodes + prize_node_ensemble_df = node_ensemble.copy() + + # TODO what if the node_ensemble is all frequency = 0.0, that will be the new source/target/prize/ baseline? + + # Set frequency to 1.0 for matching nodes + prize_node_ensemble_df.loc[ + prize_node_ensemble_df['Node'].isin(prize_gold_intersection), + 'Frequency' + ] = 1.0 + print(prize_node_ensemble_df) + + y_scores_prizes = prize_node_ensemble_df['Frequency'].tolist() + + precision_prizes, recall_prizes, thresholds_prizes = precision_recall_curve(y_true, y_scores_prizes) + plt.plot(recall_prizes, precision_prizes, color=color_palette[label], marker='o', linestyle=':', + label=f'{label.capitalize()} with prizes') + + + precision, recall, thresholds = precision_recall_curve(y_true, y_scores) # avg precision summarizes a precision-recall curve as the weighted mean of precisions achieved at each threshold avg_precision = average_precision_score(y_true, y_scores) From 7202a990dfbf11920fe52a38bbc9b7727813ec7a Mon Sep 17 00:00:00 2001 From: ntalluri Date: Tue, 19 Aug 2025 17:34:23 -0500 Subject: [PATCH 02/15] updating the baseline --- spras/evaluation.py | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/spras/evaluation.py b/spras/evaluation.py index 0b99308c3..7c3117459 100644 --- a/spras/evaluation.py +++ b/spras/evaluation.py @@ -382,37 +382,36 @@ def precision_recall_curve_node_ensemble(node_ensembles: dict, node_table: pd.Da metric_dfs = [] baseline = None + y_scores_input_nodes = None + for label, node_ensemble in node_ensembles.items(): if not node_ensemble.empty: y_true = [1 if node in gold_standard_nodes else 0 for node in node_ensemble['Node']] y_scores = node_ensemble['Frequency'].tolist() - # TODO: add a new one here for y_scores_baseline where the sources and targets frequency are set to 1 - # set the scores for nodes in node_ensemble that are in both the gold_standard_nodes and sources/targets/prizes to 1.0 - # then make that into a new node_ensemble with altered values that are then plotted. - - pickle = Evaluation.from_file(dataset_file) - prizes_df = pickle.get_node_columns(["sources", "targets", "prize"]) - prizes = set(prizes_df['NODEID']) - prize_gold_intersection = prizes & gold_standard_nodes - prize_node_ensemble_df = node_ensemble.copy() - - # TODO what if the node_ensemble is all frequency = 0.0, that will be the new source/target/prize/ baseline? + if y_scores_input_nodes is None: + pickle = Evaluation.from_file(dataset_file) + input_nodes_df = pickle.get_node_columns(["sources", "targets", "prize", "active"]) + input_nodes = set(input_nodes_df['NODEID']) + input_nodes_gold_intersection = input_nodes & gold_standard_nodes # TODO should this be all inputs or the intersection with the gold standard for this baseline? + input_nodes_ensemble_df = node_ensemble.copy() - # Set frequency to 1.0 for matching nodes - prize_node_ensemble_df.loc[ - prize_node_ensemble_df['Node'].isin(prize_gold_intersection), - 'Frequency' - ] = 1.0 - print(prize_node_ensemble_df) + input_nodes_ensemble_df.loc[ + input_nodes_ensemble_df['Node'].isin(input_nodes_gold_intersection), + 'Frequency' + ] = 1.0 - y_scores_prizes = prize_node_ensemble_df['Frequency'].tolist() + input_nodes_ensemble_df.loc[ + ~input_nodes_ensemble_df['Node'].isin(input_nodes_gold_intersection), + 'Frequency' + ] = 0.0 - precision_prizes, recall_prizes, thresholds_prizes = precision_recall_curve(y_true, y_scores_prizes) - plt.plot(recall_prizes, precision_prizes, color=color_palette[label], marker='o', linestyle=':', - label=f'{label.capitalize()} with prizes') + y_scores_input_nodes = input_nodes_ensemble_df['Frequency'].tolist() + precision_input_nodes, recall_input_nodes, thresholds_input_nodes = precision_recall_curve(y_true, y_scores_input_nodes) + plt.plot(recall_input_nodes, precision_input_nodes, color='black', marker='o', linestyle='--', + label=f'Input Nodes Baseline') precision, recall, thresholds = precision_recall_curve(y_true, y_scores) @@ -472,6 +471,7 @@ def precision_recall_curve_node_ensemble(node_ensembles: dict, node_table: pd.Da combined_prc_df = pd.concat(prc_dfs, ignore_index=True) combined_metrics_df = pd.concat(metric_dfs, ignore_index=True) combined_metrics_df['Baseline'] = baseline + # TODO add new input_node baseline to the txt # merge dfs and NaN out metric values except for first row of each Ensemble_Source complete_df = combined_prc_df.merge(combined_metrics_df, on='Ensemble_Source', how='left') From 1b3be75bd4460dac9edad4626d8df8cc9ee544d6 Mon Sep 17 00:00:00 2001 From: ntalluri Date: Tue, 19 Aug 2025 17:37:55 -0500 Subject: [PATCH 03/15] make it all input nodes, not the intersection between gold standard and input nodes --- spras/evaluation.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/spras/evaluation.py b/spras/evaluation.py index 7c3117459..3b66e70bc 100644 --- a/spras/evaluation.py +++ b/spras/evaluation.py @@ -394,16 +394,16 @@ def precision_recall_curve_node_ensemble(node_ensembles: dict, node_table: pd.Da pickle = Evaluation.from_file(dataset_file) input_nodes_df = pickle.get_node_columns(["sources", "targets", "prize", "active"]) input_nodes = set(input_nodes_df['NODEID']) - input_nodes_gold_intersection = input_nodes & gold_standard_nodes # TODO should this be all inputs or the intersection with the gold standard for this baseline? + # input_nodes_gold_intersection = input_nodes & gold_standard_nodes # TODO should this be all inputs or the intersection with the gold standard for this baseline? input_nodes_ensemble_df = node_ensemble.copy() input_nodes_ensemble_df.loc[ - input_nodes_ensemble_df['Node'].isin(input_nodes_gold_intersection), + input_nodes_ensemble_df['Node'].isin(input_nodes), 'Frequency' ] = 1.0 input_nodes_ensemble_df.loc[ - ~input_nodes_ensemble_df['Node'].isin(input_nodes_gold_intersection), + ~input_nodes_ensemble_df['Node'].isin(input_nodes), 'Frequency' ] = 0.0 From 8ca0e9ed10eb45b812d453640aaea9dc3c2e9030 Mon Sep 17 00:00:00 2001 From: ntalluri Date: Wed, 20 Aug 2025 12:25:50 -0500 Subject: [PATCH 04/15] update to be the baseline with gold standard intersection, added data to the txt files --- spras/evaluation.py | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/spras/evaluation.py b/spras/evaluation.py index 3b66e70bc..f05423e47 100644 --- a/spras/evaluation.py +++ b/spras/evaluation.py @@ -380,9 +380,12 @@ def precision_recall_curve_node_ensemble(node_ensembles: dict, node_table: pd.Da prc_dfs = [] metric_dfs = [] + prc_input_nodes_baseline_df = None baseline = None - y_scores_input_nodes = None + precision_input_nodes = None + recall_input_nodes = None + thresholds_input_nodes = None for label, node_ensemble in node_ensembles.items(): @@ -390,20 +393,20 @@ def precision_recall_curve_node_ensemble(node_ensembles: dict, node_table: pd.Da y_true = [1 if node in gold_standard_nodes else 0 for node in node_ensemble['Node']] y_scores = node_ensemble['Frequency'].tolist() - if y_scores_input_nodes is None: + if precision_input_nodes is None and recall_input_nodes is None and thresholds_input_nodes is None: pickle = Evaluation.from_file(dataset_file) input_nodes_df = pickle.get_node_columns(["sources", "targets", "prize", "active"]) input_nodes = set(input_nodes_df['NODEID']) - # input_nodes_gold_intersection = input_nodes & gold_standard_nodes # TODO should this be all inputs or the intersection with the gold standard for this baseline? + input_nodes_gold_intersection = input_nodes & gold_standard_nodes # TODO should this be all inputs nodes or the intersection with the gold standard for this baseline? input_nodes_ensemble_df = node_ensemble.copy() input_nodes_ensemble_df.loc[ - input_nodes_ensemble_df['Node'].isin(input_nodes), + input_nodes_ensemble_df['Node'].isin(input_nodes_gold_intersection), 'Frequency' ] = 1.0 input_nodes_ensemble_df.loc[ - ~input_nodes_ensemble_df['Node'].isin(input_nodes), + ~input_nodes_ensemble_df['Node'].isin(input_nodes_gold_intersection), 'Frequency' ] = 0.0 @@ -413,6 +416,17 @@ def precision_recall_curve_node_ensemble(node_ensembles: dict, node_table: pd.Da plt.plot(recall_input_nodes, precision_input_nodes, color='black', marker='o', linestyle='--', label=f'Input Nodes Baseline') + prc_input_nodes_baseline_data = { + 'Threshold': thresholds_input_nodes, + 'Precision': precision_input_nodes[:-1], + 'Recall': recall_input_nodes[:-1], + } + + prc_input_nodes_baseline_data = {'Ensemble_Source': ["input_nodes_baseline"] * len(thresholds_input_nodes), **prc_input_nodes_baseline_data} + + prc_input_nodes_baseline_df = pd.DataFrame.from_dict(prc_input_nodes_baseline_data) + + precision, recall, thresholds = precision_recall_curve(y_true, y_scores) # avg precision summarizes a precision-recall curve as the weighted mean of precisions achieved at each threshold @@ -471,10 +485,9 @@ def precision_recall_curve_node_ensemble(node_ensembles: dict, node_table: pd.Da combined_prc_df = pd.concat(prc_dfs, ignore_index=True) combined_metrics_df = pd.concat(metric_dfs, ignore_index=True) combined_metrics_df['Baseline'] = baseline - # TODO add new input_node baseline to the txt # merge dfs and NaN out metric values except for first row of each Ensemble_Source - complete_df = combined_prc_df.merge(combined_metrics_df, on='Ensemble_Source', how='left') + complete_df = combined_prc_df.merge(combined_metrics_df, on='Ensemble_Source', how='left').merge(prc_input_nodes_baseline_df, on=['Ensemble_Source', 'Threshold', 'Precision', 'Recall'], how='outer') not_last_rows = complete_df.duplicated(subset='Ensemble_Source', keep='first') complete_df.loc[not_last_rows, ['Average_Precision', 'Baseline']] = None complete_df.to_csv(output_file, index=False, sep='\t') From dda62645fe71b36ba1128ff1a87bf4178354f00b Mon Sep 17 00:00:00 2001 From: ntalluri Date: Wed, 20 Aug 2025 14:16:23 -0500 Subject: [PATCH 05/15] debugging error in test cases --- spras/evaluation.py | 4 ++++ test/evaluate/test_evaluate.py | 32 +++++++++++++++++++++++--------- 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/spras/evaluation.py b/spras/evaluation.py index f05423e47..a22a31c3a 100644 --- a/spras/evaluation.py +++ b/spras/evaluation.py @@ -410,12 +410,16 @@ def precision_recall_curve_node_ensemble(node_ensembles: dict, node_table: pd.Da 'Frequency' ] = 0.0 + print(input_nodes_ensemble_df) + y_scores_input_nodes = input_nodes_ensemble_df['Frequency'].tolist() precision_input_nodes, recall_input_nodes, thresholds_input_nodes = precision_recall_curve(y_true, y_scores_input_nodes) plt.plot(recall_input_nodes, precision_input_nodes, color='black', marker='o', linestyle='--', label=f'Input Nodes Baseline') + print(precision_input_nodes) + print(recall_input_nodes) prc_input_nodes_baseline_data = { 'Threshold': thresholds_input_nodes, 'Precision': precision_input_nodes[:-1], diff --git a/test/evaluate/test_evaluate.py b/test/evaluate/test_evaluate.py index ce50350e5..71616496b 100644 --- a/test/evaluate/test_evaluate.py +++ b/test/evaluate/test_evaluate.py @@ -35,6 +35,8 @@ def setup_class(cls): 'other_files': [] }) + # TODO figure out why the input-nodes file is not being included in the data.pickle file + # it keeps coming up empty with open(out_dataset, 'wb') as f: pickle.dump(dataset, f) @@ -126,8 +128,8 @@ def test_node_ensemble(self): out_path_file = Path(OUT_DIR + 'node-ensemble.csv') out_path_file.unlink(missing_ok=True) ensemble_network = [INPUT_DIR + 'ensemble-network.tsv'] - input_network = OUT_DIR + 'data.pickle' - node_ensemble_dict = Evaluation.edge_frequency_node_ensemble(GS_NODE_TABLE, ensemble_network, input_network) + input_data = OUT_DIR + 'data.pickle' + node_ensemble_dict = Evaluation.edge_frequency_node_ensemble(GS_NODE_TABLE, ensemble_network, input_data) node_ensemble_dict['ensemble'].to_csv(out_path_file, sep='\t', index=False) assert filecmp.cmp(out_path_file, EXPECT_DIR + 'expected-node-ensemble.csv', shallow=False) @@ -135,9 +137,9 @@ def test_empty_node_ensemble(self): out_path_file = Path(OUT_DIR + 'empty-node-ensemble.csv') out_path_file.unlink(missing_ok=True) empty_ensemble_network = [INPUT_DIR + 'empty-ensemble-network.tsv'] - input_network = OUT_DIR + 'data.pickle' + input_data = OUT_DIR + 'data.pickle' node_ensemble_dict = Evaluation.edge_frequency_node_ensemble(GS_NODE_TABLE, empty_ensemble_network, - input_network) + input_data) node_ensemble_dict['empty'].to_csv(out_path_file, sep='\t', index=False) assert filecmp.cmp(out_path_file, EXPECT_DIR + 'expected-empty-node-ensemble.csv', shallow=False) @@ -147,8 +149,8 @@ def test_multiple_node_ensemble(self): out_path_empty_file = Path(OUT_DIR + 'empty-node-ensemble.csv') out_path_empty_file.unlink(missing_ok=True) ensemble_networks = [INPUT_DIR + 'ensemble-network.tsv', INPUT_DIR + 'empty-ensemble-network.tsv'] - input_network = OUT_DIR + 'data.pickle' - node_ensemble_dict = Evaluation.edge_frequency_node_ensemble(GS_NODE_TABLE, ensemble_networks, input_network) + input_data = OUT_DIR + 'data.pickle' + node_ensemble_dict = Evaluation.edge_frequency_node_ensemble(GS_NODE_TABLE, ensemble_networks, input_data) node_ensemble_dict['ensemble'].to_csv(out_path_file, sep='\t', index=False) assert filecmp.cmp(out_path_file, EXPECT_DIR + 'expected-node-ensemble.csv', shallow=False) node_ensemble_dict['empty'].to_csv(out_path_empty_file, sep='\t', index=False) @@ -159,9 +161,19 @@ def test_precision_recall_curve_ensemble_nodes(self): out_path_png.unlink(missing_ok=True) out_path_file = Path(OUT_DIR + 'pr-curve-ensemble-nodes.txt') out_path_file.unlink(missing_ok=True) + input_data = OUT_DIR + 'data.pickle' + + pickle = Evaluation.from_file(input_data) + input_nodes_df = pickle.get_node_columns(["prize", "active"]) + + print(input_nodes_df) + ensemble_file = pd.read_csv(INPUT_DIR + 'node-ensemble.csv', sep='\t', header=0) + + print(ensemble_file) + node_ensembles_dict = {'ensemble': ensemble_file} - Evaluation.precision_recall_curve_node_ensemble(node_ensembles_dict, GS_NODE_TABLE, out_path_png, + Evaluation.precision_recall_curve_node_ensemble(node_ensembles_dict, GS_NODE_TABLE, input_data, out_path_png, out_path_file) assert out_path_png.exists() assert filecmp.cmp(out_path_file, EXPECT_DIR + 'expected-pr-curve-ensemble-nodes.txt', shallow=False) @@ -171,9 +183,10 @@ def test_precision_recall_curve_ensemble_nodes_empty(self): out_path_png.unlink(missing_ok=True) out_path_file = Path(OUT_DIR + 'pr-curve-ensemble-nodes-empty.txt') out_path_file.unlink(missing_ok=True) + input_data = OUT_DIR + 'data.pickle' empty_ensemble_file = pd.read_csv(INPUT_DIR + 'node-ensemble-empty.csv', sep='\t', header=0) node_ensembles_dict = {'ensemble': empty_ensemble_file} - Evaluation.precision_recall_curve_node_ensemble(node_ensembles_dict, GS_NODE_TABLE, out_path_png, + Evaluation.precision_recall_curve_node_ensemble(node_ensembles_dict, GS_NODE_TABLE, input_data, out_path_png, out_path_file) assert out_path_png.exists() assert filecmp.cmp(out_path_file, EXPECT_DIR + 'expected-pr-curve-ensemble-nodes-empty.txt', shallow=False) @@ -183,10 +196,11 @@ def test_precision_recall_curve_multiple_ensemble_nodes(self): out_path_png.unlink(missing_ok=True) out_path_file = Path(OUT_DIR + 'pr-curve-multiple-ensemble-nodes.txt') out_path_file.unlink(missing_ok=True) + input_data = OUT_DIR + 'data.pickle' ensemble_file = pd.read_csv(INPUT_DIR + 'node-ensemble.csv', sep='\t', header=0) empty_ensemble_file = pd.read_csv(INPUT_DIR + 'node-ensemble-empty.csv', sep='\t', header=0) node_ensembles_dict = {'ensemble1': ensemble_file, 'ensemble2': ensemble_file, 'ensemble3': empty_ensemble_file} - Evaluation.precision_recall_curve_node_ensemble(node_ensembles_dict, GS_NODE_TABLE, out_path_png, + Evaluation.precision_recall_curve_node_ensemble(node_ensembles_dict, GS_NODE_TABLE, input_data, out_path_png, out_path_file, True) assert out_path_png.exists() assert filecmp.cmp(out_path_file, EXPECT_DIR + 'expected-pr-curve-multiple-ensemble-nodes.txt', shallow=False) From d8a16fc6c4e5d73a97510a16434483cf0a011abd Mon Sep 17 00:00:00 2001 From: ntalluri Date: Thu, 21 Aug 2025 15:17:17 -0500 Subject: [PATCH 06/15] fix input-nodes file with tristan's update --- test/evaluate/input/input-nodes.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/evaluate/input/input-nodes.txt b/test/evaluate/input/input-nodes.txt index 1ffd8bdff..ef08aff66 100644 --- a/test/evaluate/input/input-nodes.txt +++ b/test/evaluate/input/input-nodes.txt @@ -1,3 +1,3 @@ NODEID prize active dummy sources targets -N -C 5.7 True True +N +C 5.7 True True \ No newline at end of file From 7f3c7f63b27d9b94be2e6e6852b2a02c2a3b4e9b Mon Sep 17 00:00:00 2001 From: ntalluri Date: Thu, 21 Aug 2025 15:41:23 -0500 Subject: [PATCH 07/15] update test case --- spras/evaluation.py | 4 ---- .../expected-pr-curve-ensemble-nodes-empty.txt | 2 ++ .../expected/expected-pr-curve-ensemble-nodes.txt | 2 ++ .../expected-pr-curve-multiple-ensemble-nodes.txt | 2 ++ test/evaluate/input/input-nodes.txt | 3 ++- test/evaluate/test_evaluate.py | 11 ----------- 6 files changed, 8 insertions(+), 16 deletions(-) diff --git a/spras/evaluation.py b/spras/evaluation.py index a22a31c3a..f05423e47 100644 --- a/spras/evaluation.py +++ b/spras/evaluation.py @@ -410,16 +410,12 @@ def precision_recall_curve_node_ensemble(node_ensembles: dict, node_table: pd.Da 'Frequency' ] = 0.0 - print(input_nodes_ensemble_df) - y_scores_input_nodes = input_nodes_ensemble_df['Frequency'].tolist() precision_input_nodes, recall_input_nodes, thresholds_input_nodes = precision_recall_curve(y_true, y_scores_input_nodes) plt.plot(recall_input_nodes, precision_input_nodes, color='black', marker='o', linestyle='--', label=f'Input Nodes Baseline') - print(precision_input_nodes) - print(recall_input_nodes) prc_input_nodes_baseline_data = { 'Threshold': thresholds_input_nodes, 'Precision': precision_input_nodes[:-1], diff --git a/test/evaluate/expected/expected-pr-curve-ensemble-nodes-empty.txt b/test/evaluate/expected/expected-pr-curve-ensemble-nodes-empty.txt index c9f6561c6..b57d3adfa 100644 --- a/test/evaluate/expected/expected-pr-curve-ensemble-nodes-empty.txt +++ b/test/evaluate/expected/expected-pr-curve-ensemble-nodes-empty.txt @@ -1,2 +1,4 @@ Ensemble_Source Threshold Precision Recall Average_Precision Baseline Aggregated 0.0 0.15384615384615385 1.0 0.15384615384615385 0.15384615384615385 +input_nodes_baseline 0.0 0.15384615384615385 1.0 +input_nodes_baseline 1.0 1.0 0.25 diff --git a/test/evaluate/expected/expected-pr-curve-ensemble-nodes.txt b/test/evaluate/expected/expected-pr-curve-ensemble-nodes.txt index b0e50594e..21063c107 100644 --- a/test/evaluate/expected/expected-pr-curve-ensemble-nodes.txt +++ b/test/evaluate/expected/expected-pr-curve-ensemble-nodes.txt @@ -3,3 +3,5 @@ Aggregated 0.0 0.15384615384615385 1.0 0.6666666666666666 0.15384615384615385 Aggregated 0.01 0.6666666666666666 1.0 Aggregated 0.5 0.75 0.75 Aggregated 0.66 0.5 0.25 +input_nodes_baseline 0.0 0.15384615384615385 1.0 +input_nodes_baseline 1.0 1.0 0.25 diff --git a/test/evaluate/expected/expected-pr-curve-multiple-ensemble-nodes.txt b/test/evaluate/expected/expected-pr-curve-multiple-ensemble-nodes.txt index 630a89ceb..c38848d82 100644 --- a/test/evaluate/expected/expected-pr-curve-multiple-ensemble-nodes.txt +++ b/test/evaluate/expected/expected-pr-curve-multiple-ensemble-nodes.txt @@ -8,3 +8,5 @@ Ensemble2 0.01 0.6666666666666666 1.0 Ensemble2 0.5 0.75 0.75 Ensemble2 0.66 0.5 0.25 Ensemble3 0.0 0.15384615384615385 1.0 0.15384615384615385 0.15384615384615385 +input_nodes_baseline 0.0 0.15384615384615385 1.0 +input_nodes_baseline 1.0 1.0 0.25 diff --git a/test/evaluate/input/input-nodes.txt b/test/evaluate/input/input-nodes.txt index ef08aff66..ec0e4058f 100644 --- a/test/evaluate/input/input-nodes.txt +++ b/test/evaluate/input/input-nodes.txt @@ -1,3 +1,4 @@ NODEID prize active dummy sources targets N -C 5.7 True True \ No newline at end of file +C 5.7 True True +A 5 True True \ No newline at end of file diff --git a/test/evaluate/test_evaluate.py b/test/evaluate/test_evaluate.py index 71616496b..1178f427a 100644 --- a/test/evaluate/test_evaluate.py +++ b/test/evaluate/test_evaluate.py @@ -35,8 +35,6 @@ def setup_class(cls): 'other_files': [] }) - # TODO figure out why the input-nodes file is not being included in the data.pickle file - # it keeps coming up empty with open(out_dataset, 'wb') as f: pickle.dump(dataset, f) @@ -162,16 +160,7 @@ def test_precision_recall_curve_ensemble_nodes(self): out_path_file = Path(OUT_DIR + 'pr-curve-ensemble-nodes.txt') out_path_file.unlink(missing_ok=True) input_data = OUT_DIR + 'data.pickle' - - pickle = Evaluation.from_file(input_data) - input_nodes_df = pickle.get_node_columns(["prize", "active"]) - - print(input_nodes_df) - ensemble_file = pd.read_csv(INPUT_DIR + 'node-ensemble.csv', sep='\t', header=0) - - print(ensemble_file) - node_ensembles_dict = {'ensemble': ensemble_file} Evaluation.precision_recall_curve_node_ensemble(node_ensembles_dict, GS_NODE_TABLE, input_data, out_path_png, out_path_file) From 8492f70ca3b151bae94e6e71ab67dc09c2ce29fa Mon Sep 17 00:00:00 2001 From: ntalluri Date: Fri, 22 Aug 2025 11:05:27 -0500 Subject: [PATCH 08/15] clean up code and outputs --- spras/evaluation.py | 45 ++++++++++++++++++++++++++------------------- 1 file changed, 26 insertions(+), 19 deletions(-) diff --git a/spras/evaluation.py b/spras/evaluation.py index f05423e47..8d1913d88 100644 --- a/spras/evaluation.py +++ b/spras/evaluation.py @@ -354,7 +354,7 @@ def edge_frequency_node_ensemble(node_table: pd.DataFrame, ensemble_files: list[ return node_ensembles_dict @staticmethod - def precision_recall_curve_node_ensemble(node_ensembles: dict, node_table: pd.DataFrame, dataset_file: str, output_png: str | PathLike, + def precision_recall_curve_node_ensemble(node_ensembles: dict, node_table: pd.DataFrame, input_nodes: pd.DataFrame, output_png: str | PathLike, output_file: str | PathLike, aggregate_per_algorithm: bool = False): """ Plots precision-recall (PR) curves for a set of node ensembles evaluated against a gold standard. @@ -365,6 +365,7 @@ def precision_recall_curve_node_ensemble(node_ensembles: dict, node_table: pd.Da @param node_ensembles: dict of the pre-computed node_ensemble(s) @param node_table: gold standard nodes + @param input_nodes: the input nodes (sources, targets, prizes, actives) used for a specific dataset @param output_png: filename to save the precision and recall curves as a .png image @param output_file: filename to save the precision, recall, threshold values, average precision, and baseline average precision @@ -393,22 +394,18 @@ def precision_recall_curve_node_ensemble(node_ensembles: dict, node_table: pd.Da y_true = [1 if node in gold_standard_nodes else 0 for node in node_ensemble['Node']] y_scores = node_ensemble['Frequency'].tolist() - if precision_input_nodes is None and recall_input_nodes is None and thresholds_input_nodes is None: - pickle = Evaluation.from_file(dataset_file) - input_nodes_df = pickle.get_node_columns(["sources", "targets", "prize", "active"]) - input_nodes = set(input_nodes_df['NODEID']) - input_nodes_gold_intersection = input_nodes & gold_standard_nodes # TODO should this be all inputs nodes or the intersection with the gold standard for this baseline? + # input nodes (sources, targets, prizes, actives) may be easier to recover but are still valid gold standard nodes; + # the Input_Nodes_Baseline PR curve highlights their overlap with the gold standard. + if prc_input_nodes_baseline_df is None: + # pickle = Evaluation.from_file(dataset_file) + # input_nodes_df = pickle.get_node_columns(["sources", "targets", "prize", "active"]) + input_nodes_set = set(input_nodes['NODEID']) + input_nodes_gold_intersection = input_nodes_set & gold_standard_nodes # TODO should this be all inputs nodes or the intersection with the gold standard for this baseline? I think it should be the intersection input_nodes_ensemble_df = node_ensemble.copy() - input_nodes_ensemble_df.loc[ - input_nodes_ensemble_df['Node'].isin(input_nodes_gold_intersection), - 'Frequency' - ] = 1.0 - - input_nodes_ensemble_df.loc[ - ~input_nodes_ensemble_df['Node'].isin(input_nodes_gold_intersection), - 'Frequency' - ] = 0.0 + input_nodes_ensemble_df["Frequency"] = ( + input_nodes_ensemble_df["Node"].isin(input_nodes_gold_intersection).astype(float) + ) y_scores_input_nodes = input_nodes_ensemble_df['Frequency'].tolist() @@ -416,18 +413,16 @@ def precision_recall_curve_node_ensemble(node_ensembles: dict, node_table: pd.Da plt.plot(recall_input_nodes, precision_input_nodes, color='black', marker='o', linestyle='--', label=f'Input Nodes Baseline') + # Dropping last elements because scikit-learn adds (1, 0) to precision/recall for plotting, not tied to real thresholds prc_input_nodes_baseline_data = { 'Threshold': thresholds_input_nodes, 'Precision': precision_input_nodes[:-1], 'Recall': recall_input_nodes[:-1], } - prc_input_nodes_baseline_data = {'Ensemble_Source': ["input_nodes_baseline"] * len(thresholds_input_nodes), **prc_input_nodes_baseline_data} - + prc_input_nodes_baseline_data = {'Ensemble_Source': ["Input_Nodes_Baseline"] * len(thresholds_input_nodes), **prc_input_nodes_baseline_data} prc_input_nodes_baseline_df = pd.DataFrame.from_dict(prc_input_nodes_baseline_data) - - precision, recall, thresholds = precision_recall_curve(y_true, y_scores) # avg precision summarizes a precision-recall curve as the weighted mean of precisions achieved at each threshold avg_precision = average_precision_score(y_true, y_scores) @@ -488,6 +483,18 @@ def precision_recall_curve_node_ensemble(node_ensembles: dict, node_table: pd.Da # merge dfs and NaN out metric values except for first row of each Ensemble_Source complete_df = combined_prc_df.merge(combined_metrics_df, on='Ensemble_Source', how='left').merge(prc_input_nodes_baseline_df, on=['Ensemble_Source', 'Threshold', 'Precision', 'Recall'], how='outer') + + # for each Ensemble_Source, remove Average_Precision and Baseline in all but the first row not_last_rows = complete_df.duplicated(subset='Ensemble_Source', keep='first') complete_df.loc[not_last_rows, ['Average_Precision', 'Baseline']] = None + + # move Input_Nodes_Baseline to the top of the df + complete_df.sort_values( + by='Ensemble_Source', + # x.ne('Input_Nodes_Baseline'): returns a Series of booleans; True for all rows except Input_Nodes_Baseline. + # Since False < True, baseline rows sort to the top. + key=lambda x: x.ne('Input_Nodes_Baseline'), + inplace=True + ) + complete_df.to_csv(output_file, index=False, sep='\t') From 9b8e31ce92a36eca877f34ccdc9bb5b56d397e04 Mon Sep 17 00:00:00 2001 From: ntalluri Date: Fri, 22 Aug 2025 11:16:25 -0500 Subject: [PATCH 09/15] update to use input nodes and interactome from the snakefile instead of the functions itself --- Snakefile | 12 ++++++++---- spras/evaluation.py | 19 +++++++------------ 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/Snakefile b/Snakefile index 518e7509e..068b3fcb6 100644 --- a/Snakefile +++ b/Snakefile @@ -524,8 +524,10 @@ rule evaluation_ensemble_pr_curve: pr_curve_file = SEP.join([out_dir, '{dataset_gold_standard_pairs}-eval', 'pr-curve-ensemble-nodes.txt']), run: node_table = Evaluation.from_file(input.gold_standard_file).node_table - node_ensemble_dict = Evaluation.edge_frequency_node_ensemble(node_table, input.ensemble_file, input.dataset_file) - Evaluation.precision_recall_curve_node_ensemble(node_ensemble_dict, node_table, input.dataset_file,output.pr_curve_png, output.pr_curve_file) + input_interactome = Evaluation.from_file(input.dataset_file).get_interactome() + node_ensemble_dict = Evaluation.edge_frequency_node_ensemble(node_table, input.ensemble_file, input_interactome) + input_nodes = Evaluation.from_file(input.dataset_file).get_node_columns(["sources", "targets", "prize", "active"]) + Evaluation.precision_recall_curve_node_ensemble(node_ensemble_dict, node_table, input_nodes,output.pr_curve_png, output.pr_curve_file) # Returns list of algorithm specific ensemble files per dataset def collect_ensemble_per_algo_per_dataset(wildcards): @@ -543,8 +545,10 @@ rule evaluation_per_algo_ensemble_pr_curve: pr_curve_file = SEP.join([out_dir, '{dataset_gold_standard_pairs}-eval', 'pr-curve-ensemble-nodes-per-algorithm.txt']), run: node_table = Evaluation.from_file(input.gold_standard_file).node_table - node_ensembles_dict = Evaluation.edge_frequency_node_ensemble(node_table, input.ensemble_files, input.dataset_file) - Evaluation.precision_recall_curve_node_ensemble(node_ensembles_dict, node_table, input.dataset_file,output.pr_curve_png, output.pr_curve_file, include_aggregate_algo_eval) + input_interactome = Evaluation.from_file(input.dataset_file).get_interactome() + node_ensembles_dict = Evaluation.edge_frequency_node_ensemble(node_table, input.ensemble_files, input_interactome) + input_nodes = Evaluation.from_file(input.dataset_file).get_node_columns(["sources", "targets", "prize", "active"]) + Evaluation.precision_recall_curve_node_ensemble(node_ensembles_dict, node_table, input_nodes,output.pr_curve_png, output.pr_curve_file, include_aggregate_algo_eval) # Remove the output directory diff --git a/spras/evaluation.py b/spras/evaluation.py index 8d1913d88..628c46b33 100644 --- a/spras/evaluation.py +++ b/spras/evaluation.py @@ -288,7 +288,7 @@ def pca_chosen_pathway(coordinates_files: list[Union[str, PathLike]], pathway_su return rep_pathways @staticmethod - def edge_frequency_node_ensemble(node_table: pd.DataFrame, ensemble_files: list[Union[str, PathLike]], dataset_file: str) -> dict: + def edge_frequency_node_ensemble(node_table: pd.DataFrame, ensemble_files: list[Union[str, PathLike]], input_interactome: pd.DataFrame) -> dict: """ Generates a dictionary of node ensembles using edge frequency data from a list of ensemble files. A list of ensemble files can contain an aggregated ensemble or algorithm-specific ensembles per dataset @@ -308,28 +308,25 @@ def edge_frequency_node_ensemble(node_table: pd.DataFrame, ensemble_files: list[ @param node_table: dataFrame of gold standard nodes (column: NODEID) @param ensemble_files: list of file paths containing edge ensemble outputs - @param dataset_file: path to the dataset file used to load the interactome + @param input_interactome: the input interactome used for a specific dataset @return: dictionary mapping each ensemble source to its node ensemble DataFrame """ node_ensembles_dict = dict() - pickle = Evaluation.from_file(dataset_file) - interactome = pickle.get_interactome() - - if interactome.empty: + if input_interactome.empty: raise ValueError( - f"Cannot compute PR curve or generate node ensemble. Input network for dataset \"{dataset_file.split('-')[0]}\" is empty." + f"Cannot compute PR curve or generate node ensemble. The input network is empty." ) if node_table.empty: raise ValueError( - f"Cannot compute PR curve or generate node ensemble. Gold standard associated with dataset \"{dataset_file.split('-')[0]}\" is empty." + f"Cannot compute PR curve or generate node ensemble. The gold standard is empty." ) # set the initial default frequencies to 0 for all interactome and gold standard nodes - node1_interactome = interactome[['Interactor1']].rename(columns={'Interactor1': 'Node'}) + node1_interactome = input_interactome[['Interactor1']].rename(columns={'Interactor1': 'Node'}) node1_interactome['Frequency'] = 0.0 - node2_interactome = interactome[['Interactor2']].rename(columns={'Interactor2': 'Node'}) + node2_interactome = input_interactome[['Interactor2']].rename(columns={'Interactor2': 'Node'}) node2_interactome['Frequency'] = 0.0 gs_nodes = node_table[[Evaluation.NODE_ID]].rename(columns={Evaluation.NODE_ID: 'Node'}) gs_nodes['Frequency'] = 0.0 @@ -397,8 +394,6 @@ def precision_recall_curve_node_ensemble(node_ensembles: dict, node_table: pd.Da # input nodes (sources, targets, prizes, actives) may be easier to recover but are still valid gold standard nodes; # the Input_Nodes_Baseline PR curve highlights their overlap with the gold standard. if prc_input_nodes_baseline_df is None: - # pickle = Evaluation.from_file(dataset_file) - # input_nodes_df = pickle.get_node_columns(["sources", "targets", "prize", "active"]) input_nodes_set = set(input_nodes['NODEID']) input_nodes_gold_intersection = input_nodes_set & gold_standard_nodes # TODO should this be all inputs nodes or the intersection with the gold standard for this baseline? I think it should be the intersection input_nodes_ensemble_df = node_ensemble.copy() From 04ddc1ffe3b2aefc238fb934fadf06b6299faf25 Mon Sep 17 00:00:00 2001 From: ntalluri Date: Fri, 22 Aug 2025 11:21:49 -0500 Subject: [PATCH 10/15] update test cases --- ...expected-pr-curve-ensemble-nodes-empty.txt | 4 ++-- .../expected-pr-curve-ensemble-nodes.txt | 4 ++-- ...ected-pr-curve-multiple-ensemble-nodes.txt | 4 ++-- test/evaluate/test_evaluate.py | 19 ++++++++++++------- 4 files changed, 18 insertions(+), 13 deletions(-) diff --git a/test/evaluate/expected/expected-pr-curve-ensemble-nodes-empty.txt b/test/evaluate/expected/expected-pr-curve-ensemble-nodes-empty.txt index b57d3adfa..4517ceb0e 100644 --- a/test/evaluate/expected/expected-pr-curve-ensemble-nodes-empty.txt +++ b/test/evaluate/expected/expected-pr-curve-ensemble-nodes-empty.txt @@ -1,4 +1,4 @@ Ensemble_Source Threshold Precision Recall Average_Precision Baseline +Input_Nodes_Baseline 0.0 0.15384615384615385 1.0 +Input_Nodes_Baseline 1.0 1.0 0.25 Aggregated 0.0 0.15384615384615385 1.0 0.15384615384615385 0.15384615384615385 -input_nodes_baseline 0.0 0.15384615384615385 1.0 -input_nodes_baseline 1.0 1.0 0.25 diff --git a/test/evaluate/expected/expected-pr-curve-ensemble-nodes.txt b/test/evaluate/expected/expected-pr-curve-ensemble-nodes.txt index 21063c107..9a78dbe77 100644 --- a/test/evaluate/expected/expected-pr-curve-ensemble-nodes.txt +++ b/test/evaluate/expected/expected-pr-curve-ensemble-nodes.txt @@ -1,7 +1,7 @@ Ensemble_Source Threshold Precision Recall Average_Precision Baseline +Input_Nodes_Baseline 0.0 0.15384615384615385 1.0 +Input_Nodes_Baseline 1.0 1.0 0.25 Aggregated 0.0 0.15384615384615385 1.0 0.6666666666666666 0.15384615384615385 Aggregated 0.01 0.6666666666666666 1.0 Aggregated 0.5 0.75 0.75 Aggregated 0.66 0.5 0.25 -input_nodes_baseline 0.0 0.15384615384615385 1.0 -input_nodes_baseline 1.0 1.0 0.25 diff --git a/test/evaluate/expected/expected-pr-curve-multiple-ensemble-nodes.txt b/test/evaluate/expected/expected-pr-curve-multiple-ensemble-nodes.txt index c38848d82..483e972d8 100644 --- a/test/evaluate/expected/expected-pr-curve-multiple-ensemble-nodes.txt +++ b/test/evaluate/expected/expected-pr-curve-multiple-ensemble-nodes.txt @@ -1,4 +1,6 @@ Ensemble_Source Threshold Precision Recall Average_Precision Baseline +Input_Nodes_Baseline 0.0 0.15384615384615385 1.0 +Input_Nodes_Baseline 1.0 1.0 0.25 Ensemble1 0.0 0.15384615384615385 1.0 0.6666666666666666 0.15384615384615385 Ensemble1 0.01 0.6666666666666666 1.0 Ensemble1 0.5 0.75 0.75 @@ -8,5 +10,3 @@ Ensemble2 0.01 0.6666666666666666 1.0 Ensemble2 0.5 0.75 0.75 Ensemble2 0.66 0.5 0.25 Ensemble3 0.0 0.15384615384615385 1.0 0.15384615384615385 0.15384615384615385 -input_nodes_baseline 0.0 0.15384615384615385 1.0 -input_nodes_baseline 1.0 1.0 0.25 diff --git a/test/evaluate/test_evaluate.py b/test/evaluate/test_evaluate.py index 1178f427a..3eb0ca03e 100644 --- a/test/evaluate/test_evaluate.py +++ b/test/evaluate/test_evaluate.py @@ -127,7 +127,8 @@ def test_node_ensemble(self): out_path_file.unlink(missing_ok=True) ensemble_network = [INPUT_DIR + 'ensemble-network.tsv'] input_data = OUT_DIR + 'data.pickle' - node_ensemble_dict = Evaluation.edge_frequency_node_ensemble(GS_NODE_TABLE, ensemble_network, input_data) + input_interactome = Evaluation.from_file(input_data).get_interactome() + node_ensemble_dict = Evaluation.edge_frequency_node_ensemble(GS_NODE_TABLE, ensemble_network, input_interactome) node_ensemble_dict['ensemble'].to_csv(out_path_file, sep='\t', index=False) assert filecmp.cmp(out_path_file, EXPECT_DIR + 'expected-node-ensemble.csv', shallow=False) @@ -136,8 +137,8 @@ def test_empty_node_ensemble(self): out_path_file.unlink(missing_ok=True) empty_ensemble_network = [INPUT_DIR + 'empty-ensemble-network.tsv'] input_data = OUT_DIR + 'data.pickle' - node_ensemble_dict = Evaluation.edge_frequency_node_ensemble(GS_NODE_TABLE, empty_ensemble_network, - input_data) + input_interactome = Evaluation.from_file(input_data).get_interactome() + node_ensemble_dict = Evaluation.edge_frequency_node_ensemble(GS_NODE_TABLE, empty_ensemble_network, input_interactome) node_ensemble_dict['empty'].to_csv(out_path_file, sep='\t', index=False) assert filecmp.cmp(out_path_file, EXPECT_DIR + 'expected-empty-node-ensemble.csv', shallow=False) @@ -148,7 +149,8 @@ def test_multiple_node_ensemble(self): out_path_empty_file.unlink(missing_ok=True) ensemble_networks = [INPUT_DIR + 'ensemble-network.tsv', INPUT_DIR + 'empty-ensemble-network.tsv'] input_data = OUT_DIR + 'data.pickle' - node_ensemble_dict = Evaluation.edge_frequency_node_ensemble(GS_NODE_TABLE, ensemble_networks, input_data) + input_interactome = Evaluation.from_file(input_data).get_interactome() + node_ensemble_dict = Evaluation.edge_frequency_node_ensemble(GS_NODE_TABLE, ensemble_networks, input_interactome) node_ensemble_dict['ensemble'].to_csv(out_path_file, sep='\t', index=False) assert filecmp.cmp(out_path_file, EXPECT_DIR + 'expected-node-ensemble.csv', shallow=False) node_ensemble_dict['empty'].to_csv(out_path_empty_file, sep='\t', index=False) @@ -160,9 +162,10 @@ def test_precision_recall_curve_ensemble_nodes(self): out_path_file = Path(OUT_DIR + 'pr-curve-ensemble-nodes.txt') out_path_file.unlink(missing_ok=True) input_data = OUT_DIR + 'data.pickle' + input_nodes = Evaluation.from_file(input_data).get_node_columns(["sources", "targets", "prize", "active"]) ensemble_file = pd.read_csv(INPUT_DIR + 'node-ensemble.csv', sep='\t', header=0) node_ensembles_dict = {'ensemble': ensemble_file} - Evaluation.precision_recall_curve_node_ensemble(node_ensembles_dict, GS_NODE_TABLE, input_data, out_path_png, + Evaluation.precision_recall_curve_node_ensemble(node_ensembles_dict, GS_NODE_TABLE, input_nodes, out_path_png, out_path_file) assert out_path_png.exists() assert filecmp.cmp(out_path_file, EXPECT_DIR + 'expected-pr-curve-ensemble-nodes.txt', shallow=False) @@ -173,9 +176,10 @@ def test_precision_recall_curve_ensemble_nodes_empty(self): out_path_file = Path(OUT_DIR + 'pr-curve-ensemble-nodes-empty.txt') out_path_file.unlink(missing_ok=True) input_data = OUT_DIR + 'data.pickle' + input_nodes = Evaluation.from_file(input_data).get_node_columns(["sources", "targets", "prize", "active"]) empty_ensemble_file = pd.read_csv(INPUT_DIR + 'node-ensemble-empty.csv', sep='\t', header=0) node_ensembles_dict = {'ensemble': empty_ensemble_file} - Evaluation.precision_recall_curve_node_ensemble(node_ensembles_dict, GS_NODE_TABLE, input_data, out_path_png, + Evaluation.precision_recall_curve_node_ensemble(node_ensembles_dict, GS_NODE_TABLE, input_nodes, out_path_png, out_path_file) assert out_path_png.exists() assert filecmp.cmp(out_path_file, EXPECT_DIR + 'expected-pr-curve-ensemble-nodes-empty.txt', shallow=False) @@ -186,10 +190,11 @@ def test_precision_recall_curve_multiple_ensemble_nodes(self): out_path_file = Path(OUT_DIR + 'pr-curve-multiple-ensemble-nodes.txt') out_path_file.unlink(missing_ok=True) input_data = OUT_DIR + 'data.pickle' + input_nodes = Evaluation.from_file(input_data).get_node_columns(["sources", "targets", "prize", "active"]) ensemble_file = pd.read_csv(INPUT_DIR + 'node-ensemble.csv', sep='\t', header=0) empty_ensemble_file = pd.read_csv(INPUT_DIR + 'node-ensemble-empty.csv', sep='\t', header=0) node_ensembles_dict = {'ensemble1': ensemble_file, 'ensemble2': ensemble_file, 'ensemble3': empty_ensemble_file} - Evaluation.precision_recall_curve_node_ensemble(node_ensembles_dict, GS_NODE_TABLE, input_data, out_path_png, + Evaluation.precision_recall_curve_node_ensemble(node_ensembles_dict, GS_NODE_TABLE, input_nodes, out_path_png, out_path_file, True) assert out_path_png.exists() assert filecmp.cmp(out_path_file, EXPECT_DIR + 'expected-pr-curve-multiple-ensemble-nodes.txt', shallow=False) From e98d1e2e932c12b0f2f2757614bb7aa1eef805c9 Mon Sep 17 00:00:00 2001 From: ntalluri Date: Fri, 22 Aug 2025 11:24:34 -0500 Subject: [PATCH 11/15] removed unnecessary code --- spras/evaluation.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/spras/evaluation.py b/spras/evaluation.py index 628c46b33..29749320d 100644 --- a/spras/evaluation.py +++ b/spras/evaluation.py @@ -379,12 +379,7 @@ def precision_recall_curve_node_ensemble(node_ensembles: dict, node_table: pd.Da prc_dfs = [] metric_dfs = [] prc_input_nodes_baseline_df = None - baseline = None - precision_input_nodes = None - recall_input_nodes = None - thresholds_input_nodes = None - for label, node_ensemble in node_ensembles.items(): if not node_ensemble.empty: @@ -405,8 +400,7 @@ def precision_recall_curve_node_ensemble(node_ensembles: dict, node_table: pd.Da y_scores_input_nodes = input_nodes_ensemble_df['Frequency'].tolist() precision_input_nodes, recall_input_nodes, thresholds_input_nodes = precision_recall_curve(y_true, y_scores_input_nodes) - plt.plot(recall_input_nodes, precision_input_nodes, color='black', marker='o', linestyle='--', - label=f'Input Nodes Baseline') + plt.plot(recall_input_nodes, precision_input_nodes, color='black', marker='o', linestyle='--', label=f'Input Nodes Baseline') # Dropping last elements because scikit-learn adds (1, 0) to precision/recall for plotting, not tied to real thresholds prc_input_nodes_baseline_data = { From 2004c9a61bcf9e6b8c7bc5e32b8c3358ebcc0674 Mon Sep 17 00:00:00 2001 From: ntalluri Date: Mon, 25 Aug 2025 10:59:57 -0500 Subject: [PATCH 12/15] update code to use helper function for input nodes --- Snakefile | 10 ++++++---- spras/dataset.py | 9 +++++++++ spras/evaluation.py | 1 + 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/Snakefile b/Snakefile index 068b3fcb6..7054ab6f8 100644 --- a/Snakefile +++ b/Snakefile @@ -523,10 +523,11 @@ rule evaluation_ensemble_pr_curve: pr_curve_png = SEP.join([out_dir, '{dataset_gold_standard_pairs}-eval', 'pr-curve-ensemble-nodes.png']), pr_curve_file = SEP.join([out_dir, '{dataset_gold_standard_pairs}-eval', 'pr-curve-ensemble-nodes.txt']), run: + input_dataset = Evaluation.from_file(input.dataset_file) + input_interactome = input_dataset.get_interactome() + input_nodes = input_dataset.get_interesting_input_nodes() node_table = Evaluation.from_file(input.gold_standard_file).node_table - input_interactome = Evaluation.from_file(input.dataset_file).get_interactome() node_ensemble_dict = Evaluation.edge_frequency_node_ensemble(node_table, input.ensemble_file, input_interactome) - input_nodes = Evaluation.from_file(input.dataset_file).get_node_columns(["sources", "targets", "prize", "active"]) Evaluation.precision_recall_curve_node_ensemble(node_ensemble_dict, node_table, input_nodes,output.pr_curve_png, output.pr_curve_file) # Returns list of algorithm specific ensemble files per dataset @@ -544,10 +545,11 @@ rule evaluation_per_algo_ensemble_pr_curve: pr_curve_png = SEP.join([out_dir, '{dataset_gold_standard_pairs}-eval', 'pr-curve-ensemble-nodes-per-algorithm.png']), pr_curve_file = SEP.join([out_dir, '{dataset_gold_standard_pairs}-eval', 'pr-curve-ensemble-nodes-per-algorithm.txt']), run: + input_dataset = Evaluation.from_file(input.dataset_file) + input_interactome = input_dataset.get_interactome() + input_nodes = input_dataset.get_interesting_input_nodes() node_table = Evaluation.from_file(input.gold_standard_file).node_table - input_interactome = Evaluation.from_file(input.dataset_file).get_interactome() node_ensembles_dict = Evaluation.edge_frequency_node_ensemble(node_table, input.ensemble_files, input_interactome) - input_nodes = Evaluation.from_file(input.dataset_file).get_node_columns(["sources", "targets", "prize", "active"]) Evaluation.precision_recall_curve_node_ensemble(node_ensembles_dict, node_table, input_nodes,output.pr_curve_png, output.pr_curve_file, include_aggregate_algo_eval) diff --git a/spras/dataset.py b/spras/dataset.py index 1346750e3..8b501733e 100644 --- a/spras/dataset.py +++ b/spras/dataset.py @@ -171,6 +171,15 @@ def contains_node_columns(self, col_names: list[str] | str): return False return True + def get_interesting_input_nodes(self) -> pd.DataFrame: + """ + Returns: a table listing the input nodes considered as starting points for pathway reconstruction algorithms, + restricted to nodes that have at least one of the specified attributes. + """ + interesting_input_node_columns = ["sources", "targets", "prize", "active"] + interesting_input_nodes = Dataset.get_node_columns(self, col_names = interesting_input_node_columns) + return interesting_input_nodes + def get_other_files(self): return self.other_files.copy() diff --git a/spras/evaluation.py b/spras/evaluation.py index 29749320d..af6ae965a 100644 --- a/spras/evaluation.py +++ b/spras/evaluation.py @@ -393,6 +393,7 @@ def precision_recall_curve_node_ensemble(node_ensembles: dict, node_table: pd.Da input_nodes_gold_intersection = input_nodes_set & gold_standard_nodes # TODO should this be all inputs nodes or the intersection with the gold standard for this baseline? I think it should be the intersection input_nodes_ensemble_df = node_ensemble.copy() + # TODO: make a comment on what this is doing for future use input_nodes_ensemble_df["Frequency"] = ( input_nodes_ensemble_df["Node"].isin(input_nodes_gold_intersection).astype(float) ) From 82fbfce8562cd08ddd4f217b9ac3fce77ec87fb3 Mon Sep 17 00:00:00 2001 From: ntalluri Date: Mon, 25 Aug 2025 11:04:51 -0500 Subject: [PATCH 13/15] update test case to use new helper function --- test/evaluate/test_evaluate.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/evaluate/test_evaluate.py b/test/evaluate/test_evaluate.py index 3eb0ca03e..422660bd7 100644 --- a/test/evaluate/test_evaluate.py +++ b/test/evaluate/test_evaluate.py @@ -162,7 +162,7 @@ def test_precision_recall_curve_ensemble_nodes(self): out_path_file = Path(OUT_DIR + 'pr-curve-ensemble-nodes.txt') out_path_file.unlink(missing_ok=True) input_data = OUT_DIR + 'data.pickle' - input_nodes = Evaluation.from_file(input_data).get_node_columns(["sources", "targets", "prize", "active"]) + input_nodes = Evaluation.from_file(input_data).get_interesting_input_nodes() ensemble_file = pd.read_csv(INPUT_DIR + 'node-ensemble.csv', sep='\t', header=0) node_ensembles_dict = {'ensemble': ensemble_file} Evaluation.precision_recall_curve_node_ensemble(node_ensembles_dict, GS_NODE_TABLE, input_nodes, out_path_png, @@ -176,7 +176,7 @@ def test_precision_recall_curve_ensemble_nodes_empty(self): out_path_file = Path(OUT_DIR + 'pr-curve-ensemble-nodes-empty.txt') out_path_file.unlink(missing_ok=True) input_data = OUT_DIR + 'data.pickle' - input_nodes = Evaluation.from_file(input_data).get_node_columns(["sources", "targets", "prize", "active"]) + input_nodes = Evaluation.from_file(input_data).get_interesting_input_nodes() empty_ensemble_file = pd.read_csv(INPUT_DIR + 'node-ensemble-empty.csv', sep='\t', header=0) node_ensembles_dict = {'ensemble': empty_ensemble_file} Evaluation.precision_recall_curve_node_ensemble(node_ensembles_dict, GS_NODE_TABLE, input_nodes, out_path_png, @@ -190,7 +190,7 @@ def test_precision_recall_curve_multiple_ensemble_nodes(self): out_path_file = Path(OUT_DIR + 'pr-curve-multiple-ensemble-nodes.txt') out_path_file.unlink(missing_ok=True) input_data = OUT_DIR + 'data.pickle' - input_nodes = Evaluation.from_file(input_data).get_node_columns(["sources", "targets", "prize", "active"]) + input_nodes = Evaluation.from_file(input_data).get_interesting_input_nodes() ensemble_file = pd.read_csv(INPUT_DIR + 'node-ensemble.csv', sep='\t', header=0) empty_ensemble_file = pd.read_csv(INPUT_DIR + 'node-ensemble-empty.csv', sep='\t', header=0) node_ensembles_dict = {'ensemble1': ensemble_file, 'ensemble2': ensemble_file, 'ensemble3': empty_ensemble_file} From dcab7810586482c6861a770521cb641783cf0159 Mon Sep 17 00:00:00 2001 From: ntalluri Date: Mon, 25 Aug 2025 11:06:37 -0500 Subject: [PATCH 14/15] update comment --- spras/evaluation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spras/evaluation.py b/spras/evaluation.py index af6ae965a..313727fb8 100644 --- a/spras/evaluation.py +++ b/spras/evaluation.py @@ -393,7 +393,7 @@ def precision_recall_curve_node_ensemble(node_ensembles: dict, node_table: pd.Da input_nodes_gold_intersection = input_nodes_set & gold_standard_nodes # TODO should this be all inputs nodes or the intersection with the gold standard for this baseline? I think it should be the intersection input_nodes_ensemble_df = node_ensemble.copy() - # TODO: make a comment on what this is doing for future use + # set 'Frequency' to 1.0 if the input node is also in the gold standard intersection, else set to 0.0 input_nodes_ensemble_df["Frequency"] = ( input_nodes_ensemble_df["Node"].isin(input_nodes_gold_intersection).astype(float) ) From a57ef19e1be459dee4fd5f42e929a739d1cd42c5 Mon Sep 17 00:00:00 2001 From: Neha Talluri <78840540+ntalluri@users.noreply.github.com> Date: Tue, 26 Aug 2025 08:52:07 -0700 Subject: [PATCH 15/15] fix space formatting --- Snakefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Snakefile b/Snakefile index 7054ab6f8..f009821c1 100644 --- a/Snakefile +++ b/Snakefile @@ -528,7 +528,7 @@ rule evaluation_ensemble_pr_curve: input_nodes = input_dataset.get_interesting_input_nodes() node_table = Evaluation.from_file(input.gold_standard_file).node_table node_ensemble_dict = Evaluation.edge_frequency_node_ensemble(node_table, input.ensemble_file, input_interactome) - Evaluation.precision_recall_curve_node_ensemble(node_ensemble_dict, node_table, input_nodes,output.pr_curve_png, output.pr_curve_file) + Evaluation.precision_recall_curve_node_ensemble(node_ensemble_dict, node_table, input_nodes, output.pr_curve_png, output.pr_curve_file) # Returns list of algorithm specific ensemble files per dataset def collect_ensemble_per_algo_per_dataset(wildcards): @@ -550,7 +550,7 @@ rule evaluation_per_algo_ensemble_pr_curve: input_nodes = input_dataset.get_interesting_input_nodes() node_table = Evaluation.from_file(input.gold_standard_file).node_table node_ensembles_dict = Evaluation.edge_frequency_node_ensemble(node_table, input.ensemble_files, input_interactome) - Evaluation.precision_recall_curve_node_ensemble(node_ensembles_dict, node_table, input_nodes,output.pr_curve_png, output.pr_curve_file, include_aggregate_algo_eval) + Evaluation.precision_recall_curve_node_ensemble(node_ensembles_dict, node_table, input_nodes, output.pr_curve_png, output.pr_curve_file, include_aggregate_algo_eval) # Remove the output directory