From 05c3ba05a9cccb09f5f5014519f3483136445670 Mon Sep 17 00:00:00 2001 From: arubique Date: Thu, 2 Oct 2025 15:26:32 +0200 Subject: [PATCH] Add cmd_env_var to common prefixes --- stnd/run_from_csv/__main__.py | 5 +++-- stnd/utility/logger.py | 4 +++- tests/run_from_csv/canonical_files/canonical_csv.csv | 8 ++++---- tests/run_from_csv/test_main.py | 3 +++ 4 files changed, 13 insertions(+), 7 deletions(-) diff --git a/stnd/run_from_csv/__main__.py b/stnd/run_from_csv/__main__.py index 6b525da..539f305 100644 --- a/stnd/run_from_csv/__main__.py +++ b/stnd/run_from_csv/__main__.py @@ -50,6 +50,7 @@ CONDOR_BID_KEY, ENV_VAR_PREFIX, CMD_ENV_VAR_PREFIX, + COMMON_PREFIXES, make_logger, make_gdrive_client, sync_local_file_with_gdrive, @@ -160,11 +161,11 @@ def main(): if args.cluster_type == "slurm": assert args.conda_env is not None, "Conda env is required for Slurm" make_final_cmd = make_final_cmd_slurm - allowed_prefixes = (SLURM_PREFIX, DELTA_PREFIX) + allowed_prefixes = COMMON_PREFIXES + [SLURM_PREFIX] elif args.cluster_type == "condor": assert args.conda_env is not None, "Conda env is required for Condor" make_final_cmd = make_final_cmd_condor - allowed_prefixes = (CONDOR_PREFIX, DELTA_PREFIX, ENV_VAR_PREFIX) + allowed_prefixes = COMMON_PREFIXES + [CONDOR_PREFIX, ENV_VAR_PREFIX] else: raise Exception(f"Unknown cluster type: {args.cluster_type}") diff --git a/stnd/utility/logger.py b/stnd/utility/logger.py index 8fce08c..ce5cca7 100644 --- a/stnd/utility/logger.py +++ b/stnd/utility/logger.py @@ -54,7 +54,6 @@ CONDOR_PREFIX = "condor" CONDOR_BID_KEY = "bid" ENV_VAR_PREFIX = "env_var" # used in bash script e.g. for Condor -CMD_ENV_VAR_PREFIX = "cmd_env_var" # used in command line PROGRESS_FREQUENCY = 0.01 @@ -181,6 +180,9 @@ WORKSHEET_SEPARATOR = "::" DELTA_PREFIX = "delta" SLURM_PREFIX = "slurm" +CMD_ENV_VAR_PREFIX = "cmd_env_var" # used in command line +COMMON_PREFIXES = [DELTA_PREFIX, CMD_ENV_VAR_PREFIX] + PREFIX_SEPARATOR = ":" OLD_PREFIX = "Old " PREV_RUN_FOLDER_KEY = "prev_run_folder" diff --git a/tests/run_from_csv/canonical_files/canonical_csv.csv b/tests/run_from_csv/canonical_files/canonical_csv.csv index 142304d..593fd30 100644 --- a/tests/run_from_csv/canonical_files/canonical_csv.csv +++ b/tests/run_from_csv/canonical_files/canonical_csv.csv @@ -1,4 +1,4 @@ -path_to_default_config,path_to_main,whether_to_run,slurm:cpus-per-task,slurm:partition,delta:logging/wandb/netrc_path,delta:logging/use_wandb,delta:initialization_type,delta:image/color,run_folder,status,job id,mean of latest tensor,walltime -./configs/config.yaml,./executables/executable.py,0,4,a100,~/.netrc,FALSE,zeros,red,,Completed,Not found,0.0, -./configs/config.yaml,./executables/executable.py,0,4,a100,~/.netrc,FALSE,random,red,?,?,?,?, -./configs/config.yaml,./executables/executable.py,0,4,a100,~/.netrc,FALSE,ones,red,,Completed,Not found,0.3333333333333333, +path_to_default_config,path_to_main,whether_to_run,slurm:cpus-per-task,slurm:partition,delta:logging/wandb/netrc_path,delta:logging/use_wandb,delta:initialization_type,delta:image/color,cmd_env_var:CUDA_VISIBLE_DEVICES,run_folder,status,job id,mean of latest tensor,walltime +./configs/config.yaml,./executables/executable.py,0,4,a100,~/.netrc,FALSE,zeros,red,0,,Completed,Not found,0.0, +./configs/config.yaml,./executables/executable.py,0,4,a100,~/.netrc,FALSE,random,red,0,?,?,?,?, +./configs/config.yaml,./executables/executable.py,0,4,a100,~/.netrc,FALSE,ones,red,0,,Completed,Not found,0.3333333333333333, diff --git a/tests/run_from_csv/test_main.py b/tests/run_from_csv/test_main.py index ebda55d..0ef1f1b 100644 --- a/tests/run_from_csv/test_main.py +++ b/tests/run_from_csv/test_main.py @@ -58,6 +58,7 @@ def create_test_csv(csv_path, config_path, cluster_type="slurm"): "delta:logging/use_wandb": ["FALSE"] * TOTAL_ROWS, "delta:initialization_type": ["zeros", "random", "ones"], "delta:image/color": ["red"] * TOTAL_ROWS, + "cmd_env_var:CUDA_VISIBLE_DEVICES": ["0"] * TOTAL_ROWS, } elif cluster_type == "runner": data = { @@ -92,6 +93,7 @@ def create_test_csv(csv_path, config_path, cluster_type="slurm"): "env_var:CUDA_VISIBLE_DEVICES": [ "0__COMMA__1__COMMA__2__COMMA__3__COMMA__4__COMMA__5__COMMA__6__COMMA__7" ], + "cmd_env_var:CMD_VAR": ["BAR"], } else: raise ValueError(f"Invalid cluster type: {cluster_type}") @@ -526,6 +528,7 @@ def test_column_placeholder_replacement(): assert csv_row_mixed["col3"] == "This is value1 and 123" +@pytest.mark.skipif(SKIP_TESTS, reason="Skip tests when debugging") def test_process_csv_row_includes_cmd_env_exports(tmp_path, monkeypatch): from stnd.run_from_csv import __main__ as main_mod