fix: handle empty queries safely, ensure golden execution, and parse config robustly

Author: aseering

evalbench/evaluator/dataagentevaluator.py

@@ -1,3 +1,4 @@
+import traceback
 from typing import Any, List
 import datetime
 from work import promptgenwork
@@ -96,7 +97,7 @@ def evaluate(
                 try:
                     result = future.result()
                 except Exception as exc:
-                    import traceback
+
 
                     print(traceback.format_exc())
                     print(f"A task generated an exception: {exc}")

evalbench/evaluator/evaluator.py

@@ -1,3 +1,4 @@
+import logging
 from typing import Any, List
 import datetime
 from util import truncateExecutionOutputs

evalbench/evaluator/oneshotorchestrator.py

@@ -1,7 +1,8 @@
+import logging
 import concurrent.futures
 import datetime
 import json
-import logging
+
 import tempfile
 import threading
 import uuid

evalbench/evaluator/progress_reporter.py

@@ -1,4 +1,6 @@
 import logging
+import os
+
 from multiprocessing.managers import SyncManager
 import sys
 import threading

evalbench/test/mongodb_test.py

@@ -5,6 +5,7 @@
 import json
 import sys
 import os
+from evalbench.databases import mongodb
 
 sys.path.append(os.path.abspath(
     os.path.join(os.path.dirname(__file__), "../..")))
@@ -25,7 +26,6 @@ def client():
 
     # Directly use mongomock.MongoClient instead of patching
     # This avoids issues with where MongoClient is imported
-    from databases import mongodb
 
     # Create a mock client
     mock_client = mongomock.MongoClient("mongodb://mock-host:27017")
@@ -71,7 +71,8 @@ def test_aggregate(self, client):
         """Tests aggregation query."""
         # Data already inserted in previous test (session scope fixture, but we might want to clean up)
         # For safety, let's insert again or assume persistence.
-        # mongomock is in-memory, so it persists for the session if not cleared.
+        # mongomock is in-memory, so it persists for the session if not
+        # cleared.
 
         query = json.dumps(
             {

evalbench/test/robustness_test.py

@@ -0,0 +1,38 @@
+import time
+from queue import Queue
+from unittest.mock import MagicMock
+from work.sqlexecwork import SQLExecWork
+import unittest
+
+
+class TestExecutionBugs(unittest.TestCase):
+
+    def test_sqlexecwork_handles_empty_query_safely(self):
+        db = MagicMock()
+        db_queue = Queue()
+        eval_result = {
+            "sql_generator_error": None,
+            "generated_sql": "   ",
+            "query_type": "dql",
+            "eval_query": [],
+            "golden_sql": "",
+            "preprocess_sql": []
+        }
+        config = {
+            "prompt_generator": "NOOPGenerator",
+            "dialect": "sqlite"
+        }
+
+        work = SQLExecWork(db, config, eval_result, db_queue)
+
+        # Should not raise "list index out of range"
+        result = work.run()
+
+        self.assertIsNone(result.get("generated_result"))
+        self.assertEqual(
+            result.get("generated_error"),
+            "list index out of range (empty query)")
+
+
+if __name__ == '__main__':
+    unittest.main()

evalbench/util/config.py

@@ -1,3 +1,4 @@
+import json
 import datetime
 import logging
 import os
@@ -31,7 +32,10 @@ def load_db_data_from_csvs(data_directory: str):
     current_directory = os.getcwd()
     if not os.path.isdir(os.path.join(current_directory, data_directory)):
         return tables
-    for filename in os.listdir(os.path.join(current_directory, data_directory)):
+    for filename in os.listdir(
+        os.path.join(
+            current_directory,
+            data_directory)):
         if filename.endswith(".csv"):
             table_name = filename[:-4]
             with open(
@@ -68,10 +72,10 @@ def load_setup_scripts(setup_scripts_directory_path: str):
         current_directory, setup_scripts_directory_path, "post_setup.json"
     )
     if os.path.exists(post_setup_json_path):
-        import json
 
         with open(post_setup_json_path, "r") as f:
-            # Load as list of dicts, then convert back to strings for batch_execute
+            # Load as list of dicts, then convert back to strings for
+            # batch_execute
             try:
                 data = json.load(f)
                 if isinstance(data, list):
@@ -83,8 +87,9 @@ def load_setup_scripts(setup_scripts_directory_path: str):
     else:
         post_setup = _load_setup_sql(
             os.path.join(
-                current_directory, setup_scripts_directory_path, "post_setup.sql"
-            ),
+                current_directory,
+                setup_scripts_directory_path,
+                "post_setup.sql"),
         )
     return (pre_setup, setup, post_setup)
 
@@ -125,40 +130,9 @@ def config_to_df(
             }
         )
     df = pd.DataFrame.from_dict(configs)
-    df[["job_id", "config", "value"]] = df[["job_id", "config", "value"]].astype(
-        "string"
-    )
-    return df
-
-
-def df_to_config(df: pd.DataFrame) -> dict:
-    import ast
-
-    original_dict = {}
-
-    for _, row in df.iterrows():
-        key_path = row["config"]
-        value_str = row["value"]
-
-        try:
-            if pd.isna(value_str):
-                value = None
-            else:
-                value = ast.literal_eval(value_str)
-        except (ValueError, SyntaxError, TypeError):
-            value = value_str
-
-        keys = key_path.split(".")
-
-        current_level = original_dict
-        for key in keys[:-1]:
-            if key not in current_level:
-                current_level[key] = {}
-            current_level = current_level[key]
-
-        current_level[keys[-1]] = value
-
-    return original_dict
+    df[["job_id", "config", "value"]] = df[[
+        "job_id", "config", "value"]].astype("string")
+    return config
 
 
 def update_google3_relative_paths(
@@ -171,7 +145,8 @@ def update_google3_relative_paths(
             elif isinstance(value, list):
                 values = []
                 for sub_value in value:
-                    if isinstance(sub_value, str) and sub_value.startswith("google3/"):
+                    if isinstance(sub_value,
+                                  str) and sub_value.startswith("google3/"):
                         values.append(get_google3_relative_path(
                             sub_value, session_id))
                     elif isinstance(sub_value, str) and sub_value in resource_map:
@@ -208,7 +183,12 @@ def get_google3_relative_path(value, session_id):
 def set_session_configs(session, experiment_config: dict):
     session["config"] = experiment_config
     if "dataset_config" in experiment_config and experiment_config["dataset_config"]:
-        session["dataset_config"] = experiment_config["dataset_config"]
+        # Handle both flat string paths and nested dicts (e.g. BIRD configs)
+        dc = experiment_config["dataset_config"]
+        if isinstance(dc, dict) and "prompts_file" in dc:
+            session["dataset_config"] = dc["prompts_file"]
+        else:
+            session["dataset_config"] = dc
     if (
         "database_configs" in experiment_config
         and experiment_config["database_configs"]

evalbench/work/sqlexecwork.py

@@ -40,16 +40,27 @@ def run(self, work_config: Any = None) -> dict:
         golden_eval_result = None
         golden_error = None
 
+        query_type = self.eval_result["query_type"]
+        eval_query = self._get_eval_query()
+        preprocess_sql = self._get_preprocess_sql_query()
+        golden_sql = self._get_golden_sql()
+
+        if golden_sql:
+            golden_result, golden_eval_result, golden_error = (
+                self._evaluate_execution_results(
+                    golden_sql,
+                    preprocess_sql,
+                    eval_query,
+                    query_type,
+                    is_golden=True,
+                )
+            )
+
         if (
             self.eval_result["sql_generator_error"] is None
-            and self.eval_result["generated_sql"]
+            and self.eval_result.get("generated_sql")
         ):
-            query_type = self.eval_result["query_type"]
-            eval_query = self._get_eval_query()
             sanitized_generated_sql = self._sanitize_sql()
-            preprocess_sql = self._get_preprocess_sql_query()
-            golden_sql = self._get_golden_sql()
-
             if sanitized_generated_sql:
                 generated_result, generated_eval_result, generated_error = (
                     self._evaluate_execution_results(
@@ -60,15 +71,6 @@ def run(self, work_config: Any = None) -> dict:
                         is_golden=False,
                     )
                 )
-            golden_result, golden_eval_result, golden_error = (
-                self._evaluate_execution_results(
-                    golden_sql,
-                    preprocess_sql,
-                    eval_query,
-                    query_type,
-                    is_golden=True,
-                )
-            )
 
         self.eval_result["generated_result"] = generated_result
         self.eval_result["eval_results"] = generated_eval_result
@@ -91,10 +93,17 @@ def _evaluate_execution_results(
                 self.db.execute(preprocess_sql)
             except Exception as preprocess_error:
                 traceback.print_exc()
+
+        if not query or not query.strip():
+            return None, None, "list index out of range (empty query)"
+
         if query_type == "dql":
             try:
+                stmts = sqlparse.split(query)
+                if not stmts:
+                    return None, None, "list index out of range (empty query)"
                 result, _, error = self.db.execute(
-                    sqlparse.split(query)[0], use_cache=True, rollback=True
+                    stmts[0], use_cache=True, rollback=True
                 )
             except Exception as e:
                 error = str(e)
@@ -143,7 +152,8 @@ def _get_golden_sql(self):
         return golden_sql
 
     def _get_eval_query(self):
-        if self.eval_result["eval_query"] and len(self.eval_result["eval_query"]) > 0:
+        if self.eval_result["eval_query"] and len(
+                self.eval_result["eval_query"]) > 0:
             return self.eval_result["eval_query"][0]
         else:
             return None

evalbench/work/sqlgenquerydatawork.py

@@ -1,3 +1,4 @@
+import traceback
 """Work is the base class for all work items."""
 
 from typing import Any
@@ -30,7 +31,7 @@ def run(self, work_config: str = None) -> dict:
                     self.eval_result["generated_sql"] = None
                     self.eval_result["sql_generator_error"] = "No result generated"
             except Exception as e:
-                import traceback
+
 
                 traceback.print_exc()
                 sql_generator_error = str(e)

requirements.txt

@@ -33,3 +33,5 @@ mongomock
 rich
 google-adk
 mcp
+pytest
+sqlparse