Merge pull request #40 from softpudding/codex/top-layer-visibility-guard

Codex/top layer visibility guard

Author: softpudding

eval/dataset/dataflow.yaml

@@ -4,7 +4,7 @@ difficulty: medium
 description: "Test visual understanding capabilities through realistic dashboard tasks"
 start_url: "http://localhost:16605/dataflow/"
 instruction: "Please complete the following tasks: 1) Enable the weekly reports feature in settings. 2) View the detailed report for the quarter with the highest revenue. 3) Navigate to the Revenue tab."
-time_limit: 300.0
+time_limit: 600.0
 cost_limit: 0.5
 
 criteria:
@@ -34,4 +34,4 @@ criteria:
     expected:
       event_type: tab_click
       tab: "revenue"
-      page: "/dataflow/"
+      page: "/dataflow/"

eval/evaluate_browser_agent.py

@@ -215,7 +215,9 @@ def create_conversation(
             base_url: Optional base URL override
             model_alias: Optional configured model alias
         """
-        if self.chrome_uuid and not self.wait_for_browser_validity(timeout_seconds=30.0):
+        if self.chrome_uuid and not self.wait_for_browser_validity(
+            timeout_seconds=30.0
+        ):
             return None
 
         request_json: Dict[str, Any] = {}
@@ -2031,9 +2033,11 @@ def _generate_json_report(
                                 result.get("composite_score") or 0, 4
                             ),
                             "total_score": round(
-                                result.get("total_score")
-                                if result.get("total_score") is not None
-                                else result["task_score"],
+                                (
+                                    result.get("total_score")
+                                    if result.get("total_score") is not None
+                                    else result["task_score"]
+                                ),
                                 2,
                             ),
                             "duration": round(result.get("duration") or 0, 2),
@@ -2241,9 +2245,7 @@ def main():
                 passed_float = 1.0 if result.passed else 0.0
                 eff_score = result.efficiency_score or 0.0
                 usage_score_val = result.usage_score or 0.0
-                test_composite = (
-                    passed_float * 3 + eff_score + usage_score_val
-                ) / 5.0
+                test_composite = (passed_float * 3 + eff_score + usage_score_val) / 5.0
                 print(f"  Composite score: {test_composite:.2f}/1.0")
                 print(f"  Total score: {result.total_score or result.score:.1f}")
                 print(
@@ -2282,9 +2284,7 @@ def main():
                     )
                     print(f"  Status: {'PASS' if result.passed else 'FAIL'}")
                     print(f"  Task score: {result.score:.1f}/{result.max_score:.1f}")
-                    print(
-                        f"  Efficiency score: {result.efficiency_score or 0:.2f}/1.0"
-                    )
+                    print(f"  Efficiency score: {result.efficiency_score or 0:.2f}/1.0")
                     print(f"  Usage score: {result.usage_score or 0:.2f}/1.0")
                     # Calculate composite score for this test
                     passed_float = 1.0 if result.passed else 0.0