Align default argument values with evaluation repository #377

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open

simonrosenberg wants to merge 33 commits into main from align-default-values-with-evaluation

benchmarks/commit0/build_images.py

-Original file line number
+Diff line change
@@ Expand Up / @@ -13,6 +13,7 @@ @@
     from commit0.harness.constants import SPLIT
+    from benchmarks.commit0.config import BUILD_DEFAULTS, INFER_DEFAULTS
     from benchmarks.utils.build_utils import (
         build_all_images,
         default_build_output_dir,
@@ Expand Down Expand Up / @@ -90,7 +91,6 @@ def main(argv: list[str]) -> int: @@
         parser.add_argument(
             "--repo-split",
             type=str,
-            default="lite",
             help="Commit0 repo split (lite, all, or repo name)",
         )
         parser.add_argument(
@@ Expand All / @@ -99,7 +99,12 @@ def main(argv: list[str]) -> int: @@
             default="",
             help="Override base image prefix (default: env EVAL_DOCKER_IMAGE_PREFIX)",
         )
-        parser.set_defaults(dataset="wentingzhao/commit0_combined")
+        parser.set_defaults(
+            dataset=INFER_DEFAULTS["dataset"],
+            split=INFER_DEFAULTS["split"],
+            repo_split=INFER_DEFAULTS["repo_split"],
+            **BUILD_DEFAULTS,
+        )
         args = parser.parse_args(argv)
         docker_image_prefix = args.docker_image_prefix or None
@@ Expand Down @@

benchmarks/commit0/config.py

-Original file line number
+Diff line change
@@ -0,0 +1,21 @@
+    """
+    Commit0 benchmark configuration.
+    Default values aligned with evaluation repository (OpenHands/evaluation).
+    """
+    # Inference defaults (used by run_infer.py)
+    # Note: commit0 uses max_attempts=1 and max_retries=1 (different from default of 3)
+    INFER_DEFAULTS = {
+        "dataset": "wentingzhao/commit0_combined",
+        "split": "test",
+        "repo_split": "lite",
+        "num_workers": 16,
+        "max_attempts": 1,
+        "max_retries": 3,
+    }
+    # Build defaults (used by build_images.py)
+    BUILD_DEFAULTS = {
+        "max_workers": 16,
+    }

benchmarks/commit0/run_infer.py

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -12,6 +12,7 @@
  
        extract_custom_tag,

        get_base_docker_image,

    )

    from benchmarks.commit0.config import INFER_DEFAULTS

    from benchmarks.utils.args_parser import get_parser

    from benchmarks.utils.constants import EVAL_AGENT_SERVER_IMAGE

    from benchmarks.utils.conversation import build_event_persistence_callback

    @@ -110,29 +111,29 @@ def __init__(
  
            self,

            metadata: EvalMetadata,

            num_workers: int = 1,

            repo_split: str = "lite",

            dataset_name: str = "wentingzhao/commit0_combined",

            dataset_split: str = "test",

            repo_split: str | None = None,

            dataset_name: str | None = None,

            dataset_split: str | None = None,

        ):

            super().__init__(metadata=metadata, num_workers=num_workers)

            # Store additional parameters in metadata.details for access in methods

            if not hasattr(metadata, "details") or metadata.details is None:

                metadata.details = {}

            metadata.details.update(

                {

                    "repo_split": repo_split,

                    "dataset_name": dataset_name,

                    "dataset_split": dataset_split,

                    "repo_split": repo_split or INFER_DEFAULTS["repo_split"],

                    "dataset_name": dataset_name or INFER_DEFAULTS["dataset"],

                    "dataset_split": dataset_split or INFER_DEFAULTS["split"],

                }

            )

        def prepare_instances(self) -> List[EvalInstance]:

            logger.info("Setting up Commit0 evaluation data")

            details = self.metadata.details or {}

            dataset_name = details.get("dataset_name", "wentingzhao/commit0_combined")

            dataset_split = details.get("dataset_split", "test")

            repo_split = details.get("repo_split", "lite")

            dataset_name = details.get("dataset_name", INFER_DEFAULTS["dataset"])

            dataset_split = details.get("dataset_split", INFER_DEFAULTS["split"])

            repo_split = details.get("repo_split", INFER_DEFAULTS["repo_split"])

            dataset = load_dataset(dataset_name, split=dataset_split)

            df = commit0_setup(dataset, repo_split)

    @@ -593,11 +594,10 @@ def main() -> None:
  
        parser.add_argument(

            "--repo-split",

            type=str,

            default="lite",

            help="all, lite, or each repo name",

        )

        # Override the default dataset for commit0

        parser.set_defaults(dataset="wentingzhao/commit0_combined")

        # Apply INFER_DEFAULTS from config (matches evaluation repository values.yaml)

        parser.set_defaults(**INFER_DEFAULTS)

        args = parser.parse_args()

        # Validate max_attempts

benchmarks/gaia/config.py

-Original file line number
+Diff line change
@@ -0,0 +1,18 @@
+    """
+    GAIA benchmark configuration.
+    Default values aligned with evaluation repository (OpenHands/evaluation).
+    """
+    # Inference defaults (used by run_infer.py)
+    INFER_DEFAULTS = {
+        "dataset": "gaia-benchmark/GAIA",
+        "split": "validation",
+        "level": "2023_all",
+        "num_workers": 30,
+    }
+    # Build defaults (used by build_images.py)
+    BUILD_DEFAULTS = {
+        "max_workers": 1,
+    }

benchmarks/gaia/run_infer.py

-Original file line number
+Diff line change
@@ Expand Up / @@ -11,6 +11,7 @@ @@
     from datasets import DatasetDict, load_dataset
     from PIL import Image
+    from benchmarks.gaia.config import INFER_DEFAULTS
     from benchmarks.gaia.scorer import question_scorer
     from benchmarks.gaia.utils import image_to_jpg_base64_url, image_to_png_base64_url
     from benchmarks.utils.args_parser import get_parser
@@ Expand Down Expand Up / @@ -548,9 +549,9 @@ def main() -> None: @@
         parser.add_argument(
             "--level",
             type=str,
-            required=True,
-            help="GAIA level to evaluate (e.g., 2023_level1, 2023_level2, 2023_level3)",
+            help="GAIA level to evaluate (e.g., 2023_level1, 2023_level2, 2023_level3, 2023_all)",
         )
+        parser.set_defaults(**INFER_DEFAULTS)
         args = parser.parse_args()
         # Create critic instance from parsed arguments
@@ Expand Down Expand Up / @@ -585,7 +586,7 @@ def main() -> None: @@
         # Create metadata
         metadata = EvalMetadata(
             llm=llm,
-            dataset="gaia-benchmark/GAIA",
+            dataset=args.dataset,
             dataset_split=args.split,
             max_iterations=args.max_iterations,
             eval_output_dir=structured_output_dir,
@@ Expand Down @@

benchmarks/swebench/build_images.py

-Original file line number
+Diff line change
@@ Expand Up / @@ -13,6 +13,7 @@ @@
     from pathlib import Path
     from benchmarks.swebench import constants
+    from benchmarks.swebench.config import BUILD_DEFAULTS
     from benchmarks.utils.build_utils import (
         BuildOutput,
         build_all_images,
@@ Expand Down Expand Up @@
     def main(argv: list[str]) -> int:
         parser = get_build_parser()
+        parser.set_defaults(**BUILD_DEFAULTS)
         args = parser.parse_args(argv)
         base_images: list[str] = collect_unique_base_images(
@@ Expand Down @@

benchmarks/swebench/config.py

-Original file line number
+Diff line change
@@ -0,0 +1,23 @@
+    """
+    SWE-bench benchmark configuration.
+    Default values aligned with evaluation repository (OpenHands/evaluation).
+    """
+    # Inference defaults (used by run_infer.py)
+    INFER_DEFAULTS = {
+        "dataset": "princeton-nlp/SWE-bench_Verified",
+        "split": "test",
+        "num_workers": 30,
+    }
+    # Evaluation defaults (used by eval_infer.py)
+    EVAL_DEFAULTS = {
+        "dataset": "princeton-nlp/SWE-bench_Verified",
+        "workers": 12,
+    }
+    # Build defaults (used by build_images.py)
+    BUILD_DEFAULTS = {
+        "max_workers": 32,
+    }

benchmarks/swebench/constants.py

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -1,16 +1,13 @@
  
    """

    SWE-Bench hyperparameters and constant values.

    This module serves as the single source of truth for all constant values

    used in the SWE-Bench evaluation workflow.

    This module provides constant values used in the SWE-Bench evaluation workflow.

    For dataset, model, and worker defaults, see config.py (INFER_DEFAULTS, EVAL_DEFAULTS).

    """

    from typing import Final, Literal

    # Dataset

    DEFAULT_DATASET: Final[str] = "princeton-nlp/SWE-bench_Verified"

    # Docker

    DOCKER_IMAGE_PREFIX: Final[str] = "docker.io/swebench/"

    DOCKER_IMAGE_TAG: Final[str] = "latest"

    @@ -28,9 +25,6 @@
  
    DEFAULT_RUNTIME_API_URL: Final[str] = "https://runtime.eval.all-hands.dev"

    DEFAULT_REMOTE_RUNTIME_STARTUP_TIMEOUT: Final[int] = 600

    # Evaluation

    DEFAULT_EVAL_WORKERS: Final[int] = 12

    # Model - preserving original behavior: function default is "OpenHands", CLI default is "openhands"

    DEFAULT_MODEL_NAME: Final[str] = "OpenHands"

    DEFAULT_CLI_MODEL_NAME: Final[str] = "openhands"

benchmarks/swebench/eval_infer.py

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -17,6 +17,7 @@
  
    from pathlib import Path

    from benchmarks.swebench import constants

    from benchmarks.swebench.config import EVAL_DEFAULTS

    from benchmarks.utils.laminar import LaminarService

    from benchmarks.utils.patch_utils import remove_files_from_patch

    from benchmarks.utils.report_costs import generate_cost_report

    @@ -27,7 +28,9 @@
  
    def convert_to_swebench_format(

        input_file: str, output_file: str, model_name: str = constants.DEFAULT_MODEL_NAME

        input_file: str,

        output_file: str,

        model_name: str = constants.DEFAULT_CLI_MODEL_NAME,

    ) -> None:

        """

        Convert OpenHands output.jsonl to SWE-Bench prediction format.

    @@ -116,8 +119,8 @@ def convert_to_swebench_format(
  
    def run_swebench_evaluation(

        predictions_file: str,

        dataset: str = constants.DEFAULT_DATASET,

        workers: int = constants.DEFAULT_EVAL_WORKERS,

        dataset: str = EVAL_DEFAULTS["dataset"],

        workers: int = EVAL_DEFAULTS["workers"],

    ) -> None:

        """

        Run SWE-Bench evaluation on the predictions file.

    @@ -198,8 +201,7 @@ def main() -> None:
  
        parser.add_argument(

            "--dataset",

            default=constants.DEFAULT_DATASET,

            help=f"SWE-Bench dataset to evaluate against (default: {constants.DEFAULT_DATASET})",

            help="SWE-Bench dataset to evaluate against",

        )

        parser.add_argument(

    @@ -216,17 +218,18 @@ def main() -> None:
  
        parser.add_argument(

            "--model-name",

            default=constants.DEFAULT_CLI_MODEL_NAME,

            help=f"Model name to use in the model_name_or_path field (default: {constants.DEFAULT_CLI_MODEL_NAME})",

            help="Model name to use in the model_name_or_path field",

        )

        parser.add_argument(

            "--workers",

            type=int,

            default=constants.DEFAULT_EVAL_WORKERS,

            help=f"Number of workers to use when evaluating (default: {constants.DEFAULT_EVAL_WORKERS})",

            help="Number of workers to use when evaluating",

        )

        # Apply EVAL_DEFAULTS from config

        parser.set_defaults(**EVAL_DEFAULTS)

        args = parser.parse_args()

        # Validate input file

benchmarks/swebench/run_infer.py

-Original file line number
+Diff line change
@@ Expand Up / @@ -11,6 +11,7 @@ @@
         should_wrap_instance_id,
         wrap_image,
     )
+    from benchmarks.swebench.config import INFER_DEFAULTS
     from benchmarks.utils.args_parser import get_parser
     from benchmarks.utils.build_utils import build_image
     from benchmarks.utils.constants import EVAL_AGENT_SERVER_IMAGE
@@ Expand Down Expand Up / @@ -334,6 +335,7 @@ def main() -> None: @@
             choices=choices,
             help="Path to prompt template file",
         )
+        parser.set_defaults(**INFER_DEFAULTS)
         args = parser.parse_args()
         # Validate max_attempts
@@ Expand Down @@

benchmarks/swebenchmultimodal/build_images.py

-Original file line number
+Diff line change
@@ Expand Up / @@ -10,6 +10,7 @@ @@
     import sys
+    from benchmarks.swebenchmultimodal.config import BUILD_DEFAULTS
     from benchmarks.utils.build_utils import (
         build_all_images,
         default_build_output_dir,
@@ Expand Down Expand Up / @@ -68,6 +69,7 @@ def collect_unique_base_images(dataset, split, n_limit): @@
     def main(argv: list[str]) -> int:
         parser = get_build_parser()
+        parser.set_defaults(**BUILD_DEFAULTS)
         args = parser.parse_args(argv)
         base_images: list[str] = collect_unique_base_images(
@@ Expand Down @@

benchmarks/swebenchmultimodal/config.py

-Original file line number
+Diff line change
@@ -0,0 +1,24 @@
+    """
+    SWE-bench Multimodal benchmark configuration.
+    Default values aligned with evaluation repository (OpenHands/evaluation).
+    """
+    # Inference defaults (used by run_infer.py)
+    INFER_DEFAULTS = {
+        "dataset": "princeton-nlp/SWE-bench_Multimodal",
+        "split": "dev",
+        "num_workers": 30,
+    }
+    # Evaluation defaults (used by eval_infer.py)
+    EVAL_DEFAULTS = {
+        "dataset": "princeton-nlp/SWE-bench_Multimodal",
+        "split": "dev",
+        "workers": 12,
+    }
+    # Build defaults (used by build_images.py)
+    BUILD_DEFAULTS = {
+        "max_workers": 32,
+    }

benchmarks/swebenchmultimodal/eval_infer.py

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -16,6 +16,7 @@
  
    from pathlib import Path

    from typing import Any

    from benchmarks.swebenchmultimodal.config import EVAL_DEFAULTS

    from benchmarks.utils.patch_utils import remove_files_from_patch

    from benchmarks.utils.report_costs import generate_cost_report

    from openhands.sdk import get_logger

    @@ -375,15 +376,12 @@ def main() -> None:
  
        parser.add_argument(

            "--dataset",

            default="princeton-nlp/SWE-bench_Multimodal",

            help="SWE-Bench dataset to evaluate against "

            "(default: princeton-nlp/SWE-bench_Multimodal)",

            help="SWE-Bench dataset to evaluate against",

        )

        parser.add_argument(

            "--split",

            default="dev",

            help="Dataset split to use (default: dev)",

            help="Dataset split to use",

        )

        parser.add_argument(

    @@ -406,10 +404,12 @@ def main() -> None:
  
        parser.add_argument(

            "--workers",

            default="12",

            type=int,

            help="Number of workers to use when evaluating",

        )

        parser.set_defaults(**EVAL_DEFAULTS)

        parser.add_argument(

            "--run-id",

            help="Run ID for the evaluation (default: eval_<output_filename>)",

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Align default argument values with evaluation repository #377

Uh oh!

Diff view

Diff view

There are no files selected for viewing

Uh oh!

Align default argument values with evaluation repository #377

Are you sure you want to change the base?

Uh oh!

Align default argument values with evaluation repository #377

Uh oh!

Uh oh!

Diff view

Diff view

There are no files selected for viewing

Uh oh!