diff --git a/evaluation/README.md b/evaluation/README.md index cfaf1ba36c4d..5f6963d93baf 100644 --- a/evaluation/README.md +++ b/evaluation/README.md @@ -60,6 +60,7 @@ The OpenHands evaluation harness supports a wide variety of benchmarks across [s - AiderBench: [`evaluation/benchmarks/aider_bench`](./benchmarks/aider_bench/) - Commit0: [`evaluation/benchmarks/commit0_bench`](./benchmarks/commit0_bench/) - DiscoveryBench: [`evaluation/benchmarks/discoverybench`](./benchmarks/discoverybench/) +- APPS: [`evaluation/benchmarks/apps`](./benchmarks/apps/) ### Web Browsing @@ -76,6 +77,10 @@ The OpenHands evaluation harness supports a wide variety of benchmarks across [s - Entity deduction Arena (EDA): [`evaluation/benchmarks/EDA`](./benchmarks/EDA) - ProofWriter: [`evaluation/benchmarks/logic_reasoning`](./benchmarks/logic_reasoning) - ScienceAgentBench: [`evaluation/benchmarks/scienceagentbench`](./benchmarks/scienceagentbench) +- MATH: [`evaluation/benchmarks/math`](./benchmarks/math/) +- HotpotQA: [`evaluation/benchmarks/hotpotqa`](./benchmarks/hotpotqa/) +- WikiTableQuestion: [`evaluation/benchmarks/wiki_table_question`](./benchmarks/wiki_table_question/) +- AlfWorld: [`evaluation/benchmarks/alfworld`](./benchmarks/alfworld/) ### Real World diff --git a/evaluation/benchmarks/aider_bench/README.md b/evaluation/benchmarks/aider_bench/README.md index 086cfe58160a..a011e6ec9d5c 100644 --- a/evaluation/benchmarks/aider_bench/README.md +++ b/evaluation/benchmarks/aider_bench/README.md @@ -16,7 +16,7 @@ development environment and LLM. ## Start the evaluation ```bash -./evaluation/benchmarks/aider_bench/scripts/run_infer.sh [model_config] [git-version] [agent] [eval_limit] [eval-num-workers] [eval_ids] +./evaluation/benchmarks/aider_bench/scripts/run_infer.sh [model_config] [git-version] [agent] [eval_limit] [eval-num-workers] [eval_ids] [run_evaluation] ``` - `model_config`, e.g. `eval_gpt4_1106_preview`, is the config group name for @@ -31,6 +31,7 @@ development environment and LLM. - `eval-num-workers`: the number of workers to use for evaluation. Default: `1`. - `eval_ids`, e.g. `"1,3,10"`, limits the evaluation to instances with the given IDs (comma separated). +- `run_evaluation`: set to `eval` to automatically run evaluation after the benchmark completes. There are also following optional environment variables you can set: @@ -53,7 +54,11 @@ You can update the arguments in the script - `--eval-ids`: the IDs of the examples to evaluate (comma separated). For example, `"1,3,10"`. ```bash +# Run benchmark without evaluation ./evaluation/benchmarks/aider_bench/scripts/run_infer.sh eval_gpt35_turbo HEAD CodeActAgent 100 1 "1,3,10" + +# Run benchmark with automatic evaluation +./evaluation/benchmarks/aider_bench/scripts/run_infer.sh eval_gpt35_turbo HEAD CodeActAgent 100 1 "1,3,10" eval ``` ### Run Inference on `RemoteRuntime` (experimental) diff --git a/evaluation/benchmarks/aider_bench/run_infer.py b/evaluation/benchmarks/aider_bench/run_infer.py index ae5faadc098b..4247db909b0a 100644 --- a/evaluation/benchmarks/aider_bench/run_infer.py +++ b/evaluation/benchmarks/aider_bench/run_infer.py @@ -21,6 +21,7 @@ prepare_dataset, reset_logger_for_multiprocessing, run_evaluation, + update_llm_config_for_completions_logging, ) from openhands.controller.state.state import State from openhands.core.config import ( @@ -45,6 +46,7 @@ def get_config( + instance: pd.Series, metadata: EvalMetadata, ) -> AppConfig: sandbox_config = get_default_sandbox_config_for_eval() @@ -59,7 +61,13 @@ def get_config( workspace_base=None, workspace_mount_path=None, ) - config.set_llm_config(metadata.llm_config) + # Update llm_config to enable completions logging + llm_config = update_llm_config_for_completions_logging( + metadata.llm_config, + metadata.eval_output_dir, + str(instance.instance_id) + ) + config.set_llm_config(llm_config) agent_config = config.get_agent_config(metadata.agent_class) agent_config.enable_prompt_extensions = False @@ -162,7 +170,7 @@ def process_instance( metadata: EvalMetadata, reset_logger: bool = True, ) -> EvalOutput: - config = get_config(metadata) + config = get_config(instance, metadata) # Setup the logger properly, so you can run multi-processing to parallelize the evaluation if reset_logger: @@ -277,6 +285,15 @@ def process_instance( if llm_config is None: raise ValueError(f'Could not find LLM config: --llm_config {args.llm_config}') + # Create details dictionary with agent configuration + agent_details = { + "agent_config": { + "codeact_enable_jupyter": False, + "codeact_enable_browsing": False, + "codeact_enable_llm_editor": False, + } + } + metadata = make_metadata( llm_config, 'AiderBench', @@ -284,6 +301,7 @@ def process_instance( args.max_iterations, args.eval_note, args.eval_output_dir, + details=agent_details, ) output_file = os.path.join(metadata.eval_output_dir, 'output.jsonl') diff --git a/evaluation/benchmarks/aider_bench/scripts/run_infer.sh b/evaluation/benchmarks/aider_bench/scripts/run_infer.sh index 59d53cfb1980..8160a1ea40f9 100755 --- a/evaluation/benchmarks/aider_bench/scripts/run_infer.sh +++ b/evaluation/benchmarks/aider_bench/scripts/run_infer.sh @@ -9,6 +9,21 @@ AGENT=$3 EVAL_LIMIT=$4 NUM_WORKERS=$5 EVAL_IDS=$6 +RUN_EVALUATION=$7 # New parameter to run evaluation after benchmark + +# Special case: if the 7th parameter is "eval", set RUN_EVALUATION to "eval" +if [ "$RUN_EVALUATION" = "eval" ]; then + echo "Evaluation mode enabled" +fi + +# Special case: if any parameter is "eval", set RUN_EVALUATION to "eval" +for param in "$@"; do + if [ "$param" = "eval" ]; then + RUN_EVALUATION="eval" + echo "Evaluation mode enabled" + break + fi +done if [ -z "$NUM_WORKERS" ]; then NUM_WORKERS=1 @@ -51,10 +66,59 @@ if [ -n "$EVAL_LIMIT" ]; then COMMAND="$COMMAND --eval-n-limit $EVAL_LIMIT" fi -if [ -n "$EVAL_IDS" ]; then +# Only pass eval-ids if it's not "eval" (which is a special parameter for evaluation mode) +if [ -n "$EVAL_IDS" ] && [ "$EVAL_IDS" != "eval" ]; then echo "EVAL_IDS: $EVAL_IDS" COMMAND="$COMMAND --eval-ids $EVAL_IDS" fi # Run the command eval $COMMAND + +# Get the output directory - first try the default location +OUTPUT_DIR=$(find evaluation/evaluation_outputs -path "*/AiderBench/$AGENT/*" -type d -name "*$EVAL_NOTE*" 2>/dev/null | sort -r | head -n 1) + +# If not found, try to find it anywhere under evaluation_outputs +if [ -z "$OUTPUT_DIR" ]; then + OUTPUT_DIR=$(find . -path "*/evaluation_outputs/*" -path "*/AiderBench/$AGENT/*" -type d -name "*$EVAL_NOTE*" 2>/dev/null | sort -r | head -n 1) +fi + +# If still not found, try to find any output.jsonl file +if [ -z "$OUTPUT_DIR" ]; then + OUTPUT_FILE=$(find . -name "output.jsonl" 2>/dev/null | sort -r | head -n 1) + if [ -n "$OUTPUT_FILE" ]; then + OUTPUT_DIR=$(dirname "$OUTPUT_FILE") + fi +else + OUTPUT_FILE="$OUTPUT_DIR/output.jsonl" +fi + +# Print the output directory and file for debugging +echo "" +echo "Output directory: $OUTPUT_DIR" +echo "Output file: $OUTPUT_FILE" + +# Run evaluation if requested +if [ "$RUN_EVALUATION" = "eval" ]; then + echo "" + echo "======================================" + echo "Running evaluation on results..." + echo "======================================" + echo "" + + if [ -f "$OUTPUT_FILE" ]; then + echo "Evaluating results in: $OUTPUT_FILE" + poetry run python evaluation/benchmarks/aider_bench/scripts/summarize_results.py "$OUTPUT_FILE" + + # Save the evaluation results + EVAL_RESULTS_FILE="$OUTPUT_DIR/evaluation_results.txt" + echo "Saving evaluation results to: $EVAL_RESULTS_FILE" + poetry run python evaluation/benchmarks/aider_bench/scripts/summarize_results.py "$OUTPUT_FILE" > "$EVAL_RESULTS_FILE" + + echo "" + echo "Evaluation complete. Results saved to: $EVAL_RESULTS_FILE" + else + echo "Error: Output file not found: $OUTPUT_FILE" + echo "Cannot run evaluation." + fi +fi diff --git a/evaluation/benchmarks/alfworld/__init__.py b/evaluation/benchmarks/alfworld/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/evaluation/benchmarks/apps/README.md b/evaluation/benchmarks/apps/README.md new file mode 100644 index 000000000000..339fdc377395 --- /dev/null +++ b/evaluation/benchmarks/apps/README.md @@ -0,0 +1,45 @@ +# APPS Benchmark Evaluation + +This folder contains evaluation harness for evaluating agents on the [APPS benchmark](https://huggingface.co/datasets/codeparrot/apps). + +APPS is a benchmark for code generation that consists of 10,000 problems, which range from introductory programming problems to competition-level problems. The benchmark contains natural language descriptions of problems, canonical solutions, and test cases. + +## Setup Environment and LLM Configuration + +Please follow instruction [here](../../README.md#setup) to setup your local development environment and LLM. + +## Start the evaluation + +```bash +./evaluation/benchmarks/apps/scripts/run_infer.sh [model_config] [git-version] [agent] [eval_limit] [eval-num-workers] [eval_ids] [run_evaluation] +``` + +- `model_config`, e.g. `eval_gpt4_1106_preview`, is the config group name for your LLM settings, as defined in your `config.toml`. +- `git-version`, e.g. `HEAD`, is the git commit hash of the OpenHands version you would like to evaluate. It could also be a release tag like `0.9.0`. +- `agent`, e.g. `CodeActAgent`, is the name of the agent for benchmarks, defaulting to `CodeActAgent`. +- `eval_limit`, e.g. `10`, limits the evaluation to the first `eval_limit` instances. By default, the script evaluates the entire test set. +- `eval-num-workers`: the number of workers to use for evaluation. Default: `1`. +- `eval_ids`, e.g. `"1,3,10"`, limits the evaluation to instances with the given IDs (comma separated). +- `run_evaluation`: set to `eval` to automatically run evaluation after the benchmark completes. + +Following is the basic command to start the evaluation: + +```bash +# Run benchmark without evaluation +./evaluation/benchmarks/apps/scripts/run_infer.sh eval_gpt35_turbo HEAD CodeActAgent 10 1 "1,3,10" + +# Run benchmark with automatic evaluation +./evaluation/benchmarks/apps/scripts/run_infer.sh eval_gpt35_turbo HEAD CodeActAgent 10 1 "1,3,10" eval +``` + +## Summarize Results + +```bash +poetry run python ./evaluation/benchmarks/apps/scripts/summarize_results.py [path_to_output_jsonl_file] +``` + +Full example: + +```bash +poetry run python ./evaluation/benchmarks/apps/scripts/summarize_results.py evaluation/evaluation_outputs/outputs/APPS/CodeActAgent/gpt-4o-2024-05-13@20240620_maxiter_30_N_v1.9/output.jsonl +``` \ No newline at end of file diff --git a/evaluation/benchmarks/apps/__init__.py b/evaluation/benchmarks/apps/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/evaluation/benchmarks/apps/run_infer.py b/evaluation/benchmarks/apps/run_infer.py new file mode 100644 index 000000000000..54ac0398bacc --- /dev/null +++ b/evaluation/benchmarks/apps/run_infer.py @@ -0,0 +1,319 @@ +import asyncio +import copy +import os +import tempfile +from typing import Any + +import pandas as pd +from datasets import load_dataset + +from evaluation.utils.shared import ( + EvalMetadata, + EvalOutput, + compatibility_for_eval_history_pairs, + get_default_sandbox_config_for_eval, + make_metadata, + prepare_dataset, + reset_logger_for_multiprocessing, + run_evaluation, + update_llm_config_for_completions_logging, +) +from openhands.controller.state.state import State +from openhands.core.config import ( + AppConfig, + get_llm_config_arg, + load_from_toml, + parse_arguments, +) +from openhands.core.logger import openhands_logger as logger +from openhands.core.main import create_runtime, run_controller +from openhands.events.action import CmdRunAction, MessageAction +from openhands.events.observation import CmdOutputObservation +from openhands.runtime.base import Runtime +from openhands.utils.async_utils import call_async_from_sync + +# Configure any environment variables +SKIP_NUM = os.environ.get('SKIP_NUM') +SKIP_NUM = ( + int(SKIP_NUM) if SKIP_NUM and SKIP_NUM.isdigit() and int(SKIP_NUM) >= 0 else None +) + + +def get_config( + instance: pd.Series, + metadata: EvalMetadata, +) -> AppConfig: + sandbox_config = get_default_sandbox_config_for_eval() + sandbox_config.base_container_image = 'python:3.11-bookworm' + config = AppConfig( + default_agent=metadata.agent_class, + run_as_openhands=False, + runtime=os.environ.get('RUNTIME', 'docker'), + max_iterations=metadata.max_iterations, + sandbox=sandbox_config, + # do not mount workspace + workspace_base=None, + workspace_mount_path=None, + ) + # Update llm_config to enable completions logging + llm_config = update_llm_config_for_completions_logging( + metadata.llm_config, + metadata.eval_output_dir, + str(instance.instance_id) + ) + config.set_llm_config(llm_config) + agent_config = config.get_agent_config(metadata.agent_class) + agent_config.enable_prompt_extensions = False + + # copy 'draft_editor' config if exists + config_copy = copy.deepcopy(config) + load_from_toml(config_copy) + if 'draft_editor' in config_copy.llms: + config.set_llm_config(config_copy.llms['draft_editor'], 'draft_editor') + + return config + + +def initialize_runtime( + runtime: Runtime, + instance: pd.Series, +): + """Initialize the runtime for the agent. + + This function is called before the runtime is used to run the agent. + """ + logger.info(f"\n{'-' * 50} BEGIN Runtime Initialization Fn {'-' * 50}\n") + obs: CmdOutputObservation + + # Set up workspace + action = CmdRunAction(command='mkdir -p /workspace') + logger.info(action, extra={'msg_type': 'ACTION'}) + obs = runtime.run_action(action) + assert obs.exit_code == 0 + + action = CmdRunAction(command='cd /workspace') + logger.info(action, extra={'msg_type': 'ACTION'}) + obs = runtime.run_action(action) + assert obs.exit_code == 0 + + # Create problem file + with tempfile.TemporaryDirectory() as tmpdir: + file_path = os.path.join(tmpdir, 'problem.txt') + with open(file_path, 'w') as f: + f.write(instance.problem) + runtime.copy_to( + file_path, + '/workspace', + ) + + # Create test cases file + file_path = os.path.join(tmpdir, 'test_cases.py') + with open(file_path, 'w') as f: + f.write(instance.test_cases) + runtime.copy_to( + file_path, + '/workspace', + ) + + logger.info(f"\n{'-' * 50} END Runtime Initialization Fn {'-' * 50}\n") + + +def complete_runtime( + runtime: Runtime, + instance: pd.Series, +) -> dict[str, Any]: + """Complete the runtime for the agent. + + This function is called after the agent has run. + If you need to do something in the sandbox to get the correctness metric after + the agent has run, modify this function. + """ + logger.info(f"\n{'-' * 50} BEGIN Runtime Completion Fn {'-' * 50}\n") + obs: CmdOutputObservation + + # Check if solution.py exists + action = CmdRunAction(command='ls -la /workspace') + logger.info(action, extra={'msg_type': 'ACTION'}) + obs = runtime.run_action(action) + logger.info(obs, extra={'msg_type': 'OBSERVATION'}) + + # Run test cases + action = CmdRunAction(command='python3 /workspace/test_cases.py') + logger.info(action, extra={'msg_type': 'ACTION'}) + obs = runtime.run_action(action) + logger.info(obs, extra={'msg_type': 'OBSERVATION'}) + + exit_code = 1 + if isinstance(obs, CmdOutputObservation): + exit_code = obs.exit_code + + logger.info(f"\n{'-' * 50} END Runtime Completion Fn {'-' * 50}\n") + + runtime.close() + + return { + 'test_output': obs.content, + 'exit_code': exit_code, + } + + +def process_instance( + instance: pd.Series, + metadata: EvalMetadata, + reset_logger: bool = True, +) -> EvalOutput: + config = get_config(instance, metadata) + + # Setup the logger properly, so you can run multi-processing to parallelize the evaluation + if reset_logger: + log_dir = os.path.join(metadata.eval_output_dir, 'infer_logs') + reset_logger_for_multiprocessing(logger, str(instance.instance_id), log_dir) + else: + logger.info( + f'\nStarting evaluation for instance {str(instance.instance_id)}.\n' + ) + + # ============================================= + # build instruction + # ============================================= + + # Prepare instruction + logger.info(instance) + instruction = f"""You are given a programming problem to solve. The problem description is in the file 'problem.txt'. + +Please read the problem carefully and implement a solution in Python. Save your solution in a file named 'solution.py'. + +After implementing your solution, you can test it by running 'python3 test_cases.py'. This will execute your solution against a set of test cases. + +IMPORTANT: You should ONLY interact with the environment provided to you AND NEVER ASK FOR HUMAN HELP. +""" + + # ============================================= + # create sandbox and run the agent + # ============================================= + + runtime: Runtime = create_runtime(config) + call_async_from_sync(runtime.connect) + + initialize_runtime(runtime, instance=instance) + + # Here's how you can run the agent (similar to the `main` function) and get the final task state + state: State | None = asyncio.run( + run_controller( + config=config, + initial_user_action=MessageAction(content=instruction), + runtime=runtime, + ) + ) + if state is None: + raise ValueError('State should not be None.') + + # ============================================= + # result evaluation + # ============================================= + + return_val = complete_runtime(runtime, instance) + exit_code = return_val['exit_code'] + test_output = return_val['test_output'] + + test_result = { + 'exit_code': exit_code, + 'test_output': test_output, + } + + # history is now available as a stream of events, rather than list of pairs of (Action, Observation) + # for compatibility with the existing output format, we can remake the pairs here + # remove when it becomes unnecessary + histories = compatibility_for_eval_history_pairs(state.history) + metrics = state.metrics.get() if state.metrics else None + + # Save the output + output = EvalOutput( + instance_id=str(instance.instance_id), + instance=instance.to_dict(), + instruction=instruction, + metadata=metadata, + history=histories, + metrics=metrics, + error=state.last_error if state and state.last_error else None, + test_result=test_result, + ) + return output + + +def prepare_apps_dataset(): + """Prepare the APPS dataset for evaluation.""" + # Load the APPS dataset + dataset = load_dataset('codeparrot/apps', split='test') + + # Convert to pandas DataFrame + df = dataset.to_pandas() + + # Add instance_id column + df['instance_id'] = df.index + + # Rename columns to match expected format + df = df.rename(columns={ + 'question': 'problem', + 'test': 'test_cases', + }) + + return df + + +if __name__ == '__main__': + args = parse_arguments() + + # Prepare the APPS dataset + apps_dataset = prepare_apps_dataset() + + llm_config = None + if args.llm_config: + llm_config = get_llm_config_arg(args.llm_config) + # modify_params must be False for evaluation purpose, for reproducibility and accuracy of results + llm_config.modify_params = False + + if llm_config is None: + raise ValueError(f'Could not find LLM config: --llm_config {args.llm_config}') + + # Create details dictionary with agent configuration + agent_details = { + "agent_config": { + "codeact_enable_jupyter": False, + "codeact_enable_browsing": False, + "codeact_enable_llm_editor": False, + } + } + + metadata = make_metadata( + llm_config, + 'APPS', + args.agent_cls, + args.max_iterations, + args.eval_note, + args.eval_output_dir, + details=agent_details, + ) + output_file = os.path.join(metadata.eval_output_dir, 'output.jsonl') + + # Parse dataset IDs if provided + eval_ids = None + if args.eval_ids: + eval_ids = str(args.eval_ids).split(',') + logger.info(f'\nUsing specific dataset IDs: {eval_ids}\n') + + instances = prepare_dataset( + apps_dataset, + output_file, + args.eval_n_limit, + eval_ids=eval_ids, + skip_num=SKIP_NUM, + ) + + run_evaluation( + instances, + metadata, + output_file, + args.eval_num_workers, + process_instance, + ) \ No newline at end of file diff --git a/evaluation/benchmarks/apps/scripts/run_infer.sh b/evaluation/benchmarks/apps/scripts/run_infer.sh new file mode 100755 index 000000000000..c053c6e3fbae --- /dev/null +++ b/evaluation/benchmarks/apps/scripts/run_infer.sh @@ -0,0 +1,114 @@ +#!/usr/bin/env bash +set -eo pipefail + +source "evaluation/utils/version_control.sh" + +MODEL_CONFIG=$1 +COMMIT_HASH=$2 +AGENT=$3 +EVAL_LIMIT=$4 +NUM_WORKERS=$5 +EVAL_IDS=$6 +RUN_EVALUATION=$7 # New parameter to run evaluation after benchmark + +# Special case: if the 7th parameter is "eval", set RUN_EVALUATION to "eval" +if [ "$RUN_EVALUATION" = "eval" ]; then + echo "Evaluation mode enabled" +fi + +# Special case: if any parameter is "eval", set RUN_EVALUATION to "eval" +for param in "$@"; do + if [ "$param" = "eval" ]; then + RUN_EVALUATION="eval" + echo "Evaluation mode enabled" + break + fi +done + +if [ -z "$NUM_WORKERS" ]; then + NUM_WORKERS=1 + echo "Number of workers not specified, use default $NUM_WORKERS" +fi +checkout_eval_branch + +if [ -z "$AGENT" ]; then + echo "Agent not specified, use default CodeActAgent" + AGENT="CodeActAgent" +fi + +get_openhands_version + +echo "AGENT: $AGENT" +echo "OPENHANDS_VERSION: $OPENHANDS_VERSION" +echo "MODEL_CONFIG: $MODEL_CONFIG" + +EVAL_NOTE=$OPENHANDS_VERSION + +COMMAND="export PYTHONPATH=evaluation/benchmarks/apps:\$PYTHONPATH && poetry run python evaluation/benchmarks/apps/run_infer.py \ + --agent-cls $AGENT \ + --llm-config $MODEL_CONFIG \ + --max-iterations 30 \ + --eval-num-workers $NUM_WORKERS \ + --eval-note $EVAL_NOTE" + +if [ -n "$EVAL_LIMIT" ]; then + echo "EVAL_LIMIT: $EVAL_LIMIT" + COMMAND="$COMMAND --eval-n-limit $EVAL_LIMIT" +fi + +# Only pass eval-ids if it's not "eval" (which is a special parameter for evaluation mode) +if [ -n "$EVAL_IDS" ] && [ "$EVAL_IDS" != "eval" ]; then + echo "EVAL_IDS: $EVAL_IDS" + COMMAND="$COMMAND --eval-ids $EVAL_IDS" +fi + +# Run the command +eval $COMMAND + +# Get the output directory - first try the default location +OUTPUT_DIR=$(find evaluation/evaluation_outputs -path "*/APPS/$AGENT/*" -type d -name "*$EVAL_NOTE*" 2>/dev/null | sort -r | head -n 1) + +# If not found, try to find it anywhere under evaluation_outputs +if [ -z "$OUTPUT_DIR" ]; then + OUTPUT_DIR=$(find . -path "*/evaluation_outputs/*" -path "*/APPS/$AGENT/*" -type d -name "*$EVAL_NOTE*" 2>/dev/null | sort -r | head -n 1) +fi + +# If still not found, try to find any output.jsonl file +if [ -z "$OUTPUT_DIR" ]; then + OUTPUT_FILE=$(find . -name "output.jsonl" 2>/dev/null | sort -r | head -n 1) + if [ -n "$OUTPUT_FILE" ]; then + OUTPUT_DIR=$(dirname "$OUTPUT_FILE") + fi +else + OUTPUT_FILE="$OUTPUT_DIR/output.jsonl" +fi + +# Print the output directory and file for debugging +echo "" +echo "Output directory: $OUTPUT_DIR" +echo "Output file: $OUTPUT_FILE" + +# Run evaluation if requested +if [ "$RUN_EVALUATION" = "eval" ]; then + echo "" + echo "======================================" + echo "Running evaluation on results..." + echo "======================================" + echo "" + + if [ -f "$OUTPUT_FILE" ]; then + echo "Evaluating results in: $OUTPUT_FILE" + poetry run python evaluation/benchmarks/apps/scripts/summarize_results.py "$OUTPUT_FILE" + + # Save the evaluation results + EVAL_RESULTS_FILE="$OUTPUT_DIR/evaluation_results.txt" + echo "Saving evaluation results to: $EVAL_RESULTS_FILE" + poetry run python evaluation/benchmarks/apps/scripts/summarize_results.py "$OUTPUT_FILE" > "$EVAL_RESULTS_FILE" + + echo "" + echo "Evaluation complete. Results saved to: $EVAL_RESULTS_FILE" + else + echo "Error: Output file not found: $OUTPUT_FILE" + echo "Cannot run evaluation." + fi +fi \ No newline at end of file diff --git a/evaluation/benchmarks/apps/scripts/summarize_results.py b/evaluation/benchmarks/apps/scripts/summarize_results.py new file mode 100755 index 000000000000..a661ac078d87 --- /dev/null +++ b/evaluation/benchmarks/apps/scripts/summarize_results.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +import argparse +import json +import os +from collections import defaultdict + + +def load_jsonl(file_path): + """Load a jsonl file.""" + data = [] + with open(file_path, 'r') as f: + for line in f: + data.append(json.loads(line)) + return data + + +def summarize_results(output_file): + """Summarize the results of the APPS benchmark.""" + print(f"Summarizing results from {output_file}") + + # Load the results + results = load_jsonl(output_file) + + # Count the number of instances that passed and failed + passed = [] + failed = [] + + for result in results: + instance_id = result['instance_id'] + test_result = result.get('test_result', {}) + exit_code = test_result.get('exit_code', 1) + + if exit_code == 0: + passed.append(instance_id) + else: + failed.append(instance_id) + + # Print the summary + print(f"\nTotal instances: {len(results)}") + print(f"Passed: {len(passed)} ({len(passed) / len(results) * 100:.2f}%)") + print(f"Failed: {len(failed)} ({len(failed) / len(results) * 100:.2f}%)") + + # Print the list of passed and failed instances + print("\nPassed instances:") + for instance_id in passed: + print(f" - {instance_id}") + + print("\nFailed instances:") + for instance_id in failed: + print(f" - {instance_id}") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Summarize APPS benchmark results") + parser.add_argument("output_file", help="Path to the output.jsonl file") + args = parser.parse_args() + + summarize_results(args.output_file) \ No newline at end of file diff --git a/evaluation/benchmarks/hotpotqa/README.md b/evaluation/benchmarks/hotpotqa/README.md new file mode 100644 index 000000000000..3aea9b507293 --- /dev/null +++ b/evaluation/benchmarks/hotpotqa/README.md @@ -0,0 +1,45 @@ +# HotpotQA Benchmark Evaluation + +This folder contains evaluation harness for evaluating agents on the [HotpotQA benchmark](http://curtis.ml.cmu.edu/datasets/hotpot/). + +HotpotQA is a question answering dataset featuring natural, multi-hop questions, with strong supervision for supporting facts to enable more explainable question answering systems. + +## Setup Environment and LLM Configuration + +Please follow instruction [here](../../README.md#setup) to setup your local development environment and LLM. + +## Start the evaluation + +```bash +./evaluation/benchmarks/hotpotqa/scripts/run_infer.sh [model_config] [git-version] [agent] [eval_limit] [eval-num-workers] [eval_ids] [run_evaluation] +``` + +- `model_config`, e.g. `eval_gpt4_1106_preview`, is the config group name for your LLM settings, as defined in your `config.toml`. +- `git-version`, e.g. `HEAD`, is the git commit hash of the OpenHands version you would like to evaluate. It could also be a release tag like `0.9.0`. +- `agent`, e.g. `CodeActAgent`, is the name of the agent for benchmarks, defaulting to `CodeActAgent`. +- `eval_limit`, e.g. `10`, limits the evaluation to the first `eval_limit` instances. By default, the script evaluates the entire test set. +- `eval-num-workers`: the number of workers to use for evaluation. Default: `1`. +- `eval_ids`, e.g. `"1,3,10"`, limits the evaluation to instances with the given IDs (comma separated). +- `run_evaluation`: set to `eval` to automatically run evaluation after the benchmark completes. + +Following is the basic command to start the evaluation: + +```bash +# Run benchmark without evaluation +./evaluation/benchmarks/hotpotqa/scripts/run_infer.sh eval_gpt35_turbo HEAD CodeActAgent 10 1 "1,3,10" + +# Run benchmark with automatic evaluation +./evaluation/benchmarks/hotpotqa/scripts/run_infer.sh eval_gpt35_turbo HEAD CodeActAgent 10 1 "1,3,10" eval +``` + +## Summarize Results + +```bash +poetry run python ./evaluation/benchmarks/hotpotqa/scripts/summarize_results.py [path_to_output_jsonl_file] +``` + +Full example: + +```bash +poetry run python ./evaluation/benchmarks/hotpotqa/scripts/summarize_results.py evaluation/evaluation_outputs/outputs/HotpotQA/CodeActAgent/gpt-4o-2024-05-13@20240620_maxiter_30_N_v1.9/output.jsonl +``` \ No newline at end of file diff --git a/evaluation/benchmarks/hotpotqa/__init__.py b/evaluation/benchmarks/hotpotqa/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/evaluation/benchmarks/hotpotqa/run_infer.py b/evaluation/benchmarks/hotpotqa/run_infer.py new file mode 100644 index 000000000000..b0ac799e1979 --- /dev/null +++ b/evaluation/benchmarks/hotpotqa/run_infer.py @@ -0,0 +1,354 @@ +import asyncio +import copy +import json +import os +import tempfile +from typing import Any + +import pandas as pd +import requests + +from evaluation.utils.shared import ( + EvalMetadata, + EvalOutput, + compatibility_for_eval_history_pairs, + get_default_sandbox_config_for_eval, + make_metadata, + prepare_dataset, + reset_logger_for_multiprocessing, + run_evaluation, + update_llm_config_for_completions_logging, +) +from openhands.controller.state.state import State +from openhands.core.config import ( + AppConfig, + get_llm_config_arg, + load_from_toml, + parse_arguments, +) +from openhands.core.logger import openhands_logger as logger +from openhands.core.main import create_runtime, run_controller +from openhands.events.action import CmdRunAction, MessageAction +from openhands.events.observation import CmdOutputObservation +from openhands.runtime.base import Runtime +from openhands.utils.async_utils import call_async_from_sync + +# Configure any environment variables +SKIP_NUM = os.environ.get('SKIP_NUM') +SKIP_NUM = ( + int(SKIP_NUM) if SKIP_NUM and SKIP_NUM.isdigit() and int(SKIP_NUM) >= 0 else None +) + + +def get_config( + instance: pd.Series, + metadata: EvalMetadata, +) -> AppConfig: + sandbox_config = get_default_sandbox_config_for_eval() + sandbox_config.base_container_image = 'python:3.11-bookworm' + config = AppConfig( + default_agent=metadata.agent_class, + run_as_openhands=False, + runtime=os.environ.get('RUNTIME', 'docker'), + max_iterations=metadata.max_iterations, + sandbox=sandbox_config, + # do not mount workspace + workspace_base=None, + workspace_mount_path=None, + ) + # Update llm_config to enable completions logging + llm_config = update_llm_config_for_completions_logging( + metadata.llm_config, + metadata.eval_output_dir, + str(instance.instance_id) + ) + config.set_llm_config(llm_config) + agent_config = config.get_agent_config(metadata.agent_class) + agent_config.enable_prompt_extensions = False + + # copy 'draft_editor' config if exists + config_copy = copy.deepcopy(config) + load_from_toml(config_copy) + if 'draft_editor' in config_copy.llms: + config.set_llm_config(config_copy.llms['draft_editor'], 'draft_editor') + + return config + + +def initialize_runtime( + runtime: Runtime, + instance: pd.Series, +): + """Initialize the runtime for the agent. + + This function is called before the runtime is used to run the agent. + """ + logger.info(f"\n{'-' * 50} BEGIN Runtime Initialization Fn {'-' * 50}\n") + obs: CmdOutputObservation + + # Set up workspace + action = CmdRunAction(command='mkdir -p /workspace') + logger.info(action, extra={'msg_type': 'ACTION'}) + obs = runtime.run_action(action) + assert obs.exit_code == 0 + + action = CmdRunAction(command='cd /workspace') + logger.info(action, extra={'msg_type': 'ACTION'}) + obs = runtime.run_action(action) + assert obs.exit_code == 0 + + # Create question file + with tempfile.TemporaryDirectory() as tmpdir: + file_path = os.path.join(tmpdir, 'question.txt') + with open(file_path, 'w') as f: + f.write(instance.question) + runtime.copy_to( + file_path, + '/workspace', + ) + + # Create context files + for i, context in enumerate(instance.context): + file_path = os.path.join(tmpdir, f'context_{i}.txt') + with open(file_path, 'w') as f: + f.write(context) + runtime.copy_to( + file_path, + '/workspace', + ) + + logger.info(f"\n{'-' * 50} END Runtime Initialization Fn {'-' * 50}\n") + + +def complete_runtime( + runtime: Runtime, + instance: pd.Series, +) -> dict[str, Any]: + """Complete the runtime for the agent. + + This function is called after the agent has run. + If you need to do something in the sandbox to get the correctness metric after + the agent has run, modify this function. + """ + logger.info(f"\n{'-' * 50} BEGIN Runtime Completion Fn {'-' * 50}\n") + obs: CmdOutputObservation + + # Check if answer.txt exists + action = CmdRunAction(command='ls -la /workspace') + logger.info(action, extra={'msg_type': 'ACTION'}) + obs = runtime.run_action(action) + logger.info(obs, extra={'msg_type': 'OBSERVATION'}) + + # Get the answer content + answer_content = "" + if "answer.txt" in obs.content: + action = CmdRunAction(command='cat /workspace/answer.txt') + logger.info(action, extra={'msg_type': 'ACTION'}) + obs = runtime.run_action(action) + logger.info(obs, extra={'msg_type': 'OBSERVATION'}) + answer_content = obs.content + + logger.info(f"\n{'-' * 50} END Runtime Completion Fn {'-' * 50}\n") + + runtime.close() + + # For HotpotQA, we need to evaluate the answer against the ground truth + # Here we just return the answer content for evaluation + return { + 'answer': answer_content, + 'correct_answer': instance.answer, + } + + +def process_instance( + instance: pd.Series, + metadata: EvalMetadata, + reset_logger: bool = True, +) -> EvalOutput: + config = get_config(instance, metadata) + + # Setup the logger properly, so you can run multi-processing to parallelize the evaluation + if reset_logger: + log_dir = os.path.join(metadata.eval_output_dir, 'infer_logs') + reset_logger_for_multiprocessing(logger, str(instance.instance_id), log_dir) + else: + logger.info( + f'\nStarting evaluation for instance {str(instance.instance_id)}.\n' + ) + + # ============================================= + # build instruction + # ============================================= + + # Prepare instruction + logger.info(instance) + instruction = f"""You are given a question and some context documents to help you answer it. The question is in the file 'question.txt'. + +The context documents are in files named 'context_0.txt', 'context_1.txt', etc. You should read all the context files to gather information needed to answer the question. + +Please write your answer in a file named 'answer.txt'. Your answer should be concise and directly address the question. + +IMPORTANT: You should ONLY interact with the environment provided to you AND NEVER ASK FOR HUMAN HELP. +""" + + # ============================================= + # create sandbox and run the agent + # ============================================= + + runtime: Runtime = create_runtime(config) + call_async_from_sync(runtime.connect) + + initialize_runtime(runtime, instance=instance) + + # Here's how you can run the agent (similar to the `main` function) and get the final task state + state: State | None = asyncio.run( + run_controller( + config=config, + initial_user_action=MessageAction(content=instruction), + runtime=runtime, + ) + ) + if state is None: + raise ValueError('State should not be None.') + + # ============================================= + # result evaluation + # ============================================= + + return_val = complete_runtime(runtime, instance) + answer = return_val['answer'] + correct_answer = return_val['correct_answer'] + + # Simple evaluation - check if the answer matches the correct answer + # In a real implementation, you would need a more sophisticated evaluation + is_correct = answer.strip().lower() == correct_answer.strip().lower() + + test_result = { + 'answer': answer, + 'correct_answer': correct_answer, + 'is_correct': is_correct, + } + + # history is now available as a stream of events, rather than list of pairs of (Action, Observation) + # for compatibility with the existing output format, we can remake the pairs here + # remove when it becomes unnecessary + histories = compatibility_for_eval_history_pairs(state.history) + metrics = state.metrics.get() if state.metrics else None + + # Save the output + output = EvalOutput( + instance_id=str(instance.instance_id), + instance=instance.to_dict(), + instruction=instruction, + metadata=metadata, + history=histories, + metrics=metrics, + error=state.last_error if state and state.last_error else None, + test_result=test_result, + ) + return output + + +def prepare_hotpotqa_dataset(): + """Prepare the HotpotQA dataset for evaluation.""" + # In a real implementation, you would download and process the HotpotQA dataset + # For now, we'll create a simple mock dataset + data = { + 'instance_id': list(range(10)), + 'question': [ + "What government position was held by the woman who portrayed Corliss Archer in the film Kiss and Tell?", + "Were Scott Derrickson and Ed Wood of the same nationality?", + "What is the name of the professional wrestler who had a role in the film The Princess Bride?", + "Which magazine was started first Arthur's Magazine or First for Women?", + "What city was the birthplace of the actor who played Humpty Dumpty in the 2010 adaptation of Alice in Wonderland?", + "What is the difference in years between the release of The Innocents and The Others?", + "What is the name of the actor who played the character Wolverine in the X-Men film series?", + "Which country is the birthplace of the actor who played James Bond in the film Skyfall?", + "What is the name of the director who directed the film Inception?", + "Which film won more Academy Awards, The Lord of the Rings: The Return of the King or Titanic?" + ], + 'context': [ + ["Shirley Temple was an American actress, singer, dancer, businesswoman, and diplomat who was Hollywood's number one box-office draw as a child actress from 1935 to 1938.", "Shirley Temple Black (April 23, 1928 – February 10, 2014) was an American actress, singer, dancer, businesswoman, and diplomat who was Hollywood's number one box-office draw as a child actress from 1935 to 1938. As an adult, she was named United States ambassador to Ghana and to Czechoslovakia, and also served as Chief of Protocol of the United States."], + ["Scott Derrickson (born July 16, 1966) is an American director, screenwriter and producer. He lives in Los Angeles, California. He is best known for directing horror films such as Sinister, The Exorcism of Emily Rose, and Deliver Us From Evil, as well as the 2016 Marvel Cinematic Universe installment, Doctor Strange.", "Edward Davis Wood Jr. (October 10, 1924 – December 10, 1978) was an American filmmaker, actor, writer, producer, and director."], + ["André René Roussimoff (May 19, 1946 – January 27, 1993), best known as André the Giant, was a French professional wrestler and actor.", "The Princess Bride is a 1987 American fantasy comedy film directed and co-produced by Rob Reiner, starring Cary Elwes, Robin Wright, Mandy Patinkin, Chris Sarandon, Wallace Shawn, André the Giant, and Christopher Guest."], + ["Arthur's Magazine (1844-1846) was an American literary periodical published in Philadelphia in the 19th century.", "First for Women is a woman's magazine published by Bauer Media Group in the USA. The magazine was started in 1989."], + ["Sir Sydney Smirke RA (20 October 1798 – 8 December 1877) was a British architect who was born in London, England, the younger brother of Sir Robert Smirke, also an architect. Their father, also Robert Smirke, was a well-known painter.", "Alice in Wonderland is a 2010 American dark fantasy adventure film directed by Tim Burton from a screenplay written by Linda Woolverton."], + ["The Innocents is a 1961 British supernatural gothic horror film directed and produced by Jack Clayton, and starring Deborah Kerr, Michael Redgrave, and Megs Jenkins.", "The Others (Spanish: Los Otros) is a 2001 English-language Spanish gothic supernatural psychological horror film written, directed, and scored by Alejandro Amenábar."], + ["Hugh Michael Jackman (born 12 October 1968) is an Australian actor, singer, and producer.", "Wolverine is a fictional character appearing in American comic books published by Marvel Comics, mostly in association with the X-Men."], + ["Daniel Wroughton Craig (born 2 March 1968) is an English actor.", "Skyfall is a 2012 spy film and the twenty-third in the James Bond series produced by Eon Productions."], + ["Christopher Edward Nolan CBE (born 30 July 1970) is a British-American film director, producer, and screenwriter.", "Inception is a 2010 science fiction action film written and directed by Christopher Nolan, who also produced the film with his wife, Emma Thomas."], + ["The Lord of the Rings: The Return of the King is a 2003 epic fantasy adventure film directed by Peter Jackson, based on the third volume of J. R. R. Tolkien's The Lord of the Rings.", "Titanic is a 1997 American epic romance and disaster film directed, written, co-produced, and co-edited by James Cameron."] + ], + 'answer': [ + "United States ambassador", + "Yes", + "André the Giant", + "Arthur's Magazine", + "London", + "40 years", + "Hugh Jackman", + "England", + "Christopher Nolan", + "The Lord of the Rings: The Return of the King" + ] + } + + return pd.DataFrame(data) + + +if __name__ == '__main__': + args = parse_arguments() + + # Prepare the HotpotQA dataset + hotpotqa_dataset = prepare_hotpotqa_dataset() + + llm_config = None + if args.llm_config: + llm_config = get_llm_config_arg(args.llm_config) + # modify_params must be False for evaluation purpose, for reproducibility and accuracy of results + llm_config.modify_params = False + + if llm_config is None: + raise ValueError(f'Could not find LLM config: --llm_config {args.llm_config}') + + # Create details dictionary with agent configuration + agent_details = { + "agent_config": { + "codeact_enable_jupyter": False, + "codeact_enable_browsing": False, + "codeact_enable_llm_editor": False, + } + } + + metadata = make_metadata( + llm_config, + 'HotpotQA', + args.agent_cls, + args.max_iterations, + args.eval_note, + args.eval_output_dir, + details=agent_details, + ) + output_file = os.path.join(metadata.eval_output_dir, 'output.jsonl') + + # Parse dataset IDs if provided + eval_ids = None + if args.eval_ids: + eval_ids = str(args.eval_ids).split(',') + logger.info(f'\nUsing specific dataset IDs: {eval_ids}\n') + + instances = prepare_dataset( + hotpotqa_dataset, + output_file, + args.eval_n_limit, + eval_ids=eval_ids, + skip_num=SKIP_NUM, + ) + + run_evaluation( + instances, + metadata, + output_file, + args.eval_num_workers, + process_instance, + ) \ No newline at end of file diff --git a/evaluation/benchmarks/hotpotqa/scripts/run_infer.sh b/evaluation/benchmarks/hotpotqa/scripts/run_infer.sh new file mode 100755 index 000000000000..434f2f35dd45 --- /dev/null +++ b/evaluation/benchmarks/hotpotqa/scripts/run_infer.sh @@ -0,0 +1,114 @@ +#!/usr/bin/env bash +set -eo pipefail + +source "evaluation/utils/version_control.sh" + +MODEL_CONFIG=$1 +COMMIT_HASH=$2 +AGENT=$3 +EVAL_LIMIT=$4 +NUM_WORKERS=$5 +EVAL_IDS=$6 +RUN_EVALUATION=$7 # New parameter to run evaluation after benchmark + +# Special case: if the 7th parameter is "eval", set RUN_EVALUATION to "eval" +if [ "$RUN_EVALUATION" = "eval" ]; then + echo "Evaluation mode enabled" +fi + +# Special case: if any parameter is "eval", set RUN_EVALUATION to "eval" +for param in "$@"; do + if [ "$param" = "eval" ]; then + RUN_EVALUATION="eval" + echo "Evaluation mode enabled" + break + fi +done + +if [ -z "$NUM_WORKERS" ]; then + NUM_WORKERS=1 + echo "Number of workers not specified, use default $NUM_WORKERS" +fi +checkout_eval_branch + +if [ -z "$AGENT" ]; then + echo "Agent not specified, use default CodeActAgent" + AGENT="CodeActAgent" +fi + +get_openhands_version + +echo "AGENT: $AGENT" +echo "OPENHANDS_VERSION: $OPENHANDS_VERSION" +echo "MODEL_CONFIG: $MODEL_CONFIG" + +EVAL_NOTE=$OPENHANDS_VERSION + +COMMAND="export PYTHONPATH=evaluation/benchmarks/hotpotqa:\$PYTHONPATH && poetry run python evaluation/benchmarks/hotpotqa/run_infer.py \ + --agent-cls $AGENT \ + --llm-config $MODEL_CONFIG \ + --max-iterations 30 \ + --eval-num-workers $NUM_WORKERS \ + --eval-note $EVAL_NOTE" + +if [ -n "$EVAL_LIMIT" ]; then + echo "EVAL_LIMIT: $EVAL_LIMIT" + COMMAND="$COMMAND --eval-n-limit $EVAL_LIMIT" +fi + +# Only pass eval-ids if it's not "eval" (which is a special parameter for evaluation mode) +if [ -n "$EVAL_IDS" ] && [ "$EVAL_IDS" != "eval" ]; then + echo "EVAL_IDS: $EVAL_IDS" + COMMAND="$COMMAND --eval-ids $EVAL_IDS" +fi + +# Run the command +eval $COMMAND + +# Get the output directory - first try the default location +OUTPUT_DIR=$(find evaluation/evaluation_outputs -path "*/HotpotQA/$AGENT/*" -type d -name "*$EVAL_NOTE*" 2>/dev/null | sort -r | head -n 1) + +# If not found, try to find it anywhere under evaluation_outputs +if [ -z "$OUTPUT_DIR" ]; then + OUTPUT_DIR=$(find . -path "*/evaluation_outputs/*" -path "*/HotpotQA/$AGENT/*" -type d -name "*$EVAL_NOTE*" 2>/dev/null | sort -r | head -n 1) +fi + +# If still not found, try to find any output.jsonl file +if [ -z "$OUTPUT_DIR" ]; then + OUTPUT_FILE=$(find . -name "output.jsonl" 2>/dev/null | sort -r | head -n 1) + if [ -n "$OUTPUT_FILE" ]; then + OUTPUT_DIR=$(dirname "$OUTPUT_FILE") + fi +else + OUTPUT_FILE="$OUTPUT_DIR/output.jsonl" +fi + +# Print the output directory and file for debugging +echo "" +echo "Output directory: $OUTPUT_DIR" +echo "Output file: $OUTPUT_FILE" + +# Run evaluation if requested +if [ "$RUN_EVALUATION" = "eval" ]; then + echo "" + echo "======================================" + echo "Running evaluation on results..." + echo "======================================" + echo "" + + if [ -f "$OUTPUT_FILE" ]; then + echo "Evaluating results in: $OUTPUT_FILE" + poetry run python evaluation/benchmarks/hotpotqa/scripts/summarize_results.py "$OUTPUT_FILE" + + # Save the evaluation results + EVAL_RESULTS_FILE="$OUTPUT_DIR/evaluation_results.txt" + echo "Saving evaluation results to: $EVAL_RESULTS_FILE" + poetry run python evaluation/benchmarks/hotpotqa/scripts/summarize_results.py "$OUTPUT_FILE" > "$EVAL_RESULTS_FILE" + + echo "" + echo "Evaluation complete. Results saved to: $EVAL_RESULTS_FILE" + else + echo "Error: Output file not found: $OUTPUT_FILE" + echo "Cannot run evaluation." + fi +fi \ No newline at end of file diff --git a/evaluation/benchmarks/hotpotqa/scripts/summarize_results.py b/evaluation/benchmarks/hotpotqa/scripts/summarize_results.py new file mode 100755 index 000000000000..9c7bba2fbbfc --- /dev/null +++ b/evaluation/benchmarks/hotpotqa/scripts/summarize_results.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +import argparse +import json +import os +from collections import defaultdict + + +def load_jsonl(file_path): + """Load a jsonl file.""" + data = [] + with open(file_path, 'r') as f: + for line in f: + data.append(json.loads(line)) + return data + + +def summarize_results(output_file): + """Summarize the results of the HotpotQA benchmark.""" + print(f"Summarizing results from {output_file}") + + # Load the results + results = load_jsonl(output_file) + + # Count the number of instances that passed and failed + correct = [] + incorrect = [] + + for result in results: + instance_id = result['instance_id'] + test_result = result.get('test_result', {}) + is_correct = test_result.get('is_correct', False) + + if is_correct: + correct.append(instance_id) + else: + incorrect.append(instance_id) + + # Print the summary + print(f"\nTotal instances: {len(results)}") + print(f"Correct: {len(correct)} ({len(correct) / len(results) * 100:.2f}%)") + print(f"Incorrect: {len(incorrect)} ({len(incorrect) / len(results) * 100:.2f}%)") + + # Print the list of correct and incorrect instances + print("\nCorrect instances:") + for instance_id in correct: + print(f" - {instance_id}") + + print("\nIncorrect instances:") + for instance_id in incorrect: + print(f" - {instance_id}") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Summarize HotpotQA benchmark results") + parser.add_argument("output_file", help="Path to the output.jsonl file") + args = parser.parse_args() + + summarize_results(args.output_file) \ No newline at end of file diff --git a/evaluation/benchmarks/math/README.md b/evaluation/benchmarks/math/README.md new file mode 100644 index 000000000000..46589f56621c --- /dev/null +++ b/evaluation/benchmarks/math/README.md @@ -0,0 +1,45 @@ +# MATH Benchmark Evaluation + +This folder contains evaluation harness for evaluating agents on the [MATH benchmark](https://github.com/hendrycks/math). + +MATH is a dataset of 12,500 challenging competition mathematics problems. Each problem in MATH has a full step-by-step solution which can be used to teach models to generate answer derivations and explanations. + +## Setup Environment and LLM Configuration + +Please follow instruction [here](../../README.md#setup) to setup your local development environment and LLM. + +## Start the evaluation + +```bash +./evaluation/benchmarks/math/scripts/run_infer.sh [model_config] [git-version] [agent] [eval_limit] [eval-num-workers] [eval_ids] [run_evaluation] +``` + +- `model_config`, e.g. `eval_gpt4_1106_preview`, is the config group name for your LLM settings, as defined in your `config.toml`. +- `git-version`, e.g. `HEAD`, is the git commit hash of the OpenHands version you would like to evaluate. It could also be a release tag like `0.9.0`. +- `agent`, e.g. `CodeActAgent`, is the name of the agent for benchmarks, defaulting to `CodeActAgent`. +- `eval_limit`, e.g. `10`, limits the evaluation to the first `eval_limit` instances. By default, the script evaluates the entire test set. +- `eval-num-workers`: the number of workers to use for evaluation. Default: `1`. +- `eval_ids`, e.g. `"1,3,10"`, limits the evaluation to instances with the given IDs (comma separated). +- `run_evaluation`: set to `eval` to automatically run evaluation after the benchmark completes. + +Following is the basic command to start the evaluation: + +```bash +# Run benchmark without evaluation +./evaluation/benchmarks/math/scripts/run_infer.sh eval_gpt35_turbo HEAD CodeActAgent 10 1 "1,3,10" + +# Run benchmark with automatic evaluation +./evaluation/benchmarks/math/scripts/run_infer.sh eval_gpt35_turbo HEAD CodeActAgent 10 1 "1,3,10" eval +``` + +## Summarize Results + +```bash +poetry run python ./evaluation/benchmarks/math/scripts/summarize_results.py [path_to_output_jsonl_file] +``` + +Full example: + +```bash +poetry run python ./evaluation/benchmarks/math/scripts/summarize_results.py evaluation/evaluation_outputs/outputs/MATH/CodeActAgent/gpt-4o-2024-05-13@20240620_maxiter_30_N_v1.9/output.jsonl +``` \ No newline at end of file diff --git a/evaluation/benchmarks/math/__init__.py b/evaluation/benchmarks/math/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/evaluation/benchmarks/math/run_infer.py b/evaluation/benchmarks/math/run_infer.py new file mode 100644 index 000000000000..18089778329a --- /dev/null +++ b/evaluation/benchmarks/math/run_infer.py @@ -0,0 +1,336 @@ +import asyncio +import copy +import os +import tempfile +from typing import Any + +import pandas as pd +from datasets import load_dataset + +from evaluation.utils.shared import ( + EvalMetadata, + EvalOutput, + compatibility_for_eval_history_pairs, + get_default_sandbox_config_for_eval, + make_metadata, + prepare_dataset, + reset_logger_for_multiprocessing, + run_evaluation, + update_llm_config_for_completions_logging, +) +from openhands.controller.state.state import State +from openhands.core.config import ( + AppConfig, + get_llm_config_arg, + load_from_toml, + parse_arguments, +) +from openhands.core.logger import openhands_logger as logger +from openhands.core.main import create_runtime, run_controller +from openhands.events.action import CmdRunAction, MessageAction +from openhands.events.observation import CmdOutputObservation +from openhands.runtime.base import Runtime +from openhands.utils.async_utils import call_async_from_sync + +# Configure any environment variables +SKIP_NUM = os.environ.get('SKIP_NUM') +SKIP_NUM = ( + int(SKIP_NUM) if SKIP_NUM and SKIP_NUM.isdigit() and int(SKIP_NUM) >= 0 else None +) + + +def get_config( + instance: pd.Series, + metadata: EvalMetadata, +) -> AppConfig: + sandbox_config = get_default_sandbox_config_for_eval() + sandbox_config.base_container_image = 'python:3.11-bookworm' + config = AppConfig( + default_agent=metadata.agent_class, + run_as_openhands=False, + runtime=os.environ.get('RUNTIME', 'docker'), + max_iterations=metadata.max_iterations, + sandbox=sandbox_config, + # do not mount workspace + workspace_base=None, + workspace_mount_path=None, + ) + # Update llm_config to enable completions logging + llm_config = update_llm_config_for_completions_logging( + metadata.llm_config, + metadata.eval_output_dir, + str(instance.instance_id) + ) + config.set_llm_config(llm_config) + agent_config = config.get_agent_config(metadata.agent_class) + agent_config.enable_prompt_extensions = False + + # copy 'draft_editor' config if exists + config_copy = copy.deepcopy(config) + load_from_toml(config_copy) + if 'draft_editor' in config_copy.llms: + config.set_llm_config(config_copy.llms['draft_editor'], 'draft_editor') + + return config + + +def initialize_runtime( + runtime: Runtime, + instance: pd.Series, +): + """Initialize the runtime for the agent. + + This function is called before the runtime is used to run the agent. + """ + logger.info(f"\n{'-' * 50} BEGIN Runtime Initialization Fn {'-' * 50}\n") + obs: CmdOutputObservation + + # Set up workspace + action = CmdRunAction(command='mkdir -p /workspace') + logger.info(action, extra={'msg_type': 'ACTION'}) + obs = runtime.run_action(action) + assert obs.exit_code == 0 + + action = CmdRunAction(command='cd /workspace') + logger.info(action, extra={'msg_type': 'ACTION'}) + obs = runtime.run_action(action) + assert obs.exit_code == 0 + + # Create problem file + with tempfile.TemporaryDirectory() as tmpdir: + file_path = os.path.join(tmpdir, 'problem.txt') + with open(file_path, 'w') as f: + f.write(instance.problem) + runtime.copy_to( + file_path, + '/workspace', + ) + + logger.info(f"\n{'-' * 50} END Runtime Initialization Fn {'-' * 50}\n") + + +def complete_runtime( + runtime: Runtime, + instance: pd.Series, +) -> dict[str, Any]: + """Complete the runtime for the agent. + + This function is called after the agent has run. + If you need to do something in the sandbox to get the correctness metric after + the agent has run, modify this function. + """ + logger.info(f"\n{'-' * 50} BEGIN Runtime Completion Fn {'-' * 50}\n") + obs: CmdOutputObservation + + # Check if solution.txt exists + action = CmdRunAction(command='ls -la /workspace') + logger.info(action, extra={'msg_type': 'ACTION'}) + obs = runtime.run_action(action) + logger.info(obs, extra={'msg_type': 'OBSERVATION'}) + + # Get the solution content + solution_content = "" + if "solution.txt" in obs.content: + action = CmdRunAction(command='cat /workspace/solution.txt') + logger.info(action, extra={'msg_type': 'ACTION'}) + obs = runtime.run_action(action) + logger.info(obs, extra={'msg_type': 'OBSERVATION'}) + solution_content = obs.content + + logger.info(f"\n{'-' * 50} END Runtime Completion Fn {'-' * 50}\n") + + runtime.close() + + # For MATH problems, we need to manually evaluate the solution + # Here we just return the solution content for manual evaluation + return { + 'solution': solution_content, + 'correct_answer': instance.answer, + } + + +def process_instance( + instance: pd.Series, + metadata: EvalMetadata, + reset_logger: bool = True, +) -> EvalOutput: + config = get_config(instance, metadata) + + # Setup the logger properly, so you can run multi-processing to parallelize the evaluation + if reset_logger: + log_dir = os.path.join(metadata.eval_output_dir, 'infer_logs') + reset_logger_for_multiprocessing(logger, str(instance.instance_id), log_dir) + else: + logger.info( + f'\nStarting evaluation for instance {str(instance.instance_id)}.\n' + ) + + # ============================================= + # build instruction + # ============================================= + + # Prepare instruction + logger.info(instance) + instruction = f"""You are given a mathematics problem to solve. The problem is in the file 'problem.txt'. + +Please read the problem carefully and solve it step by step. Write your solution in a file named 'solution.txt'. + +Your solution should include: +1. A clear understanding of the problem +2. Step-by-step working +3. The final answer + +IMPORTANT: You should ONLY interact with the environment provided to you AND NEVER ASK FOR HUMAN HELP. +""" + + # ============================================= + # create sandbox and run the agent + # ============================================= + + runtime: Runtime = create_runtime(config) + call_async_from_sync(runtime.connect) + + initialize_runtime(runtime, instance=instance) + + # Here's how you can run the agent (similar to the `main` function) and get the final task state + state: State | None = asyncio.run( + run_controller( + config=config, + initial_user_action=MessageAction(content=instruction), + runtime=runtime, + ) + ) + if state is None: + raise ValueError('State should not be None.') + + # ============================================= + # result evaluation + # ============================================= + + return_val = complete_runtime(runtime, instance) + solution = return_val['solution'] + correct_answer = return_val['correct_answer'] + + # Simple evaluation - check if the correct answer appears in the solution + # In a real implementation, you would need a more sophisticated evaluation + is_correct = correct_answer in solution + + test_result = { + 'solution': solution, + 'correct_answer': correct_answer, + 'is_correct': is_correct, + } + + # history is now available as a stream of events, rather than list of pairs of (Action, Observation) + # for compatibility with the existing output format, we can remake the pairs here + # remove when it becomes unnecessary + histories = compatibility_for_eval_history_pairs(state.history) + metrics = state.metrics.get() if state.metrics else None + + # Save the output + output = EvalOutput( + instance_id=str(instance.instance_id), + instance=instance.to_dict(), + instruction=instruction, + metadata=metadata, + history=histories, + metrics=metrics, + error=state.last_error if state and state.last_error else None, + test_result=test_result, + ) + return output + + +def prepare_math_dataset(): + """Prepare the MATH dataset for evaluation.""" + # In a real implementation, you would load the MATH dataset + # For now, we'll create a simple mock dataset + data = { + 'instance_id': list(range(10)), + 'problem': [ + "Find the value of x in the equation 2x + 3 = 7.", + "Solve for y: 3y - 5 = 10.", + "Calculate the area of a circle with radius 5 cm.", + "Find the derivative of f(x) = x^2 + 3x + 2.", + "Solve the system of equations: 2x + y = 5, x - y = 1.", + "Find the indefinite integral of g(x) = 2x + 3.", + "Calculate the limit of (x^2 - 1)/(x - 1) as x approaches 1.", + "Find the value of sin(30°) + cos(60°).", + "Solve the quadratic equation x^2 - 5x + 6 = 0.", + "Find the sum of the first 10 terms of the arithmetic sequence with a_1 = 3 and d = 2." + ], + 'answer': [ + "x = 2", + "y = 5", + "78.54 cm²", + "f'(x) = 2x + 3", + "x = 2, y = 1", + "∫(2x + 3)dx = x² + 3x + C", + "2", + "1", + "x = 2, x = 3", + "75" + ], + 'level': ['Algebra'] * 10, + 'type': ['Equation'] * 5 + ['Calculus'] * 3 + ['Equation'] * 2 + } + + return pd.DataFrame(data) + + +if __name__ == '__main__': + args = parse_arguments() + + # Prepare the MATH dataset + math_dataset = prepare_math_dataset() + + llm_config = None + if args.llm_config: + llm_config = get_llm_config_arg(args.llm_config) + # modify_params must be False for evaluation purpose, for reproducibility and accuracy of results + llm_config.modify_params = False + + if llm_config is None: + raise ValueError(f'Could not find LLM config: --llm_config {args.llm_config}') + + # Create details dictionary with agent configuration + agent_details = { + "agent_config": { + "codeact_enable_jupyter": False, + "codeact_enable_browsing": False, + "codeact_enable_llm_editor": False, + } + } + + metadata = make_metadata( + llm_config, + 'MATH', + args.agent_cls, + args.max_iterations, + args.eval_note, + args.eval_output_dir, + details=agent_details, + ) + output_file = os.path.join(metadata.eval_output_dir, 'output.jsonl') + + # Parse dataset IDs if provided + eval_ids = None + if args.eval_ids: + eval_ids = str(args.eval_ids).split(',') + logger.info(f'\nUsing specific dataset IDs: {eval_ids}\n') + + instances = prepare_dataset( + math_dataset, + output_file, + args.eval_n_limit, + eval_ids=eval_ids, + skip_num=SKIP_NUM, + ) + + run_evaluation( + instances, + metadata, + output_file, + args.eval_num_workers, + process_instance, + ) \ No newline at end of file diff --git a/evaluation/benchmarks/math/scripts/run_infer.sh b/evaluation/benchmarks/math/scripts/run_infer.sh new file mode 100755 index 000000000000..b157b2cd0df6 --- /dev/null +++ b/evaluation/benchmarks/math/scripts/run_infer.sh @@ -0,0 +1,114 @@ +#!/usr/bin/env bash +set -eo pipefail + +source "evaluation/utils/version_control.sh" + +MODEL_CONFIG=$1 +COMMIT_HASH=$2 +AGENT=$3 +EVAL_LIMIT=$4 +NUM_WORKERS=$5 +EVAL_IDS=$6 +RUN_EVALUATION=$7 # New parameter to run evaluation after benchmark + +# Special case: if the 7th parameter is "eval", set RUN_EVALUATION to "eval" +if [ "$RUN_EVALUATION" = "eval" ]; then + echo "Evaluation mode enabled" +fi + +# Special case: if any parameter is "eval", set RUN_EVALUATION to "eval" +for param in "$@"; do + if [ "$param" = "eval" ]; then + RUN_EVALUATION="eval" + echo "Evaluation mode enabled" + break + fi +done + +if [ -z "$NUM_WORKERS" ]; then + NUM_WORKERS=1 + echo "Number of workers not specified, use default $NUM_WORKERS" +fi +checkout_eval_branch + +if [ -z "$AGENT" ]; then + echo "Agent not specified, use default CodeActAgent" + AGENT="CodeActAgent" +fi + +get_openhands_version + +echo "AGENT: $AGENT" +echo "OPENHANDS_VERSION: $OPENHANDS_VERSION" +echo "MODEL_CONFIG: $MODEL_CONFIG" + +EVAL_NOTE=$OPENHANDS_VERSION + +COMMAND="export PYTHONPATH=evaluation/benchmarks/math:\$PYTHONPATH && poetry run python evaluation/benchmarks/math/run_infer.py \ + --agent-cls $AGENT \ + --llm-config $MODEL_CONFIG \ + --max-iterations 30 \ + --eval-num-workers $NUM_WORKERS \ + --eval-note $EVAL_NOTE" + +if [ -n "$EVAL_LIMIT" ]; then + echo "EVAL_LIMIT: $EVAL_LIMIT" + COMMAND="$COMMAND --eval-n-limit $EVAL_LIMIT" +fi + +# Only pass eval-ids if it's not "eval" (which is a special parameter for evaluation mode) +if [ -n "$EVAL_IDS" ] && [ "$EVAL_IDS" != "eval" ]; then + echo "EVAL_IDS: $EVAL_IDS" + COMMAND="$COMMAND --eval-ids $EVAL_IDS" +fi + +# Run the command +eval $COMMAND + +# Get the output directory - first try the default location +OUTPUT_DIR=$(find evaluation/evaluation_outputs -path "*/MATH/$AGENT/*" -type d -name "*$EVAL_NOTE*" 2>/dev/null | sort -r | head -n 1) + +# If not found, try to find it anywhere under evaluation_outputs +if [ -z "$OUTPUT_DIR" ]; then + OUTPUT_DIR=$(find . -path "*/evaluation_outputs/*" -path "*/MATH/$AGENT/*" -type d -name "*$EVAL_NOTE*" 2>/dev/null | sort -r | head -n 1) +fi + +# If still not found, try to find any output.jsonl file +if [ -z "$OUTPUT_DIR" ]; then + OUTPUT_FILE=$(find . -name "output.jsonl" 2>/dev/null | sort -r | head -n 1) + if [ -n "$OUTPUT_FILE" ]; then + OUTPUT_DIR=$(dirname "$OUTPUT_FILE") + fi +else + OUTPUT_FILE="$OUTPUT_DIR/output.jsonl" +fi + +# Print the output directory and file for debugging +echo "" +echo "Output directory: $OUTPUT_DIR" +echo "Output file: $OUTPUT_FILE" + +# Run evaluation if requested +if [ "$RUN_EVALUATION" = "eval" ]; then + echo "" + echo "======================================" + echo "Running evaluation on results..." + echo "======================================" + echo "" + + if [ -f "$OUTPUT_FILE" ]; then + echo "Evaluating results in: $OUTPUT_FILE" + poetry run python evaluation/benchmarks/math/scripts/summarize_results.py "$OUTPUT_FILE" + + # Save the evaluation results + EVAL_RESULTS_FILE="$OUTPUT_DIR/evaluation_results.txt" + echo "Saving evaluation results to: $EVAL_RESULTS_FILE" + poetry run python evaluation/benchmarks/math/scripts/summarize_results.py "$OUTPUT_FILE" > "$EVAL_RESULTS_FILE" + + echo "" + echo "Evaluation complete. Results saved to: $EVAL_RESULTS_FILE" + else + echo "Error: Output file not found: $OUTPUT_FILE" + echo "Cannot run evaluation." + fi +fi \ No newline at end of file diff --git a/evaluation/benchmarks/math/scripts/summarize_results.py b/evaluation/benchmarks/math/scripts/summarize_results.py new file mode 100755 index 000000000000..0880de840254 --- /dev/null +++ b/evaluation/benchmarks/math/scripts/summarize_results.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +import argparse +import json +import os +from collections import defaultdict + + +def load_jsonl(file_path): + """Load a jsonl file.""" + data = [] + with open(file_path, 'r') as f: + for line in f: + data.append(json.loads(line)) + return data + + +def summarize_results(output_file): + """Summarize the results of the MATH benchmark.""" + print(f"Summarizing results from {output_file}") + + # Load the results + results = load_jsonl(output_file) + + # Count the number of instances that passed and failed + correct = [] + incorrect = [] + + for result in results: + instance_id = result['instance_id'] + test_result = result.get('test_result', {}) + is_correct = test_result.get('is_correct', False) + + if is_correct: + correct.append(instance_id) + else: + incorrect.append(instance_id) + + # Print the summary + print(f"\nTotal instances: {len(results)}") + print(f"Correct: {len(correct)} ({len(correct) / len(results) * 100:.2f}%)") + print(f"Incorrect: {len(incorrect)} ({len(incorrect) / len(results) * 100:.2f}%)") + + # Print the list of correct and incorrect instances + print("\nCorrect instances:") + for instance_id in correct: + print(f" - {instance_id}") + + print("\nIncorrect instances:") + for instance_id in incorrect: + print(f" - {instance_id}") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Summarize MATH benchmark results") + parser.add_argument("output_file", help="Path to the output.jsonl file") + args = parser.parse_args() + + summarize_results(args.output_file) \ No newline at end of file diff --git a/evaluation/benchmarks/polyglot_benchmark/Dockerfile b/evaluation/benchmarks/polyglot_benchmark/Dockerfile new file mode 100644 index 000000000000..ed789e6d8000 --- /dev/null +++ b/evaluation/benchmarks/polyglot_benchmark/Dockerfile @@ -0,0 +1,63 @@ +FROM ubuntu:22.04 + +# Avoid prompts from apt +ENV DEBIAN_FRONTEND=noninteractive + +# Install common dependencies +RUN apt-get update && apt-get install -y \ + build-essential \ + curl \ + git \ + python3 \ + python3-pip \ + python3-dev \ + python3-venv \ + wget \ + software-properties-common \ + apt-transport-https \ + ca-certificates \ + gnupg \ + lsb-release \ + libboost-all-dev \ + cmake \ + && rm -rf /var/lib/apt/lists/* + +# Install Python packages +RUN pip3 install --no-cache-dir pytest pytest-timeout + +# Install Node.js and npm +RUN curl -fsSL https://deb.nodesource.com/setup_18.x | bash - \ + && apt-get install -y nodejs \ + && rm -rf /var/lib/apt/lists/* + +# Install Rust +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y +ENV PATH="/root/.cargo/bin:${PATH}" + +# Install Go +RUN wget https://go.dev/dl/go1.20.5.linux-amd64.tar.gz \ + && tar -C /usr/local -xzf go1.20.5.linux-amd64.tar.gz \ + && rm go1.20.5.linux-amd64.tar.gz +ENV PATH="/usr/local/go/bin:${PATH}" + +# Install Java +RUN apt-get update && apt-get install -y openjdk-17-jdk \ + && rm -rf /var/lib/apt/lists/* +ENV JAVA_HOME=/usr/lib/jvm/java-17-openjdk-amd64 + +# Install Gradle +RUN wget https://services.gradle.org/distributions/gradle-7.6-bin.zip \ + && mkdir /opt/gradle \ + && unzip -d /opt/gradle gradle-7.6-bin.zip \ + && rm gradle-7.6-bin.zip +ENV PATH="/opt/gradle/gradle-7.6/bin:${PATH}" + +# Create workspace directory +RUN mkdir -p /workspace +WORKDIR /workspace + +# Set environment variables +ENV PYTHONUNBUFFERED=1 +ENV PYTHONIOENCODING=UTF-8 + +CMD ["/bin/bash"] \ No newline at end of file diff --git a/evaluation/benchmarks/polyglot_benchmark/README.md b/evaluation/benchmarks/polyglot_benchmark/README.md new file mode 100644 index 000000000000..f5e8ee6a2903 --- /dev/null +++ b/evaluation/benchmarks/polyglot_benchmark/README.md @@ -0,0 +1,207 @@ +# Polyglot Benchmark + +This benchmark is based on the [Aider Polyglot Benchmark](https://github.com/Aider-AI/polyglot-benchmark), which evaluates how effectively an agent can translate natural language coding requests into executable code that passes unit tests across multiple programming languages. + +> **Note**: This benchmark has been modified to use only the same tools as SWE-Bench: +> - execute_bash +> - finish +> - str_replace_editor +> +> This restriction ensures consistent tool usage across benchmarks for more accurate comparisons. + +## Features + +- Supports multiple programming languages (Python, JavaScript, Rust, Go, C++, Java) +- End-to-end evaluation of code editing capabilities +- Automated test execution and validation +- Parallel evaluation with multiple workers +- Detailed metrics and logging + +## Setup + +1. Clone the polyglot-benchmark repository: + ```bash + git clone https://github.com/Aider-AI/polyglot-benchmark.git /workspace/polyglot-benchmark + ``` + +2. Build the Docker image for the benchmark: + ```bash + ./evaluation/benchmarks/polyglot_benchmark/scripts/build_docker.sh + ``` + +## Usage + +1. Make sure you have the required dependencies installed: + ```bash + pip install -e .[dev] + ``` + +2. To test one instance per language (quick verification): + ```bash + # Without evaluation + ./evaluation/benchmarks/polyglot_benchmark/scripts/run_infer.sh --one-per-language --model eval_gpt35_turbo + + # With automatic evaluation + ./evaluation/benchmarks/polyglot_benchmark/scripts/run_infer.sh --one-per-language --model eval_gpt35_turbo --eval + ``` + + This will run one test for each supported language (Python, Rust, Go, JavaScript, C++, and Java) and provide a summary of results. + +3. Run the full benchmark: + ```bash + # Using named arguments (recommended) + ./evaluation/benchmarks/polyglot_benchmark/scripts/run_infer.sh --model eval_gpt35_turbo --agent CodeActAgent --limit 10 --workers 4 --languages python,javascript + + # With automatic evaluation + ./evaluation/benchmarks/polyglot_benchmark/scripts/run_infer.sh --model eval_gpt35_turbo --agent CodeActAgent --limit 10 --workers 4 --languages python,javascript --eval + + # Or using positional arguments (legacy) + ./evaluation/benchmarks/polyglot_benchmark/scripts/run_infer.sh + ``` + +4. Available command-line options: + ``` + --help Show help message + --model MODEL Model configuration (default: eval_gpt4_1106_preview) + --agent AGENT Agent class (default: CodeActAgent) + --limit LIMIT Evaluation limit (default: -1 for all) + --workers WORKERS Number of workers (default: 1) + --ids IDS Comma-separated list of instance IDs + --languages LANGUAGES Comma-separated list of languages + --one-per-language Test one instance per language + --eval Run evaluation after benchmark completes + ``` + +### Command Line Arguments + +- `model_config`: The LLM configuration to use (e.g., `eval_gpt4_1106_preview`) +- `git-version`: Git commit or note to append to output directory (e.g., `HEAD`) +- `agent`: Agent class name (e.g., `CodeActAgent`) +- `eval_limit`: Limit the number of examples to evaluate (default: `-1` for all) +- `eval-num-workers`: Number of parallel workers (default: `1`) +- `eval_ids`: Comma-separated list of specific test IDs to run (e.g., `"1,3,10"`) +- `eval_languages`: Comma-separated list of languages to test (e.g., `"python,javascript,rust"`) + +### Environment Variables + +You can also set the following environment variables: + +```bash +export POLYGLOT_BENCHMARK_PATH="/path/to/polyglot-benchmark" # Path to the polyglot-benchmark repository +export USE_UNIT_TESTS="true" # Whether to run unit tests (default: true) +export NO_DOCKER="true" # Skip Docker container creation and use local runtime (default: false) +export POLYGLOT_DOCKER_IMAGE="image:tag" # Custom Docker image to use (default: ghcr.io/opendevin/eval-polyglot:v1.0.0) +export BUILD_LOCAL_DOCKER="false" # Build a local Docker image if one doesn't exist (default: true) +``` + +### Docker Support + +The benchmark uses Docker to create isolated environments for running code in different programming languages. By default, the script will: + +1. Try to pull the specified Docker image from the registry +2. If the pull fails, automatically build a local Docker image + +You have several options for customizing this behavior: + +#### Option 1: Use the Default Behavior (Recommended) + +Simply run the benchmark script, and it will handle the Docker image automatically: + +```bash +./evaluation/benchmarks/polyglot_benchmark/scripts/run_infer.sh eval_gpt4_1106_preview HEAD CodeActAgent 1 1 +``` + +#### Option 2: Manually Build a Local Docker Image + +You can explicitly build a local Docker image before running the benchmark: + +```bash +# Build the Docker image +./evaluation/benchmarks/polyglot_benchmark/scripts/build_local_docker.sh + +# Run the benchmark with the local image +./evaluation/benchmarks/polyglot_benchmark/scripts/run_infer.sh eval_gpt4_1106_preview HEAD CodeActAgent 1 1 +``` + +#### Option 3: Disable Automatic Docker Image Building + +If you want to disable the automatic building of a Docker image: + +```bash +BUILD_LOCAL_DOCKER=false ./evaluation/benchmarks/polyglot_benchmark/scripts/run_infer.sh eval_gpt4_1106_preview HEAD CodeActAgent 1 1 +``` + +#### Option 4: Use a Custom Docker Image + +You can specify a custom Docker image to use: + +```bash +POLYGLOT_DOCKER_IMAGE="your-custom-image:tag" ./evaluation/benchmarks/polyglot_benchmark/scripts/run_infer.sh eval_gpt4_1106_preview HEAD CodeActAgent 1 1 +``` + +### Troubleshooting + +#### Docker Issues + +If you encounter Docker-related errors like: + +``` +Command 'docker buildx build ...' returned non-zero exit status 1 +``` + +You can try the following solutions: + +1. Build a local Docker image as described above. + +2. Run with `NO_DOCKER=true` to use the local runtime instead: + ```bash + NO_DOCKER=true ./evaluation/benchmarks/polyglot_benchmark/scripts/run_infer.sh eval_gpt4_1106_preview HEAD CodeActAgent 1 1 + ``` + +3. Make sure Docker is installed and running: + ```bash + docker --version + docker ps + ``` + +4. Check if you have permission to use Docker: + ```bash + sudo usermod -aG docker $USER + # Then log out and log back in + ``` + +### Example + +```bash +# Run evaluation on CodeActAgent for all Python instances with 2 workers +export POLYGLOT_BENCHMARK_PATH="/workspace/polyglot-benchmark" +./evaluation/benchmarks/polyglot_benchmark/scripts/run_infer.sh eval_gpt4_1106_preview HEAD CodeActAgent -1 2 "" "python" +``` + +## Summarize Results + +After running the benchmark, you can summarize the results: + +```bash +poetry run python ./evaluation/benchmarks/polyglot_benchmark/scripts/summarize_results.py +``` + +Example: + +```bash +poetry run python ./evaluation/benchmarks/polyglot_benchmark/scripts/summarize_results.py evaluation/evaluation_outputs/outputs/PolyglotBenchmark/CodeActAgent/gpt-4-1106-preview_maxiter_30/output.jsonl +``` + +## Supported Languages + +The benchmark supports the following languages and test frameworks: +- Python: pytest +- JavaScript: npm test +- Rust: cargo test +- Go: go test +- C++: make test +- Java: Gradle test + +## Docker Support + +The benchmark runs in a Docker container to safely execute untrusted code. The container image includes all necessary language toolchains and test frameworks. \ No newline at end of file diff --git a/evaluation/benchmarks/polyglot_benchmark/helper/__init__.py b/evaluation/benchmarks/polyglot_benchmark/helper/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/evaluation/benchmarks/polyglot_benchmark/helper/prompts.py b/evaluation/benchmarks/polyglot_benchmark/helper/prompts.py new file mode 100644 index 000000000000..61bc0e54cb11 --- /dev/null +++ b/evaluation/benchmarks/polyglot_benchmark/helper/prompts.py @@ -0,0 +1,28 @@ +"""Prompts used in the polyglot benchmark.""" + +INSTRUCTIONS_ADDENDUM = """ +I've provided the following files that need to be modified: +{file_list} + +Please help me implement the necessary changes to meet the requirements. +You should ONLY modify these files, and NOT create any new files. +""" + +TEST_FAILURES = """ +The tests failed. Please fix the issues and try again. +Remember to only modify the following files: +{file_list} +""" + +# Dictionary mapping agent class names to their specific instruction suffixes +INST_SUFFIXES = { + 'CodeActAgent': ( + 'REMEMBER: All edits must be made directly in the files. Do NOT send' + ' the edited file as output to the user.\n' + ) +} + +# Dictionary mapping agent class names to their fake response functions +FAKE_RESPONSES = { + 'CodeActAgent': lambda _: None, # Will be replaced with codeact_user_response from shared.py +} \ No newline at end of file diff --git a/evaluation/benchmarks/polyglot_benchmark/run_infer.py b/evaluation/benchmarks/polyglot_benchmark/run_infer.py new file mode 100644 index 000000000000..334a0a769bcc --- /dev/null +++ b/evaluation/benchmarks/polyglot_benchmark/run_infer.py @@ -0,0 +1,548 @@ +import asyncio +import copy +import json +import os +import shutil +import subprocess +import tempfile +from pathlib import Path +from typing import Any, Dict, List, Optional + +# NOTE: This benchmark has been modified to use only the same tools as SWE-Bench: +# - execute_bash +# - finish +# - str_replace_editor + +import pandas as pd + +from evaluation.benchmarks.polyglot_benchmark.helper.prompts import ( + INSTRUCTIONS_ADDENDUM, + INST_SUFFIXES, + TEST_FAILURES, + FAKE_RESPONSES, +) +from evaluation.utils.shared import ( + EvalMetadata, + EvalOutput, + compatibility_for_eval_history_pairs, + make_metadata, + prepare_dataset, + reset_logger_for_multiprocessing, + run_evaluation, + update_llm_config_for_completions_logging, + codeact_user_response, +) +from openhands.controller.state.state import State +from openhands.core.config import ( + AppConfig, + SandboxConfig, + get_llm_config_arg, + load_from_toml, + parse_arguments, +) +from openhands.core.logger import openhands_logger as logger +from openhands.core.main import create_runtime, run_controller +from openhands.events.action import CmdRunAction, MessageAction +from openhands.events.observation import CmdOutputObservation +from openhands.runtime.base import Runtime +from openhands.utils.async_utils import call_async_from_sync + +# Configure visibility of unit tests to the Agent. +USE_UNIT_TESTS = os.environ.get('USE_UNIT_TESTS', 'true').lower() == 'true' + +# Map of file extensions to test commands +TEST_COMMANDS = { + ".py": ["python3", "-m", "pytest"], + ".rs": ["cargo", "test", "--", "--include-ignored"], + ".go": ["go", "test", "./..."], + ".js": ["npm", "test"], + ".cpp": ["make", "test"], + ".java": ["./gradlew", "test"], +} + +# Update fake responses with the actual function +FAKE_RESPONSES['CodeActAgent'] = codeact_user_response + +def get_config( + instance: pd.Series, + metadata: EvalMetadata, +) -> AppConfig: + # Determine runtime type based on environment variable + runtime_type = os.environ.get('RUNTIME', 'docker') + + # Check if NO_DOCKER is set to skip Docker container creation + if os.environ.get('NO_DOCKER', 'false').lower() == 'true': + runtime_type = 'local' + logger.info("Using local runtime instead of Docker due to NO_DOCKER=true") + + config = AppConfig( + default_agent=metadata.agent_class, + run_as_openhands=False, + runtime=runtime_type, + max_iterations=metadata.max_iterations, + sandbox=SandboxConfig( + base_container_image=os.environ.get('POLYGLOT_DOCKER_IMAGE', 'ghcr.io/opendevin/eval-polyglot:v1.0.0'), + enable_auto_lint=True, + use_host_network=False, + timeout=300, # Longer timeout for compilation + api_key=os.environ.get('ALLHANDS_API_KEY', None), + remote_runtime_api_url=os.environ.get('SANDBOX_REMOTE_RUNTIME_API_URL'), + keep_runtime_alive=False, + remote_runtime_init_timeout=1800, + remote_runtime_enable_retries=True, + ), + # do not mount workspace + workspace_base=None, + workspace_mount_path=None, + ) + + # Update llm_config to enable completions logging + llm_config = update_llm_config_for_completions_logging( + metadata.llm_config, + metadata.eval_output_dir, + str(instance.instance_id) + ) + config.set_llm_config(llm_config) + + agent_config = config.get_agent_config(metadata.agent_class) + agent_config.enable_prompt_extensions = False + + # Restrict tools to match SWE-Bench (only execute_bash, finish, and str_replace_editor) + agent_config.codeact_enable_jupyter = False + agent_config.codeact_enable_browsing = False + agent_config.codeact_enable_llm_editor = False + + # copy 'draft_editor' config if exists + config_copy = copy.deepcopy(config) + load_from_toml(config_copy) + if 'draft_editor' in config_copy.llms: + config.set_llm_config(config_copy.llms['draft_editor'], 'draft_editor') + + return config + +def initialize_runtime( + runtime: Runtime, + instance: pd.Series, +): + """Initialize the runtime for the agent.""" + logger.info('-' * 30) + logger.info('BEGIN Runtime Initialization Fn') + logger.info('-' * 30) + obs: CmdOutputObservation + + # Create workspace + action = CmdRunAction(command='mkdir -p /workspace') + logger.info(action, extra={'msg_type': 'ACTION'}) + obs = runtime.run_action(action) + assert obs.exit_code == 0 + + action = CmdRunAction(command='cd /workspace') + logger.info(action, extra={'msg_type': 'ACTION'}) + obs = runtime.run_action(action) + assert obs.exit_code == 0 + + # Copy files to workspace + with tempfile.TemporaryDirectory() as tmpdir: + # Copy solution files + for file_path in instance.solution_files: + file_path = Path(file_path) + temp_file = Path(tmpdir) / file_path.name + with open(temp_file, 'w') as f: + f.write(instance.solution_content[file_path.name]) + runtime.copy_to( + str(temp_file), + '/workspace', + ) + + # Copy test files if enabled + if USE_UNIT_TESTS: + for file_path in instance.test_files: + file_path = Path(file_path) + temp_file = Path(tmpdir) / file_path.name + with open(temp_file, 'w') as f: + f.write(instance.test_content[file_path.name]) + runtime.copy_to( + str(temp_file), + '/workspace', + ) + + logger.info('-' * 30) + logger.info('END Runtime Initialization Fn') + logger.info('-' * 30) + +def complete_runtime( + runtime: Runtime, + instance: pd.Series, +) -> Dict[str, Any]: + """Complete the runtime for the agent.""" + logger.info('-' * 30) + logger.info('BEGIN Runtime Completion Fn') + logger.info('-' * 30) + + # Run tests + test_output = "" + exit_code = 1 + + if USE_UNIT_TESTS: + # Get unique file extensions from test files + extensions = {Path(f).suffix for f in instance.test_files} + + # Find matching test command + command = None + for ext in extensions: + if ext in TEST_COMMANDS: + command = TEST_COMMANDS[ext] + break + + if command: + try: + # Use the runtime to run the command inside the Docker container + cmd_str = " ".join(command) + logger.info(f"Running test command: {cmd_str}") + + action = CmdRunAction(command=cmd_str) + logger.info(action, extra={'msg_type': 'ACTION'}) + + obs = runtime.run_action(action) + logger.info(obs, extra={'msg_type': 'OBSERVATION'}) + + if isinstance(obs, CmdOutputObservation): + exit_code = obs.exit_code + test_output = obs.content + else: + logger.error(f"Unexpected observation type: {type(obs)}") + exit_code = 1 + test_output = f"Error: Unexpected observation type: {type(obs)}" + + # Clean up output + test_output = test_output.replace("/workspace", "workspace") + + # Log test output to history file + with tempfile.TemporaryDirectory() as tmpdir: + history_path = os.path.join(tmpdir, ".aider.chat.history.md") + with open(history_path, 'w') as f: + f.write(f"```\n{test_output}\n```") + runtime.copy_to( + history_path, + '/workspace', + ) + + except Exception as e: + logger.error(f"Error running tests: {e}") + test_output = f"Tests failed with error: {e}" + exit_code = 1 + + logger.info('-' * 30) + logger.info('END Runtime Completion Fn') + logger.info('-' * 30) + + runtime.close() + + return { + 'test_output': test_output, + 'exit_code': exit_code, + } + +def process_instance( + instance: pd.Series, + metadata: EvalMetadata, + reset_logger: bool = True, +) -> EvalOutput: + config = get_config(instance, metadata) + + # Setup the logger properly, so you can run multi-processing to parallelize the evaluation + if reset_logger: + log_dir = os.path.join(metadata.eval_output_dir, 'infer_logs') + reset_logger_for_multiprocessing(logger, str(instance.instance_id), log_dir) + else: + logger.info( + f'\nStarting evaluation for instance {str(instance.instance_id)}.\n' + ) + + # ============================================= + # build instruction + # ============================================= + + # Prepare instruction + logger.info(instance) + instruction = instance.instruction + + # Add file list to instruction + file_list = " ".join(instance.solution_files) + instruction += INSTRUCTIONS_ADDENDUM.format(file_list=file_list) + + if USE_UNIT_TESTS: + test_files = " ".join(instance.test_files) + logger.info(f'\nTest files: {test_files}\n') + instruction += ( + f'Use the appropriate test command to run the tests and verify your solution. ' + 'DO NOT EDIT the test files.\n\n' + ) + + instruction += ( + 'IMPORTANT: You should ONLY interact with the environment provided ' + 'to you AND NEVER ASK FOR HUMAN HELP.\n' + ) + + # Add agent-specific instruction suffix + if metadata.agent_class in INST_SUFFIXES: + instruction += INST_SUFFIXES[metadata.agent_class] + + # ============================================= + # create sandbox and run the agent + # ============================================= + + runtime: Runtime = create_runtime(config) + call_async_from_sync(runtime.connect) + + initialize_runtime(runtime, instance=instance) + + # Here's how you can run the agent (similar to the `main` function) and get the final task state + state: State | None = asyncio.run( + run_controller( + config=config, + initial_user_action=MessageAction(content=instruction), + runtime=runtime, + fake_user_response_fn=FAKE_RESPONSES[metadata.agent_class], + ) + ) + if state is None: + raise ValueError('State should not be None.') + + # ============================================= + # result evaluation + # ============================================= + + return_val = complete_runtime(runtime, instance) + exit_code = return_val['exit_code'] + test_output = return_val['test_output'] + + errors = [] + test_cases = None + if test_output: + if 'SyntaxError' in test_output: + errors.append('SyntaxError') + elif 'IndentationError' in test_output: + errors.append('IndentationError') + else: + test_cases = test_output + + test_result = { + 'exit_code': exit_code, + 'test_cases': test_cases, + 'errors': errors, + } + + # history is now available as a stream of events, rather than list of pairs of (Action, Observation) + # for compatibility with the existing output format, we can remake the pairs here + histories = compatibility_for_eval_history_pairs(state.history) + metrics = state.metrics.get() if state.metrics else None + + # Save the output + output = EvalOutput( + instance_id=str(instance.instance_id), + instance=instance.to_dict(), + instruction=instruction, + metadata=metadata, + history=histories, + metrics=metrics, + error=state.last_error if state and state.last_error else None, + test_result=test_result, + ) + return output + +def load_polyglot_dataset(): + """Load the polyglot benchmark dataset from the repository.""" + import glob + import json + import os + from pathlib import Path + + # Try to find the polyglot-benchmark repository + # First check the environment variable + repo_path = os.environ.get('POLYGLOT_BENCHMARK_PATH') + + # If not set, try common locations + if not repo_path or not os.path.exists(repo_path): + possible_paths = [ + '/workspace/polyglot-benchmark', + str(Path.home() / 'polyglot-benchmark'), + str(Path.home() / 'thereal' / 'polyglot-benchmark'), + str(Path(__file__).parent.parent.parent.parent.parent / 'polyglot-benchmark'), + str(Path.cwd() / 'polyglot-benchmark'), + ] + + for path in possible_paths: + if os.path.exists(path): + repo_path = path + logger.info(f"Found polyglot-benchmark repository at: {repo_path}") + break + + if not repo_path or not os.path.exists(repo_path): + logger.error("Could not find polyglot-benchmark repository. Please set POLYGLOT_BENCHMARK_PATH environment variable.") + return pd.DataFrame() + + all_tests = [] + instance_id = 0 + + # Process each language directory + for lang_dir in ['python', 'javascript', 'rust', 'go', 'cpp', 'java']: + lang_path = os.path.join(repo_path, lang_dir, 'exercises', 'practice') + if not os.path.exists(lang_path): + logger.warning(f"Language directory not found: {lang_path}") + continue + + # Process each exercise directory + for exercise_dir in os.listdir(lang_path): + exercise_path = os.path.join(lang_path, exercise_dir) + if not os.path.isdir(exercise_path): + continue + + # Check for config.json + config_file = os.path.join(exercise_path, '.meta', 'config.json') + if not os.path.exists(config_file): + logger.warning(f"Config file not found: {config_file}") + continue + + # Load config + with open(config_file, 'r') as f: + config = json.load(f) + + # Get solution and test files + solution_files = config.get('files', {}).get('solution', []) + test_files = config.get('files', {}).get('test', []) + + if not solution_files or not test_files: + logger.warning(f"Missing solution or test files in {exercise_path}") + continue + + # Load instructions + instruction = "" + intro_file = os.path.join(exercise_path, '.docs', 'introduction.md') + if os.path.exists(intro_file): + with open(intro_file, 'r') as f: + instruction += f.read() + "\n\n" + + instructions_file = os.path.join(exercise_path, '.docs', 'instructions.md') + if os.path.exists(instructions_file): + with open(instructions_file, 'r') as f: + instruction += f.read() + "\n\n" + + if not instruction: + logger.warning(f"No instructions found for {exercise_path}") + continue + + # Load solution and test content + solution_content = {} + for file_path in solution_files: + full_path = os.path.join(exercise_path, file_path) + if os.path.exists(full_path): + with open(full_path, 'r') as f: + solution_content[os.path.basename(file_path)] = f.read() + + test_content = {} + for file_path in test_files: + full_path = os.path.join(exercise_path, file_path) + if os.path.exists(full_path): + with open(full_path, 'r') as f: + test_content[os.path.basename(file_path)] = f.read() + + # Create test instance + test_instance = { + 'instance_id': instance_id, + 'instance_name': exercise_dir, + 'language': lang_dir, + 'instruction': instruction, + 'solution_files': [os.path.basename(f) for f in solution_files], + 'test_files': [os.path.basename(f) for f in test_files], + 'solution_content': solution_content, + 'test_content': test_content, + } + + all_tests.append(test_instance) + instance_id += 1 + + return pd.DataFrame(all_tests) + +def add_arguments(parser): + """Add polyglot benchmark specific arguments to the parser.""" + parser.add_argument( + '--eval-languages', + type=str, + help='Comma-separated list of languages to test (e.g., "python,javascript,rust")', + ) + return parser + +if __name__ == '__main__': + # Get the argument parser and add custom arguments + import argparse + from openhands.core.config import get_parser + + parser = get_parser() + add_arguments(parser) + args = parse_arguments() + + # Load the polyglot benchmark dataset + polyglot_tests = load_polyglot_dataset() + + if polyglot_tests.empty: + logger.error("Failed to load polyglot benchmark dataset") + exit(1) + + logger.info(f"Loaded {len(polyglot_tests)} test instances from polyglot benchmark") + + llm_config = None + if args.llm_config: + llm_config = get_llm_config_arg(args.llm_config) + # modify_params must be False for evaluation purpose, for reproducibility and accuracy of results + llm_config.modify_params = False + + if llm_config is None: + raise ValueError(f'Could not find LLM config: --llm_config {args.llm_config}') + + # Create details dictionary with agent configuration + agent_details = { + "agent_config": { + "codeact_enable_jupyter": False, + "codeact_enable_browsing": False, + "codeact_enable_llm_editor": False, + } + } + + metadata = make_metadata( + llm_config, + 'PolyglotBenchmark', + args.agent_cls, + args.max_iterations, + args.eval_note, + args.eval_output_dir, + details=agent_details, + ) + output_file = os.path.join(metadata.eval_output_dir, 'output.jsonl') + + # Parse dataset IDs if provided + eval_ids = None + if args.eval_ids: + eval_ids = str(args.eval_ids).split(',') + logger.info(f'\nUsing specific dataset IDs: {eval_ids}\n') + + # Filter by language if specified + if hasattr(args, 'eval_languages') and args.eval_languages: + languages = [lang.strip().lower() for lang in args.eval_languages.split(',')] + polyglot_tests = polyglot_tests[polyglot_tests['language'].str.lower().isin(languages)] + logger.info(f'\nFiltered to languages: {languages}, {len(polyglot_tests)} instances remaining\n') + + instances = prepare_dataset( + polyglot_tests, + output_file, + args.eval_n_limit, + eval_ids=eval_ids, + ) + + run_evaluation( + instances, + metadata, + output_file, + args.eval_num_workers, + process_instance, + ) \ No newline at end of file diff --git a/evaluation/benchmarks/polyglot_benchmark/scripts/build_docker.sh b/evaluation/benchmarks/polyglot_benchmark/scripts/build_docker.sh new file mode 100755 index 000000000000..1c6a2dfff7a1 --- /dev/null +++ b/evaluation/benchmarks/polyglot_benchmark/scripts/build_docker.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +set -e + +# Get the directory of this script +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +BENCHMARK_DIR="$( cd "${SCRIPT_DIR}/.." && pwd )" + +# Build the Docker image +docker build -t ghcr.io/opendevin/eval-polyglot:v1.0.0 -f "${BENCHMARK_DIR}/Dockerfile" "${BENCHMARK_DIR}" + +echo "Docker image built successfully: ghcr.io/opendevin/eval-polyglot:v1.0.0" \ No newline at end of file diff --git a/evaluation/benchmarks/polyglot_benchmark/scripts/build_local_docker.sh b/evaluation/benchmarks/polyglot_benchmark/scripts/build_local_docker.sh new file mode 100755 index 000000000000..0f93c82164a0 --- /dev/null +++ b/evaluation/benchmarks/polyglot_benchmark/scripts/build_local_docker.sh @@ -0,0 +1,93 @@ +#!/bin/bash + +set -e + +# Get the directory of this script +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +BENCHMARK_DIR="$( cd "${SCRIPT_DIR}/.." && pwd )" +REPO_ROOT="$( cd "${BENCHMARK_DIR}/../../.." && pwd )" + +# Create a temporary directory for the Docker build +BUILD_DIR=$(mktemp -d) +trap "rm -rf $BUILD_DIR" EXIT + +echo "Creating Docker build context in $BUILD_DIR" + +# Create a simple Dockerfile that includes all the necessary tools +cat > "$BUILD_DIR/Dockerfile" << 'EOF' +FROM ubuntu:22.04 + +# Avoid prompts from apt +ENV DEBIAN_FRONTEND=noninteractive + +# Install common dependencies +RUN apt-get update && apt-get install -y \ + build-essential \ + curl \ + git \ + python3 \ + python3-pip \ + python3-dev \ + python3-venv \ + wget \ + unzip \ + zip \ + software-properties-common \ + apt-transport-https \ + ca-certificates \ + gnupg \ + lsb-release \ + libboost-all-dev \ + cmake \ + && rm -rf /var/lib/apt/lists/* + +# Install Python packages +RUN pip3 install --no-cache-dir pytest pytest-timeout + +# Install Node.js and npm +RUN curl -fsSL https://deb.nodesource.com/setup_18.x | bash - \ + && apt-get install -y nodejs \ + && rm -rf /var/lib/apt/lists/* + +# Install Rust +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y +ENV PATH="/root/.cargo/bin:${PATH}" + +# Install Go +RUN wget https://go.dev/dl/go1.20.5.linux-amd64.tar.gz \ + && tar -C /usr/local -xzf go1.20.5.linux-amd64.tar.gz \ + && rm go1.20.5.linux-amd64.tar.gz +ENV PATH="/usr/local/go/bin:${PATH}" + +# Install Java +RUN apt-get update && apt-get install -y openjdk-17-jdk \ + && rm -rf /var/lib/apt/lists/* +ENV JAVA_HOME=/usr/lib/jvm/java-17-openjdk-amd64 + +# Install Gradle +RUN apt-get update && apt-get install -y gradle \ + && rm -rf /var/lib/apt/lists/* + +# Create workspace directory +RUN mkdir -p /workspace +WORKDIR /workspace + +# Set environment variables +ENV PYTHONUNBUFFERED=1 +ENV PYTHONIOENCODING=UTF-8 + +CMD ["/bin/bash"] +EOF + +# Build the Docker image +IMAGE_NAME="polyglot-benchmark:local" +echo "Building Docker image: $IMAGE_NAME" +docker build -t "$IMAGE_NAME" "$BUILD_DIR" + +# Export the image name as an environment variable +echo "export POLYGLOT_DOCKER_IMAGE=$IMAGE_NAME" > "$BENCHMARK_DIR/docker_image.env" + +echo "Docker image built successfully: $IMAGE_NAME" +echo "To use this image, run:" +echo "source $BENCHMARK_DIR/docker_image.env" +echo "Then run the benchmark as usual." \ No newline at end of file diff --git a/evaluation/benchmarks/polyglot_benchmark/scripts/run_infer.sh b/evaluation/benchmarks/polyglot_benchmark/scripts/run_infer.sh new file mode 100755 index 000000000000..757cee5ac3bb --- /dev/null +++ b/evaluation/benchmarks/polyglot_benchmark/scripts/run_infer.sh @@ -0,0 +1,330 @@ +#!/bin/bash + +set -e + +# Display usage information +function show_usage { + echo "Usage: $0 [options]" + echo "" + echo "Options:" + echo " --help Show this help message" + echo " --model MODEL Model configuration (default: eval_gpt4_1106_preview)" + echo " --agent AGENT Agent class (default: CodeActAgent)" + echo " --limit LIMIT Evaluation limit (default: -1 for all)" + echo " --workers WORKERS Number of workers (default: 1)" + echo " --ids IDS Comma-separated list of instance IDs" + echo " --languages LANGUAGES Comma-separated list of languages" + echo " --one-per-language Test one instance per language" + echo " --eval Run evaluation after benchmark" + echo "" + echo "Legacy positional arguments are still supported:" + echo " $0 MODEL_CONFIG GIT_VERSION AGENT EVAL_LIMIT EVAL_NUM_WORKERS EVAL_IDS EVAL_LANGUAGES" + exit 0 +} + +# Parse named arguments +ONE_PER_LANGUAGE=false +RUN_EVALUATION=false +POSITIONAL_ARGS=() + +while [[ $# -gt 0 ]]; do + case $1 in + --help) + show_usage + ;; + --model) + MODEL_CONFIG="$2" + shift 2 + ;; + --agent) + AGENT="$2" + shift 2 + ;; + --limit) + EVAL_LIMIT="$2" + shift 2 + ;; + --workers) + EVAL_NUM_WORKERS="$2" + shift 2 + ;; + --ids) + EVAL_IDS="$2" + shift 2 + ;; + --languages) + EVAL_LANGUAGES="$2" + shift 2 + ;; + --one-per-language) + ONE_PER_LANGUAGE=true + shift + ;; + --eval) + RUN_EVALUATION=true + shift + ;; + eval) + # Special case for the 'eval' parameter in the positional arguments + RUN_EVALUATION=true + shift + ;; + *) + POSITIONAL_ARGS+=("$1") + shift + ;; + esac +done + +# Restore positional parameters +set -- "${POSITIONAL_ARGS[@]}" + +# Default values (if not set by named arguments) +MODEL_CONFIG=${MODEL_CONFIG:-${1:-"eval_gpt4_1106_preview"}} +GIT_VERSION=${2:-"HEAD"} +AGENT=${AGENT:-${3:-"CodeActAgent"}} +EVAL_LIMIT=${EVAL_LIMIT:-${4:-"-1"}} +EVAL_NUM_WORKERS=${EVAL_NUM_WORKERS:-${5:-"1"}} +EVAL_IDS=${EVAL_IDS:-${6:-""}} +EVAL_LANGUAGES=${EVAL_LANGUAGES:-${7:-""}} + +# Set environment variables +export USE_UNIT_TESTS=${USE_UNIT_TESTS:-"true"} +export NO_DOCKER=${NO_DOCKER:-"false"} + +# Check if we have a local Docker image env file +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +BENCHMARK_DIR="$( cd "${SCRIPT_DIR}/.." && pwd )" +DOCKER_ENV_FILE="${BENCHMARK_DIR}/docker_image.env" + +# Set BUILD_LOCAL_DOCKER to true by default if not specified +export BUILD_LOCAL_DOCKER=${BUILD_LOCAL_DOCKER:-"true"} + +if [ -f "$DOCKER_ENV_FILE" ]; then + echo "Loading Docker image configuration from $DOCKER_ENV_FILE" + source "$DOCKER_ENV_FILE" +else + # If no local image is available, use the default + export POLYGLOT_DOCKER_IMAGE=${POLYGLOT_DOCKER_IMAGE:-"ghcr.io/opendevin/eval-polyglot:v1.0.0"} + + # Try to pull the image first + echo "Trying to pull Docker image: $POLYGLOT_DOCKER_IMAGE" + if ! docker pull "$POLYGLOT_DOCKER_IMAGE" 2>/dev/null; then + echo "Failed to pull Docker image: $POLYGLOT_DOCKER_IMAGE" + + # Build a local Docker image if pulling fails and BUILD_LOCAL_DOCKER is true + if [ "$BUILD_LOCAL_DOCKER" = "true" ]; then + echo "Building local Docker image..." + "${SCRIPT_DIR}/build_local_docker.sh" + source "$DOCKER_ENV_FILE" + else + echo "WARNING: Docker image not found and BUILD_LOCAL_DOCKER is not set to true." + echo "You can build a local Docker image by running:" + echo " ${SCRIPT_DIR}/build_local_docker.sh" + echo "Or set BUILD_LOCAL_DOCKER=true to build it automatically." + fi + else + echo "Successfully pulled Docker image: $POLYGLOT_DOCKER_IMAGE" + fi +fi + +echo "Using Docker image: $POLYGLOT_DOCKER_IMAGE" + +# Try to find the polyglot-benchmark repository +if [ -z "$POLYGLOT_BENCHMARK_PATH" ]; then + # Check common locations + POSSIBLE_PATHS=( + "/workspace/polyglot-benchmark" + "$HOME/polyglot-benchmark" + "$HOME/thereal/polyglot-benchmark" + "$(git rev-parse --show-toplevel)/polyglot-benchmark" + "$(pwd)/polyglot-benchmark" + ) + + for path in "${POSSIBLE_PATHS[@]}"; do + if [ -d "$path" ]; then + export POLYGLOT_BENCHMARK_PATH="$path" + echo "Found polyglot-benchmark repository at: $POLYGLOT_BENCHMARK_PATH" + break + fi + done +fi + +# If still not found, try to clone it +if [ -z "$POLYGLOT_BENCHMARK_PATH" ] || [ ! -d "$POLYGLOT_BENCHMARK_PATH" ]; then + echo "Polyglot benchmark repository not found. Attempting to clone it..." + CLONE_DIR="$(git rev-parse --show-toplevel)/polyglot-benchmark" + git clone https://github.com/Aider-AI/polyglot-benchmark.git "$CLONE_DIR" + if [ $? -eq 0 ]; then + export POLYGLOT_BENCHMARK_PATH="$CLONE_DIR" + echo "Successfully cloned polyglot-benchmark to $POLYGLOT_BENCHMARK_PATH" + else + echo "Failed to clone polyglot-benchmark. Please set POLYGLOT_BENCHMARK_PATH manually." + exit 1 + fi +fi + +# Add additional arguments based on provided parameters +ARGS="--agent-cls ${AGENT} --llm-config ${MODEL_CONFIG} --max-iterations 30 --eval-num-workers ${EVAL_NUM_WORKERS}" + +if [ "${EVAL_LIMIT}" != "-1" ]; then + ARGS="${ARGS} --eval-n-limit ${EVAL_LIMIT}" +fi + +# Only pass eval-ids if it's not "eval" (which is a special parameter for evaluation mode) +if [ -n "${EVAL_IDS}" ] && [ "${EVAL_IDS}" != "eval" ]; then + ARGS="${ARGS} --eval-ids ${EVAL_IDS}" +fi + +if [ -n "${EVAL_LANGUAGES}" ]; then + ARGS="${ARGS} --eval-languages ${EVAL_LANGUAGES}" +fi + +# Change to the repository root directory +cd "$(git rev-parse --show-toplevel)" + +# If one-per-language mode is enabled +if [ "$ONE_PER_LANGUAGE" = true ]; then + echo "Running one instance per language mode..." + + # Define the languages to test + LANGUAGES=("python" "javascript" "rust" "go" "cpp" "java") + + # Create a temporary directory for results + RESULTS_DIR="evaluation/evaluation_outputs/one_per_language_test" + mkdir -p "$RESULTS_DIR" + + # Summary file + SUMMARY_FILE="$RESULTS_DIR/summary.txt" + echo "POLYGLOT BENCHMARK - ONE INSTANCE PER LANGUAGE TEST" > "$SUMMARY_FILE" + echo "=================================================" >> "$SUMMARY_FILE" + echo "Model: $MODEL_CONFIG" >> "$SUMMARY_FILE" + echo "Agent: $AGENT" >> "$SUMMARY_FILE" + echo "Date: $(date)" >> "$SUMMARY_FILE" + echo "=================================================" >> "$SUMMARY_FILE" + echo "" >> "$SUMMARY_FILE" + + # Test each language + for LANG in "${LANGUAGES[@]}"; do + echo "" + echo "===== Testing language: $LANG =====" + echo "" + + # Run with one instance for this language + LANG_ARGS="--agent-cls ${AGENT} --llm-config ${MODEL_CONFIG} --max-iterations 30 --eval-num-workers 1 --eval-n-limit 1 --eval-languages ${LANG} --eval-note one_per_language_${LANG}" + + # Run the evaluation for this language + if poetry run python -m evaluation.benchmarks.polyglot_benchmark.run_infer ${LANG_ARGS}; then + RESULT="PASSED" + else + RESULT="FAILED" + fi + + # Add to summary + echo "${LANG}: ${RESULT}" >> "$SUMMARY_FILE" + done + + # Display summary + echo "" + echo "===== TEST SUMMARY =====" + cat "$SUMMARY_FILE" + echo "" + echo "Detailed results available in: $RESULTS_DIR" + + # Run evaluation if requested + if [ "$RUN_EVALUATION" = true ]; then + echo "" + echo "======================================" + echo "Running detailed evaluation on results..." + echo "======================================" + echo "" + + # Evaluate each language's results + for LANG in "${LANGUAGES[@]}"; do + # Try to find the output directory for this language + LANG_OUTPUT_DIR=$(find evaluation/evaluation_outputs -type d -name "*one_per_language_${LANG}*" 2>/dev/null | sort -r | head -n 1) + + if [ -z "$LANG_OUTPUT_DIR" ]; then + LANG_OUTPUT_DIR=$(find . -path "*/evaluation_outputs/*" -type d -name "*one_per_language_${LANG}*" 2>/dev/null | sort -r | head -n 1) + fi + + if [ -z "$LANG_OUTPUT_DIR" ]; then + LANG_OUTPUT_DIR="evaluation/evaluation_outputs/one_per_language_${LANG}" + fi + + LANG_OUTPUT_FILE="${LANG_OUTPUT_DIR}/output.jsonl" + + # Print the language output directory and file for debugging + echo "" + echo "Language: $LANG" + echo "Output directory: $LANG_OUTPUT_DIR" + echo "Output file: $LANG_OUTPUT_FILE" + + if [ -f "$LANG_OUTPUT_FILE" ]; then + echo "" + echo "===== Evaluating $LANG results =====" + echo "" + echo "Evaluating results in: $LANG_OUTPUT_FILE" + + # Save the evaluation results + EVAL_RESULTS_FILE="${LANG_OUTPUT_DIR}/evaluation_results.txt" + echo "Saving evaluation results to: $EVAL_RESULTS_FILE" + poetry run python evaluation/benchmarks/polyglot_benchmark/scripts/summarize_results.py "$LANG_OUTPUT_FILE" > "$EVAL_RESULTS_FILE" + fi + done + + echo "" + echo "Detailed evaluation complete." + fi +else + # Run the normal evaluation + poetry run python -m evaluation.benchmarks.polyglot_benchmark.run_infer ${ARGS} + + # Run evaluation if requested + if [ "$RUN_EVALUATION" = true ]; then + echo "" + echo "======================================" + echo "Running evaluation on results..." + echo "======================================" + echo "" + + # Get the output directory - first try the default location + OUTPUT_DIR=$(find evaluation/evaluation_outputs -path "*/PolyglotBenchmark/$AGENT/*" -type d -name "*tools_bash+finish+str_replace*" 2>/dev/null | sort -r | head -n 1) + + # If not found, try to find it anywhere under evaluation_outputs + if [ -z "$OUTPUT_DIR" ]; then + OUTPUT_DIR=$(find . -path "*/evaluation_outputs/*" -path "*/PolyglotBenchmark/$AGENT/*" -type d -name "*tools_bash+finish+str_replace*" 2>/dev/null | sort -r | head -n 1) + fi + + # If still not found, try to find any output.jsonl file + if [ -z "$OUTPUT_DIR" ]; then + OUTPUT_FILE=$(find . -name "output.jsonl" 2>/dev/null | sort -r | head -n 1) + if [ -n "$OUTPUT_FILE" ]; then + OUTPUT_DIR=$(dirname "$OUTPUT_FILE") + fi + else + OUTPUT_FILE="$OUTPUT_DIR/output.jsonl" + fi + + # Print the output directory and file for debugging + echo "" + echo "Output directory: $OUTPUT_DIR" + echo "Output file: $OUTPUT_FILE" + + if [ -f "$OUTPUT_FILE" ]; then + echo "Evaluating results in: $OUTPUT_FILE" + poetry run python evaluation/benchmarks/polyglot_benchmark/scripts/summarize_results.py "$OUTPUT_FILE" + + # Save the evaluation results + EVAL_RESULTS_FILE="$OUTPUT_DIR/evaluation_results.txt" + echo "Saving evaluation results to: $EVAL_RESULTS_FILE" + poetry run python evaluation/benchmarks/polyglot_benchmark/scripts/summarize_results.py "$OUTPUT_FILE" > "$EVAL_RESULTS_FILE" + + echo "" + echo "Evaluation complete. Results saved to: $EVAL_RESULTS_FILE" + else + echo "Error: Output file not found: $OUTPUT_FILE" + echo "Cannot run evaluation." + fi + fi +fi \ No newline at end of file diff --git a/evaluation/benchmarks/polyglot_benchmark/scripts/summarize_results.py b/evaluation/benchmarks/polyglot_benchmark/scripts/summarize_results.py new file mode 100755 index 000000000000..988f3a618bff --- /dev/null +++ b/evaluation/benchmarks/polyglot_benchmark/scripts/summarize_results.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 + +import argparse +import json +import os +from collections import defaultdict + +def load_jsonl(file_path): + """Load data from a jsonl file.""" + data = [] + with open(file_path, 'r') as f: + for line in f: + data.append(json.loads(line)) + return data + +def summarize_results(output_file): + """Summarize the results of the polyglot benchmark evaluation.""" + if not os.path.exists(output_file): + print(f"Error: Output file {output_file} does not exist.") + return + + results = load_jsonl(output_file) + + # Count total instances + total_instances = len(results) + print(f"Total instances: {total_instances}") + + # Count by language + language_counts = defaultdict(int) + language_passed = defaultdict(int) + + # Count passed and failed instances + passed_instances = [] + failed_instances = [] + + for result in results: + instance = result.get('instance', {}) + language = instance.get('language', 'unknown') + instance_name = instance.get('instance_name', 'unknown') + instance_id = result.get('instance_id', 'unknown') + + language_counts[language] += 1 + + # Check if all tests passed + test_result = result.get('test_result', {}) + exit_code = test_result.get('exit_code', 1) + + if exit_code == 0: + passed_instances.append((instance_id, language, instance_name)) + language_passed[language] += 1 + else: + failed_instances.append((instance_id, language, instance_name)) + + # Print summary + print("\nResults by language:") + print("--------------------") + for language, count in sorted(language_counts.items()): + passed = language_passed[language] + percentage = (passed / count) * 100 if count > 0 else 0 + print(f"{language}: {passed}/{count} ({percentage:.1f}%)") + + # Overall pass rate + total_passed = len(passed_instances) + overall_percentage = (total_passed / total_instances) * 100 if total_instances > 0 else 0 + print(f"\nOverall pass rate: {total_passed}/{total_instances} ({overall_percentage:.1f}%)") + + # Print passed instances + print("\nPassed instances:") + print("----------------") + for instance_id, language, instance_name in sorted(passed_instances): + print(f"ID: {instance_id}, Language: {language}, Name: {instance_name}") + + # Print failed instances + print("\nFailed instances:") + print("----------------") + for instance_id, language, instance_name in sorted(failed_instances): + print(f"ID: {instance_id}, Language: {language}, Name: {instance_name}") + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Summarize polyglot benchmark results") + parser.add_argument("output_file", help="Path to the output.jsonl file") + args = parser.parse_args() + + summarize_results(args.output_file) \ No newline at end of file diff --git a/evaluation/benchmarks/polyglot_benchmark/test_all_languages.py b/evaluation/benchmarks/polyglot_benchmark/test_all_languages.py new file mode 100755 index 000000000000..f196651b890d --- /dev/null +++ b/evaluation/benchmarks/polyglot_benchmark/test_all_languages.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 + +import os +import sys +import argparse +from pathlib import Path + +# Add the parent directory to the Python path +sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent)) + +from evaluation.benchmarks.polyglot_benchmark.run_infer import ( + load_polyglot_dataset, + process_instance, + make_metadata, + get_llm_config_arg, +) +from openhands.core.logger import openhands_logger as logger + +def test_language(language, model, agent): + """Test the first instance of a specific language.""" + print(f"\n{'=' * 50}") + print(f"Testing language: {language}") + print(f"{'=' * 50}\n") + + # Set the environment variable for the polyglot benchmark path + os.environ['POLYGLOT_BENCHMARK_PATH'] = '/workspace/polyglot-benchmark' + + # Load the dataset + dataset = load_polyglot_dataset() + + # Filter by language + dataset = dataset[dataset['language'].str.lower() == language.lower()] + if dataset.empty: + print(f"No instances found for language: {language}") + return False + + # Get the first instance + instance = dataset.iloc[0] + print(f"Testing instance {instance.instance_id}: {instance.instance_name} ({instance.language})") + + # Get LLM config + llm_config = get_llm_config_arg(model) + if llm_config is None: + print(f"Could not find LLM config: {model}") + return False + + # Create details dictionary with agent configuration + agent_details = { + "agent_config": { + "codeact_enable_jupyter": False, + "codeact_enable_browsing": False, + "codeact_enable_llm_editor": False, + } + } + + # Create metadata + metadata = make_metadata( + llm_config, + 'PolyglotBenchmark', + agent, + 30, # max_iterations + f"test_{language}", + f"evaluation/evaluation_outputs/test_{language}", + details=agent_details, + ) + + # Process the instance + try: + output = process_instance(instance, metadata, reset_logger=False) + print("\nTest completed successfully!") + print(f"Exit code: {output.test_result['exit_code']}") + print(f"Passed: {output.test_result['exit_code'] == 0}") + return output.test_result['exit_code'] == 0 + except Exception as e: + print(f"Error processing instance: {e}") + return False + +def main(): + parser = argparse.ArgumentParser(description="Test the polyglot benchmark with one instance per language") + parser.add_argument("--model", default="eval_gpt35_turbo", help="Model configuration name") + parser.add_argument("--agent", default="CodeActAgent", help="Agent class name") + parser.add_argument("--languages", default="python,rust,go,javascript,cpp,java", + help="Comma-separated list of languages to test") + args = parser.parse_args() + + languages = args.languages.split(',') + results = {} + + for language in languages: + language = language.strip() + if not language: + continue + + success = test_language(language, args.model, args.agent) + results[language] = "PASSED" if success else "FAILED" + + # Print summary + print("\n" + "=" * 50) + print("SUMMARY OF RESULTS") + print("=" * 50) + + for language, result in results.items(): + print(f"{language.ljust(12)}: {result}") + + # Check if all tests passed + all_passed = all(result == "PASSED" for result in results.values()) + print("\nOverall result:", "PASSED" if all_passed else "FAILED") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/evaluation/benchmarks/polyglot_benchmark/test_load_dataset.py b/evaluation/benchmarks/polyglot_benchmark/test_load_dataset.py new file mode 100755 index 000000000000..708259732b02 --- /dev/null +++ b/evaluation/benchmarks/polyglot_benchmark/test_load_dataset.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 + +import os +import sys +from pathlib import Path + +# Add the parent directory to the Python path +sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent)) + +from evaluation.benchmarks.polyglot_benchmark.run_infer import load_polyglot_dataset + +def main(): + # Set the environment variable for the polyglot benchmark path + os.environ['POLYGLOT_BENCHMARK_PATH'] = '/workspace/polyglot-benchmark' + + # Load the dataset + dataset = load_polyglot_dataset() + + # Print summary + print(f"Loaded {len(dataset)} test instances") + + # Print language distribution + language_counts = dataset['language'].value_counts() + print("\nLanguage distribution:") + for language, count in language_counts.items(): + print(f"{language}: {count}") + + # Print a sample instance + if not dataset.empty: + print("\nSample instance:") + sample = dataset.iloc[0] + print(f"ID: {sample.instance_id}") + print(f"Name: {sample.instance_name}") + print(f"Language: {sample.language}") + print(f"Solution files: {sample.solution_files}") + print(f"Test files: {sample.test_files}") + print(f"Instruction (first 100 chars): {sample.instruction[:100]}...") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/evaluation/benchmarks/polyglot_benchmark/test_run.py b/evaluation/benchmarks/polyglot_benchmark/test_run.py new file mode 100755 index 000000000000..c946356e90d6 --- /dev/null +++ b/evaluation/benchmarks/polyglot_benchmark/test_run.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 + +import os +import sys +import argparse +from pathlib import Path + +# Add the parent directory to the Python path +sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent)) + +from evaluation.benchmarks.polyglot_benchmark.run_infer import ( + load_polyglot_dataset, + process_instance, + make_metadata, + get_llm_config_arg, +) +from openhands.core.logger import openhands_logger as logger + +def main(): + parser = argparse.ArgumentParser(description="Test the polyglot benchmark with a single instance") + parser.add_argument("--model", default="eval_gpt35_turbo", help="Model configuration name") + parser.add_argument("--agent", default="CodeActAgent", help="Agent class name") + parser.add_argument("--instance-id", type=int, default=0, help="Instance ID to test") + parser.add_argument("--language", help="Filter by language") + args = parser.parse_args() + + # Set the environment variable for the polyglot benchmark path + os.environ['POLYGLOT_BENCHMARK_PATH'] = '/workspace/polyglot-benchmark' + + # Load the dataset + dataset = load_polyglot_dataset() + + if args.language: + dataset = dataset[dataset['language'].str.lower() == args.language.lower()] + if dataset.empty: + print(f"No instances found for language: {args.language}") + return + + # Get the instance to test + if args.instance_id >= len(dataset): + print(f"Instance ID {args.instance_id} is out of range. Max ID: {len(dataset) - 1}") + return + + instance = dataset.iloc[args.instance_id] + print(f"Testing instance {instance.instance_id}: {instance.instance_name} ({instance.language})") + + # Get LLM config + llm_config = get_llm_config_arg(args.model) + if llm_config is None: + print(f"Could not find LLM config: {args.model}") + return + + # Create details dictionary with agent configuration + agent_details = { + "agent_config": { + "codeact_enable_jupyter": False, + "codeact_enable_browsing": False, + "codeact_enable_llm_editor": False, + } + } + + # Create metadata + metadata = make_metadata( + llm_config, + 'PolyglotBenchmark', + args.agent, + 30, # max_iterations + "test", + "evaluation/evaluation_outputs/test", + details=agent_details, + ) + + # Process the instance + try: + output = process_instance(instance, metadata, reset_logger=False) + print("\nTest completed successfully!") + print(f"Exit code: {output.test_result['exit_code']}") + print(f"Passed: {output.test_result['exit_code'] == 0}") + except Exception as e: + print(f"Error processing instance: {e}") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/evaluation/benchmarks/swe_bench/run_infer.py b/evaluation/benchmarks/swe_bench/run_infer.py index 89fe618a6c34..56448fe36c3a 100644 --- a/evaluation/benchmarks/swe_bench/run_infer.py +++ b/evaluation/benchmarks/swe_bench/run_infer.py @@ -540,7 +540,14 @@ def filter_dataset(dataset: pd.DataFrame, filter_column: str) -> pd.DataFrame: if llm_config is None: raise ValueError(f'Could not find LLM config: --llm_config {args.llm_config}') - details = {} + # Create details dictionary with agent configuration + details = { + "agent_config": { + "codeact_enable_jupyter": False, + "codeact_enable_browsing": RUN_WITH_BROWSING, + "codeact_enable_llm_editor": False, + } + } _agent_cls = openhands.agenthub.Agent.get_cls(args.agent_cls) dataset_descrption = ( diff --git a/evaluation/benchmarks/wiki_table_question/__init__.py b/evaluation/benchmarks/wiki_table_question/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/evaluation/utils/shared.py b/evaluation/utils/shared.py index 566fbbd71bb3..55ac1e3dd37c 100644 --- a/evaluation/utils/shared.py +++ b/evaluation/utils/shared.py @@ -160,6 +160,35 @@ def cleanup(): process.join() +def get_tools_string(agent_class: str, details: dict[str, Any] | None = None) -> str: + """Generate a string representation of the tools used by the agent. + + Args: + agent_class: The agent class name. + details: Additional details that might contain tool configuration. + + Returns: + A string representation of the tools used, e.g., "bash+finish+str_replace". + """ + # Default tools for CodeActAgent + if agent_class == "CodeActAgent": + tools = ["bash", "finish", "str_replace"] + + # Check if additional tools are enabled + if details and "agent_config" in details: + agent_config = details.get("agent_config", {}) + if agent_config.get("codeact_enable_browsing", False): + tools.extend(["web_read", "browser"]) + if agent_config.get("codeact_enable_jupyter", False): + tools.append("ipython") + if agent_config.get("codeact_enable_llm_editor", False): + tools[-1] = "llm_editor" # Replace str_replace with llm_editor + + return "+".join(tools) + + # For other agents, return a default string + return "default_tools" + def make_metadata( llm_config: LLMConfig, dataset_name: str, @@ -175,12 +204,15 @@ def make_metadata( model_name = llm_config.model.split('/')[-1] model_path = model_name.replace(':', '_').replace('@', '-') eval_note = f'_N_{eval_note}' if eval_note else '' - + + # Get tools string + tools_string = get_tools_string(agent_class, details) + eval_output_path = os.path.join( eval_output_dir, dataset_name, agent_class, - f'{model_path}_maxiter_{max_iterations}{eval_note}', + f'{model_path}_maxiter_{max_iterations}_tools_{tools_string}{eval_note}', ) pathlib.Path(eval_output_path).mkdir(parents=True, exist_ok=True) @@ -484,14 +516,15 @@ def update_llm_config_for_completions_logging( instance_id: str, ) -> LLMConfig: """Update the LLM config for logging completions.""" - if llm_config.log_completions: - llm_config.log_completions_folder = os.path.join( - eval_output_dir, 'llm_completions', instance_id - ) - logger.info( - f'Logging LLM completions for instance {instance_id} to ' - f'{llm_config.log_completions_folder}' - ) + # Always enable completions logging + llm_config.log_completions = True + llm_config.log_completions_folder = os.path.join( + eval_output_dir, 'llm_completions', instance_id + ) + logger.info( + f'Logging LLM completions for instance {instance_id} to ' + f'{llm_config.log_completions_folder}' + ) return llm_config