Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 27 additions & 6 deletions tests/integration/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
MessageEvent,
)
from openhands.sdk.tool import Tool
from openhands.tools.preset.default import get_default_condenser, get_default_tools
from tests.integration.early_stopper import EarlyStopperBase, EarlyStopResult


Expand Down Expand Up @@ -195,19 +196,39 @@ def run_instruction(self) -> TestResult:
self.teardown()

@property
@abstractmethod
def enable_browser(self) -> bool:
"""Whether to enable browser tools. Override to enable browser.

Returns:
True to enable browser tools, False otherwise (default)
"""
return False

@property
def tools(self) -> list[Tool]:
"""List of tools available to the agent."""
pass
"""List of tools available to the agent.

By default, uses the default preset tools from openhands.tools.preset.default.
Override this property to provide custom tools.

Returns:
List of Tool instances
"""
return get_default_tools(enable_browser=self.enable_browser)

@property
def condenser(self) -> CondenserBase | None:
"""Optional condenser for the agent. Override to provide a custom condenser.
"""Optional condenser for the agent.

By default, uses the default condenser from openhands.tools.preset.default.
Override to provide a custom condenser or return None to disable.

Returns:
CondenserBase instance or None (default)
CondenserBase instance (default condenser) or None
"""
return None
return get_default_condenser(
llm=self.llm.model_copy(update={"usage_id": "condenser"})
)

@property
def max_iteration_per_run(self) -> int:
Expand Down
14 changes: 4 additions & 10 deletions tests/integration/tests/b05_do_not_create_redundant_files.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,6 @@
from textwrap import dedent

from openhands.sdk import get_logger
from openhands.sdk.tool import Tool, register_tool
from openhands.tools.file_editor import FileEditorTool
from openhands.tools.terminal import TerminalTool
from tests.integration.base import BaseIntegrationTest, SkipTest, TestResult
from tests.integration.behavior_utils import (
get_conversation_summary,
Expand All @@ -28,15 +25,12 @@

class NoRedundantFilesTest(BaseIntegrationTest):
"""Ensure the agent does not create any redundant files (e.g., .md files)
that are not asked by users when performing the task."""
that are not asked by users when performing the task.

INSTRUCTION: str = INSTRUCTION
Uses the default agent preset (TerminalTool, FileEditorTool, TaskTrackerTool).
"""

@property
def tools(self) -> list[Tool]:
register_tool("TerminalTool", TerminalTool)
register_tool("FileEditorTool", FileEditorTool)
return [Tool(name="TerminalTool"), Tool(name="FileEditorTool")]
INSTRUCTION: str = INSTRUCTION

def setup(self) -> None: # noqa: D401
"""Set up a realistic codebase by cloning the lerobot repo."""
Expand Down
18 changes: 4 additions & 14 deletions tests/integration/tests/t01_fix_simple_typo.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,6 @@
import os

from openhands.sdk import get_logger
from openhands.sdk.tool import Tool, register_tool
from openhands.tools.file_editor import FileEditorTool
from openhands.tools.terminal import TerminalTool
from tests.integration.base import BaseIntegrationTest, TestResult


Expand All @@ -26,24 +23,17 @@


class TypoFixTest(BaseIntegrationTest):
"""Test that an agent can fix typos in a text file."""
"""Test that an agent can fix typos in a text file.

Uses the default agent preset (TerminalTool, FileEditorTool, TaskTrackerTool).
"""

INSTRUCTION: str = INSTRUCTION

def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.document_path: str = os.path.join(self.workspace, "document.txt")

@property
def tools(self) -> list[Tool]:
"""List of tools available to the agent."""
register_tool("TerminalTool", TerminalTool)
register_tool("FileEditorTool", FileEditorTool)
return [
Tool(name="TerminalTool"),
Tool(name="FileEditorTool"),
]

def setup(self) -> None:
"""Create a text file with typos for the agent to fix."""
# Create the test file with typos
Expand Down
18 changes: 4 additions & 14 deletions tests/integration/tests/t02_add_bash_hello.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,6 @@
import os

from openhands.sdk import get_logger
from openhands.sdk.tool import Tool, register_tool
from openhands.tools.file_editor import FileEditorTool
from openhands.tools.terminal import TerminalTool
from tests.integration.base import BaseIntegrationTest, TestResult


Expand All @@ -16,24 +13,17 @@


class BashHelloTest(BaseIntegrationTest):
"""Test that an agent can write a shell script that prints 'hello'."""
"""Test that an agent can write a shell script that prints 'hello'.

Uses the default agent preset (TerminalTool, FileEditorTool, TaskTrackerTool).
"""

INSTRUCTION: str = INSTRUCTION

def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.script_path: str = os.path.join(self.workspace, "shell", "hello.sh")

@property
def tools(self) -> list[Tool]:
"""List of tools available to the agent."""
register_tool("TerminalTool", TerminalTool)
register_tool("FileEditorTool", FileEditorTool)
return [
Tool(name="TerminalTool"),
Tool(name="FileEditorTool"),
]

def setup(self) -> None:
"""Setup is not needed - agent will create directories as needed."""

Expand Down
18 changes: 4 additions & 14 deletions tests/integration/tests/t03_jupyter_write_file.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,6 @@
import os

from openhands.sdk import get_logger
from openhands.sdk.tool import Tool, register_tool
from openhands.tools.file_editor import FileEditorTool
from openhands.tools.terminal import TerminalTool
from tests.integration.base import BaseIntegrationTest, TestResult


Expand All @@ -19,24 +16,17 @@


class JupyterWriteFileTest(BaseIntegrationTest):
"""Test that an agent can use Jupyter IPython to write a text file."""
"""Test that an agent can use Jupyter IPython to write a text file.

Uses the default agent preset (TerminalTool, FileEditorTool, TaskTrackerTool).
"""

INSTRUCTION: str = INSTRUCTION

def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.file_path: str = os.path.join(self.workspace, "test.txt")

@property
def tools(self) -> list[Tool]:
"""List of tools available to the agent."""
register_tool("TerminalTool", TerminalTool)
register_tool("FileEditorTool", FileEditorTool)
return [
Tool(name="TerminalTool"),
Tool(name="FileEditorTool"),
]

def setup(self) -> None:
"""Setup is not needed - agent will create directories as needed."""

Expand Down
18 changes: 4 additions & 14 deletions tests/integration/tests/t04_git_staging.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,6 @@
import subprocess

from openhands.sdk import get_logger
from openhands.sdk.tool import Tool, register_tool
from openhands.tools.file_editor import FileEditorTool
from openhands.tools.terminal import TerminalTool
from tests.integration.base import BaseIntegrationTest, TestResult


Expand All @@ -19,19 +16,12 @@


class GitStagingTest(BaseIntegrationTest):
"""Test that an agent can write a git commit message and commit changes."""
"""Test that an agent can write a git commit message and commit changes.

INSTRUCTION: str = INSTRUCTION
Uses the default agent preset (TerminalTool, FileEditorTool, TaskTrackerTool).
"""

@property
def tools(self) -> list[Tool]:
"""List of tools available to the agent."""
register_tool("TerminalTool", TerminalTool)
register_tool("FileEditorTool", FileEditorTool)
return [
Tool(name="TerminalTool"),
Tool(name="FileEditorTool"),
]
INSTRUCTION: str = INSTRUCTION

def setup(self) -> None:
"""Set up git repository with staged changes."""
Expand Down
19 changes: 7 additions & 12 deletions tests/integration/tests/t05_simple_browsing.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,6 @@

from openhands.sdk import get_logger
from openhands.sdk.conversation import get_agent_final_response
from openhands.sdk.tool import Tool, register_tool
from openhands.tools.file_editor import FileEditorTool
from openhands.tools.terminal import TerminalTool
from tests.integration.base import BaseIntegrationTest, TestResult


Expand Down Expand Up @@ -92,7 +89,10 @@


class SimpleBrowsingTest(BaseIntegrationTest):
"""Test that an agent can browse a local web page and extract information."""
"""Test that an agent can browse a local web page and extract information.

Uses the default agent preset with browser enabled.
"""

INSTRUCTION: str = INSTRUCTION

Expand All @@ -101,14 +101,9 @@ def __init__(self, *args, **kwargs):
self.server_process: subprocess.Popen[bytes] | None = None

@property
def tools(self) -> list[Tool]:
"""List of tools available to the agent."""
register_tool("TerminalTool", TerminalTool)
register_tool("FileEditorTool", FileEditorTool)
return [
Tool(name="TerminalTool"),
Tool(name="FileEditorTool"),
]
def enable_browser(self) -> bool:
"""Enable browser tools for this browsing test."""
return True

def setup(self) -> None:
"""Set up a local web server with the HTML file."""
Expand Down
19 changes: 7 additions & 12 deletions tests/integration/tests/t06_github_pr_browsing.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,6 @@

from openhands.sdk import get_logger
from openhands.sdk.conversation import get_agent_final_response
from openhands.sdk.tool import Tool, register_tool
from openhands.tools.file_editor import FileEditorTool
from openhands.tools.terminal import TerminalTool
from tests.integration.base import BaseIntegrationTest, TestResult


Expand All @@ -18,19 +15,17 @@


class GitHubPRBrowsingTest(BaseIntegrationTest):
"""Test that an agent can browse a GitHub PR and extract information."""
"""Test that an agent can browse a GitHub PR and extract information.

Uses the default agent preset with browser enabled.
"""

INSTRUCTION: str = INSTRUCTION

@property
def tools(self) -> list[Tool]:
"""List of tools available to the agent."""
register_tool("TerminalTool", TerminalTool)
register_tool("FileEditorTool", FileEditorTool)
return [
Tool(name="TerminalTool"),
Tool(name="FileEditorTool"),
]
def enable_browser(self) -> bool:
"""Enable browser tools for this browsing test."""
return True

def setup(self) -> None:
"""No special setup needed for GitHub PR browsing."""
Expand Down
18 changes: 4 additions & 14 deletions tests/integration/tests/t07_interactive_commands.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,6 @@
import os

from openhands.sdk import get_logger
from openhands.sdk.tool import Tool, register_tool
from openhands.tools.file_editor import FileEditorTool
from openhands.tools.terminal import TerminalTool
from tests.integration.base import BaseIntegrationTest, TestResult


Expand All @@ -32,24 +29,17 @@


class InteractiveCommandsTest(BaseIntegrationTest):
"""Test that an agent can execute interactive Python scripts with input."""
"""Test that an agent can execute interactive Python scripts with input.

Uses the default agent preset (TerminalTool, FileEditorTool, TaskTrackerTool).
"""

INSTRUCTION: str = INSTRUCTION

def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.script_path: str = os.path.join(self.workspace, "python_script.py")

@property
def tools(self) -> list[Tool]:
"""List of tools available to the agent."""
register_tool("TerminalTool", TerminalTool)
register_tool("FileEditorTool", FileEditorTool)
return [
Tool(name="TerminalTool"),
Tool(name="FileEditorTool"),
]

def setup(self) -> None:
"""Set up the interactive Python script."""

Expand Down
18 changes: 4 additions & 14 deletions tests/integration/tests/t08_image_file_viewing.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,6 @@

from openhands.sdk import get_logger
from openhands.sdk.conversation.response_utils import get_agent_final_response
from openhands.sdk.tool import Tool, register_tool
from openhands.tools.file_editor import FileEditorTool
from openhands.tools.terminal import TerminalTool
from tests.integration.base import BaseIntegrationTest, SkipTest, TestResult


Expand All @@ -23,7 +20,10 @@


class ImageFileViewingTest(BaseIntegrationTest):
"""Test that an agent can view and analyze image files."""
"""Test that an agent can view and analyze image files.

Uses the default agent preset (TerminalTool, FileEditorTool, TaskTrackerTool).
"""

INSTRUCTION: str = INSTRUCTION

Expand All @@ -38,16 +38,6 @@ def __init__(self, *args, **kwargs):
"Please use a model that supports image input."
)

@property
def tools(self) -> list[Tool]:
"""List of tools available to the agent."""
register_tool("TerminalTool", TerminalTool)
register_tool("FileEditorTool", FileEditorTool)
return [
Tool(name="TerminalTool"),
Tool(name="FileEditorTool"),
]

def setup(self) -> None:
"""Download the OpenHands logo for the agent to analyze."""
try:
Expand Down
Loading
Loading