Terminal Bench Integration Test (#521)

* integrate terminal bench * fix ci * add ci * fix ci * fix ci * parallel ci * handle timeout * fix lint * trigger
2025-12-20 08:47:44 +00:00 · 2025-09-05 17:02:03 +08:00
parent b54fd6df06
commit 010ad5ddc4
21 changed files with 788 additions and 3 deletions
--- a/integration-tests/terminal-bench/qwen_code.py
+++ b/integration-tests/terminal-bench/qwen_code.py
@@ -0,0 +1,70 @@
+import os
+import shlex
+
+from terminal_bench.agents.installed_agents.abstract_installed_agent import (
+    AbstractInstalledAgent,
+)
+from terminal_bench.terminal.models import TerminalCommand
+
+
+class QwenCodeAgent(AbstractInstalledAgent):
+    @staticmethod
+    def name() -> str:
+        return "Qwen Code"
+
+    def __init__(self, model_name: str | None = None, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._model_name = model_name
+        self._version = kwargs.get("version", "latest")
+        
+        # Configurable API settings through agent_kwargs
+        self._api_key = kwargs.get("api_key")
+        self._base_url = kwargs.get("base_url")
+
+    @property
+    def _env(self) -> dict[str, str]:
+        env = {}
+        
+        # API Key - prefer agent_kwargs over environment variables
+        if self._api_key:
+            env["OPENAI_API_KEY"] = self._api_key
+        elif "OPENAI_API_KEY" in os.environ:
+            env["OPENAI_API_KEY"] = os.environ["OPENAI_API_KEY"]
+        else:
+            raise ValueError(
+                "OPENAI_API_KEY must be provided either via environment variable "
+                "or --agent-kwarg api_key=your_key"
+            )
+        
+        # Model - use model_name parameter or fallback
+        if self._model_name:
+            env["OPENAI_MODEL"] = self._model_name
+        elif "OPENAI_MODEL" in os.environ:
+            env["OPENAI_MODEL"] = os.environ["OPENAI_MODEL"]
+        else:
+            env["OPENAI_MODEL"] = "qwen3-coder-plus"
+        
+        # Base URL - prefer agent_kwargs over environment variables  
+        if self._base_url:
+            env["OPENAI_BASE_URL"] = self._base_url
+        elif "OPENAI_BASE_URL" in os.environ:
+            env["OPENAI_BASE_URL"] = os.environ["OPENAI_BASE_URL"]
+        else:
+            env["OPENAI_BASE_URL"] = "https://dashscope.aliyuncs.com/compatible-mode/v1"
+        
+        return env
+
+    @property
+    def _install_agent_script_path(self) -> os.PathLike:
+        return self._get_templated_script_path("qwen-code-setup.sh.j2")
+
+    def _run_agent_commands(self, task_description: str) -> list[TerminalCommand]:
+        escaped_description = shlex.quote(task_description)
+        return [
+            TerminalCommand(
+                command=f"qwen -y --prompt {escaped_description}",
+                max_timeout_sec=float("inf"),
+                block=True,
+                append_enter=True
+            )
+        ]