logiguard fork v3: full patch set on verified 8c74db0 tree
Includes prior-session patches (carry forward so the app compiles): - crates/gpui/build.rs: cross-compile manifest fix - crates/gpui/src/platform.rs: PlatformWindow::activate_with_token trait method - crates/gpui/src/window.rs: Window::activate_with_token public API - crates/gpui_linux/src/linux/wayland/window.rs: WaylandWindow::activate_with_token + activate() keyboard-serial fix Plus the focus-serial fix: - serial.rs: SerialKind::KeyboardEnter - client.rs: store wl_keyboard.enter serial; latest_serial_of() Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Mohamad Khani
4
crates/eval_cli/.gitignore
vendored
Normal file
4
crates/eval_cli/.gitignore
vendored
Normal file
@@ -0,0 +1,4 @@
|
||||
**/jobs
|
||||
**/*.egg-info
|
||||
**/__pycache__
|
||||
uv.lock
|
||||
51
crates/eval_cli/Cargo.toml
Normal file
51
crates/eval_cli/Cargo.toml
Normal file
@@ -0,0 +1,51 @@
|
||||
[package]
|
||||
name = "eval_cli"
|
||||
version = "0.1.0"
|
||||
publish.workspace = true
|
||||
edition.workspace = true
|
||||
license = "GPL-3.0-or-later"
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[[bin]]
|
||||
name = "eval-cli"
|
||||
path = "src/main.rs"
|
||||
|
||||
[dependencies]
|
||||
acp_thread.workspace = true
|
||||
agent.workspace = true
|
||||
agent-client-protocol.workspace = true
|
||||
agent_ui.workspace = true
|
||||
anyhow.workspace = true
|
||||
clap.workspace = true
|
||||
client.workspace = true
|
||||
ctrlc = { version = "3.5", features = ["termination"] }
|
||||
db.workspace = true
|
||||
debug_adapter_extension.workspace = true
|
||||
env_logger.workspace = true
|
||||
extension.workspace = true
|
||||
feature_flags.workspace = true
|
||||
fs.workspace = true
|
||||
futures.workspace = true
|
||||
gpui.workspace = true
|
||||
gpui_platform.workspace = true
|
||||
gpui_tokio.workspace = true
|
||||
language.workspace = true
|
||||
language_extension.workspace = true
|
||||
language_model.workspace = true
|
||||
language_models.workspace = true
|
||||
languages = { workspace = true, features = ["load-grammars"] }
|
||||
node_runtime.workspace = true
|
||||
paths.workspace = true
|
||||
project.workspace = true
|
||||
prompt_store.workspace = true
|
||||
release_channel.workspace = true
|
||||
reqwest_client.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
settings.workspace = true
|
||||
shellexpand.workspace = true
|
||||
terminal_view.workspace = true
|
||||
util.workspace = true
|
||||
watch.workspace = true
|
||||
50
crates/eval_cli/Dockerfile
Normal file
50
crates/eval_cli/Dockerfile
Normal file
@@ -0,0 +1,50 @@
|
||||
# Build eval-cli for Linux.
|
||||
#
|
||||
# Usage (from the zed repo root):
|
||||
# docker build --platform linux/amd64 -f crates/eval_cli/Dockerfile -t eval-cli-builder .
|
||||
# docker cp "$(docker create eval-cli-builder)":/eval-cli ./target/eval-cli
|
||||
#
|
||||
# Or use the helper script:
|
||||
# crates/eval_cli/script/build-linux
|
||||
|
||||
FROM rust:1.95.0 AS builder
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Pre-install the toolchain specified in rust-toolchain.toml so it is cached.
|
||||
RUN rustup toolchain install 1.95.0 --profile minimal \
|
||||
--component rustfmt --component clippy --component rust-analyzer --component rust-src \
|
||||
--target wasm32-wasip2 --target wasm32-unknown-unknown --target x86_64-unknown-linux-musl --target x86_64-unknown-linux-gnu
|
||||
|
||||
# Install build tools. cmake + build-essential are needed for vendored C
|
||||
# libraries (libgit2-sys, zstd-sys, libsqlite3-sys). No audio/GUI -dev
|
||||
# packages required — eval-cli runs headless with those features disabled.
|
||||
#
|
||||
# cargo-zigbuild cross-compiles against musl libc, producing a fully
|
||||
# static binary that runs on any Linux distro (glibc or musl / Alpine).
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
cmake \
|
||||
build-essential \
|
||||
curl \
|
||||
xz-utils \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
RUN mkdir -p /opt/zig \
|
||||
&& curl -fsSL https://ziglang.org/download/0.15.2/zig-x86_64-linux-0.15.2.tar.xz \
|
||||
| tar -xJ -C /opt/zig --strip-components=1 \
|
||||
&& ln -s /opt/zig/zig /usr/local/bin/zig
|
||||
|
||||
RUN cargo install --locked cargo-zigbuild
|
||||
|
||||
COPY . .
|
||||
|
||||
RUN --mount=type=cache,target=/usr/local/cargo/registry \
|
||||
--mount=type=cache,target=/usr/local/cargo/git \
|
||||
--mount=type=cache,target=/app/target \
|
||||
cargo zigbuild --release --package eval_cli \
|
||||
--target x86_64-unknown-linux-musl && \
|
||||
cp /app/target/x86_64-unknown-linux-musl/release/eval-cli /eval-cli && \
|
||||
strip /eval-cli
|
||||
|
||||
FROM scratch
|
||||
COPY --from=builder /eval-cli /eval-cli
|
||||
21
crates/eval_cli/Dockerfile.dockerignore
Normal file
21
crates/eval_cli/Dockerfile.dockerignore
Normal file
@@ -0,0 +1,21 @@
|
||||
.git
|
||||
.github
|
||||
**/.gitignore
|
||||
**/.gitkeep
|
||||
.gitattributes
|
||||
.mailmap
|
||||
**/target
|
||||
zed.xcworkspace
|
||||
.DS_Store
|
||||
compose.yml
|
||||
plugins/bin
|
||||
script/node_modules
|
||||
styles/node_modules
|
||||
crates/collab/static/styles.css
|
||||
vendor/bin
|
||||
assets/themes/
|
||||
**/jobs
|
||||
|
||||
**/*.egg-info
|
||||
**/__pycache__
|
||||
**/.venv
|
||||
1
crates/eval_cli/LICENSE-GPL
Symbolic link
1
crates/eval_cli/LICENSE-GPL
Symbolic link
@@ -0,0 +1 @@
|
||||
../../LICENSE-GPL
|
||||
108
crates/eval_cli/README.md
Normal file
108
crates/eval_cli/README.md
Normal file
@@ -0,0 +1,108 @@
|
||||
# eval-cli
|
||||
|
||||
Headless CLI binary for running Zed's agent in evaluation/benchmark
|
||||
environments. Designed to work inside containerized environments like
|
||||
[Harbor](https://harborframework.com/) where the repository is already
|
||||
checked out and API keys are provided via environment variables.
|
||||
|
||||
Uses the same `NativeAgent` + `AcpThread` pipeline as the production Zed
|
||||
editor — full agentic loop with tool calls, subagents, and retries, just
|
||||
without a GUI.
|
||||
|
||||
## Building
|
||||
|
||||
### Native (for local testing on the same OS)
|
||||
|
||||
```
|
||||
cargo build --release -p eval_cli
|
||||
```
|
||||
|
||||
### Cross-compile for Linux x86_64 (from macOS or other hosts)
|
||||
|
||||
Harbor containers run Linux x86_64. Use the Docker-based build script:
|
||||
|
||||
```
|
||||
crates/eval_cli/script/build-linux
|
||||
```
|
||||
|
||||
This produces `target/eval-cli` (an x86_64 Linux ELF binary). You can
|
||||
also specify a custom output path:
|
||||
|
||||
```
|
||||
crates/eval_cli/script/build-linux --output ~/bin/eval-cli-linux
|
||||
```
|
||||
|
||||
## Standalone usage
|
||||
|
||||
```
|
||||
eval-cli \
|
||||
--workdir /testbed \
|
||||
--model anthropic/claude-sonnet-4-6-latest \
|
||||
--instruction "Fix the bug described in..." \
|
||||
--timeout 600 \
|
||||
--output-dir /logs/agent
|
||||
```
|
||||
|
||||
Reads API keys from environment variables (`ANTHROPIC_API_KEY`,
|
||||
`OPENAI_API_KEY`, etc.). Writes `result.json`, `thread.md`, and
|
||||
`thread.json` to the output directory.
|
||||
|
||||
### Exit codes
|
||||
|
||||
| Code | Meaning |
|
||||
| ---- | ---------------------------------- |
|
||||
| 0 | Agent finished |
|
||||
| 1 | Error (model/auth/runtime failure) |
|
||||
| 2 | Timeout |
|
||||
| 3 | Interrupted (SIGTERM/SIGINT) |
|
||||
|
||||
## Harbor integration
|
||||
|
||||
The `zed_eval/` directory contains a Python package that
|
||||
implements Harbor's `BaseInstalledAgent` interface, allowing eval-cli to
|
||||
be used with `--agent-import-path` without modifying Harbor's source code.
|
||||
|
||||
### Setup
|
||||
|
||||
```
|
||||
pip install -e crates/eval_cli/harbor/
|
||||
```
|
||||
|
||||
### Running with a local binary
|
||||
|
||||
Build for Linux first, then pass the binary path:
|
||||
|
||||
```
|
||||
crates/eval_cli/script/build-linux
|
||||
|
||||
harbor run -d "swebench_verified@latest" \
|
||||
--agent-import-path zed_eval.agent:ZedAgent \
|
||||
--ae binary_path=target/eval-cli \
|
||||
-m anthropic/claude-sonnet-4-6-latest
|
||||
```
|
||||
|
||||
The agent uploads the binary into the container during setup — no
|
||||
download URL needed during local iteration.
|
||||
|
||||
### Running with a download URL
|
||||
|
||||
For CI or when the binary is hosted somewhere:
|
||||
|
||||
```
|
||||
harbor run -d "swebench_verified@latest" \
|
||||
--agent-import-path zed_eval.agent:ZedAgent \
|
||||
--ak download_url=https://example.com/eval-cli \
|
||||
-m anthropic/claude-sonnet-4-6-latest
|
||||
```
|
||||
|
||||
### Setting a timeout
|
||||
|
||||
Pass `EVAL_CLI_TIMEOUT` via `--ae`:
|
||||
|
||||
```
|
||||
harbor run -d "swebench_verified@latest" \
|
||||
--agent-import-path zed_eval.agent:ZedAgent \
|
||||
--ak binary_path=target/eval-cli \
|
||||
--ae EVAL_CLI_TIMEOUT=600 \
|
||||
-m anthropic/claude-sonnet-4-6-latest
|
||||
```
|
||||
15
crates/eval_cli/build.rs
Normal file
15
crates/eval_cli/build.rs
Normal file
@@ -0,0 +1,15 @@
|
||||
fn main() {
|
||||
let cargo_toml =
|
||||
std::fs::read_to_string("../zed/Cargo.toml").expect("Failed to read crates/zed/Cargo.toml");
|
||||
let version = cargo_toml
|
||||
.lines()
|
||||
.find(|line| line.starts_with("version = "))
|
||||
.expect("Version not found in crates/zed/Cargo.toml")
|
||||
.split('=')
|
||||
.nth(1)
|
||||
.expect("Invalid version format")
|
||||
.trim()
|
||||
.trim_matches('"');
|
||||
println!("cargo:rerun-if-changed=../zed/Cargo.toml");
|
||||
println!("cargo:rustc-env=ZED_PKG_VERSION={}", version);
|
||||
}
|
||||
59
crates/eval_cli/script/build-linux
Executable file
59
crates/eval_cli/script/build-linux
Executable file
@@ -0,0 +1,59 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Build eval-cli for x86_64 Linux from any host (macOS, Linux, etc.)
|
||||
# using Docker + cargo-zigbuild. Targets musl libc, producing a fully
|
||||
# static binary that runs on any Linux distro (glibc or musl / Alpine).
|
||||
# The resulting binary is placed at the path printed on completion
|
||||
# (default: target/eval-cli).
|
||||
#
|
||||
# Usage:
|
||||
# crates/eval_cli/script/build-linux [--output PATH]
|
||||
#
|
||||
# Examples:
|
||||
# crates/eval_cli/script/build-linux
|
||||
# crates/eval_cli/script/build-linux --output ~/bin/eval-cli
|
||||
#
|
||||
# Prerequisites: Docker must be installed and running.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
REPO_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)"
|
||||
OUTPUT="${REPO_ROOT}/target/eval-cli"
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
--output)
|
||||
OUTPUT="$2"
|
||||
shift 2
|
||||
;;
|
||||
*)
|
||||
echo "Unknown option: $1" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
cd "$REPO_ROOT"
|
||||
|
||||
IMAGE_TAG="eval-cli-builder"
|
||||
|
||||
echo "Building eval-cli for x86_64-unknown-linux-musl (static binary)..."
|
||||
echo " Repo root: $REPO_ROOT"
|
||||
echo " Output: $OUTPUT"
|
||||
echo ""
|
||||
|
||||
docker build \
|
||||
--platform linux/amd64 \
|
||||
-f crates/eval_cli/Dockerfile \
|
||||
-t "$IMAGE_TAG" \
|
||||
.
|
||||
|
||||
CONTAINER_ID=$(docker create "$IMAGE_TAG" /eval-cli)
|
||||
mkdir -p "$(dirname "$OUTPUT")"
|
||||
docker cp "$CONTAINER_ID":/eval-cli "$OUTPUT"
|
||||
docker rm "$CONTAINER_ID" > /dev/null
|
||||
|
||||
echo ""
|
||||
echo "Built successfully: $OUTPUT"
|
||||
echo " $(file "$OUTPUT")"
|
||||
137
crates/eval_cli/src/headless.rs
Normal file
137
crates/eval_cli/src/headless.rs
Normal file
@@ -0,0 +1,137 @@
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
|
||||
use client::{Client, ProxySettings, RefreshLlmTokenListener, UserStore};
|
||||
use db::AppDatabase;
|
||||
use extension::ExtensionHostProxy;
|
||||
use fs::RealFs;
|
||||
use gpui::http_client::read_proxy_from_env;
|
||||
use gpui::{App, AppContext as _, Entity};
|
||||
use gpui_tokio::Tokio;
|
||||
use language::LanguageRegistry;
|
||||
use language_extension::LspAccess;
|
||||
use node_runtime::{NodeBinaryOptions, NodeRuntime};
|
||||
use project::project_settings::ProjectSettings;
|
||||
use prompt_store::PromptBuilder;
|
||||
use release_channel::{AppCommitSha, AppVersion};
|
||||
use reqwest_client::ReqwestClient;
|
||||
use settings::{Settings, SettingsStore};
|
||||
use util::ResultExt as _;
|
||||
|
||||
pub struct AgentCliAppState {
|
||||
pub languages: Arc<LanguageRegistry>,
|
||||
pub client: Arc<Client>,
|
||||
pub user_store: Entity<UserStore>,
|
||||
pub fs: Arc<dyn fs::Fs>,
|
||||
pub node_runtime: NodeRuntime,
|
||||
}
|
||||
|
||||
pub fn init(cx: &mut App) -> Arc<AgentCliAppState> {
|
||||
let app_commit_sha = option_env!("ZED_COMMIT_SHA").map(|s| AppCommitSha::new(s.to_owned()));
|
||||
|
||||
let app_version = AppVersion::load(
|
||||
env!("ZED_PKG_VERSION"),
|
||||
option_env!("ZED_BUILD_ID"),
|
||||
app_commit_sha,
|
||||
);
|
||||
|
||||
release_channel::init(app_version.clone(), cx);
|
||||
gpui_tokio::init(cx);
|
||||
|
||||
let settings_store = SettingsStore::new(cx, &settings::default_settings());
|
||||
cx.set_global(settings_store);
|
||||
|
||||
let user_agent = format!(
|
||||
"Zed Agent CLI/{} ({}; {})",
|
||||
app_version,
|
||||
std::env::consts::OS,
|
||||
std::env::consts::ARCH
|
||||
);
|
||||
let proxy_str = ProxySettings::get_global(cx).proxy.to_owned();
|
||||
let proxy_url = proxy_str
|
||||
.as_ref()
|
||||
.and_then(|input| input.parse().ok())
|
||||
.or_else(read_proxy_from_env);
|
||||
let http = {
|
||||
let _guard = Tokio::handle(cx).enter();
|
||||
ReqwestClient::proxy_and_user_agent(proxy_url, &user_agent)
|
||||
.expect("could not start HTTP client")
|
||||
};
|
||||
cx.set_http_client(Arc::new(http));
|
||||
|
||||
let client = Client::production(cx);
|
||||
cx.set_http_client(client.http_client());
|
||||
|
||||
let app_db = AppDatabase::new();
|
||||
cx.set_global(app_db);
|
||||
|
||||
let git_binary_path = None;
|
||||
let fs = Arc::new(RealFs::new(
|
||||
git_binary_path,
|
||||
cx.background_executor().clone(),
|
||||
));
|
||||
<dyn fs::Fs>::set_global(fs.clone(), cx);
|
||||
|
||||
let mut languages = LanguageRegistry::new(cx.background_executor().clone());
|
||||
languages.set_language_server_download_dir(paths::languages_dir().clone());
|
||||
let languages = Arc::new(languages);
|
||||
|
||||
let user_store = cx.new(|cx| UserStore::new(client.clone(), cx));
|
||||
|
||||
extension::init(cx);
|
||||
|
||||
let (mut node_options_tx, node_options_rx) = watch::channel(None);
|
||||
cx.observe_global::<SettingsStore>(move |cx| {
|
||||
let settings = &ProjectSettings::get_global(cx).node;
|
||||
let options = NodeBinaryOptions {
|
||||
allow_path_lookup: !settings.ignore_system_version,
|
||||
allow_binary_download: true,
|
||||
use_paths: settings.path.as_ref().map(|node_path| {
|
||||
let node_path = PathBuf::from(shellexpand::tilde(node_path).as_ref());
|
||||
let npm_path = settings
|
||||
.npm_path
|
||||
.as_ref()
|
||||
.map(|path| PathBuf::from(shellexpand::tilde(&path).as_ref()));
|
||||
(
|
||||
node_path.clone(),
|
||||
npm_path.unwrap_or_else(|| {
|
||||
let base_path = PathBuf::new();
|
||||
node_path.parent().unwrap_or(&base_path).join("npm")
|
||||
}),
|
||||
)
|
||||
}),
|
||||
};
|
||||
node_options_tx.send(Some(options)).log_err();
|
||||
})
|
||||
.detach();
|
||||
let node_runtime = NodeRuntime::new(client.http_client(), None, node_options_rx);
|
||||
|
||||
let extension_host_proxy = ExtensionHostProxy::global(cx);
|
||||
debug_adapter_extension::init(extension_host_proxy.clone(), cx);
|
||||
language_extension::init(LspAccess::Noop, extension_host_proxy, languages.clone());
|
||||
language_model::init(cx);
|
||||
RefreshLlmTokenListener::register(client.clone(), user_store.clone(), cx);
|
||||
language_models::init(user_store.clone(), client.clone(), cx);
|
||||
languages::init(languages.clone(), fs.clone(), node_runtime.clone(), cx);
|
||||
prompt_store::init(cx);
|
||||
terminal_view::init(cx);
|
||||
|
||||
let stdout_is_a_pty = false;
|
||||
let prompt_builder = PromptBuilder::load(fs.clone(), stdout_is_a_pty, cx);
|
||||
agent_ui::init(
|
||||
fs.clone(),
|
||||
prompt_builder,
|
||||
languages.clone(),
|
||||
true,
|
||||
true,
|
||||
cx,
|
||||
);
|
||||
|
||||
Arc::new(AgentCliAppState {
|
||||
languages,
|
||||
client,
|
||||
user_store,
|
||||
fs,
|
||||
node_runtime,
|
||||
})
|
||||
}
|
||||
574
crates/eval_cli/src/main.rs
Normal file
574
crates/eval_cli/src/main.rs
Normal file
@@ -0,0 +1,574 @@
|
||||
//! Headless CLI binary for running Zed's agent in evaluation/benchmark environments.
|
||||
//!
|
||||
//! Designed to work inside containerized environments (like Harbor/termbench) where:
|
||||
//! - The repository is already checked out at the working directory
|
||||
//! - The model API key is provided via environment variables
|
||||
//! - Results are written to an output directory (default: `/logs/agent/`)
|
||||
//!
|
||||
//! ## Usage
|
||||
//!
|
||||
//! ```text
|
||||
//! eval-cli --workdir /testbed --model anthropic/claude-sonnet-4-6-latest \
|
||||
//! --instruction "Fix the bug described in..." --timeout 600
|
||||
//! ```
|
||||
//!
|
||||
//! ## Output
|
||||
//!
|
||||
//! Writes to `--output-dir` (default `/logs/agent/`):
|
||||
//! - `result.json` — structured result with status, timing, and token usage
|
||||
//! - `thread.md` — full conversation as markdown
|
||||
//! - `thread.json` — raw thread state as JSON
|
||||
//!
|
||||
//! ## Exit codes
|
||||
//!
|
||||
//! | Code | Meaning |
|
||||
//! |------|---------|
|
||||
//! | 0 | Agent finished |
|
||||
//! | 1 | Error (model/auth/runtime failure) |
|
||||
//! | 2 | Timeout |
|
||||
//! | 3 | Interrupted (SIGTERM/SIGINT) |
|
||||
|
||||
mod headless;
|
||||
|
||||
use std::path::PathBuf;
|
||||
use std::process;
|
||||
use std::rc::Rc;
|
||||
use std::str::FromStr;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use acp_thread::AgentConnection as _;
|
||||
use agent::{NativeAgent, NativeAgentConnection, Templates, ThreadStore};
|
||||
use agent_client_protocol::schema as acp;
|
||||
use anyhow::{Context, Result};
|
||||
use clap::Parser;
|
||||
use feature_flags::FeatureFlagAppExt as _;
|
||||
|
||||
use futures::{FutureExt, select_biased};
|
||||
use gpui::{AppContext as _, AsyncApp, Entity, UpdateGlobal};
|
||||
use language_model::{LanguageModelRegistry, SelectedModel};
|
||||
use project::Project;
|
||||
use settings::SettingsStore;
|
||||
use util::path_list::PathList;
|
||||
|
||||
use crate::headless::AgentCliAppState;
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
#[command(
|
||||
name = "eval-cli",
|
||||
about = "Run Zed's agent headlessly in evaluation/benchmark environments"
|
||||
)]
|
||||
struct Args {
|
||||
/// Output current environment variables as JSON to stdout.
|
||||
/// Used internally by Zed's shell environment capture.
|
||||
#[arg(long, hide = true)]
|
||||
printenv: bool,
|
||||
|
||||
/// Path to the repository working directory. Defaults to the current directory.
|
||||
#[arg(long, default_value = ".")]
|
||||
workdir: PathBuf,
|
||||
|
||||
/// Instruction/prompt text. If omitted, read from --instruction-file or stdin.
|
||||
#[arg(long, allow_hyphen_values = true)]
|
||||
instruction: Option<String>,
|
||||
|
||||
/// Language model to use, in `provider/model` format.
|
||||
#[arg(long, default_value = "anthropic/claude-sonnet-4-6-latest")]
|
||||
model: String,
|
||||
|
||||
/// Maximum wall-clock time in seconds for the agent run.
|
||||
#[arg(long)]
|
||||
timeout: Option<u64>,
|
||||
|
||||
/// Directory for output artifacts (result.json, thread.md, thread.json).
|
||||
#[arg(long, default_value = ".")]
|
||||
output_dir: PathBuf,
|
||||
|
||||
/// Disable staff mode (staff mode is enabled by default).
|
||||
#[arg(long)]
|
||||
no_staff: bool,
|
||||
|
||||
/// Reasoning effort level for models that support thinking (low, medium, high).
|
||||
/// Defaults to "high" for thinking-capable models.
|
||||
#[arg(long)]
|
||||
reasoning_effort: Option<String>,
|
||||
|
||||
/// Enable or disable extended thinking. Defaults to model auto-detection if omitted.
|
||||
#[arg(long)]
|
||||
thinking: Option<bool>,
|
||||
}
|
||||
|
||||
enum AgentOutcome {
|
||||
Completed,
|
||||
Timeout { seconds: u64 },
|
||||
Interrupted,
|
||||
}
|
||||
|
||||
#[derive(serde::Serialize)]
|
||||
struct EvalResult {
|
||||
status: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
error: Option<String>,
|
||||
duration_secs: f64,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
timeout_secs: Option<u64>,
|
||||
model: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
input_tokens: Option<u64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
output_tokens: Option<u64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
cache_creation_input_tokens: Option<u64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
cache_read_input_tokens: Option<u64>,
|
||||
}
|
||||
|
||||
const EXIT_OK: i32 = 0;
|
||||
const EXIT_ERROR: i32 = 1;
|
||||
const EXIT_TIMEOUT: i32 = 2;
|
||||
const EXIT_INTERRUPTED: i32 = 3;
|
||||
|
||||
static TERMINATED: AtomicBool = AtomicBool::new(false);
|
||||
|
||||
fn main() {
|
||||
let args = Args::parse();
|
||||
|
||||
if args.printenv {
|
||||
util::shell_env::print_env();
|
||||
return;
|
||||
}
|
||||
|
||||
env_logger::init();
|
||||
|
||||
ctrlc::set_handler(|| {
|
||||
TERMINATED.store(true, Ordering::SeqCst);
|
||||
})
|
||||
.expect("failed to set signal handler");
|
||||
|
||||
let instruction = read_instruction(&args).unwrap_or_else(|e| {
|
||||
eprintln!("Error reading instruction: {e}");
|
||||
process::exit(EXIT_ERROR);
|
||||
});
|
||||
|
||||
let workdir = args.workdir.canonicalize().unwrap_or_else(|e| {
|
||||
eprintln!("Invalid --workdir {:?}: {e}", args.workdir);
|
||||
process::exit(EXIT_ERROR);
|
||||
});
|
||||
|
||||
let output_dir = args.output_dir.clone();
|
||||
if let Err(e) = std::fs::create_dir_all(&output_dir) {
|
||||
eprintln!("Error creating output dir {}: {e}", output_dir.display());
|
||||
process::exit(EXIT_ERROR);
|
||||
}
|
||||
|
||||
let http_client = Arc::new(reqwest_client::ReqwestClient::new());
|
||||
let app = gpui_platform::headless().with_http_client(http_client);
|
||||
|
||||
app.run(move |cx| {
|
||||
let app_state = headless::init(cx);
|
||||
cx.set_staff(!args.no_staff);
|
||||
|
||||
let auth_tasks = LanguageModelRegistry::global(cx).update(cx, |registry, cx| {
|
||||
registry
|
||||
.providers()
|
||||
.iter()
|
||||
.map(|p| p.authenticate(cx))
|
||||
.collect::<Vec<_>>()
|
||||
});
|
||||
|
||||
let model_name = args.model.clone();
|
||||
let timeout = args.timeout;
|
||||
let thinking_override = args.thinking;
|
||||
let reasoning_effort = args.reasoning_effort.clone();
|
||||
|
||||
cx.spawn(async move |cx| {
|
||||
futures::future::join_all(auth_tasks).await;
|
||||
|
||||
let start = Instant::now();
|
||||
|
||||
let (outcome, token_usage) = run_agent(
|
||||
&app_state,
|
||||
&workdir,
|
||||
&instruction,
|
||||
&model_name,
|
||||
timeout,
|
||||
thinking_override,
|
||||
reasoning_effort.as_deref(),
|
||||
Some(&output_dir),
|
||||
cx,
|
||||
)
|
||||
.await;
|
||||
|
||||
let duration = start.elapsed();
|
||||
|
||||
let (status, error, exit_code) = match &outcome {
|
||||
Ok(AgentOutcome::Completed) => ("completed".to_string(), None, EXIT_OK),
|
||||
Ok(AgentOutcome::Timeout { seconds }) => {
|
||||
eprintln!("Timeout: agent exceeded {seconds}s time limit");
|
||||
("timeout".to_string(), None, EXIT_TIMEOUT)
|
||||
}
|
||||
Ok(AgentOutcome::Interrupted) => {
|
||||
eprintln!("Interrupted: received SIGTERM, saved partial output");
|
||||
("interrupted".to_string(), None, EXIT_INTERRUPTED)
|
||||
}
|
||||
Err(e) => {
|
||||
eprintln!("Error: {e:#}");
|
||||
("error".to_string(), Some(format!("{e:#}")), EXIT_ERROR)
|
||||
}
|
||||
};
|
||||
|
||||
let result = EvalResult {
|
||||
status,
|
||||
error,
|
||||
duration_secs: duration.as_secs_f64(),
|
||||
timeout_secs: timeout,
|
||||
model: model_name.clone(),
|
||||
input_tokens: token_usage.as_ref().map(|u| u.input_tokens),
|
||||
output_tokens: token_usage.as_ref().map(|u| u.output_tokens),
|
||||
cache_creation_input_tokens: token_usage
|
||||
.as_ref()
|
||||
.filter(|u| u.cache_creation_input_tokens > 0)
|
||||
.map(|u| u.cache_creation_input_tokens),
|
||||
cache_read_input_tokens: token_usage
|
||||
.as_ref()
|
||||
.filter(|u| u.cache_read_input_tokens > 0)
|
||||
.map(|u| u.cache_read_input_tokens),
|
||||
};
|
||||
|
||||
match serde_json::to_string_pretty(&result) {
|
||||
Ok(json) => {
|
||||
if let Err(e) = std::fs::write(output_dir.join("result.json"), &json) {
|
||||
eprintln!("Error writing result.json: {e:#}");
|
||||
}
|
||||
eprintln!("[eval-cli] result: {json}");
|
||||
}
|
||||
Err(e) => eprintln!("Error serializing result: {e:#}"),
|
||||
}
|
||||
|
||||
cx.update(|cx| cx.quit());
|
||||
process::exit(exit_code);
|
||||
})
|
||||
.detach();
|
||||
});
|
||||
}
|
||||
|
||||
fn read_instruction(args: &Args) -> Result<String> {
|
||||
let text = if let Some(text) = &args.instruction {
|
||||
text.clone()
|
||||
} else {
|
||||
use std::io::Read;
|
||||
let mut buf = String::new();
|
||||
std::io::stdin()
|
||||
.read_to_string(&mut buf)
|
||||
.context("reading instruction from stdin")?;
|
||||
buf
|
||||
};
|
||||
anyhow::ensure!(!text.trim().is_empty(), "instruction is empty");
|
||||
Ok(text)
|
||||
}
|
||||
|
||||
async fn run_agent(
|
||||
app_state: &Arc<AgentCliAppState>,
|
||||
workdir: &std::path::Path,
|
||||
instruction: &str,
|
||||
model_name: &str,
|
||||
timeout: Option<u64>,
|
||||
thinking_override: Option<bool>,
|
||||
reasoning_effort: Option<&str>,
|
||||
output_dir: Option<&std::path::Path>,
|
||||
cx: &mut AsyncApp,
|
||||
) -> (Result<AgentOutcome>, Option<language_model::TokenUsage>) {
|
||||
let setup_result: Result<()> = cx.update(|cx| {
|
||||
let selected = SelectedModel::from_str(model_name).map_err(|e| anyhow::anyhow!("{e}"))?;
|
||||
let registry = LanguageModelRegistry::global(cx);
|
||||
let model = registry
|
||||
.read(cx)
|
||||
.available_models(cx)
|
||||
.find(|m| m.id() == selected.model && m.provider_id() == selected.provider)
|
||||
.ok_or_else(|| {
|
||||
let available = registry
|
||||
.read(cx)
|
||||
.available_models(cx)
|
||||
.map(|m| format!("{}/{}", m.provider_id().0, m.id().0))
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ");
|
||||
anyhow::anyhow!("Model {model_name} not found. Available: {available}")
|
||||
})?;
|
||||
|
||||
let supports_thinking = model.supports_thinking();
|
||||
|
||||
registry.update(cx, |registry, cx| {
|
||||
registry.set_default_model(
|
||||
Some(language_model::ConfiguredModel {
|
||||
provider: registry
|
||||
.provider(&model.provider_id())
|
||||
.context("Provider not found")?,
|
||||
model,
|
||||
}),
|
||||
cx,
|
||||
);
|
||||
anyhow::Ok(())
|
||||
})?;
|
||||
|
||||
let enable_thinking = thinking_override.unwrap_or(supports_thinking);
|
||||
let effort = if enable_thinking {
|
||||
match reasoning_effort {
|
||||
Some(level) => format!("\"{level}\""),
|
||||
None => "\"high\"".to_string(),
|
||||
}
|
||||
} else {
|
||||
"null".to_string()
|
||||
};
|
||||
let provider_id = selected.provider.0.to_string();
|
||||
let model_id = selected.model.0.to_string();
|
||||
SettingsStore::update_global(cx, |store, cx| {
|
||||
let settings = format!(
|
||||
r#"{{
|
||||
"agent": {{
|
||||
"tool_permissions": {{"default": "allow"}},
|
||||
"default_model": {{
|
||||
"provider": "{provider_id}",
|
||||
"model": "{model_id}",
|
||||
"enable_thinking": {enable_thinking},
|
||||
"effort": {effort}
|
||||
}}
|
||||
}},
|
||||
"autosave": "off",
|
||||
"format_on_save": "off"
|
||||
}}"
|
||||
"#
|
||||
);
|
||||
store.set_user_settings(&settings, cx).ok();
|
||||
});
|
||||
|
||||
anyhow::Ok(())
|
||||
});
|
||||
|
||||
if let Err(e) = setup_result {
|
||||
return (Err(e), None);
|
||||
}
|
||||
|
||||
let project = cx.update(|cx| {
|
||||
Project::local(
|
||||
app_state.client.clone(),
|
||||
app_state.node_runtime.clone(),
|
||||
app_state.user_store.clone(),
|
||||
app_state.languages.clone(),
|
||||
app_state.fs.clone(),
|
||||
None,
|
||||
project::LocalProjectFlags {
|
||||
init_worktree_trust: false,
|
||||
..Default::default()
|
||||
},
|
||||
cx,
|
||||
)
|
||||
});
|
||||
|
||||
let worktree = project.update(cx, |project, cx| project.create_worktree(workdir, true, cx));
|
||||
let worktree = match worktree.await {
|
||||
Ok(w) => w,
|
||||
Err(e) => return (Err(e).context("creating worktree"), None),
|
||||
};
|
||||
|
||||
let scan_result = worktree.update(cx, |tree, _cx| {
|
||||
tree.as_local()
|
||||
.context("expected local worktree")
|
||||
.map(|local| local.scan_complete())
|
||||
});
|
||||
match scan_result {
|
||||
Ok(future) => future.await,
|
||||
Err(e) => return (Err(e), None),
|
||||
};
|
||||
|
||||
let agent = cx.update(|cx| {
|
||||
let thread_store = cx.new(|cx| ThreadStore::new(cx));
|
||||
NativeAgent::new(
|
||||
thread_store,
|
||||
Templates::new(),
|
||||
None,
|
||||
app_state.fs.clone(),
|
||||
cx,
|
||||
)
|
||||
});
|
||||
|
||||
let connection = Rc::new(NativeAgentConnection(agent.clone()));
|
||||
let acp_thread = match cx
|
||||
.update(|cx| {
|
||||
connection
|
||||
.clone()
|
||||
.new_session(project, PathList::new(&[workdir]), cx)
|
||||
})
|
||||
.await
|
||||
{
|
||||
Ok(t) => t,
|
||||
Err(e) => return (Err(e).context("creating ACP session"), None),
|
||||
};
|
||||
|
||||
let _subscription = cx.subscribe(&acp_thread, |acp_thread, event, cx| {
|
||||
log_acp_thread_event(&acp_thread, event, cx);
|
||||
});
|
||||
|
||||
let message = vec![acp::ContentBlock::Text(acp::TextContent::new(
|
||||
instruction.to_string(),
|
||||
))];
|
||||
|
||||
let send_future = acp_thread.update(cx, |acp_thread: &mut acp_thread::AcpThread, cx| {
|
||||
acp_thread.send(message, cx)
|
||||
});
|
||||
|
||||
let timeout_future = if let Some(timeout_secs) = timeout {
|
||||
futures::future::Either::Left(
|
||||
cx.background_executor()
|
||||
.timer(Duration::from_secs(timeout_secs)),
|
||||
)
|
||||
} else {
|
||||
futures::future::Either::Right(futures::future::pending::<()>())
|
||||
};
|
||||
|
||||
let sigterm_future = {
|
||||
let executor = cx.background_executor().clone();
|
||||
async move {
|
||||
while !TERMINATED.load(Ordering::Relaxed) {
|
||||
executor.timer(Duration::from_millis(100)).await;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
let outcome = select_biased! {
|
||||
result = send_future.fuse() => match result {
|
||||
Ok(Some(response)) => {
|
||||
eprintln!("[eval-cli] stopped: {:?}", response.stop_reason);
|
||||
if response.stop_reason == acp::StopReason::MaxTokens {
|
||||
Err(anyhow::anyhow!("Model hit maximum token limit"))
|
||||
} else {
|
||||
Ok(AgentOutcome::Completed)
|
||||
}
|
||||
}
|
||||
Ok(None) => {
|
||||
eprintln!("[eval-cli] completed (no response)");
|
||||
Ok(AgentOutcome::Completed)
|
||||
}
|
||||
Err(e) => Err(e).context("agent run failed"),
|
||||
},
|
||||
_ = sigterm_future.fuse() => {
|
||||
eprintln!("[eval-cli] received SIGTERM, cancelling...");
|
||||
acp_thread.update(cx, |t: &mut acp_thread::AcpThread, cx| t.cancel(cx)).await;
|
||||
Ok(AgentOutcome::Interrupted)
|
||||
},
|
||||
_ = timeout_future.fuse() => {
|
||||
acp_thread.update(cx, |t: &mut acp_thread::AcpThread, cx| t.cancel(cx)).await;
|
||||
Ok(AgentOutcome::Timeout { seconds: timeout.unwrap_or(0) })
|
||||
}
|
||||
};
|
||||
|
||||
let thread = cx.update(|cx| {
|
||||
let session_id = acp_thread.read(cx).session_id().clone();
|
||||
connection.thread(&session_id, cx)
|
||||
});
|
||||
|
||||
let cumulative_usage = if let Some(thread) = &thread {
|
||||
let db_thread = thread.read_with(cx, |thread, cx| thread.to_db(cx));
|
||||
let db_thread = db_thread.await;
|
||||
let usage = db_thread.cumulative_token_usage;
|
||||
if usage.input_tokens > 0 || usage.output_tokens > 0 {
|
||||
Some(usage)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let acp_usage = cx.update(|cx| {
|
||||
acp_thread
|
||||
.read(cx)
|
||||
.token_usage()
|
||||
.map(|usage| language_model::TokenUsage {
|
||||
input_tokens: usage.input_tokens,
|
||||
output_tokens: usage.output_tokens,
|
||||
..Default::default()
|
||||
})
|
||||
});
|
||||
|
||||
let final_usage = cumulative_usage.or(acp_usage);
|
||||
|
||||
if let (Some(thread), Some(dir)) = (&thread, output_dir) {
|
||||
let markdown = thread.read_with(cx, |thread, _cx| thread.to_markdown());
|
||||
if let Err(e) = std::fs::write(dir.join("thread.md"), markdown) {
|
||||
eprintln!("Error writing thread.md: {e:#}");
|
||||
}
|
||||
|
||||
let db_thread = thread.read_with(cx, |thread, cx| thread.to_db(cx));
|
||||
let db_thread = db_thread.await;
|
||||
match serde_json::to_string_pretty(&db_thread) {
|
||||
Ok(json) => {
|
||||
if let Err(e) = std::fs::write(dir.join("thread.json"), json) {
|
||||
eprintln!("Error writing thread.json: {e:#}");
|
||||
}
|
||||
}
|
||||
Err(e) => eprintln!("Error serializing thread.json: {e:#}"),
|
||||
}
|
||||
}
|
||||
|
||||
(outcome, final_usage)
|
||||
}
|
||||
|
||||
fn log_acp_thread_event(
|
||||
acp_thread: &Entity<acp_thread::AcpThread>,
|
||||
event: &acp_thread::AcpThreadEvent,
|
||||
cx: &mut gpui::App,
|
||||
) {
|
||||
match event {
|
||||
acp_thread::AcpThreadEvent::NewEntry => {
|
||||
let entries = acp_thread.read(cx).entries();
|
||||
if let Some(acp_thread::AgentThreadEntry::AssistantMessage(message)) = entries.last() {
|
||||
for chunk in &message.chunks {
|
||||
if let acp_thread::AssistantMessageChunk::Message { block } = chunk {
|
||||
if let acp_thread::ContentBlock::Markdown { markdown } = block {
|
||||
let text = markdown.read(cx).source().to_string();
|
||||
if !text.is_empty() {
|
||||
eprint!("{text}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
acp_thread::AcpThreadEvent::EntryUpdated(index) => {
|
||||
let entries = acp_thread.read(cx).entries();
|
||||
if let Some(acp_thread::AgentThreadEntry::ToolCall(tool_call)) = entries.get(*index) {
|
||||
if let Some(name) = &tool_call.tool_name {
|
||||
match &tool_call.status {
|
||||
acp_thread::ToolCallStatus::Completed => {
|
||||
eprintln!("[tool] {name} ✓");
|
||||
}
|
||||
acp_thread::ToolCallStatus::Failed => {
|
||||
eprintln!("[tool] {name} ✗");
|
||||
}
|
||||
acp_thread::ToolCallStatus::Rejected => {
|
||||
eprintln!("[tool] {name} rejected");
|
||||
}
|
||||
acp_thread::ToolCallStatus::Canceled => {
|
||||
eprintln!("[tool] {name} canceled");
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
acp_thread::AcpThreadEvent::Stopped(reason) => {
|
||||
eprintln!("\n[eval-cli] stopped: {reason:?}");
|
||||
}
|
||||
acp_thread::AcpThreadEvent::Error => {
|
||||
eprintln!("[eval-cli] error event");
|
||||
}
|
||||
acp_thread::AcpThreadEvent::Retry(status) => {
|
||||
eprintln!("[eval-cli] retry: {status:?}");
|
||||
}
|
||||
acp_thread::AcpThreadEvent::SubagentSpawned(session_id) => {
|
||||
eprintln!("[eval-cli] subagent spawned: {session_id}");
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
3
crates/eval_cli/zed_eval/__init__.py
Normal file
3
crates/eval_cli/zed_eval/__init__.py
Normal file
@@ -0,0 +1,3 @@
|
||||
from zed_eval.agent import ZedAgent
|
||||
|
||||
__all__ = ["ZedAgent"]
|
||||
460
crates/eval_cli/zed_eval/agent.py
Normal file
460
crates/eval_cli/zed_eval/agent.py
Normal file
@@ -0,0 +1,460 @@
|
||||
"""Harbor agent wrapper for Zed's eval-cli binary.
|
||||
|
||||
Usage:
|
||||
# Build eval-cli locally first:
|
||||
cargo build --release -p eval_cli
|
||||
|
||||
# Run via Harbor with a local binary:
|
||||
harbor run -d "dataset@version" \
|
||||
--agent-import-path zed_eval.agent:ZedAgent \
|
||||
--ae binary_path=/path/to/target/release/eval-cli \
|
||||
--agent-model anthropic/claude-sonnet-4-6-latest
|
||||
|
||||
# Or with a download URL (for CI):
|
||||
harbor run -d "dataset@version" \
|
||||
--agent-import-path zed_eval.agent:ZedAgent \
|
||||
--ae download_url=https://example.com/eval-cli \
|
||||
--agent-model anthropic/claude-sonnet-4-6-latest
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import shlex
|
||||
from pathlib import Path
|
||||
|
||||
from harbor.agents.installed.base import BaseInstalledAgent, with_prompt_template
|
||||
from harbor.environments.base import BaseEnvironment
|
||||
from harbor.models.agent.context import AgentContext
|
||||
|
||||
|
||||
class ZedAgent(BaseInstalledAgent):
|
||||
"""Runs Zed's headless AI agent (eval-cli) to solve tasks.
|
||||
|
||||
The eval-cli binary boots a headless GPUI application and uses the same
|
||||
NativeAgent + AcpThread pipeline as the production Zed editor, driving
|
||||
the full agentic loop (tool calls, subagents, retries) without a GUI.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
logs_dir: Path,
|
||||
binary_path: str | None = None,
|
||||
download_url: str | None = None,
|
||||
*args,
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__(logs_dir, *args, **kwargs)
|
||||
self._binary_path = binary_path
|
||||
self._download_url = download_url or os.environ.get("EVAL_CLI_DOWNLOAD_URL")
|
||||
|
||||
@staticmethod
|
||||
def name() -> str:
|
||||
return "zed"
|
||||
|
||||
async def _detect_workdir(self, environment: BaseEnvironment) -> str:
|
||||
"""Detect the working directory inside the container.
|
||||
|
||||
Checks, in order:
|
||||
1. Explicit ``EVAL_CLI_WORKDIR`` extra-env override
|
||||
2. Well-known dirs with a ``.git`` subdirectory (SWE-bench style)
|
||||
3. First git repo found under ``/`` (max depth 3)
|
||||
4. Well-known dirs that exist at all (terminal-bench style)
|
||||
5. The container's default working directory (``pwd``)
|
||||
"""
|
||||
override = self._extra_env.get("EVAL_CLI_WORKDIR")
|
||||
if override:
|
||||
return override
|
||||
|
||||
# First: try to find a git repo (SWE-bench, etc.)
|
||||
result = await self.exec_as_agent(
|
||||
environment,
|
||||
command=(
|
||||
"for d in /app /testbed /repo; do "
|
||||
' if [ -d "$d/.git" ]; then echo "$d"; exit 0; fi; '
|
||||
"done; "
|
||||
"find / -maxdepth 3 -name .git -type d 2>/dev/null "
|
||||
'| head -1 | sed "s|/.git$||"'
|
||||
),
|
||||
)
|
||||
workdir = (result.stdout or "").strip()
|
||||
if workdir:
|
||||
return workdir
|
||||
|
||||
# Fallback: use the first well-known directory that exists,
|
||||
# even without .git (terminal-bench containers aren't git repos).
|
||||
result = await self.exec_as_agent(
|
||||
environment,
|
||||
command=(
|
||||
"for d in /app /testbed /repo /root /home; do "
|
||||
' if [ -d "$d" ]; then echo "$d"; exit 0; fi; '
|
||||
"done; "
|
||||
"pwd"
|
||||
),
|
||||
)
|
||||
workdir = (result.stdout or "").strip()
|
||||
if workdir:
|
||||
return workdir
|
||||
|
||||
raise RuntimeError(
|
||||
"Could not detect a working directory in the container. "
|
||||
"Set EVAL_CLI_WORKDIR explicitly via --ae EVAL_CLI_WORKDIR=/path/to/repo"
|
||||
)
|
||||
|
||||
async def install(self, environment: BaseEnvironment) -> None:
|
||||
# Detect the package manager and install base dependencies.
|
||||
# Supports Debian/Ubuntu (apt-get), Alpine (apk), and
|
||||
# Fedora/RHEL/CentOS (dnf/yum).
|
||||
await self.exec_as_root(
|
||||
environment,
|
||||
command=(
|
||||
"if command -v apt-get >/dev/null 2>&1; then "
|
||||
" apt-get update && "
|
||||
" apt-get install -y --no-install-recommends ca-certificates curl git; "
|
||||
"elif command -v apk >/dev/null 2>&1; then "
|
||||
" apk add --no-cache ca-certificates curl git bash coreutils gcompat libstdc++; "
|
||||
"elif command -v dnf >/dev/null 2>&1; then "
|
||||
" dnf install -y ca-certificates curl git; "
|
||||
"elif command -v yum >/dev/null 2>&1; then "
|
||||
" yum install -y ca-certificates curl git; "
|
||||
"else "
|
||||
" echo 'WARNING: No supported package manager found (apt-get, apk, dnf, yum)' >&2; "
|
||||
"fi"
|
||||
),
|
||||
env={"DEBIAN_FRONTEND": "noninteractive"},
|
||||
)
|
||||
|
||||
# ── Non-essential tooling ─────────────────────────────────────
|
||||
# Everything below here (Node.js, LSPs, uv/ruff) is nice-to-have.
|
||||
# If any step fails (e.g. musl incompatibility, network issues),
|
||||
# log a warning and continue — the agent can still work without
|
||||
# pre-installed language servers.
|
||||
|
||||
await self._install_node(environment)
|
||||
await self._install_lsps(environment)
|
||||
await self._install_uv_and_ruff(environment)
|
||||
|
||||
if self._binary_path:
|
||||
binary = Path(self._binary_path)
|
||||
if not binary.exists():
|
||||
raise FileNotFoundError(
|
||||
f"eval-cli binary not found at {binary}. "
|
||||
"Build it with: cargo build --release -p eval_cli"
|
||||
)
|
||||
await environment.upload_file(
|
||||
source_path=binary,
|
||||
target_path="/usr/local/bin/eval-cli",
|
||||
)
|
||||
await self.exec_as_root(
|
||||
environment,
|
||||
command="chmod +x /usr/local/bin/eval-cli && eval-cli --help",
|
||||
)
|
||||
return
|
||||
|
||||
if self._download_url:
|
||||
await self.exec_as_root(
|
||||
environment,
|
||||
command=(
|
||||
f"curl -fsSL {shlex.quote(self._download_url)} "
|
||||
"-o /usr/local/bin/eval-cli && "
|
||||
"chmod +x /usr/local/bin/eval-cli && "
|
||||
"eval-cli --help"
|
||||
),
|
||||
)
|
||||
return
|
||||
|
||||
raise ValueError(
|
||||
"No eval-cli binary provided. "
|
||||
"Either pass binary_path=/path/to/target/release/eval-cli "
|
||||
"or set download_url=/EVAL_CLI_DOWNLOAD_URL."
|
||||
)
|
||||
|
||||
async def _install_node(self, environment: BaseEnvironment) -> None:
|
||||
"""Install Node.js from official binary tarballs.
|
||||
|
||||
Uses the musl build on Alpine and the glibc build elsewhere.
|
||||
Skips if node is already on PATH.
|
||||
"""
|
||||
try:
|
||||
await self.exec_as_root(
|
||||
environment,
|
||||
command=(
|
||||
"if command -v node >/dev/null 2>&1; then "
|
||||
' echo "Node.js already available: $(node --version)"; '
|
||||
"else "
|
||||
" NODE_VER=v22.14.0; "
|
||||
" ARCH=$(uname -m); "
|
||||
' case "$ARCH" in '
|
||||
" x86_64) NODE_ARCH=x64 ;; "
|
||||
" aarch64) NODE_ARCH=arm64 ;; "
|
||||
' *) echo "WARNING: unsupported arch $ARCH for Node.js" >&2; exit 0 ;; '
|
||||
" esac; "
|
||||
" if ldd /bin/sh 2>&1 | grep -qi musl; then "
|
||||
' NODE_URL="https://unofficial-builds.nodejs.org/download/release/${NODE_VER}/node-${NODE_VER}-linux-${NODE_ARCH}-musl.tar.gz"; '
|
||||
" else "
|
||||
' NODE_URL="https://nodejs.org/dist/${NODE_VER}/node-${NODE_VER}-linux-${NODE_ARCH}.tar.gz"; '
|
||||
" fi; "
|
||||
' echo "Downloading Node.js from $NODE_URL"; '
|
||||
' curl -fsSL "$NODE_URL" | tar -xz -C /usr/local --strip-components=1; '
|
||||
' echo "Installed Node.js $(node --version)"; '
|
||||
"fi"
|
||||
),
|
||||
)
|
||||
except Exception as exc:
|
||||
self.logger.warning("Node.js installation failed (non-fatal): %s", exc)
|
||||
|
||||
async def _install_lsps(self, environment: BaseEnvironment) -> None:
|
||||
"""Pre-install language servers so Zed doesn't download them at runtime.
|
||||
|
||||
Each LSP is installed independently so one failure doesn't block the rest.
|
||||
"""
|
||||
# npm-based LSPs — skip all if npm is not available.
|
||||
try:
|
||||
await self.exec_as_agent(
|
||||
environment,
|
||||
command="command -v npm >/dev/null 2>&1",
|
||||
)
|
||||
except Exception:
|
||||
self.logger.warning("npm not available — skipping npm-based LSP installs")
|
||||
return
|
||||
|
||||
lsp_installs = [
|
||||
(
|
||||
"basedpyright",
|
||||
'DIR="$ZED_DATA_DIR/languages/basedpyright"; '
|
||||
'mkdir -p "$DIR" && npm install --prefix "$DIR" --save-exact basedpyright',
|
||||
),
|
||||
(
|
||||
"typescript-language-server",
|
||||
'DIR="$ZED_DATA_DIR/languages/typescript-language-server"; '
|
||||
'mkdir -p "$DIR" && npm install --prefix "$DIR" --save-exact typescript typescript-language-server',
|
||||
),
|
||||
(
|
||||
"vtsls",
|
||||
'DIR="$ZED_DATA_DIR/languages/vtsls"; '
|
||||
'mkdir -p "$DIR" && npm install --prefix "$DIR" --save-exact @vtsls/language-server typescript',
|
||||
),
|
||||
(
|
||||
"tailwindcss-language-server",
|
||||
'DIR="$ZED_DATA_DIR/languages/tailwindcss-language-server"; '
|
||||
'mkdir -p "$DIR" && npm install --prefix "$DIR" --save-exact @tailwindcss/language-server',
|
||||
),
|
||||
]
|
||||
|
||||
for name, cmd in lsp_installs:
|
||||
try:
|
||||
await self.exec_as_agent(
|
||||
environment,
|
||||
command=(
|
||||
'ZED_DATA_DIR="${XDG_DATA_HOME:-$HOME/.local/share}/zed"; '
|
||||
+ cmd
|
||||
),
|
||||
)
|
||||
except Exception as exc:
|
||||
self.logger.warning(
|
||||
"LSP install '%s' failed (non-fatal): %s", name, exc
|
||||
)
|
||||
|
||||
# eslint — downloaded from GitHub and compiled separately.
|
||||
try:
|
||||
await self.exec_as_agent(
|
||||
environment,
|
||||
command=(
|
||||
"set -euo pipefail; "
|
||||
'ZED_DATA_DIR="${XDG_DATA_HOME:-$HOME/.local/share}/zed"; '
|
||||
'ESLINT_DIR="$ZED_DATA_DIR/languages/eslint/vscode-eslint-2.4.4"; '
|
||||
'mkdir -p "$ESLINT_DIR"; '
|
||||
'curl -fsSL "https://github.com/zed-industries/vscode-eslint/archive/refs/tags/release/2.4.4.tar.gz" '
|
||||
'| tar -xz -C "$ESLINT_DIR"; '
|
||||
'mv "$ESLINT_DIR"/vscode-eslint-release-2.4.4 "$ESLINT_DIR/vscode-eslint"; '
|
||||
'cd "$ESLINT_DIR/vscode-eslint" && npm install && npm run compile'
|
||||
),
|
||||
)
|
||||
except Exception as exc:
|
||||
self.logger.warning("eslint LSP install failed (non-fatal): %s", exc)
|
||||
|
||||
# gopls — only when Go is present. Guarded by a 120s timeout so slow
|
||||
# compilation can never eat the full setup budget.
|
||||
gopls_script = (
|
||||
"if command -v go >/dev/null 2>&1; then "
|
||||
"if go install golang.org/x/tools/gopls@latest 2>/dev/null; then "
|
||||
"echo 'Installed gopls@latest'; "
|
||||
"else "
|
||||
' MY_GO=$(go env GOVERSION | sed "s/^go//"); '
|
||||
" for v in $(curl -fsSL "
|
||||
"https://proxy.golang.org/golang.org/x/tools/gopls/@v/list 2>/dev/null"
|
||||
" | grep -E '^v[0-9]+\\.[0-9]+\\.[0-9]+$' | sort -rV | head -5); do "
|
||||
" NEED=$(curl -fsSL "
|
||||
'"https://proxy.golang.org/golang.org/x/tools/gopls/@v/${v}.mod"'
|
||||
" 2>/dev/null | awk '/^go /{print $2; exit}'); "
|
||||
' if [ -n "$NEED" ] '
|
||||
' && [ "$(printf \'%s\\n%s\\n\' "$NEED" "$MY_GO" '
|
||||
' | sort -V | head -1)" = "$NEED" ]; then '
|
||||
' echo "Installing gopls $v (compatible with Go $MY_GO)"; '
|
||||
' go install "golang.org/x/tools/gopls@$v" && break; '
|
||||
" fi; "
|
||||
" done; "
|
||||
"fi; "
|
||||
"fi"
|
||||
)
|
||||
try:
|
||||
await self.exec_as_agent(
|
||||
environment,
|
||||
command=(
|
||||
"timeout 120 bash -c "
|
||||
+ shlex.quote(gopls_script)
|
||||
+ " || echo 'WARNING: gopls installation timed out or failed -- skipping'"
|
||||
),
|
||||
)
|
||||
except Exception as exc:
|
||||
self.logger.warning("gopls install failed (non-fatal): %s", exc)
|
||||
|
||||
async def _install_uv_and_ruff(self, environment: BaseEnvironment) -> None:
|
||||
"""Install uv and ruff for Python tooling."""
|
||||
try:
|
||||
await self.exec_as_agent(
|
||||
environment,
|
||||
command=(
|
||||
"curl -LsSf https://astral.sh/uv/install.sh | sh && "
|
||||
'. "$HOME/.local/bin/env"'
|
||||
),
|
||||
)
|
||||
|
||||
agent_home_result = await self.exec_as_agent(
|
||||
environment,
|
||||
command='printf %s "$HOME"',
|
||||
)
|
||||
agent_home = agent_home_result.stdout.strip()
|
||||
if not agent_home:
|
||||
self.logger.warning(
|
||||
"Could not determine agent home directory — skipping uv symlinks"
|
||||
)
|
||||
return
|
||||
|
||||
await self.exec_as_root(
|
||||
environment,
|
||||
command=(
|
||||
f"ln -sf {shlex.quote(agent_home + '/.local/bin/uv')} /usr/local/bin/uv && "
|
||||
f"ln -sf {shlex.quote(agent_home + '/.local/bin/uvx')} /usr/local/bin/uvx"
|
||||
),
|
||||
)
|
||||
|
||||
await self.exec_as_agent(
|
||||
environment,
|
||||
command='export PATH="$HOME/.local/bin:$PATH" && uv tool install ruff',
|
||||
)
|
||||
except Exception as exc:
|
||||
self.logger.warning("uv/ruff installation failed (non-fatal): %s", exc)
|
||||
|
||||
def populate_context_post_run(self, context: AgentContext) -> None:
|
||||
result_data = None
|
||||
for json_file in self.logs_dir.rglob("result.json"):
|
||||
try:
|
||||
result_data = json.loads(json_file.read_text())
|
||||
break
|
||||
except (json.JSONDecodeError, OSError):
|
||||
continue
|
||||
|
||||
if result_data is None:
|
||||
self.logger.warning("Could not find or parse result.json from eval-cli")
|
||||
return
|
||||
|
||||
if result_data.get("input_tokens") is not None:
|
||||
context.n_input_tokens = result_data["input_tokens"]
|
||||
if result_data.get("output_tokens") is not None:
|
||||
context.n_output_tokens = result_data["output_tokens"]
|
||||
if result_data.get("cache_read_input_tokens") is not None:
|
||||
context.n_cache_tokens = result_data["cache_read_input_tokens"]
|
||||
|
||||
context.metadata = {
|
||||
"status": result_data.get("status"),
|
||||
"duration_secs": result_data.get("duration_secs"),
|
||||
"model": result_data.get("model"),
|
||||
}
|
||||
|
||||
def _get_api_env(self) -> dict[str, str]:
|
||||
env: dict[str, str] = {}
|
||||
if not self.model_name or "/" not in self.model_name:
|
||||
return env
|
||||
|
||||
provider = self.model_name.split("/", 1)[0]
|
||||
provider_env_map = {
|
||||
"anthropic": "ANTHROPIC_API_KEY",
|
||||
"openai": "OPENAI_API_KEY",
|
||||
"google": "GEMINI_API_KEY",
|
||||
"gemini": "GEMINI_API_KEY",
|
||||
"deepseek": "DEEPSEEK_API_KEY",
|
||||
"mistral": "MISTRAL_API_KEY",
|
||||
}
|
||||
|
||||
env_var = provider_env_map.get(provider)
|
||||
if env_var:
|
||||
api_key = os.environ.get(env_var, "")
|
||||
if api_key:
|
||||
env[env_var] = api_key
|
||||
|
||||
return env
|
||||
|
||||
@with_prompt_template
|
||||
async def run(
|
||||
self, instruction: str, environment: BaseEnvironment, context: AgentContext
|
||||
) -> None:
|
||||
escaped_instruction = shlex.quote(instruction)
|
||||
env = self._get_api_env()
|
||||
|
||||
workdir = await self._detect_workdir(environment)
|
||||
|
||||
parts = [
|
||||
"eval-cli",
|
||||
f"--workdir {shlex.quote(workdir)}",
|
||||
"--output-dir /logs/agent",
|
||||
]
|
||||
|
||||
if self.model_name:
|
||||
parts.append(f"--model {shlex.quote(self.model_name)}")
|
||||
|
||||
timeout = self._extra_env.get("EVAL_CLI_TIMEOUT")
|
||||
if timeout:
|
||||
parts.append(f"--timeout {shlex.quote(timeout)}")
|
||||
|
||||
staff = self._extra_env.get("EVAL_CLI_STAFF")
|
||||
if staff and staff.lower() == "false":
|
||||
parts.append("--no-staff")
|
||||
|
||||
reasoning_effort = self._extra_env.get("EVAL_CLI_REASONING_EFFORT")
|
||||
if reasoning_effort:
|
||||
parts.append(f"--reasoning-effort {shlex.quote(reasoning_effort)}")
|
||||
|
||||
enable_thinking = self._extra_env.get("EVAL_CLI_ENABLE_THINKING")
|
||||
if enable_thinking:
|
||||
if enable_thinking.lower() == "true":
|
||||
parts.append("--enable-thinking")
|
||||
elif enable_thinking.lower() == "false":
|
||||
parts.append("--disable-thinking")
|
||||
|
||||
parts.append(f"--instruction {escaped_instruction}")
|
||||
|
||||
await self.exec_as_agent(
|
||||
environment,
|
||||
command=(
|
||||
" ".join(parts) + " 2>&1 | if command -v stdbuf >/dev/null 2>&1;"
|
||||
" then stdbuf -oL tee /logs/agent/eval-cli.txt;"
|
||||
" else tee /logs/agent/eval-cli.txt; fi"
|
||||
),
|
||||
env=env,
|
||||
)
|
||||
|
||||
# Only generate a patch if the workdir is a git repo
|
||||
# (SWE-bench style). Terminal-bench containers aren't git repos.
|
||||
await self.exec_as_agent(
|
||||
environment,
|
||||
command=(
|
||||
'if [ -d ".git" ]; then '
|
||||
"git add -A && "
|
||||
"git diff --cached HEAD > /logs/agent/patch.diff && "
|
||||
'echo "Patch size: $(wc -c < /logs/agent/patch.diff) bytes"; '
|
||||
"else "
|
||||
'echo "No git repo found, skipping patch generation"; '
|
||||
"fi"
|
||||
),
|
||||
cwd=workdir,
|
||||
)
|
||||
10
crates/eval_cli/zed_eval/pyproject.toml
Normal file
10
crates/eval_cli/zed_eval/pyproject.toml
Normal file
@@ -0,0 +1,10 @@
|
||||
[project]
|
||||
name = "zed-eval"
|
||||
version = "0.1.0"
|
||||
description = "Harbor agent wrapper for Zed's eval-cli"
|
||||
requires-python = ">=3.12"
|
||||
dependencies = ["harbor==0.6.4"]
|
||||
|
||||
[build-system]
|
||||
requires = ["setuptools"]
|
||||
build-backend = "setuptools.build_meta"
|
||||
Reference in New Issue
Block a user