logiguard fork v3: full patch set on verified 8c74db0 tree
Includes prior-session patches (carry forward so the app compiles): - crates/gpui/build.rs: cross-compile manifest fix - crates/gpui/src/platform.rs: PlatformWindow::activate_with_token trait method - crates/gpui/src/window.rs: Window::activate_with_token public API - crates/gpui_linux/src/linux/wayland/window.rs: WaylandWindow::activate_with_token + activate() keyboard-serial fix Plus the focus-serial fix: - serial.rs: SerialKind::KeyboardEnter - client.rs: store wl_keyboard.enter serial; latest_serial_of() Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
18
crates/eval_utils/Cargo.toml
Normal file
18
crates/eval_utils/Cargo.toml
Normal file
@@ -0,0 +1,18 @@
|
||||
[package]
|
||||
name = "eval_utils"
|
||||
version = "0.1.0"
|
||||
edition.workspace = true
|
||||
publish.workspace = true
|
||||
license = "GPL-3.0-or-later"
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[lib]
|
||||
path = "src/eval_utils.rs"
|
||||
doctest = false
|
||||
|
||||
[dependencies]
|
||||
gpui_platform.workspace = true
|
||||
serde.workspace = true
|
||||
smol.workspace = true
|
||||
1
crates/eval_utils/LICENSE-GPL
Symbolic link
1
crates/eval_utils/LICENSE-GPL
Symbolic link
@@ -0,0 +1 @@
|
||||
../../LICENSE-GPL
|
||||
3
crates/eval_utils/README.md
Normal file
3
crates/eval_utils/README.md
Normal file
@@ -0,0 +1,3 @@
|
||||
# eval_utils
|
||||
|
||||
Utilities for evals of agents.
|
||||
146
crates/eval_utils/src/eval_utils.rs
Normal file
146
crates/eval_utils/src/eval_utils.rs
Normal file
@@ -0,0 +1,146 @@
|
||||
//! Utilities for evaluation and benchmarking.
|
||||
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
sync::{Arc, mpsc},
|
||||
};
|
||||
|
||||
fn report_progress(evaluated_count: usize, failed_count: usize, iterations: usize) {
|
||||
let passed_count = evaluated_count - failed_count;
|
||||
let passed_ratio = if evaluated_count == 0 {
|
||||
0.0
|
||||
} else {
|
||||
passed_count as f64 / evaluated_count as f64
|
||||
};
|
||||
println!(
|
||||
"\r\x1b[KEvaluated {}/{} ({:.2}% passed)",
|
||||
evaluated_count,
|
||||
iterations,
|
||||
passed_ratio * 100.0
|
||||
)
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub enum OutcomeKind {
|
||||
Passed,
|
||||
Failed,
|
||||
Error,
|
||||
}
|
||||
|
||||
pub trait EvalOutputProcessor {
|
||||
type Metadata: 'static + Send;
|
||||
fn process(&mut self, output: &EvalOutput<Self::Metadata>);
|
||||
fn assert(&mut self);
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct EvalOutput<M> {
|
||||
pub outcome: OutcomeKind,
|
||||
pub data: String,
|
||||
pub metadata: M,
|
||||
}
|
||||
|
||||
impl<M: Default> EvalOutput<M> {
|
||||
pub fn passed(message: impl Into<String>) -> Self {
|
||||
EvalOutput {
|
||||
outcome: OutcomeKind::Passed,
|
||||
data: message.into(),
|
||||
metadata: M::default(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn failed(message: impl Into<String>) -> Self {
|
||||
EvalOutput {
|
||||
outcome: OutcomeKind::Failed,
|
||||
data: message.into(),
|
||||
metadata: M::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct NoProcessor;
|
||||
impl EvalOutputProcessor for NoProcessor {
|
||||
type Metadata = ();
|
||||
|
||||
fn process(&mut self, _output: &EvalOutput<Self::Metadata>) {}
|
||||
|
||||
fn assert(&mut self) {}
|
||||
}
|
||||
|
||||
pub fn eval<P>(
|
||||
iterations: usize,
|
||||
expected_pass_ratio: f32,
|
||||
mut processor: P,
|
||||
evalf: impl Fn() -> EvalOutput<P::Metadata> + Send + Sync + 'static,
|
||||
) where
|
||||
P: EvalOutputProcessor,
|
||||
{
|
||||
let mut evaluated_count = 0;
|
||||
let mut failed_count = 0;
|
||||
let evalf = Arc::new(evalf);
|
||||
report_progress(evaluated_count, failed_count, iterations);
|
||||
|
||||
let (tx, rx) = mpsc::channel();
|
||||
|
||||
let executor = gpui_platform::background_executor();
|
||||
let semaphore = Arc::new(smol::lock::Semaphore::new(32));
|
||||
let evalf = Arc::new(evalf);
|
||||
// Warm the cache once
|
||||
let first_output = evalf();
|
||||
tx.send(first_output).ok();
|
||||
|
||||
for _ in 1..iterations {
|
||||
let tx = tx.clone();
|
||||
let semaphore = semaphore.clone();
|
||||
let evalf = evalf.clone();
|
||||
executor
|
||||
.spawn(async move {
|
||||
let _guard = semaphore.acquire().await;
|
||||
let output = evalf();
|
||||
tx.send(output).ok();
|
||||
})
|
||||
.detach();
|
||||
}
|
||||
drop(tx);
|
||||
|
||||
let mut failed_evals = Vec::new();
|
||||
let mut errored_evals = HashMap::new();
|
||||
while let Ok(output) = rx.recv() {
|
||||
processor.process(&output);
|
||||
|
||||
match output.outcome {
|
||||
OutcomeKind::Passed => {}
|
||||
OutcomeKind::Failed => {
|
||||
failed_count += 1;
|
||||
failed_evals.push(output);
|
||||
}
|
||||
OutcomeKind::Error => {
|
||||
failed_count += 1;
|
||||
*errored_evals.entry(output.data).or_insert(0) += 1;
|
||||
}
|
||||
}
|
||||
|
||||
evaluated_count += 1;
|
||||
report_progress(evaluated_count, failed_count, iterations);
|
||||
}
|
||||
|
||||
let actual_pass_ratio = (iterations - failed_count) as f32 / iterations as f32;
|
||||
println!("Actual pass ratio: {}\n", actual_pass_ratio);
|
||||
if actual_pass_ratio < expected_pass_ratio {
|
||||
for (error, count) in errored_evals {
|
||||
println!("Eval errored {} times. Error: {}", count, error);
|
||||
}
|
||||
|
||||
for failed in failed_evals {
|
||||
println!("Eval failed");
|
||||
println!("{}", failed.data);
|
||||
}
|
||||
|
||||
panic!(
|
||||
"Actual pass ratio: {}\nExpected pass ratio: {}",
|
||||
actual_pass_ratio, expected_pass_ratio
|
||||
);
|
||||
}
|
||||
|
||||
processor.assert();
|
||||
}
|
||||
Reference in New Issue
Block a user