logiguard fork: GPUI xdg-activation keyboard-focus serial fix

Single-commit orphan branch: full zed-industries/zed @ 8c74db0 source tree
with a 3-file patch applied (no upstream history).

Patch (crates/gpui_linux/src/linux/wayland/):
  - serial.rs: add SerialKind::KeyboardEnter
  - client.rs: store wl_keyboard.enter serial; add latest_serial_of()
  - window.rs: activate() uses keyboard-enter serial (Mutter focus gate)

Mutter honors window activation only when the token carries the keyboard-
focus serial from wl_keyboard.enter; GPUI used a stale mouse-press serial.
See docs/tray-window-focus-wayland.md in logiguard.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

crates/edit_prediction_metrics/Cargo.toml

@@ -0,0 +1,24 @@
+[package]
+name = "edit_prediction_metrics"
+version = "0.1.0"
+publish.workspace = true
+edition.workspace = true
+license = "GPL-3.0-or-later"
+
+[lints]
+workspace = true
+
+[lib]
+path = "src/edit_prediction_metrics.rs"
+
+[dependencies]
+imara-diff.workspace = true
+serde.workspace = true
+serde_json = "1.0"
+similar = "2.7.0"
+tree-sitter.workspace = true
+zeta_prompt.workspace = true
+
+[dev-dependencies]
+indoc.workspace = true
+pretty_assertions.workspace = true

crates/edit_prediction_metrics/LICENSE-GPL

@@ -0,0 +1 @@
+../../LICENSE-GPL

crates/edit_prediction_metrics/src/edit_prediction_metrics.rs

@@ -0,0 +1,33 @@
+mod kept_rate;
+mod patch_metrics;
+mod prediction_score;
+mod reversal;
+mod summary;
+mod tokenize;
+mod tree_sitter;
+
+pub use kept_rate::AnnotatedToken;
+pub use kept_rate::KeptRateResult;
+pub use kept_rate::TokenAnnotation;
+pub use kept_rate::annotate_kept_rate_tokens;
+pub use kept_rate::compute_kept_rate;
+pub use patch_metrics::ClassificationMetrics;
+pub use patch_metrics::Counts;
+pub use patch_metrics::DeltaChrFMetrics;
+pub use patch_metrics::TokenChangeCounts;
+pub use patch_metrics::braces_disbalance;
+pub use patch_metrics::count_patch_token_changes;
+pub use patch_metrics::delta_chr_f;
+pub use patch_metrics::delta_chr_f_beta;
+pub use patch_metrics::exact_lines_match;
+pub use patch_metrics::extract_changed_lines_from_diff;
+pub use patch_metrics::has_isolated_whitespace_changes;
+pub use patch_metrics::is_editable_region_correct;
+pub use patch_metrics::reconstruct_texts_from_diff;
+pub use prediction_score::{
+    ActualPredictionCursor, PredictionReversalContext, PredictionScore, PredictionScoringInput,
+    PrepareExpectedPatchError, PreparedExpectedPatch, prepare_expected_patches, score_prediction,
+};
+pub use reversal::compute_prediction_reversal_ratio_from_history;
+pub use summary::{PredictionSummaryInput, QaSummaryData, SummaryJson, compute_summary};
+pub use tree_sitter::count_tree_sitter_errors;

crates/edit_prediction_metrics/src/kept_rate.rs

@@ -0,0 +1,723 @@
+use crate::tokenize::tokenize;
+use serde::Serialize;
+
+const MAX_DIRTY_LENGTH_DELTA_CHARS: usize = 512;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
+#[serde(rename_all = "snake_case")]
+pub enum TokenAnnotation {
+    Context,
+    Kept,
+    Discarded,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
+pub struct AnnotatedToken {
+    pub token: String,
+    pub annotation: TokenAnnotation,
+}
+
+#[allow(dead_code)]
+#[derive(Debug, Clone, Serialize)]
+pub struct KeptRateResult {
+    /// Characters newly introduced by the candidate
+    pub candidate_new_chars: usize,
+    /// Characters newly introduced by the reference
+    pub reference_new_chars: usize,
+    /// Characters from `base` that are deleted by the candidate.
+    pub candidate_deleted_chars: usize,
+    /// Characters from `base` that are deleted by the reference.
+    pub reference_deleted_chars: usize,
+    /// Candidate new characters that are also present in the reference.
+    pub kept_chars: usize,
+    /// Base characters deleted by both the candidate and the reference.
+    pub correctly_deleted_chars: usize,
+    /// Candidate new characters that are not kept in the reference.
+    pub discarded_chars: usize,
+    /// Candidate characters treated as unchanged context
+    pub context_chars: usize,
+    /// Fraction of candidate edit characters that match the reference edit.
+    ///
+    /// This includes both kept newly introduced characters and correctly
+    /// deleted base characters.
+    pub kept_rate: f64,
+    /// Fraction of reference edit characters covered by the candidate edit.
+    ///
+    /// This includes both kept newly introduced characters and correctly
+    /// deleted base characters.
+    pub recall_rate: f64,
+    /// Per-token classification for candidate tokens.
+    pub token_annotations: Vec<TokenAnnotation>,
+}
+
+fn dp_index(width: usize, row: usize, column: usize) -> usize {
+    row * width + column
+}
+
+/// Fill masks over `a` and `b` using one-sided LCS tie-breaking for each side
+/// while sharing a single DP table construction.
+fn fill_lcs_keep_masks<T: Eq>(
+    a: &[T],
+    b: &[T],
+    mut keep_a: Option<&mut [bool]>,
+    mut keep_b: Option<&mut [bool]>,
+) {
+    if a.is_empty() || b.is_empty() {
+        return;
+    }
+
+    if a == b {
+        if let Some(keep_a) = keep_a.as_mut() {
+            keep_a.fill(true);
+        }
+        if let Some(keep_b) = keep_b.as_mut() {
+            keep_b.fill(true);
+        }
+        return;
+    }
+
+    let prefix_len = a
+        .iter()
+        .zip(b.iter())
+        .take_while(|(left, right)| left == right)
+        .count();
+    let suffix_len = {
+        let max_suffix = (a.len() - prefix_len).min(b.len() - prefix_len);
+        let mut suffix_len = 0;
+
+        while suffix_len < max_suffix {
+            let a_index = a.len() - 1 - suffix_len;
+            let b_index = b.len() - 1 - suffix_len;
+            if a[a_index] != b[b_index] {
+                break;
+            }
+            suffix_len += 1;
+        }
+
+        suffix_len
+    };
+
+    for index in 0..prefix_len {
+        if let Some(keep_a) = keep_a.as_mut() {
+            keep_a[index] = true;
+        }
+        if let Some(keep_b) = keep_b.as_mut() {
+            keep_b[index] = true;
+        }
+    }
+
+    for offset in 0..suffix_len {
+        let a_index = a.len() - suffix_len + offset;
+        let b_index = b.len() - suffix_len + offset;
+        if let Some(keep_a) = keep_a.as_mut() {
+            keep_a[a_index] = true;
+        }
+        if let Some(keep_b) = keep_b.as_mut() {
+            keep_b[b_index] = true;
+        }
+    }
+
+    let a_mid = &a[prefix_len..a.len() - suffix_len];
+    let b_mid = &b[prefix_len..b.len() - suffix_len];
+
+    if a_mid.is_empty() || b_mid.is_empty() {
+        return;
+    }
+
+    let row_count = a_mid.len() + 1;
+    let column_count = b_mid.len() + 1;
+    let mut dp = vec![0u32; row_count * column_count];
+
+    for i in 1..row_count {
+        let token_a = &a_mid[i - 1];
+        for j in 1..column_count {
+            let index = dp_index(column_count, i, j);
+            if token_a == &b_mid[j - 1] {
+                dp[index] = dp[dp_index(column_count, i - 1, j - 1)] + 1;
+            } else {
+                let up = dp[dp_index(column_count, i - 1, j)];
+                let left = dp[dp_index(column_count, i, j - 1)];
+                dp[index] = up.max(left);
+            }
+        }
+    }
+
+    if let Some(keep_a) = keep_a.as_mut() {
+        let mut i = a_mid.len();
+        let mut j = b_mid.len();
+
+        while i > 0 && j > 0 {
+            if a_mid[i - 1] == b_mid[j - 1] {
+                keep_a[prefix_len + i - 1] = true;
+                i -= 1;
+                j -= 1;
+            } else {
+                let up = dp[dp_index(column_count, i - 1, j)];
+                let left = dp[dp_index(column_count, i, j - 1)];
+                if up >= left {
+                    i -= 1;
+                } else {
+                    j -= 1;
+                }
+            }
+        }
+    }
+
+    if let Some(keep_b) = keep_b.as_mut() {
+        let mut i = a_mid.len();
+        let mut j = b_mid.len();
+
+        while i > 0 && j > 0 {
+            if a_mid[i - 1] == b_mid[j - 1] {
+                keep_b[prefix_len + j - 1] = true;
+                i -= 1;
+                j -= 1;
+            } else {
+                let up = dp[dp_index(column_count, i - 1, j)];
+                let left = dp[dp_index(column_count, i, j - 1)];
+                if left >= up {
+                    j -= 1;
+                } else {
+                    i -= 1;
+                }
+            }
+        }
+    }
+}
+
+fn lcs_keep_mask<T: Eq>(a: &[T], b: &[T]) -> Vec<bool> {
+    let mut keep_a = vec![false; a.len()];
+    fill_lcs_keep_masks(a, b, Some(&mut keep_a), None);
+    keep_a
+}
+
+fn lcs_keep_masks<T: Eq>(a: &[T], b: &[T]) -> (Vec<bool>, Vec<bool>) {
+    let mut keep_a = vec![false; a.len()];
+    let mut keep_b = vec![false; b.len()];
+    fill_lcs_keep_masks(a, b, Some(&mut keep_a), Some(&mut keep_b));
+    (keep_a, keep_b)
+}
+
+#[derive(Debug, Clone)]
+struct ComparisonUnit {
+    text: String,
+    token_start: usize,
+    token_end: usize,
+}
+
+fn is_identifier_token(token: &str) -> bool {
+    !token.is_empty()
+        && token
+            .chars()
+            .all(|character| character.is_alphanumeric() || character == '_')
+}
+
+fn build_comparison_units(tokens: &[&str]) -> Vec<ComparisonUnit> {
+    let mut units = Vec::new();
+    let mut index = 0;
+
+    while index < tokens.len() {
+        let token_start = index;
+
+        if is_identifier_token(tokens[index]) {
+            let mut text = String::new();
+
+            while index < tokens.len() && is_identifier_token(tokens[index]) {
+                text.push_str(tokens[index]);
+                index += 1;
+            }
+
+            units.push(ComparisonUnit {
+                text,
+                token_start,
+                token_end: index,
+            });
+        } else {
+            units.push(ComparisonUnit {
+                text: tokens[index].to_string(),
+                token_start,
+                token_end: index + 1,
+            });
+            index += 1;
+        }
+    }
+
+    units
+}
+
+fn analyze_masked_units<'a>(
+    units: &'a [ComparisonUnit],
+    mask: &[bool],
+) -> (Vec<&'a str>, usize, usize) {
+    let mut unmasked_units = Vec::with_capacity(units.len());
+    let mut unmasked_chars = 0;
+    let mut masked_chars = 0;
+
+    for (unit, &is_masked) in units.iter().zip(mask.iter()) {
+        if is_masked {
+            masked_chars += unit.text.len();
+        } else {
+            unmasked_units.push(unit.text.as_str());
+            unmasked_chars += unit.text.len();
+        }
+    }
+
+    (unmasked_units, unmasked_chars, masked_chars)
+}
+
+fn count_unmasked_unit_chars(units: &[ComparisonUnit], mask: &[bool]) -> usize {
+    units
+        .iter()
+        .zip(mask.iter())
+        .filter_map(|(unit, &is_masked)| (!is_masked).then_some(unit.text.len()))
+        .sum()
+}
+
+fn should_bail_for_dirty_final(base: &str, candidate: &str, reference: &str) -> bool {
+    let candidate_delta_chars = candidate.len().abs_diff(base.len());
+    let reference_delta_chars = reference.len().abs_diff(base.len());
+    candidate_delta_chars.abs_diff(reference_delta_chars) > MAX_DIRTY_LENGTH_DELTA_CHARS
+}
+
+pub fn compute_kept_rate(base: &str, candidate: &str, reference: &str) -> KeptRateResult {
+    if base == candidate && candidate == reference {
+        let candidate_tokens = tokenize(candidate);
+        let context_chars = candidate_tokens.iter().map(|token| token.len()).sum();
+        return KeptRateResult {
+            candidate_new_chars: 0,
+            reference_new_chars: 0,
+            candidate_deleted_chars: 0,
+            reference_deleted_chars: 0,
+            kept_chars: 0,
+            correctly_deleted_chars: 0,
+            discarded_chars: 0,
+            context_chars,
+            kept_rate: 1.0,
+            recall_rate: 1.0,
+            token_annotations: vec![TokenAnnotation::Context; candidate_tokens.len()],
+        };
+    }
+
+    if should_bail_for_dirty_final(base, candidate, reference) {
+        let candidate_new_chars = candidate.len().abs_diff(base.len());
+        let reference_new_chars = reference.len().abs_diff(base.len());
+        return KeptRateResult {
+            candidate_new_chars,
+            reference_new_chars,
+            candidate_deleted_chars: 0,
+            reference_deleted_chars: 0,
+            kept_chars: 0,
+            correctly_deleted_chars: 0,
+            discarded_chars: candidate_new_chars,
+            context_chars: 0,
+            kept_rate: 0.0,
+            recall_rate: 0.0,
+            token_annotations: vec![TokenAnnotation::Discarded; tokenize(candidate).len()],
+        };
+    }
+
+    let base_tokens = tokenize(base);
+    let candidate_tokens = tokenize(candidate);
+    let reference_tokens = tokenize(reference);
+
+    let candidate_units = build_comparison_units(&candidate_tokens);
+    let base_units = build_comparison_units(&base_tokens);
+    let reference_units = build_comparison_units(&reference_tokens);
+
+    let candidate_unit_texts: Vec<&str> = candidate_units
+        .iter()
+        .map(|unit| unit.text.as_str())
+        .collect();
+    let base_unit_texts: Vec<&str> = base_units.iter().map(|unit| unit.text.as_str()).collect();
+    let reference_unit_texts: Vec<&str> = reference_units
+        .iter()
+        .map(|unit| unit.text.as_str())
+        .collect();
+
+    let (candidate_base_mask, base_candidate_mask) =
+        lcs_keep_masks(&candidate_unit_texts, &base_unit_texts);
+    let (stripped_candidate, candidate_new_chars, context_chars) =
+        analyze_masked_units(&candidate_units, &candidate_base_mask);
+
+    let (reference_base_mask, base_reference_mask) =
+        lcs_keep_masks(&reference_unit_texts, &base_unit_texts);
+    let (stripped_reference, reference_new_chars, _) =
+        analyze_masked_units(&reference_units, &reference_base_mask);
+
+    let keep_mask = lcs_keep_mask(&stripped_candidate, &stripped_reference);
+
+    let kept_chars: usize = stripped_candidate
+        .iter()
+        .zip(keep_mask.iter())
+        .filter_map(|(&token, &is_kept)| is_kept.then_some(token.len()))
+        .sum();
+
+    let candidate_deleted_chars = count_unmasked_unit_chars(&base_units, &base_candidate_mask);
+    let reference_deleted_chars = count_unmasked_unit_chars(&base_units, &base_reference_mask);
+    let correctly_deleted_chars: usize = base_units
+        .iter()
+        .zip(base_candidate_mask.iter().zip(base_reference_mask.iter()))
+        .filter_map(|(unit, (&in_candidate, &in_reference))| {
+            (!in_candidate && !in_reference).then_some(unit.text.len())
+        })
+        .sum();
+
+    let discarded_chars = candidate_new_chars - kept_chars;
+    let matched_edit_chars = kept_chars + correctly_deleted_chars;
+    let candidate_edit_chars = candidate_new_chars + candidate_deleted_chars;
+    let reference_edit_chars = reference_new_chars + reference_deleted_chars;
+
+    let kept_rate = if candidate_edit_chars == 0 {
+        if reference_edit_chars == 0 { 1.0 } else { 0.0 }
+    } else {
+        matched_edit_chars as f64 / candidate_edit_chars as f64
+    };
+
+    let recall_rate = if reference_edit_chars == 0 {
+        if candidate_edit_chars == 0 { 1.0 } else { 0.0 }
+    } else {
+        matched_edit_chars as f64 / reference_edit_chars as f64
+    };
+
+    let token_annotations = {
+        let mut token_annotations = vec![TokenAnnotation::Context; candidate_tokens.len()];
+        let mut new_index = 0;
+
+        for (unit_index, unit) in candidate_units.iter().enumerate() {
+            let annotation = if candidate_base_mask[unit_index] {
+                TokenAnnotation::Context
+            } else {
+                let annotation = if keep_mask[new_index] {
+                    TokenAnnotation::Kept
+                } else {
+                    TokenAnnotation::Discarded
+                };
+                new_index += 1;
+                annotation
+            };
+
+            for token_index in unit.token_start..unit.token_end {
+                token_annotations[token_index] = annotation;
+            }
+        }
+
+        token_annotations
+    };
+
+    KeptRateResult {
+        candidate_new_chars,
+        reference_new_chars,
+        candidate_deleted_chars,
+        reference_deleted_chars,
+        kept_chars,
+        correctly_deleted_chars,
+        discarded_chars,
+        context_chars,
+        kept_rate,
+        recall_rate,
+        token_annotations,
+    }
+}
+
+pub fn annotate_kept_rate_tokens(
+    base: &str,
+    candidate: &str,
+    reference: &str,
+) -> Vec<AnnotatedToken> {
+    let result = compute_kept_rate(base, candidate, reference);
+    tokenize(candidate)
+        .into_iter()
+        .zip(result.token_annotations)
+        .map(|(token, annotation)| AnnotatedToken {
+            token: token.to_string(),
+            annotation,
+        })
+        .collect()
+}
+
+#[cfg(test)]
+mod test_kept_rate {
+    use super::*;
+    use indoc::indoc;
+
+    #[test]
+    fn test_lcs_keep_masks() {
+        let (a_mask, b_mask) = lcs_keep_masks(&["a", "b", "c", "d", "e"], &["a", "c", "e"]);
+        assert_eq!(a_mask, vec![true, false, true, false, true]);
+        assert_eq!(b_mask, vec![true, true, true]);
+
+        let (a_mask, b_mask) = lcs_keep_masks(&[], &["x"]);
+        assert!(a_mask.is_empty());
+        assert_eq!(b_mask, vec![false]);
+    }
+
+    #[test]
+    fn test_lcs_keep_masks_matches_historical_one_sided_masks() {
+        let a = ["x", "a", "x", "b"];
+        let b = ["a", "x", "b", "x"];
+        let (a_mask, b_mask) = lcs_keep_masks(&a, &b);
+        assert_eq!(a_mask, lcs_keep_mask(&a, &b));
+        assert_eq!(b_mask, lcs_keep_mask(&b, &a));
+    }
+
+    #[test]
+    fn test_rate_extremes() {
+        let no_change = compute_kept_rate("foo bar", "foo bar", "foo bar");
+        assert!((no_change.kept_rate - 1.0).abs() < 1e-6);
+        assert!((no_change.recall_rate - 1.0).abs() < 1e-6);
+        assert_eq!(no_change.candidate_new_chars, 0);
+        assert!(
+            no_change
+                .token_annotations
+                .iter()
+                .all(|&annotation| annotation == TokenAnnotation::Context)
+        );
+
+        let accepted = compute_kept_rate("old", "new", "new");
+        assert!((accepted.kept_rate - 1.0).abs() < 1e-6);
+        assert!((accepted.recall_rate - 1.0).abs() < 1e-6);
+
+        let discarded = compute_kept_rate("old", "old", "new");
+        assert!((discarded.kept_rate - 0.0).abs() < 1e-6);
+        assert!((discarded.recall_rate - 0.0).abs() < 1e-6);
+    }
+
+    #[test]
+    fn test_pure_addition() {
+        let kept = compute_kept_rate("", "brand new line\n", "brand new line\n");
+        assert_eq!(kept.kept_chars, kept.candidate_new_chars);
+        assert!(
+            kept.token_annotations
+                .iter()
+                .all(|&annotation| annotation == TokenAnnotation::Kept)
+        );
+
+        let discarded =
+            compute_kept_rate("", "brand new line\n", "something completely different\n");
+        assert!(discarded.kept_chars < discarded.candidate_new_chars);
+    }
+
+    #[test]
+    fn test_decoy_when_base_excluded() {
+        let base = "    decoy.when(mock_sync_hardware_api.sp()).then_return(SpeedStatus.IDLE)\n";
+        let candidate = "    decoy.when(mock_sync_module_hardware.speed_status).then_return(SpeedStatus.IDLE)\n";
+        let reference = "    decoy.when(mock_sync_module_hardware.speed_status).then_return(SpeedStatus.IDLE)\n";
+        let result = compute_kept_rate(base, candidate, reference);
+        let expected_new = "mock_sync_module_hardware".len() + "speed_status".len();
+        assert_eq!(result.candidate_new_chars, expected_new);
+        assert!(result.correctly_deleted_chars > 0);
+        assert!((result.kept_rate - 1.0).abs() < 1e-6);
+        assert!((result.recall_rate - 1.0).abs() < 1e-6);
+    }
+
+    #[test]
+    fn test_missing_deletion() {
+        let base = indoc! {"
+            fn example() {
+                epr
+        "};
+        let candidate = indoc! {r#"
+            fn example() {
+                epr
+            eprintln!("");
+        "#};
+        let reference = indoc! {r#"
+            fn example() {
+            eprintln!("");
+        "#};
+
+        let result = compute_kept_rate(base, candidate, reference);
+        assert!((result.kept_rate - (14.0 / 15.0)).abs() < 1e-6);
+        assert_eq!(result.kept_chars, 14);
+        assert_eq!(result.discarded_chars, 1);
+    }
+
+    #[test]
+    fn test_empty_prediction() {
+        let result = compute_kept_rate("old line\n", "", "new line\n");
+        assert_eq!(result.candidate_new_chars, 0);
+        assert!(result.candidate_deleted_chars > 0);
+        assert!(result.correctly_deleted_chars > 0);
+        assert!(result.correctly_deleted_chars < result.candidate_deleted_chars);
+        assert!(result.kept_rate > 0.0 && result.kept_rate < 1.0);
+        assert!(result.recall_rate > 0.0 && result.recall_rate < 1.0);
+    }
+
+    #[test]
+    fn test_partial_kept() {
+        let result = compute_kept_rate("old\n", "alpha\nbeta\ngamma\n", "alpha\ngamma\n");
+        assert!(result.kept_chars > 0);
+        assert!(result.discarded_chars > 0);
+        assert!(result.kept_rate > 0.0 && result.kept_rate < 1.0);
+    }
+
+    #[test]
+    fn test_bails_for_dirty_final() {
+        let base = indoc! {"
+            fn example() {
+                work();
+            }
+        "};
+        let candidate = indoc! {"
+            fn example() {
+                work();
+                predicted();
+            }
+        "};
+        let reference = format!(
+            "fn example() {{\n    work();\n    {}\n}}\n",
+            "settled();\n    ".repeat(MAX_DIRTY_LENGTH_DELTA_CHARS / 8 + 64)
+        );
+
+        let result = compute_kept_rate(base, candidate, &reference);
+        assert_eq!(result.kept_rate, 0.0);
+        assert_eq!(result.recall_rate, 0.0);
+        assert_eq!(result.kept_chars, 0);
+        assert_eq!(result.discarded_chars, result.candidate_new_chars);
+    }
+
+    #[test]
+    fn test_eprintln_token_alignment() {
+        let base = indoc! {"
+            fn example() {
+                epr
+        "};
+        let candidate = indoc! {r#"
+            fn example() {
+                eprintln!("hello world!");
+        "#};
+        let reference = indoc! {r#"
+            fn example() {
+                eprintln!("");
+        "#};
+
+        let result = compute_kept_rate(base, candidate, reference);
+        assert!(result.discarded_chars > 0);
+        assert!(result.kept_chars > 0);
+        assert!(result.kept_rate > 0.0 && result.kept_rate < 1.0);
+        assert_eq!(result.kept_chars, 14);
+        assert_eq!(result.discarded_chars, 12);
+    }
+
+    #[test]
+    fn test_kept_rate_treats_unchanged_stale_text_as_context() {
+        let base = indoc! {"
+            a=fomr
+            b=old
+        "};
+        let candidate = indoc! {"
+            a=formula;
+            b=old
+        "};
+        let reference = indoc! {"
+            a=formula;
+            b=new
+        "};
+
+        let result = compute_kept_rate(base, candidate, reference);
+        let candidate_tokens = tokenize(candidate);
+
+        assert_eq!(result.candidate_new_chars, "formula".len() + ";".len());
+        assert_eq!(result.kept_chars, "formula".len() + ";".len());
+        assert_eq!(result.discarded_chars, 0);
+        assert_eq!(result.candidate_deleted_chars, "fomr".len());
+        assert_eq!(result.correctly_deleted_chars, "fomr".len());
+        assert!((result.kept_rate - 1.0).abs() < 1e-6);
+        assert!((result.recall_rate - (2.0 / 3.0)).abs() < 1e-6);
+
+        let old_index = candidate_tokens
+            .iter()
+            .position(|&token| token == "old")
+            .expect("old token not found");
+        assert_eq!(
+            result.token_annotations[old_index],
+            TokenAnnotation::Context
+        );
+    }
+
+    #[test]
+    fn test_annotations_rename() {
+        let base = "    foo(old_name)\n";
+        let candidate = "    foo(new_name)\n";
+        let reference = "    foo(new_name)\n";
+        let result = compute_kept_rate(base, candidate, reference);
+
+        assert_eq!(result.candidate_new_chars, "new_name".len());
+        assert_eq!(result.candidate_deleted_chars, "old_name".len());
+        assert_eq!(result.reference_deleted_chars, "old_name".len());
+        assert_eq!(result.correctly_deleted_chars, "old_name".len());
+        assert!((result.recall_rate - 1.0).abs() < 1e-6);
+        assert_eq!(result.token_annotations.len(), tokenize(candidate).len());
+
+        for (&token, &annotation) in tokenize(candidate).iter().zip(&result.token_annotations) {
+            if matches!(token, "new" | "_" | "name") {
+                assert_eq!(annotation, TokenAnnotation::Kept);
+            } else {
+                assert_eq!(annotation, TokenAnnotation::Context);
+            }
+        }
+    }
+
+    #[test]
+    fn test_annotations_eprintln_coloring() {
+        let base = indoc! {"
+            fn example() {
+                epr
+        "};
+        let candidate = indoc! {r#"
+            fn example() {
+                eprintln!("hello world!");
+        "#};
+        let reference = indoc! {r#"
+            fn example() {
+                eprintln!("");
+        "#};
+        let result = compute_kept_rate(base, candidate, reference);
+        let candidate_tokens = tokenize(candidate);
+
+        let eprintln_index = candidate_tokens
+            .iter()
+            .position(|&token| token == "eprintln")
+            .expect("eprintln token not found");
+
+        for annotation in &result.token_annotations[..eprintln_index] {
+            assert_eq!(*annotation, TokenAnnotation::Context);
+        }
+
+        assert_eq!(
+            &result.token_annotations[eprintln_index..=eprintln_index + 10],
+            &[
+                TokenAnnotation::Kept,
+                TokenAnnotation::Kept,
+                TokenAnnotation::Kept,
+                TokenAnnotation::Kept,
+                TokenAnnotation::Discarded,
+                TokenAnnotation::Discarded,
+                TokenAnnotation::Discarded,
+                TokenAnnotation::Discarded,
+                TokenAnnotation::Kept,
+                TokenAnnotation::Kept,
+                TokenAnnotation::Kept,
+            ]
+        );
+        assert_eq!(
+            result.token_annotations.last(),
+            Some(&TokenAnnotation::Context)
+        );
+    }
+
+    #[test]
+    fn test_repetitive_tokens_remain_discarded() {
+        let base = "foo + foo + foo + foo + foo\n".repeat(16);
+        let candidate = "foo + foo + prediction_token + foo + foo\n".repeat(16);
+        let reference = "foo + foo + kept_token + foo + foo\n".repeat(16);
+        let result = compute_kept_rate(&base, &candidate, &reference);
+
+        assert_eq!(result.kept_chars, 0);
+        assert_eq!(result.correctly_deleted_chars, "foo".len() * 16);
+        assert_eq!(result.discarded_chars, result.candidate_new_chars);
+        assert_eq!(result.candidate_new_chars, "prediction_token".len() * 16);
+        assert!(result.kept_rate > 0.0);
+        assert!(result.recall_rate > 0.0);
+    }
+}

crates/edit_prediction_metrics/src/main.rs

@@ -0,0 +1,710 @@
+use std::env;
+use std::fmt::Write as _;
+use std::fs;
+use std::path::Path;
+use std::process;
+
+use edit_prediction_metrics::{
+    ClassificationMetrics, DeltaChrFMetrics, KeptRateResult, TokenAnnotation,
+    annotate_kept_rate_tokens, braces_disbalance, compute_kept_rate, count_patch_token_changes,
+    delta_chr_f, exact_lines_match, extract_changed_lines_from_diff,
+    has_isolated_whitespace_changes, is_editable_region_correct,
+};
+use serde::Deserialize;
+
+fn main() {
+    if let Err(error) = run() {
+        eprintln!("error: {error}");
+        process::exit(1);
+    }
+}
+
+fn run() -> Result<(), String> {
+    let args: Vec<String> = env::args().skip(1).collect();
+    if args.is_empty() {
+        print_usage();
+        return Err("missing arguments".to_string());
+    }
+
+    let input = CliInput::parse(&args)?;
+    let report = match input {
+        CliInput::Files {
+            base_path,
+            expected_patch_path,
+            actual_patch_path,
+        } => {
+            let base = fs::read_to_string(&base_path)
+                .map_err(|err| format!("failed to read {}: {err}", base_path.display()))?;
+            let expected_patch = fs::read_to_string(&expected_patch_path).map_err(|err| {
+                format!("failed to read {}: {err}", expected_patch_path.display())
+            })?;
+            let actual_patch = fs::read_to_string(&actual_patch_path)
+                .map_err(|err| format!("failed to read {}: {err}", actual_patch_path.display()))?;
+
+            let expected = apply_patch_to_excerpt(&base, &expected_patch, 0)?;
+            let actual = apply_patch_to_excerpt(&base, &actual_patch, 0)?;
+
+            EvaluationReport::new(base, expected_patch, actual_patch, expected, actual)
+        }
+        CliInput::Json {
+            json_path,
+            prediction_index,
+        } => {
+            let json = fs::read_to_string(&json_path)
+                .map_err(|err| format!("failed to read {}: {err}", json_path.display()))?;
+            let example: JsonExample = serde_json::from_str(&json)
+                .map_err(|err| format!("failed to parse {}: {err}", json_path.display()))?;
+
+            let base = example.prompt_inputs.cursor_excerpt;
+            let excerpt_start_row = example.prompt_inputs.excerpt_start_row;
+            let expected_patch = example
+                .expected_patches
+                .into_iter()
+                .next()
+                .ok_or_else(|| "JSON input is missing expected_patches[0]".to_string())?;
+            let actual_patch = example
+                .predictions
+                .into_iter()
+                .nth(prediction_index)
+                .ok_or_else(|| {
+                    format!("JSON input does not contain predictions[{prediction_index}]")
+                })?
+                .actual_patch;
+
+            let expected = apply_patch_to_excerpt(&base, &expected_patch, excerpt_start_row)?;
+            let actual = apply_patch_to_excerpt(&base, &actual_patch, excerpt_start_row)?;
+
+            EvaluationReport::new(base, expected_patch, actual_patch, expected, actual)
+        }
+    };
+
+    print_report(&report);
+    Ok(())
+}
+
+fn print_usage() {
+    eprintln!(
+        "Usage:\n  edit_prediction_metrics --base <base.txt> --expected-patch <expected.diff> --actual-patch <actual.diff>\n  edit_prediction_metrics --json <example.json> [--prediction-index <n>]"
+    );
+}
+
+enum CliInput {
+    Files {
+        base_path: std::path::PathBuf,
+        expected_patch_path: std::path::PathBuf,
+        actual_patch_path: std::path::PathBuf,
+    },
+    Json {
+        json_path: std::path::PathBuf,
+        prediction_index: usize,
+    },
+}
+
+impl CliInput {
+    fn parse(args: &[String]) -> Result<Self, String> {
+        let mut base_path = None;
+        let mut expected_patch_path = None;
+        let mut actual_patch_path = None;
+        let mut json_path = None;
+        let mut prediction_index = 0usize;
+
+        let mut index = 0;
+        while index < args.len() {
+            match args[index].as_str() {
+                "--base" => {
+                    index += 1;
+                    base_path = Some(path_arg(args, index, "--base")?);
+                }
+                "--expected-patch" => {
+                    index += 1;
+                    expected_patch_path = Some(path_arg(args, index, "--expected-patch")?);
+                }
+                "--actual-patch" => {
+                    index += 1;
+                    actual_patch_path = Some(path_arg(args, index, "--actual-patch")?);
+                }
+                "--json" => {
+                    index += 1;
+                    json_path = Some(path_arg(args, index, "--json")?);
+                }
+                "--prediction-index" => {
+                    index += 1;
+                    let raw = string_arg(args, index, "--prediction-index")?;
+                    prediction_index = raw.parse::<usize>().map_err(|err| {
+                        format!("invalid value for --prediction-index ({raw}): {err}")
+                    })?;
+                }
+                "--help" | "-h" => {
+                    print_usage();
+                    process::exit(0);
+                }
+                unknown => {
+                    return Err(format!("unrecognized argument: {unknown}"));
+                }
+            }
+            index += 1;
+        }
+
+        if let Some(json_path) = json_path {
+            if base_path.is_some() || expected_patch_path.is_some() || actual_patch_path.is_some() {
+                return Err(
+                    "--json cannot be combined with --base/--expected-patch/--actual-patch"
+                        .to_string(),
+                );
+            }
+            return Ok(CliInput::Json {
+                json_path,
+                prediction_index,
+            });
+        }
+
+        match (base_path, expected_patch_path, actual_patch_path) {
+            (Some(base_path), Some(expected_patch_path), Some(actual_patch_path)) => {
+                Ok(CliInput::Files {
+                    base_path,
+                    expected_patch_path,
+                    actual_patch_path,
+                })
+            }
+            _ => Err(
+                "expected either --json <file> or all of --base, --expected-patch, and --actual-patch"
+                    .to_string(),
+            ),
+        }
+    }
+}
+
+fn path_arg(args: &[String], index: usize, flag: &str) -> Result<std::path::PathBuf, String> {
+    Ok(Path::new(string_arg(args, index, flag)?).to_path_buf())
+}
+
+fn string_arg<'a>(args: &'a [String], index: usize, flag: &str) -> Result<&'a str, String> {
+    args.get(index)
+        .map(|value| value.as_str())
+        .ok_or_else(|| format!("missing value for {flag}"))
+}
+
+#[derive(Debug)]
+struct EvaluationReport {
+    base: String,
+    expected: String,
+    actual: String,
+    kept_rate: KeptRateResult,
+    exact_lines: ClassificationMetrics,
+    delta_chr_f: DeltaChrFMetrics,
+    expected_changed_lines: usize,
+    actual_changed_lines: usize,
+    token_changes: edit_prediction_metrics::TokenChangeCounts,
+    isolated_whitespace_changes: bool,
+    editable_region_correct: bool,
+    expected_braces_disbalance: usize,
+    actual_braces_disbalance: usize,
+}
+
+impl EvaluationReport {
+    fn new(
+        base: String,
+        expected_patch: String,
+        actual_patch: String,
+        expected: String,
+        actual: String,
+    ) -> Self {
+        let kept_rate = compute_kept_rate(&base, &actual, &expected);
+        let exact_lines = exact_lines_match(&expected_patch, &actual_patch);
+        let delta_chr_f = delta_chr_f(&base, &expected, &actual);
+        let expected_changed_lines = extract_changed_lines_from_diff(&expected_patch)
+            .values()
+            .sum();
+        let actual_changed_lines = extract_changed_lines_from_diff(&actual_patch)
+            .values()
+            .sum();
+        let token_changes = count_patch_token_changes(&actual_patch);
+        let isolated_whitespace_changes = has_isolated_whitespace_changes(&actual_patch, None);
+        let editable_region_correct = is_editable_region_correct(&actual_patch);
+        let expected_braces_disbalance = braces_disbalance(&expected);
+        let actual_braces_disbalance = braces_disbalance(&actual);
+
+        Self {
+            base,
+            expected,
+            actual,
+            kept_rate,
+            exact_lines,
+            delta_chr_f,
+            expected_changed_lines,
+            actual_changed_lines,
+            token_changes,
+            isolated_whitespace_changes,
+            editable_region_correct,
+            expected_braces_disbalance,
+            actual_braces_disbalance,
+        }
+    }
+}
+
+fn print_report(report: &EvaluationReport) {
+    println!("Metrics");
+    println!("=======");
+    println!("kept_rate: {:.6}", report.kept_rate.kept_rate);
+    println!("kept_rate_recall: {:.6}", report.kept_rate.recall_rate);
+    println!("delta_chr_f: {:.6}", report.delta_chr_f.score);
+    println!("delta_chr_f_precision: {:.6}", report.delta_chr_f.precision);
+    println!("delta_chr_f_recall: {:.6}", report.delta_chr_f.recall);
+    println!("delta_chr_f_beta: {:.6}", report.delta_chr_f.beta);
+    println!();
+
+    println!("Exact line match");
+    println!("----------------");
+    println!("true_positives: {}", report.exact_lines.true_positives);
+    println!("false_positives: {}", report.exact_lines.false_positives);
+    println!("false_negatives: {}", report.exact_lines.false_negatives);
+    println!("precision: {:.6}", report.exact_lines.precision());
+    println!("recall: {:.6}", report.exact_lines.recall());
+    println!("f1: {:.6}", report.exact_lines.f1());
+    println!("expected_changed_lines: {}", report.expected_changed_lines);
+    println!("actual_changed_lines: {}", report.actual_changed_lines);
+    println!();
+
+    println!("Patch structure");
+    println!("---------------");
+    println!("inserted_tokens: {}", report.token_changes.inserted_tokens);
+    println!("deleted_tokens: {}", report.token_changes.deleted_tokens);
+    println!(
+        "isolated_whitespace_changes: {}",
+        report.isolated_whitespace_changes
+    );
+    println!(
+        "editable_region_correct: {}",
+        report.editable_region_correct
+    );
+    println!();
+
+    println!("Final text checks");
+    println!("-----------------");
+    println!(
+        "expected_braces_disbalance: {}",
+        report.expected_braces_disbalance
+    );
+    println!(
+        "actual_braces_disbalance: {}",
+        report.actual_braces_disbalance
+    );
+    println!();
+
+    println!("Kept-rate breakdown");
+    println!("-------------------");
+    println!(
+        "candidate_new_chars: {}",
+        report.kept_rate.candidate_new_chars
+    );
+    println!(
+        "reference_new_chars: {}",
+        report.kept_rate.reference_new_chars
+    );
+    println!(
+        "candidate_deleted_chars: {}",
+        report.kept_rate.candidate_deleted_chars
+    );
+    println!(
+        "reference_deleted_chars: {}",
+        report.kept_rate.reference_deleted_chars
+    );
+    println!("kept_chars: {}", report.kept_rate.kept_chars);
+    println!(
+        "correctly_deleted_chars: {}",
+        report.kept_rate.correctly_deleted_chars
+    );
+    println!("discarded_chars: {}", report.kept_rate.discarded_chars);
+    println!("context_chars: {}", report.kept_rate.context_chars);
+    println!();
+
+    print_kept_rate_explanation(&report.base, &report.actual, &report.expected);
+}
+
+fn print_kept_rate_explanation(base: &str, actual: &str, expected: &str) {
+    println!("Kept-rate explanation");
+    println!("---------------------");
+    println!("Legend: context = default, kept = green background, discarded = red background");
+    println!();
+
+    let annotated = annotate_kept_rate_tokens(base, actual, expected);
+    println!("Actual final text with token annotations:");
+    println!("{}", render_annotated_tokens(&annotated));
+    println!();
+}
+
+fn render_annotated_tokens(tokens: &[edit_prediction_metrics::AnnotatedToken]) -> String {
+    const RESET: &str = "\x1b[0m";
+    const KEPT_STYLE: &str = "\x1b[30;42m";
+    const DISCARDED_STYLE: &str = "\x1b[30;41m";
+
+    let mut rendered = String::new();
+    for token in tokens {
+        let style = match token.annotation {
+            TokenAnnotation::Context => "",
+            TokenAnnotation::Kept => KEPT_STYLE,
+            TokenAnnotation::Discarded => DISCARDED_STYLE,
+        };
+
+        if style.is_empty() {
+            rendered.push_str(&visualize_whitespace(&token.token));
+        } else {
+            rendered.push_str(style);
+            rendered.push_str(&visualize_whitespace(&token.token));
+            rendered.push_str(RESET);
+        }
+    }
+    rendered
+}
+
+fn visualize_whitespace(token: &str) -> String {
+    let mut rendered = String::new();
+    for ch in token.chars() {
+        match ch {
+            ' ' => rendered.push('·'),
+            '\t' => rendered.push('⇥'),
+            '\n' => rendered.push_str("↵\n"),
+            _ => rendered.push(ch),
+        }
+    }
+    rendered
+}
+
+#[derive(Debug, Deserialize)]
+struct JsonExample {
+    prompt_inputs: PromptInputs,
+    expected_patches: Vec<String>,
+    predictions: Vec<Prediction>,
+}
+
+#[derive(Debug, Deserialize)]
+struct PromptInputs {
+    cursor_excerpt: String,
+    excerpt_start_row: u32,
+}
+
+#[derive(Debug, Deserialize)]
+struct Prediction {
+    actual_patch: String,
+}
+
+#[derive(Debug, Clone)]
+struct ParsedHunk {
+    old_start: u32,
+    lines: Vec<HunkLine>,
+}
+
+#[derive(Debug, Clone)]
+enum HunkLine {
+    Context(String),
+    Addition(String),
+    Deletion(String),
+}
+
+fn apply_patch_to_excerpt(
+    base: &str,
+    patch: &str,
+    excerpt_start_row: u32,
+) -> Result<String, String> {
+    let hunks = parse_diff_hunks(patch);
+
+    let result = try_apply_hunks(base, &hunks, excerpt_start_row);
+
+    // Predicted patches may use excerpt-relative line numbers instead of
+    // file-global ones. When all hunks fall outside the excerpt window the
+    // result is identical to the base text. Retry with a zero offset so the
+    // line numbers are interpreted relative to the excerpt.
+    if excerpt_start_row > 0 && !hunks.is_empty() {
+        let should_retry = match &result {
+            Ok(text) => text == base,
+            Err(_) => true,
+        };
+
+        if should_retry {
+            let fallback = try_apply_hunks(base, &hunks, 0);
+            if matches!(&fallback, Ok(text) if text != base) {
+                return fallback;
+            }
+        }
+    }
+
+    result
+}
+
+fn try_apply_hunks(
+    base: &str,
+    hunks: &[ParsedHunk],
+    excerpt_start_row: u32,
+) -> Result<String, String> {
+    let base_has_trailing_newline = base.ends_with('\n');
+    let mut lines = split_preserving_final_empty_line(base);
+    let original_line_count = lines.len() as u32;
+
+    let excerpt_end_row = excerpt_start_row + original_line_count;
+    let mut line_delta: i64 = 0;
+
+    for hunk in hunks {
+        let filtered = match filter_hunk_to_excerpt(hunk, excerpt_start_row, excerpt_end_row) {
+            Some(filtered) => filtered,
+            None => continue,
+        };
+
+        let local_start = filtered.old_start.saturating_sub(excerpt_start_row) as i64 + line_delta;
+        if local_start < 0 {
+            return Err(format!(
+                "patch application moved before excerpt start at source row {}",
+                filtered.old_start
+            ));
+        }
+        let local_start = local_start as usize;
+
+        if local_start > lines.len() {
+            return Err(format!(
+                "patch application starts past excerpt end at local line {}",
+                local_start + 1
+            ));
+        }
+
+        let old_len = filtered
+            .lines
+            .iter()
+            .filter(|line| !matches!(line, HunkLine::Addition(_)))
+            .count();
+        let new_len = filtered
+            .lines
+            .iter()
+            .filter(|line| !matches!(line, HunkLine::Deletion(_)))
+            .count();
+
+        let old_segment: Vec<&str> = filtered
+            .lines
+            .iter()
+            .filter_map(|line| match line {
+                HunkLine::Context(text) | HunkLine::Deletion(text) => Some(text.as_str()),
+                HunkLine::Addition(_) => None,
+            })
+            .collect();
+
+        let new_segment: Vec<String> = filtered
+            .lines
+            .iter()
+            .filter_map(|line| match line {
+                HunkLine::Context(text) | HunkLine::Addition(text) => Some(text.clone()),
+                HunkLine::Deletion(_) => None,
+            })
+            .collect();
+
+        if local_start + old_len > lines.len() {
+            return Err(format!(
+                "patch application exceeds excerpt bounds near source row {}",
+                filtered.old_start
+            ));
+        }
+
+        let current_segment: Vec<&str> = lines[local_start..local_start + old_len]
+            .iter()
+            .map(String::as_str)
+            .collect();
+
+        if current_segment != old_segment {
+            let mut details = String::new();
+            let _ = write!(
+                details,
+                "patch context mismatch near source row {}: expected {:?}, found {:?}",
+                filtered.old_start, old_segment, current_segment
+            );
+            return Err(details);
+        }
+
+        lines.splice(local_start..local_start + old_len, new_segment);
+        line_delta += new_len as i64 - old_len as i64;
+    }
+
+    Ok(join_lines(&lines, base_has_trailing_newline))
+}
+
+fn split_preserving_final_empty_line(text: &str) -> Vec<String> {
+    let mut lines: Vec<String> = text.lines().map(ToString::to_string).collect();
+    if text.ends_with('\n') {
+        if lines.last().is_some_and(|line| !line.is_empty()) || lines.is_empty() {
+            lines.push(String::new());
+        }
+    }
+    lines
+}
+
+fn join_lines(lines: &[String], had_trailing_newline: bool) -> String {
+    if lines.is_empty() {
+        return String::new();
+    }
+
+    let mut joined = lines.join("\n");
+    if had_trailing_newline && !joined.ends_with('\n') {
+        joined.push('\n');
+    }
+    if !had_trailing_newline && joined.ends_with('\n') {
+        joined.pop();
+    }
+    joined
+}
+
+fn filter_hunk_to_excerpt(
+    hunk: &ParsedHunk,
+    excerpt_start_row: u32,
+    excerpt_end_row: u32,
+) -> Option<ParsedHunk> {
+    let mut filtered_lines = Vec::new();
+    let mut current_old_row = hunk.old_start.saturating_sub(1);
+    let mut filtered_old_start = None;
+    let mut has_overlap = false;
+
+    for line in &hunk.lines {
+        match line {
+            HunkLine::Context(text) => {
+                let in_excerpt =
+                    current_old_row >= excerpt_start_row && current_old_row < excerpt_end_row;
+                if in_excerpt {
+                    filtered_old_start.get_or_insert(current_old_row);
+                    filtered_lines.push(HunkLine::Context(text.clone()));
+                    has_overlap = true;
+                }
+                current_old_row += 1;
+            }
+            HunkLine::Deletion(text) => {
+                let in_excerpt =
+                    current_old_row >= excerpt_start_row && current_old_row < excerpt_end_row;
+                if in_excerpt {
+                    filtered_old_start.get_or_insert(current_old_row);
+                    filtered_lines.push(HunkLine::Deletion(text.clone()));
+                    has_overlap = true;
+                }
+                current_old_row += 1;
+            }
+            HunkLine::Addition(text) => {
+                let insertion_in_excerpt =
+                    current_old_row >= excerpt_start_row && current_old_row <= excerpt_end_row;
+                if insertion_in_excerpt {
+                    filtered_old_start.get_or_insert(current_old_row);
+                    filtered_lines.push(HunkLine::Addition(text.clone()));
+                    has_overlap = true;
+                }
+            }
+        }
+    }
+
+    if !has_overlap {
+        return None;
+    }
+
+    Some(ParsedHunk {
+        old_start: filtered_old_start.unwrap_or(excerpt_start_row),
+        lines: filtered_lines,
+    })
+}
+
+fn parse_diff_hunks(diff: &str) -> Vec<ParsedHunk> {
+    let mut hunks = Vec::new();
+    let mut current_hunk: Option<ParsedHunk> = None;
+
+    for line in diff.lines() {
+        if let Some((old_start, old_count, _new_start, _new_count)) = parse_hunk_header(line) {
+            if let Some(hunk) = current_hunk.take() {
+                hunks.push(hunk);
+            }
+            let _ = old_count;
+            current_hunk = Some(ParsedHunk {
+                old_start,
+                lines: Vec::new(),
+            });
+            continue;
+        }
+
+        let Some(hunk) = current_hunk.as_mut() else {
+            continue;
+        };
+
+        if let Some(text) = line.strip_prefix('+') {
+            if !line.starts_with("+++") {
+                hunk.lines.push(HunkLine::Addition(text.to_string()));
+            }
+        } else if let Some(text) = line.strip_prefix('-') {
+            if !line.starts_with("---") {
+                hunk.lines.push(HunkLine::Deletion(text.to_string()));
+            }
+        } else if let Some(text) = line.strip_prefix(' ') {
+            hunk.lines.push(HunkLine::Context(text.to_string()));
+        } else if line.is_empty() {
+            hunk.lines.push(HunkLine::Context(String::new()));
+        }
+    }
+
+    if let Some(hunk) = current_hunk {
+        hunks.push(hunk);
+    }
+
+    hunks
+}
+
+fn parse_hunk_header(line: &str) -> Option<(u32, u32, u32, u32)> {
+    let line = line.strip_prefix("@@ -")?;
+    let (old_part, rest) = line.split_once(' ')?;
+    let rest = rest.strip_prefix('+')?;
+    let (new_part, _) = rest.split_once(" @@")?;
+
+    let (old_start, old_count) = parse_hunk_range(old_part)?;
+    let (new_start, new_count) = parse_hunk_range(new_part)?;
+    Some((old_start, old_count, new_start, new_count))
+}
+
+fn parse_hunk_range(part: &str) -> Option<(u32, u32)> {
+    if let Some((start, count)) = part.split_once(',') {
+        Some((start.parse().ok()?, count.parse().ok()?))
+    } else {
+        Some((part.parse().ok()?, 1))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn applies_patch_in_file_mode() {
+        let base = "fn main() {\n    println!(\"hello\");\n}\n";
+        let patch = "@@ -1,3 +1,3 @@\n fn main() {\n-    println!(\"hello\");\n+    println!(\"world\");\n }\n";
+
+        let actual = apply_patch_to_excerpt(base, patch, 0).unwrap();
+        assert_eq!(actual, "fn main() {\n    println!(\"world\");\n}\n");
+    }
+
+    #[test]
+    fn applies_patch_in_json_excerpt_mode() {
+        let base = "b\nc\nd\n";
+        let patch = "@@ -2,2 +2,2 @@\n-b\n-c\n+x\n+y\n";
+
+        let actual = apply_patch_to_excerpt(base, patch, 1).unwrap();
+        assert_eq!(actual, "x\ny\nd\n");
+    }
+
+    #[test]
+    fn applies_patch_with_excerpt_relative_line_numbers() {
+        let base = "a\nb\nc\nd\n";
+        // Patch uses excerpt-relative line numbers (line 2 of excerpt)
+        // even though the excerpt starts at file row 100.
+        let patch = "@@ -2,2 +2,2 @@\n-b\n-c\n+x\n+y\n";
+
+        let actual = apply_patch_to_excerpt(base, patch, 100).unwrap();
+        assert_eq!(actual, "a\nx\ny\nd\n");
+    }
+
+    #[test]
+    fn prefers_file_global_line_numbers_over_excerpt_relative() {
+        let base = "a\nb\nc\n";
+        // Patch uses file-global line numbers: excerpt starts at row 5,
+        // hunk targets line 6 (1-based) = row 5 (0-based) = first line.
+        let patch = "@@ -6,2 +6,2 @@\n-a\n-b\n+x\n+y\n";
+
+        let actual = apply_patch_to_excerpt(base, patch, 5).unwrap();
+        assert_eq!(actual, "x\ny\nc\n");
+    }
+}

crates/edit_prediction_metrics/src/prediction_score.rs

@@ -0,0 +1,351 @@
+use serde::{Deserialize, Serialize};
+use std::error::Error;
+use std::fmt;
+use std::path::Path;
+use std::sync::Arc;
+use zeta_prompt::udiff::{apply_diff_to_string, apply_diff_to_string_with_hunk_offset};
+
+use crate::patch_metrics::{
+    ClassificationMetrics, DeltaChrFMetrics, braces_disbalance, count_patch_token_changes,
+    delta_chr_f, delta_chr_f_beta, exact_lines_match, has_isolated_whitespace_changes,
+    is_editable_region_correct,
+};
+use crate::reversal::compute_prediction_reversal_ratio_from_history;
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct PredictionScore {
+    pub delta_chr_f: f32,
+    #[serde(default)]
+    pub delta_chr_f_true_positives: usize,
+    #[serde(default)]
+    pub delta_chr_f_false_positives: usize,
+    #[serde(default)]
+    pub delta_chr_f_false_negatives: usize,
+    #[serde(default)]
+    pub delta_chr_f_precision: f64,
+    #[serde(default)]
+    pub delta_chr_f_recall: f64,
+    #[serde(default)]
+    pub delta_chr_f_beta: f64,
+    pub braces_disbalance: usize,
+    #[serde(default)]
+    pub exact_lines_tp: usize,
+    #[serde(default)]
+    pub exact_lines_fp: usize,
+    #[serde(default)]
+    pub exact_lines_fn: usize,
+    #[serde(default)]
+    pub reversal_ratio: f32,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub cursor_distance: Option<usize>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub cursor_exact_match: Option<bool>,
+    pub wrong_editable_region: Option<bool>,
+    #[serde(default)]
+    pub has_isolated_whitespace_changes: bool,
+    #[serde(default)]
+    pub inserted_tokens: usize,
+    #[serde(default)]
+    pub deleted_tokens: usize,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub kept_rate: Option<f64>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub recall_rate: Option<f64>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub kept_chars: Option<usize>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub correctly_deleted_chars: Option<usize>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub discarded_chars: Option<usize>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub cumulative_logprob: Option<f64>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub avg_logprob: Option<f64>,
+}
+
+impl PredictionScore {
+    pub fn zero() -> Self {
+        Self {
+            delta_chr_f: 0.0,
+            delta_chr_f_true_positives: 0,
+            delta_chr_f_false_positives: 0,
+            delta_chr_f_false_negatives: 0,
+            delta_chr_f_precision: 0.0,
+            delta_chr_f_recall: 0.0,
+            delta_chr_f_beta: delta_chr_f_beta(),
+            braces_disbalance: 0,
+            exact_lines_tp: 0,
+            exact_lines_fp: 0,
+            exact_lines_fn: 0,
+            reversal_ratio: 0.0,
+            cursor_distance: None,
+            cursor_exact_match: None,
+            wrong_editable_region: None,
+            has_isolated_whitespace_changes: false,
+            inserted_tokens: 0,
+            deleted_tokens: 0,
+            kept_rate: None,
+            recall_rate: None,
+            kept_chars: None,
+            correctly_deleted_chars: None,
+            discarded_chars: None,
+            cumulative_logprob: None,
+            avg_logprob: None,
+        }
+    }
+
+    pub fn delta_chr_f_counts(&self) -> ClassificationMetrics {
+        ClassificationMetrics {
+            true_positives: self.delta_chr_f_true_positives,
+            false_positives: self.delta_chr_f_false_positives,
+            false_negatives: self.delta_chr_f_false_negatives,
+        }
+    }
+
+    pub fn exact_lines_counts(&self) -> ClassificationMetrics {
+        ClassificationMetrics {
+            true_positives: self.exact_lines_tp,
+            false_positives: self.exact_lines_fp,
+            false_negatives: self.exact_lines_fn,
+        }
+    }
+}
+
+impl Default for PredictionScore {
+    fn default() -> Self {
+        Self::zero()
+    }
+}
+
+#[derive(Clone, Debug)]
+pub struct PreparedExpectedPatch {
+    pub patch: String,
+    pub text: String,
+    pub cursor_editable_region_offset: Option<usize>,
+}
+
+#[derive(Clone, Debug)]
+pub struct PrepareExpectedPatchError {
+    message: String,
+}
+
+impl fmt::Display for PrepareExpectedPatchError {
+    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+        self.message.fmt(formatter)
+    }
+}
+
+impl Error for PrepareExpectedPatchError {}
+
+pub fn prepare_expected_patches(
+    expected_patches_with_cursors: &[(String, Option<usize>)],
+    original_text: &str,
+    old_editable_region: Option<&str>,
+) -> Result<Vec<PreparedExpectedPatch>, PrepareExpectedPatchError> {
+    expected_patches_with_cursors
+        .iter()
+        .map(|(patch, cursor_in_patch)| {
+            let text = apply_diff_to_string(patch, original_text).map_err(|error| {
+                PrepareExpectedPatchError {
+                    message: error.to_string(),
+                }
+            })?;
+            let cursor_editable_region_offset =
+                if let (Some(editable_region), Some(cursor_in_patch)) =
+                    (old_editable_region, *cursor_in_patch)
+                {
+                    match apply_diff_to_string_with_hunk_offset(patch, editable_region) {
+                        Ok((_, hunk_offset)) => Some(hunk_offset.unwrap_or(0) + cursor_in_patch),
+                        Err(_) => None,
+                    }
+                } else {
+                    *cursor_in_patch
+                };
+
+            Ok(PreparedExpectedPatch {
+                patch: patch.clone(),
+                text,
+                cursor_editable_region_offset,
+            })
+        })
+        .collect()
+}
+
+#[derive(Clone, Copy, Debug)]
+pub struct ActualPredictionCursor {
+    pub row: u32,
+    pub editable_region_offset: Option<usize>,
+}
+
+#[derive(Clone, Copy, Debug)]
+pub struct PredictionReversalContext<'a> {
+    pub edit_history: &'a [Arc<zeta_prompt::Event>],
+    pub excerpt_start_row: Option<u32>,
+    pub cursor_path: &'a Path,
+}
+
+#[derive(Clone, Copy, Debug)]
+pub struct PredictionScoringInput<'a> {
+    pub original_text: &'a str,
+    pub expected_patches: &'a [PreparedExpectedPatch],
+    pub actual_patch: Option<&'a str>,
+    pub actual_cursor: Option<ActualPredictionCursor>,
+    pub reversal_context: Option<PredictionReversalContext<'a>>,
+    pub cumulative_logprob: Option<f64>,
+    pub avg_logprob: Option<f64>,
+}
+
+pub fn score_prediction(input: PredictionScoringInput<'_>) -> PredictionScore {
+    let Some(actual_patch) = input.actual_patch else {
+        return PredictionScore::zero();
+    };
+
+    let token_changes = count_patch_token_changes(actual_patch);
+
+    let actual_text = match apply_diff_to_string(actual_patch, input.original_text) {
+        Ok(text) => text,
+        Err(_) => {
+            let mut score = PredictionScore::zero();
+            score.inserted_tokens = token_changes.inserted_tokens;
+            score.deleted_tokens = token_changes.deleted_tokens;
+            return score;
+        }
+    };
+
+    let mut best_delta_chr_f_metrics = DeltaChrFMetrics::default();
+    let mut best_expected_cursor = None;
+    let mut best_expected_text = None;
+
+    for expected in input.expected_patches {
+        let delta_chr_f_metrics = delta_chr_f(input.original_text, &expected.text, &actual_text);
+        if best_expected_text.is_none()
+            || delta_chr_f_metrics.score > best_delta_chr_f_metrics.score
+        {
+            best_delta_chr_f_metrics = delta_chr_f_metrics;
+            best_expected_cursor = expected.cursor_editable_region_offset;
+            best_expected_text = Some(expected.text.as_str());
+        }
+    }
+
+    let disbalance_before = braces_disbalance(input.original_text);
+    let disbalance_after = braces_disbalance(&actual_text);
+    let braces_disbalance = disbalance_after.saturating_sub(disbalance_before);
+
+    let best_exact_lines = input
+        .expected_patches
+        .iter()
+        .map(|expected| exact_lines_match(&expected.patch, actual_patch))
+        .max_by_key(|metrics| metrics.true_positives)
+        .unwrap_or_default();
+
+    let reversal_ratio = input
+        .reversal_context
+        .map(|context| {
+            compute_prediction_reversal_ratio_from_history(
+                input.original_text,
+                context.edit_history,
+                context.excerpt_start_row,
+                &actual_text,
+                context.cursor_path,
+            )
+        })
+        .unwrap_or(0.0);
+
+    let (cursor_distance, cursor_exact_match) =
+        compute_cursor_metrics(best_expected_cursor, input.actual_cursor);
+
+    let wrong_editable_region = Some(!is_editable_region_correct(actual_patch));
+    let has_isolated_whitespace_changes =
+        has_isolated_whitespace_changes(actual_patch, input.actual_cursor.map(|cursor| cursor.row));
+
+    let (kept_rate, recall_rate, kept_chars, correctly_deleted_chars, discarded_chars) =
+        best_expected_text
+            .map(|reference_text| {
+                let result = crate::kept_rate::compute_kept_rate(
+                    input.original_text,
+                    &actual_text,
+                    reference_text,
+                );
+                (
+                    Some(result.kept_rate),
+                    Some(result.recall_rate),
+                    Some(result.kept_chars),
+                    Some(result.correctly_deleted_chars),
+                    Some(result.discarded_chars),
+                )
+            })
+            .unwrap_or((None, None, None, None, None));
+
+    PredictionScore {
+        delta_chr_f: best_delta_chr_f_metrics.score as f32,
+        delta_chr_f_true_positives: best_delta_chr_f_metrics.counts.true_positives,
+        delta_chr_f_false_positives: best_delta_chr_f_metrics.counts.false_positives,
+        delta_chr_f_false_negatives: best_delta_chr_f_metrics.counts.false_negatives,
+        delta_chr_f_precision: best_delta_chr_f_metrics.precision,
+        delta_chr_f_recall: best_delta_chr_f_metrics.recall,
+        delta_chr_f_beta: best_delta_chr_f_metrics.beta,
+        braces_disbalance,
+        exact_lines_tp: best_exact_lines.true_positives,
+        exact_lines_fp: best_exact_lines.false_positives,
+        exact_lines_fn: best_exact_lines.false_negatives,
+        reversal_ratio,
+        cursor_distance,
+        cursor_exact_match,
+        wrong_editable_region,
+        has_isolated_whitespace_changes,
+        inserted_tokens: token_changes.inserted_tokens,
+        deleted_tokens: token_changes.deleted_tokens,
+        kept_rate,
+        recall_rate,
+        kept_chars,
+        correctly_deleted_chars,
+        discarded_chars,
+        cumulative_logprob: input.cumulative_logprob,
+        avg_logprob: input.avg_logprob,
+    }
+}
+
+fn compute_cursor_metrics(
+    expected_cursor_editable_region_offset: Option<usize>,
+    actual_cursor: Option<ActualPredictionCursor>,
+) -> (Option<usize>, Option<bool>) {
+    match (expected_cursor_editable_region_offset, actual_cursor) {
+        (Some(expected), Some(actual)) => {
+            let distance = expected.abs_diff(actual.editable_region_offset.unwrap_or_default());
+            let exact_match = distance == 0;
+            (Some(distance), Some(exact_match))
+        }
+        (None, None) => (None, None),
+        (Some(_), None) | (None, Some(_)) => (None, Some(false)),
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_kept_rate_is_computed_when_best_delta_chr_f_score_is_zero() {
+        let original_text = "";
+        let actual_patch = "--- a/file.txt\n+++ b/file.txt\n@@ -0,0 +1 @@\n+bbbbbb\n";
+        let expected_patch = "--- a/file.txt\n+++ b/file.txt\n@@ -0,0 +1 @@\n+cccccc\n";
+        let expected_patches = [PreparedExpectedPatch {
+            patch: expected_patch.to_string(),
+            text: "cccccc".to_string(),
+            cursor_editable_region_offset: None,
+        }];
+
+        let score = score_prediction(PredictionScoringInput {
+            original_text,
+            expected_patches: &expected_patches,
+            actual_patch: Some(actual_patch),
+            actual_cursor: None,
+            reversal_context: None,
+            cumulative_logprob: None,
+            avg_logprob: None,
+        });
+
+        assert_eq!(score.delta_chr_f, 0.0);
+        assert_eq!(score.kept_rate, Some(0.0));
+    }
+}

crates/edit_prediction_metrics/src/summary.rs

@@ -0,0 +1,293 @@
+use serde::Serialize;
+
+use crate::patch_metrics::ClassificationMetrics;
+use crate::prediction_score::PredictionScore;
+
+#[derive(Clone, Copy, Debug, Default)]
+pub struct QaSummaryData {
+    pub reverts_edits: Option<bool>,
+    pub confidence: Option<u8>,
+}
+
+#[derive(Clone, Copy, Debug)]
+pub struct PredictionSummaryInput<'a> {
+    pub score: &'a PredictionScore,
+    pub qa: Option<QaSummaryData>,
+}
+
+#[derive(Clone, Debug, Serialize)]
+pub struct SummaryJson {
+    pub total_examples: usize,
+    pub avg_delta_chr_f: f32,
+    pub delta_chr_f_beta: f64,
+    pub delta_chr_f_true_positives: usize,
+    pub delta_chr_f_false_positives: usize,
+    pub delta_chr_f_false_negatives: usize,
+    pub delta_chr_f_precision: f64,
+    pub delta_chr_f_recall: f64,
+    pub avg_braces_disbalance: f32,
+    pub exact_lines_true_positives: usize,
+    pub exact_lines_false_positives: usize,
+    pub exact_lines_false_negatives: usize,
+    pub exact_lines_precision: f64,
+    pub exact_lines_recall: f64,
+    pub exact_lines_f1: f64,
+    pub avg_reversal_ratio: f32,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub qa_avg_reverts_edits: Option<f32>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub qa_avg_confidence: Option<f32>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub cursor_exact_match_rate: Option<f32>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub cursor_avg_distance: Option<f32>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub cursor_total_evaluated: Option<usize>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub wrong_editable_region_rate: Option<f32>,
+    pub isolated_whitespace_rate: Option<f32>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub avg_kept_rate: Option<f64>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub avg_recall_rate: Option<f64>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub total_kept_chars: Option<usize>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub total_correctly_deleted_chars: Option<usize>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub total_discarded_chars: Option<usize>,
+}
+
+pub fn compute_summary<'a>(
+    predictions: impl IntoIterator<Item = PredictionSummaryInput<'a>>,
+) -> SummaryJson {
+    let mut all_delta_chr_f_scores = Vec::new();
+    let mut all_reversal_ratios = Vec::new();
+    let mut braces_disbalance_sum: usize = 0;
+    let mut total_delta_chr_f = ClassificationMetrics::default();
+    let mut total_delta_chr_f_precision = 0.0;
+    let mut total_delta_chr_f_recall = 0.0;
+    let mut delta_chr_f_beta = 0.0;
+    let mut total_exact_lines = ClassificationMetrics::default();
+    let mut total_scores: usize = 0;
+    let mut qa_reverts_count: usize = 0;
+    let mut qa_reverts_total: usize = 0;
+    let mut qa_confidence_sum: u64 = 0;
+    let mut qa_confidence_count: usize = 0;
+    let mut cursor_exact_matches: usize = 0;
+    let mut cursor_total: usize = 0;
+    let mut cursor_distance_sum: usize = 0;
+    let mut cursor_distance_count: usize = 0;
+    let mut wrong_editable_region_count: usize = 0;
+    let mut wrong_editable_region_total: usize = 0;
+    let mut isolated_whitespace_count: usize = 0;
+    let mut kept_rate_sum: f64 = 0.0;
+    let mut kept_rate_count: usize = 0;
+    let mut kept_chars_total: usize = 0;
+    let mut kept_chars_count: usize = 0;
+    let mut correctly_deleted_chars_total: usize = 0;
+    let mut correctly_deleted_chars_count: usize = 0;
+    let mut discarded_chars_total: usize = 0;
+    let mut discarded_chars_count: usize = 0;
+    let mut recall_rate_sum: f64 = 0.0;
+    let mut recall_rate_count: usize = 0;
+
+    for prediction in predictions {
+        let score = prediction.score;
+
+        all_delta_chr_f_scores.push(score.delta_chr_f);
+        all_reversal_ratios.push(score.reversal_ratio);
+        total_scores += 1;
+        braces_disbalance_sum += score.braces_disbalance;
+        total_delta_chr_f.accumulate(&score.delta_chr_f_counts());
+        total_delta_chr_f_precision += score.delta_chr_f_precision;
+        total_delta_chr_f_recall += score.delta_chr_f_recall;
+        delta_chr_f_beta = score.delta_chr_f_beta;
+        total_exact_lines.accumulate(&score.exact_lines_counts());
+
+        if let Some(qa) = prediction.qa {
+            if let Some(reverts) = qa.reverts_edits {
+                qa_reverts_total += 1;
+                if reverts {
+                    qa_reverts_count += 1;
+                }
+            }
+            if let Some(confidence) = qa.confidence {
+                qa_confidence_sum += confidence as u64;
+                qa_confidence_count += 1;
+            }
+        }
+
+        if let Some(wrong) = score.wrong_editable_region {
+            wrong_editable_region_total += 1;
+            if wrong {
+                wrong_editable_region_count += 1;
+            }
+        }
+
+        if score.has_isolated_whitespace_changes {
+            isolated_whitespace_count += 1;
+        }
+
+        if let Some(kept_rate) = score.kept_rate {
+            kept_rate_sum += kept_rate;
+            kept_rate_count += 1;
+        }
+        if let Some(kept_chars) = score.kept_chars {
+            kept_chars_total += kept_chars;
+            kept_chars_count += 1;
+        }
+        if let Some(correctly_deleted_chars) = score.correctly_deleted_chars {
+            correctly_deleted_chars_total += correctly_deleted_chars;
+            correctly_deleted_chars_count += 1;
+        }
+        if let Some(discarded_chars) = score.discarded_chars {
+            discarded_chars_total += discarded_chars;
+            discarded_chars_count += 1;
+        }
+        if let Some(recall_rate) = score.recall_rate {
+            recall_rate_sum += recall_rate;
+            recall_rate_count += 1;
+        }
+
+        if let Some(exact_match) = score.cursor_exact_match {
+            cursor_total += 1;
+            if exact_match {
+                cursor_exact_matches += 1;
+            }
+        }
+        if let Some(distance) = score.cursor_distance {
+            cursor_distance_sum += distance;
+            cursor_distance_count += 1;
+        }
+    }
+
+    let avg_delta_chr_f = if all_delta_chr_f_scores.is_empty() {
+        0.0
+    } else {
+        all_delta_chr_f_scores.iter().sum::<f32>() / all_delta_chr_f_scores.len() as f32
+    };
+
+    let avg_reversal_ratio = if all_reversal_ratios.is_empty() {
+        0.0
+    } else {
+        all_reversal_ratios.iter().sum::<f32>() / all_reversal_ratios.len() as f32
+    };
+
+    let avg_braces_disbalance = if total_scores == 0 {
+        0.0
+    } else {
+        braces_disbalance_sum as f32 / total_scores as f32
+    };
+
+    let qa_avg_reverts_edits = if qa_reverts_total > 0 {
+        Some(qa_reverts_count as f32 / qa_reverts_total as f32)
+    } else {
+        None
+    };
+
+    let qa_avg_confidence = if qa_confidence_count > 0 {
+        Some(qa_confidence_sum as f32 / qa_confidence_count as f32)
+    } else {
+        None
+    };
+
+    let cursor_exact_match_rate = if cursor_total > 0 {
+        Some(cursor_exact_matches as f32 / cursor_total as f32)
+    } else {
+        None
+    };
+
+    let cursor_avg_distance = if cursor_distance_count > 0 {
+        Some(cursor_distance_sum as f32 / cursor_distance_count as f32)
+    } else {
+        None
+    };
+
+    let cursor_total_evaluated = if cursor_total > 0 {
+        Some(cursor_total)
+    } else {
+        None
+    };
+
+    let wrong_editable_region_rate = if wrong_editable_region_total > 0 {
+        Some(wrong_editable_region_count as f32 / wrong_editable_region_total as f32)
+    } else {
+        None
+    };
+
+    let isolated_whitespace_rate = if total_scores > 0 {
+        Some(isolated_whitespace_count as f32 / total_scores as f32)
+    } else {
+        None
+    };
+
+    let avg_kept_rate = if kept_rate_count > 0 {
+        Some(kept_rate_sum / kept_rate_count as f64)
+    } else {
+        None
+    };
+
+    let avg_recall_rate = if recall_rate_count > 0 {
+        Some(recall_rate_sum / recall_rate_count as f64)
+    } else {
+        None
+    };
+
+    let total_kept_chars = if kept_chars_count > 0 {
+        Some(kept_chars_total)
+    } else {
+        None
+    };
+
+    let total_correctly_deleted_chars = if correctly_deleted_chars_count > 0 {
+        Some(correctly_deleted_chars_total)
+    } else {
+        None
+    };
+
+    let total_discarded_chars = if discarded_chars_count > 0 {
+        Some(discarded_chars_total)
+    } else {
+        None
+    };
+
+    SummaryJson {
+        total_examples: total_scores,
+        avg_delta_chr_f,
+        delta_chr_f_beta,
+        delta_chr_f_true_positives: total_delta_chr_f.true_positives,
+        delta_chr_f_false_positives: total_delta_chr_f.false_positives,
+        delta_chr_f_false_negatives: total_delta_chr_f.false_negatives,
+        delta_chr_f_precision: if total_scores == 0 {
+            0.0
+        } else {
+            total_delta_chr_f_precision / total_scores as f64
+        },
+        delta_chr_f_recall: if total_scores == 0 {
+            0.0
+        } else {
+            total_delta_chr_f_recall / total_scores as f64
+        },
+        avg_braces_disbalance,
+        exact_lines_true_positives: total_exact_lines.true_positives,
+        exact_lines_false_positives: total_exact_lines.false_positives,
+        exact_lines_false_negatives: total_exact_lines.false_negatives,
+        exact_lines_precision: total_exact_lines.precision(),
+        exact_lines_recall: total_exact_lines.recall(),
+        exact_lines_f1: total_exact_lines.f1(),
+        avg_reversal_ratio,
+        qa_avg_reverts_edits,
+        qa_avg_confidence,
+        cursor_exact_match_rate,
+        cursor_avg_distance,
+        cursor_total_evaluated,
+        wrong_editable_region_rate,
+        isolated_whitespace_rate,
+        avg_kept_rate,
+        avg_recall_rate,
+        total_kept_chars,
+        total_correctly_deleted_chars,
+        total_discarded_chars,
+    }
+}

crates/edit_prediction_metrics/src/tokenize.rs

@@ -0,0 +1,220 @@
+use std::{iter::Peekable, str::CharIndices};
+
+#[derive(Clone, Copy, PartialEq, Eq)]
+enum CharClass {
+    Identifier,
+    Newline,
+    Whitespace,
+    Punctuation,
+}
+
+const MULTI_CHAR_PUNCTUATION: &[&str] = &[
+    ">>>=", "<<=", ">>=", "...", "..=", "??=", "**=", ">>>", "::", "->", "=>", "==", "!=", "<=",
+    ">=", "&&", "||", "<<", ">>", "..", "+=", "-=", "*=", "/=", "%=", "&=", "|=", "^=", "++", "--",
+    "**", "??", "?.", ":=", "<-", "//", "/*", "*/",
+];
+
+fn char_class(character: char) -> CharClass {
+    if character == '\n' || character == '\r' {
+        CharClass::Newline
+    } else if character.is_whitespace() {
+        CharClass::Whitespace
+    } else if character.is_alphanumeric() || character == '_' {
+        CharClass::Identifier
+    } else {
+        CharClass::Punctuation
+    }
+}
+
+fn is_identifier_boundary(previous: char, current: char, next: Option<char>) -> bool {
+    (current.is_uppercase() && (previous.is_lowercase() || previous.is_numeric()))
+        || (current.is_uppercase()
+            && previous.is_uppercase()
+            && next.is_some_and(|next| next.is_lowercase()))
+}
+
+fn push_identifier_tokens<'a>(identifier: &'a str, tokens: &mut Vec<&'a str>) {
+    let characters: Vec<(usize, char)> = identifier.char_indices().collect();
+    let mut segment_start = 0;
+    let mut index = 0;
+
+    while index < characters.len() {
+        let (byte_index, character) = characters[index];
+
+        if character == '_' {
+            if segment_start < byte_index {
+                tokens.push(&identifier[segment_start..byte_index]);
+            }
+
+            let mut underscore_end = byte_index + character.len_utf8();
+            index += 1;
+
+            while index < characters.len() && characters[index].1 == '_' {
+                underscore_end = characters[index].0 + characters[index].1.len_utf8();
+                index += 1;
+            }
+
+            tokens.push(&identifier[byte_index..underscore_end]);
+            segment_start = underscore_end;
+            continue;
+        }
+
+        if byte_index > segment_start {
+            let previous = characters[index - 1].1;
+            let next = characters.get(index + 1).map(|(_, character)| *character);
+
+            if is_identifier_boundary(previous, character, next) {
+                tokens.push(&identifier[segment_start..byte_index]);
+                segment_start = byte_index;
+            }
+        }
+
+        index += 1;
+    }
+
+    if segment_start < identifier.len() {
+        tokens.push(&identifier[segment_start..]);
+    }
+}
+
+fn push_punctuation_token<'a>(
+    text: &'a str,
+    start: usize,
+    character: char,
+    characters: &mut Peekable<CharIndices<'a>>,
+    tokens: &mut Vec<&'a str>,
+) {
+    let remaining = &text[start..];
+
+    for punctuation in MULTI_CHAR_PUNCTUATION {
+        if remaining.starts_with(punctuation) {
+            for _ in punctuation.chars().skip(1) {
+                characters.next();
+            }
+
+            tokens.push(&remaining[..punctuation.len()]);
+            return;
+        }
+    }
+
+    let end = start + character.len_utf8();
+    tokens.push(&text[start..end]);
+}
+
+pub(crate) fn tokenize(text: &str) -> Vec<&str> {
+    let mut tokens = Vec::new();
+    let mut characters = text.char_indices().peekable();
+
+    while let Some((start, character)) = characters.next() {
+        match char_class(character) {
+            CharClass::Identifier => {
+                let mut end = start + character.len_utf8();
+
+                while let Some(&(next_start, next_character)) = characters.peek() {
+                    if char_class(next_character) != CharClass::Identifier {
+                        break;
+                    }
+
+                    end = next_start + next_character.len_utf8();
+                    characters.next();
+                }
+
+                push_identifier_tokens(&text[start..end], &mut tokens);
+            }
+            CharClass::Newline => {
+                let mut end = start + character.len_utf8();
+
+                while let Some(&(next_start, next_character)) = characters.peek() {
+                    if char_class(next_character) != CharClass::Newline {
+                        break;
+                    }
+
+                    end = next_start + next_character.len_utf8();
+                    characters.next();
+                }
+
+                tokens.push(&text[start..end]);
+            }
+            CharClass::Whitespace => {
+                let mut end = start + character.len_utf8();
+
+                while let Some(&(next_start, next_character)) = characters.peek() {
+                    if char_class(next_character) != CharClass::Whitespace {
+                        break;
+                    }
+
+                    end = next_start + next_character.len_utf8();
+                    characters.next();
+                }
+
+                tokens.push(&text[start..end]);
+            }
+            CharClass::Punctuation => {
+                push_punctuation_token(text, start, character, &mut characters, &mut tokens);
+            }
+        }
+    }
+
+    tokens
+}
+
+#[cfg(test)]
+mod tests {
+    use super::tokenize;
+
+    #[test]
+    fn tokenizes_code() {
+        assert_eq!(tokenize("hello world"), vec!["hello", " ", "world"]);
+        assert_eq!(
+            tokenize("foo_bar123 + baz"),
+            vec!["foo", "_", "bar123", " ", "+", " ", "baz"]
+        );
+        assert_eq!(
+            tokenize("print(\"hello\")"),
+            vec!["print", "(", "\"", "hello", "\"", ")"]
+        );
+        assert_eq!(tokenize("hello_world"), vec!["hello", "_", "world"]);
+        assert_eq!(tokenize("fn();"), vec!["fn", "(", ")", ";"]);
+    }
+
+    #[test]
+    fn tokenizes_identifier_case_styles() {
+        assert_eq!(
+            tokenize("camelCase PascalCase snake_case"),
+            vec![
+                "camel", "Case", " ", "Pascal", "Case", " ", "snake", "_", "case"
+            ]
+        );
+        assert_eq!(
+            tokenize("myHTTPServer __private_value foo__bar"),
+            vec![
+                "my", "HTTP", "Server", " ", "__", "private", "_", "value", " ", "foo", "__", "bar"
+            ]
+        );
+        assert_eq!(
+            tokenize("XMLHttpRequest Version2Update"),
+            vec!["XML", "Http", "Request", " ", "Version2", "Update"]
+        );
+    }
+
+    #[test]
+    fn tokenizes_grouped_punctuation() {
+        assert_eq!(
+            tokenize("a::b -> c != d ..= e"),
+            vec![
+                "a", "::", "b", " ", "->", " ", "c", " ", "!=", " ", "d", " ", "..=", " ", "e"
+            ]
+        );
+        assert_eq!(
+            tokenize("foo?.bar ?? baz"),
+            vec!["foo", "?.", "bar", " ", "??", " ", "baz"]
+        );
+    }
+
+    #[test]
+    fn tokenize_whitespace_runs() {
+        assert_eq!(tokenize("  "), vec!["  "]);
+        assert_eq!(tokenize("  \n   foo"), vec!["  ", "\n", "   ", "foo"]);
+        assert_eq!(tokenize("\r\n\nfoo"), vec!["\r\n\n", "foo"]);
+    }
+}

crates/edit_prediction_metrics/src/tree_sitter.rs

@@ -0,0 +1,27 @@
+pub fn count_tree_sitter_errors<'a>(nodes: impl Iterator<Item = tree_sitter::Node<'a>>) -> usize {
+    let mut total_count: usize = 0;
+    for node in nodes {
+        let mut cursor = node.walk();
+        'node: loop {
+            let current = cursor.node();
+            if current.is_error() || current.is_missing() {
+                total_count += 1;
+            }
+            if current.has_error() && cursor.goto_first_child() {
+                continue;
+            }
+            if cursor.goto_next_sibling() {
+                continue;
+            }
+            loop {
+                if !cursor.goto_parent() {
+                    break 'node;
+                }
+                if cursor.goto_next_sibling() {
+                    continue;
+                }
+            }
+        }
+    }
+    total_count
+}