use anyhow::Result; use html_to_markdown::markdown::{ CodeHandler, HeadingHandler, ListHandler, ParagraphHandler, StyledTextHandler, TableHandler, WebpageChromeRemover, }; use html_to_markdown::{TagHandler, convert_html_to_markdown}; use std::cell::RefCell; use std::rc::Rc; /// Convert HTML to Markdown for rendering in the REPL. pub fn html_to_markdown(html: &str) -> Result { let mut handlers: Vec = vec![ // WebpageChromeRemover must come first to skip style, script, head, nav tags Rc::new(RefCell::new(WebpageChromeRemover)), Rc::new(RefCell::new(ParagraphHandler)), Rc::new(RefCell::new(HeadingHandler)), Rc::new(RefCell::new(ListHandler)), Rc::new(RefCell::new(TableHandler::new())), Rc::new(RefCell::new(StyledTextHandler)), Rc::new(RefCell::new(CodeHandler)), ]; let markdown = convert_html_to_markdown(html.as_bytes(), &mut handlers)?; Ok(clean_markdown_tables(&markdown)) } /// Clean up markdown table formatting and ensure tables have separator rows. fn clean_markdown_tables(markdown: &str) -> String { let lines: Vec<&str> = markdown.lines().collect(); let mut result: Vec = Vec::new(); let mut in_table = false; let mut has_separator = false; for (i, line) in lines.iter().enumerate() { let trimmed = line.trim(); if trimmed.starts_with('|') { let normalized = normalize_table_row(trimmed); if !in_table { // Starting a new table in_table = true; has_separator = false; } // Check if this line is a separator row if trimmed.contains("---") { has_separator = true; } result.push(normalized.clone()); // If this is the first row and no separator exists yet, // check if next row is a table row (not separator) and add one if !has_separator { let next_is_table_row = i + 1 < lines.len() && lines[i + 1].trim().starts_with('|') && !lines[i + 1].contains("---"); if next_is_table_row { // Insert separator after first row let col_count = normalized.matches('|').count().saturating_sub(1); if col_count > 0 { let separator = (0..col_count) .map(|_| "---") .collect::>() .join(" | "); result.push(format!("| {} |", separator)); has_separator = true; } } } } else { // Not a table row if !trimmed.is_empty() { result.push(trimmed.to_string()); } in_table = false; has_separator = false; } } result.join("\n") } /// Normalize a table row by trimming cells and ensuring consistent spacing. fn normalize_table_row(row: &str) -> String { let parts: Vec<&str> = row.split('|').collect(); let normalized: Vec = parts.iter().map(|cell| cell.trim().to_string()).collect(); normalized.join(" | ").trim().to_string() } #[cfg(test)] mod tests { use super::*; #[test] fn test_html_table_to_markdown() { let html = r#"
AB
1x
"#; let md = html_to_markdown(html).unwrap(); assert!(md.contains("|")); assert!(md.contains("---")); } #[test] fn test_html_with_headings() { let html = "

Title

Content

"; let md = html_to_markdown(html).unwrap(); assert!(md.contains("# Title")); } #[test] fn test_pandas_dataframe_html() { let html = r#"
AB
01x
12y
"#; let md = html_to_markdown(html).unwrap(); assert!(md.contains("|")); // Verify table rows are properly formatted (start with |) for line in md.lines() { if line.contains("|") { assert!( line.starts_with("|"), "Table line should start with |: {:?}", line ); } } } #[test] fn test_table_format_normalized() { let html = r#"
NameAge
Alice25
"#; let md = html_to_markdown(html).unwrap(); // Should have clean table format assert!(md.contains("| Name | Age |")); assert!(md.contains("| --- | --- |")); assert!(md.contains("| Alice | 25 |")); } #[test] fn test_style_tags_are_filtered() { let html = r#"
A
1
"#; let md = html_to_markdown(html).unwrap(); // Style content should not appear in output assert!(!md.contains("dataframe")); assert!(!md.contains("border")); // Table should still be present assert!(md.contains("| A |")); } #[test] fn test_table_without_thead() { // Tables without should still get a separator row let html = r#"
FeatureSupported
Tables
Lists
"#; let md = html_to_markdown(html).unwrap(); // Should have separator row inserted after first row assert!( md.contains("| --- | --- |"), "Missing separator row: {}", md ); assert!(md.contains("| Feature | Supported |")); assert!(md.contains("| Tables | ✓ |")); } }