logiguard fork: GPUI xdg-activation keyboard-focus serial fix
Some checks failed
Update All Top Ranking Issues / update_top_ranking_issues (push) Has been cancelled
Triage Project Sync (#84) / Sync triage project (push) Has been cancelled
release_nightly / notify_on_failure (push) Has been cancelled
release_nightly / check_style (push) Has been cancelled
release_nightly / run_tests_windows (push) Has been cancelled
release_nightly / clippy_windows (push) Has been cancelled
release_nightly / bundle_linux_aarch64 (push) Has been cancelled
release_nightly / bundle_linux_x86_64 (push) Has been cancelled
release_nightly / bundle_mac_aarch64 (push) Has been cancelled
release_nightly / bundle_mac_x86_64 (push) Has been cancelled
release_nightly / bundle_windows_aarch64 (push) Has been cancelled
release_nightly / bundle_windows_x86_64 (push) Has been cancelled
release_nightly / build_nix_linux_x86_64 (push) Has been cancelled
release_nightly / build_nix_mac_aarch64 (push) Has been cancelled
release_nightly / update_nightly_tag (push) Has been cancelled
Hotfix Review Monitor / check-hotfix-reviews (push) Has been cancelled
Stale PR Review Reminder / check-stale-prs (push) Has been cancelled
Update Weekly Top Ranking Issues / update_top_ranking_issues (push) Has been cancelled
Bump collab-staging Tag / update-collab-staging-tag (push) Has been cancelled
compliance_check / scheduled_compliance_check (push) Has been cancelled
Some checks failed
Update All Top Ranking Issues / update_top_ranking_issues (push) Has been cancelled
Triage Project Sync (#84) / Sync triage project (push) Has been cancelled
release_nightly / notify_on_failure (push) Has been cancelled
release_nightly / check_style (push) Has been cancelled
release_nightly / run_tests_windows (push) Has been cancelled
release_nightly / clippy_windows (push) Has been cancelled
release_nightly / bundle_linux_aarch64 (push) Has been cancelled
release_nightly / bundle_linux_x86_64 (push) Has been cancelled
release_nightly / bundle_mac_aarch64 (push) Has been cancelled
release_nightly / bundle_mac_x86_64 (push) Has been cancelled
release_nightly / bundle_windows_aarch64 (push) Has been cancelled
release_nightly / bundle_windows_x86_64 (push) Has been cancelled
release_nightly / build_nix_linux_x86_64 (push) Has been cancelled
release_nightly / build_nix_mac_aarch64 (push) Has been cancelled
release_nightly / update_nightly_tag (push) Has been cancelled
Hotfix Review Monitor / check-hotfix-reviews (push) Has been cancelled
Stale PR Review Reminder / check-stale-prs (push) Has been cancelled
Update Weekly Top Ranking Issues / update_top_ranking_issues (push) Has been cancelled
Bump collab-staging Tag / update-collab-staging-tag (push) Has been cancelled
compliance_check / scheduled_compliance_check (push) Has been cancelled
Single-commit orphan branch: full zed-industries/zed @ 8c74db0 source tree with a 3-file patch applied (no upstream history). Patch (crates/gpui_linux/src/linux/wayland/): - serial.rs: add SerialKind::KeyboardEnter - client.rs: store wl_keyboard.enter serial; add latest_serial_of() - window.rs: activate() uses keyboard-enter serial (Mutter focus gate) Mutter honors window activation only when the token carries the keyboard- focus serial from wl_keyboard.enter; GPUI used a stale mouse-press serial. See docs/tray-window-focus-wayland.md in logiguard. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
26
crates/html_to_markdown/Cargo.toml
Normal file
26
crates/html_to_markdown/Cargo.toml
Normal file
@@ -0,0 +1,26 @@
|
||||
[package]
|
||||
name = "html_to_markdown"
|
||||
version = "0.1.0"
|
||||
description = "Convert HTML to Markdown"
|
||||
repository = "https://github.com/zed-industries/zed"
|
||||
documentation = "https://docs.rs/html_to_markdown"
|
||||
keywords = ["html", "markdown", "html-to-markdown"]
|
||||
edition.workspace = true
|
||||
publish = true
|
||||
license = "Apache-2.0"
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[lib]
|
||||
path = "src/html_to_markdown.rs"
|
||||
|
||||
[dependencies]
|
||||
anyhow.workspace = true
|
||||
html5ever.workspace = true
|
||||
markup5ever_rcdom.workspace = true
|
||||
regex.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
indoc.workspace = true
|
||||
pretty_assertions.workspace = true
|
||||
1
crates/html_to_markdown/LICENSE-APACHE
Symbolic link
1
crates/html_to_markdown/LICENSE-APACHE
Symbolic link
@@ -0,0 +1 @@
|
||||
../../LICENSE-APACHE
|
||||
83
crates/html_to_markdown/src/html_element.rs
Normal file
83
crates/html_to_markdown/src/html_element.rs
Normal file
@@ -0,0 +1,83 @@
|
||||
use std::cell::RefCell;
|
||||
use std::collections::HashSet;
|
||||
use std::sync::OnceLock;
|
||||
|
||||
use html5ever::Attribute;
|
||||
|
||||
/// Returns a [`HashSet`] containing the HTML elements that are inline by default.
|
||||
///
|
||||
/// [MDN: List of "inline" elements](https://yari-demos.prod.mdn.mozit.cloud/en-US/docs/Web/HTML/Inline_elements)
|
||||
fn inline_elements() -> &'static HashSet<&'static str> {
|
||||
static INLINE_ELEMENTS: OnceLock<HashSet<&str>> = OnceLock::new();
|
||||
INLINE_ELEMENTS.get_or_init(|| {
|
||||
HashSet::from_iter([
|
||||
"a", "abbr", "acronym", "audio", "b", "bdi", "bdo", "big", "br", "button", "canvas",
|
||||
"cite", "code", "data", "datalist", "del", "dfn", "em", "embed", "i", "iframe", "img",
|
||||
"input", "ins", "kbd", "label", "map", "mark", "meter", "noscript", "object", "output",
|
||||
"picture", "progress", "q", "ruby", "s", "samp", "script", "select", "slot", "small",
|
||||
"span", "strong", "sub", "sup", "svg", "template", "textarea", "time", "tt", "u",
|
||||
"var", "video", "wbr",
|
||||
])
|
||||
})
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct HtmlElement {
|
||||
tag: String,
|
||||
pub(crate) attrs: RefCell<Vec<Attribute>>,
|
||||
}
|
||||
|
||||
impl HtmlElement {
|
||||
pub fn new(tag: String, attrs: RefCell<Vec<Attribute>>) -> Self {
|
||||
Self { tag, attrs }
|
||||
}
|
||||
|
||||
pub fn tag(&self) -> &str {
|
||||
&self.tag
|
||||
}
|
||||
|
||||
/// Returns whether this [`HtmlElement`] is an inline element.
|
||||
pub fn is_inline(&self) -> bool {
|
||||
inline_elements().contains(self.tag.as_str())
|
||||
}
|
||||
|
||||
/// Returns the attribute with the specified name.
|
||||
pub fn attr(&self, name: &str) -> Option<String> {
|
||||
self.attrs
|
||||
.borrow()
|
||||
.iter()
|
||||
.find(|attr| attr.name.local.to_string() == name)
|
||||
.map(|attr| attr.value.to_string())
|
||||
}
|
||||
|
||||
/// Returns the list of classes on this [`HtmlElement`].
|
||||
pub fn classes(&self) -> Vec<String> {
|
||||
self.attrs
|
||||
.borrow()
|
||||
.iter()
|
||||
.find(|attr| attr.name.local.to_string() == "class")
|
||||
.map(|attr| {
|
||||
attr.value
|
||||
.split(' ')
|
||||
.map(|class| class.trim().to_string())
|
||||
.collect::<Vec<_>>()
|
||||
})
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
/// Returns whether this [`HtmlElement`] has the specified class.
|
||||
pub fn has_class(&self, class: &str) -> bool {
|
||||
self.has_any_classes(&[class])
|
||||
}
|
||||
|
||||
/// Returns whether this [`HtmlElement`] has any of the specified classes.
|
||||
pub fn has_any_classes(&self, classes: &[&str]) -> bool {
|
||||
self.attrs.borrow().iter().any(|attr| {
|
||||
attr.name.local.to_string() == "class"
|
||||
&& attr
|
||||
.value
|
||||
.split(' ')
|
||||
.any(|class| classes.contains(&class.trim()))
|
||||
})
|
||||
}
|
||||
}
|
||||
46
crates/html_to_markdown/src/html_to_markdown.rs
Normal file
46
crates/html_to_markdown/src/html_to_markdown.rs
Normal file
@@ -0,0 +1,46 @@
|
||||
//! Convert HTML to Markdown.
|
||||
|
||||
mod html_element;
|
||||
pub mod markdown;
|
||||
mod markdown_writer;
|
||||
pub mod structure;
|
||||
|
||||
use std::io::Read;
|
||||
|
||||
use anyhow::{Context as _, Result};
|
||||
use html5ever::driver::ParseOpts;
|
||||
use html5ever::parse_document;
|
||||
use html5ever::tendril::TendrilSink;
|
||||
use html5ever::tree_builder::TreeBuilderOpts;
|
||||
use markup5ever_rcdom::RcDom;
|
||||
|
||||
pub use crate::html_element::*;
|
||||
pub use crate::markdown_writer::*;
|
||||
|
||||
/// Converts the provided HTML to Markdown.
|
||||
pub fn convert_html_to_markdown(html: impl Read, handlers: &mut [TagHandler]) -> Result<String> {
|
||||
let dom = parse_html(html).context("failed to parse HTML")?;
|
||||
|
||||
let markdown_writer = MarkdownWriter::new();
|
||||
let markdown = markdown_writer
|
||||
.run(&dom.document, handlers)
|
||||
.context("failed to convert HTML to Markdown")?;
|
||||
|
||||
Ok(markdown)
|
||||
}
|
||||
|
||||
fn parse_html(mut html: impl Read) -> Result<RcDom> {
|
||||
let parse_options = ParseOpts {
|
||||
tree_builder: TreeBuilderOpts {
|
||||
drop_doctype: true,
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
let dom = parse_document(RcDom::default(), parse_options)
|
||||
.from_utf8()
|
||||
.read_from(&mut html)
|
||||
.context("failed to parse HTML document")?;
|
||||
|
||||
Ok(dom)
|
||||
}
|
||||
276
crates/html_to_markdown/src/markdown.rs
Normal file
276
crates/html_to_markdown/src/markdown.rs
Normal file
@@ -0,0 +1,276 @@
|
||||
use crate::html_element::HtmlElement;
|
||||
use crate::markdown_writer::{HandleTag, HandlerOutcome, MarkdownWriter, StartTagOutcome};
|
||||
|
||||
pub struct WebpageChromeRemover;
|
||||
|
||||
impl HandleTag for WebpageChromeRemover {
|
||||
fn should_handle(&self, tag: &str) -> bool {
|
||||
matches!(tag, "head" | "script" | "style" | "nav")
|
||||
}
|
||||
|
||||
fn handle_tag_start(
|
||||
&mut self,
|
||||
tag: &HtmlElement,
|
||||
_writer: &mut MarkdownWriter,
|
||||
) -> StartTagOutcome {
|
||||
match tag.tag() {
|
||||
"head" | "script" | "style" | "nav" => return StartTagOutcome::Skip,
|
||||
_ => {}
|
||||
}
|
||||
|
||||
StartTagOutcome::Continue
|
||||
}
|
||||
}
|
||||
|
||||
pub struct ParagraphHandler;
|
||||
|
||||
impl HandleTag for ParagraphHandler {
|
||||
fn should_handle(&self, _tag: &str) -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
fn handle_tag_start(
|
||||
&mut self,
|
||||
tag: &HtmlElement,
|
||||
writer: &mut MarkdownWriter,
|
||||
) -> StartTagOutcome {
|
||||
if tag.is_inline()
|
||||
&& writer.is_inside("p")
|
||||
&& let Some(parent) = writer.current_element_stack().iter().last()
|
||||
&& !(parent.is_inline()
|
||||
|| writer.markdown.ends_with(' ')
|
||||
|| writer.markdown.ends_with('\n'))
|
||||
{
|
||||
writer.push_str(" ");
|
||||
}
|
||||
|
||||
if tag.tag() == "p" {
|
||||
writer.push_blank_line()
|
||||
}
|
||||
StartTagOutcome::Continue
|
||||
}
|
||||
}
|
||||
|
||||
pub struct HeadingHandler;
|
||||
|
||||
impl HandleTag for HeadingHandler {
|
||||
fn should_handle(&self, tag: &str) -> bool {
|
||||
matches!(tag, "h1" | "h2" | "h3" | "h4" | "h5" | "h6")
|
||||
}
|
||||
|
||||
fn handle_tag_start(
|
||||
&mut self,
|
||||
tag: &HtmlElement,
|
||||
writer: &mut MarkdownWriter,
|
||||
) -> StartTagOutcome {
|
||||
match tag.tag() {
|
||||
"h1" => writer.push_str("\n\n# "),
|
||||
"h2" => writer.push_str("\n\n## "),
|
||||
"h3" => writer.push_str("\n\n### "),
|
||||
"h4" => writer.push_str("\n\n#### "),
|
||||
"h5" => writer.push_str("\n\n##### "),
|
||||
"h6" => writer.push_str("\n\n###### "),
|
||||
_ => {}
|
||||
}
|
||||
|
||||
StartTagOutcome::Continue
|
||||
}
|
||||
|
||||
fn handle_tag_end(&mut self, tag: &HtmlElement, writer: &mut MarkdownWriter) {
|
||||
match tag.tag() {
|
||||
"h1" | "h2" | "h3" | "h4" | "h5" | "h6" => writer.push_blank_line(),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct ListHandler;
|
||||
|
||||
impl HandleTag for ListHandler {
|
||||
fn should_handle(&self, tag: &str) -> bool {
|
||||
matches!(tag, "ul" | "ol" | "li")
|
||||
}
|
||||
|
||||
fn handle_tag_start(
|
||||
&mut self,
|
||||
tag: &HtmlElement,
|
||||
writer: &mut MarkdownWriter,
|
||||
) -> StartTagOutcome {
|
||||
match tag.tag() {
|
||||
"ul" | "ol" => writer.push_newline(),
|
||||
"li" => writer.push_str("- "),
|
||||
_ => {}
|
||||
}
|
||||
|
||||
StartTagOutcome::Continue
|
||||
}
|
||||
|
||||
fn handle_tag_end(&mut self, tag: &HtmlElement, writer: &mut MarkdownWriter) {
|
||||
match tag.tag() {
|
||||
"ul" | "ol" => writer.push_newline(),
|
||||
"li" => writer.push_newline(),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct TableHandler {
|
||||
/// The number of columns in the current `<table>`.
|
||||
current_table_columns: usize,
|
||||
is_first_th: bool,
|
||||
is_first_td: bool,
|
||||
}
|
||||
|
||||
impl TableHandler {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
current_table_columns: 0,
|
||||
is_first_th: true,
|
||||
is_first_td: true,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for TableHandler {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl HandleTag for TableHandler {
|
||||
fn should_handle(&self, tag: &str) -> bool {
|
||||
matches!(tag, "table" | "thead" | "tbody" | "tr" | "th" | "td")
|
||||
}
|
||||
|
||||
fn handle_tag_start(
|
||||
&mut self,
|
||||
tag: &HtmlElement,
|
||||
writer: &mut MarkdownWriter,
|
||||
) -> StartTagOutcome {
|
||||
match tag.tag() {
|
||||
"thead" => writer.push_blank_line(),
|
||||
"tr" => writer.push_newline(),
|
||||
"th" => {
|
||||
self.current_table_columns += 1;
|
||||
if self.is_first_th {
|
||||
self.is_first_th = false;
|
||||
} else {
|
||||
writer.push_str(" ");
|
||||
}
|
||||
writer.push_str("| ");
|
||||
}
|
||||
"td" => {
|
||||
if self.is_first_td {
|
||||
self.is_first_td = false;
|
||||
} else {
|
||||
writer.push_str(" ");
|
||||
}
|
||||
writer.push_str("| ");
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
|
||||
StartTagOutcome::Continue
|
||||
}
|
||||
|
||||
fn handle_tag_end(&mut self, tag: &HtmlElement, writer: &mut MarkdownWriter) {
|
||||
match tag.tag() {
|
||||
"thead" => {
|
||||
writer.push_newline();
|
||||
for ix in 0..self.current_table_columns {
|
||||
if ix > 0 {
|
||||
writer.push_str(" ");
|
||||
}
|
||||
writer.push_str("| ---");
|
||||
}
|
||||
writer.push_str(" |");
|
||||
self.is_first_th = true;
|
||||
}
|
||||
"tr" => {
|
||||
writer.push_str(" |");
|
||||
self.is_first_td = true;
|
||||
}
|
||||
"table" => {
|
||||
self.current_table_columns = 0;
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct StyledTextHandler;
|
||||
|
||||
impl HandleTag for StyledTextHandler {
|
||||
fn should_handle(&self, tag: &str) -> bool {
|
||||
matches!(tag, "strong" | "em")
|
||||
}
|
||||
|
||||
fn handle_tag_start(
|
||||
&mut self,
|
||||
tag: &HtmlElement,
|
||||
writer: &mut MarkdownWriter,
|
||||
) -> StartTagOutcome {
|
||||
match tag.tag() {
|
||||
"strong" => writer.push_str("**"),
|
||||
"em" => writer.push_str("_"),
|
||||
_ => {}
|
||||
}
|
||||
|
||||
StartTagOutcome::Continue
|
||||
}
|
||||
|
||||
fn handle_tag_end(&mut self, tag: &HtmlElement, writer: &mut MarkdownWriter) {
|
||||
match tag.tag() {
|
||||
"strong" => writer.push_str("**"),
|
||||
"em" => writer.push_str("_"),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct CodeHandler;
|
||||
|
||||
impl HandleTag for CodeHandler {
|
||||
fn should_handle(&self, tag: &str) -> bool {
|
||||
matches!(tag, "pre" | "code")
|
||||
}
|
||||
|
||||
fn handle_tag_start(
|
||||
&mut self,
|
||||
tag: &HtmlElement,
|
||||
writer: &mut MarkdownWriter,
|
||||
) -> StartTagOutcome {
|
||||
match tag.tag() {
|
||||
"code" => {
|
||||
if !writer.is_inside("pre") {
|
||||
writer.push_str("`");
|
||||
}
|
||||
}
|
||||
"pre" => writer.push_str("\n\n```\n"),
|
||||
_ => {}
|
||||
}
|
||||
|
||||
StartTagOutcome::Continue
|
||||
}
|
||||
|
||||
fn handle_tag_end(&mut self, tag: &HtmlElement, writer: &mut MarkdownWriter) {
|
||||
match tag.tag() {
|
||||
"code" => {
|
||||
if !writer.is_inside("pre") {
|
||||
writer.push_str("`");
|
||||
}
|
||||
}
|
||||
"pre" => writer.push_str("\n```\n"),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
fn handle_text(&mut self, text: &str, writer: &mut MarkdownWriter) -> HandlerOutcome {
|
||||
if writer.is_inside("pre") {
|
||||
writer.push_str(text);
|
||||
return HandlerOutcome::Handled;
|
||||
}
|
||||
|
||||
HandlerOutcome::NoOp
|
||||
}
|
||||
}
|
||||
199
crates/html_to_markdown/src/markdown_writer.rs
Normal file
199
crates/html_to_markdown/src/markdown_writer.rs
Normal file
@@ -0,0 +1,199 @@
|
||||
use std::collections::VecDeque;
|
||||
use std::rc::Rc;
|
||||
use std::{cell::RefCell, sync::LazyLock};
|
||||
|
||||
use anyhow::Result;
|
||||
use markup5ever_rcdom::{Handle, NodeData};
|
||||
use regex::Regex;
|
||||
|
||||
use crate::html_element::HtmlElement;
|
||||
|
||||
fn empty_line_regex() -> &'static Regex {
|
||||
static REGEX: LazyLock<Regex> =
|
||||
LazyLock::new(|| Regex::new(r"^\s*$").expect("Failed to create empty_line_regex"));
|
||||
®EX
|
||||
}
|
||||
|
||||
fn more_than_three_newlines_regex() -> &'static Regex {
|
||||
static REGEX: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"\n{3,}").unwrap());
|
||||
®EX
|
||||
}
|
||||
|
||||
pub enum StartTagOutcome {
|
||||
Continue,
|
||||
Skip,
|
||||
}
|
||||
|
||||
pub type TagHandler = Rc<RefCell<dyn HandleTag>>;
|
||||
|
||||
pub struct MarkdownWriter {
|
||||
current_element_stack: VecDeque<HtmlElement>,
|
||||
pub(crate) markdown: String,
|
||||
}
|
||||
|
||||
impl Default for MarkdownWriter {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl MarkdownWriter {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
current_element_stack: VecDeque::new(),
|
||||
markdown: String::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn current_element_stack(&self) -> &VecDeque<HtmlElement> {
|
||||
&self.current_element_stack
|
||||
}
|
||||
|
||||
pub fn is_inside(&self, tag: &str) -> bool {
|
||||
self.current_element_stack
|
||||
.iter()
|
||||
.any(|parent_element| parent_element.tag() == tag)
|
||||
}
|
||||
|
||||
/// Appends the given string slice onto the end of the Markdown output.
|
||||
pub fn push_str(&mut self, str: &str) {
|
||||
self.markdown.push_str(str);
|
||||
}
|
||||
|
||||
/// Appends a newline to the end of the Markdown output.
|
||||
pub fn push_newline(&mut self) {
|
||||
self.push_str("\n");
|
||||
}
|
||||
|
||||
/// Appends a blank line to the end of the Markdown output.
|
||||
pub fn push_blank_line(&mut self) {
|
||||
self.push_str("\n\n");
|
||||
}
|
||||
|
||||
pub fn run(mut self, root_node: &Handle, handlers: &mut [TagHandler]) -> Result<String> {
|
||||
self.visit_node(root_node, handlers)?;
|
||||
Ok(Self::prettify_markdown(self.markdown))
|
||||
}
|
||||
|
||||
fn prettify_markdown(markdown: String) -> String {
|
||||
let markdown = empty_line_regex().replace_all(&markdown, "");
|
||||
let markdown = more_than_three_newlines_regex().replace_all(&markdown, "\n\n");
|
||||
|
||||
markdown.trim().to_string()
|
||||
}
|
||||
|
||||
fn visit_node(&mut self, node: &Handle, handlers: &mut [TagHandler]) -> Result<()> {
|
||||
let mut current_element = None;
|
||||
|
||||
match node.data {
|
||||
NodeData::Document
|
||||
| NodeData::Doctype { .. }
|
||||
| NodeData::ProcessingInstruction { .. }
|
||||
| NodeData::Comment { .. } => {
|
||||
// Currently left unimplemented, as we're not interested in this data
|
||||
// at this time.
|
||||
}
|
||||
NodeData::Element {
|
||||
ref name,
|
||||
ref attrs,
|
||||
..
|
||||
} => {
|
||||
let tag_name = name.local.to_string();
|
||||
if !tag_name.is_empty() {
|
||||
current_element = Some(HtmlElement::new(tag_name, attrs.clone()));
|
||||
}
|
||||
}
|
||||
NodeData::Text { ref contents } => {
|
||||
let text = contents.borrow().to_string();
|
||||
self.visit_text(text, handlers)?;
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(current_element) = current_element.as_ref() {
|
||||
match self.start_tag(current_element, handlers) {
|
||||
StartTagOutcome::Continue => {}
|
||||
StartTagOutcome::Skip => return Ok(()),
|
||||
}
|
||||
|
||||
self.current_element_stack
|
||||
.push_back(current_element.clone());
|
||||
}
|
||||
|
||||
if self.current_element_stack.len() < 200 {
|
||||
for child in node.children.borrow().iter() {
|
||||
self.visit_node(child, handlers)?;
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(current_element) = current_element {
|
||||
self.current_element_stack.pop_back();
|
||||
self.end_tag(¤t_element, handlers);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn start_tag(&mut self, tag: &HtmlElement, handlers: &mut [TagHandler]) -> StartTagOutcome {
|
||||
for handler in handlers {
|
||||
if handler.borrow().should_handle(tag.tag()) {
|
||||
match handler.borrow_mut().handle_tag_start(tag, self) {
|
||||
StartTagOutcome::Continue => {}
|
||||
StartTagOutcome::Skip => return StartTagOutcome::Skip,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
StartTagOutcome::Continue
|
||||
}
|
||||
|
||||
fn end_tag(&mut self, tag: &HtmlElement, handlers: &mut [TagHandler]) {
|
||||
for handler in handlers {
|
||||
if handler.borrow().should_handle(tag.tag()) {
|
||||
handler.borrow_mut().handle_tag_end(tag, self);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn visit_text(&mut self, text: String, handlers: &mut [TagHandler]) -> Result<()> {
|
||||
for handler in handlers {
|
||||
match handler.borrow_mut().handle_text(&text, self) {
|
||||
HandlerOutcome::Handled => return Ok(()),
|
||||
HandlerOutcome::NoOp => {}
|
||||
}
|
||||
}
|
||||
|
||||
let text = text
|
||||
.trim_matches(|char| char == '\n' || char == '\r' || char == '\t')
|
||||
.replace('\n', " ");
|
||||
|
||||
self.push_str(&text);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
pub enum HandlerOutcome {
|
||||
Handled,
|
||||
NoOp,
|
||||
}
|
||||
|
||||
pub trait HandleTag {
|
||||
/// Returns whether this handler should handle the given tag.
|
||||
fn should_handle(&self, tag: &str) -> bool;
|
||||
|
||||
/// Handles the start of the given tag.
|
||||
fn handle_tag_start(
|
||||
&mut self,
|
||||
_tag: &HtmlElement,
|
||||
_writer: &mut MarkdownWriter,
|
||||
) -> StartTagOutcome {
|
||||
StartTagOutcome::Continue
|
||||
}
|
||||
|
||||
/// Handles the end of the given tag.
|
||||
fn handle_tag_end(&mut self, _tag: &HtmlElement, _writer: &mut MarkdownWriter) {}
|
||||
|
||||
fn handle_text(&mut self, _text: &str, _writer: &mut MarkdownWriter) -> HandlerOutcome {
|
||||
HandlerOutcome::NoOp
|
||||
}
|
||||
}
|
||||
1
crates/html_to_markdown/src/structure.rs
Normal file
1
crates/html_to_markdown/src/structure.rs
Normal file
@@ -0,0 +1 @@
|
||||
pub mod wikipedia;
|
||||
181
crates/html_to_markdown/src/structure/wikipedia.rs
Normal file
181
crates/html_to_markdown/src/structure/wikipedia.rs
Normal file
@@ -0,0 +1,181 @@
|
||||
use crate::HandleTag;
|
||||
use crate::html_element::HtmlElement;
|
||||
use crate::markdown_writer::{HandlerOutcome, MarkdownWriter, StartTagOutcome};
|
||||
|
||||
pub struct WikipediaChromeRemover;
|
||||
|
||||
impl HandleTag for WikipediaChromeRemover {
|
||||
fn should_handle(&self, _tag: &str) -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
fn handle_tag_start(
|
||||
&mut self,
|
||||
tag: &HtmlElement,
|
||||
_writer: &mut MarkdownWriter,
|
||||
) -> StartTagOutcome {
|
||||
match tag.tag() {
|
||||
"head" | "script" | "style" | "nav" => return StartTagOutcome::Skip,
|
||||
"sup" => {
|
||||
if tag.has_class("reference") {
|
||||
return StartTagOutcome::Skip;
|
||||
}
|
||||
}
|
||||
"div" | "span" | "a" => {
|
||||
if tag.attr("id").as_deref() == Some("p-lang-btn") {
|
||||
return StartTagOutcome::Skip;
|
||||
}
|
||||
|
||||
if tag.attr("id").as_deref() == Some("p-search") {
|
||||
return StartTagOutcome::Skip;
|
||||
}
|
||||
|
||||
let classes_to_skip = ["noprint", "mw-editsection", "mw-jump-link"];
|
||||
if tag.has_any_classes(&classes_to_skip) {
|
||||
return StartTagOutcome::Skip;
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
|
||||
StartTagOutcome::Continue
|
||||
}
|
||||
}
|
||||
|
||||
pub struct WikipediaInfoboxHandler;
|
||||
|
||||
impl HandleTag for WikipediaInfoboxHandler {
|
||||
fn should_handle(&self, tag: &str) -> bool {
|
||||
tag == "table"
|
||||
}
|
||||
|
||||
fn handle_tag_start(
|
||||
&mut self,
|
||||
tag: &HtmlElement,
|
||||
_writer: &mut MarkdownWriter,
|
||||
) -> StartTagOutcome {
|
||||
if tag.tag() == "table" && tag.has_class("infobox") {
|
||||
return StartTagOutcome::Skip;
|
||||
}
|
||||
|
||||
StartTagOutcome::Continue
|
||||
}
|
||||
}
|
||||
|
||||
pub struct WikipediaCodeHandler {
|
||||
language: Option<String>,
|
||||
}
|
||||
|
||||
impl WikipediaCodeHandler {
|
||||
pub fn new() -> Self {
|
||||
Self { language: None }
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for WikipediaCodeHandler {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl HandleTag for WikipediaCodeHandler {
|
||||
fn should_handle(&self, tag: &str) -> bool {
|
||||
matches!(tag, "div" | "pre" | "code")
|
||||
}
|
||||
|
||||
fn handle_tag_start(
|
||||
&mut self,
|
||||
tag: &HtmlElement,
|
||||
writer: &mut MarkdownWriter,
|
||||
) -> StartTagOutcome {
|
||||
match tag.tag() {
|
||||
"code" => {
|
||||
if !writer.is_inside("pre") {
|
||||
writer.push_str("`");
|
||||
}
|
||||
}
|
||||
"div" => {
|
||||
let classes = tag.classes();
|
||||
self.language = classes.iter().find_map(|class| {
|
||||
if let Some((_, language)) = class.split_once("mw-highlight-lang-") {
|
||||
Some(language.trim().to_owned())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
});
|
||||
}
|
||||
"pre" => {
|
||||
writer.push_blank_line();
|
||||
writer.push_str("```");
|
||||
if let Some(language) = self.language.take() {
|
||||
writer.push_str(&language);
|
||||
}
|
||||
writer.push_newline();
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
|
||||
StartTagOutcome::Continue
|
||||
}
|
||||
|
||||
fn handle_tag_end(&mut self, tag: &HtmlElement, writer: &mut MarkdownWriter) {
|
||||
match tag.tag() {
|
||||
"code" => {
|
||||
if !writer.is_inside("pre") {
|
||||
writer.push_str("`");
|
||||
}
|
||||
}
|
||||
"pre" => writer.push_str("\n```\n"),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
fn handle_text(&mut self, text: &str, writer: &mut MarkdownWriter) -> HandlerOutcome {
|
||||
if writer.is_inside("pre") {
|
||||
writer.push_str(text);
|
||||
return HandlerOutcome::Handled;
|
||||
}
|
||||
|
||||
HandlerOutcome::NoOp
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::cell::RefCell;
|
||||
use std::rc::Rc;
|
||||
|
||||
use indoc::indoc;
|
||||
use pretty_assertions::assert_eq;
|
||||
|
||||
use crate::{TagHandler, convert_html_to_markdown, markdown};
|
||||
|
||||
use super::*;
|
||||
|
||||
fn wikipedia_handlers() -> Vec<TagHandler> {
|
||||
vec![
|
||||
Rc::new(RefCell::new(markdown::ParagraphHandler)),
|
||||
Rc::new(RefCell::new(markdown::HeadingHandler)),
|
||||
Rc::new(RefCell::new(markdown::ListHandler)),
|
||||
Rc::new(RefCell::new(markdown::StyledTextHandler)),
|
||||
Rc::new(RefCell::new(WikipediaChromeRemover)),
|
||||
]
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_citation_references_get_removed() {
|
||||
let html = indoc! {r##"
|
||||
<p>Rust began as a personal project in 2006 by <a href="/wiki/Mozilla" title="Mozilla">Mozilla</a> Research employee Graydon Hoare.<sup id="cite_ref-MITTechReview_23-0" class="reference"><a href="#cite_note-MITTechReview-23">[20]</a></sup> Mozilla began sponsoring the project in 2009 as a part of the ongoing development of an experimental <a href="/wiki/Browser_engine" title="Browser engine">browser engine</a> called <a href="/wiki/Servo_(software)" title="Servo (software)">Servo</a>,<sup id="cite_ref-infoq2012_24-0" class="reference"><a href="#cite_note-infoq2012-24">[21]</a></sup> which was officially announced by Mozilla in 2010.<sup id="cite_ref-MattAsay_25-0" class="reference"><a href="#cite_note-MattAsay-25">[22]</a></sup><sup id="cite_ref-26" class="reference"><a href="#cite_note-26">[23]</a></sup> Rust's memory and ownership system was influenced by <a href="/wiki/Region-based_memory_management" title="Region-based memory management">region-based memory management</a> in languages such as <a href="/wiki/Cyclone_(programming_language)" title="Cyclone (programming language)">Cyclone</a> and ML Kit.<sup id="cite_ref-influences_8-13" class="reference"><a href="#cite_note-influences-8">[5]</a></sup>
|
||||
</p>
|
||||
"##};
|
||||
let expected = indoc! {"
|
||||
Rust began as a personal project in 2006 by Mozilla Research employee Graydon Hoare. Mozilla began sponsoring the project in 2009 as a part of the ongoing development of an experimental browser engine called Servo, which was officially announced by Mozilla in 2010. Rust's memory and ownership system was influenced by region-based memory management in languages such as Cyclone and ML Kit.
|
||||
"}
|
||||
.trim();
|
||||
|
||||
assert_eq!(
|
||||
convert_html_to_markdown(html.as_bytes(), &mut wikipedia_handlers()).unwrap(),
|
||||
expected
|
||||
)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user