diff --git a/.sampo/changesets/cunning-thunderbearer-vipunen.md b/.sampo/changesets/cunning-thunderbearer-vipunen.md new file mode 100644 index 00000000..e0bab576 --- /dev/null +++ b/.sampo/changesets/cunning-thunderbearer-vipunen.md @@ -0,0 +1,7 @@ +--- +cargo/satteri-ast: patch +cargo/satteri-napi: patch +npm/satteri: patch +--- + +Fixed `rawHtml: true` losing code-fence `data.lang` and `data.meta` in `markdownToHast` and MDX compilation, so HAST plugins can read fence options. diff --git a/.sampo/changesets/raw-html-mdx-component-overrides.md b/.sampo/changesets/raw-html-mdx-component-overrides.md new file mode 100644 index 00000000..3ee877dc --- /dev/null +++ b/.sampo/changesets/raw-html-mdx-component-overrides.md @@ -0,0 +1,6 @@ +--- +cargo/satteri-ast: patch +npm/satteri: patch +--- + +Fixed `export const components` being ignored in MDX when `rawHtml` and `optimizeStatic` are both enabled. diff --git a/.sampo/changesets/raw-html-positions.md b/.sampo/changesets/raw-html-positions.md new file mode 100644 index 00000000..1b13308b --- /dev/null +++ b/.sampo/changesets/raw-html-positions.md @@ -0,0 +1,6 @@ +--- +cargo/satteri-ast: patch +npm/satteri: patch +--- + +Fixed `rawHtml` dropping the original source and root span. Markdown-derived nodes and self-contained raw blocks now retain their positions when their source association is unambiguous. diff --git a/crates/satteri-ast/src/convert.rs b/crates/satteri-ast/src/convert.rs index 7ca6c949..e9e0c69f 100644 --- a/crates/satteri-ast/src/convert.rs +++ b/crates/satteri-ast/src/convert.rs @@ -397,7 +397,7 @@ pub struct ConvertOptions { /// Reparse raw HTML embedded in the converted tree into real HAST nodes /// (see [`raw_to_hast_arena`](crate::hast::from_html::raw_to_hast_arena)). /// Applied as the final conversion step so every pipeline that converts - /// MDAST to HAST gets it. Positions are not preserved through the reparse. + /// MDAST to HAST gets it. Unambiguously associated nodes retain their positions. /// Default: `false`. #[cfg(feature = "from-html")] pub raw_html: bool, diff --git a/crates/satteri-ast/src/hast/from_html.rs b/crates/satteri-ast/src/hast/from_html.rs index 4d5bd9fc..d36fe6fa 100644 --- a/crates/satteri-ast/src/hast/from_html.rs +++ b/crates/satteri-ast/src/hast/from_html.rs @@ -6,14 +6,19 @@ //! that is then emitted into the builder in document order. use std::cell::{Cell, Ref, RefCell}; +use std::collections::{HashMap, HashSet}; +use std::ops::Range; +use html5ever::buffer_queue::BufferQueue; use html5ever::interface::{ElementFlags, NodeOrText, QuirksMode, TreeSink}; use html5ever::tendril::{StrTendril, TendrilSink}; +use html5ever::tokenizer::{TagKind, Token, TokenSink, TokenSinkResult, Tokenizer}; +use html5ever::tree_builder::TreeBuilder; use html5ever::{ Attribute, LocalName, Namespace, ParseOpts, QualName, parse_document, parse_fragment, tree_builder::TreeBuilderOpts, }; -use satteri_arena::{Arena, ArenaBuilder, Hast, StringRef}; +use satteri_arena::{Arena, ArenaBuilder, Hast, NodePosition, StringRef}; use satteri_property_info::{PropKind, find_property}; use crate::hast::codec::{ @@ -103,6 +108,11 @@ struct Node { parent: Option, children: Vec, data: NodeData, + /// Original opening token that created this handle, retained through parser moves. + origin: Option, + position_origin: Option, + /// Serialized bytes that produced this node, including any coalesced text. + span: Option>, } enum NodeData { @@ -130,6 +140,7 @@ struct HtmlSink { nodes: RefCell>, quirks_mode: Cell, stitch: Option, + token_span: Cell>, } impl HtmlSink { @@ -139,9 +150,13 @@ impl HtmlSink { parent: None, children: Vec::new(), data: NodeData::Document, + origin: None, + position_origin: None, + span: None, }]), quirks_mode: Cell::new(QuirksMode::NoQuirks), stitch, + token_span: Cell::new(None), } } } @@ -205,6 +220,9 @@ fn new_node(nodes: &mut Vec, data: NodeData) -> usize { parent: None, children: Vec::new(), data, + origin: None, + position_origin: None, + span: None, }); id } @@ -236,9 +254,13 @@ fn append_node(nodes: &mut [Node], parent: usize, child: usize) { } /// The tree builder expects adjacent text to coalesce into a single node. -fn push_text(nodes: &mut [Node], target: usize, text: &str) -> bool { +fn push_text(nodes: &mut [Node], target: usize, text: &str, span: Option<(usize, usize)>) -> bool { if let NodeData::Text { contents } = &mut nodes[target].data { contents.push_slice(text); + if let (Some(existing), Some((start, end))) = (&mut nodes[target].span, span) { + existing.start = existing.start.min(start); + existing.end = existing.end.max(end); + } true } else { false @@ -304,7 +326,7 @@ impl TreeSink for HtmlSink { let parent = *parent; if let NodeOrText::AppendText(text) = &child && let Some(&last) = nodes[parent].children.last() - && push_text(&mut nodes, last, text) + && push_text(&mut nodes, last, text, self.token_span.get()) { return; } @@ -327,7 +349,7 @@ impl TreeSink for HtmlSink { } (NodeOrText::AppendText(text), index) => { let prev = nodes[parent].children[index - 1]; - if push_text(&mut nodes, prev, &text) { + if push_text(&mut nodes, prev, &text, self.token_span.get()) { return; } new_node(&mut nodes, NodeData::Text { contents: text }) @@ -483,7 +505,12 @@ fn emit( } } NodeData::Doctype => { - builder.add_leaf(HastNodeType::Doctype as u8); + let leaf = builder.add_leaf(HastNodeType::Doctype as u8); + copy_position( + builder, + leaf, + nodes[id].position_origin.zip(stitching.map(|s| s.src)), + ); } NodeData::Text { contents } => { let text = scrub_markers(contents, leaked); @@ -492,6 +519,11 @@ fn emit( builder .arena_mut() .set_type_data(leaf, &text_ref.as_bytes()); + copy_position( + builder, + leaf, + nodes[id].position_origin.zip(stitching.map(|s| s.src)), + ); } NodeData::Comment { contents } => { let text = scrub_markers(contents, leaked); @@ -500,6 +532,11 @@ fn emit( builder .arena_mut() .set_type_data(leaf, &text_ref.as_bytes()); + copy_position( + builder, + leaf, + nodes[id].position_origin.zip(stitching.map(|s| s.src)), + ); } NodeData::Stitch(index) => { let stitches = stitching.expect("Stitch node without stitching").stitches; @@ -526,6 +563,17 @@ fn emit( let element = builder.open_node(HastNodeType::Element as u8); let data = encode_element_data(tag_ref, &props); builder.arena_mut().set_type_data(element, &data); + copy_position( + builder, + element, + nodes[id].position_origin.zip(stitching.map(|s| s.src)), + ); + if &*name.local == "code" + && let Some(origin) = nodes[id].origin + && let Some(data) = stitching.and_then(|s| s.src.get_node_data(origin)) + { + builder.arena_mut().set_node_data(element, data.to_vec()); + } stack.push(EmitTask::Close); // `