From 75bcb809e9f95829739988008de14055cbc17a5d Mon Sep 17 00:00:00 2001 From: Princesseuh <3019731+Princesseuh@users.noreply.github.com> Date: Sun, 23 Aug 2026 13:51:37 +0200 Subject: [PATCH 1/5] fix(ast): carry the source and root span through the rawHtml reparse --- .sampo/changesets/raw-html-root-position.md | 6 ++++ crates/satteri-ast/src/hast/from_html.rs | 16 ++++++++-- packages/satteri/src/compile.ts | 3 +- .../satteri/test/raw-html-pipelines.test.ts | 30 ++++++++++++++++++- website/content/docs/entry-points.md | 2 +- website/content/docs/features.md | 2 +- 6 files changed, 52 insertions(+), 7 deletions(-) create mode 100644 .sampo/changesets/raw-html-root-position.md diff --git a/.sampo/changesets/raw-html-root-position.md b/.sampo/changesets/raw-html-root-position.md new file mode 100644 index 00000000..90f121aa --- /dev/null +++ b/.sampo/changesets/raw-html-root-position.md @@ -0,0 +1,6 @@ +--- +cargo/satteri-ast: patch +npm/satteri: patch +--- + +Fixes `rawHtml` dropping the root node's position, and MDX component overrides being ignored when `rawHtml` is enabled. diff --git a/crates/satteri-ast/src/hast/from_html.rs b/crates/satteri-ast/src/hast/from_html.rs index b0e5802f..14288ffb 100644 --- a/crates/satteri-ast/src/hast/from_html.rs +++ b/crates/satteri-ast/src/hast/from_html.rs @@ -807,11 +807,21 @@ pub fn html_fragment_to_wrap_arena(html: &str) -> Result, String> { /// no synthesised ``/``/`` wrapper. /// /// MDX nodes have no HTML form; they are carried through as placeholder -/// comments and spliced back afterwards. Positions are not preserved: the -/// tree is rebuilt from serialised HTML. +/// comments and spliced back afterwards. Child positions do not survive the +/// rebuild from serialised HTML; the source string and the root's own span +/// describe the document rather than the reparse, so both carry over. pub fn raw_to_hast_arena(arena: &Arena) -> Arena { - let mut builder = ArenaBuilder::::new(String::new()); + let mut builder = ArenaBuilder::::new(arena.source().to_string()); builder.open_node_raw(HastNodeType::Root as u8); + let root = arena.get_node(0); + builder.set_position_current( + root.start_offset, + root.end_offset, + root.start_line, + root.start_column, + root.end_line, + root.end_column, + ); reparse_children_into(arena, 0, &mut builder); builder.close_node(); builder.finish() diff --git a/packages/satteri/src/compile.ts b/packages/satteri/src/compile.ts index e5c757be..6b398cb1 100644 --- a/packages/satteri/src/compile.ts +++ b/packages/satteri/src/compile.ts @@ -591,7 +591,8 @@ export interface Features { * (re-emitted verbatim on stringify). With `rawHtml: true`, the tree is * reparsed so raw HTML becomes structured `element`/`text`/`comment` nodes * with normalized properties, including tags that open in one raw block - * and close in another. Positions are not preserved through the reparse. + * and close in another. Positions are not preserved through the reparse, + * except on the root, which keeps the document's own span. */ rawHtml?: boolean; } diff --git a/packages/satteri/test/raw-html-pipelines.test.ts b/packages/satteri/test/raw-html-pipelines.test.ts index a6abc2be..6e7b7a58 100644 --- a/packages/satteri/test/raw-html-pipelines.test.ts +++ b/packages/satteri/test/raw-html-pipelines.test.ts @@ -1,5 +1,12 @@ import { test, expect } from "vitest"; -import { markdownToHtml, markdownToJs, defineMdastPlugin, defineHastPlugin } from "../src/index.js"; +import { + markdownToHtml, + markdownToHast, + markdownToJs, + mdxToJs, + defineMdastPlugin, + defineHastPlugin, +} from "../src/index.js"; // `rawHtml` is applied during MDAST→HAST conversion, so every pipeline (the // no-plugin fast path, the MDAST-plugin fused tail, and the full @@ -131,6 +138,27 @@ test("plugin-spliced raw HTML is reparsed too", () => { expect(html).toContain(''); }); +test("rawHtml keeps the document span on the root", () => { + const md = "# Hi\n\n
hello
\n\nBye\n"; + expect(markdownToHast(md, { features: { rawHtml: true } }).position).toEqual( + markdownToHast(md).position, + ); +}); + +// Offsets are UTF-16 code units, so only a multibyte document exposes a lost source. +test("rawHtml root offsets stay in UTF-16 code units", () => { + const md = "# 👋 héllo\n\n
x
\n\nBye\n"; + expect(markdownToHast(md, { features: { rawHtml: true } }).position?.end.offset).toBe(md.length); +}); + +test("rawHtml keeps component overrides visible to optimizeStatic", () => { + const mdx = 'export const components = { h1: "h2" };\n\n# Hello\n\nSome text.\n'; + const optimizeStatic = { component: "Fragment", prop: "set:html" }; + expect(mdxToJs(mdx, { optimizeStatic, features: { rawHtml: true } }).code).toBe( + mdxToJs(mdx, { optimizeStatic }).code, + ); +}); + test("rawHtml keeps namespaced SVG attributes through the round trip", () => { const sprite = '\n'; diff --git a/website/content/docs/entry-points.md b/website/content/docs/entry-points.md index 8b9aff28..34439d94 100644 --- a/website/content/docs/entry-points.md +++ b/website/content/docs/entry-points.md @@ -148,4 +148,4 @@ const tree = markdownToHast(`
\n\n**hi**\n\n
`, { //
is now a real element wrapping the parsed

hi

``` -The whole tree is reparsed through the HTML parser, so a tag opened in one raw block and closed in another is resolved against the surrounding Markdown. Positions are not preserved through the reparse. +The whole tree is reparsed through the HTML parser, so a tag opened in one raw block and closed in another is resolved against the surrounding Markdown. Positions are not preserved through the reparse, except on the root, which keeps the document's own span. diff --git a/website/content/docs/features.md b/website/content/docs/features.md index e9a738be..78f50b71 100644 --- a/website/content/docs/features.md +++ b/website/content/docs/features.md @@ -286,7 +286,7 @@ rawHtml?: boolean By default, raw HTML embedded in Markdown is kept as opaque `raw` nodes and re-emitted verbatim. `rawHtml: true` reparses it into real HAST element, text, and comment nodes. The reparse runs during the mdast→hast conversion, so `markdownToHast`, `markdownToHtml`, and the plugin pipelines all reparse identically, and HAST plugins always see the reparsed elements. -The whole tree goes through the HTML parser, so a tag opened in one raw block and closed in another is resolved against the surrounding Markdown. Attributes are normalised into typed hast properties (`class` → `className: ["…"]`, `disabled` → `true`, `tabindex` → a number, `data-foo-bar` → `dataFooBar`). In MDX, JSX elements and expressions are preserved in place while the raw HTML around them is still resolved. Positions are not preserved through the reparse. +The whole tree goes through the HTML parser, so a tag opened in one raw block and closed in another is resolved against the surrounding Markdown. Attributes are normalised into typed hast properties (`class` → `className: ["…"]`, `disabled` → `true`, `tabindex` → a number, `data-foo-bar` → `dataFooBar`). In MDX, JSX elements and expressions are preserved in place while the raw HTML around them is still resolved. Positions are not preserved through the reparse, except on the root, which keeps the document's own span. ```js import { markdownToHast } from "satteri"; From c6d72a42e44adad13b527b58dc4158a6c4f75a3c Mon Sep 17 00:00:00 2001 From: Princesseuh <3019731+Princesseuh@users.noreply.github.com> Date: Sun, 23 Aug 2026 15:09:38 +0200 Subject: [PATCH 2/5] fix(ast): keep Markdown-derived positions through the rawHtml reparse --- .sampo/changesets/raw-html-root-position.md | 2 +- crates/satteri-ast/src/hast/from_html.rs | 362 ++++++++++++++---- crates/satteri-ast/src/hast/render.rs | 43 +-- packages/satteri/src/compile.ts | 4 +- .../satteri/test/raw-html-conformance.test.ts | 43 +++ website/content/docs/entry-points.md | 2 +- website/content/docs/features.md | 2 +- 7 files changed, 350 insertions(+), 108 deletions(-) diff --git a/.sampo/changesets/raw-html-root-position.md b/.sampo/changesets/raw-html-root-position.md index 90f121aa..ce6799aa 100644 --- a/.sampo/changesets/raw-html-root-position.md +++ b/.sampo/changesets/raw-html-root-position.md @@ -3,4 +3,4 @@ cargo/satteri-ast: patch npm/satteri: patch --- -Fixes `rawHtml` dropping the root node's position, and MDX component overrides being ignored when `rawHtml` is enabled. +Fixes `rawHtml` dropping every node's position, so nodes that came from Markdown keep their source positions like `rehype-raw`, and fixes MDX component overrides being ignored when `rawHtml` is enabled. diff --git a/crates/satteri-ast/src/hast/from_html.rs b/crates/satteri-ast/src/hast/from_html.rs index 14288ffb..c432610f 100644 --- a/crates/satteri-ast/src/hast/from_html.rs +++ b/crates/satteri-ast/src/hast/from_html.rs @@ -9,8 +9,9 @@ use std::cell::{Cell, Ref, RefCell}; use html5ever::interface::{ElementFlags, NodeOrText, QuirksMode, TreeSink}; use html5ever::tendril::{StrTendril, TendrilSink}; +use html5ever::tokenizer::{Doctype, Tag, TagKind, Token, TokenSink}; use html5ever::{ - Attribute, LocalName, Namespace, ParseOpts, QualName, parse_document, parse_fragment, + Attribute, LocalName, Namespace, ParseOpts, Parser, QualName, parse_document, parse_fragment, tree_builder::TreeBuilderOpts, }; use satteri_arena::{Arena, ArenaBuilder, Hast, StringRef}; @@ -21,7 +22,8 @@ use crate::hast::codec::{ decode_element_prop, decode_element_prop_count, decode_element_tag, decode_text_data, encode_element_data, }; -use crate::hast::render::{is_void_element, render_node_inner}; +use crate::hast::properties::property_to_attribute; +use crate::hast::render::is_void_element; #[cfg(feature = "mdx")] use crate::mdast::codec::{ decode_mdx_jsx_attr, decode_mdx_jsx_attr_count, decode_mdx_jsx_element_name, @@ -63,6 +65,8 @@ struct Node { parent: Option, children: Vec, data: NodeData, + /// Source-arena node this was fed from; `None` when it came from raw HTML. + origin: Option, } enum NodeData { @@ -99,6 +103,7 @@ impl HtmlSink { parent: None, children: Vec::new(), data: NodeData::Document, + origin: None, }]), quirks_mode: Cell::new(QuirksMode::NoQuirks), stitch, @@ -117,13 +122,20 @@ struct StitchRecognizer { } impl StitchRecognizer { - fn new(prefix: String, count: usize) -> Self { + fn new(prefix: String) -> Self { StitchRecognizer { prefix, - claimed: RefCell::new(vec![false; count]), + claimed: RefCell::new(Vec::new()), } } + /// Reserve the next marker index; the reparse discovers MDX nodes as it feeds them. + fn register(&self) -> usize { + let mut claimed = self.claimed.borrow_mut(); + claimed.push(false); + claimed.len() - 1 + } + fn claim(&self, contents: &str) -> Option { let index: usize = contents.strip_prefix(self.prefix.as_str())?.parse().ok()?; let mut claimed = self.claimed.borrow_mut(); @@ -165,6 +177,7 @@ fn new_node(nodes: &mut Vec, data: NodeData) -> usize { parent: None, children: Vec::new(), data, + origin: None, }); id } @@ -421,6 +434,8 @@ fn emit( EmitTask::Emit(id) => id, }; + let origin = nodes[id].origin.zip(src); + match &nodes[id].data { NodeData::Document => { for &child in nodes[id].children.iter().rev() { @@ -428,7 +443,8 @@ fn emit( } } NodeData::Doctype => { - builder.add_leaf_raw(HastNodeType::Doctype as u8); + let leaf = builder.add_leaf_raw(HastNodeType::Doctype as u8); + copy_position(builder, leaf, origin); } NodeData::Text { contents } => { let text = scrub_markers(contents, leaked); @@ -437,6 +453,7 @@ fn emit( builder .arena_mut() .set_type_data(leaf, &text_ref.as_bytes()); + copy_position(builder, leaf, origin); } NodeData::Comment { contents } => { let text = scrub_markers(contents, leaked); @@ -445,6 +462,7 @@ fn emit( builder .arena_mut() .set_type_data(leaf, &text_ref.as_bytes()); + copy_position(builder, leaf, origin); } NodeData::Stitch(index) => { stack.push(EmitTask::EmitArena(stitches[*index])); @@ -457,31 +475,39 @@ fn emit( attrs, template_contents, } => { - let tag_ref = builder.alloc_string(&scrub_markers(&name.local, leaked)); - // The SVG property schema keeps attribute casing (`viewBox`); - // the HTML schema normalises it. - let in_svg = &*name.ns == SVG_NAMESPACE; - let props: Vec<(StringRef, u8, StringRef)> = attrs - .iter() - // An attribute name containing a leaked marker is junk the - // tokenizer minted from marker text; drop it. - .filter(|attr| { - leaked.is_empty() - || !leaked.iter().any(|m| attr.name.local.contains(m.as_str())) - }) - .map(|attr| { - let attr_name = qualified_attr_name(&attr.name); - let (property, prop_kind) = find_property(&attr_name, in_svg); - let name_ref = builder.alloc_string(&property); - let value = scrub_markers(&attr.value, leaked); - let (kind, value_ref) = - coerce_value(builder, prop_kind, &attr_name, &value); - (name_ref, kind, value_ref) - }) - .collect(); - let element = builder.open_node_raw(HastNodeType::Element as u8); - let data = encode_element_data(tag_ref, &props); - builder.arena_mut().set_type_data(element, &data); + // Fed properties are already normalised; copying beats round-tripping. + match origin.filter(|&(aid, src)| src.get_type_data(aid).len() >= 16) { + Some((aid, src)) => { + open_arena_element(src, aid, builder); + } + None => { + let tag_ref = builder.alloc_string(&scrub_markers(&name.local, leaked)); + // The SVG property schema keeps attribute casing (`viewBox`); + // the HTML schema normalises it. + let in_svg = &*name.ns == SVG_NAMESPACE; + let props: Vec<(StringRef, u8, StringRef)> = attrs + .iter() + // An attribute name containing a leaked marker is junk the + // tokenizer minted from marker text; drop it. + .filter(|attr| { + leaked.is_empty() + || !leaked.iter().any(|m| attr.name.local.contains(m.as_str())) + }) + .map(|attr| { + let attr_name = qualified_attr_name(&attr.name); + let (property, prop_kind) = find_property(&attr_name, in_svg); + let name_ref = builder.alloc_string(&property); + let value = scrub_markers(&attr.value, leaked); + let (kind, value_ref) = + coerce_value(builder, prop_kind, &attr_name, &value); + (name_ref, kind, value_ref) + }) + .collect(); + let element = builder.open_node_raw(HastNodeType::Element as u8); + let data = encode_element_data(tag_ref, &props); + builder.arena_mut().set_type_data(element, &data); + } + } stack.push(EmitTask::Close); // `