diff --git a/README.md b/README.md index 676c448..d9ccee2 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ xhtmlmd is largely implemented using AI, except for the tests. The tests are lar - Definition lists: PHP Markdown Extra/Pandoc-style `Term` followed by `: definition` or `~ definition`. - Footnotes: `[^id]` references to defined `[^id]:` definitions with indented continuation blocks. - Abbreviations: `*[HTML]: Hyper Text Markup Language` definitions render matching text as ``. +- Underline (opt-in): with `underline=True`, Discord-style `__x__` renders as `x` while `**x**` stays ``. - Fenced divs: Pandoc/Quarto/Djot-style `:::` containers with attributes or a single class word. ## Usage @@ -54,7 +55,7 @@ Python callers can override rendered nodes with callbacks. Each callback receive Callback names: - Blocks: `paragraph`, `heading`, `block_quote`, `list`, `definition_list`, `code_block`, `html_block`, `html_container`, `thematic_break`, `table`, `div`, `math_block` -- Inlines: `text`, `soft_break`, `hard_break`, `emph`, `strong`, `strike`, `superscript`, `subscript`, `highlight`, `code`, `link`, `image`, `autolink`, `abbr`, `html_inline`, `math_inline`, `footnote_ref`, `span` +- Inlines: `text`, `soft_break`, `hard_break`, `emph`, `strong`, `underline`, `strike`, `superscript`, `subscript`, `highlight`, `code`, `link`, `image`, `autolink`, `abbr`, `html_inline`, `math_inline`, `footnote_ref`, `span` ```python from fastpylight import highlight diff --git a/docs/DIALECT.md b/docs/DIALECT.md index d5501aa..cb5e8b2 100644 --- a/docs/DIALECT.md +++ b/docs/DIALECT.md @@ -8,6 +8,7 @@ When Markdown extensions disagree, this crate chooses the behavior closest to Pa - Definition lists follow PHP Markdown Extra/Pandoc: one-line terms with one or more `:` or `~` definitions. - Footnotes follow Pandoc/kramdown label rules and render as XHTML endnotes with backlinks. The endnotes `
` has no leading `
` (unlike cmark-gfm): separators are a styling concern, so add one with CSS if wanted. - Inline `~~x~~` renders as strikethrough. Inline `~x~` renders as subscript, using the same no-whitespace rule as superscript `^x^`. +- The opt-in `underline` option follows Discord: `__x__` renders as `x` instead of `x`, with the usual underscore emphasis rules (no intraword, `___x___` nests as `x`). Off by default because CommonMark defines `__x__` as strong. - `` parses block Markdown inside the balanced tag. `markdown="span"` parses inline content into a single paragraph child. - Raw HTML passes through unbalanced, per CommonMark. The opt-in `balance` option closes unclosed elements at the fragment end, drops stray closes, and self-closes void tags, without HTML5 implied-end-tag rules. - Math defaults to `MathMode::Brackets`, which recognizes `\(...\)`, `\[...\]`, and `$$...$$`. `MathMode::Dollars` also recognizes `$...$` with Pandoc's guard against currency-like spans. `MathMode::On` preserves backslashes before `[]()` so client-side renderers such as KaTeX can see TeX delimiters. `MathMode::Off` treats TeX delimiters as ordinary Markdown text. diff --git a/python/xhtmlmd/__init__.py b/python/xhtmlmd/__init__.py index f8eb135..557410c 100644 --- a/python/xhtmlmd/__init__.py +++ b/python/xhtmlmd/__init__.py @@ -3,11 +3,12 @@ __all__ = ["to_xhtml", "render", "blocks"] -def to_xhtml(markdown: str, *, math: str = "brackets", tagfilter: bool = False, balance: bool = False, callbacks: dict | None = None, +def to_xhtml(markdown: str, *, math: str = "brackets", tagfilter: bool = False, balance: bool = False, underline: bool = False, + callbacks: dict | None = None, max_inline_depth: int | None = None, max_block_depth: int | None = None, max_link_paren_depth: int | None = None) -> str: "Render Markdown to an XHTML fragment." - return _to_xhtml(markdown, math=math, tagfilter=tagfilter, balance=balance, callbacks=callbacks, max_inline_depth=max_inline_depth, - max_block_depth=max_block_depth, max_link_paren_depth=max_link_paren_depth) + return _to_xhtml(markdown, math=math, tagfilter=tagfilter, balance=balance, underline=underline, callbacks=callbacks, + max_inline_depth=max_inline_depth, max_block_depth=max_block_depth, max_link_paren_depth=max_link_paren_depth) render = to_xhtml diff --git a/python/xhtmlmd/__main__.py b/python/xhtmlmd/__main__.py index dac345f..12c24fa 100644 --- a/python/xhtmlmd/__main__.py +++ b/python/xhtmlmd/__main__.py @@ -3,20 +3,22 @@ from . import to_xhtml -USAGE = ("usage: xhtmlmd [--math=off|on|brackets|dollars] [--balance] [file.md]\n\n" +USAGE = ("usage: xhtmlmd [--math=off|on|brackets|dollars] [--balance] [--underline] [file.md]\n\n" "Reads Markdown from a file or stdin and writes XHTML fragment output. Math defaults to brackets.\n" - "--balance closes unclosed raw HTML tags and drops stray closing tags.") + "--balance closes unclosed raw HTML tags and drops stray closing tags.\n" + "--underline renders Discord-style __x__ as x instead of x.") def main(argv=None): argv = sys.argv[1:] if argv is None else argv - math, balance, file = "brackets", False, None + math, balance, underline, file = "brackets", False, False, None for arg in argv: if arg in ("--math=off", "--math=on", "--math=brackets", "--math=dollars"): math = arg.split("=", 1)[1] elif arg == "--balance": balance = True + elif arg == "--underline": underline = True elif arg in ("-h", "--help"): print(USAGE); return elif arg.startswith("--"): print(f"unknown option: {arg}", file=sys.stderr); sys.exit(2) else: file = arg src = open(file, encoding="utf-8").read() if file else sys.stdin.read() - sys.stdout.write(to_xhtml(src, math=math, balance=balance)) + sys.stdout.write(to_xhtml(src, math=math, balance=balance, underline=underline)) if __name__ == "__main__": main() diff --git a/src/ast.rs b/src/ast.rs index d04a147..74290fb 100644 --- a/src/ast.rs +++ b/src/ast.rs @@ -222,6 +222,10 @@ pub enum Inline { attrs: Attr, children: Vec, }, + Underline { + attrs: Attr, + children: Vec, + }, Strike { attrs: Attr, children: Vec, @@ -283,6 +287,7 @@ impl Inline { match self { Inline::Emph { attrs, .. } | Inline::Strong { attrs, .. } + | Inline::Underline { attrs, .. } | Inline::Strike { attrs, .. } | Inline::Superscript { attrs, .. } | Inline::Subscript { attrs, .. } diff --git a/src/inline.rs b/src/inline.rs index 3c822b7..19888a9 100644 --- a/src/inline.rs +++ b/src/inline.rs @@ -247,7 +247,7 @@ fn parse_inner(src: &str, ctx: &InlineContext<'_>, depth: usize) -> Vec } } scanner.flush_text(); - process_delimiters(&mut nodes, &mut delimiters); + process_delimiters(&mut nodes, &mut delimiters, ctx.options.underline); nodes_to_inlines(&nodes) } @@ -412,7 +412,13 @@ impl InlineScanner<'_, '_> { return None; } }; - process_delimiters_range(self.nodes, self.delimiters, opener.node + 1, target_end); + process_delimiters_range( + self.nodes, + self.delimiters, + opener.node + 1, + target_end, + self.ctx.options.underline, + ); let children = collect_node_inlines(self.nodes, opener.node + 1, target_end); match &mut item { Inline::Link { children: dst, .. } | Inline::Span { children: dst, .. } => { @@ -617,8 +623,8 @@ fn delimiter_run_flags(ch: char, before: char, after: char) -> (bool, bool) { } } -fn process_delimiters(nodes: &mut [Node], delimiters: &mut [Delimiter]) { - process_delimiters_range(nodes, delimiters, 0, usize::MAX); +fn process_delimiters(nodes: &mut [Node], delimiters: &mut [Delimiter], underline: bool) { + process_delimiters_range(nodes, delimiters, 0, usize::MAX, underline); } fn process_delimiters_range( @@ -626,6 +632,7 @@ fn process_delimiters_range( delimiters: &mut [Delimiter], start_node: usize, end_node: usize, + underline: bool, ) { // cmark's openers_bottom: when no opener matches a closer, remember how far // the search went per (char, can_open, len % 3) so later closers of the same @@ -653,7 +660,7 @@ fn process_delimiters_range( closer += 1; continue; }; - if wrap_delimiters(nodes, delimiters, opener, closer, use_len) { + if wrap_delimiters(nodes, delimiters, opener, closer, use_len, underline) { if delimiters[closer].len == 0 { closer += 1; } @@ -720,6 +727,7 @@ fn wrap_delimiters( opener: usize, closer: usize, use_len: usize, + underline: bool, ) -> bool { let open_node = delimiters[opener].node; let close_node = delimiters[closer].node; @@ -738,6 +746,10 @@ fn wrap_delimiters( attrs: Attr::default(), children, }, + '_' if use_len == 2 && underline => Inline::Underline { + attrs: Attr::default(), + children, + }, _ if use_len == 2 => Inline::Strong { attrs: Attr::default(), children, @@ -1754,6 +1766,13 @@ fn normalize_inline(item: Inline) -> Inline { } Inline::Strong { attrs, children } } + Inline::Underline { attrs, children } => { + let mut children = coalesce(children); + if attrs.is_empty() { + children = flatten_empty_underline(children); + } + Inline::Underline { attrs, children } + } Inline::Strike { attrs, children } => Inline::Strike { attrs, children: coalesce(children), @@ -1806,3 +1825,18 @@ fn flatten_empty_strong(children: Vec) -> Vec { } out } + +fn flatten_empty_underline(children: Vec) -> Vec { + let mut out = Vec::with_capacity(children.len()); + for child in children { + match child { + Inline::Underline { attrs, children } if attrs.is_empty() => { + for grandchild in children { + push_coalesced(&mut out, grandchild); + } + } + x => push_coalesced(&mut out, x), + } + } + out +} diff --git a/src/lib.rs b/src/lib.rs index 6bb3fb1..6f02fec 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -36,6 +36,7 @@ pub struct Options { pub math: MathMode, pub tagfilter: bool, pub balance: bool, + pub underline: bool, pub max_inline_depth: usize, pub max_block_depth: usize, pub max_link_paren_depth: usize, @@ -47,6 +48,7 @@ impl Default for Options { math: MathMode::Brackets, tagfilter: false, balance: false, + underline: false, max_inline_depth: 64, max_block_depth: 128, max_link_paren_depth: 32, diff --git a/src/python.rs b/src/python.rs index 945291e..6149c2e 100644 --- a/src/python.rs +++ b/src/python.rs @@ -15,6 +15,7 @@ use crate::{MathMode, Options}; math = "brackets", tagfilter = false, balance = false, + underline = false, callbacks = None, max_inline_depth = None, max_block_depth = None, @@ -25,6 +26,7 @@ fn to_xhtml( math: &str, tagfilter: bool, balance: bool, + underline: bool, callbacks: Option>, max_inline_depth: Option, max_block_depth: Option, @@ -34,6 +36,7 @@ fn to_xhtml( math: parse_math_mode(math)?, tagfilter, balance, + underline, ..Options::default() }; if let Some(depth) = max_inline_depth { @@ -194,6 +197,7 @@ fn transform_inline(item: &mut Inline, callbacks: &Bound<'_, PyDict>) -> PyResul match item { Inline::Emph { children, .. } | Inline::Strong { children, .. } + | Inline::Underline { children, .. } | Inline::Strike { children, .. } | Inline::Highlight { children, .. } | Inline::Link { children, .. } @@ -276,6 +280,7 @@ fn inline_kind(item: &Inline) -> &'static str { Inline::HardBreak => "hard_break", Inline::Emph { .. } => "emph", Inline::Strong { .. } => "strong", + Inline::Underline { .. } => "underline", Inline::Strike { .. } => "strike", Inline::Superscript { .. } => "superscript", Inline::Subscript { .. } => "subscript", @@ -401,6 +406,7 @@ fn inline_node<'py>(py: Python<'py>, item: &Inline) -> PyResult {} Inline::Emph { attrs, children } | Inline::Strong { attrs, children } + | Inline::Underline { attrs, children } | Inline::Strike { attrs, children } | Inline::Highlight { attrs, children } | Inline::Span { attrs, children } => { diff --git a/src/render.rs b/src/render.rs index 721e15b..92a33db 100644 --- a/src/render.rs +++ b/src/render.rs @@ -332,6 +332,13 @@ impl<'a> Renderer<'a> { self.inlines(children, out); out.push_str("
"); } + Inline::Underline { attrs, children } => { + out.push_str("'); + self.inlines(children, out); + out.push_str(""); + } Inline::Strike { attrs, children } => { out.push_str(" String { Inline::SoftBreak | Inline::HardBreak => out.push(' '), Inline::Emph { children, .. } | Inline::Strong { children, .. } + | Inline::Underline { children, .. } | Inline::Strike { children, .. } | Inline::Highlight { children, .. } | Inline::Span { children, .. } diff --git a/tests/test_focused.py b/tests/test_focused.py index c0bca57..7fec1da 100644 --- a/tests/test_focused.py +++ b/tests/test_focused.py @@ -90,6 +90,22 @@ def test_balance_ignores_rawtext_and_voids(): assert "" not in html assert "
" in html +def test_underline_is_opt_in(): + assert to_xhtml("__x__") == "

x

\n" + assert to_xhtml("__x__", underline=True) == "

x

\n" + assert to_xhtml("**x** _y_ __z__", underline=True) == "

x y z

\n" + +def test_underline_follows_underscore_emphasis_rules(): + assert to_xhtml("___x___", underline=True) == "

x

\n" + assert to_xhtml("____x____", underline=True) == "

x

\n" + assert to_xhtml("intra__word__", underline=True) == "

intra__word__

\n" + assert to_xhtml("__a **b** c__", underline=True) == "

a b c

\n" + +def test_underline_callback(): + cb = lambda node, default: default.replace("", '') if node["type"] == "underline" else None + html = to_xhtml("__x__", underline=True, callbacks={"underline": cb}) + assert html == '

x

\n' + def test_long_nonascii_words_near_autolink_cap_do_not_error(): for boundary in ("(", "a: ", "x '"): for count in (126, 127, 128, 129, 130, 200):