From 8b2d4ec3e5473b327f0b8f3061d1c2a839c22468 Mon Sep 17 00:00:00 2001 From: Hadley Wickham Date: Sun, 5 Jul 2026 22:01:49 -0500 Subject: [PATCH 1/4] Scan verbatim tags directly instead of building a set per call tokenizeMd() built a std::set of the 49 verbatim tag names on every call (~13us), dominating its runtime on typical tag-sized inputs. A linear scan over the R vector is cheaper because each text contains only a handful of Rd tags. ~2.5x faster per call. --- src/tokenizeMd.cpp | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/src/tokenizeMd.cpp b/src/tokenizeMd.cpp index 9d6aa4dd..4e416edc 100644 --- a/src/tokenizeMd.cpp +++ b/src/tokenizeMd.cpp @@ -1,7 +1,6 @@ #include #include #include -#include #include #include @@ -87,14 +86,21 @@ static bool is_ascii_punct(char c) { (c >= '[' && c <= '`') || (c >= '{' && c <= '~'); } -[[cpp11::register]] -cpp11::writable::list tokenizeMd(std::string text, - cpp11::strings verbatim) { - std::set vtags; +// Linear scan beats building a lookup structure: texts contain few Rd +// tags, but tokenizeMd() is called for every tag of every block +static bool is_verbatim_tag(const cpp11::strings& verbatim, + const std::string& name) { for (R_xlen_t v = 0; v < verbatim.size(); v++) { - vtags.insert(std::string(verbatim[v])); + if (name == CHAR(STRING_ELT(verbatim, v))) { + return true; + } } + return false; +} +[[cpp11::register]] +cpp11::writable::list tokenizeMd(std::string text, + cpp11::strings verbatim) { static const std::string OPEN = "\xEE\x80\x80"; // U+E000 static const std::string CLOSE = "\xEE\x80\x81"; // U+E001 @@ -133,7 +139,7 @@ cpp11::writable::list tokenizeMd(std::string text, end = j - 1; std::string name = text.substr(i + 1, j - i - 1); - if (vtags.count(name)) { + if (is_verbatim_tag(verbatim, name)) { type = "verbatim"; // Consume all complete brace groups; an incomplete group is left // to markdown, like the brace groups of non-verbatim tags, and From c182be4b7b8465a7dfb924cce9ca3d5814bba9b6 Mon Sep 17 00:00:00 2001 From: Hadley Wickham Date: Sun, 5 Jul 2026 22:03:07 -0500 Subject: [PATCH 2/4] Only parse markdown in markdown_evaluate() when R code is plausible The old gate parsed the text with commonmark + xml2 whenever it contained any backtick or tilde, which is nearly every doc. The new gate looks for the shapes that can actually trigger evaluation (inline `r ...`, fenced blocks with braced info, code blocks whose first line starts with "r "), cutting ~90% of the wasted parses. --- R/markdown-code.R | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/R/markdown-code.R b/R/markdown-code.R index 86a026e2..c6b71078 100644 --- a/R/markdown-code.R +++ b/R/markdown-code.R @@ -49,10 +49,18 @@ #' @keywords internal markdown_evaluate <- function(text) { text <- paste(text, collapse = "\n") - # Code is delimited by backticks (`r ...`, ``` fences) or tilde fences - # (~~~{r}), so without either we can skip the (relatively expensive) - # markdown parsing altogether - if (!grepl("[`~]", text)) { + # Evaluation is triggered by an inline code span whose text starts with + # "r ", or a fenced block with braced chunk options. Only parse the + # (relatively expensive) markdown when the raw text shows one of those + # shapes; each pattern may over-match, which just means a wasted parse. + has_code <- + # `r ...` (commonmark strips one leading space from a code span) + grepl("`[ ]?r[ \n]", text) || + # ```{r ...} or ~~~{r ...} fences + grepl("(```+|~~~+)[^\n]*\\{", text) || + # a code block whose first line starts with "r ", e.g. indented blocks + grepl("(^|\n)[ \t]*r[ \t\n]", text) + if (!has_code) { return(text) } mdxml <- xml_ns_strip(md_to_mdxml(text, sourcepos = TRUE)) From 229643947d191ce23040f59facea1c5114ad9832 Mon Sep 17 00:00:00 2001 From: Hadley Wickham Date: Sun, 5 Jul 2026 22:11:35 -0500 Subject: [PATCH 3/4] Translate the markdown XML tree to Rd in C++ The R tree walk over xml2 nodes was the largest remaining cost of markdown processing (~24% of parse_package() on testthat). The walk now happens in C++ (src/mdxmlToRd.cpp), directly over the XML string that commonmark::markdown_xml() returns, so the xml2 parse and the per-node R dispatch disappear entirely. Token restoration happens inline during the walk, with the same final text-mode pass over the assembled Rd. Everything that needs package state stays in R and is supplied to the walk as a callback: link resolution (parse_link(), which now receives the link pieces precomputed instead of xml2 nodes), R code detection for \code vs \verb, and warnings for unsupported constructs. --- R/cpp11.R | 4 + R/markdown-link.R | 25 +- R/markdown-tokenize.R | 48 +- R/markdown.R | 397 ++------------ R/rd-include-rmd.R | 6 +- src/cpp11.cpp | 8 + src/mdxmlToRd.cpp | 692 ++++++++++++++++++++++++ tests/testthat/test-markdown-tokenize.R | 27 +- 8 files changed, 790 insertions(+), 417 deletions(-) create mode 100644 src/mdxmlToRd.cpp diff --git a/R/cpp11.R b/R/cpp11.R index 6adf9bf6..760915cd 100644 --- a/R/cpp11.R +++ b/R/cpp11.R @@ -16,6 +16,10 @@ leadingSpaces <- function(lines) { .Call(`_roxygen2_leadingSpaces`, lines) } +mdxmlToRd <- function(xml, tokens, types, has_sections, section_tag, restrict_images, resolve_link, is_r_code, warn) { + .Call(`_roxygen2_mdxmlToRd`, xml, tokens, types, has_sections, section_tag, restrict_images, resolve_link, is_r_code, warn) +} + tokenise_block <- function(lines, file, offset) { .Call(`_roxygen2_tokenise_block`, lines, file, offset) } diff --git a/R/markdown-link.R b/R/markdown-link.R index 7478e73f..096c4a5a 100644 --- a/R/markdown-link.R +++ b/R/markdown-link.R @@ -96,23 +96,17 @@ get_md_linkrefs <- function(text) { # Link parsing ----------------------------------------------------------------- -parse_link <- function(destination, contents, state) { - ## Not a [] or [][] type link, remove prefix if it is - if (!grepl("^R:", destination)) { - return(NULL) - } +# Called back from the C++ tree walk (src/mdxmlToRd.cpp) for every +# `[topic]` or `[text][topic]` style link. `text` is the concatenated +# plain text of the link contents, `rendered` is the contents already +# translated to Rd, and `is_code` signals that the contents was a single +# code span (whose \code becomes the outermost layer). +parse_link <- function(destination, text, has_nontext, is_code, rendered, state) { destination <- sub("^R:", "", URLdecode(destination)) Encoding(destination) <- "UTF-8" # restore encoding dropped URLdecodse - ## if contents is a `code tag`, then we need to move this outside - is_code <- FALSE - if (length(contents) == 1 && xml_name(contents) == "code") { - is_code <- TRUE - - contents <- xml_contents(contents) + if (is_code) { destination <- sub("`$", "", sub("^`", "", destination)) - - local_bindings(.env = state, in_link_code = TRUE) } ## If the supplied link text is the same as the reference text, @@ -120,8 +114,7 @@ parse_link <- function(destination, contents, state) { ## it was not specified explicitly. In this case `()` links are ## turned to `\\code{}`. ## We also assume link text if we see a non-text XML tag in contents. - has_link_text <- paste(xml_text(contents), collapse = "") != destination || - any(xml_name(contents) != "text") + has_link_text <- text != destination || has_nontext ## if (is_code) then we'll need \\code ## `pkg` is package or NA @@ -158,7 +151,7 @@ parse_link <- function(destination, contents, state) { } text <- escape(text) } else { - text <- mdxml_link_text(contents, state) + text <- rendered } rd_link(pkg, escape(topic), text, code = is_code) } diff --git a/R/markdown-tokenize.R b/R/markdown-tokenize.R index 83ba1abb..f3803691 100644 --- a/R/markdown-tokenize.R +++ b/R/markdown-tokenize.R @@ -3,21 +3,10 @@ # `md_tokenize()` replaces every backslash-initiated construct with an # inert placeholder, "\uE000\uE001", so that commonmark can parse # the remaining text as pure markdown, with no escaping. See -# src/tokenizeMd.cpp for the grammar. `restore_tokens()` substitutes the -# original constructs back while the parsed markdown is translated to Rd; -# the `mode` argument selects how a token is rendered in the Rd context -# it lands in: -# -# * `text`: regular Rd text. Verbatim tags come back as live Rd; the -# escaped bracket escapes `\\[` and `\\]` drop one backslash (matching -# what the markdown escape `\[` does to a bare bracket); everything -# else comes back as typed. -# * `verb`: inside `\verb{}`, `\code{}` or `\preformatted{}`. Everything -# renders literally, so token text is Rd-escaped -- except verbatim Rd -# tags, which are inserted as typed: the Rd parser keeps unknown macros -# in verbatim contexts as literal text, and this is how the escaping -# code always behaved. -# * `raw`: unprocessed output, e.g. the body of a generated `\Sexpr{}`. +# src/tokenizeMd.cpp for the grammar. The C++ tree walk +# (src/mdxmlToRd.cpp) substitutes the original constructs back while the +# parsed markdown is translated to Rd, rendering each token according to +# the Rd context it lands in. verbatim_rd_tags <- c( "acronym", @@ -89,32 +78,3 @@ md_tokenize <- function(text, tag = NULL) { out } -restore_tokens <- function(x, state, mode = c("text", "verb", "raw")) { - if (length(state$tokens) == 0 || !grepl("\uE000", x, fixed = TRUE)) { - return(x) - } - mode <- match.arg(mode) - - m <- gregexpr("\uE000[0-9]+\uE001", x, perl = TRUE) - regmatches(x, m) <- lapply(regmatches(x, m), function(placeholder) { - idx <- as.integer(gsub("[\uE000\uE001]", "", placeholder)) - src <- state$tokens[idx] - type <- state$types[idx] - - switch( - mode, - raw = src, - text = ifelse(src %in% c("\\\\[", "\\\\]"), substring(src, 2), src), - verb = ifelse(type == "verbatim", src, escape_rd_verb(src)) - ) - }) - x -} - -escape_rd_verb <- function(x) { - x <- gsub("\\", "\\\\", x, fixed = TRUE) - x <- gsub("%", "\\%", x, fixed = TRUE) - x <- gsub("{", "\\{", x, fixed = TRUE) - x <- gsub("}", "\\}", x, fixed = TRUE) - x -} diff --git a/R/markdown.R b/R/markdown.R index 16767442..bf4ef6e3 100644 --- a/R/markdown.R +++ b/R/markdown.R @@ -19,26 +19,18 @@ markdown <- function(text, tag = NULL, sections = FALSE) { markdown_pass2 <- function(tokens, tag = NULL, sections = FALSE) { text_linkrefs <- add_linkrefs_to_md(tokens$text) - mdxml <- md_to_mdxml(text_linkrefs) + xml <- commonmark::markdown_xml( + text_linkrefs, + hardbreaks = TRUE, + extensions = "table" + ) + state <- new.env(parent = emptyenv()) state$tag <- tag state$has_sections <- sections state$tokens <- tokens$tokens state$types <- tokens$types - rd <- mdxml_children_to_rd_top(mdxml, state) - - # Restore the tokens that landed in regular text; verbatim and raw - # contexts have already restored theirs during the tree walk - rd[] <- map_chr(rd, restore_tokens, state = state, mode = "text") - if (!is.null(names(rd))) { - names(rd) <- map_chr( - names(rd), - restore_tokens, - state = state, - mode = "text" - ) - } - rd + mdxml_children_to_rd_top(xml, state) } md_to_mdxml <- function(x, ...) { @@ -51,14 +43,29 @@ md_to_mdxml <- function(x, ...) { xml2::read_xml(md) } +# The XML -> Rd tree walk happens in C++ (src/mdxmlToRd.cpp); everything +# that needs package state -- link resolution, R code detection, and +# warnings -- is supplied as a callback mdxml_children_to_rd_top <- function(xml, state) { state$section_tag <- uuid() - out <- map_chr(xml_children(xml), mdxml_node_to_rd, state) - out <- c(out, mdxml_close_sections(state)) - rd <- trimws(paste0(out, collapse = "")) + result <- mdxmlToRd( + xml, + tokens = state$tokens %||% character(), + types = state$types %||% character(), + has_sections = isTRUE(state$has_sections), + section_tag = state$section_tag, + restrict_images = roxy_meta_get("restrict_image_formats") %||% TRUE, + resolve_link = function(destination, text, has_nontext, is_code, rendered) { + parse_link(destination, text, has_nontext, is_code, rendered, state) + }, + is_r_code = function(code) can_parse(code) || code %in% special, + warn = function(kind, detail) mdxml_warn(kind, detail, state) + ) + + rd <- result$rd if (state$has_sections) { secs <- strsplit(rd, state$section_tag, fixed = TRUE)[[1]] %||% "" - titles <- c("", state$titles) + titles <- c("", result$titles) # strsplit drops trailing empty strings, so pad to match titles length secs <- c(secs, rep("", length(titles) - length(secs))) rd <- structure(trimws(secs), names = titles) @@ -66,113 +73,49 @@ mdxml_children_to_rd_top <- function(xml, state) { rd } -mdxml_children_to_rd <- function(xml, state) { - out <- map_chr(xml_children(xml), mdxml_node_to_rd, state) - paste0(out, collapse = "") -} - -mdxml_node_to_rd <- function(xml, state) { - if ( - !inherits(xml, "xml_node") || - !xml_type(xml) %in% c("text", "element") - ) { +mdxml_warn <- function(kind, detail, state) { + if (kind == "unsupported") { warn_roxy_tag( state$tag, c( "markdown translation failed", - x = "Unexpected internal error", - i = "Please file an issue at https://github.com/r-lib/roxygen2/issues" + x = "{detail} are not currently supported" ) ) - return("") - } - - switch( - xml_name(xml), - html = , - document = , - unknown = mdxml_children_to_rd(xml, state), - - paragraph = paste0("\n\n", mdxml_children_to_rd(xml, state)), - text = if (is_true(state$in_link_code)) { - restore_tokens(escape_verb(xml_text(xml)), state, "verb") + } else if (kind == "heading") { + if (is.null(state$tag)) { + tag_name <- "this tag" } else { - escape_comment(xml_text(xml)) - }, - emph = paste0("\\emph{", mdxml_children_to_rd(xml, state), "}"), - strong = paste0("\\strong{", mdxml_children_to_rd(xml, state), "}"), - softbreak = mdxml_break(state), - linebreak = mdxml_break(state), - - code = mdxml_code(xml, state), - code_block = mdxml_code_block(xml, state), - - table = mdxml_table(xml, state), - list = mdxml_list(xml, state), - item = mdxml_item(xml, state), - link = mdxml_link(xml, state), - image = mdxml_image(xml), - heading = mdxml_heading(xml, state), - - # Only supported when including Rmds - html_block = mdxml_html_block(xml, state), - html_inline = mdxml_html_inline(xml, state), - - # Not supported - block_quote = mdxml_unsupported(xml, state$tag, "block quotes"), - thematic_break = mdxml_unsupported(xml, state$tag, "horizontal rules"), - mdxml_unknown(xml, state$tag) - ) -} - -mdxml_unknown <- function(xml, tag) { - warn_roxy_tag( - tag, - c( - "markdown translation failed", - x = "Internal error: unknown xml node {xml_name(xml)}", - i = "Please file an issue at https://github.com/r-lib/roxygen2/issues" + tag_name <- paste0("@", state$tag$tag) + } + warn_roxy_tag( + state$tag, + c( + "markdown translation failed", + x = "Level 1 headings are not supported in {tag_name}", + i = "Do you want to put the heading in @description or @details?" + ) ) - ) - escape_comment(xml_text(xml)) -} -mdxml_unsupported <- function(xml, tag, feature) { - warn_roxy_tag( - tag, - c( - "markdown translation failed", - x = "{feature} are not currently supported" + } else { + warn_roxy_tag( + state$tag, + c( + "markdown translation failed", + x = "Internal error: unknown xml node {detail}", + i = "Please file an issue at https://github.com/r-lib/roxygen2/issues" + ) ) - ) - escape_comment(xml_text(xml)) -} - -mdxml_break <- function(state) { - if (isTRUE(state$inlink)) " " else "\n" + } } -mdxml_code <- function(xml, state) { - code <- xml_text(xml) - # Decide what the code is based on its original source - raw_code <- restore_tokens(code, state, "raw") - - if (grepl("^Rd ", raw_code)) { - paste0( - "\\Sexpr[stage=render,results=rd]{", - substr(raw_code, 4, nchar(raw_code)), - "}" - ) - } else if (can_parse(raw_code) || raw_code %in% special) { - # See escaping details at - # https://cran.rstudio.com/doc/manuals/r-devel/R-exts.html#Insertions - paste0( - "\\code{", - restore_tokens(gsub("%", "\\\\%", code), state, "verb"), - "}" - ) - } else { - paste0("\\verb{", restore_tokens(escape_verb(code), state, "verb"), "}") - } +can_parse <- function(x) { + tryCatch( + { + parse_expr(x) + TRUE + }, + error = function(x) FALSE + ) } special <- c( @@ -214,227 +157,3 @@ special <- c( "repeat", "while" ) - -mdxml_code_block <- function(xml, state) { - info <- xml_attr(xml, "info", default = "")[1] - if (nchar(info[1]) == 0) { - info <- NA_character_ - } - paste0( - "\n\n", - "\\if{html}{\\out{
}}", - "\\preformatted{", - restore_tokens(escape_verb(xml_text(xml)), state, "verb"), - "}", - "\\if{html}{\\out{
}}" - ) -} - -can_parse <- function(x) { - tryCatch( - { - parse_expr(x) - TRUE - }, - error = function(x) FALSE - ) -} - -escape_verb <- function(x) { - # No need to escape backslashes: they are all tokenized, and - # restore_tokens() escapes them as it puts them back - x <- gsub("%", "\\%", x, fixed = TRUE) - x <- gsub("{", "\\{", x, fixed = TRUE) - x <- gsub("}", "\\}", x, fixed = TRUE) - x -} - -mdxml_table <- function(xml, state) { - head <- xml_children(xml)[[1]] - align <- substr(xml_attr(xml_children(head), "align", default = "left"), 1, 1) - - rows <- xml_find_all(xml, "d1:table_row|d1:table_header") - cells <- map(rows, xml_find_all, "d1:table_cell") - - cells_rd <- map(cells, \(x) map(x, mdxml_children_to_rd, state = state)) - rows_rd <- map_chr(cells_rd, paste0, collapse = " \\tab ") - - paste0( - "\\tabular{", - paste(align, collapse = ""), - "}{\n", - paste(" ", rows_rd, "\\cr\n", collapse = ""), - "}\n" - ) -} - -# A list, either bulleted or numbered -mdxml_list <- function(xml, state) { - type <- xml_attr(xml, "type") - if (type == "ordered") { - paste0("\n\\enumerate{", mdxml_children_to_rd(xml, state), "\n}") - } else { - paste0("\n\\itemize{", mdxml_children_to_rd(xml, state), "\n}") - } -} - -mdxml_item <- function(xml, state) { - ## A single item within a list. We remove the first paragraph - ## tag, to avoid an empty line at the beginning of the first item. - children <- xml_children(xml) - if (length(children) == 0) { - cnts <- "" - } else if (xml_name(children[[1]]) == "paragraph") { - cnts <- paste0( - mdxml_children_to_rd(children[[1]], state), - paste0(map_chr(children[-1], mdxml_node_to_rd, state), collapse = "") - ) - } else { - cnts <- mdxml_children_to_rd(xml, state) - } - paste0("\n\\item ", cnts) -} - -mdxml_link <- function(xml, state) { - ## Hyperlink, this can also be a link to a function - dest <- xml_attr(xml, "destination") - contents <- xml_contents(xml) - - link <- parse_link(dest, contents, state) - - if (!is.null(link)) { - link - } else if (dest == "" || dest == xml_text(xml)) { - paste0("\\url{", escape_comment(xml_text(xml)), "}") - } else { - paste_c( - c("\\href{", escape_comment(dest), "}"), - c("{", mdxml_link_text(contents, state), "}") - ) - } -} - -mdxml_link_text <- function(xml_contents, state) { - # Newlines in markdown get converted to softbreaks/linebreaks by - # markdown_xml(), which then get interpreted as empty strings by - # xml_text(). So we preserve newlines as spaces. - inlink <- state$inlink - on.exit(state$inlink <- inlink, add = TRUE) - state$inlink <- TRUE - - text <- map_chr(xml_contents, mdxml_node_to_rd, state) - paste0(text, collapse = "") -} - -mdxml_image <- function(xml) { - dest <- xml_attr(xml, "destination") - title <- xml_attr(xml, "title") - fmt <- get_image_format(dest) - paste0( - if (fmt == "html") "\\if{html}{", - if (fmt == "pdf") "\\if{pdf}{", - "\\figure{", - dest, - "}", - if (nchar(title)) paste0("{", title, "}"), - if (fmt %in% c("html", "pdf")) "}" - ) -} - -get_image_format <- function(path) { - should_restrict <- roxy_meta_get("restrict_image_formats") %||% TRUE - if (!should_restrict) { - return("all") - } - - path <- tolower(path) - rx <- default_image_formats() - html <- grepl(rx$html, path) - pdf <- grepl(rx$pdf, path) - if (html && pdf) { - "all" - } else if (html) { - "html" - } else if (pdf) { - "pdf" - } else { - "all" - } -} - -default_image_formats <- function() { - list( - html = "[.](jpg|jpeg|gif|png|svg)$", - pdf = "[.](jpg|jpeg|gif|png|pdf)$" - ) -} - -escape_comment <- function(x) { - gsub("%", "\\%", x, fixed = TRUE) -} - -mdxml_heading <- function(xml, state) { - level <- xml_attr(xml, "level") - if (!state$has_sections && level == 1) { - if (is.null(state$tag)) { - tag_name <- "this tag" - } else { - tag_name <- paste0("@", state$tag$tag) - } - warn_roxy_tag( - state$tag, - c( - "markdown translation failed", - x = "Level 1 headings are not supported in {tag_name}", - i = "Do you want to put the heading in @description or @details?" - ) - ) - return(escape_comment(xml_text(xml))) - } - - txt <- map_chr(xml_contents(xml), mdxml_node_to_rd, state) - if (level == 1) { - state$titles <- c(state$titles, paste(txt, collapse = "")) - } - head <- paste0( - mdxml_close_sections(state, level), - "\n", - if (level == 1) state$section_tag else "\\subsection{", - if (level > 1) paste(txt, collapse = ""), - if (level > 1) "}{" - ) - state$section <- c(state$section, level) - head -} - -mdxml_html_block <- function(xml, state) { - txt <- xml_text(xml) - txt <- gsub("}", "\\}", txt, fixed = TRUE) - txt <- gsub("{", "\\{", txt, fixed = TRUE) - paste0( - "\\if{html}{\\out{\n", - txt, - "}}\n" - ) -} - -mdxml_html_inline <- function(xml, state) { - paste0( - "\\if{html}{\\out{", - gsub("}", "\\}", xml_text(xml), fixed = TRUE), - "}}" - ) -} - -mdxml_close_sections <- function(state, upto = 1L) { - hmy <- 0L - upto <- max(upto, 2L) - while (length(state$section) && tail(state$section, 1) >= upto) { - hmy <- hmy + 1L - state$section <- head(state$section, -1L) - } - - paste0(rep("\n}\n", hmy), collapse = "") -} diff --git a/R/rd-include-rmd.R b/R/rd-include-rmd.R index e472cb18..b1c91a04 100644 --- a/R/rd-include-rmd.R +++ b/R/rd-include-rmd.R @@ -100,7 +100,11 @@ rmd_linkrefs_from_file <- function(path) { rmd_eval_rd <- function(path, tag) { mdtxt <- paste(read_lines(path), collapse = "\n") mdesc <- add_linkrefs_to_md(mdtxt) - mdxml <- md_to_mdxml(mdesc) + mdxml <- commonmark::markdown_xml( + mdesc, + hardbreaks = TRUE, + extensions = "table" + ) state <- new.env(parent = emptyenv()) state$tag <- tag state$has_sections <- TRUE diff --git a/src/cpp11.cpp b/src/cpp11.cpp index 17701453..cee54ffe 100644 --- a/src/cpp11.cpp +++ b/src/cpp11.cpp @@ -33,6 +33,13 @@ extern "C" SEXP _roxygen2_leadingSpaces(SEXP lines) { return cpp11::as_sexp(leadingSpaces(cpp11::as_cpp>(lines))); END_CPP11 } +// mdxmlToRd.cpp +cpp11::writable::list mdxmlToRd(std::string xml, cpp11::strings tokens, cpp11::strings types, bool has_sections, std::string section_tag, bool restrict_images, cpp11::function resolve_link, cpp11::function is_r_code, cpp11::function warn); +extern "C" SEXP _roxygen2_mdxmlToRd(SEXP xml, SEXP tokens, SEXP types, SEXP has_sections, SEXP section_tag, SEXP restrict_images, SEXP resolve_link, SEXP is_r_code, SEXP warn) { + BEGIN_CPP11 + return cpp11::as_sexp(mdxmlToRd(cpp11::as_cpp>(xml), cpp11::as_cpp>(tokens), cpp11::as_cpp>(types), cpp11::as_cpp>(has_sections), cpp11::as_cpp>(section_tag), cpp11::as_cpp>(restrict_images), cpp11::as_cpp>(resolve_link), cpp11::as_cpp>(is_r_code), cpp11::as_cpp>(warn))); + END_CPP11 +} // parser2.cpp cpp11::list tokenise_block(cpp11::strings lines, std::string file, int offset); extern "C" SEXP _roxygen2_tokenise_block(SEXP lines, SEXP file, SEXP offset) { @@ -68,6 +75,7 @@ static const R_CallMethodDef CallEntries[] = { {"_roxygen2_findEndOfTag", (DL_FUNC) &_roxygen2_findEndOfTag, 3}, {"_roxygen2_find_includes", (DL_FUNC) &_roxygen2_find_includes, 1}, {"_roxygen2_leadingSpaces", (DL_FUNC) &_roxygen2_leadingSpaces, 1}, + {"_roxygen2_mdxmlToRd", (DL_FUNC) &_roxygen2_mdxmlToRd, 9}, {"_roxygen2_rdComplete", (DL_FUNC) &_roxygen2_rdComplete, 2}, {"_roxygen2_tokenise_block", (DL_FUNC) &_roxygen2_tokenise_block, 3}, {"_roxygen2_tokenizeMd", (DL_FUNC) &_roxygen2_tokenizeMd, 2}, diff --git a/src/mdxmlToRd.cpp b/src/mdxmlToRd.cpp new file mode 100644 index 00000000..fa830bea --- /dev/null +++ b/src/mdxmlToRd.cpp @@ -0,0 +1,692 @@ +#include +#include +#include +#include +#include +#include + +using namespace cpp11::literals; + +// Render the commonmark XML representation of a roxygen comment as Rd. +// +// This is a C++ port of the old R tree walk over xml2 nodes; it exists +// purely for speed and must produce byte-identical output. The input is +// the string returned by commonmark::markdown_xml(), which uses a small, +// regular subset of XML: elements, double-quoted attributes, and +// character data escaped with the four entities < > & " +// (other characters are raw UTF-8). All text content lives inside leaf +// elements marked xml:space="preserve"; whitespace between structural +// tags is pretty-printing and is dropped, like xml2's NOBLANKS. +// +// Backslash constructs were replaced by "" placeholders (see +// tokenizeMd.cpp) before the markdown parse, so the walk restores each +// token as it lands in its final Rd context: +// +// * text: regular Rd text. Verbatim tags come back as live Rd; the +// escaped bracket escapes "\\[" and "\\]" drop one backslash (matching +// what the markdown escape "\[" does to a bare bracket); everything +// else comes back as typed. +// * verb: inside \verb{}, \code{} or \preformatted{}. Everything +// renders literally, so token text is Rd-escaped -- except verbatim Rd +// tags, which are inserted as typed: the Rd parser keeps unknown +// macros in verbatim contexts as literal text. +// * raw: unprocessed output, e.g. the body of a generated \Sexpr{}. +// +// Link resolution, R code detection, and warnings need package state and +// so stay in R; they are supplied as callbacks. + +static const std::string OPEN = "\xEE\x80\x80"; // U+E000 +static const std::string CLOSE = "\xEE\x80\x81"; // U+E001 + +// XML parsing ---------------------------------------------------------- + +struct MdNode { + std::string name; + std::string text; // direct character data, entities decoded + std::vector> attrs; + std::vector children; // element children only + + const std::string* attr(const char* name) const { + for (size_t i = 0; i < attrs.size(); i++) { + if (attrs[i].first == name) { + return &attrs[i].second; + } + } + return NULL; + } +}; + +// Append the UTF-8 encoding of a code point +static void append_utf8(std::string& out, unsigned long cp) { + if (cp <= 0x7F) { + out += (char)cp; + } else if (cp <= 0x7FF) { + out += (char)(0xC0 | (cp >> 6)); + out += (char)(0x80 | (cp & 0x3F)); + } else if (cp <= 0xFFFF) { + out += (char)(0xE0 | (cp >> 12)); + out += (char)(0x80 | ((cp >> 6) & 0x3F)); + out += (char)(0x80 | (cp & 0x3F)); + } else { + out += (char)(0xF0 | (cp >> 18)); + out += (char)(0x80 | ((cp >> 12) & 0x3F)); + out += (char)(0x80 | ((cp >> 6) & 0x3F)); + out += (char)(0x80 | (cp & 0x3F)); + } +} + +// Decode s[start, end) into out, replacing XML entities +static void decode_chardata(const std::string& s, size_t start, size_t end, + std::string& out) { + for (size_t i = start; i < end; i++) { + if (s[i] != '&') { + out += s[i]; + continue; + } + size_t semi = s.find(';', i + 1); + if (semi == std::string::npos || semi > i + 10) { + out += '&'; + continue; + } + std::string ent = s.substr(i + 1, semi - i - 1); + if (ent == "lt") { + out += '<'; + } else if (ent == "gt") { + out += '>'; + } else if (ent == "amp") { + out += '&'; + } else if (ent == "quot") { + out += '"'; + } else if (ent == "apos") { + out += '\''; + } else if (ent.size() > 1 && ent[0] == '#') { + unsigned long cp = (ent[1] == 'x' || ent[1] == 'X') + ? strtoul(ent.c_str() + 2, NULL, 16) + : strtoul(ent.c_str() + 1, NULL, 10); + append_utf8(out, cp); + } else { + out += '&'; + continue; + } + i = semi; + } +} + +static bool is_blank(const std::string& s) { + for (size_t i = 0; i < s.size(); i++) { + char c = s[i]; + if (c != ' ' && c != '\t' && c != '\r' && c != '\n') { + return false; + } + } + return true; +} + +static bool is_name_char(char c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || + (c >= '0' && c <= '9') || c == '_' || c == '-' || c == ':'; +} + +// Parse commonmark's XML output into nodes; returns the root index or -1 +static int parse_xml(const std::string& s, std::vector& nodes) { + size_t i = 0, n = s.size(); + int root = -1; + std::vector stack; + + while (i < n) { + if (s[i] != '<') { + size_t start = i; + while (i < n && s[i] != '<') { + i++; + } + if (!stack.empty()) { + std::string text; + decode_chardata(s, start, i, text); + MdNode& top = nodes[stack.back()]; + if (!is_blank(text) || top.attr("xml:space") != NULL) { + top.text += text; + } + } + continue; + } + + if (i + 1 < n && (s[i + 1] == '?' || s[i + 1] == '!')) { + // or + while (i < n && s[i] != '>') { + i++; + } + i++; + } else if (i + 1 < n && s[i + 1] == '/') { + while (i < n && s[i] != '>') { + i++; + } + i++; + if (!stack.empty()) { + stack.pop_back(); + } + } else { + i++; + MdNode node; + while (i < n && is_name_char(s[i])) { + node.name += s[i++]; + } + bool self_closing = false; + while (i < n) { + while (i < n && (s[i] == ' ' || s[i] == '\t' || s[i] == '\r' || + s[i] == '\n')) { + i++; + } + if (i < n && s[i] == '/') { + self_closing = true; + i++; + continue; + } + if (i >= n || s[i] == '>') { + i++; + break; + } + std::string name; + while (i < n && is_name_char(s[i]) && s[i] != '=') { + name += s[i++]; + } + std::string value; + if (i < n && s[i] == '=') { + i++; + char quote = s[i]; + size_t vstart = ++i; + while (i < n && s[i] != quote) { + i++; + } + decode_chardata(s, vstart, i, value); + i++; + } + node.attrs.push_back(std::make_pair(name, value)); + } + + int id = nodes.size(); + nodes.push_back(node); + if (!stack.empty()) { + nodes[stack.back()].children.push_back(id); + } else if (root < 0) { + root = id; + } + if (!self_closing) { + stack.push_back(id); + } + } + } + + return root; +} + +// Rd rendering --------------------------------------------------------- + +// How the characters around placeholders are escaped, and how restored +// tokens are rendered, in each Rd context (see tokenizeMd.cpp) +enum class Esc { None, Comment, Verb, Braces }; +enum class Mode { Text, Verb, Raw }; + +class Walker { +public: + Walker(const std::vector& nodes, cpp11::strings tokens, + cpp11::strings types, bool has_sections, std::string section_tag, + bool restrict_images, cpp11::function resolve_link, + cpp11::function is_r_code, cpp11::function warn) + : nodes_(nodes), + has_sections_(has_sections), + section_tag_(section_tag), + restrict_images_(restrict_images), + resolve_link_(resolve_link), + is_r_code_(is_r_code), + warn_(warn) { + for (R_xlen_t t = 0; t < tokens.size(); t++) { + tokens_.push_back(std::string(tokens[t])); + verbatim_.push_back(std::string(types[t]) == "verbatim"); + } + } + + std::string walk(int root) { + std::string out = children_to_rd(root); + out += close_sections(1); + // Tokens can also enter the output through the resolved links, e.g. + // the destination of [\code{x}] contains a placeholder, so finish + // with a text-mode restore over the assembled Rd + out = restore_text(out); + // trimws() + size_t from = out.find_first_not_of(" \t\r\n"); + size_t to = out.find_last_not_of(" \t\r\n"); + if (from == std::string::npos) { + return ""; + } + return out.substr(from, to - from + 1); + } + + std::string restore_text(const std::string& s) { + return emit(s, Esc::None, Mode::Text); + } + + std::vector titles; + +private: + const std::vector& nodes_; + std::vector tokens_; + std::vector verbatim_; + bool has_sections_; + std::string section_tag_; + bool restrict_images_; + cpp11::function resolve_link_; + cpp11::function is_r_code_; + cpp11::function warn_; + + bool inlink_ = false; + bool in_link_code_ = false; + std::vector sections_; + + std::string token_rd(int idx, Mode mode) { + if (idx < 1 || idx > (int)tokens_.size()) { + // Can't happen: placeholders are generated alongside the tokens + return OPEN + std::to_string(idx) + CLOSE; + } + const std::string& src = tokens_[idx - 1]; + switch (mode) { + case Mode::Raw: + return src; + case Mode::Text: + // \\[ renders as \[, matching the markdown escape \[ + if (src == "\\\\[" || src == "\\\\]") { + return src.substr(1); + } + return src; + case Mode::Verb: + default: + if (verbatim_[idx - 1]) { + return src; + } + std::string out; + for (size_t i = 0; i < src.size(); i++) { + switch (src[i]) { + case '\\': + out += "\\\\"; + break; + case '%': + out += "\\%"; + break; + case '{': + out += "\\{"; + break; + case '}': + out += "\\}"; + break; + default: + out += src[i]; + } + } + return out; + } + } + + // Escape s for an Rd context and restore the tokens that land in it + std::string emit(const std::string& s, Esc esc, Mode mode) { + std::string out; + out.reserve(s.size()); + size_t n = s.size(); + + for (size_t i = 0; i < n; i++) { + char c = s[i]; + if ((unsigned char)c == 0xEE && i + 2 < n && + (unsigned char)s[i + 1] == 0x80 && (unsigned char)s[i + 2] == 0x80) { + size_t j = i + 3; + int idx = 0; + while (j < n && s[j] >= '0' && s[j] <= '9') { + idx = idx * 10 + (s[j] - '0'); + j++; + } + if (j > i + 3 && j + 2 < n && (unsigned char)s[j] == 0xEE && + (unsigned char)s[j + 1] == 0x80 && + (unsigned char)s[j + 2] == 0x81) { + out += token_rd(idx, mode); + i = j + 2; + continue; + } + } + switch (c) { + case '%': + out += (esc == Esc::Comment || esc == Esc::Verb) ? "\\%" : "%"; + break; + case '{': + out += (esc == Esc::Verb || esc == Esc::Braces) ? "\\{" : "{"; + break; + case '}': + out += (esc == Esc::Verb || esc == Esc::Braces) ? "\\}" : "}"; + break; + default: + out += c; + } + } + return out; + } + + // All descendant character data, like xml2::xml_text() + void text_of(int id, std::string& out) { + const MdNode& nd = nodes_[id]; + out += nd.text; + for (size_t i = 0; i < nd.children.size(); i++) { + text_of(nd.children[i], out); + } + } + + std::string text_of(int id) { + std::string out; + text_of(id, out); + return out; + } + + std::string children_to_rd(int id) { + const MdNode& nd = nodes_[id]; + std::string out; + for (size_t i = 0; i < nd.children.size(); i++) { + out += node_to_rd(nd.children[i]); + } + return out; + } + + std::string children_to_rd_inlink(int id) { + bool old = inlink_; + inlink_ = true; + std::string out = children_to_rd(id); + inlink_ = old; + return out; + } + + std::string node_to_rd(int id) { + const MdNode& nd = nodes_[id]; + const std::string& name = nd.name; + + if (name == "html" || name == "document" || name == "unknown") { + return children_to_rd(id); + } else if (name == "paragraph") { + return "\n\n" + children_to_rd(id); + } else if (name == "text") { + return in_link_code_ ? emit(nd.text, Esc::Verb, Mode::Verb) + : emit(nd.text, Esc::Comment, Mode::Text); + } else if (name == "emph") { + return "\\emph{" + children_to_rd(id) + "}"; + } else if (name == "strong") { + return "\\strong{" + children_to_rd(id) + "}"; + } else if (name == "softbreak" || name == "linebreak") { + return inlink_ ? " " : "\n"; + } else if (name == "code") { + return code_to_rd(nd); + } else if (name == "code_block") { + return code_block_to_rd(nd); + } else if (name == "table") { + return table_to_rd(id); + } else if (name == "list") { + return list_to_rd(id); + } else if (name == "item") { + return item_to_rd(id); + } else if (name == "link") { + return link_to_rd(id); + } else if (name == "image") { + return image_to_rd(nd); + } else if (name == "heading") { + return heading_to_rd(id); + } else if (name == "html_block") { + return "\\if{html}{\\out{\n" + emit(nd.text, Esc::Braces, Mode::Text) + + "}}\n"; + } else if (name == "html_inline") { + return "\\if{html}{\\out{" + emit(nd.text, Esc::Braces, Mode::Text) + + "}}"; + } else if (name == "block_quote") { + warn_("unsupported", "block quotes"); + return emit(text_of(id), Esc::Comment, Mode::Text); + } else if (name == "thematic_break") { + warn_("unsupported", "horizontal rules"); + return emit(text_of(id), Esc::Comment, Mode::Text); + } else { + warn_("unknown", name); + return emit(text_of(id), Esc::Comment, Mode::Text); + } + } + + std::string code_to_rd(const MdNode& nd) { + // Decide what the code is based on its original source + std::string raw_code = emit(nd.text, Esc::None, Mode::Raw); + + if (raw_code.compare(0, 3, "Rd ") == 0) { + return "\\Sexpr[stage=render,results=rd]{" + raw_code.substr(3) + "}"; + } else if (cpp11::as_cpp(is_r_code_(raw_code))) { + // See escaping details at + // https://cran.rstudio.com/doc/manuals/r-devel/R-exts.html#Insertions + return "\\code{" + emit(nd.text, Esc::Comment, Mode::Verb) + "}"; + } else { + return "\\verb{" + emit(nd.text, Esc::Verb, Mode::Verb) + "}"; + } + } + + std::string code_block_to_rd(const MdNode& nd) { + const std::string* info = nd.attr("info"); + std::string out = "\n\n\\if{html}{\\out{
empty()) { + out += " " + *info; + } + out += "\">}}\\preformatted{"; + out += emit(nd.text, Esc::Verb, Mode::Verb); + out += "}\\if{html}{\\out{
}}"; + return out; + } + + std::string table_to_rd(int id) { + const MdNode& nd = nodes_[id]; + if (nd.children.empty()) { + return ""; + } + + std::string align; + const MdNode& head = nodes_[nd.children[0]]; + for (size_t i = 0; i < head.children.size(); i++) { + const std::string* a = nodes_[head.children[i]].attr("align"); + align += (a != NULL && !a->empty()) ? (*a)[0] : 'l'; + } + + std::string body; + for (size_t i = 0; i < nd.children.size(); i++) { + const MdNode& row = nodes_[nd.children[i]]; + if (row.name != "table_row" && row.name != "table_header") { + continue; + } + std::string cells; + for (size_t j = 0; j < row.children.size(); j++) { + if (nodes_[row.children[j]].name != "table_cell") { + continue; + } + if (!cells.empty()) { + cells += " \\tab "; + } + cells += children_to_rd(row.children[j]); + } + body += " " + cells + " \\cr\n"; + } + + return "\\tabular{" + align + "}{\n" + body + "}\n"; + } + + std::string list_to_rd(int id) { + const MdNode& nd = nodes_[id]; + const std::string* type = nd.attr("type"); + if (type != NULL && *type == "ordered") { + return "\n\\enumerate{" + children_to_rd(id) + "\n}"; + } else { + return "\n\\itemize{" + children_to_rd(id) + "\n}"; + } + } + + std::string item_to_rd(int id) { + // Remove the first paragraph tag, to avoid an empty line at the + // beginning of the first item + const MdNode& nd = nodes_[id]; + std::string cnts; + if (!nd.children.empty() && + nodes_[nd.children[0]].name == "paragraph") { + cnts = children_to_rd(nd.children[0]); + for (size_t i = 1; i < nd.children.size(); i++) { + cnts += node_to_rd(nd.children[i]); + } + } else { + cnts = children_to_rd(id); + } + return "\n\\item " + cnts; + } + + std::string link_to_rd(int id) { + const MdNode& nd = nodes_[id]; + const std::string* d = nd.attr("destination"); + std::string dest = d != NULL ? *d : ""; + + if (dest.compare(0, 2, "R:") == 0) { + // A [topic] or [text][topic] link; resolution needs package state, + // so it happens in R + bool is_code = + nd.children.size() == 1 && nodes_[nd.children[0]].name == "code"; + std::string contents_text; + bool has_nontext = false; + std::string rendered; + if (is_code) { + // A [`code`][topic] link: the \code becomes the outermost layer, + // so the link text is the rendered content of the code span + contents_text = nodes_[nd.children[0]].text; + rendered = emit(contents_text, Esc::Verb, Mode::Verb); + } else { + contents_text = text_of(id); + for (size_t i = 0; i < nd.children.size(); i++) { + if (nodes_[nd.children[i]].name != "text") { + has_nontext = true; + } + } + rendered = children_to_rd_inlink(id); + } + cpp11::strings out( + resolve_link_(dest, contents_text, has_nontext, is_code, rendered)); + return std::string(out[0]); + } + + std::string txt = text_of(id); + if (dest.empty() || dest == txt) { + return "\\url{" + emit(txt, Esc::Comment, Mode::Text) + "}"; + } else { + return "\\href{" + emit(dest, Esc::Comment, Mode::Text) + "}{" + + children_to_rd_inlink(id) + "}"; + } + } + + std::string image_to_rd(const MdNode& nd) { + const std::string* d = nd.attr("destination"); + const std::string* t = nd.attr("title"); + std::string dest = d != NULL ? *d : ""; + std::string title = t != NULL ? *t : ""; + + // pdf can't display svg and html can't display pdf + std::string fmt = "all"; + if (restrict_images_) { + std::string lower = dest; + for (size_t i = 0; i < lower.size(); i++) { + if (lower[i] >= 'A' && lower[i] <= 'Z') { + lower[i] += 'a' - 'A'; + } + } + bool html = has_extension(lower, "jpg") || has_extension(lower, "jpeg") || + has_extension(lower, "gif") || has_extension(lower, "png"); + bool pdf = html; + html = html || has_extension(lower, "svg"); + pdf = pdf || has_extension(lower, "pdf"); + fmt = (html && pdf) ? "all" : html ? "html" : pdf ? "pdf" : "all"; + } + + std::string out; + if (fmt == "html") { + out += "\\if{html}{"; + } + if (fmt == "pdf") { + out += "\\if{pdf}{"; + } + out += "\\figure{" + emit(dest, Esc::None, Mode::Text) + "}"; + if (!title.empty()) { + out += "{" + emit(title, Esc::None, Mode::Text) + "}"; + } + if (fmt == "html" || fmt == "pdf") { + out += "}"; + } + return out; + } + + static bool has_extension(const std::string& path, const std::string& ext) { + if (path.size() < ext.size() + 1) { + return false; + } + return path[path.size() - ext.size() - 1] == '.' && + path.compare(path.size() - ext.size(), ext.size(), ext) == 0; + } + + std::string heading_to_rd(int id) { + const MdNode& nd = nodes_[id]; + const std::string* l = nd.attr("level"); + int level = l != NULL ? atoi(l->c_str()) : 0; + + if (!has_sections_ && level == 1) { + warn_("heading", ""); + return emit(text_of(id), Esc::Comment, Mode::Text); + } + + std::string txt = children_to_rd(id); + if (level == 1) { + titles.push_back(txt); + } + std::string head = close_sections(level) + "\n"; + if (level == 1) { + head += section_tag_; + } else { + head += "\\subsection{" + txt + "}{"; + } + sections_.push_back(level); + return head; + } + + std::string close_sections(int upto) { + if (upto < 2) { + upto = 2; + } + std::string out; + while (!sections_.empty() && sections_.back() >= upto) { + out += "\n}\n"; + sections_.pop_back(); + } + return out; + } +}; + +[[cpp11::register]] +cpp11::writable::list mdxmlToRd(std::string xml, cpp11::strings tokens, + cpp11::strings types, bool has_sections, + std::string section_tag, bool restrict_images, + cpp11::function resolve_link, + cpp11::function is_r_code, + cpp11::function warn) { + std::vector nodes; + int root = parse_xml(xml, nodes); + + std::string rd; + Walker walker(nodes, tokens, types, has_sections, section_tag, + restrict_images, resolve_link, is_r_code, warn); + if (root >= 0) { + rd = walker.walk(root); + } + + cpp11::writable::strings rtitles(walker.titles.size()); + for (size_t t = 0; t < walker.titles.size(); t++) { + rtitles[t] = walker.restore_text(walker.titles[t]); + } + + return cpp11::writable::list( + {"rd"_nm = cpp11::writable::strings({rd}), + "titles"_nm = rtitles}); +} diff --git a/tests/testthat/test-markdown-tokenize.R b/tests/testthat/test-markdown-tokenize.R index fd5369f6..da4633b9 100644 --- a/tests/testthat/test-markdown-tokenize.R +++ b/tests/testthat/test-markdown-tokenize.R @@ -61,25 +61,18 @@ test_that("pre-existing sentinel characters are stripped with a warning", { expect_equal(tk$tokens, character()) }) -test_that("restore_tokens restores according to context", { - tk <- md_tokenize("\\code{x} \\emph \\% \\\\[ \\") - state <- as.environment(tk) - - expect_equal( - restore_tokens(tk$text, state, "text"), - "\\code{x} \\emph \\% \\[ \\" - ) +test_that("tokens are restored according to context", { + # text context: verbatim tags and escapes come back as typed, except + # \\[ which loses a backslash like the markdown escape \[ does expect_equal( - restore_tokens(tk$text, state, "verb"), - "\\code{x} \\\\emph \\\\\\% \\\\\\\\[ \\\\" + markdown("\\code{x} \\emph \\% \\\\[ x"), + "\\code{x} \\emph \\% \\[ x" ) + # verb context: everything is Rd-escaped, except verbatim tags expect_equal( - restore_tokens(tk$text, state, "raw"), - "\\code{x} \\emph \\% \\\\[ \\" + markdown("`\\code{x} \\emph \\% \\\\[`"), + "\\verb{\\code{x} \\\\emph \\\\\\% \\\\\\\\[}" ) -}) - -test_that("restore_tokens leaves token-free text alone", { - state <- as.environment(list(tokens = character(), types = character())) - expect_equal(restore_tokens("plain *text*", state, "text"), "plain *text*") + # raw context: tokens come back as typed + expect_equal(markdown("`Rd \\out{z} \\% x`"), "\\Sexpr[stage=render,results=rd]{\\out{z} \\% x}") }) From 05ef6f8bc69537cef5b471b3cb8302bb6d02446d Mon Sep 17 00:00:00 2001 From: Hadley Wickham Date: Mon, 6 Jul 2026 07:25:51 -0500 Subject: [PATCH 4/4] Add NEWS bullet for markdown performance work --- NEWS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/NEWS.md b/NEWS.md index 0a10473d..0ae2e779 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,6 +1,7 @@ # roxygen2 (development version) * Markdown processing has been rewritten around a single tokenizer for the combined markdown/Rd grammar, replacing the old escape/unescape passes. Processing large documentation files is now substantially faster, and Rd tags behave consistently in every markdown context. +* Markdown processing is also considerably faster: the markdown to Rd translation now happens in C++, and text is only parsed for inline R code when it might contain some. Together this makes parsing a markdown-heavy package like testthat about 1.4x faster. * Markdown text containing `\%` now renders as `%`; previously it produced `\\%` in the Rd file, which truncated the displayed line at the `%`. * Markdown links with an escaped closing bracket (e.g. `[bar\]`) no longer leak a link reference definition into the generated Rd. * An Rd tag with an unterminated argument (e.g. `\code{a % b}`, where the unescaped `%` comments out the closing brace) now generates a warning instead of being silently reinterpreted as markdown.