Skip to content

Optimize inline closer indexing #129

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
bobzhang wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Optimize inline closer indexing #129

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/cmark/block_struct_wbtest.mbt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -702,7 +702,7 @@ test "should tokenize inline image across multiple lines" {
tokenize_only(doc),
content=(
#|(
#| CloserIndex({ RightBrack: <Set: [25]>, RightParen: <Set: [64, 65]> }),
#| CloserIndex({ RightBrack: [25], RightParen: [64, 65] }),
#| Tokens(
#| <Deque:
#| [
Expand Down
26 changes: 21 additions & 5 deletions src/cmark/closer.mbt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,10 @@ priv enum Closer {
} derive(Eq, Hash, Debug)

///|
type PosSet = Set[Int]
type Positions = Array[Int]

///|
priv struct CloserIndex(Map[Closer, PosSet]) derive(Debug)
priv struct CloserIndex(Map[Closer, Positions]) derive(Debug)

///|
test {
Expand All @@ -36,16 +36,32 @@ fn CloserIndex::add(self : CloserIndex, key : Closer, pos : Int) -> Unit {
(match self.0.get(key) {
Some(ps) => ps
None => {
let ps = Set([])
let ps = []
self.0[key] = ps
ps
}
}).add(pos)
}).push(pos)
}

///|
fn CloserIndex::pos(self : CloserIndex, key : Closer, after~ : Int) -> Int? {
self.0.get(key).bind(fn(s) { s.iter().find_first(fn(pos) { pos > after }) })
guard self.0.get(key) is Some(positions) else { return None }
let len = positions.length()
let first = for lo = 0, hi = len {
if lo >= hi {
break lo
}
let mid = (lo + hi) / 2
if positions[mid] <= after {
continue mid + 1, hi
}
continue lo, mid
}
if first < len {
Some(positions[first])
} else {
None
}
}

///|
Expand Down
126 changes: 73 additions & 53 deletions src/cmark/inline_struct.mbt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ priv enum Token {
LinkStart(TokenLinkStart)
Newline(TokenNewline)
RightBrack(TokenStart)
RightParen(TokenStart)
StrikethroughMarks(TokenStrikethroughMarks)
MathSpanMarks(TokenMathSpanMarks)
} derive(Debug)
Expand Down Expand Up @@ -93,7 +92,6 @@ fn Token::start(self : Token) -> CharCodePos {
LinkStart(t) => t.start
Newline(t) => t.start
RightBrack(t) => t.start
RightParen(t) => t.start
StrikethroughMarks(t) => t.start
MathSpanMarks(t) => t.start
}
Expand Down Expand Up @@ -151,29 +149,6 @@ fn CloserIndex::has_math_span(
}

///|
fn make_closer_index(toks : Tokens) -> CloserIndex {
let cidx = CloserIndex::new()
guard !toks.0.is_empty() else { cidx }
toks.0.retain(fn(curr) {
match curr {
Backticks({ count, start, .. }) => cidx.add(Backticks(count), start)
RightBrack({ start }) => cidx.add(RightBrack, start)
RightParen({ start }) => {
cidx.add(RightParen, start)
return false // Discard the token since it's not used for parsing
}
EmphasisMarks({ char, start, may_close: true, .. }) =>
cidx.add(EmphasisMarks(char), start)
StrikethroughMarks({ start, may_close: true, .. }) =>
cidx.add(StrikethroughMarks, start)
MathSpanMarks({ count, start, may_close: true, .. }) =>
cidx.add(MathSpanMarks(count), start)
_ => ()
}
true
})
cidx
}

///|
/// Used to make the text delimitation precise for nested inlines.
Expand Down Expand Up @@ -256,6 +231,7 @@ fn Token::newline(
///|
fn tokens_add_backtick(
toks : Tokens,
cidx : CloserIndex,
s : String,
line : LineSpan,
prev_bslash~ : Bool,
Expand All @@ -264,6 +240,7 @@ fn tokens_add_backtick(
let last = @cmark_base.run_of(char='`', s, last=line.last, start=start + 1)
let count = last - start + 1
toks.push(Backticks({ start, count, escaped: prev_bslash }))
cidx.add(Backticks(count), start)
last + 1
}

Expand All @@ -283,6 +260,7 @@ fn tokens_try_add_image_link_start(
///|
fn tokens_try_add_emphasis(
toks : Tokens,
cidx : CloserIndex,
s : String,
line : LineSpan,
start~ : Int,
Expand All @@ -309,12 +287,16 @@ fn tokens_try_add_emphasis(
(char == '_' && is_right_flanking && (!is_left_flanking || is_next_punct))
guard may_open || may_close else { next }
toks.push(EmphasisMarks({ start, char, count, may_open, may_close }))
if may_close {
cidx.add(EmphasisMarks(char), start)
}
next
}

///|
fn tokens_try_add_strikethrough_marks(
toks : Tokens,
cidx : CloserIndex,
s : String,
line : LineSpan,
start~ : Int,
Expand All @@ -330,12 +312,16 @@ fn tokens_try_add_strikethrough_marks(
let may_open = !@char.is_ascii_whitespace(next_char)
let may_close = !@char.is_ascii_whitespace(prev_char)
toks.push(StrikethroughMarks({ start, may_open, may_close }))
if may_close {
cidx.add(StrikethroughMarks, start)
}
next
}

///|
fn tokens_try_add_math_span_marks(
toks : Tokens,
cidx : CloserIndex,
s : String,
line : LineSpan,
start~ : Int,
Expand All @@ -356,6 +342,9 @@ fn tokens_try_add_math_span_marks(
}
guard may_open || may_close else { next }
toks.push(MathSpanMarks({ start, count, may_open, may_close }))
if may_close {
cidx.add(MathSpanMarks(count), start)
}
next
}

Expand All @@ -364,49 +353,73 @@ fn tokenize(
exts~ : Bool,
s : String,
lines : Array[LineSpan],
) -> (CloserIndex, Tokens, LineSpan) {
) -> (CloserIndex, Tokens, LineSpan, Bool) {
guard lines is [line, .. lines] else { abort("expected at least one line") }
let toks : Tokens = Deque([])
let cidx = for lines = lines, line = line, prev_bslash = false, k = line.first {
let cidx = CloserIndex::new()
let only_newlines = for lines = lines, line = line, prev_bslash = false, k = line.first, only_newlines = true {
if k > line.last {
break match lines {
[] => make_closer_index(toks)
match lines {
[] => break only_newlines
[newline, .. lines] => {
let t = Token::newline(s, line, newline)
toks.push(t)
continue lines, newline, false, newline.first
continue lines, newline, false, newline.first, only_newlines
}
}
}
let next = match s.code_unit_at(k) {
'\\' => continue lines, line, !prev_bslash, k + 1
'`' => tokens_add_backtick(toks, s, line, prev_bslash~, start=k)
_ if prev_bslash => k + 1
'*' | '_' => tokens_try_add_emphasis(toks, s, line, start=k)
let (next, only_newlines) = match s.code_unit_at(k) {
'\\' => continue lines, line, !prev_bslash, k + 1, only_newlines
'`' =>
(tokens_add_backtick(toks, cidx, s, line, prev_bslash~, start=k), false)
_ if prev_bslash => (k + 1, only_newlines)
'*' | '_' => {
let before = toks.0.length()
let next = tokens_try_add_emphasis(toks, cidx, s, line, start=k)
(next, only_newlines && toks.0.length() == before)
}
']' => {
toks.push(RightBrack({ start: k }))
k + 1
cidx.add(RightBrack, k)
(k + 1, false)
}
'[' => {
toks.push(LinkStart({ start: k, image: false }))
k + 1
(k + 1, false)
}
'!' => {
let next = tokens_try_add_image_link_start(toks, s, line, start=k)
(next, only_newlines && next == k + 1)
}
'!' => tokens_try_add_image_link_start(toks, s, line, start=k)
'<' => {
toks.push(AutolinkOrHtmlStart({ start: k }))
k + 1
(k + 1, false)
}
')' => {
toks.push(RightParen({ start: k }))
k + 1
cidx.add(RightParen, k)
(k + 1, only_newlines)
}
'~' if exts => tokens_try_add_strikethrough_marks(toks, s, line, start=k)
'$' if exts => tokens_try_add_math_span_marks(toks, s, line, start=k)
_ => k + 1
'~' if exts => {
let before = toks.0.length()
let next = tokens_try_add_strikethrough_marks(
toks,
cidx,
s,
line,
start=k,
)
(next, only_newlines && toks.0.length() == before)
}
'$' if exts => {
let before = toks.0.length()
let next = tokens_try_add_math_span_marks(toks, cidx, s, line, start=k)
(next, only_newlines && toks.0.length() == before)
}
_ => (k + 1, only_newlines)
}
continue lines, line, false, next
continue lines, line, false, next, only_newlines
}
(cidx, toks, line)
(cidx, toks, line, only_newlines)
}

// Making inlines and inline tokens
Expand Down Expand Up @@ -942,7 +955,6 @@ fn Parser::find_link_text_tokens(
let mut line = start_line
let mut nest = 0
let acc : Tokens = Deque([])
let old_rev_toks = rev_toks.val.0.copy()
for ;; {
match (rev_toks.val.pop(), nest) {
(Some(RightBrack({ start: last })), 0) => {
Expand Down Expand Up @@ -987,7 +999,6 @@ fn Parser::find_link_text_tokens(
(None, _) => break
}
}
rev_toks.val = old_rev_toks
None
}

Expand Down Expand Up @@ -1090,8 +1101,8 @@ fn Parser::try_link(
image~ : Bool,
start~ : CharCodePos,
) -> (LineSpan, Token, Bool)? {
let rev_toks : Ref[RevTokens] = Ref(start_rev_toks.val.0.copy())
guard self.cidx.has_right_brack(after=start) else { return None }
let rev_toks : Ref[RevTokens] = Ref(start_rev_toks.val.0.copy())
guard self.find_link_text_tokens( // text_last with ] delim
rev_toks,
start_line,
Expand Down Expand Up @@ -1495,9 +1506,18 @@ fn Parser::parse_inline(
lines : Array[LineSpan],
) -> ((Col, String), Inline) {
let (layout, meta, lines) = self.strip_paragraph(lines)
let (cidx, toks, first_line) = tokenize(self.i, lines, exts=self.exts)
self.cidx = cidx
let (is_, _) = self.parse_tokens(toks, first_line)
let (cidx, toks, first_line, only_newlines) = tokenize(
self.i,
lines,
exts=self.exts,
)
let is_ = if only_newlines {
self.last_pass(toks, first_line)
} else {
self.cidx = cidx
let (is_, _) = self.parse_tokens(toks, first_line)
is_
}
let inline = match is_ {
[i] => i
_ => Inlines({ v: is_, meta })
Expand Down Expand Up @@ -1663,7 +1683,7 @@ fn Parser::parse_table_row(
self : Parser,
line : LineSpan,
) -> Array[(Inline, TableCellLayout)] {
let (cidx, toks, first_line) = tokenize(self.i, [line], exts=self.exts)
let (cidx, toks, first_line, _) = tokenize(self.i, [line], exts=self.exts)
self.cidx = cidx
let toks : Ref[Tokens] = Ref(toks)
let _ = self.first_pass(toks, first_line)
Expand Down
16 changes: 6 additions & 10 deletions src/cmark/inline_struct_wbtest.mbt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,8 @@ fn tokenize_only(
char_code_pos += ln.length() + 1 // considering the trailing '\n'
}
let (_, _, lines) = p.strip_paragraph(lines1)
tokenize(p.i, lines, exts=p.exts)
let (cidx, toks, line, _) = tokenize(p.i, lines, exts=p.exts)
(cidx, toks, line)
}

///|
Expand Down Expand Up @@ -550,7 +551,7 @@ test "should tokenize codespans" {
),
content=(
#|(
#| CloserIndex({ Backticks(1): <Set: [25, 71]>, Backticks(2): <Set: [41]> }),
#| CloserIndex({ Backticks(1): [25, 71], Backticks(2): [41] }),
#| Tokens(
#| <Deque:
#| [
Expand Down Expand Up @@ -2160,7 +2161,7 @@ test "should tokenize broken links across lines" {
),
content=(
#|(
#| CloserIndex({ RightBrack: <Set: [22]> }),
#| CloserIndex({ RightBrack: [22] }),
#| Tokens(
#| <Deque:
#| [
Expand Down Expand Up @@ -2856,12 +2857,7 @@ test "should tokenize nested strikethroughs and emphases" {
tokenize_only(doc, strict=false),
content=(
#|(
#| CloserIndex(
#| {
#| EmphasisMarks('*'): <Set: [19, 25, 69]>,
#| StrikethroughMarks: <Set: [35, 67, 72]>,
#| },
#| ),
#| CloserIndex({ EmphasisMarks('*'): [19, 25, 69], StrikethroughMarks: [35, 67, 72] }),
#| Tokens(
#| <Deque:
#| [
Expand Down Expand Up @@ -3063,7 +3059,7 @@ test "should tokenize inline math" {
),
content=(
#|(
#| CloserIndex({ MathSpanMarks(1): <Set: [16]> }),
#| CloserIndex({ MathSpanMarks(1): [16] }),
#| Tokens(
#| <Deque:
#| [
Expand Down
Loading