Adding a custom reference link syntax

[TheCloudlessSky]

Hello,

Our application supports a custom syntax that is similar to a reference link:

[my label][foo:123]

The user doesn't have to define a reference to foo:123, we automatically map it to something like this:

<a href="/foos/123">my label</a>

There's several key factors of this special syntax:

• The label should be the same support as a normal reference link label. It can contain nested inline markdown like bold/italics, new lines, images, nested brackets, etc.
• It should be able to be nested inside of other constructs like GFM-tables.
• The foo:### is a consistent prefix. The pattern is literally foo:\d+. So, it should take priority over any normal reference links.

We previously implemented this in marked but I'm having trouble implementing it in micromark. The biggest culprit that I struggle with is properly capturing the label so that it handles nested markdown, line breaks, etc. As soon as I fix one problem, it introduces others. I'm having trouble getting into the core mechanics of, for example, label parsing. Of note, I have remark-breaks and rembark-gfm extensions enabled too.

Here's my setup:

import { markdownLineEnding } from 'micromark-util-character';

const CHAR_CODES = {
  BRACKET_OPEN: '['.charCodeAt(0),
  BRACKET_CLOSE: ']'.charCodeAt(0),
  F: 'f'.charCodeAt(0),
  O: 'o'.charCodeAt(0),
  COLON: ':'.charCodeAt(0),
  DIGIT_0: '0'.charCodeAt(0),
  DIGIT_9: '9'.charCodeAt(0),
  CR: '\r'.charCodeAt(0),
  LF: '\n'.charCodeAt(0),
  EOF: null
};

function tokenizeFooLink(effects, ok, nok) {
  let labelBracketCount = 0;
  let inChunk = false;

  return start;

  function start(code) {
    if (code !== CHAR_CODES.BRACKET_OPEN) {
      return nok(code);
    }

    effects.enter('fooLink');
    effects.enter('fooLinkLabel');
    effects.enter('fooLabelMarker');
    effects.consume(code);
    effects.exit('fooLabelMarker');

    labelBracketCount = 1;
    return labelContent;
  }

  function labelContent(code) {
    if (code === null) {
      return nok(code);
    }

    // End of the label.
    if (code === CHAR_CODES.BRACKET_CLOSE && labelBracketCount === 1) {
      if (inChunk) {
        effects.exit('chunkText');
        inChunk = false;
      }
      effects.enter('fooLabelMarker');
      effects.consume(code);
      effects.exit('fooLabelMarker');
      effects.exit('fooLinkLabel');
      return afterLabel;
    }

    // Newlines: Must exit chunk, consume newline, then restart chunk if needed.
    if (markdownLineEnding(code)) {
      if (inChunk) {
        effects.exit('chunkText');
        inChunk = false;
      }
      effects.enter('lineEnding');
      effects.consume(code);
      effects.exit('lineEnding');
      return labelContent;
    }

    if (!inChunk) {
      effects.enter('chunkText', { contentType: 'text' });
      inChunk = true;
    }

    if (code === CHAR_CODES.BRACKET_OPEN) {
      labelBracketCount++;
    }

    if (code === CHAR_CODES.BRACKET_CLOSE) {
      labelBracketCount--;
    }

    effects.consume(code);
    return labelContent;
  }

  function afterLabel(code) {
    if (code !== CHAR_CODES.BRACKET_OPEN) {
      return nok(code);
    }
    effects.enter('fooIdContainer');
    effects.enter('fooIdMarker');
    effects.consume(code);
    effects.exit('fooIdMarker');
    return matchF;
  }
  
  function matchF(code) { if (code === CHAR_CODES.F) { effects.enter('fooIdPrefix'); effects.consume(code); return matchO; } return nok(code); }
  function matchO(code) { if (code === CHAR_CODES.O) { effects.consume(code); return matchColon; } return nok(code); }
  function matchColon(code) { if (code === CHAR_CODES.COLON) { effects.consume(code); effects.exit('fooIdPrefix'); return matchId; } return nok(code); }

  function matchId(code) {
    if (code >= 48 && code <= 57) {
      effects.enter('fooIdValue');
      effects.consume(code);
      return matchIdMore;
    }
    return nok(code);
  }
  
  function matchIdMore(code) {
    if (code >= CHAR_CODES.DIGIT_0 && code <= CHAR_CODES.DIGIT_9) {
      effects.consume(code);
      return matchIdMore;
    }
    if (code === CHAR_CODES.BRACKET_CLOSE) {
      effects.exit('fooIdValue');
      effects.enter('fooIdMarker');
      effects.consume(code);
      effects.exit('fooIdMarker');
      effects.exit('fooIdContainer');
      effects.exit('fooLink');
      return ok;
    }
    return nok(code);
  }
}

const fooLink = {

  tokenize: function(effects, ok, nok) {
    return (code) => {
      return effects.attempt(
        { tokenize: tokenizeFooLink, partial: true }, 
        ok, 
        nok
      )(code);
    }
  },

  resolveAll: (events, context) => {

    let index = -1;

    // Ensure the content type is set to text. This allows the sub-tokenizer to find **bold**, etc.
    while (++index < events.length) {
      const event = events[index];
      if (event[1].type === 'fooLabelContent' && event[0] === 'enter') {
        event[1].contentType = 'text';
      }
    }

    // Now let the standard resolver handle the newly marked 'text' content.
    return events;
  }
}

function fooReferenceLinkMicromarkExtension() {
  return {
    // Look for "inline" foo link starts.
    text: {
      [CHAR_CODES.BRACKET_OPEN]: {
        name: 'fooLink',
        ...fooLink,
      },
    }

  };
}

const fromMarkdownExtension = {
  enter: {
    fooLink(token) {
      console.log('ENTER foo LINK');
      this.enter({
        type: 'fooLink',
        fooId: '', 
        data: {
          hName: 'a',
          hProperties: {
            // TODO...
            // className: ['foo-link']
          }
        },
        children: []
      }, token);
    },
  },
  exit: {
    fooIdValue(token) {
      const id = this.sliceSerialize(token);
      // Look at the top of the stack (which should be our fooLink)
      const node = this.stack[this.stack.length - 1];
      if (node && node.type === 'fooLink') {
        node.fooId = id;
        node.data.hProperties.href = `/foos/${id}`;
      }
    },

    lineEnding(token) {

      const isInFooLink = this.stack.some(node => node.type === 'fooLink');
  
      if (isInFooLink) {
        this.config.enter.data.call(this, token);
        this.config.exit.data.call(this, token);
        // return;
      }
    },

    fooLink(token) {
      console.log('EXIT foo LINK');
      this.exit(token);
    }
  },

};

const toMarkdownExtension = {};

function fooReferenceLinkMdastExtension() {
  return {
    fromMarkdown: fromMarkdownExtension,
    toMarkdown: toMarkdownExtension
  };
};

This works for a majority of situations. But I'm finding several cases failing to parse as I'd expect:

[**hello world**][foo:123]   => <a href="/foos/123"><strong>hello world</strong></a> (expected)
[**hello\nworld**][foo:123]  => <a href="/foos/123">**hello<br>\nworld**</a> (unexpected unformatted bold)
foo\nbar                     => foobar (unexpected line break collapsed even with remark-breaks)

Is there core documentation (or an obvious fix) for how to properly let micromark re-parse (sub-tokenize) the label's content to support markdown, new lines, etc just like normal links?

[wooorm]

Hi!

it’s not really recommended to make a markdown flavor that does not work in other places. See https://github.com/micromark/micromark#extensions. And the recommendations there. In this case, directives seems good?

Media (links and images) are notorious. They are parsed in 2 bits, starts and closes. And only later it is checked if there are matching open/closes.

So this will never work like links, if you attempt to parse an open and close together.

I recommend turning DEBUG on, that can catch errors, and show how things are progressing! https://github.com/micromark/micromark#size--debug