Merge pull request #621 from Shopify/jm/doc_tag

Add basic {% doc %} tag parsing support

Author: jamesmengo

packages/liquid-html-parser/grammar/liquid-html.ohm

@@ -37,6 +37,7 @@ Liquid <: Helpers {
   endOfIdentifier = endOfTagName | endOfVarName
 
   liquidNode =
+  	| liquidDoc
     | liquidBlockComment
     | liquidRawTag
     | liquidDrop
@@ -216,6 +217,15 @@ Liquid <: Helpers {
   commentBlockStart = "{%" "-"? space* ("comment"    endOfIdentifier) space* tagMarkup "-"? "%}"
   commentBlockEnd   = "{%" "-"? space* ("endcomment" endOfIdentifier) space* tagMarkup "-"? "%}"
 
+  liquidDoc = 
+    liquidDocStart
+      liquidDocBody
+    liquidDocEnd
+
+  liquidDocStart = "{%" "-"? space* ("doc" endOfIdentifier) space* tagMarkup "-"? "%}"
+  liquidDocEnd = "{%" "-"? space* ("enddoc" endOfIdentifier) space* tagMarkup "-"? "%}"
+  liquidDocBody = anyExceptStar<(liquidDocStart | liquidDocEnd)>
+
   // In order for the grammar to "fallback" to the base case, this
   // rule must pass if and only if we support what we parse. This
   // implies that—since we don't support filters yet—we have a
@@ -378,6 +388,10 @@ LiquidStatement <: Liquid {
   delimTag := liquidStatementEnd
 }
 
+LiquidDoc <: Helpers {
+  Node := (TextNode)*
+}
+
 LiquidHTML <: Liquid {
   Node := yamlFrontmatter? (HtmlNode | liquidNode | TextNode)*
   openControl += "<"

packages/liquid-html-parser/src/grammar.ts

@@ -3,6 +3,7 @@ import ohm from 'ohm-js';
 export const liquidHtmlGrammars = ohm.grammars(require('../grammar/liquid-html.ohm.js'));
 
 export const TextNodeGrammar = liquidHtmlGrammars['Helpers'];
+export const LiquidDocGrammar = liquidHtmlGrammars['LiquidDoc'];
 
 export interface LiquidGrammars {
   Liquid: ohm.Grammar;
@@ -52,4 +53,5 @@ export const TAGS_WITHOUT_MARKUP = [
   'continue',
   'comment',
   'raw',
+  'doc',
 ];

packages/liquid-html-parser/src/stage-1-cst.spec.ts

@@ -982,6 +982,34 @@ describe('Unit: Stage 1 (CST)', () => {
         }
       });
 
+      it('should parse doc tags', () => {
+        for (const { toCST, expectPath } of testCases) {
+          const testStr = `{% doc -%} Renders loading-spinner. {%- enddoc %}`;
+
+          cst = toCST(testStr);
+          expectPath(cst, '0.type').to.equal('LiquidRawTag');
+          expectPath(cst, '0.name').to.equal('doc');
+          expectPath(cst, '0.body').to.include('Renders loading-spinner');
+          expectPath(cst, '0.whitespaceStart').to.equal('');
+          expectPath(cst, '0.whitespaceEnd').to.equal('-');
+          expectPath(cst, '0.delimiterWhitespaceStart').to.equal('-');
+          expectPath(cst, '0.delimiterWhitespaceEnd').to.equal('');
+          expectPath(cst, '0.blockStartLocStart').to.equal(0);
+          expectPath(cst, '0.blockStartLocEnd').to.equal(0 + '{% doc -%}'.length);
+          expectPath(cst, '0.blockEndLocStart').to.equal(testStr.length - '{%- enddoc %}'.length);
+          expectPath(cst, '0.blockEndLocEnd').to.equal(testStr.length);
+          expectPath(cst, '0.children').to.deep.equal([
+            {
+              locEnd: 35,
+              locStart: 11,
+              source: '{% doc -%} Renders loading-spinner. {%- enddoc %}',
+              type: 'TextNode',
+              value: 'Renders loading-spinner.',
+            },
+          ]);
+        }
+      });
+
       it('should parse tag open / close', () => {
         BLOCKS.forEach((block: string) => {
           for (const { toCST, expectPath } of testCases) {

packages/liquid-html-parser/src/stage-1-cst.ts

@@ -34,6 +34,7 @@ import { Parser } from 'prettier';
 import ohm, { Node } from 'ohm-js';
 import { toAST } from 'ohm-js/extras';
 import {
+  LiquidDocGrammar,
   LiquidGrammars,
   TextNodeGrammar,
   placeholderGrammars,
@@ -626,6 +627,30 @@ function toCST<T>(
       blockEndLocStart: (tokens: Node[]) => tokens[2].source.startIdx,
       blockEndLocEnd: (tokens: Node[]) => tokens[2].source.endIdx,
     },
+    liquidDoc: {
+      type: ConcreteNodeTypes.LiquidRawTag,
+      name: 'doc',
+      body: (tokens: Node[]) => tokens[1].sourceString,
+      children: (tokens: Node[]) => {
+        const contentNode = tokens[1];
+        return toLiquidDocAST(
+          source,
+          contentNode.sourceString,
+          offset + contentNode.source.startIdx,
+        );
+      },
+      whitespaceStart: (tokens: Node[]) => tokens[0].children[1].sourceString,
+      whitespaceEnd: (tokens: Node[]) => tokens[0].children[7].sourceString,
+      delimiterWhitespaceStart: (tokens: Node[]) => tokens[2].children[1].sourceString,
+      delimiterWhitespaceEnd: (tokens: Node[]) => tokens[2].children[7].sourceString,
+      locStart,
+      locEnd,
+      source,
+      blockStartLocStart: (tokens: Node[]) => tokens[0].source.startIdx,
+      blockStartLocEnd: (tokens: Node[]) => tokens[0].source.endIdx,
+      blockEndLocStart: (tokens: Node[]) => tokens[2].source.startIdx,
+      blockEndLocEnd: (tokens: Node[]) => tokens[2].source.endIdx,
+    },
     liquidInlineComment: {
       type: ConcreteNodeTypes.LiquidTag,
       name: 3,
@@ -1262,3 +1287,35 @@ function toCST<T>(
 
   return toAST(res, selectedMappings) as T;
 }
+
+/**
+ * Builds an AST for LiquidDoc content.
+ *
+ * `toCST` includes mappings and logic that are not needed for LiquidDoc so we're separating this logic
+ */
+function toLiquidDocAST(source: string, matchingSource: string, offset: number) {
+  // When we switch parser, our locStart and locEnd functions must account
+  // for the offset of the {% doc %} markup
+  const locStart = (tokens: Node[]) => offset + tokens[0].source.startIdx;
+  const locEnd = (tokens: Node[]) => offset + tokens[tokens.length - 1].source.endIdx;
+
+  const res = LiquidDocGrammar.match(matchingSource, 'Node');
+  if (res.failed()) {
+    throw new LiquidHTMLCSTParsingError(res);
+  }
+
+  const LiquidDocMappings: Mapping = {
+    Node: 0,
+    TextNode: {
+      type: ConcreteNodeTypes.TextNode,
+      value: function () {
+        return (this as any).sourceString;
+      },
+      locStart,
+      locEnd,
+      source,
+    },
+  };
+
+  return toAST(res, LiquidDocMappings);
+}

packages/liquid-html-parser/src/stage-2-ast.spec.ts

@@ -1220,6 +1220,33 @@ describe('Unit: Stage 2 (AST)', () => {
       expectPath(ast, 'children.0.markup.1.children.0.children.1.markup.name').to.eql('var3');
     });
 
+    it(`should parse doc tags`, () => {
+      ast = toLiquidAST(`{% doc %}{% enddoc %}`);
+      expectPath(ast, 'children.0.type').to.eql('LiquidRawTag');
+      expectPath(ast, 'children.0.name').to.eql('doc');
+      expectPath(ast, 'children.0.markup').toEqual('');
+      expectPath(ast, 'children.0.body.value').to.eql('');
+      expectPath(ast, 'children.0.body.type').toEqual('RawMarkup');
+      expectPath(ast, 'children.0.body.nodes').toEqual([]);
+
+      ast = toLiquidAST(`{% doc -%} single line doc {%- enddoc %}`);
+      expectPath(ast, 'children.0.type').to.eql('LiquidRawTag');
+      expectPath(ast, 'children.0.name').to.eql('doc');
+      expectPath(ast, 'children.0.body.value').to.eql(' single line doc ');
+      expectPath(ast, 'children.0.body.nodes.0.type').toEqual('TextNode');
+
+      ast = toLiquidAST(`{% doc -%}
+        multi line doc
+        multi line doc
+        {%- enddoc %}`);
+      expectPath(ast, 'children.0.type').to.eql('LiquidRawTag');
+      expectPath(ast, 'children.0.name').to.eql('doc');
+      expectPath(ast, 'children.0.body.nodes.0.value').to.eql(
+        `multi line doc\n        multi line doc`,
+      );
+      expectPath(ast, 'children.0.body.nodes.0.type').toEqual('TextNode');
+    });
+
     it('should parse unclosed tables with assignments', () => {
       ast = toLiquidAST(`
         {%- liquid

packages/theme-check-common/src/checks/liquid-html-syntax-error/index.spec.ts

@@ -86,7 +86,7 @@ describe('Module: LiquidHTMLSyntaxError', () => {
     const offenses = await runLiquidCheck(LiquidHTMLSyntaxError, sourceCode);
     expect(offenses).to.have.length(1);
     expect(offenses[0].message).to.equal(
-      `SyntaxError: expected "#", a letter, "when", "sections", "section", "render", "liquid", "layout", "increment", "include", "elsif", "else", "echo", "decrement", "content_for", "cycle", "continue", "break", "assign", "tablerow", "unless", "if", "ifchanged", "for", "case", "capture", "paginate", "form", "end", "style", "stylesheet", "schema", "javascript", "raw", or "comment"`,
+      `SyntaxError: expected "#", a letter, "when", "sections", "section", "render", "liquid", "layout", "increment", "include", "elsif", "else", "echo", "decrement", "content_for", "cycle", "continue", "break", "assign", "tablerow", "unless", "if", "ifchanged", "for", "case", "capture", "paginate", "form", "end", "style", "stylesheet", "schema", "javascript", "raw", "comment", or "doc"`,
     );
   });