Let the parse hook only override tag body types and write trailing body comments

The built in node kinds (core statements, style blocks, attribute tags,
void/text lists) are now decided before the getTagType hook is consulted, so
consumers which always return a type get the same tree as the language
server. Trailing comments in comment-only or attr-tag-only bodies are now
written to the generated script, the parser returns the filename verbatim
(normalization moved back to language-tools), and @marko/parse publishes as
a minor while its API settles.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

Author: DylanPiercey

.changeset/parse-package.md

@@ -1,5 +1,5 @@
 ---
-"@marko/parse": major
+"@marko/parse": minor
 ---
 
-New package: the Marko CST parser previously embedded in `@marko/language-tools`, extracted so it can be shared by other tooling. Node types are strings, syntax errors and comments are exposed on the parse result, comments and static statements are part of `program.body`, and a `getTagType` option allows overriding how tag bodies parse.
+New package: the Marko CST parser previously embedded in `@marko/language-tools`, extracted so it can be shared by other tooling. Node types are strings, syntax errors and comments are exposed on the parse result, comments and static statements are part of `program.body`, the filename is returned verbatim, and a `getTagType` option allows overriding how tag bodies parse (the built in statement and attribute tag node kinds are decided first; the hook only overrides a tag's body type or forces a generic `Static` statement).

AGENTS.md

@@ -2,11 +2,12 @@
 
 ## Repo overview
 
-Monorepo for the Marko Language Server and related tooling. Four pnpm workspaces under `packages/`:
+Monorepo for the Marko Language Server and related tooling. Five pnpm workspaces under `packages/`:
 
 | Package           | Published as                                | Purpose                                                     |
 | ----------------- | ------------------------------------------- | ----------------------------------------------------------- |
-| `language-tools`  | `@marko/language-tools`                     | Core extraction/analysis of Marko files (leaf dependency)   |
+| `parse`           | `@marko/parse`                              | CST parser for Marko templates (leaf dependency)            |
+| `language-tools`  | `@marko/language-tools`                     | Core extraction/analysis of Marko files; depends on parse   |
 | `language-server` | `@marko/language-server`                    | LSP implementation; depends on language-tools               |
 | `type-check`      | `@marko/type-check`                         | CLI type-checker (`mtc`); depends on language-tools         |
 | `vscode`          | `marko-vscode` (VS Code extension, private) | VS Code client; depends on language-server + language-tools |
@@ -18,8 +19,8 @@ TypeScript emits **only `.d.ts` files** (`emitDeclarationOnly: true`); esbuild (
 **Build order matters** due to project references:
 
 ```
-language-tools -> language-server -> vscode
-language-tools -> type-check
+parse -> language-tools -> language-server -> vscode
+parse -> language-tools -> type-check
 ```
 
 Commands:
@@ -33,7 +34,13 @@ The vscode build is the most complex: it bundles 4 entry points (including tests
 
 **Build is always required before testing.** `pnpm test` at root runs `pnpm run build && pnpm -r --if-present run test`.
 
-Only two packages have tests:
+Only three packages have tests:
+
+### parse (mocha)
+
+```sh
+pnpm --filter @marko/parse run test
+```
 
 ### language-server (mocha + mocha-snap)
 

packages/language-server/src/__tests__/fixtures/script/comments-trailing/__snapshots__/comments-trailing.expected/index.html

@@ -0,0 +1,8 @@
+<div>
+  
+</div><div data-marko-node-id="2">
+  
+  
+</div><span data-marko-node-id="3">
+  
+</span><div data-marko-node-id="4">text</div>

packages/language-server/src/__tests__/fixtures/script/comments-trailing/__snapshots__/comments-trailing.expected/index.md

@@ -0,0 +1,32 @@
+## Diagnostics
+### Ln 2, Col 9
+```marko
+  1 | <const/data = { a: 1 }/>
+> 2 | <loader|data|>
+    |         ^^^^ 'data' is declared but its value is never read.
+  3 |   //  ^?
+  4 | </loader>
+  5 | <div>
+```
+
+### Ln 2, Col 9
+```marko
+  1 | <const/data = { a: 1 }/>
+> 2 | <loader|data|>
+    |         ^^^^ Parameter 'data' implicitly has an 'any' type.
+  3 |   //  ^?
+  4 | </loader>
+  5 | <div>
+```
+
+### Ln 6, Col 4
+```marko
+  4 | </loader>
+  5 | <div>
+> 6 |   <@x/>
+    |    ^^ Object literal may only specify known properties, and '["x"]' does not exist in type 'Directives & Div'.
+  7 |   // ^?
+  8 | </div>
+  9 | <span>
+```
+

packages/language-server/src/__tests__/fixtures/script/comments-trailing/__snapshots__/comments-trailing.expected/index.ts

@@ -0,0 +1,90 @@
+export interface Input {}
+(function (this: void) {
+  const input = Marko._.any as Input;
+  const $signal = Marko._.any as AbortSignal;
+  const $global = Marko._.getGlobal(
+    // @ts-expect-error We expect the compiler to error because we are checking if the MarkoRun.Context is defined.
+    (Marko._.error, Marko._.any as MarkoRun.Context),
+  );
+  const data = Marko._.hoist(() => __marko_internal_hoist__data);
+  const __marko_internal_tag_1 = Marko._.resolveTemplate(
+    import("@marko/runtime-tags/tags/const.d.marko"),
+  );
+  {
+    const data = Marko._.returned(() => __marko_internal_rendered_1);
+    const __marko_internal_rendered_1 = Marko._.renderTemplate(
+      __marko_internal_tag_1,
+    )()()({
+      value: { a: 1 },
+    });
+    const __marko_internal_tag_2 = Marko._.interpolated`loader`;
+    Marko._.renderDynamicTag(__marko_internal_tag_2)()()({
+      [Marko._.contentFor(__marko_internal_tag_2)]: (data) => {
+        //  ^?
+        return Marko._.voidReturn;
+      },
+    });
+    Marko._.renderNativeTag("div")()()({
+      ["x"]: {
+        [Symbol.iterator]: Marko._.any,
+      },
+      // ^?
+    });
+    Marko._.renderNativeTag("span")()()({
+      [Marko._.content]: (() => {
+        // after
+        return () => {
+          return Marko._.voidReturn;
+        };
+      })(),
+    });
+    Marko._.renderNativeTag("div")()()({
+      [Marko._.content]: (() => {
+        // c
+        return () => {
+          return Marko._.voidReturn;
+        };
+      })(),
+    });
+    var __marko_internal_hoist__data = data;
+  }
+  Marko._.noop({ data, input, $global, $signal });
+  return;
+})();
+const __marko_internal_api = "tags";
+export { __marko_internal_api as "~api" };
+export default new (class Template extends Marko._.Template<{
+  render(
+    input: Marko.TemplateInput<Input>,
+    stream?: {
+      write: (chunk: string) => void;
+      end: (chunk?: string) => void;
+    },
+  ): Marko.Out<never>;
+
+  render(
+    input: Marko.TemplateInput<Input>,
+    cb?: (err: Error | null, result: Marko.RenderResult<never>) => void,
+  ): Marko.Out<never>;
+
+  renderSync(input: Marko.TemplateInput<Input>): Marko.RenderResult<never>;
+
+  renderToString(input: Marko.TemplateInput<Input>): string;
+
+  stream(
+    input: Marko.TemplateInput<Input>,
+  ): ReadableStream<string> & NodeJS.ReadableStream;
+
+  mount(
+    input: Marko.TemplateInput<Input>,
+    reference: Node,
+    position?: "afterbegin" | "afterend" | "beforebegin" | "beforeend",
+  ): Marko.MountedTemplate<typeof input>;
+
+  api: typeof __marko_internal_api;
+  _(): () => <__marko_internal_input extends unknown>(
+    input: Marko.Directives &
+      Input &
+      Marko._.Relate<__marko_internal_input, Marko.Directives & Input>,
+  ) => Marko._.ReturnWithScope<__marko_internal_input, void>;
+}> {})();

packages/language-server/src/__tests__/fixtures/script/comments-trailing/index.marko

@@ -0,0 +1,14 @@
+<const/data = { a: 1 }/>
+<loader|data|>
+  //  ^?
+</loader>
+<div>
+  <@x/>
+  // ^?
+</div>
+<span>
+  // after
+</span>
+div
+  // c
+  -- text

packages/language-server/src/__tests__/fixtures/script/comments/__snapshots__/comments.expected/index.ts

@@ -11,7 +11,27 @@ export { type Component };
     // @ts-expect-error We expect the compiler to error because we are checking if the MarkoRun.Context is defined.
     (Marko._.error, Marko._.any as MarkoRun.Context),
   );
-  Marko._.noop({ component, state, out, input, $global, $signal });
+  // Single-line comment */ // \n
+  /*
+multi-line
+  comment
+  <!--
+
+    */ /*
+  HTML
+    Comment
+        *\/
+  *\/ //
+\n
+    *\/
+                  */ Marko._.noop({
+    component,
+    state,
+    out,
+    input,
+    $global,
+    $signal,
+  });
   return;
 })();
 const __marko_internal_api = "class";

packages/language-server/src/__tests__/fixtures/script/el-ref-hoist/__snapshots__/el-ref-hoist.expected/index.ts

@@ -20,6 +20,7 @@ $el
 });
 {const $el = Marko._.el("div");
 Marko._.renderNativeTag("div")()()({
+// ^?
 
 });
 const __marko_internal_tag_3 = (

packages/language-server/src/__tests__/fixtures/script/for-tag/__snapshots__/for-tag.expected/index.ts

@@ -66,6 +66,7 @@ export interface Input {}
         by: (item, index) => `${item}-${index}`,
       },
       () => {
+        //               ^?    ^?
         return Marko._.voidReturn;
       },
     );
@@ -138,6 +139,7 @@ export interface Input {}
           by: (value, key) => `${value}-${key}`,
         },
         () => {
+          //                 ^?     ^?
           return Marko._.voidReturn;
         },
       );
@@ -191,6 +193,7 @@ export interface Input {}
           by: (index) => `${index}`,
         },
         () => {
+          //             ^?
           return Marko._.voidReturn;
         },
       );
@@ -266,6 +269,7 @@ export interface Input {}
           by: (index) => `${index}`,
         },
         () => {
+          //             ^?
           return Marko._.voidReturn;
         },
       );

packages/language-server/src/__tests__/fixtures/script/mutate-within-attr-tag/__snapshots__/mutate-within-attr-tag.expected/index.ts

@@ -33,6 +33,7 @@ export interface Input {}
         },
         [Symbol.iterator]: Marko._.any,
       },
+      // ^?
     });
     var __marko_internal_hoist__count = count;
   }