diff --git a/server/api/services/remixer-service.ts b/server/api/services/remixer-service.ts index 9d528ffb..1331b07b 100644 --- a/server/api/services/remixer-service.ts +++ b/server/api/services/remixer-service.ts @@ -22,6 +22,7 @@ import { shouldSkipPage, } from "../../util/remixerutils"; import * as cheerio from "cheerio"; +import { detectTranscludeStub } from "../../util/transclusion.js"; import { RemixerSubPage } from "../../types/Remixer"; import BookService from "./book-service"; const remixerLog = childLogger("remixer"); @@ -1015,25 +1016,18 @@ const renamePageToIntended = async ( } }; -// Matches CrossTransclude/Web template — captures Library (group 1) and PageID (group 2) -const CROSS_TRANSLUDE_SOURCE_RE = - /template\(\s*['"]CrossTransclude\/Web['"]\s*,\s*\{[\s\S]*?['"]Library['"]\s*:\s*['"]([^'"]+)['"][\s\S]*?['"]PageID['"]\s*:\s*(\d+)/i; - -// Matches rendered HTML form of content-reuse widget — captures data-page value (group 1) -const CONTENT_REUSE_WIDGET_RE = - /]+class=["'][^"']*mt-contentreuse-widget[^"']*["'][^>]+data-page=["']([^"']+)["']/i; - -// Matches raw wikitext wiki.page() call — captures path argument (group 1) -const WIKI_PAGE_REUSE_RE = /wiki\.page\s*\(\s*(?:["']|")([^"'&]+)/i; - /** - * Resolves the true source of a transcluded/reused page by inspecting its raw - * wikitext. Handles three cases: - * 1. CrossTransclude/Web template — cross-library; extracts Library + PageID directly. - * 2. Content-reuse widget HTML (data-page attribute) — same-library path reference. - * 3. wiki.page() raw wikitext — same-library path reference. - * Recursively follows the chain (e.g. a reuse that itself points at another - * reuse) until reaching a page with no transclusion, or returns the fallback. + * Resolves the true source of a transcluded page by inspecting its raw + * wikitext. `detectTranscludeStub` only reports a source when the page body is + * *nothing but* a pointer at another page, in either the cross-library + * (CrossTransclude/Web) or same-library (whole-page content reuse) form. + * + * A page that merely *embeds* content-reuse blocks owns its content and is its + * own source — resolving it to one of its embedded blocks would publish that + * block in place of the page. + * + * Recursively follows the chain (a stub that itself points at another stub) + * until reaching a page that owns its content, or returns the fallback. */ const resolveTranscludeSource = async ({ subdomain, @@ -1077,12 +1071,20 @@ const resolveTranscludeSource = async ({ const rawContents = await rawRes.text(); - // ── Case 1: CrossTransclude/Web (cross-library) ─────────────────────────── - const crossMatch = rawContents.match(CROSS_TRANSLUDE_SOURCE_RE); - if (crossMatch) { - const nestedSubdomain = crossMatch[1]; - const nestedId = parseInt(crossMatch[2], 10); - if (!nestedSubdomain || Number.isNaN(nestedId)) return fallback; + // A page that owns its content — including one that embeds content-reuse + // blocks — is the source. Only a pure pointer page resolves onward. + const stub = detectTranscludeStub(rawContents); + if (!stub) return fallback; + + remixerLog.debug( + { subdomain, pageId, stub }, + "Resolved transclusion stub to its source", + ); + + // ── Cross-library (CrossTransclude/Web) ─────────────────────────────────── + if (stub.kind === "cross-library") { + const nestedSubdomain = stub.subdomain; + const nestedId = stub.pageID; const nestedHeaders = await generateAPIRequestHeaders(nestedSubdomain); const nestedPageRes = await CXOneFetch({ @@ -1113,22 +1115,13 @@ const resolveTranscludeSource = async ({ }); } - // ── Cases 2 & 3: same-library content-reuse widget or wiki.page() ───────── - // data-page and wiki.page() both carry a URL path whose first segment is the - // library subdomain: //<...rest> OR /<...rest> - const dataPageMatch = rawContents.match(CONTENT_REUSE_WIDGET_RE); - const wikiPageMatch = rawContents.match(WIKI_PAGE_REUSE_RE); - const rawSourcePath = - dataPageMatch?.[1] != null - ? decodeURIComponent(dataPageMatch[1]) - : wikiPageMatch?.[1] != null - ? decodeURIComponent(wikiPageMatch[1]) - : null; - - if (!rawSourcePath) return fallback; + // ── Same-library whole-page content reuse ───────────────────────────────── + // The path carries a URL whose first segment is the library subdomain: + // //<...rest> OR /<...rest> + // detectTranscludeStub only emits this variant with a non-empty path. // Resolve the path to a real page ID on MindTouch - const pageInfo = await getPage(rawSourcePath, subdomain); + const pageInfo = await getPage(stub.path, subdomain); if (!pageInfo) return fallback; const resolvedId = parseInt(pageInfo["@id"]?.toString() ?? "", 10); diff --git a/server/util/Restackerutil.ts b/server/util/Restackerutil.ts index a1e72958..cc91af63 100644 --- a/server/util/Restackerutil.ts +++ b/server/util/Restackerutil.ts @@ -5,15 +5,9 @@ import { PageTag } from "../types/Book"; import { sleep } from "./helpers"; import { libraryKeys } from "./libraries"; import * as cheerio from "cheerio"; +import { containsReuseMarkup, detectTranscludeStub } from "./transclusion.js"; const restackerLog = childLogger("restacker"); -const CROSS_TRANSLUDE_SOURCE_RE = - /template\(\s*['"]CrossTransclude\/Web['"]\s*,\s*\{[\s\S]*?['"]Library['"]\s*:\s*['"]([^'"]+)['"][\s\S]*?['"]PageID['"]\s*:\s*(\d+)/i; -// Matches rendered HTML form of content-reuse widget -const CONTENT_REUSE_WIDGET_RE = - /]+class=["'][^"']*mt-contentreuse-widget[^"']*["'][^>]+data-page=["'][^"']+["']/i; -// Matches raw wikitext form: wiki.page("...", NULL) stored inside 
-const WIKI_PAGE_REUSE_RE = /wiki\.page\s*\(\s*["'"]/i;
 class RestackerService {
   private pageTags: Map;
 
@@ -229,94 +223,86 @@ class RestackerService {
   private async isTranscluded(
     pageID: string,
     library: string,
-  ): Promise<{isTranscluded: boolean, sourceLicense: { label: string; raw: string; version: string } | undefined}> {
+  ): Promise<{
+    isTranscluded: boolean;
+    /**
+     * The page carries reuse markup of any kind, stub or embedded. Broader than
+     * `isTranscluded` and kept separate because the quotation rate treats any
+     * reused content as quoted, while tagging and license inheritance apply
+     * only to a page that is wholly someone else's.
+     */
+    reusesContent: boolean;
+    sourceLicense: { label: string; raw: string; version: string } | undefined;
+  }> {
+    const notTranscluded = (reusesContent = false) => ({
+      isTranscluded: false,
+      reusesContent,
+      sourceLicense: undefined,
+    });
+
     try {
       const bookService = new BookService({ bookID: `${library}-${pageID}` });
       const rawContents = await bookService.getPageRawContent(pageID);
       if (!rawContents) {
-        return {isTranscluded: false, sourceLicense: undefined};
+        return notTranscluded();
+      }
+      // Only a page whose whole body is a pointer at another page is
+      // transcluded. A page that merely embeds content-reuse blocks owns its
+      // content: tagging it `transcluded:yes` or giving it the embedded
+      // block's license would both be wrong.
+      const stub = detectTranscludeStub(rawContents);
+      if (!stub) {
+        return notTranscluded(containsReuseMarkup(rawContents));
       }
-      // The API returns a JSON envelope; extract the HTML body for all further matching
-      let content = rawContents;
-      try {
-        const parsed = JSON.parse(rawContents);
-        if (typeof parsed?.body === "string") content = parsed.body;
-      } catch {}
-
-      const isCrossTranscluded =
-        CROSS_TRANSLUDE_SOURCE_RE.test(content);
-      const isContentReused =
-        CONTENT_REUSE_WIDGET_RE.test(content) ||
-        WIKI_PAGE_REUSE_RE.test(content);
-
-        if(isContentReused||isCrossTranscluded){
-          const tags = this.pageTags.get(this.pageTagsKey(library, pageID));
-          // check if transcluded tag is set
-          const transcludedTag = tags?.find((tag) => tag["@value"].startsWith("transcluded:"));
-          if(!transcludedTag){
-          // if not add it to the page on cxone
-            const existingTags = tags?.map((tag) => tag["@value"]) ?? [];
-            const newTags = existingTags.includes("transcluded:yes")
-              ? existingTags
-              : existingTags.concat("transcluded:yes");
-            await bookService.updatePageDetails(pageID, undefined, newTags);
-          }
-        }
 
-      if (isContentReused) {
-        // Extract source page path — try data-page attribute first (rendered HTML),
-        // then fall back to wiki.page() argument (" is the HTML-entity form of " in the body)
-        const dataPageMatch = content.match(/data-page=["']([^"']+)["']/i);
-        const wikiPageMatch = content.match(/wiki\.page\s*\(\s*(?:["']|")([^"'&]+)/i);
-        const rawSourcePath = dataPageMatch?.[1] ?? wikiPageMatch?.[1];
-
-        if (rawSourcePath) {
-          const sourcePath = decodeURIComponent(rawSourcePath);
-          const tags = await this.getCachedPageTags(
-            library,
-            sourcePath,
-            `${library}-${pageID}`,
-          );
-          const licenseTag = tags?.find((tag) => tag["@value"].startsWith("license:"));
-          const licenseVersionTag = tags?.find((tag) =>
-            tag["@value"].startsWith("licenseversion:"),
-          );
-          if (licenseTag) {
-            return {
-              isTranscluded: true,
-              sourceLicense: {
-                label: licenseTag["@value"],
-                raw: licenseVersionTag?.["@value"] ?? "",
-                version: licenseVersionTag?.["@value"] ?? "",
-              },
-            };
-          }
-        }
+      const tags = this.pageTags.get(this.pageTagsKey(library, pageID));
+      // check if transcluded tag is set
+      const transcludedTag = tags?.find((tag) =>
+        tag["@value"].startsWith("transcluded:"),
+      );
+      if (!transcludedTag) {
+        // if not add it to the page on cxone
+        const existingTags = tags?.map((tag) => tag["@value"]) ?? [];
+        const newTags = existingTags.includes("transcluded:yes")
+          ? existingTags
+          : existingTags.concat("transcluded:yes");
+        await bookService.updatePageDetails(pageID, undefined, newTags);
       }
-      if(isCrossTranscluded){
-        
-          // extract the library and pageID
-          const crossLibrary = content.match(/Library':['"]([^'"]+)['"]/i)?.[1];
-          const crossPageID = content.match(/PageID':(\d+)/i)?.[1];
-          if (crossLibrary && crossPageID) {
-            const tags = await this.getCachedPageTags(
-              crossLibrary,
-              crossPageID,
-              `${crossLibrary}-${crossPageID}`,
-            );
-            const licenseTag = tags?.find((tag) => tag["@value"].startsWith("license:"));
-            const licenseVersionTag = tags?.find((tag) =>
-              tag["@value"].startsWith("licenseversion:"),
+
+      const sourceTags =
+        stub.kind === "cross-library"
+          ? await this.getCachedPageTags(
+              stub.subdomain,
+              String(stub.pageID),
+              `${stub.subdomain}-${stub.pageID}`,
+            )
+          : await this.getCachedPageTags(
+              library,
+              stub.path,
+              `${library}-${pageID}`,
             );
-            if(licenseTag ){
-              return {isTranscluded: true, sourceLicense: {label: licenseTag["@value"], raw: licenseVersionTag?.["@value"] || "", version: licenseVersionTag?.["@value"] || ""}};
-            }
-          }
-        }
-   
-      return {isTranscluded: isCrossTranscluded || isContentReused, sourceLicense: undefined};
+
+      const licenseTag = sourceTags?.find((tag) =>
+        tag["@value"].startsWith("license:"),
+      );
+      const licenseVersionTag = sourceTags?.find((tag) =>
+        tag["@value"].startsWith("licenseversion:"),
+      );
+      if (licenseTag) {
+        return {
+          isTranscluded: true,
+          reusesContent: true,
+          sourceLicense: {
+            label: licenseTag["@value"],
+            raw: licenseVersionTag?.["@value"] ?? "",
+            version: licenseVersionTag?.["@value"] ?? "",
+          },
+        };
+      }
+
+      return { isTranscluded: true, reusesContent: true, sourceLicense: undefined };
     } catch (error) {
-      return {isTranscluded: false, sourceLicense: undefined};
+      return notTranscluded();
     }
   }
 
@@ -384,7 +370,12 @@ class RestackerService {
       }
     }
     const transcludedInfo = await this.isTranscluded(pageID, library);
-    const quotationRate = transcludedInfo.isTranscluded?1:this.getQuotationRate(page);
+    // A page built entirely from someone else's content is 100% quotation.
+    // getQuotationRate only counts lt-- classed text nodes, so it
+    // reads reuse widgets as 0 — hence the pin for any page carrying them.
+    const quotationRate = transcludedInfo.reusesContent
+      ? 1
+      : this.getQuotationRate(page);
 
     const licenseMap = new Map();
     for (const license of licenses) {
diff --git a/server/util/transclusion.ts b/server/util/transclusion.ts
new file mode 100644
index 00000000..d2324d15
--- /dev/null
+++ b/server/util/transclusion.ts
@@ -0,0 +1,231 @@
+import * as cheerio from "cheerio";
+
+/**
+ * Detection for MindTouch transclusion "stub" pages — pages whose entire body is
+ * a pointer at another page, as produced by `RemixerTemplates.POST_Transclude*`.
+ *
+ * The distinction that matters: a page may *contain* content-reuse blocks
+ * (an authored page pulling in a shared definition, a prerequisites list, etc.)
+ * without *being* a transclusion of anything. Treating the two the same makes a
+ * copy of the page resolve to whatever its first embedded block happened to
+ * point at, discarding the page's real content.
+ */
+
+/** Matches CrossTransclude/Web — captures Library (group 1) and PageID (group 2). */
+export const CROSS_TRANSCLUDE_SOURCE_RE =
+  /template\(\s*['"]CrossTransclude\/Web['"]\s*,\s*\{[\s\S]*?['"]Library['"]\s*:\s*['"]([^'"]+)['"][\s\S]*?['"]PageID['"]\s*:\s*(\d+)/i;
+
+/** Matches the rendered content-reuse widget — captures the data-page value (group 1). */
+export const CONTENT_REUSE_WIDGET_RE =
+  /]+class=["'][^"']*mt-contentreuse-widget[^"']*["'][^>]+data-page=["']([^"']+)["']/i;
+
+/** Matches a raw wikitext `wiki.page("path", ...)` call — captures the path (group 1). */
+export const WIKI_PAGE_REUSE_RE =
+  /wiki\.page\s*\(\s*(?:["']|")([^"'&]+)/i;
+
+/**
+ * Whole-page form only: the second argument is NULL. A string second argument
+ * names a section, which means the call pulls a fragment into a larger page.
+ */
+const WIKI_PAGE_WHOLE_RE =
+  /wiki\.page\s*\(\s*(?:["']|")([^"'&]+)(?:["']|")\s*,\s*NULL\s*\)?/i;
+
+/**
+ * MindTouch stores page paths only partially encoded — the fixtures carry raw
+ * `:` in `data-page` — so a title containing a bare `%` reaches us as an
+ * invalid escape and `decodeURIComponent` throws. A path we cannot decode is
+ * still usable as-is, and is far better than an exception unwinding into the
+ * remixer publish job.
+ */
+const safeDecodePath = (value: string): string => {
+  try {
+    return decodeURIComponent(value);
+  } catch {
+    return value;
+  }
+};
+
+export type TranscludeStub =
+  | { kind: "cross-library"; subdomain: string; pageID: number }
+  | { kind: "same-library"; path: string };
+
+/**
+ * The page-contents API sometimes hands back a JSON envelope rather than bare
+ * HTML. Both callers need the HTML body, so unwrap once here.
+ */
+export const unwrapPageBody = (rawContents: string): string => {
+  try {
+    const parsed = JSON.parse(rawContents);
+    const body = parsed?.body;
+    if (typeof body === "string") return body;
+    if (Array.isArray(body) && typeof body[0] === "string") return body[0];
+  } catch {
+    // Not JSON — already raw HTML.
+  }
+  return rawContents;
+};
+
+/**
+ * True when the body carries any transclusion or content-reuse markup at all,
+ * whether the page is a stub or an authored page embedding reused blocks.
+ */
+export const containsReuseMarkup = (rawContents: string): boolean => {
+  const html = unwrapPageBody(rawContents);
+  return (
+    CROSS_TRANSCLUDE_SOURCE_RE.test(html) ||
+    CONTENT_REUSE_WIDGET_RE.test(html) ||
+    WIKI_PAGE_REUSE_RE.test(html)
+  );
+};
+
+const countOccurrences = (haystack: string, needle: string): number => {
+  let count = 0;
+  let index = haystack.indexOf(needle);
+  while (index !== -1) {
+    count += 1;
+    index = haystack.indexOf(needle, index + needle.length);
+  }
+  return count;
+};
+
+/** Scaffolding a generated stub is allowed to carry alongside its pointer. */
+const STUB_SCAFFOLD_SELECTORS = [
+  // Attribute form, not an escaped-colon class selector — css-select rejects
+  // `.template\:tag-insert`.
+  'p[class~="template:tag-insert"]',
+  "p.mt-script-comment",
+  "div.comment",
+  "script",
+  "style",
+];
+
+/**
+ * Layout wrappers a WYSIWYG editor leaves behind. Only ignorable when
+ * completely empty \u2014 no text, no attributes, no children. Anything else,
+ * including an `
` or a `
`, is authored content.
+ */
+const EMPTY_LAYOUT_TAGS = new Set(["p", "div", "span", "br"]);
+
+/**
+ * True when anything survives after the pointer and its scaffolding are
+ * removed. Enumerating "content" element types is a losing game \u2014 ``,
+ * ``, `
`, and form controls are all real, text-free content \u2014 so
+ * the rule is inverted: every remaining element counts unless it is a provably
+ * empty layout wrapper. Over-counting only costs a chain collapse; under-
+ * counting publishes a pointer in place of a real page.
+ */
+const hasMeaningfulRemainder = ($: cheerio.CheerioAPI): boolean => {
+  const text = ($.root().text() ?? "").replace(/[\s\u00a0]+/g, "");
+  if (text.length > 0) return true;
+
+  return (
+    $.root()
+      .find("*")
+      .toArray()
+      .some((el) => {
+        const node = $(el);
+        const tag = (el as { tagName?: string }).tagName?.toLowerCase() ?? "";
+        if (!EMPTY_LAYOUT_TAGS.has(tag)) return true;
+        if (Object.keys(node.attr() ?? {}).length > 0) return true;
+        return node.children().length > 0;
+      })
+  );
+};
+
+/**
+ * Returns a stub descriptor iff the body is *nothing but* transclusion
+ * machinery. Any authored content alongside the pointer, or a reuse widget that
+ * names a section, means the page owns its content and is its own source.
+ */
+export const detectTranscludeStub = (
+  rawContents: string,
+): TranscludeStub | null => {
+  const html = unwrapPageBody(rawContents);
+  if (!html.trim()) return null;
+
+  // An unterminated `")) {
+    return null;
+  }
+
+  // Fragment parse so the body is not wrapped in .
+  const $ = cheerio.load(html, null, false);
+
+  // Drop comments through the DOM rather than by regex on the source. A regex
+  // mis-handles `