Skip to content

fix(remixer): only treat pure pointer pages as transclusion stubs #959

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jakeaturner merged 1 commit into from
Sep 2, 2026
Merged

fix(remixer): only treat pure pointer pages as transclusion stubs #959

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
69 changes: 31 additions & 38 deletions server/api/services/remixer-service.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import {
shouldSkipPage,
} from "../../util/remixerutils";
import * as cheerio from "cheerio";
import { detectTranscludeStub } from "../../util/transclusion.js";
import { RemixerSubPage } from "../../types/Remixer";
import BookService from "./book-service";
const remixerLog = childLogger("remixer");
Expand Down Expand Up @@ -1015,25 +1016,18 @@ const renamePageToIntended = async (
}
};

// Matches CrossTransclude/Web template — captures Library (group 1) and PageID (group 2)
const CROSS_TRANSLUDE_SOURCE_RE =
/template\(\s*['"]CrossTransclude\/Web['"]\s*,\s*\{[\s\S]*?['"]Library['"]\s*:\s*['"]([^'"]+)['"][\s\S]*?['"]PageID['"]\s*:\s*(\d+)/i;

// Matches rendered HTML form of content-reuse widget — captures data-page value (group 1)
const CONTENT_REUSE_WIDGET_RE =
/<div[^>]+class=["'][^"']*mt-contentreuse-widget[^"']*["'][^>]+data-page=["']([^"']+)["']/i;

// Matches raw wikitext wiki.page() call — captures path argument (group 1)
const WIKI_PAGE_REUSE_RE = /wiki\.page\s*\(\s*(?:["']|&quot;)([^"'&]+)/i;

/**
* Resolves the true source of a transcluded/reused page by inspecting its raw
* wikitext. Handles three cases:
* 1. CrossTransclude/Web template — cross-library; extracts Library + PageID directly.
* 2. Content-reuse widget HTML (data-page attribute) — same-library path reference.
* 3. wiki.page() raw wikitext — same-library path reference.
* Recursively follows the chain (e.g. a reuse that itself points at another
* reuse) until reaching a page with no transclusion, or returns the fallback.
* Resolves the true source of a transcluded page by inspecting its raw
* wikitext. `detectTranscludeStub` only reports a source when the page body is
* *nothing but* a pointer at another page, in either the cross-library
* (CrossTransclude/Web) or same-library (whole-page content reuse) form.
*
* A page that merely *embeds* content-reuse blocks owns its content and is its
* own source — resolving it to one of its embedded blocks would publish that
* block in place of the page.
*
* Recursively follows the chain (a stub that itself points at another stub)
* until reaching a page that owns its content, or returns the fallback.
*/
const resolveTranscludeSource = async ({
subdomain,
Expand Down Expand Up @@ -1077,12 +1071,20 @@ const resolveTranscludeSource = async ({

const rawContents = await rawRes.text();

// ── Case 1: CrossTransclude/Web (cross-library) ───────────────────────────
const crossMatch = rawContents.match(CROSS_TRANSLUDE_SOURCE_RE);
if (crossMatch) {
const nestedSubdomain = crossMatch[1];
const nestedId = parseInt(crossMatch[2], 10);
if (!nestedSubdomain || Number.isNaN(nestedId)) return fallback;
// A page that owns its content — including one that embeds content-reuse
// blocks — is the source. Only a pure pointer page resolves onward.
const stub = detectTranscludeStub(rawContents);
if (!stub) return fallback;

remixerLog.debug(
{ subdomain, pageId, stub },
"Resolved transclusion stub to its source",
);

// ── Cross-library (CrossTransclude/Web) ───────────────────────────────────
if (stub.kind === "cross-library") {
const nestedSubdomain = stub.subdomain;
const nestedId = stub.pageID;

const nestedHeaders = await generateAPIRequestHeaders(nestedSubdomain);
const nestedPageRes = await CXOneFetch({
Expand Down Expand Up @@ -1113,22 +1115,13 @@ const resolveTranscludeSource = async ({
});
}

// ── Cases 2 & 3: same-library content-reuse widget or wiki.page() ─────────
// data-page and wiki.page() both carry a URL path whose first segment is the
// library subdomain: /<subdomain>/<...rest> OR <subdomain>/<...rest>
const dataPageMatch = rawContents.match(CONTENT_REUSE_WIDGET_RE);
const wikiPageMatch = rawContents.match(WIKI_PAGE_REUSE_RE);
const rawSourcePath =
dataPageMatch?.[1] != null
? decodeURIComponent(dataPageMatch[1])
: wikiPageMatch?.[1] != null
? decodeURIComponent(wikiPageMatch[1])
: null;

if (!rawSourcePath) return fallback;
// ── Same-library whole-page content reuse ─────────────────────────────────
// The path carries a URL whose first segment is the library subdomain:
// /<subdomain>/<...rest> OR <subdomain>/<...rest>
// detectTranscludeStub only emits this variant with a non-empty path.

// Resolve the path to a real page ID on MindTouch
const pageInfo = await getPage(rawSourcePath, subdomain);
const pageInfo = await getPage(stub.path, subdomain);
if (!pageInfo) return fallback;

const resolvedId = parseInt(pageInfo["@id"]?.toString() ?? "", 10);
Expand Down
165 changes: 78 additions & 87 deletions server/util/Restackerutil.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,15 +5,9 @@ import { PageTag } from "../types/Book";
import { sleep } from "./helpers";
import { libraryKeys } from "./libraries";
import * as cheerio from "cheerio";
import { containsReuseMarkup, detectTranscludeStub } from "./transclusion.js";
const restackerLog = childLogger("restacker");

const CROSS_TRANSLUDE_SOURCE_RE =
/template\(\s*['"]CrossTransclude\/Web['"]\s*,\s*\{[\s\S]*?['"]Library['"]\s*:\s*['"]([^'"]+)['"][\s\S]*?['"]PageID['"]\s*:\s*(\d+)/i;
// Matches rendered HTML form of content-reuse widget
const CONTENT_REUSE_WIDGET_RE =
/<div[^>]+class=["'][^"']*mt-contentreuse-widget[^"']*["'][^>]+data-page=["'][^"']+["']/i;
// Matches raw wikitext form: wiki.page("...", NULL) stored inside <pre class="script">
const WIKI_PAGE_REUSE_RE = /wiki\.page\s*\(\s*["'&quot;]/i;
class RestackerService {
private pageTags: Map<string, PageTag[]>;

Expand Down Expand Up @@ -229,94 +223,86 @@ class RestackerService {
private async isTranscluded(
pageID: string,
library: string,
): Promise<{isTranscluded: boolean, sourceLicense: { label: string; raw: string; version: string } | undefined}> {
): Promise<{
isTranscluded: boolean;
/**
* The page carries reuse markup of any kind, stub or embedded. Broader than
* `isTranscluded` and kept separate because the quotation rate treats any
* reused content as quoted, while tagging and license inheritance apply
* only to a page that is wholly someone else's.
*/
reusesContent: boolean;
sourceLicense: { label: string; raw: string; version: string } | undefined;
}> {
const notTranscluded = (reusesContent = false) => ({
isTranscluded: false,
reusesContent,
sourceLicense: undefined,
});

try {
const bookService = new BookService({ bookID: `${library}-${pageID}` });
const rawContents = await bookService.getPageRawContent(pageID);
if (!rawContents) {
return {isTranscluded: false, sourceLicense: undefined};
return notTranscluded();
}
// Only a page whose whole body is a pointer at another page is
// transcluded. A page that merely embeds content-reuse blocks owns its
// content: tagging it `transcluded:yes` or giving it the embedded
// block's license would both be wrong.
const stub = detectTranscludeStub(rawContents);
if (!stub) {
return notTranscluded(containsReuseMarkup(rawContents));
}
// The API returns a JSON envelope; extract the HTML body for all further matching
let content = rawContents;
try {
const parsed = JSON.parse(rawContents);
if (typeof parsed?.body === "string") content = parsed.body;
} catch {}

const isCrossTranscluded =
CROSS_TRANSLUDE_SOURCE_RE.test(content);
const isContentReused =
CONTENT_REUSE_WIDGET_RE.test(content) ||
WIKI_PAGE_REUSE_RE.test(content);

if(isContentReused||isCrossTranscluded){
const tags = this.pageTags.get(this.pageTagsKey(library, pageID));
// check if transcluded tag is set
const transcludedTag = tags?.find((tag) => tag["@value"].startsWith("transcluded:"));
if(!transcludedTag){
// if not add it to the page on cxone
const existingTags = tags?.map((tag) => tag["@value"]) ?? [];
const newTags = existingTags.includes("transcluded:yes")
? existingTags
: existingTags.concat("transcluded:yes");
await bookService.updatePageDetails(pageID, undefined, newTags);
}
}

if (isContentReused) {
// Extract source page path — try data-page attribute first (rendered HTML),
// then fall back to wiki.page() argument (&quot; is the HTML-entity form of " in the body)
const dataPageMatch = content.match(/data-page=["']([^"']+)["']/i);
const wikiPageMatch = content.match(/wiki\.page\s*\(\s*(?:["']|&quot;)([^"'&]+)/i);
const rawSourcePath = dataPageMatch?.[1] ?? wikiPageMatch?.[1];

if (rawSourcePath) {
const sourcePath = decodeURIComponent(rawSourcePath);
const tags = await this.getCachedPageTags(
library,
sourcePath,
`${library}-${pageID}`,
);
const licenseTag = tags?.find((tag) => tag["@value"].startsWith("license:"));
const licenseVersionTag = tags?.find((tag) =>
tag["@value"].startsWith("licenseversion:"),
);
if (licenseTag) {
return {
isTranscluded: true,
sourceLicense: {
label: licenseTag["@value"],
raw: licenseVersionTag?.["@value"] ?? "",
version: licenseVersionTag?.["@value"] ?? "",
},
};
}
}
const tags = this.pageTags.get(this.pageTagsKey(library, pageID));
// check if transcluded tag is set
const transcludedTag = tags?.find((tag) =>
tag["@value"].startsWith("transcluded:"),
);
if (!transcludedTag) {
// if not add it to the page on cxone
const existingTags = tags?.map((tag) => tag["@value"]) ?? [];
const newTags = existingTags.includes("transcluded:yes")
? existingTags
: existingTags.concat("transcluded:yes");
await bookService.updatePageDetails(pageID, undefined, newTags);
}
if(isCrossTranscluded){

// extract the library and pageID
const crossLibrary = content.match(/Library':['"]([^'"]+)['"]/i)?.[1];
const crossPageID = content.match(/PageID':(\d+)/i)?.[1];
if (crossLibrary && crossPageID) {
const tags = await this.getCachedPageTags(
crossLibrary,
crossPageID,
`${crossLibrary}-${crossPageID}`,
);
const licenseTag = tags?.find((tag) => tag["@value"].startsWith("license:"));
const licenseVersionTag = tags?.find((tag) =>
tag["@value"].startsWith("licenseversion:"),

const sourceTags =
stub.kind === "cross-library"
? await this.getCachedPageTags(
stub.subdomain,
String(stub.pageID),
`${stub.subdomain}-${stub.pageID}`,
)
: await this.getCachedPageTags(
library,
stub.path,
`${library}-${pageID}`,
);
if(licenseTag ){
return {isTranscluded: true, sourceLicense: {label: licenseTag["@value"], raw: licenseVersionTag?.["@value"] || "", version: licenseVersionTag?.["@value"] || ""}};
}
}
}

return {isTranscluded: isCrossTranscluded || isContentReused, sourceLicense: undefined};

const licenseTag = sourceTags?.find((tag) =>
tag["@value"].startsWith("license:"),
);
const licenseVersionTag = sourceTags?.find((tag) =>
tag["@value"].startsWith("licenseversion:"),
);
if (licenseTag) {
return {
isTranscluded: true,
reusesContent: true,
sourceLicense: {
label: licenseTag["@value"],
raw: licenseVersionTag?.["@value"] ?? "",
version: licenseVersionTag?.["@value"] ?? "",
},
};
}

return { isTranscluded: true, reusesContent: true, sourceLicense: undefined };
} catch (error) {
return {isTranscluded: false, sourceLicense: undefined};
return notTranscluded();
}
}

Expand Down Expand Up @@ -384,7 +370,12 @@ class RestackerService {
}
}
const transcludedInfo = await this.isTranscluded(pageID, library);
const quotationRate = transcludedInfo.isTranscluded?1:this.getQuotationRate(page);
// A page built entirely from someone else's content is 100% quotation.
// getQuotationRate only counts lt-<library>-<id> classed text nodes, so it
// reads reuse widgets as 0 — hence the pin for any page carrying them.
const quotationRate = transcludedInfo.reusesContent
? 1
: this.getQuotationRate(page);

const licenseMap = new Map<string, { label: string; raw: string; version: string }>();
for (const license of licenses) {
Expand Down
Loading
Loading