⬆️ Add simple upgrade/downgrade package for MyST AST (#1802)

Co-authored-by: Rowan Cockett <[email protected]>

Author: agoose77

.changeset/eight-tools-live.md

@@ -0,0 +1,9 @@
+---
+"myst-transforms": patch
+"myst-spec-ext": patch
+"myst-to-docx": patch
+"myst-config": patch
+"myst-cli": patch
+---
+
+Change footnotes to use enumerator over number

.changeset/eleven-keys-explain.md

@@ -0,0 +1,5 @@
+---
+"myst-config": patch
+---
+
+Add version to config file

.changeset/red-suits-hug.md

@@ -0,0 +1,5 @@
+---
+"myst-cli": patch
+---
+
+Add version to site content outputs

.changeset/sour-spiders-push.md

@@ -0,0 +1,5 @@
+---
+'myst-migrate': patch
+---
+
+Add myst-migrate package

docs/myst.yml

@@ -153,6 +153,7 @@ project:
         - file: document-parts.md
         - file: settings.md
         - file: glossary.md
+        - file: versions.md
     - title: Contribute
       children:
         - file: contributing.md

docs/versions.md

@@ -0,0 +1,18 @@
+---
+title: MyST Content Versions
+---
+
+The following page describes changes to the MyST content that is served from the `.json` endpoint of a MyST Site. The `myst-compat` package can be used to translate between versions by upgrading and downgrading between versions.
+
+The version is a string integer (i.e. `'1'` or `'2'`) and is incremented with every change to the content of a MyST page, which includes metadata as well as the MyST AST.
+
+# MyST Versions
+
+## Version 1 - 2025-02-07 - Footnote Numbering
+
+The footnotes have dropped backwards compatibility with `number`, instead using `enumerator` on both the `FootnoteReference` and `FootnoteDefinition` nodes.
+Previous versions of the AST had both of these defined. The `enumerator` property is used in all other numberings of figures, sections, equations, etc.
+
+## Version 0 - Pre 2025-02-01
+
+This is the first version of MyST AST considered to be versioned, subsequent releases will have changes for migrating the content between versions.

package-lock.json

@@ -15,7 +15,6 @@ import type { ExportWithOutput, ExportFnOptions } from './types.js';
 import { cleanOutput } from './utils/cleanOutput.js';
 import { getFileContent } from './utils/getFileContent.js';
 import { resolveFrontmatterParts } from '../utils/resolveFrontmatterParts.js';
-import { parseMyst } from '../process/myst.js';
 
 function exportOptionsToCFF(exportOptions: ExportWithOutput): CFF {
   // Handle overlap of key "format" between CFF and export

packages/myst-cli/src/build/cff.ts

@@ -26,6 +26,7 @@ import { collectExportOptions } from '../utils/collectExportOptions.js';
 import { filterPages } from '../../project/load.js';
 import { getRawFrontmatterFromFile } from '../../process/file.js';
 import { castSession } from '../../session/cache.js';
+import { SPEC_VERSION } from '../../spec-version.js';
 
 type ManifestProject = Required<SiteManifest>['projects'][0];
 
@@ -417,9 +418,10 @@ export async function getSiteManifest(
   validatedFrontmatter.options = resolvedOptions;
   const parts = resolveFrontmatterParts(session, validatedFrontmatter);
   const manifest: SiteManifest = {
+    version: SPEC_VERSION,
+    myst: version,
     ...validatedFrontmatter,
     parts,
-    myst: version,
     nav: nav || [],
     actions: actions || [],
     projects: siteProjects,

packages/myst-cli/src/build/site/manifest.ts

@@ -10,4 +10,5 @@ export * from './session/index.js';
 export * from './store/index.js';
 export * from './transforms/index.js';
 export * from './utils/index.js';
+export * from './spec-version.js';
 export { default as version } from './version.js';

packages/myst-cli/src/index.ts

@@ -45,6 +45,7 @@ import { loadReferences } from './loadReferences.js';
 import type { TransformFn } from './mdast.js';
 import { finalizeMdast, postProcessMdast, transformMdast } from './mdast.js';
 import { toSectionedParts, buildHierarchy, sectionToHeadingLevel } from './search.js';
+import { SPEC_VERSION } from '../spec-version.js';
 
 const WEB_IMAGE_EXTENSIONS = [
   ImageExtensions.mp4,
@@ -412,6 +413,7 @@ export async function writeFile(
   const parts = resolveFrontmatterParts(session, frontmatter);
   const frontmatterWithExports = { ...frontmatter, exports, downloads, parts };
   const mystData: MystData = {
+    version: SPEC_VERSION,
     kind,
     sha256,
     slug,

packages/myst-cli/src/process/site.ts

@@ -0,0 +1 @@
+export const SPEC_VERSION = 1;

packages/myst-cli/src/spec-version.ts

@@ -17,6 +17,7 @@ function mystDataFilename(dataUrl: string) {
 }
 
 export type MystData = {
+  version: number;
   kind?: SourceFileKind;
   sha256?: string;
   slug?: string;

packages/myst-cli/src/transforms/crossReferences.ts

@@ -71,6 +71,7 @@ type ManifestProject = {
 } & Omit<ProjectFrontmatter, 'downloads' | 'exports' | 'parts'>;
 
 export type SiteManifest = Omit<SiteFrontmatter, 'parts'> & {
+  version: number;
   myst: string;
   id?: string;
   projects?: ManifestProject[];

packages/myst-config/src/site/types.ts

@@ -0,0 +1,4 @@
+module.exports = {
+  root: true,
+  extends: ['curvenote'],
+};

packages/myst-migrate/.eslintrc.cjs

@@ -0,0 +1,3 @@
+# myst-migrate
+
+Utilities for upgrading and downgrading MyST ASTs

packages/myst-migrate/README.md

@@ -0,0 +1,30 @@
+{
+  "name": "myst-migrate",
+  "sideEffects": false,
+  "version": "0.0.0",
+  "type": "module",
+  "exports": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "files": [
+    "dist"
+  ],
+  "license": "MIT",
+  "scripts": {
+    "clean": "rimraf dist",
+    "lint": "eslint \"src/**/!(*.spec).ts\" -c ./.eslintrc.cjs",
+    "lint:format": "prettier --check \"src/**/*.{ts,tsx,md}\"",
+    "test": "vitest run",
+    "test:watch": "vitest watch",
+    "build:esm": "tsc",
+    "build": "npm-run-all -l clean -p build:esm"
+  },
+  "dependencies": {
+    "unist-util-select": "^4.0.3",
+    "unist-util-visit": "^4.1.2",
+    "vfile": "^5.0.0",
+    "vfile-message": "^3.0.0"
+  },
+  "devDependencies": {
+    "@jupyterlab/nbformat": "^3.5.2"
+  }
+}

packages/myst-migrate/package.json

@@ -0,0 +1,50 @@
+import type { IFile, Options } from './types.js';
+import { MIGRATIONS } from './migrations.js';
+
+export { MIGRATIONS } from './migrations.js';
+
+/**
+ * Migrate function:
+ * @param src - The page to be migrated
+ * @param options - to: desired target version, log: Logger
+ */
+export async function migrate(src: IFile, opts?: Options): Promise<IFile> {
+  const to = opts?.to ?? MIGRATIONS.length;
+  let currentVersion = src.version || 0;
+
+  // If the current version is already at the target, do nothing
+  if (currentVersion === to) {
+    opts?.log?.debug(`Already at version ${to}. No migration needed.`);
+    return src;
+  }
+
+  // If currentVersion < toVersion, apply upgrades forward
+  while (currentVersion < to) {
+    if (currentVersion >= MIGRATIONS.length) {
+      throw new Error(
+        `No migration available to go from version ${currentVersion} to ${currentVersion + 1}`,
+      );
+    }
+    const migration = MIGRATIONS[currentVersion];
+    opts?.log?.debug(`Upgrading from v${currentVersion} to v${currentVersion + 1}...`);
+    await migration.upgrade(src);
+    currentVersion++;
+    src.version = currentVersion;
+  }
+
+  // If currentVersion > toVersion, apply downgrades backward
+  while (currentVersion > to) {
+    if (currentVersion - 1 >= MIGRATIONS.length) {
+      throw new Error(
+        `No migration available to go from version ${currentVersion} down to ${currentVersion - 1}`,
+      );
+    }
+    const migration = MIGRATIONS[currentVersion - 1];
+    opts?.log?.debug(`Downgrading from v${currentVersion} to v${currentVersion - 1}...`);
+    await migration.downgrade(src);
+    currentVersion--;
+    src.version = currentVersion;
+  }
+
+  return src;
+}

packages/myst-migrate/src/index.ts

@@ -0,0 +1,4 @@
+import type { Migration } from './types.js';
+import * as v1 from './v1_footnotes.js';
+
+export const MIGRATIONS: Migration[] = [v1];