📓 Add ipynb as export format

.changeset/config.json

@@ -6,6 +6,7 @@
     ["myst-common", "myst-config", "myst-frontmatter", "myst-spec-ext"],
     ["myst-to-jats", "jats-to-myst"],
     ["myst-to-tex", "tex-to-myst"],
+    ["myst-to-md", "myst-to-ipynb"],
     ["myst-parser", "myst-roles", "myst-directives", "myst-to-html"],
     ["mystmd", "myst-cli", "myst-migrate"]
   ],

.changeset/witty-tigers-hunt.md

@@ -0,0 +1,7 @@
+---
+"myst-frontmatter": patch
+"myst-to-ipynb": patch
+"myst-cli": patch
+---
+
+Add ipynb as export format

packages/myst-cli/package.json

@@ -85,6 +85,7 @@
     "myst-spec": "^0.0.5",
     "myst-spec-ext": "^1.7.11",
     "myst-templates": "^1.0.25",
+    "myst-to-ipynb": "^1.0.15",
     "myst-to-docx": "^1.0.14",
     "myst-to-jats": "^1.0.34",
     "myst-to-md": "^1.0.15",

packages/myst-cli/src/build/build.spec.ts

@@ -36,6 +36,7 @@ describe('get export formats', () => {
       ExportFormats.tex,
       ExportFormats.xml,
       ExportFormats.md,
+      ExportFormats.ipynb,
       ExportFormats.meca,
       ExportFormats.cff,
     ]);

packages/myst-cli/src/build/build.ts

@@ -26,6 +26,7 @@ type FormatBuildOpts = {
   typst?: boolean;
   xml?: boolean;
   md?: boolean;
+  ipynb?: boolean;
   meca?: boolean;
   cff?: boolean;
   html?: boolean;
@@ -37,8 +38,8 @@ type FormatBuildOpts = {
 export type BuildOpts = FormatBuildOpts & CollectionOptions & RunExportOptions & StartOptions;
 
 export function hasAnyExplicitExportFormat(opts: BuildOpts): boolean {
-  const { docx, pdf, tex, typst, xml, md, meca, cff } = opts;
-  return docx || pdf || tex || typst || xml || md || meca || cff || false;
+  const { docx, pdf, tex, typst, xml, md, ipynb, meca, cff } = opts;
+  return docx || pdf || tex || typst || xml || md || ipynb || meca || cff || false;
 }
 
 /**
@@ -50,12 +51,13 @@ export function hasAnyExplicitExportFormat(opts: BuildOpts): boolean {
  * @param opts.typst
  * @param opts.xml
  * @param opts.md
+ * @param opts.ipynb
  * @param opts.meca
  * @param opts.all all exports requested with --all option
  * @param opts.explicit explicit input file was provided
  */
 export function getAllowedExportFormats(opts: FormatBuildOpts & { explicit?: boolean }) {
-  const { docx, pdf, tex, typst, xml, md, meca, cff, all, explicit } = opts;
+  const { docx, pdf, tex, typst, xml, md, ipynb, meca, cff, all, explicit } = opts;
   const formats = [];
   const any = hasAnyExplicitExportFormat(opts);
   const override = all || (!any && explicit);
@@ -69,6 +71,7 @@ export function getAllowedExportFormats(opts: FormatBuildOpts & { explicit?: boo
   if (typst || override) formats.push(ExportFormats.typst);
   if (xml || override) formats.push(ExportFormats.xml);
   if (md || override) formats.push(ExportFormats.md);
+  if (ipynb || override) formats.push(ExportFormats.ipynb);
   if (meca || override) formats.push(ExportFormats.meca);
   if (cff || override) formats.push(ExportFormats.cff);
   return [...new Set(formats)];
@@ -78,14 +81,15 @@ export function getAllowedExportFormats(opts: FormatBuildOpts & { explicit?: boo
  * Return requested formats from CLI options
  */
 export function getRequestedExportFormats(opts: FormatBuildOpts) {
-  const { docx, pdf, tex, typst, xml, md, meca, cff } = opts;
+  const { docx, pdf, tex, typst, xml, md, ipynb, meca, cff } = opts;
   const formats = [];
   if (docx) formats.push(ExportFormats.docx);
   if (pdf) formats.push(ExportFormats.pdf);
   if (tex) formats.push(ExportFormats.tex);
   if (typst) formats.push(ExportFormats.typst);
   if (xml) formats.push(ExportFormats.xml);
   if (md) formats.push(ExportFormats.md);
+  if (ipynb) formats.push(ExportFormats.ipynb);
   if (meca) formats.push(ExportFormats.meca);
   if (cff) formats.push(ExportFormats.cff);
   return formats;
@@ -239,7 +243,8 @@ export async function build(session: ISession, files: string[], opts: BuildOpts)
       // Print out the kinds that are filtered
       const kinds = Object.entries(opts)
         .filter(
-          ([k, v]) => ['docx', 'pdf', 'tex', 'typst', 'xml', 'md', 'meca', 'cff'].includes(k) && v,
+          ([k, v]) =>
+            ['docx', 'pdf', 'tex', 'typst', 'xml', 'md', 'ipynb', 'meca', 'cff'].includes(k) && v,
         )
         .map(([k]) => k);
       session.log.info(

packages/myst-cli/src/build/ipynb/index.ts

@@ -0,0 +1,51 @@
+import path from 'node:path';
+import { tic, writeFileToFolder } from 'myst-cli-utils';
+import { FRONTMATTER_ALIASES, PAGE_FRONTMATTER_KEYS } from 'myst-frontmatter';
+import { writeIpynb } from 'myst-to-ipynb';
+import { filterKeys } from 'simple-validators';
+import { VFile } from 'vfile';
+import { finalizeMdast } from '../../process/mdast.js';
+import type { ISession } from '../../session/types.js';
+import { logMessagesFromVFile } from '../../utils/logging.js';
+import { KNOWN_IMAGE_EXTENSIONS } from '../../utils/resolveExtension.js';
+import type { ExportWithOutput, ExportFnOptions } from '../types.js';
+import { cleanOutput } from '../utils/cleanOutput.js';
+import { getFileContent } from '../utils/getFileContent.js';
+
+export async function runIpynbExport(
+  session: ISession,
+  sourceFile: string,
+  exportOptions: ExportWithOutput,
+  opts?: ExportFnOptions,
+) {
+  const toc = tic();
+  const { output, articles } = exportOptions;
+  const { clean, projectPath, extraLinkTransformers, execute } = opts ?? {};
+  // At this point, export options are resolved to contain one-and-only-one article
+  const article = articles[0];
+  if (!article?.file) return { tempFolders: [] };
+  if (clean) cleanOutput(session, output);
+  const [{ mdast, frontmatter }] = await getFileContent(session, [article.file], {
+    projectPath,
+    imageExtensions: KNOWN_IMAGE_EXTENSIONS,
+    extraLinkTransformers,
+    preFrontmatters: [
+      filterKeys(article, [...PAGE_FRONTMATTER_KEYS, ...Object.keys(FRONTMATTER_ALIASES)]),
+    ],
+    execute,
+  });
+  await finalizeMdast(session, mdast, frontmatter, article.file, {
+    imageWriteFolder: path.join(path.dirname(output), 'files'),
+    imageAltOutputFolder: 'files/',
+    imageExtensions: KNOWN_IMAGE_EXTENSIONS,
+    simplifyFigures: false,
+    useExistingImages: true,
+  });
+  const vfile = new VFile();
+  vfile.path = output;
+  const mdOut = writeIpynb(vfile, mdast as any, frontmatter);
+  logMessagesFromVFile(session, mdOut);
+  session.log.info(toc(`📑 Exported MD in %s, copying to ${output}`));
+  writeFileToFolder(output, mdOut.result as string);
+  return { tempFolders: [] };
+}

packages/myst-cli/src/build/utils/collectExportOptions.ts

@@ -271,6 +271,7 @@ export function resolveArticles(
 export const ALLOWED_EXTENSIONS: Record<ExportFormats, string[]> = {
   [ExportFormats.docx]: ['.doc', '.docx'],
   [ExportFormats.md]: ['.md'],
+  [ExportFormats.ipynb]: ['.ipynb'],
   [ExportFormats.meca]: ['.zip', '.meca'],
   [ExportFormats.pdf]: ['.pdf'],
   [ExportFormats.pdftex]: ['.pdf', '.tex', '.zip'],

packages/myst-cli/src/build/utils/localArticleExport.ts

@@ -20,6 +20,7 @@ import { texExportOptionsFromPdf } from '../pdf/single.js';
 import { createPdfGivenTexExport } from '../pdf/create.js';
 import { runMecaExport } from '../meca/index.js';
 import { runMdExport } from '../md/index.js';
+import { runIpynbExport } from '../ipynb/index.js';
 import { selectors, watch as watchReducer } from '../../store/index.js';
 import { runCffExport } from '../cff.js';
 
@@ -113,6 +114,8 @@ async function _localArticleExport(
         exportFn = runJatsExport;
       } else if (format === ExportFormats.md) {
         exportFn = runMdExport;
+      } else if (format === ExportFormats.ipynb) {
+        exportFn = runIpynbExport;
       } else if (format === ExportFormats.meca) {
         exportFn = runMecaExport;
       } else if (format === ExportFormats.cff) {

packages/myst-cli/src/cli/build.ts

@@ -20,6 +20,7 @@ import {
   makeMaxSizeWebpOption,
   makeDOIBibOption,
   makeCffOption,
+  makeIpynbOption,
 } from './options.js';
 import { readableName } from '../utils/whiteLabelling.js';
 
@@ -33,6 +34,7 @@ export function makeBuildCommand() {
     .addOption(makeTypstOption('Build Typst outputs'))
     .addOption(makeDocxOption('Build Docx output'))
     .addOption(makeMdOption('Build MD output'))
+    .addOption(makeIpynbOption('Build IPYNB output'))
     .addOption(makeJatsOption('Build JATS xml output'))
     .addOption(makeMecaOptions('Build MECA zip output'))
     .addOption(makeCffOption('Build CFF output'))

packages/myst-cli/src/cli/options.ts

@@ -28,6 +28,10 @@ export function makeMdOption(description: string) {
   return new Option('--md', description).default(false);
 }
 
+export function makeIpynbOption(description: string) {
+  return new Option('--ipynb', description).default(false);
+}
+
 export function makeJatsOption(description: string) {
   return new Option('--jats, --xml', description).default(false);
 }

packages/myst-frontmatter/src/exports/types.ts

@@ -8,6 +8,7 @@ export enum ExportFormats {
   docx = 'docx',
   xml = 'xml',
   md = 'md',
+  ipynb = 'ipynb',
   meca = 'meca',
   cff = 'cff',
 }

packages/myst-to-ipynb/.eslintrc.cjs

@@ -0,0 +1,4 @@
+module.exports = {
+  root: true,
+  extends: ['curvenote'],
+};

packages/myst-to-ipynb/CHANGELOG.md

@@ -0,0 +1 @@
+# myst-to-ipynb

packages/myst-to-ipynb/README.md

@@ -0,0 +1,3 @@
+# myst-to-ipynb
+
+Convert a MyST AST to ipynb notebook.

packages/myst-to-ipynb/package.json

@@ -0,0 +1,50 @@
+{
+  "name": "myst-to-ipynb",
+  "version": "1.0.15",
+  "description": "Export from MyST mdast to ipynb",
+  "author": "Rowan Cockett <[email protected]>",
+  "homepage": "https://github.com/jupyter-book/mystmd/tree/main/packages/myst-to-md",
+  "license": "MIT",
+  "type": "module",
+  "exports": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "files": [
+    "src",
+    "dist"
+  ],
+  "keywords": [
+    "myst-plugin",
+    "markdown"
+  ],
+  "publishConfig": {
+    "access": "public"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/jupyter-book/mystmd.git"
+  },
+  "scripts": {
+    "clean": "rimraf dist",
+    "lint": "eslint \"src/**/*.ts\" -c .eslintrc.cjs --max-warnings 1",
+    "lint:format": "prettier --check src/*.ts src/**/*.ts",
+    "test": "vitest run",
+    "test:watch": "vitest watch",
+    "build:esm": "tsc",
+    "build": "npm-run-all -l clean -p build:esm"
+  },
+  "bugs": {
+    "url": "https://github.com/jupyter-book/mystmd/issues"
+  },
+  "dependencies": {
+    "js-yaml": "^4.1.0",
+    "mdast-util-gfm-footnote": "^1.0.2",
+    "mdast-util-gfm-table": "^1.0.7",
+    "mdast-util-to-markdown": "^1.5.0",
+    "myst-common": "^1.7.6",
+    "myst-frontmatter": "^1.7.6",
+    "myst-to-md": "^1.0.15",
+    "unist-util-select": "^4.0.3",
+    "vfile": "^5.3.7",
+    "vfile-reporter": "^7.0.4"
+  }
+}

packages/myst-to-ipynb/src/index.ts

@@ -0,0 +1,61 @@
+import type { Root } from 'myst-spec';
+import type { Block, Code } from 'myst-spec-ext';
+import type { Plugin } from 'unified';
+import type { VFile } from 'vfile';
+import type { PageFrontmatter } from 'myst-frontmatter';
+import { writeMd } from 'myst-to-md';
+import { select } from 'unist-util-select';
+
+function sourceToStringList(src: string): string[] {
+  const lines = src.split('\n').map((s) => `${s}\n`);
+  lines[lines.length - 1] = lines[lines.length - 1].trimEnd();
+  return lines;
+}
+
+export function writeIpynb(file: VFile, node: Root, frontmatter?: PageFrontmatter) {
+  const cells = (node.children as Block[]).map((block: Block) => {
+    if (block.type === 'block' && block.kind === 'notebook-code') {
+      const code = select('code', block) as Code;
+      return {
+        cell_type: 'code',
+        execution_count: null,
+        metadata: {},
+        outputs: [],
+        source: sourceToStringList(code.value),
+      };
+    }
+    const md = writeMd(file, { type: 'root', children: [block] }).result as string;
+    return {
+      cell_type: 'markdown',
+      metadata: {},
+      source: sourceToStringList(md),
+    };
+  });
+
+  const ipynb = {
+    cells,
+    metadata: {
+      language_info: {
+        name: 'python',
+      },
+    },
+    nbformat: 4,
+    nbformat_minor: 2,
+  };
+
+  file.result = JSON.stringify(ipynb, null, 2);
+  return file;
+}
+
+const plugin: Plugin<[PageFrontmatter?], Root, VFile> = function (frontmatter?) {
+  this.Compiler = (node, file) => {
+    return writeIpynb(file, node, frontmatter);
+  };
+
+  return (node: Root) => {
+    // Preprocess
+    return node;
+  };
+};
+
+export default plugin;