Add source map symbolication and source view support (#6018)

The bugzilla part of this PR:
https://bugzilla.mozilla.org/show_bug.cgi?id=2035493

This requires a Firefox that has patches applied in [bug
2035493](https://bugzilla.mozilla.org/show_bug.cgi?id=2035493). And also
it requires the "JavaScript Sources" features to be on.

It adds a pipeline that maps compiled/bundled JS frame positions back to
their original source files using source maps fetched from the browser
after profile load, similarly to our native symbolication step. And it
adds a way to see the source mapped source contents.

After this PR, the call tree, flame graph tooltip, source view, and line
timings show original TS/JS positions and function names for frames
recorded in minified bundles.

I split the work in multiple patches so it's easier to review, but
admittedly it's a lot of changes. The initial commits in the PR don't
change the behavior until the commit that wires everything up and
updates the visualization.

Example STR:
- Use a Firefox that includes my patches. And enable the "JavaScript
Sources" feature in about:profiling.
- Start the profiler
- Load an example website that has JS source maps, for example the
Firefox Profiler frontend
- Capture a profile
- Wait until symbolication and source map resolution is complete

You should then see the symbolicated functions on the frames that are
coming from Firefox Profiler source code. (Note that the react code will
still be minified as it doesn't have source maps). You should also be
able to double click on a JS function function to be able to see the
source mapped JS source.

Author: canova

docs-developer/CHANGELOG-formats.md

@@ -6,6 +6,21 @@ Note that this is not an exhaustive list. Processed profile format upgraders can
 
 ## Processed profile format
 
+### Version 64
+
+A new `SourceLocationTable` has been added to `profile.shared.sourceLocationTable`. It holds the original (pre-compilation) source positions produced by source map symbolication, paired with the generated `line`/`column` already on `FrameTable`.
+
+- `source: IndexIntoSourceTable[]`: source file index. Set independently for func entries (the function's definition file) and frame entries (the execution point's file).
+- `line: number[]`: 1-based line number
+- `column: number[]`: 1-based column number
+
+Two new columns were added that index into this table:
+
+- `FrameTable.originalLocation: Array<IndexIntoSourceLocationTable | null>`: the original execution point for the frame
+- `FuncTable.originalLocation: Array<IndexIntoSourceLocationTable | null>`: the original definition site for the function
+
+A new `content: Array<string | null>` column was added to `SourceTable`.
+
 ### Version 63
 
 A new `tooltipRows` field was added to `CounterDisplayConfig`.

jest.config.js

@@ -30,6 +30,7 @@ const browserEnvConfig = {
 
   globals: {
     AVAILABLE_STAGING_LOCALES: null,
+    SOURCE_MAP_WORKER_PATH: 'src/test/fixtures/source-map.worker.stub.js',
   },
 
   snapshotFormat: {

package.json

@@ -76,6 +76,7 @@
     "@fluent/langneg": "^0.7.0",
     "@fluent/react": "^0.15.2",
     "@lezer/highlight": "^1.2.3",
+    "@lezer/javascript": "^1.5.4",
     "@streamparser/json": "^0.0.22",
     "@tgwf/co2": "^0.18.0",
     "array-move": "^3.0.1",
@@ -109,6 +110,8 @@
     "redux-logger": "^3.0.6",
     "redux-thunk": "^3.1.0",
     "reselect": "^4.1.8",
+    "source-map": "^0.7.6",
+    "url": "^0.11.4",
     "valibot": "^1.4.1",
     "workbox-window": "^7.4.1"
   },

scripts/build-profiler-cli.mjs

@@ -23,6 +23,9 @@ const profilerCliConfig = {
   define: {
     __BUILD_HASH__: JSON.stringify(BUILD_HASH),
     __VERSION__: JSON.stringify(version),
+    // SOURCE_MAP_WORKER_PATH is injected by the browser build. The CLI doesn't
+    // use source map workers but the shared code references this constant.
+    SOURCE_MAP_WORKER_PATH: JSON.stringify('/source-map.worker.js'),
   },
   external: [...nodeBaseConfig.external, 'gecko-profiler-demangle'],
 };

scripts/build.mjs

@@ -3,14 +3,32 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 import esbuild from 'esbuild';
 
-import { mainBundleConfig } from './lib/esbuild-configs.mjs';
+import {
+  mainBundleConfig,
+  sourceMapWorkerConfig,
+  getSourceMapWorkerPath,
+} from './lib/esbuild-configs.mjs';
 import { cleanDist, saveMetafile } from './lib/build-utils.mjs';
 
 async function build() {
   cleanDist();
-  const buildResult = await esbuild.build(mainBundleConfig);
+
+  // Build the worker first so we can read its output path from the metafile
+  // and inject it into the main bundle via SOURCE_MAP_WORKER_PATH.
+  const workerResult = await esbuild.build(sourceMapWorkerConfig);
+
+  const buildResult = await esbuild.build({
+    ...mainBundleConfig,
+    define: {
+      ...mainBundleConfig.define,
+      SOURCE_MAP_WORKER_PATH: JSON.stringify(
+        getSourceMapWorkerPath(workerResult.metafile)
+      ),
+    },
+  });
+
   saveMetafile(buildResult);
-  console.log('✅ Main browser build completed');
+  console.log('✅ Main browser build and source map worker completed');
 }
 
 build().catch(console.error);

scripts/lib/dev-server.mjs

@@ -56,6 +56,7 @@ export async function startDevServer(buildConfig, options = {}) {
     fallback = 'index.html',
     onServerStart,
     cleanDist = true,
+    extraWatchConfigs = [],
   } = options;
 
   // Clean dist directory first
@@ -77,6 +78,12 @@ export async function startDevServer(buildConfig, options = {}) {
   // Start watching for changes
   await buildContext.watch();
 
+  // Watch extra configs (no serving needed, just watch for rebuilds)
+  const extraContexts = await Promise.all(
+    extraWatchConfigs.map((config) => esbuild.context(config))
+  );
+  await Promise.all(extraContexts.map((ctx) => ctx.watch()));
+
   // Create HTTP server
   const server = http.createServer((req, res) => {
     // Validate Host header
@@ -135,7 +142,9 @@ export async function startDevServer(buildConfig, options = {}) {
     isShuttingDown = true;
 
     console.log('\nShutting down...');
-    await buildContext.dispose();
+    await Promise.all(
+      [buildContext, ...extraContexts].map((ctx) => ctx.dispose())
+    );
     server.close();
     process.exit(0);
   });

scripts/lib/esbuild-configs.mjs

@@ -81,6 +81,9 @@ export const mainBundleConfig = {
       : 'undefined',
     // no need to define NODE_ENV:
     // esbuild automatically defines NODE_ENV based on the value for "minify"
+    // In dev, the worker is not hashed so the path is predictable.
+    // In production, build.mjs overrides this after building the worker first.
+    SOURCE_MAP_WORKER_PATH: JSON.stringify('/source-map.worker.js'),
   },
   external: ['zlib'],
   plugins: [
@@ -98,6 +101,10 @@ export const mainBundleConfig = {
         { from: ['res/img/favicon.png'], to: ['dist/res/img'] },
         { from: ['docs-user/**/*'], to: ['dist/docs'] },
         { from: ['locales/**/*'], to: ['dist/locales'] },
+        {
+          from: ['node_modules/source-map/lib/mappings.wasm'],
+          to: ['dist'],
+        },
       ],
     }),
     generateHtmlPlugin({
@@ -108,6 +115,35 @@ export const mainBundleConfig = {
   ],
 };
 
+// Source map worker bundle configuration.
+// Built as a standalone IIFE so that npm dependencies (lezer, source-map) are
+// bundled into a single file that can be loaded as a Web Worker without needing
+// ES module support. In production the output filename includes a content hash
+// (e.g. source-map-ABCD1234.worker.js). The path is then injected into the main
+// bundle via the SOURCE_MAP_WORKER_PATH define. In dev there is no hash since the
+// dev server always serves fresh content and the define can't be updated mid-watch.
+export const sourceMapWorkerConfig = {
+  ...baseConfig,
+  entryPoints: ['src/profile-logic/source-map.worker.ts'],
+  outdir: 'dist',
+  format: 'iife',
+  platform: 'browser',
+  target: browserslistToEsbuild(),
+  sourcemap: true,
+  splitting: false,
+  entryNames: isProduction ? '[name]-[hash]' : '[name]',
+  metafile: true,
+  plugins: [wasmLoader()],
+};
+
+export function getSourceMapWorkerPath(metafile) {
+  const [entryPoint] = sourceMapWorkerConfig.entryPoints;
+  const [outputPath] = Object.entries(metafile.outputs).find(
+    ([, output]) => output.entryPoint === entryPoint
+  );
+  return '/' + path.basename(outputPath);
+}
+
 // Photon styling build configuration
 const photonTemplateHTML = fs.readFileSync(
   path.join(projectRoot, 'res', 'photon', 'index.html'),

scripts/run-dev-server.mjs

@@ -2,7 +2,10 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 import path from 'path';
-import { mainBundleConfig } from './lib/esbuild-configs.mjs';
+import {
+  mainBundleConfig,
+  sourceMapWorkerConfig,
+} from './lib/esbuild-configs.mjs';
 import { startDevServer } from './lib/dev-server.mjs';
 import { serveAndOpenProfile } from './lib/profile-server.mjs';
 import yargs from 'yargs';
@@ -22,6 +25,7 @@ startDevServer(mainBundleConfig, {
   host,
   distDir: 'dist',
   cleanDist: true,
+  extraWatchConfigs: [sourceMapWorkerConfig],
   onServerStart: (profilerUrl) => {
     const barAscii =
       '------------------------------------------------------------------------------------------';

src/actions/receive-profile.ts

@@ -76,7 +76,9 @@ import {
   determineTimelineType,
   hasUsefulSamples,
 } from 'firefox-profiler/profile-logic/profile-data';
+import { doSourceMapSymbolication } from './source-map-symbolication';
 
+import type { RawSourceMap } from 'source-map';
 import type {
   RequestedLib,
   ImplementationFilter,
@@ -89,6 +91,7 @@ import type {
   TabID,
   PageList,
   MixedObject,
+  IndexIntoSourceTable,
 } from 'firefox-profiler/types';
 
 import type { SymbolicationStepInfo } from '../profile-logic/symbolication';
@@ -245,7 +248,28 @@ export function finalizeProfileView(
       }
     }
 
-    await Promise.all([faviconsPromise, symbolicationPromise]);
+    // Fetch source maps for all JS sources with a sourceMapURL, then run the
+    // source-map worker. Runs fully in parallel with native symbolication:
+    // native only touches funcs/frames belonging to library resources (JS
+    // funcs aren't in those sets), and the JS apply step reads current
+    // shared state at dispatch time so it composes with whatever native
+    // has committed by then. Requires WebChannel version 7+.
+    let sourceMapSymbolicationPromise: Promise<void> | null = null;
+    if (browserConnection !== null && browserConnection.supportsGetSourceMap) {
+      sourceMapSymbolicationPromise = doResolveSourceMaps(
+        profile,
+        browserConnection,
+        dispatch
+      ).then(({ resolvedSourceMaps, compiledSources }) =>
+        dispatch(doSourceMapSymbolication(resolvedSourceMaps, compiledSources))
+      );
+    }
+
+    await Promise.all([
+      faviconsPromise,
+      symbolicationPromise,
+      sourceMapSymbolicationPromise,
+    ]);
   };
 }
 
@@ -811,6 +835,85 @@ export async function doSymbolicateProfile(
   dispatch(doneSymbolicating());
 }
 
+/**
+ * Resolve JS source maps for every source in the profile that has both a
+ * sourceMapURL and a UUID. Fetches source maps via the browser WebChannel.
+ *
+ * Also fetches the compiled source text which is required by the scope-tree
+ * name resolution in symbolicateWithSourceMaps.
+ */
+async function doResolveSourceMaps(
+  profile: Profile,
+  browserConnection: BrowserConnection,
+  dispatch: Dispatch
+): Promise<{
+  resolvedSourceMaps: Map<IndexIntoSourceTable, RawSourceMap>;
+  compiledSources: Map<IndexIntoSourceTable, string>;
+}> {
+  const { sources, stringArray } = profile.shared;
+
+  // Collect every source with a sourceMapURL and a UUID. Only UUID-bearing
+  // sources can be fetched via the browser WebChannel.
+  const sourceIndexesWithSourceMaps = new Set<IndexIntoSourceTable>();
+  for (let sourceIndex = 0; sourceIndex < sources.length; sourceIndex++) {
+    if (
+      sources.sourceMapURL[sourceIndex] !== null &&
+      sources.sourceMapURL[sourceIndex] !== undefined &&
+      typeof sources.id[sourceIndex] === 'string'
+    ) {
+      sourceIndexesWithSourceMaps.add(sourceIndex);
+    }
+  }
+
+  if (sourceIndexesWithSourceMaps.size === 0) {
+    return { resolvedSourceMaps: new Map(), compiledSources: new Map() };
+  }
+
+  // Fetch source maps and compiled sources in parallel, ignoring individual failures.
+  const resolvedSourceMaps: Map<IndexIntoSourceTable, RawSourceMap> = new Map();
+  const compiledSources: Map<IndexIntoSourceTable, string> = new Map();
+
+  dispatch({ type: 'START_SOURCE_MAP_FETCHING' });
+  try {
+    await Promise.all(
+      Array.from(sourceIndexesWithSourceMaps).map(async (sourceIndex) => {
+        const filename = stringArray[sources.filename[sourceIndex]];
+        // sourceId is guaranteed non-null by the filter above.
+        const sourceId = sources.id[sourceIndex] as string;
+
+        await Promise.all([
+          browserConnection
+            .getSourceMap(sourceId)
+            .then((result) => {
+              resolvedSourceMaps.set(sourceIndex, result);
+            })
+            .catch((e) => {
+              console.warn(
+                `Failed to fetch source map for "${filename}" (id=${sourceId}):`,
+                e
+              );
+            }),
+          browserConnection
+            .getJSSource(sourceId)
+            .then((text) => {
+              compiledSources.set(sourceIndex, text);
+            })
+            .catch((e) => {
+              console.warn(
+                `Failed to fetch compiled source for "${filename}" (id=${sourceId}):`,
+                e
+              );
+            }),
+        ]);
+      })
+    );
+  } finally {
+    dispatch({ type: 'DONE_SOURCE_MAP_FETCHING' });
+  }
+
+  return { resolvedSourceMaps, compiledSources };
+}
+
 export async function retrievePageFaviconsFromBrowser(
   dispatch: Dispatch,
   pages: PageList,