[rum-privacy] add sourcemaps and telemetry

LICENSES-3rdparty.csv

@@ -187,6 +187,7 @@ Component,Origin,Licence,Copyright
 @rollup/plugin-json,virtual,MIT,rollup (https://github.com/rollup/plugins/tree/master/packages/json#readme)
 @rollup/plugin-node-resolve,virtual,MIT,Rich Harris (https://github.com/rollup/plugins/tree/master/packages/node-resolve/#readme)
 @rollup/plugin-terser,virtual,MIT,Peter Placzek (https://github.com/rollup/plugins/tree/master/packages/terser#readme)
+@rollup/plugin-typescript,virtual,MIT,Oskar Segersvärd (https://github.com/rollup/plugins/tree/master/packages/typescript/#readme)
 @rollup/pluginutils,virtual,MIT,Rich Harris (https://github.com/rollup/plugins/tree/master/packages/pluginutils#readme)
 @rollup/rollup-darwin-arm64,npm,MIT,Lukas Taegert-Atkinson (https://rollupjs.org/)
 @rollup/rollup-darwin-x64,npm,MIT,Lukas Taegert-Atkinson (https://rollupjs.org/)
@@ -903,6 +904,7 @@ to-regex-range,npm,MIT,Jon Schlinkert (https://github.com/micromatch/to-regex-ra
 tough-cookie,npm,BSD-3-Clause,Jeremy Stashewsky (https://github.com/salesforce/tough-cookie)
 ts-api-utils,virtual,MIT,JoshuaKGoldberg (https://www.npmjs.com/package/ts-api-utils)
 ts-jest,virtual,MIT,Kulshekhar Kabra (https://kulshekhar.github.io/ts-jest)
+ts-loader,virtual,MIT,John Reilly (https://github.com/TypeStrong/ts-loader)
 ts-node,virtual,MIT,Blake Embrey (https://typestrong.org/ts-node)
 tsconfig-paths,npm,MIT,Jonas Kello (https://www.npmjs.com/package/tsconfig-paths)
 tslib,npm,0BSD,Microsoft Corp. (https://www.typescriptlang.org/)

packages/plugins/injection/src/esbuild.ts

@@ -14,7 +14,7 @@ import os from 'os';
 import path from 'path';
 
 import { PLUGIN_NAME } from './constants';
-import { getContentToInject } from './helpers';
+import { getContentToInject, isNodeSystemError } from './helpers';
 import type { ContentsToInject } from './types';
 
 const fsp = fs.promises;
@@ -118,15 +118,25 @@ export const getEsbuildPlugin = (
 
             // Write the content.
             const proms = outputs.map(async (output) => {
-                const source = await fsp.readFile(output, 'utf-8');
-                const data = await esbuild.transform(source, {
-                    loader: 'default',
-                    banner,
-                    footer,
-                });
-
-                // FIXME: Handle sourcemaps.
-                await fsp.writeFile(output, data.code);
+                try {
+                    const source = await fsp.readFile(output, 'utf-8');
+                    const data = await esbuild.transform(source, {
+                        loader: 'default',
+                        banner,
+                        footer,
+                    });
+
+                    // FIXME: Handle sourcemaps.
+                    await fsp.writeFile(output, data.code);
+                } catch (e) {
+                    if (isNodeSystemError(e) && e.code === 'ENOENT') {
+                        // When we are using sub-builds, the entry file of sub-builds may not exist
+                        // Hence we should skip the file injection in this case.
+                        log.warn(`Could not inject content in ${output}: ${e}`);
+                    } else {
+                        throw e;
+                    }
+                }
             });
 
             await Promise.all(proms);

packages/plugins/injection/src/helpers.ts

@@ -133,3 +133,11 @@ export const addInjections = async (
         contentsToInject[value.position].set(id, value.value);
     }
 };
+
+export interface NodeSystemError extends Error {
+    code: string;
+}
+
+export const isNodeSystemError = (e: unknown): e is NodeSystemError => {
+    return e instanceof Error && 'code' in e;
+};

packages/plugins/rum/package.json

@@ -31,7 +31,7 @@
         "typecheck": "tsc --noEmit"
     },
     "dependencies": {
-        "@datadog/js-instrumentation-wasm": "0.9.4",
+        "@datadog/js-instrumentation-wasm": "1.0.3",
         "@dd/core": "workspace:*",
         "@rollup/pluginutils": "5.1.4",
         "chalk": "2.3.1"

packages/plugins/rum/src/built/privacy-helpers.ts

@@ -7,14 +7,44 @@
 const globalAny: any = globalThis;
 globalAny.$DD_ALLOW = new Set();
 
-export function $(newValues: string[] | TemplateStringsArray) {
+/* __PURE__ */ const $DD_ADD_TO_DICTIONARY = (newValues: string[] | TemplateStringsArray) => {
     const initialSize = globalAny.$DD_ALLOW.size;
-    newValues.forEach((value) => globalAny.$DD_ALLOW.add(value));
+    if ((newValues as unknown as TemplateStringsArray).raw) {
+        // We're being used as a template tag function. The invocation will look like this:
+        //   const D = $('foo', $`bar${0}`, 'baz');
+        // In this context, our only role is to extract the TemplateStringsArray array so that
+        // the top-level call to $ can make use of it. So, we just need to return our first
+        // argument.
+        return newValues;
+    }
+
+    newValues.flat().forEach((value) => {
+        globalAny.$DD_ALLOW.add(value.toLocaleLowerCase());
+    });
+
     if (globalAny.$DD_ALLOW.size !== initialSize) {
         if (globalAny.$DD_ALLOW_OBSERVERS) {
             globalAny.$DD_ALLOW_OBSERVERS.forEach((cb: () => void) => cb());
         }
     }
 
     return newValues;
-}
+};
+
+// Process any queued items and set up the queue mechanism
+(() => {
+    const queueName = '$DD_A_Q';
+    const addToDictionary = $DD_ADD_TO_DICTIONARY;
+
+    // Initialize queue if it doesn't exist
+    globalAny[queueName] = globalAny[queueName] || [];
+
+    // Process all existing items in the queue
+    globalAny[queueName].forEach(addToDictionary);
+
+    // Clear the queue
+    globalAny[queueName].length = 0;
+
+    // Replace push method with our add function
+    globalAny[queueName].push = addToDictionary;
+})();

packages/plugins/rum/src/index.ts

@@ -57,10 +57,13 @@ export const getPlugins: GetPlugins = ({ options, context }) => {
 
     if (validatedOptions.privacy) {
         // Add the privacy plugin.
-        const privacyPlugin = getPrivacyPlugin(validatedOptions.privacy);
-        if (privacyPlugin) {
-            plugins.push(privacyPlugin);
-        }
+        context.inject({
+            type: 'file',
+            position: InjectPosition.BEFORE,
+            value: path.join(__dirname, './privacy-helpers.js'),
+        });
+        const privacyPlugin = getPrivacyPlugin(validatedOptions.privacy, context);
+        plugins.push(privacyPlugin);
     }
 
     return plugins;

packages/plugins/rum/src/privacy/constants.ts

@@ -5,4 +5,5 @@
 import type { PluginName } from '@dd/core/types';
 
 export const PLUGIN_NAME: PluginName = 'datadog-rum-privacy-plugin' as const;
-export const PRIVACY_HELPERS_MODULE_ID = '\0datadog:privacy-helpers';
+export const PRIVACY_HELPERS_FILE_NAME = 'privacy-helpers';
+export const PRIVACY_HELPERS_MODULE_ID = `\0datadog:${PRIVACY_HELPERS_FILE_NAME}`;

packages/plugins/rum/src/privacy/index.test.ts

@@ -28,12 +28,7 @@ describe('Rum Privacy Plugin', () => {
                 getPlugins(
                     getGetPluginsArg({
                         rum: {
-                            privacy: {
-                                disabled: true,
-                                exclude: [],
-                                include: [],
-                                module: 'esm',
-                            },
+                            // privacy plugin isdisabled without the privacy option
                         },
                     }),
                 ),
@@ -46,10 +41,8 @@ describe('Rum Privacy Plugin', () => {
                 getGetPluginsArg({
                     rum: {
                         privacy: {
-                            disabled: false,
                             exclude: [],
                             include: [],
-                            module: 'esm',
                         },
                     },
                 }),

packages/plugins/rum/src/privacy/index.ts

@@ -3,57 +3,42 @@
 // Copyright 2019-Present Datadog, Inc.
 
 import { instrument } from '@datadog/js-instrumentation-wasm';
-import type { PluginOptions } from '@dd/core/types';
+import type { GlobalContext, PluginOptions } from '@dd/core/types';
 import { createFilter } from '@rollup/pluginutils';
-import fs from 'node:fs';
-import path from 'node:path';
 
-import { PRIVACY_HELPERS_MODULE_ID, PLUGIN_NAME } from './constants';
+import { PLUGIN_NAME } from './constants';
 import { buildTransformOptions } from './transform';
-import type { PrivacyOptions } from './types';
+import type { PrivacyOptionsWithDefaults } from './types';
 
-export const getPrivacyPlugin = (pluginOptions: PrivacyOptions): PluginOptions | undefined => {
-    if (pluginOptions.disabled) {
-        return;
-    }
+export const getPrivacyPlugin = (
+    pluginOptions: PrivacyOptionsWithDefaults,
+    context: GlobalContext,
+): PluginOptions => {
+    const log = context.getLogger(PLUGIN_NAME);
 
-    const transformOptions = buildTransformOptions(pluginOptions);
-    const transformFilter = createFilter(pluginOptions.include, pluginOptions.exclude);
-
-    // Read the privacy helpers code
-    const privacyHelpersPath = path.join(
-        __dirname,
-        pluginOptions.module === 'cjs' ? './privacy-helpers.js' : './privacy-helpers.mjs',
+    const transformOptions = buildTransformOptions(
+        pluginOptions.helperCodeExpression,
+        context.bundler.name,
     );
-
+    const transformFilter = createFilter(pluginOptions.include, pluginOptions.exclude);
     return {
         name: PLUGIN_NAME,
         // Enforce when the plugin will be executed.
         // Not supported by Rollup and ESBuild.
         // https://vitejs.dev/guide/api-plugin.html#plugin-ordering
-        enforce: 'pre',
-        // webpack's id filter is outside of loader logic,
-        // an additional hook is needed for better perf on webpack
-        async resolveId(source) {
-            if (source === PRIVACY_HELPERS_MODULE_ID) {
-                return { id: PRIVACY_HELPERS_MODULE_ID };
-            }
-            return null;
-        },
-
-        async load(id) {
-            if (id === PRIVACY_HELPERS_MODULE_ID) {
-                return { code: fs.readFileSync(privacyHelpersPath, 'utf8') };
-            }
-            return null;
-        },
-        // webpack's id filter is outside of loader logic,
-        // an additional hook is needed for better perf on webpack
+        enforce: 'post',
         transformInclude(id) {
             return transformFilter(id);
         },
         async transform(code, id) {
-            return instrument({ id, code }, transformOptions);
+            try {
+                return instrument({ id, code }, transformOptions);
+            } catch (e) {
+                log.error(`Instrumentation Error: ${e}`, { forward: true });
+                return {
+                    code,
+                };
+            }
         },
     };
 };

packages/plugins/rum/src/privacy/transform.ts

@@ -4,28 +4,30 @@
 
 import type { InstrumentationOptions } from '@datadog/js-instrumentation-wasm';
 
-import { PRIVACY_HELPERS_MODULE_ID } from './constants';
-import type { PrivacyOptions } from './types';
-
 export interface TransformOutput {
     code: string;
     map?: string;
 }
 
-export function buildTransformOptions(pluginOptions: PrivacyOptions): InstrumentationOptions {
-    return {
-        input: {
-            module: pluginOptions.module,
-            jsx: pluginOptions.jsx,
-            typescript: pluginOptions.typescript,
-        },
+export function buildTransformOptions(
+    helperCodeExpression: string,
+    bundlerName: string,
+): InstrumentationOptions {
+    const transformOptions: InstrumentationOptions = {
         privacy: {
             addToDictionaryHelper: {
-                import: {
-                    module: PRIVACY_HELPERS_MODULE_ID,
-                    func: '$',
+                expression: {
+                    code: helperCodeExpression,
                 },
             },
         },
     };
+    if (['esbuild', 'webpack', 'rspack'].includes(bundlerName)) {
+        transformOptions.output = {
+            ...transformOptions.output,
+            inlineSourceMap: false,
+            embedCodeInSourceMap: true,
+        };
+    }
+    return transformOptions;
 }

packages/plugins/rum/src/privacy/types.ts

@@ -7,14 +7,11 @@ import type { Assign } from '@dd/core/types';
 export interface PrivacyOptions {
     exclude?: RegExp[] | string[];
     include?: RegExp[] | string[];
-    module?: 'cjs' | 'esm';
-    jsx?: boolean;
-    transformStrategy?: 'ast';
-    typescript?: boolean;
-    disabled?: boolean | undefined;
+    addToDictionaryFunctionName?: string;
+    helperCodeExpression?: string;
 }
 
 export type PrivacyOptionsWithDefaults = Assign<
     PrivacyOptions,
-    Pick<Required<PrivacyOptions>, 'exclude' | 'include' | 'module' | 'transformStrategy'>
+    Pick<Required<PrivacyOptions>, 'exclude' | 'include' | 'helperCodeExpression'>
 >;

packages/plugins/rum/src/sdk.test.ts

@@ -17,7 +17,10 @@ const doRequestMock = jest.mocked(doRequest);
 describe('RUM Plugin - SDK', () => {
     describe('getInjectionValue', () => {
         const options = validateOptions(
-            { ...defaultPluginOptions, rum: { sdk: { applicationId: 'app_id' } } },
+            {
+                ...defaultPluginOptions,
+                rum: { sdk: { applicationId: 'app_id' } },
+            },
             mockLogger,
         ) as RumOptionsWithSdk;
         const context = getContextMock();

packages/plugins/rum/src/validate.test.ts

@@ -0,0 +1,36 @@
+// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License.
+// This product includes software developed at Datadog (https://www.datadoghq.com/).
+// Copyright 2019-Present Datadog, Inc.
+
+import { defaultPluginOptions } from '@dd/tests/_jest/helpers/mocks';
+import { createFilter } from '@rollup/pluginutils';
+
+import { validatePrivacyOptions } from './validate';
+
+describe('Test privacy plugin option exclude regex', () => {
+    let filter: (path: string) => boolean;
+    const testCases = [
+        { description: 'exclude .preval files', path: '.preval.js', expected: false },
+        { description: 'exclude node_modules', path: '/node_modules/test.js', expected: false },
+        {
+            description: 'exclude all files that start with special characters',
+            path: '!test.js',
+            expected: false,
+        },
+        {
+            description: 'exclude all files that start with special characters',
+            path: '@test.js',
+            expected: false,
+        },
+    ];
+
+    beforeAll(() => {
+        const pluginOptions = { ...defaultPluginOptions, rum: { privacy: {} } };
+        const { config } = validatePrivacyOptions(pluginOptions);
+        filter = createFilter(config?.include, config?.exclude);
+    });
+
+    test.each(testCases)('Should $description', ({ path, expected }) => {
+        expect(filter(path)).toBe(expected);
+    });
+});