Merge pull request #185 from DataDog/yoann/add-submit-log-flag

[internal] Add submit log flag to forward logs to DD

Author: yoannmoinet

.github/CODEOWNERS

@@ -33,3 +33,6 @@ packages/plugins/custom-hooks                                         @yoannmoin
 
 # True End
 packages/plugins/true-end                                            @yoannmoinet
+
+# Async Queue
+packages/plugins/async-queue                                          @yoannmoinet

README.md

@@ -206,7 +206,7 @@ type GlobalContext = {
     // The list of all the plugin instances that are currently running in the ecosystem.
     plugins: Plugin[];
     // Send a log to Datadog.
-    sendLog: (message: string, context?: Record<string, string>) => Promise<void>;
+    sendLog: ({ message: string, context?: Record<string, string> }) => Promise<void>;
     // The start time of the build.
     start: number;
     // The version of the plugin.

packages/core/src/helpers/log.ts

@@ -0,0 +1,46 @@
+// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License.
+// This product includes software developed at Datadog (https://www.datadoghq.com/).
+// Copyright 2019-Present Datadog, Inc.
+
+import { HOST_NAME } from '../constants';
+import type { GlobalData, DdLogOptions } from '../types';
+
+import { doRequest } from './request';
+
+export const INTAKE_PATH = 'v1/input/pub44d5f4eb86e1392037b7501f7adc540e';
+export const INTAKE_HOST = 'browser-http-intake.logs.datadoghq.com';
+
+export const getSendLog =
+    (data: GlobalData) =>
+    ({ message, context }: DdLogOptions): Promise<void> => {
+        return doRequest({
+            // Don't delay the build too much on error.
+            retries: 2,
+            minTimeout: 100,
+            url: `https://${INTAKE_HOST}/${INTAKE_PATH}`,
+            method: 'POST',
+            type: 'json',
+            getData: async () => {
+                const payload = {
+                    ddsource: data.packageName || HOST_NAME,
+                    message,
+                    service: 'build-plugins',
+                    team: 'language-foundations',
+                    env: data.env,
+                    version: data.version,
+                    bundler: {
+                        name: data.bundler.name,
+                        version: data.bundler.version,
+                    },
+                    metadata: data.metadata,
+                    ...context,
+                };
+                return {
+                    data: JSON.stringify(payload),
+                    headers: {
+                        'Content-Type': 'application/json',
+                    },
+                };
+            },
+        });
+    };

packages/core/src/types.ts

@@ -56,6 +56,13 @@ export type SerializedEntry = Assign<Entry, { inputs: string[]; outputs: string[
 export type SerializedInput = Assign<Input, { dependencies: string[]; dependents: string[] }>;
 export type SerializedOutput = Assign<Output, { inputs: string[] }>;
 
+export type Log = {
+    bundler?: BundlerFullName;
+    pluginName: string;
+    type: LogLevel;
+    message: string;
+    time: number;
+};
 export type LogTags = string[];
 export type Timer = {
     label: string;
@@ -71,18 +78,12 @@ export type BuildMetadata = {
 };
 
 export type BuildReport = {
-    bundler: Omit<BundlerReport, 'outDir' | 'rawConfig'>;
-    errors: string[];
-    warnings: string[];
-    logs: {
-        bundler: BundlerFullName;
-        pluginName: string;
-        type: LogLevel;
-        message: string;
-        time: number;
-    }[];
-    metadata: BuildMetadata;
-    timings: Timer[];
+    bundler: GlobalData['bundler'];
+    errors: GlobalStores['errors'];
+    warnings: GlobalStores['warnings'];
+    logs: GlobalStores['logs'];
+    timings: GlobalStores['timings'];
+    metadata: GlobalData['metadata'];
     entries?: Entry[];
     inputs?: Input[];
     outputs?: Output[];
@@ -104,13 +105,9 @@ export type SerializedBuildReport = Assign<
 
 export type BundlerFullName = (typeof FULL_NAME_BUNDLERS)[number];
 export type BundlerName = (typeof SUPPORTED_BUNDLERS)[number];
-export type BundlerReport = {
-    name: BundlerName;
-    fullName: BundlerFullName;
+export type BundlerReport = GlobalData['bundler'] & {
     outDir: string;
     rawConfig?: any;
-    variant?: string; // e.g. Major version of the bundler (webpack 4, webpack 5)
-    version: string;
 };
 
 export type InjectedValue = string | (() => Promise<string>);
@@ -140,13 +137,21 @@ export type TimeLog = (
     opts?: { level?: LogLevel; start?: boolean | number; log?: boolean; tags?: LogTags },
 ) => TimeLogger;
 export type GetLogger = (name: string) => Logger;
+export type LogOptions = { forward?: boolean };
+export type LoggerFn = (text: any, opts?: LogOptions) => void;
 export type Logger = {
     getLogger: GetLogger;
     time: TimeLog;
-    error: (text: any) => void;
-    warn: (text: any) => void;
-    info: (text: any) => void;
-    debug: (text: any) => void;
+    error: LoggerFn;
+    warn: LoggerFn;
+    info: LoggerFn;
+    debug: LoggerFn;
+};
+type RestContext = string | string[] | number | boolean;
+export type LogData = Record<string, RestContext | Record<string, RestContext>>;
+export type DdLogOptions = {
+    message: string;
+    context?: LogData;
 };
 export type Env = (typeof ALL_ENVS)[number];
 export type TriggerHook<R> = <K extends keyof CustomHooks>(
@@ -159,16 +164,17 @@ export type GlobalContext = {
     build: BuildReport;
     bundler: BundlerReport;
     cwd: string;
-    env: Env;
+    env: GlobalData['env'];
     getLogger: GetLogger;
     git?: RepositoryData;
     hook: TriggerHook<void>;
     inject: (item: ToInjectItem) => void;
     pluginNames: string[];
     plugins: (PluginOptions | CustomPluginOptions)[];
-    sendLog: (message: string, ctx?: any) => Promise<void>;
+    queue: (promise: Promise<any>) => void;
+    sendLog: (args: DdLogOptions) => Promise<void>;
     start: number;
-    version: string;
+    version: GlobalData['version'];
 };
 
 export type FactoryMeta = {
@@ -206,6 +212,8 @@ export type GetPluginsArg = {
     bundler: any;
     context: GlobalContext;
     options: Options;
+    data: GlobalData;
+    stores: GlobalStores;
 };
 export type GetPlugins = (arg: GetPluginsArg) => PluginOptions[];
 export type GetCustomPlugins = (arg: GetPluginsArg) => CustomPluginOptions[];
@@ -265,3 +273,24 @@ export type FileValidity = {
     empty: boolean;
     exists: boolean;
 };
+
+export type GlobalData = {
+    bundler: {
+        name: BundlerName;
+        fullName: BundlerFullName;
+        variant: string; // e.g. Major version of the bundler (webpack 4, webpack 5)
+        version: string;
+    };
+    env: Env;
+    metadata: BuildMetadata;
+    packageName: string;
+    version: string;
+};
+
+export type GlobalStores = {
+    errors: string[];
+    logs: Log[];
+    queue: Promise<any>[];
+    timings: Timer[];
+    warnings: string[];
+};

packages/factory/README.md

@@ -10,6 +10,7 @@ This is used to aggregate all the plugins and expose them to the bundler.
 <!-- #toc -->
 -   [Internal Plugins](#internal-plugins)
     -   [Analytics](#analytics)
+    -   [Async Queue](#async-queue)
     -   [Build Report](#build-report)
     -   [Bundler Report](#bundler-report)
     -   [Custom Hooks](#custom-hooks)
@@ -35,11 +36,18 @@ Most of the time they will interact via the global context.
 
 > Send some analytics data to Datadog internally.
 > <br/>
-> It gives you acces to the `context.sendLog()` function.
+> Will send a log at the beginning of a build.
 
 #### [📝 Full documentation ➡️](/packages/plugins/analytics#readme)
 
 
+### Async Queue
+
+> An internal queue for async actions that we want to finish before quitting the build.
+
+#### [📝 Full documentation ➡️](/packages/plugins/async-queue#readme)
+
+
 ### Build Report
 
 > This will populate `context.build` with a bunch of data coming from the build.
@@ -271,7 +279,7 @@ type GlobalContext = {
     // The list of all the plugin instances that are currently running in the ecosystem.
     plugins: Plugin[];
     // Send a log to Datadog.
-    sendLog: (message: string, context?: Record<string, string>) => Promise<void>;
+    sendLog: ({ message: string, context?: Record<string, string> }) => Promise<void>;
     // The start time of the build.
     start: number;
     // The version of the plugin.
@@ -281,7 +289,7 @@ type GlobalContext = {
 
 > [!NOTE]
 > Some parts of the context are only available after certain hooks:
->   - all the helper functions, `asyncHook`, `getLogger`, `hook`, `inject`, `sendLog`, are available in the `init` hook.
+>   - some helper functions, `asyncHook`, `hook`, `inject` and `queue` are available in the `init` hook.
 >   - `cwd` is available in the `cwd` hook.
 >   - `context.bundler.rawConfig` is available in the `bundlerReport` hook.
 >   - `context.build.*` is available in the `buildReport` hook.

packages/factory/package.json

@@ -22,6 +22,7 @@
         "@dd/core": "workspace:*",
         "@dd/error-tracking-plugin": "workspace:*",
         "@dd/internal-analytics-plugin": "workspace:*",
+        "@dd/internal-async-queue-plugin": "workspace:*",
         "@dd/internal-build-report-plugin": "workspace:*",
         "@dd/internal-bundler-report-plugin": "workspace:*",
         "@dd/internal-custom-hooks-plugin": "workspace:*",

packages/factory/src/helpers/context.ts

@@ -2,14 +2,12 @@
 // This product includes software developed at Datadog (https://www.datadoghq.com/).
 // Copyright 2019-Present Datadog, Inc.
 
-import { ALL_ENVS } from '@dd/core/constants';
+import { getSendLog } from '@dd/core/helpers/log';
 import type {
     BuildReport,
-    BundlerFullName,
-    BundlerName,
-    Env,
-    FactoryMeta,
     GlobalContext,
+    GlobalData,
+    GlobalStores,
     OptionsWithDefaults,
 } from '@dd/core/types';
 
@@ -18,36 +16,23 @@ import { getLoggerFactory } from './logger';
 export const getContext = ({
     start,
     options,
-    bundlerName,
-    bundlerVersion,
-    version,
+    data,
+    stores,
 }: {
     start: number;
     options: OptionsWithDefaults;
-    bundlerName: BundlerName;
-    bundlerVersion: string;
-    version: FactoryMeta['version'];
+    data: GlobalData;
+    stores: GlobalStores;
 }): GlobalContext => {
     const cwd = process.cwd();
-    const variant = bundlerName === 'webpack' ? bundlerVersion.split('.')[0] : '';
     const build: BuildReport = {
-        errors: [],
-        warnings: [],
-        logs: [],
-        metadata: options.metadata || {},
-        timings: [],
-        bundler: {
-            name: bundlerName,
-            fullName: `${bundlerName}${variant}` as BundlerFullName,
-            variant,
-            version: bundlerVersion,
-        },
+        errors: stores.errors,
+        warnings: stores.warnings,
+        logs: stores.logs,
+        metadata: data.metadata,
+        timings: stores.timings,
+        bundler: data.bundler,
     };
-
-    // Use "production" if there is no env passed.
-    const passedEnv: Env = (process.env.BUILD_PLUGINS_ENV as Env) || 'production';
-    // Fallback to "development" if the passed env is wrong.
-    const env: Env = ALL_ENVS.includes(passedEnv) ? passedEnv : 'development';
     const context: GlobalContext = {
         auth: options.auth,
         pluginNames: [],
@@ -59,8 +44,8 @@ export const getContext = ({
         build,
         // This will be updated in the bundler-report plugin once we have the configuration.
         cwd,
-        env,
-        getLogger: getLoggerFactory(build, options.logLevel),
+        env: data.env,
+        getLogger: getLoggerFactory(data, stores, options.logLevel),
         // This will be updated in the injection plugin on initialization.
         asyncHook: () => {
             throw new Error('AsyncHook function called before it was initialized.');
@@ -72,12 +57,14 @@ export const getContext = ({
         inject: () => {
             throw new Error('Inject function called before it was initialized.');
         },
-        sendLog: () => {
-            throw new Error('SendLog function called before it was initialized.');
-        },
         plugins: [],
+        // This will be updated in the async-queue plugin on initialization.
+        queue: () => {
+            throw new Error('Queue function called before it was initialized.');
+        },
+        sendLog: getSendLog(data),
         start,
-        version,
+        version: data.version,
     };
 
     return context;