feat(cli): support multithreading in all cli commands (#2320)

Co-authored-by: Zxilly <[email protected]>

Author: timofei-iatsenko

CONTRIBUTING.md

@@ -117,6 +117,12 @@ There are two documented ways to do this: first is a generic way described in th
    YARN_NPM_REGISTRY_SERVER=http://0.0.0.0:4873/ yarn up "@lingui/*"
    ```
 
+   with PNPM:
+
+   ```sh
+   pnpm -r up "@lingui/*" --latest --registry=http://0.0.0.0:4873/
+   ```
+
 5. After you make some changes, you need to run the same process. (Releasing + yarn upgrade)
 
 6. When finished testing, restore default registry (only for NPM)

packages/cli/package.json

@@ -76,19 +76,23 @@
     "esbuild": "^0.25.1",
     "glob": "^11.0.0",
     "micromatch": "^4.0.7",
+    "ms": "^2.1.3",
     "normalize-path": "^3.0.0",
     "ora": "^5.1.0",
     "picocolors": "^1.1.1",
     "pofile": "^1.1.4",
     "pseudolocale": "^2.0.0",
-    "source-map": "^0.8.0-beta.0"
+    "source-map": "^0.8.0-beta.0",
+    "threads": "^1.7.0"
   },
   "devDependencies": {
     "@lingui/jest-mocks": "*",
     "@types/convert-source-map": "^2.0.0",
     "@types/micromatch": "^4.0.1",
+    "@types/ms": "^2.1.0",
     "@types/normalize-path": "^3.0.0",
     "mock-fs": "^5.2.0",
-    "mockdate": "^3.0.5"
+    "mockdate": "^3.0.5",
+    "ts-node": "^10.9.2"
   }
 }

packages/cli/src/api/ProgramExit.ts

@@ -0,0 +1,7 @@
+export class ProgramExit extends Error {
+  constructor() {
+    super()
+
+    this.name = "ProgramExit"
+  }
+}

packages/cli/src/api/catalog.test.ts

@@ -4,7 +4,7 @@ import mockFs from "mock-fs"
 import { mockConsole } from "@lingui/jest-mocks"
 import { LinguiConfig, makeConfig } from "@lingui/conf"
 
-import { Catalog, cleanObsolete, order } from "./catalog"
+import { Catalog, cleanObsolete, order, writeCompiled } from "./catalog"
 import { createCompiledCatalog } from "./compile"
 
 import {
@@ -688,9 +688,9 @@ describe("writeCompiled", () => {
     async ({ namespace, extension }) => {
       const { source } = createCompiledCatalog("en", {}, { namespace })
       // Test that the file extension of the compiled catalog is `.mjs`
-      expect(await catalog.writeCompiled("en", source, namespace)).toMatch(
-        extension
-      )
+      expect(
+        await writeCompiled(catalog.path, "en", source, namespace)
+      ).toMatch(extension)
     }
   )
 })

packages/cli/src/api/catalog.ts

@@ -6,16 +6,16 @@ import normalize from "normalize-path"
 import { LinguiConfigNormalized, OrderBy } from "@lingui/conf"
 
 import { FormatterWrapper } from "./formats"
-import { CliExtractOptions } from "../lingui-extract"
-import { CliExtractTemplateOptions } from "../lingui-extract-template"
 import { CompiledCatalogNamespace } from "./compile"
 import {
   getTranslationsForCatalog,
   GetTranslationsOptions,
-  TranslationMissingEvent,
 } from "./catalog/getTranslationsForCatalog"
 import { mergeCatalog } from "./catalog/mergeCatalog"
-import { extractFromFiles } from "./catalog/extractFromFiles"
+import {
+  extractFromFiles,
+  extractFromFilesWithWorkerPool,
+} from "./catalog/extractFromFiles"
 import {
   isDirectory,
   makePathRegexSafe,
@@ -24,16 +24,24 @@ import {
   writeFile,
 } from "./utils"
 import { AllCatalogsType, CatalogType, ExtractedCatalogType } from "./types"
+import { ExtractWorkerPool } from "./extractWorkerPool"
 
 const LOCALE = "{locale}"
 const LOCALE_SUFFIX_RE = /\{locale\}.*$/
 
-export type MakeOptions = CliExtractOptions & {
+export type MakeOptions = {
+  files?: string[]
+  clean: boolean
+  overwrite: boolean
+  locale: string[]
   orderBy?: OrderBy
+  workerPool?: ExtractWorkerPool
 }
 
-export type MakeTemplateOptions = CliExtractTemplateOptions & {
+export type MakeTemplateOptions = {
+  files?: string[]
   orderBy?: OrderBy
+  workerPool?: ExtractWorkerPool
 }
 
 export type MergeOptions = {
@@ -81,7 +89,7 @@ export class Catalog {
 
   async make(options: MakeOptions): Promise<AllCatalogsType | false> {
     const [nextCatalog, prevCatalogs] = await Promise.all([
-      this.collect({ files: options.files }),
+      this.collect({ files: options.files, workerPool: options.workerPool }),
       this.readAll(),
     ])
 
@@ -116,7 +124,10 @@ export class Catalog {
   async makeTemplate(
     options: MakeTemplateOptions
   ): Promise<CatalogType | false> {
-    const catalog = await this.collect({ files: options.files })
+    const catalog = await this.collect({
+      files: options.files,
+      workerPool: options.workerPool,
+    })
     if (!catalog) return false
     const sorted = order(options.orderBy, catalog as CatalogType)
 
@@ -128,7 +139,7 @@ export class Catalog {
    * Collect messages from source paths. Return a raw message catalog as JSON.
    */
   async collect(
-    options: { files?: string[] } = {}
+    options: { files?: string[]; workerPool?: ExtractWorkerPool } = {}
   ): Promise<ExtractedCatalogType | undefined> {
     let paths = this.sourcePaths
     if (options.files) {
@@ -140,6 +151,14 @@ export class Catalog {
       paths = paths.filter((path: string) => regex.test(normalize(path)))
     }
 
+    if (options.workerPool) {
+      return await extractFromFilesWithWorkerPool(
+        options.workerPool,
+        paths,
+        this.config
+      )
+    }
+
     return await extractFromFiles(paths, this.config)
   }
 
@@ -181,23 +200,8 @@ export class Catalog {
     )
   }
 
-  async getTranslations(
-    locale: string,
-    options: Omit<GetTranslationsOptions, "onMissing">
-  ) {
-    const missing: TranslationMissingEvent[] = []
-
-    const messages = await getTranslationsForCatalog(this, locale, {
-      ...options,
-      onMissing: (event) => {
-        missing.push(event)
-      },
-    })
-
-    return {
-      missing,
-      messages,
-    }
+  async getTranslations(locale: string, options: GetTranslationsOptions) {
+    return await getTranslationsForCatalog(this, locale, options)
   }
 
   async write(
@@ -217,40 +221,15 @@ export class Catalog {
     await this.format.write(filename, messages, undefined)
   }
 
-  async writeCompiled(
-    locale: string,
-    compiledCatalog: string,
-    namespace?: CompiledCatalogNamespace
-  ) {
-    let ext: string
-    switch (namespace) {
-      case "es":
-        ext = "mjs"
-        break
-      case "ts":
-      case "json":
-        ext = namespace
-        break
-      default:
-        ext = "js"
-    }
-
-    const filename = `${replacePlaceholders(this.path, { locale })}.${ext}`
-    await writeFile(filename, compiledCatalog)
-    return filename
-  }
-
   async read(locale: string): Promise<CatalogType> {
     return await this.format.read(this.getFilename(locale), locale)
   }
 
-  async readAll(): Promise<AllCatalogsType> {
+  async readAll(locales: string[] = this.locales): Promise<AllCatalogsType> {
     const res: AllCatalogsType = {}
 
     await Promise.all(
-      this.locales.map(
-        async (locale) => (res[locale] = await this.read(locale))
-      )
+      locales.map(async (locale) => (res[locale] = await this.read(locale)))
     )
 
     // statement above will save locales in object in undetermined order
@@ -364,6 +343,30 @@ function orderByOrigin<T extends ExtractedCatalogType>(messages: T): T {
     }, {} as T)
 }
 
+export async function writeCompiled(
+  path: string,
+  locale: string,
+  compiledCatalog: string,
+  namespace?: CompiledCatalogNamespace
+) {
+  let ext: string
+  switch (namespace) {
+    case "es":
+      ext = "mjs"
+      break
+    case "ts":
+    case "json":
+      ext = namespace
+      break
+    default:
+      ext = "js"
+  }
+
+  const filename = `${replacePlaceholders(path, { locale })}.${ext}`
+  await writeFile(filename, compiledCatalog)
+  return filename
+}
+
 export function orderByMessage<T extends ExtractedCatalogType>(messages: T): T {
   // hardcoded en-US locale to have consistent sorting
   // @see https://github.com/lingui/js-lingui/pull/1808