Linter CLI: Implement a Linter Worker to parallelize file processing

Author: marcoroth

javascript/packages/linter/rollup.config.mjs

@@ -10,6 +10,13 @@ const external = [
   "url",
   "fs",
   "module",
+  "os",
+  "worker_threads",
+  "node:path",
+  "node:url",
+  "node:fs",
+  "node:os",
+  "node:worker_threads",
 ]
 
 function isExternal(id) {
@@ -41,6 +48,27 @@ export default [
     ],
   },
 
+  // Lint worker entry point (CommonJS - used by worker_threads)
+  {
+    input: "src/cli/lint-worker.ts",
+    output: {
+      file: "dist/lint-worker.js",
+      format: "cjs",
+      sourcemap: true,
+    },
+    external: isExternal,
+    plugins: [
+      nodeResolve(),
+      commonjs(),
+      json(),
+      typescript({
+        tsconfig: "./tsconfig.json",
+        rootDir: "src/",
+        module: "esnext",
+      }),
+    ],
+  },
+
   // Library exports (ESM)
   {
     input: "src/index.ts",

javascript/packages/linter/src/cli.ts

@@ -144,7 +144,7 @@ export class CLI {
     const startTime = Date.now()
     const startDate = new Date()
 
-    const { patterns, configFile, formatOption, showTiming, theme, wrapLines, truncateLines, useGitHubActions, fix, fixUnsafe, ignoreDisableComments, force, init, loadCustomRules, failLevel } = this.argumentParser.parse(process.argv)
+    const { patterns, configFile, formatOption, showTiming, theme, wrapLines, truncateLines, useGitHubActions, fix, fixUnsafe, ignoreDisableComments, force, init, loadCustomRules, failLevel, jobs } = this.argumentParser.parse(process.argv)
 
     this.determineProjectPath(patterns)
 
@@ -246,13 +246,15 @@ export class CLI {
 
       const context: ProcessingContext = {
         projectPath: this.projectPath,
+        configPath: configFile,
         pattern: patterns.join(' '),
         fix,
         fixUnsafe,
         ignoreDisableComments,
         linterConfig,
         config: processingConfig,
-        loadCustomRules
+        loadCustomRules,
+        jobs
       }
 
       const results = await this.fileProcessor.processFiles(files, formatOption, context)

javascript/packages/linter/src/cli/argument-parser.ts

@@ -1,5 +1,6 @@
 import dedent from "dedent"
 
+import { availableParallelism } from "node:os"
 import { parseArgs } from "util"
 import { Herb } from "@herb-tools/node-wasm"
 
@@ -27,6 +28,7 @@ export interface ParsedArguments {
   init: boolean
   loadCustomRules: boolean
   failLevel?: DiagnosticSeverity
+  jobs: number
 }
 
 export class ArgumentParser {
@@ -53,6 +55,8 @@ export class ArgumentParser {
       --github                      enable GitHub Actions annotations (combines with --format)
       --no-github                   disable GitHub Actions annotations (even in GitHub Actions environment)
       --no-custom-rules             disable loading custom rules from project (custom rules are loaded by default from .herb/rules/**/*.{mjs,js})
+      -j, --jobs <n>                number of parallel workers for linting files [default: auto]
+                                    use "auto" to detect based on available CPU cores
       --theme                       syntax highlighting theme (${THEME_NAMES.join("|")}) or path to custom theme file [default: ${DEFAULT_THEME}]
       --no-color                    disable colored output
       --no-timing                   hide timing information
@@ -83,7 +87,8 @@ export class ArgumentParser {
         "no-timing": { type: "boolean" },
         "no-wrap-lines": { type: "boolean" },
         "truncate-lines": { type: "boolean" },
-        "no-custom-rules": { type: "boolean" }
+        "no-custom-rules": { type: "boolean" },
+        jobs: { type: "string", short: "j" }
       },
       allowPositionals: true
     })
@@ -163,7 +168,20 @@ export class ArgumentParser {
       }
     }
 
-    return { patterns, configFile, formatOption, showTiming, theme, wrapLines, truncateLines, useGitHubActions, fix, fixUnsafe, ignoreDisableComments, force, init, loadCustomRules, failLevel }
+    let jobs = availableParallelism()
+
+    if (values.jobs && values.jobs !== "auto") {
+      const parsed = parseInt(values.jobs, 10)
+
+      if (isNaN(parsed) || parsed < 1) {
+        console.error(`Error: Invalid --jobs value "${values.jobs}". Must be a positive integer or "auto".`)
+        process.exit(1)
+      }
+
+      jobs = parsed
+    }
+
+    return { patterns, configFile, formatOption, showTiming, theme, wrapLines, truncateLines, useGitHubActions, fix, fixUnsafe, ignoreDisableComments, force, init, loadCustomRules, failLevel, jobs }
   }
 
   private getFilePatterns(positionals: string[]): string[] {

javascript/packages/linter/src/cli/file-processor.ts

@@ -3,13 +3,17 @@ import { Linter } from "../linter.js"
 import { loadCustomRules } from "../loader.js"
 import { Config } from "@herb-tools/config"
 
-import { readFileSync, writeFileSync } from "fs"
-import { resolve } from "path"
+import { Worker } from "node:worker_threads"
+import { readFileSync, writeFileSync } from "node:fs"
+import { resolve, dirname, join } from "node:path"
+import { fileURLToPath } from "node:url"
+import { availableParallelism } from "node:os"
 import { colorize } from "@herb-tools/highlighter"
 
 import type { Diagnostic } from "@herb-tools/core"
 import type { FormatOption } from "./argument-parser.js"
 import type { HerbConfigOptions } from "@herb-tools/config"
+import type { WorkerInput, WorkerResult } from "./lint-worker.js"
 
 export interface ProcessedFile {
   filename: string
@@ -20,13 +24,15 @@ export interface ProcessedFile {
 
 export interface ProcessingContext {
   projectPath?: string
+  configPath?: string
   pattern?: string
   fix?: boolean
   fixUnsafe?: boolean
   ignoreDisableComments?: boolean
   linterConfig?: HerbConfigOptions['linter']
   config?: Config
   loadCustomRules?: boolean
+  jobs?: number
 }
 
 export interface ProcessingResult {
@@ -44,6 +50,13 @@ export interface ProcessingResult {
   context?: ProcessingContext
 }
 
+/**
+ * Minimum number of files required to use parallel processing.
+ * Below this threshold, sequential processing is faster due to
+ * worker thread startup overhead (loading WASM, config, etc.).
+ */
+const PARALLEL_FILE_THRESHOLD = 10
+
 export class FileProcessor {
   private linter: Linter | null = null
   private customRulesLoaded: boolean = false
@@ -61,6 +74,17 @@ export class FileProcessor {
   }
 
   async processFiles(files: string[], formatOption: FormatOption = 'detailed', context?: ProcessingContext): Promise<ProcessingResult> {
+    const jobs = context?.jobs ?? 1
+    const shouldParallelize = jobs > 1 && files.length >= PARALLEL_FILE_THRESHOLD
+
+    if (shouldParallelize) {
+      return this.processFilesInParallel(files, jobs, formatOption, context)
+    }
+
+    return this.processFilesSequentially(files, formatOption, context)
+  }
+
+  private async processFilesSequentially(files: string[], formatOption: FormatOption = 'detailed', context?: ProcessingContext): Promise<ProcessingResult> {
     let totalErrors = 0
     let totalWarnings = 0
     let totalInfo = 0
@@ -223,4 +247,161 @@ export class FileProcessor {
 
     return result
   }
+
+  private async processFilesInParallel(files: string[], jobs: number, formatOption: FormatOption, context?: ProcessingContext): Promise<ProcessingResult> {
+    const workerCount = Math.min(jobs, files.length)
+    const chunks = this.splitIntoChunks(files, workerCount)
+    const workerPath = this.resolveWorkerPath()
+
+    const workerPromises = chunks.map(chunk => this.runWorker(workerPath, chunk, context))
+    const workerResults = await Promise.all(workerPromises)
+
+    for (const result of workerResults) {
+      if (result.error) {
+        throw new Error(`Worker error: ${result.error}`)
+      }
+    }
+
+    return this.aggregateWorkerResults(workerResults, formatOption, context)
+  }
+
+  private resolveWorkerPath(): string {
+    try {
+      const currentDir = dirname(fileURLToPath(import.meta.url))
+
+      return join(currentDir, "lint-worker.js")
+    } catch {
+      return join(__dirname, "lint-worker.js")
+    }
+  }
+
+  private splitIntoChunks(files: string[], chunkCount: number): string[][] {
+    const chunks: string[][] = Array.from({ length: chunkCount }, () => [])
+
+    for (let i = 0; i < files.length; i++) {
+      chunks[i % chunkCount].push(files[i])
+    }
+
+    return chunks.filter(chunk => chunk.length > 0)
+  }
+
+  private runWorker(workerPath: string, files: string[], context?: ProcessingContext): Promise<WorkerResult> {
+    return new Promise((resolve, reject) => {
+      const workerData: WorkerInput = {
+        files,
+        projectPath: context?.projectPath || process.cwd(),
+        configPath: context?.configPath,
+        fix: context?.fix || false,
+        fixUnsafe: context?.fixUnsafe || false,
+        ignoreDisableComments: context?.ignoreDisableComments || false,
+        loadCustomRules: context?.loadCustomRules || false,
+      }
+
+      const worker = new Worker(workerPath, { workerData })
+
+      worker.on("message", (result: WorkerResult) => {
+        resolve(result)
+      })
+
+      worker.on("error", (error) => {
+        reject(error)
+      })
+
+      worker.on("exit", (code) => {
+        if (code !== 0) {
+          reject(new Error(`Worker exited with code ${code}`))
+        }
+      })
+    })
+  }
+
+  private aggregateWorkerResults(results: WorkerResult[], formatOption: FormatOption, context?: ProcessingContext): ProcessingResult {
+    let totalErrors = 0
+    let totalWarnings = 0
+    let totalInfo = 0
+    let totalHints = 0
+    let totalIgnored = 0
+    let totalWouldBeIgnored = 0
+    let filesWithOffenses = 0
+    let filesFixed = 0
+    let ruleCount = 0
+
+    const allOffenses: ProcessedFile[] = []
+    const ruleOffenses = new Map<string, { count: number, files: Set<string> }>()
+
+    for (const result of results) {
+      totalErrors += result.totalErrors
+      totalWarnings += result.totalWarnings
+      totalInfo += result.totalInfo
+      totalHints += result.totalHints
+      totalIgnored += result.totalIgnored
+      totalWouldBeIgnored += result.totalWouldBeIgnored
+      filesWithOffenses += result.filesWithOffenses
+      filesFixed += result.filesFixed
+
+      if (result.ruleCount > 0) {
+        ruleCount = result.ruleCount
+      }
+
+      for (const offense of result.offenses) {
+        allOffenses.push({
+          filename: offense.filename,
+          offense: offense.offense,
+          content: offense.content,
+          autocorrectable: offense.autocorrectable
+        })
+      }
+
+      for (const [rule, data] of result.ruleOffenses) {
+        const existing = ruleOffenses.get(rule) || { count: 0, files: new Set<string>() }
+        existing.count += data.count
+
+        for (const file of data.files) {
+          existing.files.add(file)
+        }
+
+        ruleOffenses.set(rule, existing)
+      }
+
+      if (formatOption !== 'json') {
+        for (const fixMessage of result.fixMessages) {
+          const [filename, countStr] = fixMessage.split("\t")
+          const count = parseInt(countStr, 10)
+          console.log(`${colorize("\u2713", "brightGreen")} ${colorize(filename, "cyan")} - ${colorize(`Fixed ${count} ${count === 1 ? "offense" : "offenses"}`, "green")}`)
+        }
+      }
+    }
+
+    const processingResult: ProcessingResult = {
+      totalErrors,
+      totalWarnings,
+      totalInfo,
+      totalHints,
+      totalIgnored,
+      filesWithOffenses,
+      filesFixed,
+      ruleCount,
+      allOffenses,
+      ruleOffenses,
+      context
+    }
+
+    if (totalWouldBeIgnored > 0) {
+      processingResult.totalWouldBeIgnored = totalWouldBeIgnored
+    }
+
+    return processingResult
+  }
+
+  /**
+   * Returns the default number of parallel jobs based on available CPU cores.
+   * Returns 1 if parallelism detection fails.
+   */
+  static defaultJobs(): number {
+    try {
+      return availableParallelism()
+    } catch {
+      return 1
+    }
+  }
 }

javascript/packages/linter/src/cli/index.ts

@@ -3,4 +3,6 @@ export { FileProcessor } from "./file-processor.js"
 export { SummaryReporter } from "./summary-reporter.js"
 export { OutputManager } from "./output-manager.js"
 
+export type { WorkerInput, WorkerResult, WorkerOffense } from "./lint-worker.js"
+
 export * from "./formatters/index.js"