[INT-78] visual-snapshots: parallel processing of PDF pages

visual-js/replace_pkg_version.sh

@@ -34,7 +34,7 @@ FILE_MAP=(
     ["@saucelabs/nightwatch-sauce-visual-service"]="visual-nightwatch/src/utils/constants.ts"
     ["@saucelabs/visual-storybook"]="visual-storybook/src/api.ts"
     ["@saucelabs/wdio-sauce-visual-service"]="visual-wdio/src/SauceVisualService.ts"
-    ["@saucelabs/visual-snapshots"]="visual-snapshots/src/index.ts"
+    ["@saucelabs/visual-snapshots"]="visual-snapshots/src/version.ts"
     ["@saucelabs/visual-playwright"]="visual-playwright/src/api.ts"
     # Add more mappings as needed
 )

visual-js/visual-snapshots/package.json

@@ -30,11 +30,14 @@
   },
   "dependencies": {
     "@saucelabs/visual": "^0.13.0",
+    "async-lock": "^1.4.1",
     "commander": "^12.0.0",
     "glob": "^11.0.1",
-    "pdf-to-img": "~4.4.0"
+    "pdf-to-img": "~4.4.0",
+    "workerpool": "^9.2.0"
   },
   "devDependencies": {
+    "@types/async-lock": "^1.4.2",
     "@types/jest": "29.5.14",
     "eslint": "^8.0.1",
     "eslint-config-prettier": "^8.3.0",

visual-js/visual-snapshots/src/api/visual-snapshots-api.ts

@@ -1,67 +1,38 @@
 import { BuildStatus, DiffingMethod, VisualApi } from "@saucelabs/visual";
-import { formatString } from "../utils/format.js";
-import path from "path";
-import { PdfFile } from "../app/pdf-file.js";
+import { __dirname } from "../utils/helpers.js";
 
 export interface CreateVisualSnapshotsParams {
-  branch: string;
-  buildName: string;
-  defaultBranch: string;
-  project: string;
-  customId: string;
+  branch?: string;
+  buildName?: string;
+  defaultBranch?: string;
+  project?: string;
+  customId?: string;
   buildId?: string;
   suiteName?: string;
   testName?: string;
   snapshotName?: string;
 }
 
-export class VisualSnapshotsApi {
-  private api: VisualApi;
-
-  constructor(api: VisualApi) {
-    this.api = api;
-  }
-
-  public async generateAndSendPdfFileSnapshots(
-    pdfFiles: PdfFile[],
-    params: CreateVisualSnapshotsParams
-  ) {
-    const buildId = params.buildId ?? (await this.createBuild(params));
-
-    for (const pdfFile of pdfFiles) {
-      console.info(`Processing file: ${pdfFile.path}`);
-
-      const filename = path.basename(pdfFile.path);
-      const testName = params.testName
-        ? formatString(params.testName, { filename })
-        : undefined;
-
-      const snapshotFormat = this.getSnapshotFormat(params.snapshotName);
-
-      let pageNumber = 1;
-      for await (const pdfPageImage of pdfFile.convertPagesToImages()) {
-        const snapshotName = formatString(snapshotFormat, {
-          filename,
-          page: pageNumber,
-        });
+export interface CreateBuildParams {
+  readonly buildName?: string;
+  readonly branch?: string;
+  readonly defaultBranch?: string;
+  readonly project?: string;
+  readonly customId?: string;
+}
 
-        await this.uploadImageAndCreateSnapshot(
-          pdfPageImage,
-          buildId,
-          snapshotName,
-          testName,
-          params.suiteName
-        );
-        pageNumber++;
-      }
-    }
+export interface UploadSnapshotParams {
+  readonly buildId: string;
+  readonly snapshot: Buffer;
+  readonly snapshotName: string;
+  readonly suiteName?: string;
+  readonly testName?: string;
+}
 
-    await this.finishBuild(buildId);
-  }
+export class VisualSnapshotsApi {
+  constructor(private readonly api: VisualApi) {}
 
-  private async createBuild(
-    params: CreateVisualSnapshotsParams
-  ): Promise<string> {
+  public async createBuild(params: CreateBuildParams): Promise<string> {
     const build = await this.api.createBuild({
       name: params.buildName,
       branch: params.branch,
@@ -73,62 +44,48 @@
     return build.id;
   }
 
-  private async uploadImageAndCreateSnapshot(
-    snapshot: Buffer,
-    buildId: string,
-    snapshotName: string,
-    testName?: string,
-    suiteName?: string
-  ) {
-    const uploadId = await this.api.uploadSnapshot({
-      buildId,
-      image: { data: snapshot },
-    });
-
-    console.info(`Uploaded image to build ${buildId}: upload id=${uploadId}.`);
-
-    await this.api.createSnapshot({
-      buildId,
-      uploadId,
-      name: snapshotName,
-      diffingMethod: DiffingMethod.Balanced,
-      testName,
-      suiteName,
-    });
-
-    console.info(`Created a snapshot ${snapshotName} for build ${buildId}.`);
-  }
-
-  private async finishBuild(buildId: string) {
-    await this.api.finishBuild({
+  public async finishBuild(buildId: string) {
+    const { status: buildStatus } = await this.api.finishBuild({
       uuid: buildId,
     });
-    console.info(`Build ${buildId} finished.`);
 
-    const buildStatus = (await this.api.buildStatus(buildId))!;
-    if (
-      [BuildStatus.Running, BuildStatus.Queued].includes(buildStatus.status)
-    ) {
+    if ([BuildStatus.Running, BuildStatus.Queued].includes(buildStatus)) {
       console.info(
         `Build ${buildId} finished but snapshots haven't been compared yet. Check the build status in a few moments.`
       );
     } else {
+      const { unapprovedCount, errorCount } = (await this.api.buildStatus(
+        buildId
+      ))!;
       console.info(
-        `Build ${buildId} finished (status=${buildStatus.status}, unapprovedCount=${buildStatus.unapprovedCount}, errorCount=${buildStatus.errorCount}).`
+        `Build ${buildId} finished (status=${buildStatus}, unapprovedCount=${unapprovedCount}, errorCount=${errorCount}).`
       );
     }
   }
 
-  private getSnapshotFormat(format: string | undefined) {
-    if (!format) {
-      return `page-{page}`;
-    }
+  public async uploadImageAndCreateSnapshot(params: UploadSnapshotParams) {
+    const uploadId = await this.api.uploadSnapshot({
+      buildId: params.buildId,
+      image: { data: params.snapshot },
+    });
 
-    // Page number is always required to make the snapshot names unique
-    if (!format.includes("{page}")) {
-      format = format += "-{page}";
-    }
+    console.info(
+      `Uploaded image to build ${params.buildId}: upload id=${uploadId}.`
+    );
+
+    await this.api.createSnapshot({
+      buildId: params.buildId,
+      uploadId,
+      name: params.snapshotName,
+      diffingMethod: DiffingMethod.Balanced,
+      testName: params.testName,
+      suiteName: params.suiteName,
+    });
+
+    console.info(
+      `Created a snapshot ${params.snapshotName} for build ${params.buildId}.`
+    );
 
-    return format;
+    return uploadId;
   }
 }

visual-js/visual-snapshots/src/app/pdf-file-loader.ts

@@ -0,0 +1,22 @@
+import { pdf } from "pdf-to-img";
+import { PdfFile } from "./pdf-file.js";
+
+export interface PdfFileLoader {
+  loadPdfFile(path: string): Promise<PdfFile>;
+}
+
+export class LibPdfFileLoader implements PdfFileLoader {
+  constructor(private readonly _pdf: typeof pdf = pdf) {}
+
+  public async loadPdfFile(pdfFilePath: string): Promise<PdfFile> {
+    const pdfFile = await this._pdf(pdfFilePath, {
+      scale: 1,
+    });
+
+    return {
+      path: pdfFilePath,
+      pages: pdfFile.length,
+      getPage: (page) => pdfFile.getPage(page),
+    };
+  }
+}

visual-js/visual-snapshots/src/app/pdf-file.ts

@@ -1,4 +1,5 @@
 export interface PdfFile {
   readonly path: string;
-  convertPagesToImages(): AsyncGenerator<Buffer>;
+  readonly pages: number;
+  getPage(page: number): Promise<Buffer>;
 }

visual-js/visual-snapshots/src/app/pdf-files-snapshot-uploader.ts

@@ -0,0 +1,11 @@
+export interface UploadPdfSnapshotsParams {
+  readonly buildId: string;
+  readonly pdfFilePaths: string[];
+  readonly suiteName?: string;
+  readonly testNameFormat?: string;
+  readonly snapshotNameFormat?: string;
+}
+
+export interface PdfSnapshotUploader {
+  uploadSnapshots(params: UploadPdfSnapshotsParams): Promise<void>;
+}

visual-js/visual-snapshots/src/app/pdf-handler.ts

@@ -2,30 +2,38 @@ import {
   CreateVisualSnapshotsParams,
   VisualSnapshotsApi,
 } from "../api/visual-snapshots-api.js";
-import { initializeVisualApi } from "../api/visual-client.js";
-import { PdfConverter } from "./pdf-converter.js";
 import { VisualConfig } from "@saucelabs/visual";
 import { getFiles } from "../utils/glob.js";
+import { PdfSnapshotUploader } from "./pdf-files-snapshot-uploader.js";
 
 export interface PdfCommandParams
   extends VisualConfig,
-    CreateVisualSnapshotsParams {}
+    CreateVisualSnapshotsParams {
+  concurrency: number;
+}
 
 export class PdfCommandHandler {
-  private clientVersion: string;
-
-  constructor(clientVersion: string) {
-    this.clientVersion = clientVersion;
-  }
+  constructor(
+    private readonly visualSnapshotsApi: VisualSnapshotsApi,
+    private readonly pdfSnapshotUploader: PdfSnapshotUploader
+  ) {}
 
   public async handle(globsOrDirs: string[], params: PdfCommandParams) {
-    const visualApi = initializeVisualApi(params, this.clientVersion);
-    const visualSnapshots = new VisualSnapshotsApi(visualApi);
-    const pdfConverter = new PdfConverter();
-
     const pdfFilePaths = await getFiles(globsOrDirs, "*.pdf");
 
-    const pdfFiles = pdfFilePaths.map((p) => pdfConverter.createPdfFile(p));
-    await visualSnapshots.generateAndSendPdfFileSnapshots(pdfFiles, params);
+    const buildId =
+      params.buildId ?? (await this.visualSnapshotsApi.createBuild(params));
+
+    await this.pdfSnapshotUploader.uploadSnapshots({
+      buildId,
+      pdfFilePaths,
+      suiteName: params.suiteName,
+      testNameFormat: params.testName,
+      snapshotNameFormat: params.snapshotName,
+    });
+
+    if (!params.buildId) {
+      await this.visualSnapshotsApi.finishBuild(buildId);
+    }
   }
 }