chore: extend logging (#77)

This PR is a clone of
Unstructured-IO/unstructured-python-client#96.

### Summary:
- Added logs to split pdf hook to improve transparency on the process.
- Added retry counter to retry hook logs

Tested on _sample_docs/layout-parser-paper-fast.pdf and
_sample_docs/layout-parser-paper.pdf.
First one will log that it cannot split if run with pdf split.

Author: christinestraub

.eslintrc.js

@@ -24,5 +24,6 @@ module.exports = {
     "import/no-named-as-default-member": "off",
 
     "import/no-default-export": "error",
+    "object-curly-spacing": ["error", "always"],
   },
 };

.gitignore

@@ -14,3 +14,5 @@ node_modules/
 # Custom ignores
 openapi.json
 openapi_client.json
+
+.idea/

src/hooks/custom/LogRetryHook.ts

@@ -5,6 +5,27 @@ import { AfterErrorContext, AfterErrorHook } from "../types";
  * after encountering a 5xx error.
  */
 export class LogRetryHook implements AfterErrorHook {
+  private retriesCounter: Map<string, number> = new Map();
+
+  /**
+   * Log retries to give users visibility into requests.
+   * @param response - The response object received from the server.
+   * @param operationID - The unique identifier for the operation being logged.
+   */
+  private logRetries(response: Response | null, operationID: string): void {
+    if (response && response.status >= 500) {
+      console.warn(
+          "Failed to process a request due to API server error with status code %d. " +
+          "Attempting retry number %d after sleep.",
+          response.status,
+          this.retriesCounter.get(operationID)
+      );
+      if (response.statusText) {
+        console.warn("Server message - %s", response.statusText);
+      }
+    }
+  }
+
   /**
    * Executes after an error occurs during a request.
    * @param _context - The context object containing information about the request.
@@ -17,11 +38,9 @@ export class LogRetryHook implements AfterErrorHook {
     response: Response | null,
     error: unknown
   ): { response: Response | null; error: unknown } {
-    if (response && response.status >= 500) {
-      console.warn(
-        `Got an error: '${response.status} ${response.statusText}'. Retrying request!`
-      );
-    }
+    const currentCount = this.retriesCounter.get(_context.operationID) || 0;
+    this.retriesCounter.set(_context.operationID, currentCount + 1);
+    this.logRetries(response, _context.operationID);
     return { response, error };
   }
 }

src/hooks/custom/SplitPdfHook.ts

@@ -12,6 +12,7 @@ import {
   SDKInitOptions,
 } from "../types";
 import {
+  getOptimalSplitSize,
   getSplitPdfConcurrencyLevel,
   getStartingPageNumber,
   loadPdf,
@@ -83,38 +84,76 @@ export class SplitPdfHook
       (formData.get(PARTITION_FORM_SPLIT_PDF_PAGE_KEY) as string) ?? "false"
     );
     const file = formData.get(PARTITION_FORM_FILES_KEY) as File | null;
-    const startingPageNumber = getStartingPageNumber(formData);
 
     if (!splitPdfPage) {
+      console.info("Partitioning without split.")
       return request;
     }
 
+    console.info("Preparing to split document for partition.")
     if (!this.client) {
-      console.warn("HTTP client not accessible! Continuing without splitting.");
+      console.warn("HTTP client not accessible! Partitioning without split.");
       return request;
     }
 
     const [error, pdf, pagesCount] = await loadPdf(file);
     if (file === null || pdf === null || error) {
+      console.warn("File could not be split. Partitioning without split.")
       return request;
     }
 
     if (pagesCount < MIN_PAGES_PER_THREAD) {
       console.warn(
-        `PDF has less than ${MIN_PAGES_PER_THREAD} pages. Continuing without splitting.`
+        `PDF has less than ${MIN_PAGES_PER_THREAD} pages. Partitioning without split.`
       );
       return request;
     }
 
+    const startingPageNumber = getStartingPageNumber(formData);
+    console.info("Starting page number set to %d", startingPageNumber);
+
     const concurrencyLevel = getSplitPdfConcurrencyLevel(formData);
-    const splits = await splitPdf(pdf, concurrencyLevel);
+    console.info("Concurrency level set to %d", concurrencyLevel)
+
+    const splitSize = await getOptimalSplitSize(pagesCount, concurrencyLevel);
+    console.info("Determined optimal split size of %d pages.", splitSize)
+
+    if (splitSize >= pagesCount) {
+      console.warn(
+          "Document has too few pages (%d) to be split efficiently. Partitioning without split.",
+          pagesCount,
+      )
+      return request;
+    }
+
+    const splits = await splitPdf(pdf, splitSize);
+    const numberOfSplits = splits.length
+    console.info(
+        "Document split into %d, %d-paged sets.",
+        numberOfSplits,
+        splitSize,
+    )
+    console.info(
+        "Partitioning %d, %d-paged sets.",
+        numberOfSplits,
+        splitSize,
+    )
+
     const headers = prepareRequestHeaders(request);
 
     const requests: Request[] = [];
 
+    let setIndex = 1
     for (const { content, startPage } of splits) {
       // Both startPage and startingPageNumber are 1-based, so we need to subtract 1
       const firstPageNumber = startPage + startingPageNumber - 1;
+      console.info(
+          "Partitioning set #%d (pages %d-%d).",
+          setIndex,
+          firstPageNumber,
+          Math.min(firstPageNumber + splitSize - 1, pagesCount),
+      );
+
       const body = await prepareRequestBody(
         formData,
         content,
@@ -126,6 +165,7 @@ export class SplitPdfHook
         body,
       });
       requests.push(req);
+      setIndex+=1;
     }
 
     this.partitionResponses[operationID] = new Array(requests.length);
@@ -173,6 +213,8 @@ export class SplitPdfHook
 
     this.clearOperation(operationID);
 
+    console.info("Successfully processed the request.")
+
     return new Response(body, {
       headers: headers,
       status: response.status,
@@ -199,6 +241,7 @@ export class SplitPdfHook
     const responses = await this.awaitAllRequests(operationID);
 
     if (!responses?.length) {
+      console.error("Failed to process the request.");
       this.clearOperation(operationID);
       return { response, error };
     }
@@ -214,6 +257,7 @@ export class SplitPdfHook
     });
 
     this.clearOperation(operationID);
+    console.info("Successfully processed the request.");
 
     return { response: finalResponse, error: null };
   }

src/hooks/custom/utils/pdf.ts

@@ -38,28 +38,43 @@ export async function pdfPagesToBlob(
   });
 }
 
+/**
+ * Calculates the optimal split size for processing pages with a specified concurrency level.
+ *
+ * @param pagesCount - The total number of pages to process.
+ * @param concurrencyLevel - The level of concurrency to be used.
+ * @returns A promise that resolves to the optimal number of pages per split,
+ * ensuring it does not exceed the maximum or fall below the minimum threshold.
+ */
+export async function getOptimalSplitSize(
+    pagesCount: number,
+    concurrencyLevel: number
+): Promise<number> {
+  let splitSize = MAX_PAGES_PER_THREAD;
+  if (pagesCount < MAX_PAGES_PER_THREAD * concurrencyLevel) {
+    splitSize = Math.ceil(pagesCount / concurrencyLevel);
+  }
+  splitSize = Math.max(splitSize, MIN_PAGES_PER_THREAD);
+
+  return splitSize;
+}
+
+
 /**
  * Retrieves an array of splits, with the start and end page numbers, from a PDF file.
  * Distribution of pages per split is made in as much uniform manner as possible.
  *
  * @param pdf - The PDF file to extract pages from.
- * @param concurrencyLevel - Number of maximum parallel requests.
+ * @param splitSize - The number of pages per split.
  * @returns A promise that resolves to an array of objects containing Blob files and
  * start and end page numbers from the original document.
  */
 export async function splitPdf(
   pdf: PDFDocument,
-  concurrencyLevel: number
+  splitSize: number
 ): Promise<PdfSplit[]> {
   const pdfSplits: PdfSplit[] = [];
   const pagesCount = pdf.getPages().length;
-
-  let splitSize = MAX_PAGES_PER_THREAD;
-  if (pagesCount < MAX_PAGES_PER_THREAD * concurrencyLevel) {
-    splitSize = Math.ceil(pagesCount / concurrencyLevel);
-  }
-  splitSize = Math.max(splitSize, MIN_PAGES_PER_THREAD);
-
   const numberOfSplits = Math.ceil(pagesCount / splitSize);
 
   for (let i = 0; i < numberOfSplits; ++i) {

src/hooks/custom/utils/request.ts

@@ -28,9 +28,17 @@ export async function prepareResponseBody(
   responses: Response[]
 ): Promise<string> {
   const allElements: any[] = [];
+  let index = 1;
   for (const res of responses) {
+    if (res.status == 200) {
+      console.info("Successfully partitioned set #%d, elements added to the final result.", index);
+    } else {
+      console.warn("Failed to partition set #%d, its elements will be omitted in the final result.", index);
+    }
+
     const resElements = await res.json();
     allElements.push(resElements);
+    index++;
   }
   return JSON.stringify(allElements.flat());
 }

test/integration/SplitPdfHook.test.ts

@@ -170,10 +170,12 @@ describe("SplitPdfHook integration tests check splitted file is same as not spli
         });
       } catch (e) {
         if (!expectedOk) {
-          expect(consoleErrorSpy).toHaveBeenCalledTimes(1);
+          expect(consoleErrorSpy).toHaveBeenCalledTimes(2);
+          expect(consoleErrorSpy).toHaveBeenCalledWith("Failed to process the request.");
           expect(consoleWarnSpy).toHaveBeenCalledWith(
             "Attempted to interpret file as pdf, but error arose when splitting by pages. Reverting to non-split pdf handling path."
           );
+          expect(consoleWarnSpy).toHaveBeenCalledWith("File could not be split. Partitioning without split.");
           return;
         } else {
           throw e;

test/unit/utils/pdf.test.ts

@@ -6,6 +6,8 @@ import {
   pdfPagesToBlob,
   splitPdf,
 } from "../../../src/hooks/custom/utils";
+import { getOptimalSplitSize } from "../../../hooks/custom/utils";
+import { MAX_PAGES_PER_THREAD, MIN_PAGES_PER_THREAD } from "../../../hooks/custom/common";
 
 describe("Pdf utility functions", () => {
   const filename = "test/data/layout-parser-paper.pdf";
@@ -39,17 +41,34 @@ describe("Pdf utility functions", () => {
     });
   });
 
+  describe("getOptimalSplitSize", () => {
+    it("should return the maximum pages per thread when pagesCount is high", async () => {
+      const result = await getOptimalSplitSize(100, 4);
+      expect(result).toBe(MAX_PAGES_PER_THREAD);
+    });
+
+    it("should return the minimum pages per thread when pagesCount is low", async () => {
+      const result = await getOptimalSplitSize(2, 4);
+      expect(result).toBe(MIN_PAGES_PER_THREAD);
+    });
+
+    it("should return an appropriate split size for a given pagesCount and concurrencyLevel", async () => {
+      const result = await getOptimalSplitSize(10, 3);
+      expect(result).toBe(Math.ceil(10 / 3));
+    });
+  });
+
   describe("splitPdf", () => {
     it("should split the PDF into one batch", async () => {
-      const result = await splitPdf(pdf, 1);
+      const result = await splitPdf(pdf, 16);
 
       expect(result).toHaveLength(1);
       expect(result[0]?.startPage).toBe(1);
       expect(result[0]?.endPage).toBe(16);
     });
 
     it("should split the PDF into 3 batches", async () => {
-      const result = await splitPdf(pdf, 3);
+      const result = await splitPdf(pdf, 6);
 
       // Verify the expected behavior
       expect(result).toHaveLength(3);
@@ -62,7 +81,7 @@ describe("Pdf utility functions", () => {
     });
 
     it("should split the PDF into 4 batches", async () => {
-      const result = await splitPdf(pdf, 5);
+      const result = await splitPdf(pdf, 4);
 
       // Verify the expected behavior
       expect(result).toHaveLength(4);