refactor: optimize jwpub parsing

Author: mtdvlpr

app/pages/translate/outlines/index.vue

@@ -53,7 +53,7 @@ const importOutlines = async () => {
   try {
     const body = new FormData();
     body.append("file", jwpubFile.value);
-    const parsedOutlines = await $fetch<Outline[]>("/api/outlines", {
+    const parsedOutlines = await $fetch<Outline[]>("/api/outlines/stream", {
       body,
       method: "POST",
     });

app/pages/translate/outlines/new.vue

@@ -0,0 +1,24 @@
+export default defineEventHandler(async (event) => {
+  console.log("Getting outlines...");
+  const [file] = await receiveFiles(event, {
+    ensure: {
+      maxSize: "128MB",
+      types: ["application/octet-stream"],
+    },
+    formKey: "file",
+    multiple: false,
+  });
+
+  if (!file) {
+    throw createError({
+      statusCode: 400,
+      statusMessage: "No file received",
+    });
+  }
+
+  console.log("File received");
+
+  const database = await getJWPUBDatabaseFromBuffer(await file.arrayBuffer());
+
+  return await getOutlinesFromJWPUB(database);
+});

server/api/outlines/buffer.post.ts

@@ -0,0 +1,9 @@
+export default defineEventHandler(async (event) => {
+  console.log("Getting outlines...");
+  const database = await processFileUpload(event, {
+    maxSize: 256 * 1024 * 1024,
+    processor: getJWPUBDatabaseFromStream,
+  });
+
+  return await getOutlinesFromJWPUB(database);
+});

server/api/outlines/stream.post.ts

@@ -0,0 +1,142 @@
+import type {
+  BlobEnsureOptions,
+  BlobSize,
+  BlobUploadOptions,
+  FileSizeUnit,
+} from "#server/types/blob";
+import type { H3Event } from "h3";
+
+import { defu } from "defu";
+
+// Credits from shared utils of https://github.com/pingdotgg/uploadthing
+const FILESIZE_UNITS = ["B", "KB", "MB", "GB"];
+
+/**
+ * Utility to receive a file or files from body's FormData without storing it.
+ *
+ * @throws
+ * If the files are invalid or don't meet the ensure conditions.
+ */
+export async function receiveFiles(
+  event: H3Event,
+  options: BlobUploadOptions = {},
+) {
+  options = defu(options, {
+    formKey: "files",
+    multiple: true,
+  } satisfies BlobUploadOptions);
+
+  try {
+    const form = await readFormData(event);
+    const files = form.getAll(options.formKey!) as File[];
+
+    if (!files?.length)
+      throw createError({ message: "No files received", status: 400 });
+
+    if (!options.multiple && files.length > 1)
+      throw createError({
+        message: "Multiple files are not allowed",
+        status: 400,
+      });
+
+    if (typeof options.multiple === "number" && files.length > options.multiple)
+      throw createError({
+        message: `Number of files exceeded. Maximum allowed: ${options.multiple}`,
+        status: 400,
+      });
+
+    if (options.ensure?.maxSize || options.ensure?.types?.length) {
+      for (const file of files) {
+        ensureBlob(file, options.ensure);
+      }
+    }
+    return files;
+  } catch (e) {
+    console.error(e);
+    throw createError({
+      cause: e,
+      message: "Error receiving files",
+      statusCode: 500,
+    });
+  }
+}
+
+/**
+ * Ensure the blob is valid and meets the specified requirements.
+ *
+ * @param blob The blob to check
+ * @param options The options to check against
+ * @param options.maxSize The maximum size of the blob (e.g. '1MB')
+ * @param options.types The allowed types of the blob (e.g. ['image/png', 'application/json', 'video'])
+ *
+ * @throws If the blob does not meet the requirements
+ */
+function ensureBlob(blob: Blob, options: BlobEnsureOptions = {}) {
+  if (!(blob instanceof Blob)) {
+    throw createError({
+      message: "Received invalid file",
+      statusCode: 400,
+    });
+  }
+
+  if (options.maxSize) {
+    const maxFileSizeBytes = fileSizeToBytes(options.maxSize);
+
+    if (blob.size > maxFileSizeBytes) {
+      throw createError({
+        message: `File too heavy. Max size is: ${options.maxSize}`,
+        statusCode: 400,
+      });
+    }
+  }
+
+  const [blobType, blobSubtype] = blob.type.split("/");
+
+  if (
+    options.types?.length &&
+    !options.types?.includes(blob.type) &&
+    !options.types?.includes(blobType ?? "") &&
+    !options.types?.includes(blobSubtype ?? "")
+  ) {
+    throw createError({
+      message: `Invalid file type: ${blob.type}. Only allowed: ${options.types.join(", ")}`,
+      statusCode: 400,
+    });
+  }
+}
+
+/**
+ * Helper function that converts any valid BlobSize into numeric bytes value
+ *
+ * @example "1MB", "1500B", "1.2GB"
+ *
+ * @throws If the input is not a valid BlobSize
+ */
+function fileSizeToBytes(input: BlobSize) {
+  // eslint-disable-next-line security/detect-non-literal-regexp
+  const regex = new RegExp(
+    `^(\\d+)(\\.\\d+)?\\s*(${FILESIZE_UNITS.join("|")})$`,
+    "i",
+  );
+  const match = input.match(regex);
+
+  if (!match) {
+    throw createError({
+      message: `Invalid file size format: ${input}`,
+      statusCode: 500,
+    });
+  }
+
+  const sizeValue = Number.parseFloat(match[1]!);
+  const sizeUnit = match[3]!.toUpperCase() as FileSizeUnit;
+
+  if (!FILESIZE_UNITS.includes(sizeUnit)) {
+    throw createError({
+      message: `Invalid file size unit: ${sizeUnit}`,
+      statusCode: 500,
+    });
+  }
+
+  const bytes = sizeValue * Math.pow(1024, FILESIZE_UNITS.indexOf(sizeUnit));
+  return Math.floor(bytes);
+}

server/utils/blob.ts

@@ -12,7 +12,9 @@ import unzipper from "unzipper";
  * @param stream The readable stream of the .jwpub file.
  * @returns The loaded database.
  */
-export const getJWPUBDatabase = async (stream: Readable): Promise<Database> => {
+export const getJWPUBDatabaseFromStream = async (
+  stream: Readable,
+): Promise<Database> => {
   try {
     const dbBuffer = await pipeline(
       stream,
@@ -39,8 +41,49 @@ export const getJWPUBDatabase = async (stream: Readable): Promise<Database> => {
   }
 };
 
-export const parseJWPUB = async (db: Database) => {
-  const htmlDocs = await getHTMLDocs(db);
+export const getJWPUBDatabaseFromBuffer = async (
+  buffer: ArrayBuffer,
+): Promise<Database> => {
+  try {
+    const outerZip = await extractZipFiles(buffer);
+    if (!outerZip.files["contents"]) {
+      throw createError({
+        message: "No contents file found in the JWPUB file",
+        status: 400,
+      });
+    }
+
+    const innerZip = await extractZipFiles(
+      await outerZip.files["contents"]!.async("uint8array"),
+    );
+
+    const dbFile = Object.keys(innerZip.files).find((file) =>
+      file.endsWith(".db"),
+    );
+    if (!dbFile)
+      throw createError({
+        message: "No database file found in the JWPUB file",
+        status: 400,
+      });
+
+    const sqlDb = await innerZip.files[dbFile]!.async("uint8array");
+
+    return loadDatabase(sqlDb);
+  } catch (e) {
+    console.error(e);
+    throw createError({
+      cause: e,
+      message: "Failed to get database from .jwpub file",
+      status: 500,
+    });
+  }
+};
+
+export const parseOutlines = async (
+  db: Database,
+  outlines: { Content: BufferSource }[],
+) => {
+  const htmlDocs = await getDocsFromOutlines(db, outlines);
   return htmlDocs.map((htmlDoc) => {
     // "Nr. {nr} {title}"
     let header = htmlDoc.querySelector("header > h1 strong")?.textContent;
@@ -264,22 +307,39 @@ const getDocs = async (db: Database, key: string, iv: string) => {
 
   for (const row of data.at(0)!.values) {
     const content = row.at(0) as BufferSource;
-    const text = await getRawContent(content, key, iv);
-    const htmlDoc = parseHTML(text);
-
-    htmlDoc.querySelectorAll("rt").forEach((rt) => rt.remove());
-
-    files.push(htmlDoc);
+    files.push(await getDoc(content, key, iv));
   }
 
   return files;
 };
 
-const getHTMLDocs = async (db: Database) => {
+const getDoc = async (content: BufferSource, key: string, iv: string) => {
+  const text = await getRawContent(content, key, iv);
+  const htmlDoc = parseHTML(text);
+
+  htmlDoc.querySelectorAll("rt").forEach((rt) => rt.remove());
+
+  return htmlDoc;
+};
+
+export const getHTMLDocs = async (db: Database) => {
   const pubCard = getPubCard(db);
 
   const { iv, key } = await getPubKeyIv(pubCard);
 
   const files = await getDocs(db, key, iv);
   return files;
 };
+
+const getDocsFromOutlines = async (
+  db: Database,
+  contents: { Content: BufferSource }[],
+) => {
+  const pubCard = getPubCard(db);
+
+  const { iv, key } = await getPubKeyIv(pubCard);
+
+  return await Promise.all(
+    contents.map((content) => getDoc(content.Content, key, iv)),
+  );
+};