[fix] read large files in node, there will be a wrong package in koa, memory overflow

[yi-xiaobai]

Describe the bug

Node.js version: v18.20.4

Description: in koa environment read large files in node, there will be a wrong package in koa, memory overflow but Why doesn't directly executing node xxx.js report an error

[jonathanong]

can you provide a reproducible case or example?

[yi-xiaobai]

const { ZSTDDecoder } = require('zstddec');
const fs = require('fs');
const pako = require('pako');
const kiwi = require('kiwi-schema');
const zip = require('@zip.js/zip.js');

const uncompressChunk = async (chunkBytes) => {
  try {
    return pako.inflateRaw(chunkBytes);
  } catch (err) {
    try {
      const decoder = new ZSTDDecoder();
      await decoder.init();
      return decoder.decode(chunkBytes);
    } catch {
      throw err;
    }
  }
};
async function decodeFile(arrayBuffer) {
  const bytes = arrayBuffer;
  const header = String.fromCharCode(...bytes.slice(0, 8));
  console.log('헤더:', header);
  const view = new DataView(arrayBuffer.buffer);
  const version = view.getUint32(8, true);
  console.log('버전:', version);
  const chunks = [];
  let offset = 12;
  while (offset < bytes.length) {
    const chunkLength = view.getUint32(offset, true);
    offset += 4;
    chunks.push(bytes.slice(offset, offset + chunkLength));
    offset += chunkLength;
  }
  if (chunks.length < 2) throw new Error('Chunk is not enough');
  const encodedSchema = await uncompressChunk(chunks[0]);
  const encodedData = await uncompressChunk(chunks[1]);
  const decodedSchema = kiwi.decodeBinarySchema(encodedSchema);
  console.log(decodedSchema.definitions[2], encodedData);
  const schema = kiwi.compileSchema(decodedSchema);
  const { nodeChanges, blobs } = schema.decodeMessage(encodedData);
  console.log('nodeChanges', nodeChanges.length);
  return {
    nodeChanges,
    blobs,
  }
  const nodes = new Map();
  const orderByPosition = ({ parentIndex: { position: a } }, { parentIndex: { position: b } }) => {
    return (a < b) - (a > b);
  };
  for (const node of nodeChanges) {
    const { sessionID, localID } = node.guid;
    nodes.set(`${sessionID}:${localID}`, node);
  }
  for (const node of nodeChanges) {
    if (node.parentIndex) {
      const { sessionID, localID } = node.parentIndex.guid;
      const parent = nodes.get(`${sessionID}:${localID}`);
      if (parent) {
        parent.children ||= [];
        parent.children.push(node);
      }
    }
  }
  for (const node of nodeChanges) {
    if (node.children) {
      node.children.sort(orderByPosition);
    }
  }
  for (const node of nodeChanges) {
    delete node.parentIndex;
  }

  console.log('schema:', encodedSchema);
  console.log('data:', encodedData);
  return { encodedSchema, encodedData, nodes, nodeChanges, version, root: nodes.get('0:0'), blobs };
}
function parseBlob(key, { bytes }) {
  const view = new DataView(bytes.buffer);
  let offset = 0;
  switch (key) {
    case 'commands':
      const path = [];
      while (offset < bytes.length) {
        switch (bytes[offset++]) {
          case 0:
            path.push('Z');
            break;
          case 1:
            if (offset + 8 > bytes.length) return;
            path.push('M', view.getFloat32(offset, true), view.getFloat32(offset + 4, true));
            offset += 8;
            break;
          case 2:
            if (offset + 8 > bytes.length) return;
            path.push('L', view.getFloat32(offset, true), view.getFloat32(offset + 4, true));
            offset += 8;
            break;
          case 3:
            if (offset + 16 > bytes.length) return;
            path.push(
              'Q',
              view.getFloat32(offset, true),
              view.getFloat32(offset + 4, true),
              view.getFloat32(offset + 8, true),
              view.getFloat32(offset + 12, true),
            );
            offset += 16;
            break;
          case 4:
            if (offset + 24 > bytes.length) return;
            path.push(
              'C',
              view.getFloat32(offset, true),
              view.getFloat32(offset + 4, true),
              view.getFloat32(offset + 8, true),
              view.getFloat32(offset + 12, true),
              view.getFloat32(offset + 16, true),
              view.getFloat32(offset + 20, true),
            );
            offset += 24;
            break;
          default:
            return;
        }
      }
      return path;
      break;
    case 'vectorNetwork':
      const vertexCount = view.getUint32(0, true);
      const segmentCount = view.getUint32(4, true);
      const regionCount = view.getUint32(8, true);
      const vertices = [];
      const segments = [];
      const regions = [];
      for (let i = 0; i < vertexCount; i++) {
        if (offset + 8 > bytes.length) break;
        vertices.push({
          styleID: view.getUint32(offset + 0, true),
          x: view.getFloat32(offset + 4, true),
          y: view.getFloat32(offset + 8, true),
        });
        offset += 12;
      }
      for (let i = 0; i < segmentCount; i++) {
        if (offset + 28 > bytes.length) break;
        const startVertex = view.getUint32(offset + 4, true);
        const endVertex = view.getUint32(offset + 16, true);
        if (startVertex >= vertexCount || endVertex >= vertexCount) continue;
        segments.push({
          styleID: view.getUint32(offset + 0, true),
          start: {
            vertex: startVertex,
            dx: view.getFloat32(offset + 8, true),
            dy: view.getFloat32(offset + 12, true),
          },
          end: {
            vertex: endVertex,
            dx: view.getFloat32(offset + 20, true),
            dy: view.getFloat32(offset + 24, true),
          },
        });
        offset += 28;
      }
      for (let i = 0; i < regionCount; i++) {
        if (offset + 8 > bytes.length) break;
        let styleID = view.getUint32(offset, true);
        const windingRule = styleID & 1 ? 'NONZERO' : 'ODD';
        styleID >>= 1;
        const loopCount = view.getUint32(offset + 4, true);
        const loops = [];
        offset += 8;
        for (let i = 0; i < loopCount; i++) {
          if (offset + 4 > bytes.length) break;
          const indexCount = view.getUint32(offset, true);
          const indices = [];
          offset += 4;
          if (offset + indexCount * 4 > bytes.length) return;
          for (let k = 0; k < indexCount; k++) {
            const segment = view.getUint32(offset, true);
            if (segment >= segmentCount) return;
            indices.push(segment);
            offset += 4;
          }
          loops.push({
            segments: indices,
            windingRule,
          });
        }
        regions.push({
          styleID,
          windingRule,
          loops,
        });
      }
      return { vertices, segments, regions };
  }
  return { type, length, data };
}
function recoverProperty(prop, blobs) {
  if (Array.isArray(prop)) {
    return prop.map((item) => recoverProperty(item, blobs));
  } else if (typeof prop === 'object' && prop !== null) {
    return recoverObject(prop, blobs);
  }
  return prop;
}
function recoverObject(obj, blobs) {
  for (const [key, value] of Object.entries(obj)) {
    if (key.endsWith('Blob')) {
      const blobKey = key.slice(0, -4); 
      const blobData = blobs[value];
      if (blobData) {
        obj[blobKey] = parseBlob(blobKey, blobData);
        delete obj[key]; 
      }
    } else {
      obj[key] = recoverProperty(value, blobs);
    }
  }
  return obj;
}

function recoverNode(node, blobs) {

  recoverObject(node, blobs);

  if (node.children) {
    node.children = node.children.map((child) => recoverNode(child, blobs));
  }
  return node;
}
async function showPreview(filename, bytes, originalFilePrefix) {

  const isPNG = String.fromCharCode(...bytes.slice(0, 8)) === String.fromCharCode(137, 80, 78, 71, 13, 10, 26, 10);
  const isJPEG = String.fromCharCode(...bytes.slice(0, 2)) === String.fromCharCode(255, 216);
  const isGIF = ['GIF87a', 'GIF89a'].includes(String.fromCharCode(...bytes.slice(0, 6)));
  if (isPNG || isJPEG || isGIF) {
    const imageType = isPNG ? 'PNG' : isJPEG ? 'JPEG' : 'GIF';
    // add preview image
    return;
  }
  // if file is json 
  if (filename.endsWith('.json')) {
    const json = JSON.parse(new TextDecoder().decode(bytes));
    console.log('JSON is checked:', json);
    // json file logic 
    return;
  }
  // if file is figma file
  const figHeader = String.fromCharCode(...bytes.slice(0, 8));
  console.log('figHeader', figHeader);
  if (figHeader === 'fig-kiwi' || figHeader === 'fig-jam.' || figHeader === 'fig-deck') {
    const fileType = figHeader === 'fig-jam.' ? 'FigJam' : figHeader === 'fig-deck' ? 'FigDeck' : 'Figma';
    console.log(`${fileType} is exists`);

    const { encodedSchema, encodedData, version, root, nodeChanges, blobs } = await decodeFile(bytes);
    // const newRoot = recoverNode(root, blobs);
    // return newRoot;
  }
}
async function startLoading(files) {
  try {
    const file = files[0];
    const originalFilePrefix = file.name.replace(/\.fig$/, '');
    const arrayBuffer = file.arrayBuffer;
    const zipHeader = String.fromCharCode(...arrayBuffer.slice(0, 2));
    let fileEntries = [];
    if (zipHeader === 'PK') {
      const zipReader = new zip.ZipReader(new zip.Uint8ArrayReader(new Uint8Array(arrayBuffer)));
      const entries = await zipReader.getEntries();
      fileEntries = entries.filter((entry) => !entry.directory);
    } else {
      fileEntries = [{ filename: file.name, getData: async () => new Uint8Array(arrayBuffer) }];
    }
    console.log(
      'file entries:',
      fileEntries.map((entry) => entry.filename),
    );

    if (fileEntries.length > 0) {
      const firstEntry = fileEntries[0];
      fileEntries.forEach(async (entry) => {
        const bytes = await entry.getData(new zip.Uint8ArrayWriter());
        const node = await showPreview(entry.filename, bytes, originalFilePrefix);
        entry.node = node;
      });
    }
    console.log('decoding file is complete:', originalFilePrefix);
  } catch (err) {
    console.error('error:', err);
  }
}
class File {
  constructor(arrayBuffer, name) {
    this.arrayBuffer = arrayBuffer;
    this.name = name;
  }
}

// module.exports = {
//   startLoading,
// };


async function test() {
  const testFile = new File(await fs.promises.readFile('your .fig file path'), 'your file name');
  await startLoading([testFile]);
}

test();

this is my code

It's fine to directly use node --max-old-space-size=32768 xxx.js, but if it's placed in a koa service, a memory overflow will be reported

Why does it lead to this?

[yi-xiaobai]

schema.decodeMessage(encodedData); lead a memory overflow will be reported

[jonathanong]

at a cursory look, you are loading the entire file in memory. of course you will run into memory problems

[yi-xiaobai]

I don't understand why node xxx.js doesn't report a memory overflow directly