Skip to content

Commit aec6b7f

Browse files
james-prejames-pre
committed
Added support for streaming zip files (#3)
1 parent 8904248 commit aec6b7f

File tree

6 files changed

+155
-32
lines changed

6 files changed

+155
-32
lines changed

‎package-lock.json‎

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎src/zip/fs.ts‎

Lines changed: 109 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,29 @@
11
// SPDX-License-Identifier: LGPL-3.0-or-later
2-
import { withErrno } from 'kerium';
3-
import { FileSystem, Inode, type UsageInfo } from '@zenfs/core';
2+
import { FileSystem, Inode, Sync, type UsageInfo } from '@zenfs/core';
43
import type { Backend } from '@zenfs/core/backends/backend.js';
54
import { Readonly } from '@zenfs/core/mixins/readonly.js';
6-
import { Sync } from '@zenfs/core/mixins/sync.js';
7-
import { S_IFDIR } from '@zenfs/core/vfs/constants.js';
85
import { parse } from '@zenfs/core/path.js';
6+
import { S_IFDIR } from '@zenfs/core/vfs/constants.js';
7+
import { withErrno } from 'kerium';
8+
import { err } from 'kerium/log';
99
import { _throw } from 'utilium';
1010
import type { Header } from './zip.js';
1111
import { computeEOCD, FileEntry } from './zip.js';
1212

13+
export interface ZipDataSource<TBuffer extends ArrayBufferLike = ArrayBuffer> {
14+
readonly size: number;
15+
get(offset: number, length: number): Uint8Array<TBuffer> | Promise<Uint8Array<TBuffer>>;
16+
set?(offset: number, data: ArrayBufferView<TBuffer>): void | Promise<void>;
17+
}
18+
1319
/**
1420
* Configuration options for a ZipFS file system.
1521
*/
1622
export interface ZipOptions<TBuffer extends ArrayBufferLike = ArrayBuffer> {
1723
/**
1824
* The zip file as a binary buffer.
1925
*/
20-
data: TBuffer | ArrayBufferView<TBuffer>;
26+
data: TBuffer | ArrayBufferView<TBuffer> | ZipDataSource<TBuffer>;
2127

2228
/**
2329
* The name of the zip file (optional).
@@ -62,16 +68,17 @@ export class ZipFS<TBuffer extends ArrayBufferLike = ArrayBuffer> extends Readon
6268
protected directories: Map<string, Set<string>> = new Map();
6369

6470
protected _time = Date.now();
71+
private _ready: boolean = false;
6572

66-
protected readonly eocd: Header<TBuffer>;
73+
protected eocd!: Header<TBuffer>;
6774

68-
public constructor(
69-
public label: string,
70-
protected data: Uint8Array<TBuffer>
71-
) {
72-
super(0x207a6970, 'zipfs');
75+
public async ready(): Promise<void> {
76+
await super.ready();
77+
78+
if (this._ready) return;
79+
this._ready = true;
7380

74-
this.eocd = computeEOCD(data);
81+
this.eocd = await computeEOCD(this.data);
7582
if (this.eocd.disk != this.eocd.entriesDisk) {
7683
throw withErrno('EINVAL', 'ZipFS does not support spanned zip files.');
7784
}
@@ -84,7 +91,8 @@ export class ZipFS<TBuffer extends ArrayBufferLike = ArrayBuffer> extends Readon
8491
const cdEnd = ptr + this.eocd.size;
8592

8693
while (ptr < cdEnd) {
87-
const cd = new FileEntry<TBuffer>(data.buffer, data.byteOffset + ptr);
94+
const entryData = await this.data.get(ptr, FileEntry.size);
95+
const cd = new FileEntry<TBuffer>(entryData.buffer, entryData.byteOffset);
8896
/* Paths must be absolute,
8997
yet zip file paths are always relative to the zip root.
9098
So we prepend '/' and call it a day. */
@@ -122,9 +130,16 @@ export class ZipFS<TBuffer extends ArrayBufferLike = ArrayBuffer> extends Readon
122130
}
123131
}
124132

133+
public constructor(
134+
public label: string,
135+
protected data: ZipDataSource<TBuffer>
136+
) {
137+
super(0x207a6970, 'zipfs');
138+
}
139+
125140
public usage(): UsageInfo {
126141
return {
127-
totalSpace: this.data.byteLength,
142+
totalSpace: this.data.size,
128143
freeSpace: 0,
129144
};
130145
}
@@ -170,23 +185,101 @@ export class ZipFS<TBuffer extends ArrayBufferLike = ArrayBuffer> extends Readon
170185
}
171186
}
172187

188+
const _isShared = (b: unknown): b is SharedArrayBuffer => typeof b == 'object' && b !== null && b.constructor.name === 'SharedArrayBuffer';
189+
190+
export function fromStream(stream: ReadableStream<Uint8Array>, size: number): ZipDataSource<ArrayBuffer> {
191+
const data = new Uint8Array(size);
192+
193+
let bytesRead = 0;
194+
const pending = new Set<{
195+
resolve(value: void | PromiseLike<void>): void;
196+
offset: number;
197+
length: number;
198+
}>();
199+
200+
const allDone = (async function __read() {
201+
for await (const chunk of stream) {
202+
data.set(chunk, bytesRead);
203+
bytesRead += chunk.byteLength;
204+
for (const promise of pending) {
205+
if (bytesRead >= promise.offset + promise.length) {
206+
promise.resolve();
207+
pending.delete(promise);
208+
}
209+
}
210+
}
211+
})();
212+
213+
return {
214+
size,
215+
async get(offset, length) {
216+
const view = data.subarray(offset, offset + length);
217+
if (bytesRead >= offset + length) return view;
218+
const { promise, resolve } = Promise.withResolvers<void>();
219+
220+
pending.add({ resolve, offset, length });
221+
await promise;
222+
return view;
223+
},
224+
};
225+
}
226+
227+
function getSource<TBuffer extends ArrayBufferLike = ArrayBuffer>(input: ZipOptions<TBuffer>['data']): ZipDataSource<TBuffer> {
228+
if (input instanceof ArrayBuffer || _isShared(input)) {
229+
return {
230+
size: input.byteLength,
231+
get(offset: number, length: number) {
232+
return new Uint8Array(input, offset, length);
233+
},
234+
set(offset, data) {
235+
new Uint8Array(input, offset, data.byteLength).set(new Uint8Array(data.buffer, data.byteOffset, data.byteLength));
236+
},
237+
};
238+
}
239+
240+
if (ArrayBuffer.isView(input)) {
241+
return {
242+
size: input.byteLength,
243+
get(offset: number, length: number) {
244+
return new Uint8Array(input.buffer, input.byteOffset + offset, length);
245+
},
246+
set(offset, data) {
247+
new Uint8Array(input.buffer, input.byteOffset + offset, data.byteLength).set(new Uint8Array(data.buffer, data.byteOffset, data.byteLength));
248+
},
249+
};
250+
}
251+
252+
if (typeof input == 'object' && input !== null && 'size' in input && typeof input.size == 'number' && typeof input.get == 'function') {
253+
return input;
254+
}
255+
256+
throw err(withErrno('EINVAL', 'Invalid zip data source'));
257+
}
258+
173259
const _Zip = {
174260
name: 'Zip',
175261

176262
options: {
177263
data: {
178-
type: [ArrayBuffer, Object.getPrototypeOf(Uint8Array) /* %TypedArray% */],
264+
type: [
265+
ArrayBuffer,
266+
Object.getPrototypeOf(Uint8Array) /* %TypedArray% */,
267+
function ZipDataSource(v: unknown): v is ZipDataSource {
268+
return typeof v == 'object' && v !== null && 'size' in v && typeof v.size == 'number' && 'get' in v && typeof v.get == 'function';
269+
},
270+
],
179271
required: true,
180272
},
181273
name: { type: 'string', required: false },
274+
lazy: { type: 'boolean', required: false },
182275
},
183276

184277
isAvailable(): boolean {
185278
return true;
186279
},
187280

188281
create<TBuffer extends ArrayBufferLike = ArrayBuffer>({ name, data }: ZipOptions<TBuffer>): ZipFS<TBuffer> {
189-
return new ZipFS<TBuffer>(name ?? '', ArrayBuffer.isView(data) ? new Uint8Array<TBuffer>(data.buffer, data.byteOffset, data.byteLength) : new Uint8Array<TBuffer>(data));
282+
return new ZipFS<TBuffer>(name ?? '', getSource(data));
190283
},
191284
} satisfies Backend<ZipFS, ZipOptions>;
192285
type _Zip = typeof _Zip;

‎src/zip/zip.ts‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import { sizeof } from 'memium';
66
import { $from, struct, types as t } from 'memium/decorators';
77
import { memoize } from 'utilium';
88
import { CompressionMethod, decompressionMethods } from './compression.js';
9+
import type { ZipDataSource } from './fs.js';
910
import { msdosDate, safeDecode } from './utils.js';
1011

1112
/**
@@ -504,13 +505,14 @@ export class Header<TBuffer extends ArrayBufferLike = ArrayBuffer> extends $from
504505
*
505506
* There is no byte alignment on the comment
506507
*/
507-
export function computeEOCD<T extends ArrayBufferLike = ArrayBuffer>(data: Uint8Array<T>): Header<T> {
508-
const view = new DataView(data.buffer, data.byteOffset, data.byteLength);
509-
for (let offset = data.byteLength - 22; offset > data.byteLength - 0xffff; offset--) {
510-
// Magic number: EOCD Signature
511-
if (view.getUint32(offset, true) === 0x6054b50) {
508+
export async function computeEOCD<T extends ArrayBufferLike = ArrayBuffer>(source: ZipDataSource<T>): Promise<Header<T>> {
509+
for (let offset = source.size - 22; offset > source.size - 0xffff; offset--) {
510+
const data = await source.get(offset, 22);
511+
const sig = (data[0] | (data[1] << 8) | (data[2] << 16) | (data[3] << 24)) >>> 0;
512+
// The magic number is the EOCD Signature
513+
if (sig === 0x6054b50) {
512514
log.debug('zipfs: found End of Central Directory signature at 0x' + offset.toString(16));
513-
return new Header<T>(data.buffer, data.byteOffset + offset);
515+
return new Header<T>(data.buffer, data.byteOffset);
514516
}
515517
}
516518
throw log.err(withErrno('EINVAL', 'zipfs: could not locate End of Central Directory signature'));

‎tests/tsconfig.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
{
22
"extends": "../tsconfig.json",
33
"compilerOptions": {
4-
"noEmit": true
4+
"noEmit": true,
5+
"rootDir": "."
56
},
67
"include": ["**/*.ts"]
78
}

‎tests/zip.test.ts‎

Lines changed: 34 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,17 @@
1+
import { fromStream, Zip } from '@zenfs/archives';
12
import { configureSingle, fs } from '@zenfs/core';
2-
import { readFileSync } from 'fs';
3-
import assert from 'node:assert';
4-
import { suite, test } from 'node:test';
5-
import { dirname } from 'path';
6-
import { fileURLToPath } from 'url';
7-
import { Zip } from '../dist/zip/fs.js';
83
// @ts-expect-error 7016
94
import { setupLogs } from '@zenfs/core/tests/logs.js';
5+
import assert from 'node:assert';
6+
import { readFileSync, statSync } from 'node:fs';
7+
import { open } from 'node:fs/promises';
8+
import { suite, test } from 'node:test';
109

1110
setupLogs();
1211

1312
suite('Basic ZIP operations', () => {
1413
test('Configure', async () => {
15-
const buffer = readFileSync(dirname(fileURLToPath(import.meta.url)) + '/files/data.zip');
14+
const buffer = readFileSync(import.meta.dirname + '/files/data.zip');
1615
const data = buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength);
1716
await configureSingle({ backend: Zip, data });
1817
});
@@ -37,3 +36,31 @@ suite('Basic ZIP operations', () => {
3736
assert.equal(fs.readFileSync('/nested/omg.txt', 'utf8'), 'This is a nested file!');
3837
});
3938
});
39+
40+
suite('ZIP Streaming', () => {
41+
test('Configure', async () => {
42+
const stream = (await open(import.meta.dirname + '/files/data.zip')).readableWebStream() as ReadableStream;
43+
const { size } = statSync(import.meta.dirname + '/files/data.zip');
44+
await configureSingle({ backend: Zip, data: fromStream(stream, size) });
45+
});
46+
47+
test('readdir /', () => {
48+
assert.equal(fs.readdirSync('/').length, 3);
49+
});
50+
51+
test('read /one.txt', () => {
52+
assert.equal(fs.readFileSync('/one.txt', 'utf8'), '1');
53+
});
54+
55+
test('read /two.txt', () => {
56+
assert.equal(fs.readFileSync('/two.txt', 'utf8'), 'two');
57+
});
58+
59+
test('readdir /nested', () => {
60+
assert.equal(fs.readdirSync('/nested').length, 1);
61+
});
62+
63+
test('readdir /nested/omg.txt', () => {
64+
assert.equal(fs.readFileSync('/nested/omg.txt', 'utf8'), 'This is a nested file!');
65+
});
66+
});

‎tsconfig.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
"module": "NodeNext",
44
"target": "ES2022",
55
"outDir": "dist",
6-
"lib": ["ESNext", "DOM"],
6+
"lib": ["ESNext", "DOM", "DOM.AsyncIterable"],
77
"moduleResolution": "NodeNext",
88
"declaration": true,
99
"sourceMap": true,

0 commit comments

Comments
 (0)