Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions packages/blake3-wasm/README.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
> ⚠️ **DEPRECATED**: This package is deprecated in favor of [`blake3-jit`](https://www.npmjs.com/package/blake3-jit). Please migrate to `blake3-jit` for better performance and maintenance. This package will continue to work but will not receive updates.

JS and WASM implementations of https://github.com/BLAKE3-team/BLAKE3

Using [AssemblyScript](https://www.assemblyscript.org/) to generate a lean WASM.
Expand Down
1 change: 1 addition & 0 deletions packages/blake3-wasm/package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
{
"name": "@huggingface/blake3-wasm",
"version": "0.0.3",
"deprecated": "This package is deprecated in favor of blake3-jit. Please migrate to blake3-jit for better performance and maintenance.",
"scripts": {
"build:debug": "asc assembly/index.ts --target debug",
"build:release": "asc assembly/index.ts --target release",
Expand Down
8 changes: 4 additions & 4 deletions packages/xetchunk-wasm/README.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
JS and WASM implementations of https://github.com/huggingface/xet-core/blob/main/deduplication/src/chunking.rs
JavaScript implementation of https://github.com/huggingface/xet-core/blob/main/deduplication/src/chunking.rs

Using [AssemblyScript](https://www.assemblyscript.org/) to generate a lean WASM.
This package uses `blake3-jit` for hashing and `@huggingface/gearhash-wasm` for gearhash matching.

## Usage

Expand All @@ -14,7 +14,7 @@ const data = new Uint8Array(1000000); // Example: 1MB of data
// ... fill data with your content ...

const chunks = getChunks(data, TARGET_CHUNK_SIZE);
console.log("xorbHash", xorbHasht(chunks));
console.log("xorbHash", xorbHash(chunks));

// Alternative, in case your data is streaming
const chunker = createChunker(TARGET_CHUNK_SIZE);
Expand All @@ -27,7 +27,7 @@ for await (const data of source) {
console.log("last chunk", finalize(chunker));
```

## Beanchmarking chunking
## Benchmarking chunking

```shell
pnpm install
Expand Down
4 changes: 0 additions & 4 deletions packages/xetchunk-wasm/assembly/tsconfig.json

This file was deleted.

40 changes: 13 additions & 27 deletions packages/xetchunk-wasm/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -2,52 +2,38 @@
"name": "@huggingface/xetchunk-wasm",
"version": "0.0.2",
"scripts": {
"build:debug": "asc assembly/index.ts --target debug",
"build:release": "asc assembly/index.ts --target release",
"build": "pnpm run build:debug && npm run build:release",
"build": "tsup src/index.ts --format esm --dts",
"test": "vitest run",
"bench": "node tests/bench.js",
"prepare": "pnpm run build"
},
"keywords": [
"xet",
"chunk",
"chunking",
"assemblyscript",
"assembly",
"wasm"
"chunking"
],
"dependencies": {
"@huggingface/blake3-wasm": "workspace:*",
"@huggingface/gearhash-wasm": "workspace:*"
"@huggingface/gearhash-wasm": "workspace:*",
"blake3-jit": "^1.0.0"
},
"type": "module",
"exports": {
".": {
"import": "./build/release.js",
"types": "./build/release.d.ts"
},
"./assembly": {
"import": "./assembly/index.ts"
},
"./wasm": {
"import": "./build/release.wasm"
"import": "./dist/index.js",
"types": "./dist/index.d.ts"
}
},
"main": "./build/release.js",
"types": "./build/release.d.ts",
"main": "./dist/index.js",
"types": "./dist/index.d.ts",
"devDependencies": {
"@huggingface/splitmix64-wasm": "workspace:*",
"assemblyscript": "0.27.36"
"tsup": "^8.0.0",
"typescript": "^5.0.0"
},
"files": [
"build/release.js",
"build/release.d.ts",
"build/release.wasm",
"build/release.wat",
"build/release.wasm.map",
"dist",
"src",
"README.md",
"asconfig.json",
"assembly"
"tsconfig.json"
]
}
Loading