Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

BREAKING(encoding/unstable): merge Base32Hex(Encoder|Decoder)Stream to Base32(Encoder|Decoder)Stream #6452

Merged
merged 33 commits into from
Mar 13, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
33 commits
Select commit Hold shift + click to select a range
d4ba5bf
perf(encoding): base64
BlackAsLight Feb 22, 2025
c1eccd4
perf(encoding): base64url
BlackAsLight Feb 22, 2025
df36993
perf(encoding): base32
BlackAsLight Feb 23, 2025
ea879eb
perf(encoding): base32hex
BlackAsLight Feb 23, 2025
2103c65
chore(encoding): add missing copyright comment
BlackAsLight Feb 23, 2025
ba88700
adjust(encoding): base64 to use `switch` instead of `if`
BlackAsLight Feb 23, 2025
8f671ae
chore(encoding): update exports
BlackAsLight Feb 23, 2025
56b6f99
chore(encoding): fmt
BlackAsLight Feb 23, 2025
71e0246
perf(encoding): hex
BlackAsLight Feb 23, 2025
a0ac03e
chore(encoding): remove file committed on accident
BlackAsLight Feb 23, 2025
927c2e7
perf(encoding): update streams to be more memory efficient
BlackAsLight Feb 23, 2025
39a3026
perf(encoding): base32crockford
BlackAsLight Feb 23, 2025
6bb70e5
Merge branch 'denoland:main' into encoding_refactor
BlackAsLight Feb 23, 2025
994124b
chore(encoding): fix docs
BlackAsLight Feb 23, 2025
01477c2
revert(encoding): perf brought to stable exports
BlackAsLight Feb 23, 2025
5acf1fc
docs(encoding): fix mismatch type
BlackAsLight Feb 23, 2025
6112d8a
chore(encoding): add missing export
BlackAsLight Feb 23, 2025
4bb9d91
chore(encoding): remove file committed on accident
BlackAsLight Feb 23, 2025
3b91390
perf(encoding): base32crockford stream
BlackAsLight Feb 23, 2025
75778c2
fix(encoding): types for deno v1.x
BlackAsLight Feb 23, 2025
d4fcd64
fix(encoding): types for deno v1.x
BlackAsLight Feb 23, 2025
ec1d78e
refactor(encoding): to bring perf to stable packages
BlackAsLight Feb 26, 2025
78e3c9d
refactor(encoding): merging same base functions & adding tests & docs
BlackAsLight Feb 26, 2025
0bec310
chore(tools): update check_docs.ts entrypoint
BlackAsLight Feb 26, 2025
8edd5e2
chore(encoding): fix types of deno v1.x
BlackAsLight Feb 26, 2025
8cffee7
Merge branch 'main' into encoding_refactor
BlackAsLight Feb 26, 2025
94eef75
revert(encoding): all changes except unstable base32
BlackAsLight Feb 27, 2025
f968715
revert(tools): change to check_docs.ts
BlackAsLight Feb 27, 2025
0f1cd37
revert(encoding): missing file for stable base32
BlackAsLight Feb 27, 2025
6e0451e
Merge branch 'main' into encoding_base32_refactor
BlackAsLight Mar 13, 2025
c737e47
fix(tools): mistake in merge conflict
BlackAsLight Mar 13, 2025
0ac170f
adjust(encoding): constructor of streams for an options argument
BlackAsLight Mar 13, 2025
f216a5c
Merge branch 'main' into encoding_base32_refactor
kt3k Mar 13, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 0 additions & 2 deletions _tools/check_docs.ts
Original file line number Diff line number Diff line change
Expand Up @@ -51,9 +51,7 @@ const ENTRY_POINTS = [
"../encoding/unstable_base32.ts",
"../encoding/unstable_base64.ts",
"../encoding/unstable_base64_stream.ts",
"../encoding/unstable_base32hex_stream.ts",
"../encoding/unstable_base32_stream.ts",
"../encoding/unstable_base32hex_stream.ts",
"../encoding/unstable_hex_stream.ts",
"../expect/mod.ts",
"../fmt/bytes.ts",
Expand Down
1 change: 0 additions & 1 deletion encoding/deno.json
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@
"./base32": "./base32.ts",
"./unstable-base32": "./unstable_base32.ts",
"./unstable-base32-stream": "./unstable_base32_stream.ts",
"./unstable-base32hex-stream": "./unstable_base32hex_stream.ts",
"./base58": "./base58.ts",
"./base64": "./base64.ts",
"./unstable-base64": "./unstable_base64.ts",
Expand Down
177 changes: 127 additions & 50 deletions encoding/unstable_base32_stream.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,110 +2,187 @@
// This module is browser compatible.

/**
* Utilities for encoding and decoding to and from base32 in a streaming manner.
* TransformStream classes to encode and decode to and from base32 data in a streaming manner.
*
* ```ts
* import { assertEquals } from "@std/assert";
* import { Base32DecoderStream } from "@std/encoding/unstable-base32-stream";
* import { toText } from "@std/streams/to-text";
* import { encodeBase32 } from "@std/encoding/unstable-base32";
* import { Base32EncoderStream } from "@std/encoding/unstable-base32-stream";
* import { toText } from "@std/streams";
*
* const stream = ReadableStream.from(["JBSWY3DPEBLW64TMMQQQ===="])
* .pipeThrough(new Base32DecoderStream())
* .pipeThrough(new TextDecoderStream());
* const readable = (await Deno.open("./deno.lock"))
* .readable
* .pipeThrough(new Base32EncoderStream({ output: "string" }));
*
* assertEquals(await toText(stream), "Hello World!");
* assertEquals(
* await toText(readable),
* encodeBase32(await Deno.readFile("./deno.lock"), "Base32"),
* );
* ```
*
* @experimental **UNSTABLE**: New API, yet to be vetted.
*
* @module
*/

import { decodeBase32, encodeBase32 } from "./base32.ts";
import type { Uint8Array_ } from "./_types.ts";
export type { Uint8Array_ };
import {
type Base32Format,
calcMax,
decodeRawBase32 as decode,
encodeRawBase32 as encode,
} from "./unstable_base32.ts";
import { detach } from "./_common_detach.ts";

type Expect<T> = T extends "bytes" ? Uint8Array_ : string;

/**
* Converts a Uint8Array stream into a base32-encoded stream.
* Transforms a {@linkcode Uint8Array<ArrayBuffer>} stream into a base32 stream.
*
* @experimental **UNSTABLE**: New API, yet to be vetted.
*
* @see {@link https://www.rfc-editor.org/rfc/rfc4648.html#section-6}
* @typeParam T The type of the base32 stream.
*
* @example Usage
* @example Basic Usage
* ```ts
* import { assertEquals } from "@std/assert";
* import { encodeBase32 } from "@std/encoding/base32";
* import { encodeBase32 } from "@std/encoding/unstable-base32";
* import { Base32EncoderStream } from "@std/encoding/unstable-base32-stream";
* import { toText } from "@std/streams/to-text";
* import { toText } from "@std/streams";
*
* const stream = ReadableStream.from(["Hello,", " world!"])
* .pipeThrough(new TextEncoderStream())
* .pipeThrough(new Base32EncoderStream());
* const readable = (await Deno.open("./deno.lock"))
* .readable
* .pipeThrough(new Base32EncoderStream({ output: "string" }));
*
* assertEquals(await toText(stream), encodeBase32(new TextEncoder().encode("Hello, world!")));
* assertEquals(
* await toText(readable),
* encodeBase32(await Deno.readFile("./deno.lock"), "Base32"),
* );
* ```
*/
export class Base32EncoderStream extends TransformStream<Uint8Array, string> {
constructor() {
let push = new Uint8Array(0);
export class Base32EncoderStream<T extends "string" | "bytes">
extends TransformStream<
Uint8Array_,
T extends "bytes" ? Uint8Array_ : string
> {
/**
* Constructs a new instance.
*
* @param options The options of the base32 stream.
*/
constructor(options: { format?: Base32Format; output?: T } = {}) {
const decode = function (): (input: Uint8Array_) => Expect<T> {
if (options.output === "bytes") return (x) => x as Expect<T>;
const decoder = new TextDecoder();
return (x) => decoder.decode(x) as Expect<T>;
}();
const push = new Uint8Array(4);
let remainder = 0;
super({
transform(chunk, controller) {
const concat = new Uint8Array(push.length + chunk.length);
concat.set(push);
concat.set(chunk, push.length);

const remainder = -concat.length % 5;
controller.enqueue(
encodeBase32(concat.slice(0, remainder || undefined)),
let [output, i] = detach(chunk, calcMax(remainder + chunk.length));
if (remainder) {
i -= remainder;
output.set(push.subarray(0, remainder), i);
}
remainder = (output.length - i) % 5;
if (remainder) push.set(output.subarray(-remainder));
const o = encode(

Check warning on line 91 in encoding/unstable_base32_stream.ts

View check run for this annotation

Codecov / codecov/patch

encoding/unstable_base32_stream.ts#L91

Added line #L91 was not covered by tests
output.subarray(0, -remainder || undefined),
i,
0,
options.format,

Check warning on line 95 in encoding/unstable_base32_stream.ts

View check run for this annotation

Codecov / codecov/patch

encoding/unstable_base32_stream.ts#L93-L95

Added lines #L93 - L95 were not covered by tests
);
push = remainder ? concat.slice(remainder) : new Uint8Array(0);
controller.enqueue(decode(output.subarray(0, o)));
},
flush(controller) {
if (push.length) {
controller.enqueue(encodeBase32(push));
if (remainder) {
const [output, i] = detach(
push.subarray(0, remainder),
calcMax(remainder),
);
encode(output, i, 0, options.format);
controller.enqueue(decode(output));
}
},
});
}
}

/**
* Decodes a base32-encoded stream into a Uint8Array stream.
* Transforms a base32 stream into a {@link Uint8Array<ArrayBuffer>} stream.
*
* @experimental **UNSTABLE**: New API, yet to be vetted.
*
* @see {@link https://www.rfc-editor.org/rfc/rfc4648.html#section-6}
* @typeParam T The type of the base32 stream.
*
* @example Usage
* @example Basic Usage
* ```ts
* import { assertEquals } from "@std/assert";
* import { Base32DecoderStream } from "@std/encoding/unstable-base32-stream";
* import { toText } from "@std/streams/to-text";
* import {
* Base32DecoderStream,
* Base32EncoderStream,
* } from "@std/encoding/unstable-base32-stream";
* import { toBytes } from "@std/streams/unstable-to-bytes";
*
* const stream = ReadableStream.from(["JBSWY3DPEBLW64TMMQQQ===="])
* .pipeThrough(new Base32DecoderStream())
* .pipeThrough(new TextDecoderStream());
* const readable = (await Deno.open("./deno.lock"))
* .readable
* .pipeThrough(new Base32EncoderStream({ output: "string" }))
* .pipeThrough(new Base32DecoderStream({ input: "string" }));
*
* assertEquals(await toText(stream), "Hello World!");
* assertEquals(
* await toBytes(readable),
* await Deno.readFile("./deno.lock"),
* );
* ```
*
* @module
*/
export class Base32DecoderStream extends TransformStream<string, Uint8Array_> {
constructor() {
let push = "";
export class Base32DecoderStream<T extends "string" | "bytes">
extends TransformStream<
T extends "bytes" ? Uint8Array_ : string,
Uint8Array_
> {
/**
* Constructs a new instance.
*
* @param options The options of the base32 stream.
*/
constructor(options: { format?: Base32Format; input?: T } = {}) {
const encode = function (): (input: Expect<T>) => Uint8Array_ {
if (options.input === "bytes") return (x) => x as Uint8Array_;
const encoder = new TextEncoder();
return (x) => encoder.encode(x as string) as Uint8Array_;
}();
const push = new Uint8Array(7);
let remainder = 0;
super({
transform(chunk, controller) {
push += chunk;
if (push.length < 8) {
return;
let output = encode(chunk);
if (remainder) {
output = detach(output, remainder + output.length)[0];
output.set(push.subarray(0, remainder));
}
const remainder = -push.length % 8;
controller.enqueue(decodeBase32(push.slice(0, remainder || undefined)));
push = remainder ? chunk.slice(remainder) : "";
remainder = output.length % 8;
if (remainder) push.set(output.subarray(-remainder));
const o = decode(
output.subarray(0, -remainder || undefined),
0,
0,
options.format,
);
controller.enqueue(output.subarray(0, o));
},
flush(controller) {
if (push.length) {
controller.enqueue(decodeBase32(push));
if (remainder) {
const o = decode(
push.subarray(0, remainder),
0,
0,
options.format,

Check warning on line 183 in encoding/unstable_base32_stream.ts

View check run for this annotation

Codecov / codecov/patch

encoding/unstable_base32_stream.ts#L179-L183

Added lines #L179 - L183 were not covered by tests
);
controller.enqueue(push.subarray(0, o));

Check warning on line 185 in encoding/unstable_base32_stream.ts

View check run for this annotation

Codecov / codecov/patch

encoding/unstable_base32_stream.ts#L185

Added line #L185 was not covered by tests
}
},
});
Expand Down
106 changes: 80 additions & 26 deletions encoding/unstable_base32_stream_test.ts
Original file line number Diff line number Diff line change
@@ -1,36 +1,90 @@
// Copyright 2018-2025 the Deno authors. MIT license.

import { assertEquals } from "@std/assert";
import { encodeBase32 } from "./base32.ts";
import { toText } from "@std/streams";
import { toBytes } from "@std/streams/unstable-to-bytes";
import { FixedChunkStream } from "@std/streams/unstable-fixed-chunk-stream";
import { encodeBase32 } from "./unstable_base32.ts";
import {
Base32DecoderStream,
Base32EncoderStream,
} from "./unstable_base32_stream.ts";
import { RandomSliceStream } from "./_random_slice_stream.ts";
import { toText } from "../streams/to_text.ts";
import { concat } from "@std/bytes/concat";

Deno.test("Base32EncoderStream() encodes stream", async () => {
const readable = (await Deno.open("./deno.lock"))
.readable
.pipeThrough(new RandomSliceStream())
.pipeThrough(new Base32EncoderStream());

assertEquals(
await toText(readable),
encodeBase32(await Deno.readFile("./deno.lock")),
);

Deno.test("Base32EncoderStream() with normal format", async () => {
for (const format of ["Base32", "Base32Hex", "Base32Crockford"] as const) {
const readable = (await Deno.open("./deno.lock"))
.readable
.pipeThrough(new FixedChunkStream(1021))
.pipeThrough(new Base32EncoderStream({ format, output: "string" }));

assertEquals(
await toText(readable),
encodeBase32(await Deno.readFile("./deno.lock"), format),
format,
);
}
});

Deno.test("Base32EncoderStream() with raw format", async () => {
for (
const format of [
"Base32",
"Base32Hex",
"Base32Crockford",
] as const
) {
const readable = (await Deno.open("./deno.lock"))
.readable
.pipeThrough(new FixedChunkStream(1021))
.pipeThrough(new Base32EncoderStream({ format, output: "bytes" }));

assertEquals(
await toBytes(readable),
new TextEncoder().encode(
encodeBase32(
await Deno.readFile("./deno.lock"),
format,
),
),
format,
);
}
});

Deno.test("Base32DecoderStream() decodes stream", async () => {
const readable = (await Deno.open("./deno.lock"))
.readable
.pipeThrough(new Base32EncoderStream())
.pipeThrough(new RandomSliceStream())
.pipeThrough(new Base32DecoderStream());

assertEquals(
concat(await Array.fromAsync(readable)),
await Deno.readFile("./deno.lock"),
);
Deno.test("Base32DecoderStream() with normal format", async () => {
for (const format of ["Base32", "Base32Hex", "Base32Crockford"] as const) {
const readable = (await Deno.open("./deno.lock"))
.readable
.pipeThrough(new Base32EncoderStream({ format, output: "string" }))
.pipeThrough(new TextEncoderStream())
.pipeThrough(new FixedChunkStream(1021))
.pipeThrough(new TextDecoderStream())
.pipeThrough(new Base32DecoderStream({ format, input: "string" }));

assertEquals(
await toBytes(readable),
await Deno.readFile("./deno.lock"),
);
}
});

Deno.test("Base32DecoderStream() with raw format", async () => {
for (
const format of [
"Base32",
"Base32Hex",
"Base32Crockford",
] as const
) {
const readable = (await Deno.open("./deno.lock"))
.readable
.pipeThrough(new Base32EncoderStream({ format, output: "bytes" }))
.pipeThrough(new FixedChunkStream(1021))
.pipeThrough(new Base32DecoderStream({ format, input: "bytes" }));

assertEquals(
await toBytes(readable),
await Deno.readFile("./deno.lock"),
);
}
});
Loading
Loading