Add additional exports for Encoding, pre-tokenizers, and models

Co-authored-by: xenova <[email protected]>

Author: Copilot

package.json

@@ -18,6 +18,28 @@
         "import": "./dist/tokenizers.min.mjs"
       },
       "default": "./dist/tokenizers.min.mjs"
+    },
+    "./pre-tokenizers": {
+      "types": "./types/pre-tokenizers.d.ts",
+      "node": {
+        "require": "./dist/pre-tokenizers.min.cjs",
+        "import": "./dist/pre-tokenizers.min.mjs"
+      },
+      "browser": {
+        "import": "./dist/pre-tokenizers.min.mjs"
+      },
+      "default": "./dist/pre-tokenizers.min.mjs"
+    },
+    "./models": {
+      "types": "./types/models.d.ts",
+      "node": {
+        "require": "./dist/models.min.cjs",
+        "import": "./dist/models.min.mjs"
+      },
+      "browser": {
+        "import": "./dist/models.min.mjs"
+      },
+      "default": "./dist/models.min.mjs"
     }
   },
   "files": [

scripts/build.mjs

@@ -25,7 +25,7 @@ const reportSize = (outfile) => {
   console.log(`⚡ Done\n`);
 };
 
-const build = async (outfile) => {
+const build = async (entryPoint, outfile) => {
   const format = outfile.endsWith(".mjs") ? "esm" : "cjs";
   const minifyOptions = /\.min\.[cm]js$/.test(outfile)
     ? { minify: true, minifySyntax: true }
@@ -35,7 +35,7 @@ const build = async (outfile) => {
     bundle: true,
     treeShaking: true,
     logLevel: "silent",
-    entryPoints: ["src/index.ts"],
+    entryPoints: [entryPoint],
     platform: "neutral",
     metafile: true,
     format,
@@ -45,7 +45,17 @@ const build = async (outfile) => {
   reportSize(outfile);
 };
 
-await build("dist/tokenizers.mjs");
-await build("dist/tokenizers.cjs");
-await build("dist/tokenizers.min.mjs");
-await build("dist/tokenizers.min.cjs");
+await build("src/index.ts", "dist/tokenizers.mjs");
+await build("src/index.ts", "dist/tokenizers.cjs");
+await build("src/index.ts", "dist/tokenizers.min.mjs");
+await build("src/index.ts", "dist/tokenizers.min.cjs");
+
+await build("src/pre-tokenizers.ts", "dist/pre-tokenizers.mjs");
+await build("src/pre-tokenizers.ts", "dist/pre-tokenizers.cjs");
+await build("src/pre-tokenizers.ts", "dist/pre-tokenizers.min.mjs");
+await build("src/pre-tokenizers.ts", "dist/pre-tokenizers.min.cjs");
+
+await build("src/models.ts", "dist/models.mjs");
+await build("src/models.ts", "dist/models.cjs");
+await build("src/models.ts", "dist/models.min.mjs");
+await build("src/models.ts", "dist/models.min.cjs");

src/index.ts

@@ -1 +1,2 @@
 export { default as Tokenizer } from "./core/Tokenizer";
+export type { Encoding } from "./static/types";

src/models.ts

@@ -0,0 +1,5 @@
+export { default as BPE } from "./core/tokenizerModelImplementations/BPE";
+export { default as LegacyTokenizerModel } from "./core/tokenizerModelImplementations/LegacyTokenizerModel";
+export { default as Unigram } from "./core/tokenizerModelImplementations/Unigram";
+export { default as WordPiece } from "./core/tokenizerModelImplementations/WordPieceTokenizer";
+export { default as WordPieceTokenizer } from "./core/tokenizerModelImplementations/WordPieceTokenizer";

src/pre-tokenizers.ts

@@ -0,0 +1,12 @@
+export { default as BertPreTokenizer } from "./core/preTokenizer/BertPreTokenizer";
+export { default as ByteLevelPreTokenizer } from "./core/preTokenizer/ByteLevelPreTokenizer";
+export { default as DigitsPreTokenizer } from "./core/preTokenizer/DigitsPreTokenizer";
+export { default as Metaspace } from "./core/preTokenizer/MetaspacePreTokenizer";
+export { default as MetaspacePreTokenizer } from "./core/preTokenizer/MetaspacePreTokenizer";
+export { default as PreTokenizerSequence } from "./core/preTokenizer/PreTokenizerSequence";
+export { default as PunctuationPreTokenizer } from "./core/preTokenizer/PunctuationPreTokenizer";
+export { default as ReplacePreTokenizer } from "./core/preTokenizer/ReplacePreTokenizer";
+export { default as SplitPreTokenizer } from "./core/preTokenizer/SplitPreTokenizer";
+export { default as Whitespace } from "./core/preTokenizer/WhitespacePreTokenizer";
+export { default as WhitespacePreTokenizer } from "./core/preTokenizer/WhitespacePreTokenizer";
+export { default as WhitespaceSplit } from "./core/preTokenizer/WhitespaceSplit";

tests/exports.test.ts

@@ -0,0 +1,94 @@
+import { Tokenizer } from "../src";
+import type { Encoding } from "../src";
+import { Metaspace, Whitespace } from "../src/pre-tokenizers";
+import { BPE } from "../src/models";
+
+describe("Additional exports", () => {
+  describe("Main exports", () => {
+    it("should export Tokenizer", () => {
+      expect(Tokenizer).toBeDefined();
+    });
+
+    it("should export Encoding type (compile-time test)", () => {
+      // This test verifies that the Encoding type can be used
+      const encoding: Encoding = {
+        ids: [1, 2, 3],
+        tokens: ["hello", "world", "!"],
+        attention_mask: [1, 1, 1],
+      };
+      expect(encoding.ids).toEqual([1, 2, 3]);
+    });
+  });
+
+  describe("Pre-tokenizer exports", () => {
+    it("should export Metaspace pre-tokenizer", () => {
+      expect(Metaspace).toBeDefined();
+      const metaspace = new Metaspace({
+        type: "Metaspace",
+        replacement: "▁",
+        add_prefix_space: true,
+      });
+      expect(metaspace).toBeInstanceOf(Metaspace);
+    });
+
+    it("should export Whitespace pre-tokenizer", () => {
+      expect(Whitespace).toBeDefined();
+      const whitespace = new Whitespace();
+      expect(whitespace).toBeInstanceOf(Whitespace);
+    });
+
+    it("Metaspace pre-tokenizer should work correctly", () => {
+      const metaspace = new Metaspace({
+        type: "Metaspace",
+        replacement: "▁",
+        add_prefix_space: true,
+      });
+      const result = metaspace.pre_tokenize_text("hello world");
+      expect(result).toEqual(["▁hello▁world"]);
+    });
+
+    it("Whitespace pre-tokenizer should work correctly", () => {
+      const whitespace = new Whitespace();
+      const result = whitespace.pre_tokenize_text("hello world!");
+      expect(result).toEqual(["hello", "world", "!"]);
+    });
+  });
+
+  describe("Model exports", () => {
+    it("should export BPE model", () => {
+      expect(BPE).toBeDefined();
+    });
+
+    it("BPE model should be instantiable", () => {
+      const bpe = new BPE({
+        type: "BPE",
+        vocab: { a: 0, b: 1, c: 2 },
+        merges: [["a", "b"]],
+        unk_token: "<unk>",
+        ignore_merges: false,
+      });
+      expect(bpe).toBeInstanceOf(BPE);
+    });
+  });
+
+  describe("Integration test - import paths", () => {
+    it("should support the documented import syntax", async () => {
+      // This test verifies that the documented import paths work
+      // import { Tokenizer, Encoding } from "@huggingface/tokenizers";
+      const { Tokenizer: T1 } = await import("../src/index");
+      expect(T1).toBeDefined();
+      // Encoding is a type-only export, so we can't test it at runtime
+
+      // import { Metaspace, Whitespace } from "@huggingface/tokenizers/pre-tokenizers";
+      const { Metaspace: M1, Whitespace: W1 } = await import(
+        "../src/pre-tokenizers"
+      );
+      expect(M1).toBeDefined();
+      expect(W1).toBeDefined();
+
+      // import { BPE } from "@huggingface/tokenizers/models";
+      const { BPE: B1 } = await import("../src/models");
+      expect(B1).toBeDefined();
+    });
+  });
+});

tsconfig.build.json

@@ -8,5 +8,5 @@
     "outDir": "types",
     "noEmit": false
   },
-  "include": ["src/index.ts"]
+  "include": ["src/index.ts", "src/pre-tokenizers.ts", "src/models.ts"]
 }