Fix exports to match Python API: remove duplicates, add base classes, use correct short names

Co-authored-by: xenova <[email protected]>

Author: Copilot

src/decoders.ts

@@ -1,10 +1,11 @@
-export { default as BPEDecoder } from "./core/decoder/BPEDecoder";
+export { default as Decoder } from "./core/Decoder";
+export { default as ByteLevel } from "./core/decoder/ByteLevelDecoder";
+export { default as Replace } from "./core/decoder/ReplaceDecoder";
+export { default as WordPiece } from "./core/decoder/WordPieceDecoder";
 export { default as ByteFallback } from "./core/decoder/ByteFallback";
-export { default as ByteLevelDecoder } from "./core/decoder/ByteLevelDecoder";
-export { default as CTCDecoder } from "./core/decoder/CTCDecoder";
-export { default as DecoderSequence } from "./core/decoder/DecoderSequence";
-export { default as FuseDecoder } from "./core/decoder/FuseDecoder";
-export { default as MetaspaceDecoder } from "./core/decoder/MetaspaceDecoder";
-export { default as ReplaceDecoder } from "./core/decoder/ReplaceDecoder";
-export { default as StripDecoder } from "./core/decoder/StripDecoder";
-export { default as WordPieceDecoder } from "./core/decoder/WordPieceDecoder";
+export { default as Fuse } from "./core/decoder/FuseDecoder";
+export { default as Strip } from "./core/decoder/StripDecoder";
+export { default as Metaspace } from "./core/decoder/MetaspaceDecoder";
+export { default as BPEDecoder } from "./core/decoder/BPEDecoder";
+export { default as CTC } from "./core/decoder/CTCDecoder";
+export { default as Sequence } from "./core/decoder/DecoderSequence";

src/models.ts

@@ -1,5 +1,4 @@
+export { default as Model } from "./core/TokenizerModel";
 export { default as BPE } from "./core/tokenizerModelImplementations/BPE";
-export { default as LegacyTokenizerModel } from "./core/tokenizerModelImplementations/LegacyTokenizerModel";
 export { default as Unigram } from "./core/tokenizerModelImplementations/Unigram";
 export { default as WordPiece } from "./core/tokenizerModelImplementations/WordPieceTokenizer";
-export { default as WordPieceTokenizer } from "./core/tokenizerModelImplementations/WordPieceTokenizer";

src/normalizers.ts

@@ -1,13 +1,13 @@
+export { default as Normalizer } from "./core/Normalizer";
 export { default as BertNormalizer } from "./core/normalizer/BertNormalizer";
-export { default as Lowercase } from "./core/normalizer/Lowercase";
-export { default as NFC } from "./core/normalizer/NFC";
 export { default as NFD } from "./core/normalizer/NFD";
-export { default as NFKC } from "./core/normalizer/NFKC";
 export { default as NFKD } from "./core/normalizer/NFKD";
-export { default as NormalizerSequence } from "./core/normalizer/NormalizerSequence";
-export { default as Precompiled } from "./core/normalizer/Precompiled";
+export { default as NFC } from "./core/normalizer/NFC";
+export { default as NFKC } from "./core/normalizer/NFKC";
+export { default as Sequence } from "./core/normalizer/NormalizerSequence";
+export { default as Lowercase } from "./core/normalizer/Lowercase";
 export { default as Prepend } from "./core/normalizer/Prepend";
-export { default as Replace } from "./core/normalizer/Replace";
+export { default as Strip } from "./core/normalizer/StripNormalizer";
 export { default as StripAccents } from "./core/normalizer/StripAccents";
-export { default as StripNormalizer } from "./core/normalizer/StripNormalizer";
-export { default as UnicodeNormalizer } from "./core/normalizer/UnicodeNormalizer";
+export { default as Precompiled } from "./core/normalizer/Precompiled";
+export { default as Replace } from "./core/normalizer/Replace";

src/post-processors.ts

@@ -1,5 +1,6 @@
+export { default as PostProcessor } from "./core/PostProcessor";
 export { default as BertProcessing } from "./core/postProcessor/BertProcessing";
-export { default as ByteLevelPostProcessor } from "./core/postProcessor/ByteLevelPostProcessor";
-export { default as PostProcessorSequence } from "./core/postProcessor/PostProcessorSequence";
+export { default as ByteLevel } from "./core/postProcessor/ByteLevelPostProcessor";
 export { default as RobertaProcessing } from "./core/postProcessor/RobertaProcessing";
+export { default as Sequence } from "./core/postProcessor/PostProcessorSequence";
 export { default as TemplateProcessing } from "./core/postProcessor/TemplateProcessing";

src/pre-tokenizers.ts

@@ -1,12 +1,10 @@
+export { default as PreTokenizer } from "./core/PreTokenizer";
 export { default as BertPreTokenizer } from "./core/preTokenizer/BertPreTokenizer";
-export { default as ByteLevelPreTokenizer } from "./core/preTokenizer/ByteLevelPreTokenizer";
-export { default as DigitsPreTokenizer } from "./core/preTokenizer/DigitsPreTokenizer";
+export { default as ByteLevel } from "./core/preTokenizer/ByteLevelPreTokenizer";
+export { default as Digits } from "./core/preTokenizer/DigitsPreTokenizer";
 export { default as Metaspace } from "./core/preTokenizer/MetaspacePreTokenizer";
-export { default as MetaspacePreTokenizer } from "./core/preTokenizer/MetaspacePreTokenizer";
-export { default as PreTokenizerSequence } from "./core/preTokenizer/PreTokenizerSequence";
-export { default as PunctuationPreTokenizer } from "./core/preTokenizer/PunctuationPreTokenizer";
-export { default as ReplacePreTokenizer } from "./core/preTokenizer/ReplacePreTokenizer";
-export { default as SplitPreTokenizer } from "./core/preTokenizer/SplitPreTokenizer";
+export { default as Punctuation } from "./core/preTokenizer/PunctuationPreTokenizer";
+export { default as Sequence } from "./core/preTokenizer/PreTokenizerSequence";
+export { default as Split } from "./core/preTokenizer/SplitPreTokenizer";
 export { default as Whitespace } from "./core/preTokenizer/WhitespacePreTokenizer";
-export { default as WhitespacePreTokenizer } from "./core/preTokenizer/WhitespacePreTokenizer";
 export { default as WhitespaceSplit } from "./core/preTokenizer/WhitespaceSplit";