File tree Expand file tree Collapse file tree 3 files changed +779
-723
lines changed Expand file tree Collapse file tree 3 files changed +779
-723
lines changed Original file line number Diff line number Diff line change @@ -14,11 +14,11 @@ The following table shows the accuracy of the token count approximation for diff
1414| Description | Actual GPT Token Count | Estimated Token Count | Token Count Deviation |
1515| --- | --- | --- | --- |
1616| Short English text | 10 | 11 | 10.00% |
17- | German text with umlauts | 56 | 49 | 12.50 % |
18- | Metamorphosis by Franz Kafka (English) | 31892 | 35705 | 11.96 % |
19- | Die Verwandlung by Franz Kafka (German) | 40621 | 35069 | 13.67 % |
20- | 道德經 by Laozi (Chinese) | 14387 | 12059 | 16.18 % |
21- | TypeScript ES5 Type Declarations (~ 4000 loc) | 48553 | 52434 | 7.99 % |
17+ | German text with umlauts | 48 | 49 | 2.08 % |
18+ | Metamorphosis by Franz Kafka (English) | 31796 | 35705 | 12.29 % |
19+ | Die Verwandlung by Franz Kafka (German) | 35309 | 35069 | 0.68 % |
20+ | 道德經 by Laozi (Chinese) | 11712 | 12059 | 2.96 % |
21+ | TypeScript ES5 Type Declarations (~ 4000 loc) | 49304 | 52615 | 6.72 % |
2222<!-- END GENERATED TOKEN COUNT TABLE -->
2323
2424## Features
Original file line number Diff line number Diff line change 22 "name" : " tokenx" ,
33 "type" : " module" ,
44 "version" : " 1.1.0" ,
5- "packageManager" : " pnpm@10.16.1 " ,
5+ "packageManager" : " pnpm@10.18.3 " ,
66 "description" : " Fast and lightweight token estimation for any LLM without requiring a full tokenizer" ,
77 "author" :
" Johann Schopplich <[email protected] >" ,
88 "license" : " MIT" ,
4545 "release" : " bumpp"
4646 },
4747 "devDependencies" : {
48- "@antfu/eslint-config" : " ^5.3 .0" ,
49- "@types/node" : " ^24.5.0 " ,
50- "bumpp" : " ^10.2.3 " ,
51- "eslint" : " ^9.35 .0" ,
52- "gpt-tokenizer" : " ^3.0.1 " ,
53- "tsdown" : " ^0.15.1 " ,
54- "tsx" : " ^4.20.5 " ,
55- "typescript" : " ^5.9.2 " ,
48+ "@antfu/eslint-config" : " ^6.0 .0" ,
49+ "@types/node" : " ^24.7.2 " ,
50+ "bumpp" : " ^10.3.1 " ,
51+ "eslint" : " ^9.37 .0" ,
52+ "gpt-tokenizer" : " ^3.2.0 " ,
53+ "tsdown" : " ^0.15.7 " ,
54+ "tsx" : " ^4.20.6 " ,
55+ "typescript" : " ^5.9.3 " ,
5656 "vitest" : " ^3.2.4"
5757 }
5858}
You can’t perform that action at this time.
0 commit comments