refactor: replace approximateTokenSize with estimateTokenCount and update tests

Author: johannschopplich

scripts/generateTable.ts

@@ -2,7 +2,7 @@ import { readFile, writeFile } from 'node:fs/promises'
 import { join } from 'node:path'
 import { fileURLToPath } from 'node:url'
 import { encode } from 'gpt-tokenizer'
-import { approximateTokenSize } from '../src/index'
+import { estimateTokenCount } from '../src/index'
 
 const rootDir = join(fileURLToPath(new URL('../', import.meta.url)))
 const readmePath = join(rootDir, 'README.md')
@@ -49,7 +49,7 @@ for (const example of tokenExamples) {
     ? (await readFile(example.input, 'utf-8'))
     : example.input
   const tokenCount = encode(text).length
-  const estimatedTokenCount = approximateTokenSize(text)
+  const estimatedTokenCount = estimateTokenCount(text)
   const errorPercentage = ((Math.abs(tokenCount - estimatedTokenCount) / tokenCount) * 100).toFixed(2)
 
   markdownTable += `| ${[

src/index.ts

@@ -34,7 +34,7 @@ export function isWithinTokenLimit(
   return estimateTokenCount(text, options) <= tokenLimit
 }
 
-// Legacy alias for backward compatibility
+/** @deprecated Use `estimateTokenCount` instead */
 export const approximateTokenSize: typeof estimateTokenCount = estimateTokenCount
 
 /**

test/index.test.ts

@@ -3,7 +3,6 @@ import { join } from 'node:path'
 import { fileURLToPath } from 'node:url'
 import { describe, expect, it } from 'vitest'
 import {
-  approximateTokenSize,
   estimateTokenCount,
   isWithinTokenLimit,
   sliceByTokens,
@@ -15,38 +14,28 @@ describe('token-related functions', () => {
   const ENGLISH_TEXT = 'Hello, world! This is a short sentence.'
   const GERMAN_TEXT = 'Die pünktlich gewünschte Trüffelfüllung im übergestülpten Würzkümmel-Würfel ist kümmerlich und dürfte fürderhin zu Rüffeln in Hülle und Fülle führen'
 
-  describe('approximateTokenSize (legacy)', () => {
-    it('should approximate the token size for short English text', () => {
-      expect(approximateTokenSize(ENGLISH_TEXT)).toMatchInlineSnapshot('11')
+  describe('estimateTokenCount', () => {
+    it('should estimate tokens for short English text', () => {
+      expect(estimateTokenCount(ENGLISH_TEXT)).toMatchInlineSnapshot('11')
     })
 
-    it('should approximate the token size for short German text with umlauts', () => {
-      expect(approximateTokenSize(GERMAN_TEXT)).toMatchInlineSnapshot('49')
+    it('should estimate tokens for German text with umlauts', () => {
+      expect(estimateTokenCount(GERMAN_TEXT)).toMatchInlineSnapshot('49')
     })
 
     it('should approximate the token size for English ebook', async () => {
       const input = await readFile(join(fixturesDir, 'ebooks/pg5200.txt'), 'utf-8')
-      expect(approximateTokenSize(input)).toMatchInlineSnapshot(`35705`)
+      expect(estimateTokenCount(input)).toMatchInlineSnapshot(`35705`)
     })
 
     it('should approximate the token size for German ebook', async () => {
       const input = await readFile(join(fixturesDir, 'ebooks/pg22367.txt'), 'utf-8')
-      expect(approximateTokenSize(input)).toMatchInlineSnapshot(`35069`)
+      expect(estimateTokenCount(input)).toMatchInlineSnapshot(`35069`)
     })
 
     it('should approximate the token size for Chinese ebook', async () => {
       const input = await readFile(join(fixturesDir, 'ebooks/pg7337.txt'), 'utf-8')
-      expect(approximateTokenSize(input)).toMatchInlineSnapshot(`12059`)
-    })
-  })
-
-  describe('estimateTokenCount', () => {
-    it('should estimate tokens for short English text', () => {
-      expect(estimateTokenCount(ENGLISH_TEXT)).toMatchInlineSnapshot('11')
-    })
-
-    it('should estimate tokens for German text with umlauts', () => {
-      expect(estimateTokenCount(GERMAN_TEXT)).toMatchInlineSnapshot('49')
+      expect(estimateTokenCount(input)).toMatchInlineSnapshot(`12059`)
     })
 
     it('should handle empty input', () => {