Add more tests

Author: cmdcolin

src/externalSort.ts

@@ -1,6 +1,7 @@
 import fs from 'fs'
 import path from 'path'
 import { Readable, Writable } from 'stream'
+import { pipeline } from 'stream/promises'
 
 const EOF = Symbol('EOF')
 
@@ -15,6 +16,8 @@ class FileParser {
   private bytesRead = 0
   private file: string
   private delimiter: string
+  private fh: fs.promises.FileHandle | undefined
+  private eof = false
 
   constructor(file: string, delimiter: string) {
     this.file = file
@@ -38,45 +41,49 @@ class FileParser {
       return this.checkBuffer()
     }
 
-    const fh = await fs.promises.open(this.file, 'r')
+    if (this.eof) {
+      return EOF
+    }
+
+    if (!this.fh) {
+      this.fh = await fs.promises.open(this.file, 'r')
+    }
+
     const cBuffer = Buffer.alloc(512)
     let readed: { bytesRead: number }
 
-    try {
-      while (
-        (readed = await fh.read(cBuffer, 0, 512, this.bytesRead)).bytesRead > 0
-      ) {
-        this.bbuffer = Buffer.concat([
-          this.bbuffer,
-          cBuffer.subarray(0, readed.bytesRead),
-        ])
-        this.bytesRead += readed.bytesRead
-        const dIndex = this.bbuffer.indexOf(this.delimiter)
-        if (dIndex === -1) {
-          continue
-        }
-        this.buffer = this.bbuffer.subarray(0, dIndex + 1).toString('utf8')
-        this.bbuffer = this.bbuffer.subarray(dIndex + 1)
-        return this.checkBuffer()
-      }
-
-      if (this.bbuffer.length > 0 && this.bbuffer.includes(this.delimiter)) {
-        this.buffer = this.bbuffer.toString('utf8')
-        this.bbuffer = Buffer.from('')
-        return this.checkBuffer()
+    while (
+      (readed = await this.fh.read(cBuffer, 0, 512, this.bytesRead)).bytesRead >
+      0
+    ) {
+      this.bbuffer = Buffer.concat([
+        this.bbuffer,
+        cBuffer.subarray(0, readed.bytesRead),
+      ])
+      this.bytesRead += readed.bytesRead
+      const dIndex = this.bbuffer.indexOf(this.delimiter)
+      if (dIndex === -1) {
+        continue
       }
+      this.buffer = this.bbuffer.subarray(0, dIndex + 1).toString('utf8')
+      this.bbuffer = this.bbuffer.subarray(dIndex + 1)
+      return this.checkBuffer()
+    }
 
-      return EOF
-    } finally {
-      await fh.close()
+    if (this.bbuffer.length > 0 && this.bbuffer.includes(this.delimiter)) {
+      this.buffer = this.bbuffer.toString('utf8')
+      this.bbuffer = Buffer.from('')
+      return this.checkBuffer()
     }
+
+    this.eof = true
+    await this.fh.close()
+    return EOF
   }
 }
 
 function swap(harr: HeapItem[], a: number, b: number) {
-  const temp = harr[a]
-  harr[a] = harr[b]!
-  harr[b] = temp
+  ;[harr[a], harr[b]] = [harr[b], harr[a]]
 }
 
 function compare(a: string | typeof EOF, b: string | typeof EOF): number {
@@ -86,13 +93,7 @@ function compare(a: string | typeof EOF, b: string | typeof EOF): number {
   if (b === EOF) {
     return -1
   }
-  if (a < b) {
-    return -1
-  }
-  if (a === b) {
-    return 0
-  }
-  return 1
+  return a < b ? -1 : a > b ? 1 : 0
 }
 
 function heapify(harr: HeapItem[], i: number, heapSize: number) {
@@ -189,19 +190,8 @@ async function mergeSortedFiles(
   const flen = filesPath.length
 
   if (flen === 1) {
-    await new Promise<void>((resolve, reject) => {
-      const rs = fs.createReadStream(filesPath[0], 'utf8')
-      rs.on('open', () => {
-        rs.pipe(output)
-      })
-      rs.on('error', err => {
-        output.end()
-        reject(err)
-      })
-      rs.on('end', () => {
-        resolve()
-      })
-    })
+    const rs = fs.createReadStream(filesPath[0], 'utf8')
+    await pipeline(rs, output)
     return
   }
 

src/makeIx.ts

@@ -1,6 +1,6 @@
 import { spawn } from 'child_process'
 import fs from 'fs'
-import { Readable } from 'stream'
+import { PassThrough, Readable } from 'stream'
 import { pipeline } from 'stream/promises'
 
 import { sync as commandExistsSync } from 'command-exists'
@@ -48,10 +48,14 @@ async function makeIxWithExternalSort(
 async function makeIxWithJsSort(fileStream: Readable, outIxFilename: string) {
   const out = fs.createWriteStream(outIxFilename)
 
-  // Transform input
-  const transformedInput = fileStream
-    .pipe(split2())
-    .pipe(new TrixInputTransform())
+  // Transform input using pipeline for proper error handling
+  const transformedInput = new PassThrough()
+  const inputDone = pipeline(
+    fileStream,
+    split2(),
+    new TrixInputTransform(),
+    transformedInput,
+  )
 
   // Sort lines using external merge sort
   const sortedOutput = split2()
@@ -60,7 +64,7 @@ async function makeIxWithJsSort(fileStream: Readable, outIxFilename: string) {
   // Transform sorted output and write to file
   const writeDone = pipeline(sortedOutput, new TrixOutputTransform(), out)
 
-  await Promise.all([sortDone, writeDone])
+  await Promise.all([inputDone, sortDone, writeDone])
 }
 
 export async function makeIxStream(

src/sortLines.ts

@@ -24,60 +24,3 @@ export async function sortLinesExternal(
     }
   }
 }
-
-/**
- * Sort lines in memory. Only use for small inputs.
- */
-export async function sortLinesInMemory(
-  input: Readable,
-  output: Writable,
-): Promise<void> {
-  const lines: string[] = []
-
-  await new Promise<void>((resolve, reject) => {
-    let buffer = ''
-
-    input.on('data', function onData(chunk: Buffer | string) {
-      buffer += chunk.toString()
-      const parts = buffer.split('\n')
-      buffer = parts.pop()!
-      for (const line of parts) {
-        if (line) {
-          lines.push(line)
-        }
-      }
-    })
-
-    input.on('end', function onEnd() {
-      if (buffer) {
-        lines.push(buffer)
-      }
-      resolve()
-    })
-
-    input.on('error', reject)
-  })
-
-  lines.sort()
-
-  await new Promise<void>((resolve, reject) => {
-    let i = 0
-
-    function writeNext() {
-      let ok = true
-      while (i < lines.length && ok) {
-        ok = output.write(lines[i] + '\n')
-        i++
-      }
-      if (i < lines.length) {
-        output.once('drain', writeNext)
-      } else {
-        output.end()
-        resolve()
-      }
-    }
-
-    output.on('error', reject)
-    writeNext()
-  })
-}

test/TrixInputTransform.test.ts

@@ -0,0 +1,59 @@
+import { Readable } from 'stream'
+import { pipeline } from 'stream/promises'
+
+import split2 from 'split2'
+import { describe, expect, test } from 'vitest'
+
+import { TrixInputTransform } from '../src/TrixInputTransform.ts'
+
+async function transformInput(lines: string[]): Promise<string> {
+  const input = Readable.from(lines.map(l => l + '\n'))
+  const chunks: string[] = []
+
+  await pipeline(input, split2(), new TrixInputTransform(), async function* (
+    source,
+  ) {
+    for await (const chunk of source) {
+      chunks.push(chunk.toString())
+    }
+  })
+
+  return chunks.join('')
+}
+
+describe('TrixInputTransform', () => {
+  test('transforms single word line', async () => {
+    const result = await transformInput(['id123 hello'])
+    expect(result).toBe('hello id123\n')
+  })
+
+  test('transforms multiple words', async () => {
+    const result = await transformInput(['id123 hello world test'])
+    expect(result).toBe('hello id123\nworld id123\ntest id123\n')
+  })
+
+  test('lowercases words', async () => {
+    const result = await transformInput(['id123 HELLO World TeSt'])
+    expect(result).toBe('hello id123\nworld id123\ntest id123\n')
+  })
+
+  test('handles line with only id (no words)', async () => {
+    const result = await transformInput(['id123'])
+    expect(result).toBe('')
+  })
+
+  test('handles multiple lines', async () => {
+    const result = await transformInput(['id1 apple', 'id2 banana'])
+    expect(result).toBe('apple id1\nbanana id2\n')
+  })
+
+  test('handles multiple whitespace between words', async () => {
+    const result = await transformInput(['id123   hello    world'])
+    expect(result).toBe('hello id123\nworld id123\n')
+  })
+
+  test('handles empty input', async () => {
+    const result = await transformInput([])
+    expect(result).toBe('')
+  })
+})

test/TrixInputTransformEdgeCases.test.ts

@@ -0,0 +1,88 @@
+import { Readable } from 'stream'
+import { pipeline } from 'stream/promises'
+
+import split2 from 'split2'
+import { describe, expect, test } from 'vitest'
+
+import { TrixInputTransform } from '../src/TrixInputTransform.ts'
+
+async function transformInput(lines: string[]): Promise<string> {
+  const input = Readable.from(lines.map(l => l + '\n'))
+  const chunks: string[] = []
+
+  await pipeline(input, split2(), new TrixInputTransform(), async function* (
+    source,
+  ) {
+    for await (const chunk of source) {
+      chunks.push(chunk.toString())
+    }
+  })
+
+  return chunks.join('')
+}
+
+describe('TrixInputTransform edge cases', () => {
+  test('filters out empty strings from split', async () => {
+    // Multiple spaces create empty strings when split
+    const result = await transformInput(['id1   word1   word2'])
+    // Should only have word1 and word2, not empty entries
+    const lines = result.trim().split('\n')
+    expect(lines.every(l => l.length > 0)).toBe(true)
+    expect(lines).toHaveLength(2)
+  })
+
+  test('handles tab characters as whitespace', async () => {
+    const result = await transformInput(['id1\tword1\tword2'])
+    expect(result).toContain('word1')
+    expect(result).toContain('word2')
+  })
+
+  test('handles mixed whitespace', async () => {
+    const result = await transformInput(['id1 \t word1 \t word2'])
+    const lines = result.trim().split('\n').filter(l => l.length > 0)
+    expect(lines).toHaveLength(2)
+  })
+
+  test('preserves numbers in words', async () => {
+    const result = await transformInput(['id1 abc123 456def'])
+    expect(result).toContain('abc123')
+    expect(result).toContain('456def')
+  })
+
+  test('handles hyphenated words', async () => {
+    const result = await transformInput(['id1 well-known self-aware'])
+    expect(result).toContain('well-known')
+    expect(result).toContain('self-aware')
+  })
+
+  test('handles words with underscores', async () => {
+    const result = await transformInput(['id1 snake_case SCREAMING_SNAKE'])
+    expect(result).toContain('snake_case')
+    expect(result).toContain('screaming_snake')
+  })
+
+  test('handles unicode words', async () => {
+    const result = await transformInput(['id1 café naïve résumé'])
+    expect(result).toContain('café')
+    expect(result).toContain('naïve')
+    expect(result).toContain('résumé')
+  })
+
+  test('handles CJK characters', async () => {
+    const result = await transformInput(['id1 日本語 中文'])
+    expect(result).toContain('日本語')
+    expect(result).toContain('中文')
+  })
+
+  test('handles emoji', async () => {
+    const result = await transformInput(['id1 hello🎉 world🌍'])
+    expect(result).toContain('hello🎉')
+    expect(result).toContain('world🌍')
+  })
+
+  test('lowercases unicode letters', async () => {
+    const result = await transformInput(['id1 MÜNCHEN ZÜRICH'])
+    expect(result).toContain('münchen')
+    expect(result).toContain('zürich')
+  })
+})