Updates

Author: cmdcolin

component_tests/cli-node18/run.mjs

@@ -162,13 +162,25 @@ ctgA\ttest\tgene\t5000\t6000\t.\t+\t.\tID=gene3;Name=TestGene3
 
     // Create HTTP server to serve the gzipped file
     const server = createServer((req, res) => {
+      console.log(`[SERVER] Received request: ${req.method} ${req.url}`)
       if (req.url === '/test.gff3.gz') {
         const stat = statSync(gzFile)
+        console.log(`[SERVER] Serving file, size: ${stat.size} bytes`)
         res.writeHead(200, {
           'Content-Type': 'application/gzip',
           'Content-Length': stat.size,
         })
-        createReadStream(gzFile).pipe(res)
+        const fileStream = createReadStream(gzFile)
+        fileStream.on('end', () => {
+          console.log('[SERVER] File stream ended')
+        })
+        fileStream.on('error', err => {
+          console.log('[SERVER] File stream error:', err)
+        })
+        fileStream.pipe(res)
+        res.on('finish', () => {
+          console.log('[SERVER] Response finished')
+        })
       } else {
         res.writeHead(404)
         res.end('Not found')
@@ -182,13 +194,27 @@ ctgA\ttest\tgene\t5000\t6000\t.\t+\t.\tID=gene3;Name=TestGene3
       })
       const addr = server.address()
       const port = addr.port
+      console.log(`[TEST] Server listening on port ${port}`)
 
       const outDir = join(tmpDir, 'indexes')
 
       // Run text-index command against the remote file
-      run(
-        `npx jbrowse text-index --file http://localhost:${port}/test.gff3.gz --out ${outDir} --quiet`,
-      )
+      console.log(`[TEST] Running text-index command...`)
+      const cmd = `npx jbrowse text-index --file http://localhost:${port}/test.gff3.gz --out ${outDir} --quiet`
+      console.log(`[TEST] Command: ${cmd}`)
+      try {
+        const result = execSync(cmd, {
+          encoding: 'utf8',
+          stdio: ['pipe', 'pipe', 'pipe'],
+          timeout: 60000,
+        })
+        console.log(`[TEST] Command stdout: ${result}`)
+      } catch (cmdErr) {
+        console.log(`[TEST] Command stderr: ${cmdErr.stderr}`)
+        console.log(`[TEST] Command stdout: ${cmdErr.stdout}`)
+        throw cmdErr
+      }
+      console.log(`[TEST] Command completed`)
 
       // Check that index files were created
       // The command creates .ix and .ixx files

packages/text-indexing-core/src/types/common.ts

@@ -50,13 +50,16 @@ export async function getLocalOrRemoteStream({
   let receivedBytes = 0
 
   if (isURL(file)) {
+    console.error(`[DEBUG] Fetching remote file: ${file}`)
     const res = await fetch(file)
+    console.error(`[DEBUG] Fetch response status: ${res.status}`)
     if (!res.ok) {
       throw new Error(
         `Failed to fetch ${file} status ${res.status} ${await res.text()}`,
       )
     }
     const totalBytes = +(res.headers.get('Content-Length') || 0)
+    console.error(`[DEBUG] Content-Length: ${totalBytes}`)
     onStart(totalBytes)
 
     const body = res.body
@@ -67,9 +70,20 @@ export async function getLocalOrRemoteStream({
 
     body.on('data', chunk => {
       receivedBytes += chunk.length
+      console.error(
+        `[DEBUG] Received chunk: ${chunk.length} bytes, total: ${receivedBytes}`,
+      )
       onUpdate(receivedBytes)
     })
 
+    body.on('end', () => {
+      console.error(`[DEBUG] Body stream ended, total bytes: ${receivedBytes}`)
+    })
+
+    body.on('error', err => {
+      console.error(`[DEBUG] Body stream error:`, err)
+    })
+
     return body as unknown as Readable
   } else {
     // Handle file:// URLs by converting to local path
@@ -94,13 +108,33 @@ export function createReadlineInterface(
   stream: Readable,
   inLocation: string,
 ): readline.Interface {
-  const inputStream = /.b?gz$/.exec(inLocation)
-    ? stream.pipe(createGunzip())
-    : stream
+  console.error(`[DEBUG] createReadlineInterface: inLocation=${inLocation}`)
+  const isGzipped = /.b?gz$/.exec(inLocation)
+  console.error(`[DEBUG] Is gzipped: ${!!isGzipped}`)
 
-  return readline.createInterface({
+  let inputStream: Readable
+  if (isGzipped) {
+    const gunzip = createGunzip()
+    gunzip.on('error', err => {
+      console.error(`[DEBUG] Gunzip error:`, err)
+    })
+    gunzip.on('end', () => {
+      console.error(`[DEBUG] Gunzip stream ended`)
+    })
+    inputStream = stream.pipe(gunzip)
+  } else {
+    inputStream = stream
+  }
+
+  const rl = readline.createInterface({
     input: inputStream,
   })
+
+  rl.on('close', () => {
+    console.error(`[DEBUG] Readline interface closed`)
+  })
+
+  return rl
 }
 
 // Efficient attribute parsing for GFF3/VCF info fields

packages/text-indexing-core/src/types/gff3Adapter.ts

@@ -23,23 +23,35 @@ export async function* indexGff3({
   onUpdate: (progressBytes: number) => void
 }) {
   const { trackId } = config
+  console.error(
+    `[DEBUG] indexGff3 starting for trackId=${trackId}, inLocation=${inLocation}`,
+  )
   const stream = await getLocalOrRemoteStream({
     file: inLocation,
     out: outDir,
     onStart,
     onUpdate,
   })
+  console.error(`[DEBUG] Got stream, creating readline interface`)
 
   const rl = createReadlineInterface(stream, inLocation)
   const excludeSet = new Set(featureTypesToExclude)
   const encodedTrackId = encodeURIComponent(trackId)
 
+  let lineCount = 0
+  let yieldCount = 0
+  console.error(`[DEBUG] Starting to read lines`)
   for await (const line of rl) {
+    lineCount++
+    if (lineCount <= 5 || lineCount % 100 === 0) {
+      console.error(`[DEBUG] Read line ${lineCount}: ${line.slice(0, 50)}...`)
+    }
     if (!line.trim()) {
       continue
     } else if (line.startsWith('#')) {
       continue
     } else if (line.startsWith('>')) {
+      console.error(`[DEBUG] Found FASTA header, breaking at line ${lineCount}`)
       break
     }
 
@@ -57,8 +69,12 @@ export async function* indexGff3({
         const record = `["${encodeURIComponent(locStr)}"|"${encodedTrackId}"|${encodedAttrs.join('|')}]`
         const uniqueAttrs = [...new Set(attrs)]
 
+        yieldCount++
         yield `${record} ${uniqueAttrs.join(' ')}\n`
       }
     }
   }
+  console.error(
+    `[DEBUG] indexGff3 finished: read ${lineCount} lines, yielded ${yieldCount} records`,
+  )
 }

products/jbrowse-cli/src/commands/text-index/indexing-utils.ts

@@ -28,6 +28,7 @@ export async function runIxIxx({
   prefixSize?: number
   quiet?: boolean
 }): Promise<void> {
+  console.error(`[DEBUG] runIxIxx starting: name=${name}`)
   const progressBar = new SingleBar(
     { format: '{bar} Sorting and writing index...', etaBuffer: 2000 },
     Presets.shades_classic,
@@ -37,12 +38,13 @@ export async function runIxIxx({
     progressBar.start(1, 0)
   }
 
-  await ixIxxStream(
-    readStream,
-    path.join(outLocation, 'trix', `${name}.ix`),
-    path.join(outLocation, 'trix', `${name}.ixx`),
-    prefixSize,
-  )
+  const ixPath = path.join(outLocation, 'trix', `${name}.ix`)
+  const ixxPath = path.join(outLocation, 'trix', `${name}.ixx`)
+  console.error(`[DEBUG] Calling ixIxxStream: ix=${ixPath}, ixx=${ixxPath}`)
+
+  await ixIxxStream(readStream, ixPath, ixxPath, prefixSize)
+
+  console.error(`[DEBUG] ixIxxStream completed`)
 
   if (!quiet) {
     progressBar.update(1)
@@ -146,6 +148,11 @@ export async function indexDriver({
   assemblyNames: string[]
   prefixSize?: number
 }): Promise<void> {
+  console.error(
+    `[DEBUG] indexDriver starting: name=${name}, outLocation=${outLocation}`,
+  )
+  console.error(`[DEBUG] trackConfigs count: ${trackConfigs.length}`)
+
   const readStream = Readable.from(
     indexFiles({
       trackConfigs,
@@ -156,6 +163,8 @@ export async function indexDriver({
     }),
   )
 
+  console.error(`[DEBUG] Created readStream from indexFiles, calling runIxIxx`)
+
   await runIxIxx({
     readStream,
     outLocation,
@@ -164,6 +173,8 @@ export async function indexDriver({
     quiet,
   })
 
+  console.error(`[DEBUG] runIxIxx completed, generating meta`)
+
   await generateMeta({
     configs: trackConfigs,
     attributesToIndex: attributes,
@@ -172,6 +183,8 @@ export async function indexDriver({
     featureTypesToExclude: typesToExclude,
     assemblyNames,
   })
+
+  console.error(`[DEBUG] indexDriver completed`)
 }
 
 export function prepareFileTrackConfigs(