chore(llmobs): use csv parser for dataset import

Author: mehulsonowal

package.json

@@ -169,6 +169,7 @@
     "version.js"
   ],
   "dependencies": {
+    "csv-parse": "^7.0.1",
     "dc-polyfill": "^0.1.11",
     "import-in-the-middle": "^3.3.1",
     "opentracing": ">=0.14.7"

packages/dd-trace/src/llmobs/experiments/DATASET_PARITY_TODO.md

@@ -2,6 +2,8 @@
 
 const fs = require('node:fs')
 
+const { parse: parseCsv } = require('csv-parse/sync')
+
 const log = require('../../log')
 const { ExperimentsClient, API_BASE_PATH } = require('./client')
 const { Dataset, DatasetRecord } = require('./dataset')
@@ -24,49 +26,6 @@ async function retryWithBackoff (attempt, { maxTotalMs = 30_000, baseDelayMs = 2
   }
 }
 
-function parseCsv (content, delimiter) {
-  const rows = []
-  let field = ''
-  let row = []
-  let inQuotes = false
-
-  for (let i = 0; i < content.length; i++) {
-    const char = content[i]
-    if (inQuotes) {
-      if (char === '"') {
-        if (content[i + 1] === '"') {
-          field += '"'
-          i++
-        } else {
-          inQuotes = false
-        }
-      } else {
-        field += char
-      }
-    } else if (char === '"') {
-      inQuotes = true
-    } else if (char === delimiter) {
-      row.push(field)
-      field = ''
-    } else if (char === '\n' || char === '\r') {
-      row.push(field)
-      field = ''
-      rows.push(row)
-      row = []
-      if (char === '\r' && content[i + 1] === '\n') i++
-    } else {
-      field += char
-    }
-  }
-
-  if (field !== '' || row.length > 0) {
-    row.push(field)
-    rows.push(row)
-  }
-
-  return rows
-}
-
 function assertColumns (name, columns, allowEmpty = true) {
   if (!Array.isArray(columns) || (!allowEmpty && columns.length === 0)) {
     throw new Error(`${name} must be a ${allowEmpty ? '' : 'non-empty '}array of column names`)
@@ -106,7 +65,11 @@ function recordsFromCsv (csvPath, options) {
   const content = fs.readFileSync(csvPath, 'utf8')
   if (content.trim().length === 0) throw new Error('CSV file appears to be empty or header is missing')
 
-  const rows = parseCsv(content, csvDelimiter)
+  const rows = parseCsv(content, {
+    delimiter: csvDelimiter,
+    bom: true,
+    relaxColumnCount: true,
+  })
   const header = rows.shift()
   if (!header || header.every(column => column === '')) {
     throw new Error('CSV file appears to be empty or header is missing')

packages/dd-trace/src/llmobs/experiments/index.js

@@ -1747,6 +1747,11 @@ crypt@0.0.2:
   resolved "https://registry.yarnpkg.com/crypt/-/crypt-0.0.2.tgz#88d7ff7ec0dfb86f713dc87bbb42d044d3e6c41b"
   integrity sha512-mCxBlsHFYh9C+HVpiEacem8FEBnMXgU9gy4zmNC+SXAZNB/1idgp/aulFJ4FgCi7GPEVbfyng092GqL2k2rmow==
 
+csv-parse@^7.0.1:
+  version "7.0.1"
+  resolved "https://registry.yarnpkg.com/csv-parse/-/csv-parse-7.0.1.tgz#77d0f216dc25af5e8006f3b7899526081dd81b07"
+  integrity sha512-+2z7Ar0APQ7Uu6fX4cn+pitRmxjZ1WPBcGmZFKmA74FCyi7Et/XZx8cjNQ5CjbZ4HCOxXCOpRBYvYH08Qa003A==
+
 data-view-buffer@^1.0.2:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/data-view-buffer/-/data-view-buffer-1.0.2.tgz#211a03ba95ecaf7798a8c7198d79536211f88570"