[Store] Add CSV loader to load data from CSV file as text documents

src/store/CHANGELOG.md

@@ -4,6 +4,7 @@ CHANGELOG
 0.4
 ---
 
+ * Add `CsvLoader` for loading documents from CSV files
  * Add `StoreInterface::remove()` method
  * Add `SourceIndexer` for indexing from sources (file paths, URLs, etc.) using a `LoaderInterface`
  * Add `DocumentIndexer` for indexing documents directly without a loader

src/store/src/Document/Loader/CsvLoader.php

@@ -0,0 +1,182 @@
+<?php
+
+/*
+ * This file is part of the Symfony package.
+ *
+ * (c) Fabien Potencier <fabien@symfony.com>
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+namespace Symfony\AI\Store\Document\Loader;
+
+use Symfony\AI\Store\Document\LoaderInterface;
+use Symfony\AI\Store\Document\Metadata;
+use Symfony\AI\Store\Document\TextDocument;
+use Symfony\AI\Store\Exception\InvalidArgumentException;
+use Symfony\AI\Store\Exception\RuntimeException;
+use Symfony\Component\Uid\Uuid;
+
+/**
+ * Loads documents from CSV files.
+ *
+ * Each row becomes a TextDocument. One column provides the content,
+ * optional columns can be mapped as metadata.
+ *
+ * @author Ramy Hakam <ramyhakam1@gmail.com>
+ */
+final class CsvLoader implements LoaderInterface
+{
+    public const OPTION_CONTENT_COLUMN = 'content_column';
+    public const OPTION_ID_COLUMN = 'id_column';
+    public const OPTION_METADATA_COLUMNS = 'metadata_columns';
+    public const OPTION_DELIMITER = 'delimiter';
+    public const OPTION_ENCLOSURE = 'enclosure';
+    public const OPTION_ESCAPE = 'escape';
+    public const OPTION_HAS_HEADER = 'has_header';
+
+    /**
+     * @param array<string|int> $metadataColumns
+     */
+    public function __construct(
+        private readonly string|int $contentColumn = 'content',
+        private readonly string|int|null $idColumn = null,
+        private readonly array $metadataColumns = [],
+        private readonly string $delimiter = ',',
+        private readonly string $enclosure = '"',
+        private readonly string $escape = '\\',
+        private readonly bool $hasHeader = true,
+    ) {
+    }
+
+    public function load(?string $source = null, array $options = []): iterable
+    {
+        if (null === $source) {
+            throw new InvalidArgumentException('CSV loader requires a file path as source.');
+        }
+
+        if (!is_file($source)) {
+            throw new RuntimeException(\sprintf('File "%s" does not exist.', $source));
+        }
+
+        $handle = fopen($source, 'r');
+        if (false === $handle) {
+            throw new RuntimeException(\sprintf('Unable to open file "%s".', $source));
+        }
+
+        $contentColumn = $options[self::OPTION_CONTENT_COLUMN] ?? $this->contentColumn;
+        $idColumn = $options[self::OPTION_ID_COLUMN] ?? $this->idColumn;
+        $metadataColumns = $options[self::OPTION_METADATA_COLUMNS] ?? $this->metadataColumns;
+        $delimiter = $options[self::OPTION_DELIMITER] ?? $this->delimiter;
+        $enclosure = $options[self::OPTION_ENCLOSURE] ?? $this->enclosure;
+        $escape = $options[self::OPTION_ESCAPE] ?? $this->escape;
+        $hasHeader = $options[self::OPTION_HAS_HEADER] ?? $this->hasHeader;
+
+        try {
+            $headers = null;
+            $rowIndex = 0;
+
+            while (false !== ($row = fgetcsv($handle, 0, $delimiter, $enclosure, $escape))) {
+                if ([null] === $row) {
+                    continue;
+                }
+
+                if ($hasHeader && null === $headers) {
+                    $headers = $row;
+
+                    if (\is_string($contentColumn) && !\in_array($contentColumn, $headers, true)) {
+                        throw new InvalidArgumentException(\sprintf('Content column "%s" not found in CSV headers.', $contentColumn));
+                    }
+
+                    continue;
+                }
+
+                $data = $this->normalizeRow($row, $headers, $source, $rowIndex);
+
+                $content = $this->getValue($data, $contentColumn);
+                if (null === $content || '' === trim($content)) {
+                    ++$rowIndex;
+                    continue;
+                }
+
+                $documentId = $this->resolveDocumentId($data, $idColumn);
+                $metadata = $this->buildMetadata($data, $metadataColumns, $source, $rowIndex);
+
+                yield new TextDocument(
+                    $documentId,
+                    trim($content),
+                    new Metadata($metadata)
+                );
+
+                ++$rowIndex;
+            }
+        } finally {
+            fclose($handle);
+        }
+    }
+
+    /**
+     * @param array<int, string>      $row
+     * @param array<int, string>|null $headers
+     *
+     * @return array<string|int, string>
+     */
+    private function normalizeRow(array $row, ?array $headers, string $source, int $rowIndex): array
+    {
+        if (null === $headers) {
+            return $row;
+        }
+
+        if (\count($row) !== \count($headers)) {
+            $row = array_pad($row, \count($headers), '');
+        }
+
+        return array_combine($headers, $row);
+    }
+
+    /**
+     * @param array<string|int, string> $data
+     */
+    private function resolveDocumentId(array $data, string|int|null $idColumn): string
+    {
+        if (null === $idColumn) {
+            return Uuid::v4();
+        }
+
+        $id = $this->getValue($data, $idColumn);
+
+        return null !== $id && '' !== $id ? $id : Uuid::v4();
+    }
+
+    /**
+     * @param array<string|int, string> $data
+     * @param array<string|int>         $metadataColumns
+     *
+     * @return array<string, mixed>
+     */
+    private function buildMetadata(array $data, array $metadataColumns, string $source, int $rowIndex): array
+    {
+        $metadata = [
+            Metadata::KEY_SOURCE => $source,
+            '_row_index' => $rowIndex,
+        ];
+
+        foreach ($metadataColumns as $column) {
+            $value = $this->getValue($data, $column);
+            if (null !== $value) {
+                $metadata[\is_int($column) ? 'column_'.$column : $column] = $value;
+            }
+        }
+
+        return $metadata;
+    }
+
+    /**
+     * @param array<string|int, string> $data
+     */
+    private function getValue(array $data, string|int $key): ?string
+    {
+        return \array_key_exists($key, $data) ? $data[$key] : null;
+    }
+}

src/store/tests/Document/Loader/CsvLoaderTest.php

@@ -0,0 +1,190 @@
+<?php
+
+/*
+ * This file is part of the Symfony package.
+ *
+ * (c) Fabien Potencier <fabien@symfony.com>
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+namespace Symfony\AI\Store\Tests\Document\Loader;
+
+use PHPUnit\Framework\TestCase;
+use Symfony\AI\Store\Document\Loader\CsvLoader;
+use Symfony\AI\Store\Document\TextDocument;
+use Symfony\AI\Store\Exception\InvalidArgumentException;
+use Symfony\AI\Store\Exception\RuntimeException;
+
+/**
+ * @author Ramy Hakam <ramyhakam1@gmail.com>
+ */
+final class CsvLoaderTest extends TestCase
+{
+    public function testLoadWithNullSource()
+    {
+        $loader = new CsvLoader();
+
+        $this->expectException(InvalidArgumentException::class);
+        $this->expectExceptionMessage('CSV loader requires a file path as source.');
+
+        iterator_to_array($loader->load(null));
+    }
+
+    public function testLoadWithInvalidSource()
+    {
+        $loader = new CsvLoader();
+
+        $this->expectException(RuntimeException::class);
+        $this->expectExceptionMessage('File "/invalid/source.csv" does not exist.');
+
+        iterator_to_array($loader->load('/invalid/source.csv'));
+    }
+
+    public function testLoadWithValidSource()
+    {
+        $loader = new CsvLoader();
+
+        $documents = iterator_to_array(
+            $loader->load($this->getFixturePath())
+        );
+
+        // 8 rows total, but row 6 has empty content and should be skipped
+        $this->assertCount(7, $documents);
+        $this->assertContainsOnlyInstancesOf(TextDocument::class, $documents);
+        $this->assertSame('This is the first document content', $documents[0]->getContent());
+        $this->assertSame('Second document with different text', $documents[1]->getContent());
+        $this->assertSame('Third document for testing purposes', $documents[2]->getContent());
+    }
+
+    public function testSourceIsPresentInMetadata()
+    {
+        $loader = new CsvLoader();
+
+        $source = $this->getFixturePath();
+        $documents = iterator_to_array($loader->load($source));
+
+        $this->assertCount(7, $documents);
+        $this->assertInstanceOf(TextDocument::class, $document = $documents[0]);
+        $this->assertSame($source, $document->getMetadata()['_source']);
+        $this->assertSame($source, $document->getMetadata()->getSource());
+    }
+
+    public function testLoadWithIdColumn()
+    {
+        $loader = new CsvLoader(idColumn: 'id');
+
+        $documents = iterator_to_array(
+            $loader->load($this->getFixturePath())
+        );
+
+        $this->assertCount(7, $documents);
+        $this->assertSame('1', (string) $documents[0]->getId());
+        $this->assertSame('2', (string) $documents[1]->getId());
+        $this->assertSame('5', (string) $documents[4]->getId());
+    }
+
+    public function testLoadWithMetadataColumns()
+    {
+        $loader = new CsvLoader(metadataColumns: ['author', 'category']);
+
+        $documents = iterator_to_array(
+            $loader->load($this->getFixturePath())
+        );
+
+        $this->assertCount(7, $documents);
+        $this->assertSame('John Doe', $documents[0]->getMetadata()['author']);
+        $this->assertSame('Technology', $documents[0]->getMetadata()['category']);
+        $this->assertSame('Jane Smith', $documents[1]->getMetadata()['author']);
+        $this->assertSame('Science', $documents[1]->getMetadata()['category']);
+    }
+
+    public function testLoadWithOptionsOverride()
+    {
+        $loader = new CsvLoader();
+
+        $documents = iterator_to_array(
+            $loader->load($this->getFixturePath(), [
+                CsvLoader::OPTION_ID_COLUMN => 'id',
+                CsvLoader::OPTION_METADATA_COLUMNS => ['author'],
+            ])
+        );
+
+        $this->assertCount(7, $documents);
+        $this->assertSame('1', (string) $documents[0]->getId());
+        $this->assertSame('John Doe', $documents[0]->getMetadata()['author']);
+    }
+
+    public function testEmptyContentRowsAreSkipped()
+    {
+        $loader = new CsvLoader(idColumn: 'id');
+
+        $documents = iterator_to_array(
+            $loader->load($this->getFixturePath())
+        );
+
+        $ids = array_map(static fn ($doc) => (string) $doc->getId(), $documents);
+
+        $this->assertContains('5', $ids);
+        $this->assertNotContains('6', $ids);
+        $this->assertContains('7', $ids);
+    }
+
+    public function testContentWithCommasIsHandledCorrectly()
+    {
+        $loader = new CsvLoader(idColumn: 'id');
+
+        $documents = iterator_to_array(
+            $loader->load($this->getFixturePath())
+        );
+
+        $doc4 = null;
+        foreach ($documents as $doc) {
+            if ('4' === (string) $doc->getId()) {
+                $doc4 = $doc;
+                break;
+            }
+        }
+
+        $this->assertNotNull($doc4);
+        $this->assertSame('Content with, comma inside', $doc4->getContent());
+    }
+
+    public function testContentWithQuotesIsHandledCorrectly()
+    {
+        $loader = new CsvLoader(idColumn: 'id');
+
+        $documents = iterator_to_array(
+            $loader->load($this->getFixturePath())
+        );
+
+        $doc8 = null;
+        foreach ($documents as $doc) {
+            if ('8' === (string) $doc->getId()) {
+                $doc8 = $doc;
+                break;
+            }
+        }
+
+        $this->assertNotNull($doc8);
+        $this->assertSame('Multi-word content with "quotes" inside', $doc8->getContent());
+    }
+
+    public function testRowIndexInMetadata()
+    {
+        $loader = new CsvLoader(idColumn: 'id');
+
+        $documents = iterator_to_array(
+            $loader->load($this->getFixturePath())
+        );
+
+        $this->assertSame(0, $documents[0]->getMetadata()['_row_index']);
+        $this->assertSame(1, $documents[1]->getMetadata()['_row_index']);
+    }
+
+    private function getFixturePath(): string
+    {
+        return \dirname(__DIR__, 2).'/Fixtures/test-data.csv';
+    }
+}

src/store/tests/Fixtures/test-data.csv

@@ -0,0 +1,9 @@
+id,content,author,category
+1,This is the first document content,John Doe,Technology
+2,Second document with different text,Jane Smith,Science
+3,Third document for testing purposes,Bob Wilson,Art
+4,"Content with, comma inside",Alice Brown,Business
+5,Document about machine learning and AI,Charlie Davis,Technology
+6,,Empty Author,Science
+7,Another valid document here,Diana Evans,Health
+8,"Multi-word content with ""quotes"" inside",Frank Green,Education