Move to lazy indexing/updating (#31)

Author: ryanmitchell

README.md

@@ -65,6 +65,12 @@ window.meilisearch = new meilisearch({
 </script>
 ```
 
+You can optionally publish the config file for this package using:
+
+```
+php artisan vendor:publish --tag=statamic-meilisearch-config
+```
+
 ### Few words about Document IDs in meilisearch
 
 When you index your Statamic Entries, the driver will always transform the ID. This is required because meilisearch only allows `id` to be a string containing alphanumeric characters (a-Z, 0-9), hyphens (-) and underscores (_). You can read more about this in the [meilisearch documentation](https://www.meilisearch.com/docs/learn/core_concepts/primary_key#invalid_document_id)

composer.json

@@ -28,7 +28,7 @@
         "guzzlehttp/guzzle": "^7.3",
         "http-interop/http-factory-guzzle": "^1.0",
         "illuminate/support": "^9.0|^10.0",
-        "statamic/cms": "^4.0"
+        "statamic/cms": "^4.41"
     },
     "require-dev": {
         "orchestra/testbench": "^7.0 || ^8.0",

config/statamic-meilisearch.php

@@ -0,0 +1,9 @@
+<?php
+
+return [
+
+    // if you make this higher it will use more memory
+    // but be quicker to update large numbers of documents
+    'insert_chunk_size' => 100,
+
+];

src/Meilisearch/Index.php

@@ -14,7 +14,7 @@ class Index extends BaseIndex
 {
     protected $client;
 
-    public function __construct(Client $client, $name, array $config, string $locale = null)
+    public function __construct(Client $client, $name, array $config, ?string $locale = null)
     {
         $this->client = $client;
 
@@ -28,11 +28,26 @@ public function search($query)
 
     public function insert($document)
     {
-        $fields = array_merge(
-            $this->searchables()->fields($document),
-            $this->getDefaultFields($document),
-        );
-        $this->getIndex()->updateDocuments([$fields]);
+        return $this->insertMultiple(collect($document));
+    }
+
+    public function insertMultiple($documents)
+    {
+        $documents
+            ->chunk(config('statamic-meilisearch.insert_chunk_size', 100))
+            ->each(function ($documents, $index) {
+                $documents = $documents
+                    ->map(fn ($document) => array_merge(
+                        $this->searchables()->fields($document),
+                        $this->getDefaultFields($document),
+                    ))
+                    ->values()
+                    ->toArray();
+
+                $this->insertDocuments(new Documents($documents));
+            });
+
+        return $this;
     }
 
     public function delete($document)
@@ -53,15 +68,7 @@ public function exists()
 
     protected function insertDocuments(Documents $documents)
     {
-        try {
-            if ($documents->isEmpty()) {
-                return true;
-            }
-
-            return $this->getIndex()->updateDocuments($documents->all());
-        } catch (\Exception $e) {
-            throw new \Exception($e->getMessage());
-        }
+        $this->getIndex()->updateDocuments($documents->all());
     }
 
     protected function deleteIndex()
@@ -94,17 +101,7 @@ public function update()
         $this->deleteIndex();
         $this->createIndex();
 
-        // Prepare documents for update
-        $searchables = $this->searchables()->all()->map(function ($entry) {
-            return array_merge(
-                $this->searchables()->fields($entry),
-                $this->getDefaultFields($entry),
-            );
-        });
-
-        // Update documents
-        $documents = new Documents($searchables);
-        $this->insertDocuments($documents);
+        $this->searchables()->lazy()->each(fn ($searchables) => $this->insertMultiple($searchables));
 
         return $this;
     }
@@ -167,4 +164,9 @@ private function getSafeDocumentID(string $entryReference)
             })
             ->implode('---');
     }
+
+    public function getCount()
+    {
+        return $this->getIndex()->stats()['numberOfDocuments'] ?? 0;
+    }
 }

src/ServiceProvider.php

@@ -11,6 +11,16 @@ class ServiceProvider extends AddonServiceProvider
 {
     public function bootAddon()
     {
+        $this->mergeConfigFrom(__DIR__.'/../config/statamic-meilisearch.php', 'statamic-meilisearch');
+
+        if ($this->app->runningInConsole()) {
+
+            $this->publishes([
+                __DIR__.'/../config/statamic-meilisearch.php' => config_path('statamic-meilisearch.php'),
+            ], 'statamic-meilisearch-config');
+
+        }
+
         Search::extend('meilisearch', function (Application $app, array $config, $name, $locale = null) {
             $client = $app->makeWith(Client::class, [
                 'url' => $config['credentials']['url'],