Merge pull request #15649 from bryanlopezinc/ImproveImporting

Improve import performance

Author: snipe

app/Console/Commands/ObjectImportCommand.php

@@ -6,6 +6,7 @@
 use Symfony\Component\Console\Input\InputArgument;
 use Symfony\Component\Console\Input\InputOption;
 use Illuminate\Support\Facades\Log;
+use Symfony\Component\Console\Helper\ProgressIndicator;
 
 ini_set('max_execution_time', env('IMPORT_TIME_LIMIT', 600)); //600 seconds = 10 minutes
 ini_set('memory_limit', env('IMPORT_MEMORY_LIMIT', '500M'));
@@ -29,6 +30,11 @@ class ObjectImportCommand extends Command
      */
     protected $description = 'Import Items from CSV';
 
+    /**
+     * The progress indicator instance.
+     */
+    protected ProgressIndicator $progressIndicator;
+
     /**
      * Create a new command instance.
      *
@@ -39,15 +45,15 @@ public function __construct()
         parent::__construct();
     }
 
-    private $bar;
-
     /**
      * Execute the console command.
      *
      * @return mixed
      */
     public function handle()
     {
+        $this->progressIndicator = new ProgressIndicator($this->output);
+
         $filename = $this->argument('filename');
         $class = title_case($this->option('item-type'));
         $classString = "App\\Importer\\{$class}Importer";
@@ -61,46 +67,25 @@ public function handle()
         // This $logFile/useFiles() bit is currently broken, so commenting it out for now
         // $logFile = $this->option('logfile');
         // Log::useFiles($logFile);
-        $this->comment('======= Importing Items from '.$filename.' =========');
-        $importer->import();
+        $this->progressIndicator->start('======= Importing Items from '.$filename.' =========');
 
-        $this->bar = null;
+        $importer->import();
 
-        if (! empty($this->errors)) {
-            $this->comment('The following Errors were encountered.');
-            foreach ($this->errors as $asset => $error) {
-                $this->comment('Error: Item: '.$asset.' failed validation: '.json_encode($error));
-            }
-        } else {
-            $this->comment('All Items imported successfully!');
-        }
-        $this->comment('');
+        $this->progressIndicator->finish('Import finished.');
     }
 
-    public function errorCallback($item, $field, $errorString)
+    public function errorCallback($item, $field, $error)
     {
-        $this->errors[$item->name][$field] = $errorString;
+        $this->output->write("\x0D\x1B[2K");
+
+        $this->warn('Error: Item: '.$item->name.' failed validation: '.json_encode($error));
     }
 
-    public function progress($count)
+    public function progress($importedItemsCount)
     {
-        if (! $this->bar) {
-            $this->bar = $this->output->createProgressBar($count);
-        }
-        static $index = 0;
-        $index++;
-        if ($index < $count) {
-            $this->bar->advance();
-        } else {
-            $this->bar->finish();
-        }
+        $this->progressIndicator->advance();
     }
 
-    // Tracks the current item for error messages
-    private $updating;
-    // An array of errors encountered while parsing
-    private $errors;
-
     /**
      * Log a message to file, configurable by the --log-file parameter.
      * If a warning message is passed, we'll spit it to the console as well.

app/Importer/Importer.php

@@ -21,7 +21,6 @@ abstract class Importer
      * Id of User performing import
      * @var
      */
-    
     protected $created_by;
     /**
      * Are we updating items in the import
@@ -149,17 +148,23 @@ public function import()
     {
         $headerRow = $this->csv->fetchOne();
         $this->csv->setHeaderOffset(0); //explicitly sets the CSV document header record
-        $results = $this->normalizeInputArray($this->csv->getRecords($headerRow));
 
         $this->populateCustomFields($headerRow);
 
-        DB::transaction(function () use (&$results) {
+        DB::transaction(function () use ($headerRow) {
+            $importedItemsCount = 0;
             Model::unguard();
-            $resultsCount = count($results);
-            foreach ($results as $row) {
+
+            foreach ($this->csv->getRecords($headerRow) as $row) {
+                //Lowercase header values to ensure we're comparing values properly.
+                $row = array_change_key_case($row, CASE_LOWER);
+
                 $this->handle($row);
+
+                $importedItemsCount++;
+
                 if ($this->progressCallback) {
-                    call_user_func($this->progressCallback, $resultsCount);
+                    call_user_func($this->progressCallback, $importedItemsCount);
                 }
 
                 $this->log('------------- Action Summary ----------------');
@@ -237,22 +242,6 @@ public function lookupCustomKey($key)
         return $key;
     }
 
-    /**
-     * Used to lowercase header values to ensure we're comparing values properly.
-     *
-     * @param $results
-     * @return array
-     */
-    public function normalizeInputArray($results)
-    {
-        $newArray = [];
-        foreach ($results as $index => $arrayToNormalize) {
-            $newArray[$index] = array_change_key_case($arrayToNormalize);
-        }
-
-        return $newArray;
-    }
-
     /**
      * Figure out the fieldname of the custom field
      *