Allow csv_collection to pass options to CSV parser (#1001)

Author: bdewater

README.md

@@ -191,8 +191,8 @@ module Maintenance
 end
 ```
 
+`posts.csv`:
 ```csv
-# posts.csv
 title,content
 My Title,Hello World!
 ```
@@ -211,6 +211,38 @@ def count(task)
 end
 ```
 
+#### CSV options
+
+Tasks can pass [options for Ruby's CSV parser][csv-parse-options] by adding
+keyword arguments to `csv_collection`:
+
+[csv-parse-options]: https://ruby-doc.org/3.3.0/stdlibs/csv/CSV.html#class-CSV-label-Options+for+Parsing
+
+```ruby
+# app/tasks/maintenance/import_posts_task.rb
+
+module Maintenance
+  class ImportPosts
+    csv_collection(skip_lines: /^#/, converters: ->(field) { field.strip })
+
+    def process(row)
+      Post.create!(title: row["title"], content: row["content"])
+    end
+  end
+end
+```
+
+These options instruct Ruby's CSV parser to skip lines that start with a `#`,
+and removes the leading and trailing spaces from any field, so that the
+following file will be processed identically as the previous example:
+
+`posts.csv`:
+```csv
+# A comment
+title,content
+ My Title ,Hello World!
+```
+
 #### Batch CSV Tasks
 
 Tasks can process CSVs in batches. Add the `in_batches` option to your task’s

app/models/maintenance_tasks/batch_csv_collection_builder.rb

@@ -12,16 +12,17 @@ class BatchCsvCollectionBuilder < CsvCollectionBuilder
     # Initialize a BatchCsvCollectionBuilder with a batch size.
     #
     # @param batch_size [Integer] the number of CSV rows in a batch.
-    def initialize(batch_size)
+    # @param csv_options [Hash] options to pass to the CSV parser.
+    def initialize(batch_size, **csv_options)
       @batch_size = batch_size
-      super()
+      super(**csv_options)
     end
 
     # Defines the collection to be iterated over, based on the provided CSV.
     # Includes the CSV and the batch size.
     def collection(task)
       BatchCsv.new(
-        csv: CSV.new(task.csv_content, headers: true),
+        csv: CSV.new(task.csv_content, **@csv_options),
         batch_size: @batch_size,
       )
     end

app/models/maintenance_tasks/csv_collection_builder.rb

@@ -5,14 +5,18 @@
 module MaintenanceTasks
   # Strategy for building a Task that processes CSV files.
   #
+  # @param csv_options [Hash] options to pass to the CSV parser.
   # @api private
   class CsvCollectionBuilder
+    def initialize(**csv_options)
+      @csv_options = csv_options
+    end
+
     # Defines the collection to be iterated over, based on the provided CSV.
     #
-    # @return [CSV] the CSV object constructed from the specified CSV content,
-    #   with headers.
+    # @return [CSV] the CSV object constructed from the specified CSV content.
     def collection(task)
-      CSV.new(task.csv_content, headers: true)
+      CSV.new(task.csv_content, **@csv_options)
     end
 
     # The number of rows to be processed.
@@ -21,7 +25,7 @@ def collection(task)
     #
     # @return [Integer] the approximate number of rows to process.
     def count(task)
-      CSV.new(task.csv_content, headers: true).count
+      CSV.new(task.csv_content, **@csv_options).count
     end
 
     # Return that the Task processes CSV content.

app/models/maintenance_tasks/task.rb

@@ -65,20 +65,24 @@ def available_tasks
       # Make this Task a task that handles CSV.
       #
       # @param in_batches [Integer] optionally, supply a batch size if the CSV
-      # should be processed in batches.
+      #   should be processed in batches.
+      # @param csv_options [Hash] optionally, supply options for the CSV parser.
+      #   If not given, defaults to: <code>{ headers: true }</code>
+      # @see https://ruby-doc.org/3.3.0/stdlibs/csv/CSV.html#class-CSV-label-Options+for+Parsing
       #
       # An input to upload a CSV will be added in the form to start a Run. The
       # collection and count method are implemented.
-      def csv_collection(in_batches: nil)
+      def csv_collection(in_batches: nil, **csv_options)
         unless defined?(ActiveStorage)
           raise NotImplementedError, "Active Storage needs to be installed\n" \
             "To resolve this issue run: bin/rails active_storage:install"
         end
 
+        csv_options[:headers] = true unless csv_options.key?(:headers)
         self.collection_builder_strategy = if in_batches
-          BatchCsvCollectionBuilder.new(in_batches)
+          BatchCsvCollectionBuilder.new(in_batches, **csv_options)
         else
-          CsvCollectionBuilder.new
+          CsvCollectionBuilder.new(**csv_options)
         end
       end
 

test/dummy/app/tasks/maintenance/import_posts_with_options_task.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Maintenance
+  class ImportPostsWithOptionsTask < MaintenanceTasks::Task
+    csv_collection(skip_lines: /^#/, converters: ->(field) { field.upcase })
+
+    def process(row)
+      Post.create!(title: row["title"], content: row["content"])
+    end
+  end
+end

test/models/maintenance_tasks/csv_collection_builder_test.rb

@@ -26,7 +26,7 @@ class CsvCollectionBuilderTest < ActiveSupport::TestCase
         3
       CSV
 
-      assert_equal(3, @builder.count(@task))
+      assert_equal(4, @builder.count(@task))
     end
 
     test "#has_csv_content?" do

test/models/maintenance_tasks/csv_task_test.rb

@@ -19,6 +19,27 @@ class CsvTaskTest < ActiveSupport::TestCase
       assert_equal "Hello World 1!", first_row["content"]
     end
 
+    test ".collection passes options to the CSV parser" do
+      csv_file = file_fixture("sample.csv")
+      csv = csv_file.read
+      csv.prepend("# Comment\n")
+      csv.concat("# Another comment\n")
+
+      csv_task = Maintenance::ImportPostsWithOptionsTask.new
+      csv_task.csv_content = csv
+      collection = csv_task.collection
+
+      assert CSV, collection.class
+      assert collection.headers
+
+      all_rows = collection.to_a
+      assert_equal 5, all_rows.count
+
+      first_row = all_rows.first
+      assert_equal "MY TITLE 1", first_row["title"]
+      assert_equal "HELLO WORLD 1!", first_row["content"]
+    end
+
     test ".count returns the number of rows to process, excluding headers and assuming a trailing newline" do
       csv_file = file_fixture("sample.csv")
 

test/models/maintenance_tasks/task_data_index_test.rb

@@ -13,6 +13,7 @@ class TaskDataIndexTest < ActiveSupport::TestCase
         "Maintenance::EnqueueErrorTask",
         "Maintenance::ErrorTask",
         "Maintenance::ImportPostsTask",
+        "Maintenance::ImportPostsWithOptionsTask",
         "Maintenance::Nested::NestedMore::NestedMoreTask",
         "Maintenance::Nested::NestedTask",
         "Maintenance::NoCollectionTask",

test/models/maintenance_tasks/task_test.rb

@@ -13,6 +13,7 @@ class TaskTest < ActiveSupport::TestCase
         "Maintenance::EnqueueErrorTask",
         "Maintenance::ErrorTask",
         "Maintenance::ImportPostsTask",
+        "Maintenance::ImportPostsWithOptionsTask",
         "Maintenance::Nested::NestedMore::NestedMoreTask",
         "Maintenance::Nested::NestedTask",
         "Maintenance::NoCollectionTask",

test/system/maintenance_tasks/tasks_test.rb

@@ -28,6 +28,7 @@ class TasksTest < ApplicationSystemTestCase
         "Maintenance::CustomEnumeratingTask\nNew",
         "Maintenance::EnqueueErrorTask\nNew",
         "Maintenance::ErrorTask\nNew",
+        "Maintenance::ImportPostsWithOptionsTask\nNew",
         "Maintenance::Nested::NestedMore::NestedMoreTask\nNew",
         "Maintenance::Nested::NestedTask\nNew",
         "Maintenance::ParamsTask\nNew",