Merge pull request #47 from cfpb/collect-view

Improve Collections implementation

Author: schbetsy

django/parse_m2/admin.py

@@ -69,7 +69,7 @@ def has_delete_permission(self, request, obj=None):
 class M2DataFileAdmin(admin.ModelAdmin):
     list_display = ['id', 'event', 'file_name', 'parsing_status',
                     'parsed_lines', 'unparseable_lines', 'timestamp',
-                    'error_message', 'parser_version']
+                    'error_message', 'collection', 'parser_version']
 
     def parsed_lines(self, obj):
         return obj.accountactivity_set.count()

django/parse_m2/m2_parser.py

@@ -15,12 +15,11 @@
 class M2FileParser():
     # Parser version is saved on each file record.
     # Increment this version for all updates to parser functionality.
-    parser_version = "3.0"
+    parser_version = "3.1"
 
     chunk_size = 2000  # TODO: determine a good number for this
     any_non_whitespace = r'\S'
 
-    collection = None
     activity_date = None
 
     def __init__(self, event: Metro2Event, filepath: str, collection: str = None) -> None:
@@ -34,13 +33,12 @@ def __init__(self, event: Metro2Event, filepath: str, collection: str = None) ->
             file_name=filepath,
             parsing_status="In progress",
             parser_version=self.parser_version,
+            # Collection, if present, will be prepended to consumer account number
+            # for all records in this file
+            collection=collection,
         )
         self.file_record.save()
 
-        # Collection, if present, will be prepended to consumer account number
-        # for all records in this file
-        self.collection = collection
-
     def update_file_record(self, status=None, msg=None) -> None:
         """
         Update the file record with the given status and error message.
@@ -190,7 +188,7 @@ def parse_line(self, line: str) -> dict:
 
             # parse the base segment into AccountActivity
             acct_activity = AccountActivity.parse_from_segment(
-                line, self.file_record, self.activity_date, self.collection)
+                line, self.file_record, self.activity_date)
             parsed["AccountActivity"] = acct_activity
 
             # parse the extra segments

django/parse_m2/migrations/0002_m2datafile_collection.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.29 on 2026-03-24 20:01
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('parse_m2', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='m2datafile',
+            name='collection',
+            field=models.CharField(blank=True, max_length=50, null=True),
+        ),
+    ]

django/parse_m2/models.py

@@ -64,6 +64,7 @@ class Meta:
     error_message = models.CharField(max_length=2000, blank=True)
     parser_version = models.CharField(max_length=200, blank=True)
     activity_date = models.DateField(null=True)
+    collection = models.CharField(max_length=50, blank=True, null=True)
 
     def __str__(self) -> str:
         return self.file_name
@@ -155,7 +156,7 @@ def __str__(self) -> str:
 
 
     @classmethod
-    def parse_from_segment(cls, base_seg: str, m2_data_file: M2DataFile, activity_date: date, collection: str = None):
+    def parse_from_segment(cls, base_seg: str, m2_data_file: M2DataFile, activity_date: date):
         # Construct the php1 field from php
         php = get_field_value(fields.base_fields, "php", base_seg)
         php1 = php[0] if php else ""
@@ -167,8 +168,8 @@ def parse_from_segment(cls, base_seg: str, m2_data_file: M2DataFile, activity_da
 
         # If 'collection' is provided, prepend to account number
         account_num = get_field_value(fields.base_fields, "cons_acct_num", base_seg)
-        if collection:
-            account_num = f"{collection}{account_num}"
+        if m2_data_file.collection:
+            account_num = f"{m2_data_file.collection}.{account_num}"
 
         return cls(
             data_file = m2_data_file,

django/parse_m2/tests/test_initiate_parsing.py

@@ -54,7 +54,7 @@ def test_open_zipfiles(self):
         self.assertEqual(AccountActivity.objects.count(), 1997)
 
     def test_prepend_collection_onto_account_num(self):
-        parse_files_from_local_filesystem(self.event, collection="HEALTH.")
+        parse_files_from_local_filesystem(self.event, collection="HEALTH")
 
         # one M2DataFile object for each file
         self.assertEqual(M2DataFile.objects.count(), 3)

django/parse_m2/tests/test_parser.py

@@ -265,7 +265,7 @@ def test_parser_saves_header_as_unparseable(self):
     # Tests for prepending a collection name to the account number
     def test_collection_prefix_on_account_number(self):
         event = Metro2Event.objects.create(name='exam_with_collections')
-        parser = M2FileParser(event=event, filepath="file.txt", collection="HEALTH.")
+        parser = M2FileParser(event=event, filepath="file.txt", collection="HEALTH")
 
         file_size = os.path.getsize(self.tiny_file)
         with open(self.tiny_file, mode='r') as filestream: