Remove the process of unused columns (#1113)

Author: AndrewThien

app/airflow/dags/libs/SR_processing/db_services.py

@@ -50,14 +50,6 @@ def create_field_entries(
                 field_name = str(row[1].value) if row[1].value is not None else ""
                 description = str(row[2].value) if row[2].value is not None else ""
                 type_column = str(row[3].value) if row[3].value is not None else ""
-                max_length = row[4].value
-                nrows = row[5].value
-                nrows_checked = row[6].value
-
-                # Calculate fractions with default values
-                fraction_empty = round(default_zero(row[7].value), 2)
-                nunique_values = row[8].value
-                fraction_unique = round(default_zero(row[9].value), 2)
 
                 pg_hook.run(
                     create_fields_query,
@@ -67,12 +59,6 @@ def create_field_entries(
                         "name": field_name,  # NOTE: without BOM removal to keep the consistency with Azure functions
                         "description_column": description,
                         "type_column": type_column,
-                        "max_length": max_length,
-                        "nrows": nrows,
-                        "nrows_checked": nrows_checked,
-                        "fraction_empty": fraction_empty,
-                        "nunique_values": nunique_values,
-                        "fraction_unique": fraction_unique,
                     },
                 )
 

app/airflow/dags/libs/queries.py

@@ -525,16 +525,12 @@
 
 create_fields_query = """
     INSERT INTO mapping_scanreportfield (
-        scan_report_table_id, name, description_column, type_column,
-        max_length, nrows, nrows_checked, fraction_empty,
-        nunique_values, fraction_unique, created_at, updated_at,
+        scan_report_table_id, name, description_column, type_column, created_at, updated_at,
         is_patient_id, is_ignore, pass_from_source
     )
     VALUES (
         %(scan_report_table_id)s, %(name)s, %(description_column)s, %(type_column)s,
-        %(max_length)s, %(nrows)s, %(nrows_checked)s, %(fraction_empty)s,
-        %(nunique_values)s, %(fraction_unique)s, NOW(), NOW(), False, False,
-        True
+        NOW(), NOW(), False, False, True
     )
 """
 

app/shared/shared/mapping/migrations/0008_remove_scanreportfield_fraction_empty_and_more.py

@@ -0,0 +1,37 @@
+# Generated by Django 5.2.1 on 2025-06-19 11:24
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("mapping", "0007_alter_dataset_hidden_alter_mappingstatus_value_and_more"),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name="scanreportfield",
+            name="fraction_empty",
+        ),
+        migrations.RemoveField(
+            model_name="scanreportfield",
+            name="fraction_unique",
+        ),
+        migrations.RemoveField(
+            model_name="scanreportfield",
+            name="max_length",
+        ),
+        migrations.RemoveField(
+            model_name="scanreportfield",
+            name="nrows",
+        ),
+        migrations.RemoveField(
+            model_name="scanreportfield",
+            name="nrows_checked",
+        ),
+        migrations.RemoveField(
+            model_name="scanreportfield",
+            name="nunique_values",
+        ),
+    ]

app/shared/shared/mapping/models.py

@@ -302,12 +302,6 @@ class ScanReportField(BaseModel):
     name = models.CharField(max_length=512, db_index=True)
     description_column = models.CharField(max_length=512)
     type_column = models.CharField(max_length=32)
-    max_length = models.IntegerField()
-    nrows = models.IntegerField()
-    nrows_checked = models.IntegerField()
-    fraction_empty = models.DecimalField(decimal_places=2, max_digits=10)
-    nunique_values = models.IntegerField()
-    fraction_unique = models.DecimalField(decimal_places=2, max_digits=10)
     ignore_column = models.CharField(max_length=64, blank=True, null=True)
     is_patient_id = models.BooleanField(default=False)
     is_ignore = models.BooleanField(default=False)

app/workers/UploadQueue/__init__.py

@@ -82,12 +82,6 @@ def _create_field_entry(row: Tuple[Cell], scan_report_table_id: str) -> ScanRepo
         name=str(row[1].value),
         description_column=str(row[2].value),
         type_column=str(row[3].value),
-        max_length=row[4].value,
-        nrows=row[5].value,
-        nrows_checked=row[6].value,
-        fraction_empty=round(helpers.default_zero(row[7].value), 2),
-        nunique_values=row[8].value,
-        fraction_unique=round(helpers.default_zero(row[9].value), 2),
     )