Add current form columns to CSV export event if field are empty and export obsolete fields ordered after current ones (#80)

Author: folix-01

CHANGES.rst

@@ -4,7 +4,12 @@ Changelog
 3.2.3 (unreleased)
 ------------------
 
-- Nothing changed yet.
+- Now in the CSV export the obsolete records fields are ordered
+  alphabetically after the current form fields.
+  [folix-01]
+
+- Add current form columns to CSV export event if field are empty.
+  [folix-01]
 
 
 3.2.2 (2025-03-05)
@@ -24,8 +29,8 @@ Changelog
 3.2.0 (2024-11-15)
 ------------------
 
-- Added an adapter (`IDataAdapter`) to allow information to be added as a return value 
-  to the form-data expander. This allows addons that integrate information to be added 
+- Added an adapter (`IDataAdapter`) to allow information to be added as a return value
+  to the form-data expander. This allows addons that integrate information to be added
   rather than overwriting the expander each time.
   [mamico]
 

src/collective/volto/formsupport/restapi/services/form_data/csv.py

@@ -1,4 +1,6 @@
 from collective.volto.formsupport.interfaces import IFormDataStore
+from collective.volto.formsupport.utils import get_blocks
+from copy import deepcopy
 from io import StringIO
 from plone.restapi.serializer.converters import json_compatible
 from plone.restapi.services import Service
@@ -23,12 +25,37 @@ def __init__(self, context, request):
             block_type = block.get("@type", "")
             if block_type == "form":
                 self.form_block = block
+                self.form_block_id = id
 
         if self.form_block:
             for field in self.form_block.get("subblocks", []):
                 field_id = field["field_id"]
                 self.form_fields_order.append(field_id)
 
+    def get_form_fields(self):
+        blocks = get_blocks(self.context)
+
+        if not blocks:
+            return {}
+        form_block = {}
+        for id, block in blocks.items():
+            if id != self.form_block_id:
+                continue
+            block_type = block.get("@type", "")
+            if block_type == "form":
+                form_block = deepcopy(block)
+        if not form_block:
+            return {}
+
+        subblocks = form_block.get("subblocks", [])
+
+        # Add the 'custom_field_id' field back in as this isn't stored with each subblock
+        for index, field in enumerate(subblocks):
+            if form_block.get(field["field_id"]):
+                subblocks[index]["custom_field_id"] = form_block.get(field["field_id"])
+
+        return subblocks
+
     def get_ordered_keys(self, record):
         """
         We need this method because we want to maintain the fields order set in the form.
@@ -61,37 +88,57 @@ def render(self):
             data = data.encode("utf-8")
         self.request.response.write(data)
 
-    def get_fields_labels(self, item):
-        return item.attrs.get("fields_labels", {})
+    def get_fields_labels(self):
+        return {
+            x["field_id"]: x.get("custom_field_id", x.get("label", x["field_id"]))
+            for x in self.get_form_fields()
+            if x.get("field_type", "") != "static_text"
+        }
 
     def get_data(self):
         store = getMultiAdapter((self.context, self.request), IFormDataStore)
         sbuf = StringIO()
         fixed_columns = ["date"]
         columns = []
-
+        legacy_columns = []
         rows = []
+        fields_labels = self.get_fields_labels()
+
         for item in store.search():
             data = {}
-            fields_labels = self.get_fields_labels(item)
+
             for k in self.get_ordered_keys(item):
                 if k in SKIP_ATTRS:
                     continue
+
                 value = item.attrs.get(k, None)
-                label = fields_labels.get(k, k)
-                if label not in columns and label not in fixed_columns:
-                    columns.append(label)
+                label = {**item.attrs.get("fields_labels", {}), **fields_labels}.get(
+                    k, k
+                )
+
+                if k not in self.form_fields_order and label not in legacy_columns:
+                    legacy_columns.append(label)
+
                 data[label] = json_compatible(value)
+
             for k in fixed_columns:
                 # add fixed columns values
                 value = item.attrs.get(k, None)
                 data[k] = json_compatible(value)
+
             rows.append(data)
+
+        columns.extend(fields_labels.values())
         columns.extend(fixed_columns)
+        columns.extend(sorted(legacy_columns))
+
         writer = csv.DictWriter(sbuf, fieldnames=columns, quoting=csv.QUOTE_ALL)
         writer.writeheader()
+
         for row in rows:
             writer.writerow(row)
+
         res = sbuf.getvalue()
         sbuf.close()
+
         return res

src/collective/volto/formsupport/tests/test_store_action_form.py

@@ -196,7 +196,7 @@ def test_store_data(self):
         response = self.export_csv()
         data = [*csv.reader(StringIO(response.text), delimiter=",")]
         self.assertEqual(len(data), 1)
-        self.assertEqual(data[0], ["date"])
+        self.assertIn("date", data[0])
 
     def test_export_csv(self):
         self.document.blocks = {