[FIX] error messages do not appear in the excel file during import and chunks are not split correctly

Author: chafique-delli

pattern_import_export/models/pattern_chunk.py

@@ -73,7 +73,15 @@ def run(self):
                     "state": "failed",
                 }
             )
-            self.with_delay().check_last()
+            if not self.messages:
+                self.messages = {"message": e.args[0]}
+            next_chunk = self.get_next_chunk()
+            if next_chunk:
+                config = self.pattern_file_id.pattern_config_id
+                priority = config.job_priority
+                next_chunk.with_delay(priority=priority).run()
+            else:
+                self.with_delay().check_last()
         return "OK"
 
     def _prepare_chunk_result(self, res):

pattern_import_export/models/pattern_config.py

@@ -128,6 +128,7 @@ def _get_header(self, use_description=False):
         header = []
         for export_line in self.export_fields:
             header.extend(export_line._get_header(use_description))
+        header.insert(0, "#Error")
         return header
 
     def generate_pattern(self):
@@ -173,7 +174,10 @@ def json2pattern_format(self, data):
                         key = key.replace(IDENTIFIER_SUFFIX, "")
                     if key == ".id":
                         key = "id"
-                    val = val[key]
+                    if key == "#Error":
+                        val = None
+                    else:
+                        val = val[key]
                     if val is None:
                         break
             except IndexError:

pattern_import_export/models/pattern_file.py

@@ -39,6 +39,8 @@ def _compute_stat(self):
                 record.progress = (record.nbr_error + record.nbr_success) * 100.0 / todo
             else:
                 record.progress = 0
+            if record.nbr_error and record.state != "failed":
+                record.state = "failed"
 
     @api.model_create_multi
     def create(self, vals):
@@ -141,7 +143,7 @@ def _prepare_chunk(self, start_idx, stop_idx, data):
     def _should_create_chunk(self, items, next_item):
         """Customise this code if you want to add some additionnal
         item after reaching the limit"""
-        return len(items) > self.pattern_config_id.chunk_size
+        return len(items) >= self.pattern_config_id.chunk_size
 
     def _create_chunk(self, start_idx, stop_idx, data):
         vals = self._prepare_chunk(start_idx, stop_idx, data)
@@ -157,7 +159,7 @@ def split_in_chunk(self):
         self.chunk_ids.unlink()
         try:
             items = []
-            start_idx = 1
+            start_idx = self.pattern_config_id.nr_of_header_rows + 1
             previous_idx = None
             # idx is the index position in the original file
             # we can have empty line that can be skipped

pattern_import_export_xlsx/models/__init__.py

@@ -1,2 +1,3 @@
 from . import pattern_config
 from . import pattern_file
+from . import pattern_chunk

pattern_import_export_xlsx/models/pattern_chunk.py

@@ -0,0 +1,16 @@
+# Copyright 2020 Akretion (https://www.akretion.com).
+# @author Sébastien BEAU <[email protected]>
+# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl).
+
+from odoo import fields, models
+
+
+class PatternChunk(models.Model):
+    _inherit = "pattern.chunk"
+
+    def check_last(self):
+        res = super().check_last()
+        if res == "There is still some running chunk":
+            for record in self:
+                record.pattern_file_id.write_error_in_xlsx()
+        return res

pattern_import_export_xlsx/models/pattern_config.py

@@ -43,6 +43,7 @@ def _build_main_sheet_structure(self, book):
         for row, lines in enumerate(self._get_output_headers(), start=1):
             for col, header in enumerate(lines.values(), start=1):
                 main_sheet.cell(row=row, column=col, value=header)
+        main_sheet.column_dimensions['A'].hidden=True
         return main_sheet
 
     def _populate_main_sheet_rows(self, main_sheet, records):
@@ -83,7 +84,7 @@ def _create_validators(self, main_sheet, records, tabs):
             formula_range_src = "=" + quote_sheetname(tab_name) + "!" + range_src
             validation = DataValidation(type="list", formula1=formula_range_src)
             for idx_col in tab["idx_col_validator"]:
-                col_letter_dst = get_column_letter(idx_col)
+                col_letter_dst = get_column_letter(idx_col+1)
                 range_dst = "${}${}:${}${}".format(
                     col_letter_dst,
                     str(self.row_start_records),

pattern_import_export_xlsx/models/pattern_file.py

@@ -70,23 +70,24 @@ def write_error_in_xlsx(self):
         infile = BytesIO(base64.b64decode(self.datas))
         wb = openpyxl.load_workbook(filename=infile)
         ws = self._get_worksheet(wb)
+        ws.column_dimensions['A'].hidden=False
 
         # we clear the error col if exist
         if ws["A1"].value == _("#Error"):
             ws.delete_cols(1)
         ws.insert_cols(1)
         ws.cell(1, 1, value=_("#Error"))
-        last_row_idx = 0
-        for chunk in self.chunk_ids:
-            for message in chunk.messages:
-                if "rows" in message:
-                    last_row_idx = message["rows"]["to"]
-                    ws.cell(message["rows"]["to"], 1, value=message["message"].strip())
+        for chunk in self.chunk_ids.filtered(lambda c: c.nbr_error > 0):
+            for key, message in chunk.messages.items():
+                if "rows" in key:
+                    ws.cell(message["rows"]["to"], 1, value=message)
                 else:
                     # If no row are specify, this is a global message
                     # that should be applied until the end of the chunk
-                    for idx in range(last_row_idx, chunk.stop_idx + 1):
-                        ws.cell(idx, 1, value=message["message"].strip())
+                    for idx in range(chunk.start_idx, chunk.stop_idx + 1):
+                        if self.pattern_config_id.nr_of_header_rows == 2 and idx == 2:
+                            idx = idx + 1
+                        ws.cell(idx, 1, value=message)
         output = BytesIO()
         wb.save(output)
         self.datas = base64.b64encode(output.getvalue())