feat: column metadata calculation (#147)

* added operation type metadata_value

* added operation type header_value

* Enabled regex without line

* Added handler for ignore_missing_values operation option
Added Tag in jcamp if data has been modified

* added CALCULATION_APPLIED_Y to test results if necessary.

* added comment box to jcamp with data operations descriptions

---------

Co-authored-by: Martin <m.starman@live.comdd>
Co-authored-by: Martin Starman <martin.starman@kit.edu>

Author: 3 people

converter_app/converters.py

@@ -9,6 +9,10 @@
 logger = logging.getLogger(__name__)
 
 
+class CalculationError(Exception):
+    pass
+
+
 class Converter:
     """
     Converter object checks if profile matches to filedata and runs the converting process
@@ -273,33 +277,74 @@ def process(self):
                                     table_index == operation.get('column', {}).get('tableIndex') and \
                                     column_index == operation.get('column', {}).get('columnIndex'):
                                 operation['rows'].append(self.get_value(row, column_index))
+            applied_operators = {
+                "applied_x_operator": False,
+                "applied_y_operator": False,
+                "applied_operator_failed": False,
+                "x_operations_description": output_table.get('table', {}).get('xOperationsDescription'),
+                "y_operations_description": output_table.get('table', {}).get('yOperationsDescription')
+            }
+            try:
+                for operation in x_operations:
+                    applied_operators["applied_x_operator"] |= self._run_operation(x_rows, operation)
+                for operation in y_operations:
+                    applied_operators["applied_y_operator"] |= self._run_operation(y_rows, operation)
+            except CalculationError:
+                applied_operators['applied_x_operator'] = applied_operators['applied_y_operator'] =  False
+                applied_operators['applied_operator_failed'] =  True
+                x_rows = []
+                y_rows = []
 
-            for operation in x_operations:
-                x_rows = self._run_operation(x_rows, operation)
-            for operation in y_operations:
-                y_rows = self._run_operation(y_rows, operation)
 
             self.tables.append({
                 'header': header,
                 'x': x_rows,
                 'y': y_rows
-            })
+            } | applied_operators)
 
     def _run_operation(self, rows, operation):
         for i, row in enumerate(rows):
-            op_value = None
+            str_value = None
             if operation.get('type') == 'column':
                 try:
-                    op_value = operation['rows'][i]
+                    str_value = operation['rows'][i]
                 except IndexError:
                     pass
             elif operation.get('type') == 'value':
-                op_value = operation.get('value')
+                str_value = operation.get('value')
+            elif operation.get('type') == 'metadata_value':
+                str_value = self.input_tables[int(operation.get('table'))]['metadata'].get(operation.get('value'))
+            elif operation.get('type') == 'header_value':
+                table = 0
+                if operation.get('table') is not None:
+                    table = int(operation.get('table'))
+                try:
+                    line_number = int(operation.get('line', ''))
+                    header = self.input_tables[table]['header'][line_number - 1].rstrip()
+                except (TypeError, ValueError, IndexError):
+                    header = os.linesep.join(self.input_tables[table]['header']).rstrip()
+                pattern = operation.get('regex')
+                if header is not None and pattern is not None:
+                    str_value = header
+                    match = re.search(pattern, str_value)
+                    if match is not None:
+                        if len(match.regs) > 1:
+                            str_value = match[1]
+                        else:
+                            str_value = match[0]
+            else:
+                raise ValueError(f"Unknown operation type: {operation.get('type')}")
+            try:
+                op_value = float(str_value)
+            except (TypeError, ValueError):
+                if not operation.get('ignore_missing_values'):
+                    raise CalculationError("Calculation could not be executed")
+                return False
 
             if op_value:
                 rows[i] = str(self.apply_operation(row, op_value, operation.get('operator')))
 
-        return rows
+        return True
 
     def _run_identifier_operation(self, value, operation):
         op_value = operation.get('value')

converter_app/readers/dta.py

@@ -16,7 +16,7 @@ def check(self):
         """
         :return: True if it fits
         """
-        return self.file.encoding != 'binary' and self.file.suffix.lower() == '.dta' and self.file.mime_type == 'text/plain'
+        return self.file.encoding != 'binary' and self.file.suffix.lower() == '.dta'
 
     def prepare_tables(self):
         tables = []

converter_app/writers/jcamp.py

@@ -2,9 +2,9 @@
 import os
 import sys
 
+from .base import Writer
 from .. import __title__, __version__
 from ..options import DATA_TYPES, DATA_CLASSES, XUNITS, YUNITS
-from .base import Writer
 
 
 class JcampWriter(Writer):
@@ -17,6 +17,7 @@ def __init__(self, converter):
         self.table = converter.tables[0]
         self.buffer = io.StringIO()
 
+
     def process(self):
         self.process_table(self.table)
 
@@ -31,12 +32,29 @@ def process_table(self, table):
             'ORIGIN': header.get('ORIGIN', ''),
             'OWNER': header.get('OWNER', '')
         }
+
+        if table.get('applied_x_operator'):
+            jcamp_header['CALCULATION_APPLIED_X'] = True
+            if table.get('x_operations_description'):
+                self.write_comment_header(['X operations description:'] + table.get('x_operations_description'))
+
+        if table.get('applied_y_operator'):
+            jcamp_header['CALCULATION_APPLIED_Y'] = True
+            if table.get('y_operations_description'):
+                self.write_comment_header(['Y operations description:'] + table.get('y_operations_description'))
+
+        if table.get('applied_operator_failed'):
+            jcamp_header['CALCULATION_FAILED'] = True
+
         for key in header:
             key_upper = key.upper()
             if key_upper not in jcamp_header:
                 jcamp_header[key_upper] = header[key]
         self.write_header(jcamp_header)
 
+        if not table.get('y') or not table.get('x'):
+            return
+
         data_class = header.get('DATA CLASS', DATA_CLASSES[0])
         if data_class == 'XYDATA':
             self.process_xydata(header, table.get('y'))
@@ -214,6 +232,14 @@ def write_header(self, header):
             if value is not None:
                 self.buffer.write('##{}={}'.format(key, value) + os.linesep)
 
+    def write_comment_header(self, header):
+        self.buffer.write('$$ ' + '-' * 20 + '\n')
+        for value in header:
+            if value is not None:
+                for line in value.split('\n'):
+                    self.buffer.write(f'$$ {line}{os.linesep}')
+        self.buffer.write('$$ ' + '-' * 20 + '\n')
+
     def write_xydata(self, y, npoints, firstx, deltax, max_decimal):
         for i in range(0, npoints, self.nline):
             x = float(firstx) + i * float(deltax)

test_manager/profile_results/CV/PalmSens/PSTrace-pssession/File053.pssession.json

@@ -4,6 +4,7 @@
 ##DATA CLASS=XYPOINTS
 ##ORIGIN=
 ##OWNER=
+##CALCULATION_APPLIED_Y=True
 ##XUNITS=Voltage in V
 ##YUNITS=Current in A
 ##CYCLES=1

test_manager/profile_results/CV/PalmSens/PSTrace-pssession/File053.pssession.zip

@@ -4,6 +4,7 @@
 ##DATA CLASS=XYPOINTS
 ##ORIGIN=
 ##OWNER=
+##CALCULATION_APPLIED_Y=True
 ##XUNITS=Voltage in V
 ##YUNITS=Current in A
 ##CYCLES=1

test_manager/profile_results/CV/PalmSens/PSTrace-pssession/File053.pssession/data/table_01.jdx

@@ -4,6 +4,7 @@
 ##DATA CLASS=XYPOINTS
 ##ORIGIN=
 ##OWNER=
+##CALCULATION_APPLIED_Y=True
 ##XUNITS=Voltage in V
 ##YUNITS=Current in A
 ##CYCLES=1

test_manager/profile_results/CV/PalmSens/PSTrace-pssession/File053.pssession/data/table_02.jdx

@@ -1,4 +1,4 @@
-f8ef9b2b69e1ae61e712ebc6d5faf860afdde3dd095062426576c288fb0dcd1e data/table_01.jdx
-b8e5436b9c021b914848c3abf05e387aa4d17ba08ac74af8beccdbf247afbb4a data/table_02.jdx
-cdb920d1aaa6ecf5e5efa15282a4c845e5cc670660ad152305750dadd2451e67 data/table_03.jdx
+4a4ac6759e69a734b4955fdc58bc5469610bee68f5a911d734851fd12572bef0 data/table_01.jdx
+573f634f7901a8484554987b6d72e82df8a097eeaa66a95d1915121824601d9d data/table_02.jdx
+e200adb008c57b899c12b3a078e04ddb8926f047dc85fd3c6c3fdb6c6cd86101 data/table_03.jdx
 d8f2e3d5c7eb1d956d2d1148d81d8ebc18865d62378d87de9d7507b644f05d6c metadata/converter.json

test_manager/profile_results/CV/PalmSens/PSTrace-pssession/File053.pssession/data/table_03.jdx

@@ -1,4 +1,4 @@
-1a0a9d5b3dc0438d709dbbd714200d8eb5aaba9a45f3bb76965109daa902dbcb089591571064695087a50ebba38db2ed9c46f35b8fa773fb426778fb9de13242 data/table_01.jdx
-a03b6d2818b55baa65aae9cf2473a004021fa782b563f8be83c377a85bbe984f1b5cb74a35e51b2b66691481b53f5bafcd4af4b9bddbb65a43856365d7144a7a data/table_02.jdx
-0629f98ce4b0e39c4b77f67cd401b48657194dd366ffc0870c203a735787fece455312dffd39cda50484c2d959b2bf85cff6f2f0be6ebe770de1665f50519115 data/table_03.jdx
+a472000bfb52a945034ce9153bf95da4d848ddfaccd81dc85ca5b76ecd7f440fb5845a16eeb96a48b439ad0fe5e93998ffc1a2de629c6949ff095fd9b4a68f21 data/table_01.jdx
+07d91df667562b38d1a7e2d3fc4bc636119262ac73a2d36569c154806132212cd9ea61f1b92d11b8ae7378b6dd9920790c08b7d1fac49b1d0f2eb8b2cf91b1aa data/table_02.jdx
+16a384b4fc6c8fa7d79a749a6bca440e3b0c0793c0eada28a4a594aa36e474d295fbbf506292e6130e08cb5fc6c159776d534c35818776f480d63cced976c6c4 data/table_03.jdx
 0eaba47a8dea7784066cb51fd2456528af7b801b4fd1c18f845810c9d0de3c7e227333aa470b1c9d133668af9cd166ad796b7251dc72b7e56a9329c49278d3f5 metadata/converter.json