Add JascoReader and refactor readers (get_value) for different number locales (#19)

* Cleanup setup.py

* Refactor CSVReader to handle "one-line" files

* Refactor readers (get_value) for different number locales

* Add jasco reader and revert csv reader

* Fix get_value

Author: jochenklar

converter_app/readers/__init__.py

@@ -7,6 +7,7 @@
 from .brml import BrmlReader
 from .dta import DtaReader
 from .pssession import PsSessionReader
+from .jasco import JascoReader
 
 logger = logging.getLogger(__name__)
 
@@ -47,3 +48,4 @@ def match_reader(self, file, file_name, content_type):
 registry.register(BrmlReader)
 registry.register(DtaReader)
 registry.register(PsSessionReader)
+registry.register(JascoReader)

converter_app/readers/ascii.py

@@ -5,16 +5,14 @@
 
 logger = logging.getLogger(__name__)
 
-PATTERNS = {
-    'text': re.compile(r'[A-Za-z]{2,}'),                   # two or more chars in row
-    'floats': re.compile(r'(-?\d+[,.]*\d*[eE+\-\d]*)\S*')  # e.g. 1.00001E-10
-}
-
 
 class AsciiReader(Reader):
     identifier = 'ascii_reader'
     priority = 1000
 
+    # two or more chars in row
+    text_pattern = re.compile(r'[A-Za-z]{2,}')
+
     def check(self):
         logger.debug('file_name=%s content_type=%s mime_type=%s encoding=%s',
                      self.file_name, self.content_type, self.mime_type, self.encoding)
@@ -38,7 +36,7 @@ def get_tables(self):
             count = None
 
             # try to match text for the header
-            text_match = PATTERNS['text'].search(row)
+            text_match = self.text_pattern.search(row)
             if text_match:
                 if table['rows']:
                     # if a table is already there, this must be a new header
@@ -48,11 +46,10 @@ def get_tables(self):
                 table['header'].append(row)
             else:
                 # try to match columns of floats
-                row = row.replace('n.a.','0')
-                float_match = PATTERNS['floats'].findall(row)
+                row = row.replace('n.a.', '')
+                float_match = self.float_pattern.findall(row)
                 if float_match:
-                    # replace , by . in floats
-                    float_match = [float_str.replace(',', '.') for float_str in float_match]
+                    float_match = [self.get_value(float_str) for float_str in float_match]
                     count = len(float_match)
 
                     if table['rows'] and count != previous_count:

converter_app/readers/base.py

@@ -1,4 +1,5 @@
 import logging
+import re
 from pathlib import Path
 
 import magic
@@ -8,6 +9,10 @@
 
 class Reader(object):
 
+    float_pattern = re.compile(r'(-?\d+[,.]*\d*[eE+\-\d]*)\S*')
+    float_de_pattern = re.compile(r'(-?[\d.]+,\d*[eE+\-\d]*)')
+    float_us_pattern = re.compile(r'(-?[\d,]+.\d*[eE+\-\d]*)')
+
     def __init__(self, file, file_name, content_type):
         self.file = file
         self.file_name = file_name
@@ -52,8 +57,25 @@ def append_table(self, tables):
         tables.append(table)
         return table
 
+    def get_shape(self, row):
+        shape = []
+        for cell in row:
+            value = cell.strip()
+            if value in self.empty_values:
+                shape.append('')
+            elif self.float_pattern.match(value):
+                shape.append('f')
+            else:
+                shape.append('s')
+
+        return shape
+
     def get_value(self, value):
-        try:
-            return float(value.replace(',', '.'))
-        except ValueError:
+        if self.float_de_pattern.match(value):
+            # remove any digit group seperators and replace the comma with a period
+            return value.replace('.', '').replace(',', '.')
+        if self.float_us_pattern.match(value):
+            # just remove the digit group seperators
+            return value.replace(',', '')
+        else:
             return value

converter_app/readers/csv.py

@@ -1,4 +1,3 @@
-import copy
 import csv
 import io
 import logging
@@ -7,25 +6,25 @@
 
 logger = logging.getLogger(__name__)
 
-TABLE_MIN_ROWS = 20
-
-DELIMITERS = {
-    '\t': 'tab',
-    ' ': 'space',
-    ';': 'semicolon',
-    ',': 'comma',
-}
-LINETERMINATORS = {
-    '\r\n': '\\r\\n',
-    '\r': '\\r',
-    '\n': '\\n',
-}
-
 
 class CSVReader(Reader):
     identifier = 'csv_reader'
     priority = 100
 
+    empty_values = ['', 'n.a.']
+    table_min_rows = 20
+    delimiters = {
+        '\t': 'tab',
+        ' ': 'space',
+        ';': 'semicolon',
+        ',': 'comma',
+    }
+    lineterminators = {
+        '\r\n': '\\r\\n',
+        '\r': '\\r',
+        '\n': '\\n',
+    }
+
     def check(self):
         logger.debug('file_name=%s content_type=%s mime_type=%s encoding=%s',
                      self.file_name, self.content_type, self.mime_type, self.encoding)
@@ -35,7 +34,7 @@ def check(self):
             file_string = self.file_content.decode(self.encoding)
 
             # check different delimiters one by one
-            for delimiter in DELIMITERS.keys():
+            for delimiter in self.delimiters.keys():
                 try:
                     self.dialect = csv.Sniffer().sniff(file_string, delimiters=delimiter)
                     result = True
@@ -44,9 +43,8 @@ def check(self):
                     pass
 
         if result:
-            io_string = io.StringIO(file_string)
-            self.lines = list(copy.copy(io_string))
-            self.rows = list(csv.reader(io_string, self.dialect))
+            self.rows = list(csv.reader(io.StringIO(file_string), self.dialect))
+            self.lines = file_string.splitlines()
 
         logger.debug('result=%s', result)
         return result
@@ -69,7 +67,7 @@ def get_tables(self):
         # loop over blocks and sort into header, table, and metadata
         prev_block = None
         for block in blocks:
-            if len(block['indexes']) < TABLE_MIN_ROWS or not block['shape']:
+            if len(block['indexes']) < self.table_min_rows or not block['shape']:
                 # this is the header
                 if table['rows']:
                     # if a table is already there, this must be a new header
@@ -92,7 +90,7 @@ def get_tables(self):
                             # remove the colum line from the header
                             table['header'] = table['header'][:-1]
 
-                table['rows'] += [self.rows[index] for index in block['indexes']]
+                table['rows'] += [[self.get_value(value) for value in self.rows[index]] for index in block['indexes']]
 
             prev_block = block
 
@@ -114,25 +112,25 @@ def get_tables(self):
 
     def get_metadata(self):
         metadata = super().get_metadata()
-        metadata['lineterminator'] = LINETERMINATORS.get(self.dialect.lineterminator, self.dialect.lineterminator)
+        metadata['lineterminator'] = self.lineterminators.get(self.dialect.lineterminator, self.dialect.lineterminator)
         metadata['quoting'] = self.dialect.quoting
         metadata['doublequote'] = self.dialect.doublequote
-        metadata['delimiter'] = DELIMITERS.get(self.dialect.delimiter, self.dialect.delimiter)
+        metadata['delimiter'] = self.delimiters.get(self.dialect.delimiter, self.dialect.delimiter)
         metadata['quotechar'] = self.dialect.quotechar
         metadata['skipinitialspace'] = self.dialect.skipinitialspace
         return metadata
 
     def get_shape(self, row):
         shape = []
         for cell in row:
-            if cell.strip() == '':
+            value = cell.strip()
+            if value in self.empty_values:
                 shape.append('')
+            elif self.float_pattern.match(value):
+                shape.append('f')
             else:
-                try:
-                    float(cell.replace(',', '.'))
-                    shape.append('f')
-                except ValueError:
-                    shape.append('s')
+                shape.append('s')
+
         return shape
 
     def compare_shape(self, shape_a, shape_b):

converter_app/readers/excel.py

@@ -86,9 +86,9 @@ def get_shape(self, row):
             if cell is None:
                 shape.append(None)
             else:
-                try:
-                    float(cell)
+
+                if isinstance(cell, (int, float)):
                     shape.append('f')
-                except (ValueError, TypeError):
+                else:
                     shape.append('s')
         return shape

converter_app/readers/jasco.py

@@ -0,0 +1,56 @@
+import io
+import logging
+import os
+
+from .base import Reader
+
+logger = logging.getLogger(__name__)
+
+
+class JascoReader(Reader):
+    identifier = 'jasco_reader'
+    priority = 99
+    header_length = 8
+
+    def check(self):
+        logger.debug('file_name=%s content_type=%s mime_type=%s encoding=%s',
+                     self.file_name, self.content_type, self.mime_type, self.encoding)
+
+        result = False
+        if self.encoding != 'binary':
+            file_string = self.file_content.decode(self.encoding)
+            if len(file_string.splitlines()) == 1:
+                file_lines = file_string.split(',')
+                if file_lines[self.header_length - 1] == str(len(file_lines) - self.header_length):
+                    result = True
+        if result:
+            self.lines = file_lines
+
+        logger.debug('result=%s', result)
+        return result
+
+    def get_tables(self):
+        tables = []
+        table = self.append_table(tables)
+
+        for i, line in enumerate(self.lines):
+            if i < self.header_length:
+                table['header'].append(line)
+            else:
+                x, y = line.split()
+                table['rows'].append((self.get_value(x), self.get_value(y)))
+
+        # build columns
+        for table in tables:
+            table['columns'] = []
+            if table['rows']:
+                for idx in range(len(table['rows'][0])):
+                    table['columns'].append({
+                        'key': str(idx),
+                        'name': 'Column #{}'.format(idx)
+                    })
+
+            table['metadata']['rows'] = len(table['rows'])
+            table['metadata']['columns'] = len(table['columns'])
+
+        return tables

converter_app/utils.py

@@ -39,6 +39,7 @@ def check_uuid(string):
     except ValueError:
         return False
 
+
 def checkpw(password, hashed_password):
     m = hashlib.sha1()
     m.update(password)

setup.py

@@ -29,10 +29,5 @@
         'License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)'
     ],
     packages=find_packages(),
-    include_package_data=True,
-    entry_points={
-        'console_scripts': [
-            'chemotion-converter=converter_app.scripts:converter',
-        ]
-    }
+    include_package_data=True
 )