CSV converted improvements and small improvements

Author: mantonovic

README.md

@@ -8,3 +8,49 @@ The project provides also a Graphical user Interface that allows for easing the
 For further information, please refer to the istSOS website: <http://istsos.org/>
 
 istSOS is released under the GPL License, and runs on all major platforms (Windows, Linux, Mac OS X), even though tests were conducted under a Linux environment.
+
+
+## Start istSOS with docker-compose
+
+The faste way to use istSOS is with docker.
+
+Create a docker-compose.yml file:
+
+```bash
+version: '3.7'
+
+services:
+  istsos-db:
+    image: postgis/postgis:12-2.5-alpine
+    restart: always
+    container_name: istsos-db
+    environment:
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: postgres
+      POSTGRES_DB: istsos
+      TZ: Europe/Zurich
+    volumes:
+      - istsos-db-data:/var/lib/postgresql/data
+
+  istsos:
+    image: istsos/istsos:2.4.1
+    container_name: istsos
+    restart: always
+    ports:
+      - 80:80
+
+volumes:
+    istsos-db-data:
+        name: istsos-db-data
+
+```
+
+And start the containers by running the following command:
+
+
+```bash
+docker-compose up -d
+```
+
+Open your browser and go to http://localhost/istsos/admin you should be able to see the web admin page.
+Before creating a new istsos service instance go to the "Database" page and set "istsos-db" as Host.

scripts/Dockerfile

@@ -0,0 +1,4 @@
+
+FROM python:2.7.18-slim-buster
+COPY ./ /app/scripts
+RUN pip install -r /app/scripts/requirements.txt

scripts/README.md

@@ -0,0 +1,25 @@
+# Create the virtualenv with docker
+
+Build the images
+
+```bash
+docker build --no-cache -t docker push docker.pkg.github.com/istsos/istsos2/istsos-scripts:2.4.1 .
+```
+
+Publish
+
+```bash
+docker push docker.pkg.github.com/istsos/istsos2/istsos-scripts:2.4.1
+```
+
+Run the script
+```bash
+docker run --rm -it \
+  --network="host" \
+  -v $PWD/acquisition:/app/acquisition  \
+  -v /home/milan/workspace/istsos/istsos2/scripts:/app/scripts  \
+  -v /mnt/acq/:/mnt/acq/  \
+  -w /app \
+  istsos/istsos-scripts:2.4.1 \
+  python ./acquisition/acquisition.py
+```

scripts/converter/csv.py

@@ -98,6 +98,61 @@
 )
 csv.execute()
 
+
+# File example: BR7_GW_____20210325000036.MIS
+# =====================================
+<STATION>BR7_GW____</STATION><SENSOR>0001</SENSOR><DATEFORMAT>YYYYMMDD</DATEFORMAT>
+20201024;181000;9.997
+20201024;182000;9.996
+20201024;183000;9.996
+<STATION>BR7_GW____</STATION><SENSOR>0002</SENSOR><DATEFORMAT>YYYYMMDD</DATEFORMAT>
+20201024;181000;8.68
+20201024;182000;8.70
+20201024;183000;8.68
+# =====================================
+
+# Block interval enable / disable
+importer = csv.CsvImporter('T_TRE', {
+        "enabled_from": {
+            "include": "<SENSOR>0001</SENSOR>"
+        },
+        "disabled_from": {
+            "include": "<SENSOR>",
+            "exclude": "<SENSOR>0001</SENSOR>"
+        },
+        skiplines: [
+            'NO DATA AVAILABLE'
+        ],
+        "headrows": 0,
+        "separator": ";",
+        "filenamedate": {
+            "format": '%Y%m%d%H%M%S',
+            "remove": ['BR7_GW_____','.MIS']
+        },
+        "datetime": {
+            "tz": '+01:00',
+            "time": {
+                "column": 1,
+                "format": '%H%M%S'
+            },
+            "date": {
+                "column": 2,
+                "format": '%Y%m%d'
+            }
+        },
+        "observations": [{
+            "observedProperty": "urn:ogc:def:parameter:x-istsos:1.0:meteo:air:temperature",
+            "column": 2
+        }]
+    },
+    'http://localhost/istsos', 'pippo',
+    "istsos/test/scripts/data/in/csv", 'BR7_GW_____20210325000036.MIS',
+    "istsos/test/scripts/data/out",
+    True
+)
+csv.execute()
+
+
 """
 
 from scripts import raw2csv
@@ -116,7 +171,19 @@ def __init__(self, procedureName, config, url, service, inputDir,
                  csvlength=5000, filenamecheck=None, archivefolder = None, extra={}):
 
         self.config = config
-        
+
+        self.max_cols = 0
+
+        if 'observations' in config:
+            for obs in config['observations']:
+                if 'column' in obs:
+                    self.max_cols = max(self.max_cols, obs['column'])
+
+        if 'enabled_from' in config:
+            self.enabled = False
+        else:
+            self.enabled = None
+
         raw2csv.Converter.__init__(self,
             procedureName, url, service, inputDir, fileNamePattern, outputDir,
             qualityIndex, exceptionBehaviour, user, password, debug, csvlength,
@@ -187,8 +254,6 @@ def setEndPositionFromFilename(self, fileName):
             
             self.setEndPosition(dt)
         """
-            
-        
 
     def parse(self, fileObj, fileName):
         # print "Filename: %s" % fileName
@@ -197,37 +262,67 @@ def parse(self, fileObj, fileName):
             cnt = cnt+1
             # Skipping header rows if present in configuration
             if "headrows" in self.config and cnt <= self.config['headrows']:
+                # print("skipping headrows (%s/%s)" % (cnt, self.config['headrows']))
                 continue
 
             elif "stopat" in self.config:
                 if isinstance(self.config["stopat"], str) and (
                         line.find(self.config["stopat"])>=0):
+                    # print("Breaking at line %s: %s" % (cnt, line))
                     break
 
-                elif isinstance(self.config["stopat"], int):
-                    pass
-
-                else:
-                    pass
-
-            # Line splitting
-            columns = line.split(self.config['separator'])
-            try:
-                date = self.parseDate(columns)
-                values = {}
-                for obs in self.config["observations"]:
-                    if obs["column"] is None:
-                        values[obs["observedProperty"]] = -999.9
-                    else:
-                        values[obs["observedProperty"]] = columns[obs["column"]]
-                self.addObservation(
-                    raw2csv.Observation(date, values)
-                )
-                self.setEndPosition(date)
-            except Exception as e:
-                print("%s [%s]:%s" % (fileName,cnt,line))
-                print(traceback.print_exc())
-                raise e
+            if "skiplines" in self.config:
+                skipline = False
+                for sl in self.config['skiplines']:
+                    if sl in line:
+                        skipline = True
+                        break
+                
+                if skipline is True:
+                    # print("Skipping at line %s, found '%s': %s" % (cnt, sl, line))
+                    continue
+
+            if self.enabled is not None and self.enabled is not True:
+
+                if self.config['enabled_from']['include'] in line:
+                    # print("Enabled at line %s, found '%s': %s" % (cnt, self.config['enabled_from']['include'], line))
+                    self.enabled = True
+                    continue
+
+            elif self.enabled is not None and self.enabled is True:
+
+                if (
+                    self.config['disabled_from']['include'] in line
+                    and self.config['disabled_from']['exclude'] not in line
+                ):
+                    self.enabled = False
+                    continue
+
+            if self.enabled is None or self.enabled is True:
+
+                # Line splitting
+                columns = line.split(self.config['separator'])
+
+                if self.max_cols > 0 and len(columns) < (self.max_cols+1):
+                    continue
+
+                try:
+                    date = self.parseDate(columns)
+                    values = {}
+                    for obs in self.config["observations"]:
+                        if obs["column"] is None:
+                            values[obs["observedProperty"]] = -999.9
+                        else:
+                            values[obs["observedProperty"]] = columns[obs["column"]]
+                    self.addObservation(
+                        raw2csv.Observation(date, values)
+                    )
+                    self.setEndPosition(date)
+
+                except Exception as e:
+                    print("%s [%s]:%s" % (fileName,cnt,line))
+                    print(traceback.print_exc())
+                    raise e
 
         self.setEndPositionFromFilename(fileName)
 

scripts/raw2csv.py

@@ -32,6 +32,7 @@
 import os
 from os import path
 import glob
+import fnmatch
 from datetime import datetime
 from datetime import timedelta
 import decimal
@@ -591,6 +592,7 @@ def prepareFiles(self):
             self.addException(msg)
             raise FileReaderError(msg)
 
+        print(" > Searching: %s" % os.path.join(self.folderIn, "%s" % (self.pattern)))
         files = list(filter(
             path.isfile,
             glob.glob(os.path.join(self.folderIn, "%s" % (self.pattern)))))

scripts/requirements.txt

@@ -6,3 +6,4 @@ pytz==2019.3
 requests==2.23.0
 six==1.14.0
 urllib3[secure]==1.25.7
+html==1.16