Initial commit

Author: tstoeter

checkers.py

@@ -0,0 +1,10 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+
+class Checker:
+    pass
+
+
+class ShellscriptChecker(Checker):
+    pass

config.py

@@ -0,0 +1,17 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+
+# global configs
+config = {}
+config["MailServerHost"] = "smtp.example.com"
+config["MailServerPort"] = 25
+config["MailServerFrom"] = "[email protected]"
+
+
+# global secrets
+secrets = {}
+secrets["RspaceApiKey"] = ""
+secrets["LimesurveyUser"] = ""
+secrets["LimesurveyPassword"] = ""
+

datenbiene.py

@@ -0,0 +1,115 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+import sys
+import pandas as pd
+import logging
+import argparse
+import json
+from jinja2 import Template
+from sources import *
+from sinks import *
+from checkers import *
+
+
+__author__ = "Torsten Stöter"
+__copyright__ = "Copyright 2025 " + __author__
+__license__ = "GPL-3.0-only"
+__version__ = "1.0"
+
+
+logger = logging.getLogger(__name__)
+
+
+def str_to_class(classname):
+    return getattr(sys.modules[__name__], classname)
+
+
+def cli(args):
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "-e",
+        "--excel",
+        required=True,
+        help="Path to the Excel file containing Sources and Sinks sheets.",
+    )
+    parser.add_argument(
+        "-j",
+        "--json",
+        required=True,
+        help="Path to the JSON file containing key-value pairs.",
+    )
+    args = parser.parse_args(args)
+
+    sources_df = pd.read_excel(args.excel, "Sources")
+    checkers_df = pd.read_excel(args.excel, "Checkers")
+    sinks_df = pd.read_excel(args.excel, "Sinks")
+
+    with open(args.json) as f:
+        keyvals = json.load(f)
+    print(keyvals)
+
+    return sources_df, checkers_df, sinks_df, keyvals
+
+
+def main(sources_df, checkers_df, sinks_df, keyvals):
+
+    # apply Jinja2 template rendering to every cell using key-value pairs
+    sources_df = sources_df.map(
+        lambda x: Template(x).render(keyvals) if not pd.isnull(x) else x
+    )
+    print(sources_df)
+    sinks_df = sinks_df.map(
+        lambda x: Template(x).render(keyvals) if not pd.isnull(x) else x
+    )
+    print(sinks_df)
+
+    # process all sources
+    sources = []
+    for idx, row in sources_df.iterrows():
+
+        if pd.isnull(row["Source"]):
+            continue
+
+        try:
+            CS = str_to_class(row["Source"] + "Source")
+        except AttributeError:  # class not found, use generic source
+            CS = GenericSource
+
+        addopt = (
+            json.loads(row["AdditionalOptions"])
+            if not pd.isnull(row["AdditionalOptions"])
+            else {}
+        )
+        path_patterns = (
+            row["PathPatterns"].split(";") if not pd.isnull(row["PathPatterns"]) else []
+        )
+        file_patterns = (
+            row["FilePatterns"].split(";") if not pd.isnull(row["FilePatterns"]) else []
+        )
+        src = CS(row["Name"], path_patterns, file_patterns, keyvals | addopt)
+
+        print(src.get_filelist())
+        sources.append(src)
+
+    # run all checkers
+    # TODO
+
+    # process all sinks
+    for idx, row in sinks_df.iterrows():
+        print(row)
+        if pd.isnull(row["Sink"]):
+            continue
+
+        try:
+            CS = str_to_class(row["Sink"] + "Sink")
+        except AttributeError:  # class not found, use generic source
+            CS = Sink
+
+        snk = CS(row["Target"], sources, keyvals)
+        sources = snk.store()
+
+
+if __name__ == "__main__":
+    inputs = cli(sys.argv[1:])
+    main(*inputs)

example.json

@@ -0,0 +1,5 @@
+{
+  "study": "multi-modal-study-01",
+  "subject": "ab12",
+  "date": 20250202
+}

example.xlsx

@@ -0,0 +1,142 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+import os
+import tarfile
+import hashlib
+import filetype
+import owncloud
+import smtplib
+from email.message import EmailMessage
+from config import config
+from sources import Source
+
+
+class Sink:
+    def __init__(self, target, sources, additional_options=None):
+        self.target = target
+        self.sources = sources
+
+    def store(self):
+        return self.sources
+
+
+class TarballSink(Sink):
+    def __init__(self, target, sources, additional_options):
+        super().__init__(target, sources, additional_options)
+
+        self.subject = additional_options["subject"].lower()
+        # date string should be in format 20250124: here we strip all non-digits
+        self.date = int(
+            "".join([c for c in str(additional_options["date"]) if c.isdigit()])
+        )
+        self.study = additional_options["study"]
+
+        self.prefix = f"{self.subject}_{self.date}"
+
+    def store(self):
+
+        if not os.path.isdir(self.target):
+            os.mkdir(self.target)
+
+        session_dir = os.path.join(self.target, self.prefix)
+        os.mkdir(session_dir)
+
+        tarfiles = []
+
+        for src in self.sources:
+            print(src.name)
+            filelist = src.get_filelist()
+            compress = 9
+
+            # disbale compression for videos
+            if "video" in src.name.lower():
+                compress = 0
+            for f in filelist:
+                if not os.path.isfile(f):
+                    continue
+                kind = filetype.guess(f)
+                print(f, kind)
+                if kind and "video" in kind.mime:
+                    compress = 0
+
+            tarfilename = os.path.join(
+                session_dir, self.prefix + "_" + src.name + ".tar.gz"
+            )
+
+            tar = tarfile.open(tarfilename, "w:gz", compresslevel=compress)
+
+            for f in src.get_filelist():
+                print(f"Storing {f} to {tar} ...")
+                tar.add(f, arcname=os.path.basename(f))
+            tar.close()
+
+            tarfiles.append(tarfilename)
+
+        print("Computing checksums for ", tarfiles)
+
+        hashlines = []
+        for tf in tarfiles:
+            with open(tf, "rb") as f:
+                digest = hashlib.file_digest(f, "sha256").hexdigest()
+                hashlines.append(f"SHA256 ({os.path.basename(tf)}) = {digest}\n")
+
+        checksum_file = os.path.join(session_dir, self.prefix + "_Checksums.txt")
+        with open(checksum_file, "w") as f:
+            f.writelines(hashlines)
+
+        tarfiles.append(checksum_file)
+
+        return [Source("TarballSink", [session_dir], tarfiles)]
+
+
+class NextcloudSink(Sink):
+    def __init__(self, target, sources, additional_options=None):
+        super().__init__(target, sources, additional_options)
+
+    def store(self):
+        print("Uploading to NexCloud: " + self.target)
+
+        oc = owncloud.Client.from_public_link(self.target)
+        for src in self.sources:
+            for f in src.get_filelist():
+                print("Uploading... " + f)
+                oc.put_file("/" + os.path.basename(f), f, chunked=False)
+
+        return self.sources
+
+
+class EmailSink(Sink):
+    def __init__(self, target, sources, additional_options=None):
+        super().__init__(target, sources, additional_options)
+
+        self.subject = additional_options["subject"].lower()
+        # date string should be in format 20250124: here we strip all non-digits
+        self.date = int(
+            "".join([c for c in str(additional_options["date"]) if c.isdigit()])
+        )
+        self.study = additional_options["study"]
+
+    def store(self):
+        msg = EmailMessage()
+        msg.set_content("Hello World!")
+        msg["Subject"] = (
+            "Datenbiene: " + self.study + " " + self.subject + " " + str(self.date)
+        )
+        msg["From"] = config["MailServerFrom"]
+        msg["To"] = self.target
+
+        # send the message via our own SMTP server
+        s = smtplib.SMTP(config["MailServerHost"], port=config["MailServerPort"])
+        s.send_message(msg)
+        s.quit()
+
+        return self.sources
+
+
+class OmeroSink(Sink):
+    pass
+
+
+class BidsSink(Sink):
+    pass