Payment processing and CI improvements

.github/workflows/rechnung_checks_and_tests.yml

@@ -16,19 +16,18 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install -r requirements.txt
+        pip install -r requirements-dev.txt
     - name: Lint with flake8
       run: |
         pip install flake8
         # stop the build if there are Python syntax errors or undefined names
         flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
         # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
-        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+        flake8 . --count --max-complexity=10 --max-line-length=127 --statistics
     - name: Stylecheck with black
       run: |
         pip install black
-        black .
+        black --check .
     - name: Test with pytest
       run: |
-        pip install pytest pytest-cov
         make test

Makefile

@@ -11,9 +11,11 @@ check: bandit black-check pip-check test ## Run all checks
 bandit: ## Run bandit
 	python -m bandit -r rechnung
 
-.PHONY: black-check
-black-check: ## Check code formatting
-	python -m black --check rechnung
+.PHONY: style-check
+style-check: ## Check code formatting
+	flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+	flake8 . --count --max-complexity=10 --max-line-length=127 --statistics
+	python -m black --check .
 
 .PHONY: black
 black: ## Format code
@@ -68,8 +70,6 @@ uninstall: ## Remove the currently installed version of rechnung
 reinstall: uninstall install 
 
 .PHONY: docs
-docs:
-	.PHONY: docs
 docs:
 	rm -f docs/rechnung.rst
 	rm -f docs/modules.rst

docs/conf.py

@@ -17,47 +17,46 @@
 
 # -- Project information -----------------------------------------------------
 
-project = 'rechnung'
-copyright = '2019, Florian Rämisch, Paul Spooren'
-author = 'Florian Rämisch, Paul Spooren'
+project = "rechnung"
+copyright = "2019, Florian Rämisch, Paul Spooren"
+author = "Florian Rämisch, Paul Spooren"
 
 
 # -- General configuration ---------------------------------------------------
 
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
-extensions = [ 'sphinx.ext.autodoc',
-]
+extensions = ["sphinx.ext.autodoc"]
 
 # Add any paths that contain templates here, relative to this directory.
-templates_path = ['_templates']
+templates_path = ["_templates"]
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
 # This pattern also affects html_static_path and html_extra_path.
-exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
 
-master_doc = 'index'
+master_doc = "index"
 
 # -- Options for HTML output -------------------------------------------------
 
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
 #
-html_theme = 'alabaster'
+html_theme = "alabaster"
 
 html_theme_options = {
     "font_family": '"Avenir Next", Calibri, "PT Sans", sans-serif',
     "head_font_family": '"Avenir Next", Calibri, "PT Sans", sans-serif',
     "font_size": "14px",
     "page_width": "980px",
     "sidebar_width": "225px",
-    "show_relbars": True
+    "show_relbars": True,
 }
 
 
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
+html_static_path = ["_static"]

rechnung/cli.py

@@ -3,6 +3,7 @@
 import os
 import rechnung.invoice as invoice
 import rechnung.contract as contract
+import rechnung.payment as payment
 
 from .settings import get_settings_from_cwd, copy_assets, create_required_settings_file
 from .transactions import read_csv_files
@@ -136,6 +137,30 @@ def send_contract(cid):
     contract.send_contract(settings, cid)
 
 
+@cli1.command()
+@click.argument("bank_statement_file", type=click.Path(exists=True))
+def import_bank_statement(bank_statement_file):
+    """
+    Import a bank statement to be annotated with cids
+    and used in reports.
+    """
+    print(f"Importing bank statement {bank_statement_file}...")
+    settings = get_settings_from_cwd(cwd)
+    outfilename = payment.import_bank_statement_from_file(bank_statement_file, settings)
+    print(f"Saved as {outfilename}")
+
+
+@cli1.command()
+@click.argument("cid")
+def report(cid):
+    """
+    Create a report for a customer.
+    """
+    print(f"Reporting customer {cid}...")
+    settings = get_settings_from_cwd(cwd)
+    payment.report_customer(cid, settings)
+
+
 cli = click.CommandCollection(sources=[cli1])
 
 if __name__ == "__main__":

rechnung/contract.py

@@ -71,7 +71,7 @@ def send_contract(settings, cid):
     """
     Sends the contract specified with the cid via email to the customer.
 
-    If set, the policy and the product description of the main product 
+    If set, the policy and the product description of the main product
     will be attached.
     """
     mail_template = get_template(settings.contract_mail_template_file)

rechnung/payment.py

@@ -0,0 +1,148 @@
+#!/usr/bin/env python3
+import arrow
+import csv
+import locale
+import yaml
+
+from dataclasses import dataclass, asdict
+from decimal import Decimal
+from pathlib import Path
+
+
+@dataclass
+class Entry:
+    date: arrow.Arrow
+    subject: str
+    amount: Decimal
+    sender: str
+    receiver: str
+    cid: str = ""
+
+
+class PostBankCsvParser:
+
+    expected_header = [
+        "Buchungsdatum",
+        "Wertstellung",
+        "Umsatzart",
+        "Buchungsdetails",
+        "Auftraggeber",
+        "Empfänger",
+        "Betrag (\x80)",
+        "Saldo (\x80)",
+    ]
+
+    def __init__(self, q):
+        self.q = q
+
+    def header_ok(self, header):
+        if header != self.expected_header:
+            return False
+        return True
+
+    def amount_to_decimal(self, value):
+        value = value.replace("\x80", "")
+        return Decimal.from_float(locale.atof(value)).quantize(Decimal(self.q))
+
+    def sanitize_subject(self, subject):
+        return subject.replace("Referenz NOTPROVIDED", "").replace(
+            "Verwendungszweck", ""
+        )
+
+    def get_entry(self, row):
+        return Entry(
+            date=arrow.get(row[0], "DD.MM.YYYY"),
+            sender=row[4],
+            receiver=row[5],
+            subject=self.sanitize_subject(row[3]),
+            amount=self.amount_to_decimal(row[6]),
+        )
+
+
+class CsvHeaderError(Exception):
+    pass
+
+
+class Entries:
+    def __init__(self, entries=None):
+        if not entries:
+            self.entries = []
+        else:
+            self.entries = entries
+
+    @classmethod
+    def from_csv(cls, csv_file, parser):
+        entries = []
+        with open(csv_file, newline="") as csvfile:
+            reader = csv.reader(csvfile, delimiter=";", quotechar='"')
+            for r, row in enumerate(reader):
+                if r == 0:
+                    if parser.header_ok(row):
+                        continue
+                    else:
+                        raise CsvHeaderError(
+                            f"expected {parser.expected_header} but got {row}"
+                        )
+                entries.append(parser.get_entry(row))
+        return cls(entries)
+
+    @classmethod
+    def from_yaml(cls, yaml_files):
+        if isinstance(yaml_files, Path):
+            yaml_files = [yaml_files]
+
+        entries = []
+        for yaml_file in yaml_files:
+            with open(yaml_file) as infile:
+                data = yaml.load(infile, Loader=yaml.FullLoader)
+                for entry in data:
+                    entries.append(Entry(**entry))
+        return cls(entries)
+
+    def to_yaml(self):
+        return yaml.dump(list(map(asdict, self.entries)))
+
+
+def import_bank_statement_from_file(bank_statement_file, settings):
+    locale.setlocale(locale.LC_ALL, "de_DE.UTF-8")
+    e = Entries.from_csv(
+        bank_statement_file, PostBankCsvParser(settings.decimal_quantization)
+    )
+    outfilename = Path(settings.payments_dir).joinpath(
+        f"{arrow.now().isoformat()}.yaml"
+    )
+    with open(outfilename, "w") as outfile:
+        outfile.write(e.to_yaml())
+    return outfilename
+
+
+def report_customer(cid, settings):
+    payments = Entries.from_yaml(settings.payments_dir.iterdir())
+    invoices = get_invoices(cid, settings)
+    total = Decimal(0)
+
+    print("\nInvoices:")
+    for i in invoices:
+        print(f"{i['date']}\t-{i['total_gross']}")
+        total -= Decimal(i["total_gross"])
+
+    print("\nPayments:")
+    for p in payments.entries:
+        if p.cid != cid:
+            continue
+        print(f"{p.date.format('DD.MM.YYYY')}\t{p.amount}")
+        total += p.amount
+
+    total = total.quantize(Decimal(settings.decimal_quantization))
+    print("-".join(["" for i in range(20)]))
+    print(f"Balance: {total}")
+    print("=".join(["" for i in range(20)]))
+
+
+def get_invoices(cid, settings):
+    invoices_dir = settings.invoices_dir / cid
+    invoices = []
+    for invoice in invoices_dir.glob("*.yaml"):
+        with open(invoice) as i_file:
+            invoices.append(yaml.safe_load(i_file))
+    return invoices

rechnung/settings.py

@@ -45,6 +45,9 @@
     "invoices_dir": "invoices",
     "logo_asset_file": "logo.svg",
     "policy_attachment_asset_file": "policy.pdf",
+    "bank_statements_dir": "bank_statements",
+    "payments_dir": "payments",
+    "decimal_quantization": ".01",
 }
 possible_settings = set(required_settings + list(optional_settings.keys()))
 
@@ -91,6 +94,28 @@ def get_settings_from_cwd(
     )
 
 
+def check_required_settings(settings, required_settings):
+    """
+    Check if all required settings are set.
+    """
+    for key in required_settings:
+        if key not in set(settings.keys()):
+            raise RequiredSettingMissingError(
+                f"Setting {key} must be set in settings.yaml!"
+            )
+
+
+def check_unknown_settings(settings, possible_settings):
+    """
+    Check for unknown config options.
+    """
+    for key in list(settings.keys()):
+        if key not in possible_settings:
+            raise UnknownSettingError(
+                f"Setting {key} is unknown, and therefore cannot be configured."
+            )
+
+
 def get_settings_from_file(
     settings_path,
     error_on_unknown=True,
@@ -109,28 +134,16 @@ def get_settings_from_file(
     with open(settings_path) as infile:
         data = yaml.safe_load(infile)
 
-        # Check if all required settings are set in yaml file
-        for key in required_settings:
-            if key not in set(data.keys()):
-                raise RequiredSettingMissingError(
-                    f"Setting {key} must be set in settings.yaml!"
-                )
-
-        # Check for unknown config options
+        check_required_settings(data, required_settings)
         if error_on_unknown:
-            for key in list(data.keys()):
-                if key not in possible_settings:
-                    raise UnknownSettingError(
-                        f"Setting {key} is unknown, and therefore cannot be configured."
-                    )
+            check_unknown_settings(data, possible_settings)
 
         # Build settings dict
         settings_data = deepcopy(optional_settings)
         settings_data.update(data)
 
         # prepend base_path to all _dir and _file settings
         for s_key, s_value in settings_data.items():
-            # print(s_key, s_value)
             if s_key.endswith(("_file", "_dir")):
                 if s_key.endswith(("_asset_file", "_template_file")):
                     s_value = base_path / settings_data["assets_dir"] / s_value

rechnung/tests/data

@@ -0,0 +1 @@
+fixtures

rechnung/tests/fixtures/bank_statements/postbank.csv

@@ -0,0 +1,5 @@
+Buchungsdatum;Wertstellung;Umsatzart;Buchungsdetails;Auftraggeber;Empfänger;Betrag (€);Saldo (€)
+18.10.2019;18.10.2019;Gutschrift;Referenz NOTPROVIDED1000.2019.10;Martha Muster;Westnetz w.V.;60,21 €;3.381,43 €
+17.10.2019;16.10.2019;Gutschrift;Referenz NOTPROVIDEDKunde 1002;Klaus Karstens;Westnetz w.V.;48,45 €;3.321,22 €
+16.10.2019;16.10.2019;Dauergutschrift;Referenz NOTPROVIDEDVerwendungszweckFrank Nord Internet;Karina Surcu;Westnetz w.V.;96,9 €;3.272,77 €
+15.10.2019;15.10.2019;Gutschrift;Referenz NOTPROVIDED1000.2019.10;Martha Muster;Westnetz w.V.;60,21 €;3.175,87 €