refactor: use src instead of flat layout (#13)

OLILHR · Konstantin · web-flow · commit d3ef696f55cc · 2024-11-19T07:29:45.000+01:00
* Use src layout and fix imports

* Add setup descriptions and contribution notes to readme

* Add license

* Create main entry point for script

* Fix `pyproject.toml` formatting

* fix: "pyproject.toml: [mypy]: Unrecognized option: truthy-bool = True"

* minor modifications to tox.ini

just to be compliant with template repo

* improve type ignore

* make black formatting compatible with python 3.11 (as long as we officially support 3.11)

* format

* test packaging in CI

* Add publishing workflow

---------

Co-authored-by: Konstantin &lt;konstantin.klein+github@hochfrequenz.de&gt;
diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml
@@ -0,0 +1,65 @@
+# This GitHub workflow is only needed for python package releases which are supposed to be published on pypi.
+# It requires the Github "environments" feature (see instructions below) it might not be available for private free accounts (but works for public or organization repos).
+# After creating the "release" environment in the Github repo settings, you need to enter your Github organization/user name + repo name + "python-publish.yml" workflow file name in the PyPI UI to make this work.
+
+# This workflow uploads a Python Package using Twine when a release is created.
+# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
+
+name: Upload Python Package
+
+on:
+  release:
+    types: [created, edited]
+
+jobs:
+  tests:
+    if: startsWith(github.ref, 'refs/tags/v')
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        python-version: ["3.12"]
+        os: [ubuntu-latest]
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install tox
+        run: |
+          python -m pip install --upgrade pip
+          pip install tox
+      - name: Run tox
+        run: |
+          tox
+
+  build-n-publish:
+    name: Build and publish Python 🐍 distributions 📦 to PyPI and TestPyPI
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        python-version: ["3.12"]
+        os: [ubuntu-latest]
+    # Specifying a GitHub environment, # Specifying a GitHub environment, which is strongly recommended by PyPI: https://docs.pypi.org/trusted-publishers/adding-a-publisher/
+    # you have to create an environment in your repository settings and add the environment name here
+    environment: release
+    permissions:
+      # IMPORTANT: this permission is mandatory for trusted publishing
+      id-token: write
+    needs: tests
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install .[packaging]
+      - name: Build wheel and source distributions
+        run: |
+          python -m build
+      - name: Publish distribution 📦 to PyPI
+        if: startsWith(github.ref, 'refs/tags/v')
+        uses: pypa/gh-action-pypi-publish@release/v1
diff --git a/.github/workflows/pythonlint.yml b/.github/workflows/pythonlint.yml
@@ -12,7 +12,7 @@ jobs:
       matrix:
         python-version: ["3.13"]
         os: [ubuntu-latest]
-        linter-env: ["linting", "type_check", "spell_check"]
+        linter-env: ["linting", "type_check", "spell_check", "test_packaging"]
     steps:
       - uses: actions/checkout@v4
       - name: Set up Python ${{ matrix.python-version }}
diff --git a/README.md b/README.md
@@ -1,15 +1,37 @@
-# ahb-diff
+<img src="https://raw.githubusercontent.com/Hochfrequenz/ahlbatross/main/ahlbatross.png" alt="ahlbatross.png">
 
-![Unittests status badge](https://github.com/Hochfrequenz/ahb-diff/workflows/Unittests/badge.svg)
-![Coverage status badge](https://github.com/Hochfrequenz/ahb-diff/workflows/Coverage/badge.svg)
-![Pylint status badge](https://github.com/Hochfrequenz/ahb-diff/workflows/Linting/badge.svg)
-![Formatting status badge](https://github.com/Hochfrequenz/ahb-diff/workflows/Formatting/badge.svg)
+# AHlBatross
+
+[![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg?labelColor=30363D&color=fccccc)](LICENSE)
+![Unittests status badge](https://github.com/Hochfrequenz/ahlbatross/workflows/Unittests/badge.svg)
+![Coverage status badge](https://github.com/Hochfrequenz/ahlbatross/workflows/Coverage/badge.svg)
+![Pylint status badge](https://github.com/Hochfrequenz/ahlbatross/workflows/Linting/badge.svg)
+![Formatting status badge](https://github.com/Hochfrequenz/ahlbatross/workflows/Formatting/badge.svg)
 
 Tool for **automatic AHB comparison** of consecutive `Formatversionen` provided by 
-[machine-readable-anwendungshandbücher](https://github.com/Hochfrequenz/machine-readable_anwendungshandbuecher/).
+[machine-readable-anwendungshandbücher](https://github.com/Hochfrequenz/machine-readable_anwendungshandbuecher/).<br>
+Highlighted changes between `PruefIDs` of various `Nachrichtenformate` are stored in the `.xlsx` files located inside 
+the `./data/output/` [directory](https://github.com/Hochfrequenz/ahlbatross/tree/main/data/output).
+
+<img width="75%" src="https://raw.githubusercontent.com/Hochfrequenz/ahlbatross/main/output.png" alt="output.png">
+
+## Development setup
+
+To set up the python development environment, install the required dependencies via
+
+```shell
+$ tox -e dev
+```
+
+For local testing and code quality maintenance, run 
+
+```shell
+tox
+```
 
-Merged `.csv` and `.xlsx` files are located inside the `./data/output/` 
-[directory](https://github.com/Hochfrequenz/ahb-diff/tree/main/data/output).
+Check out our [Python Template Repository](https://github.com/Hochfrequenz/python_template_repository#how-to-use-this-repository-on-your-machine) 
+for detailed descriptions and step-by-step instructions.
 
-<img width="90%" src="https://raw.githubusercontent.com/Hochfrequenz/ahb-diff/main/output.png" alt="output.png">
+## Contribute
 
+Feel free to contribute to this project by opening a pull request against the main branch.
diff --git a/ahlbatross.png b/ahlbatross.png
diff --git a/pyproject.toml b/pyproject.toml
@@ -1,5 +1,5 @@
 [project]
-name = "ahb-diff"
+name = "ahlbatross"
 description = ""
 version = "0.1.0"
 license = { text = "MIT" }
@@ -25,6 +25,13 @@ dependencies = [
     "xlsxwriter>=3.2.0"
 ]
 
+[project.scripts]
+ahlbatross = "ahlbatross.main:main"
+
+[project.urls]
+Changelog = "https://github.com/Hochfrequenz/ahlbatross/releases"
+Homepage = "https://github.com/Hochfrequenz/ahlbatross"
+
 [project.optional-dependencies]
 tests = [
     "pytest==8.3.3"
@@ -53,13 +60,9 @@ dev = [
     "pip-tools"
 ]
 
-[project.urls]
-Changelog = "https://github.com/Hochfrequenz/ahb-diff/releases"
-Homepage = "https://github.com/Hochfrequenz/ahb-diff"
-
 [tool.black]
 line-length = 120
-target_version = ["py312", "py313"]
+target_version = ["py311", "py312", "py313"]
 
 [tool.isort]
 line_length = 120
@@ -70,7 +73,6 @@ max-line-length = 120
 
 [mypy]
 truethy-bool = true
-
 [tool.mypy]
 disable_error_code = []
 
diff --git a/requirements.txt b/requirements.txt
@@ -9,9 +9,9 @@ numpy==2.1.3
     #   pandas
     #   pandas-stubs
 pandas==2.2.3
-    # via ahb-diff (pyproject.toml)
+    # via ahlbatross (pyproject.toml)
 pandas-stubs==2.2.3.241009
-    # via ahb-diff (pyproject.toml)
+    # via ahlbatross (pyproject.toml)
 python-dateutil==2.9.0.post0
     # via pandas
 pytz==2024.2
@@ -23,4 +23,4 @@ types-pytz==2024.2.0.20241003
 tzdata==2024.2
     # via pandas
 xlsxwriter==3.2.0
-    # via ahb-diff (pyproject.toml)
+    # via ahlbatross (pyproject.toml)
diff --git a/src/ahlbatross/__init__.py b/src/ahlbatross/__init__.py
diff --git a/src/ahlbatross/main.py b/src/ahlbatross/main.py
@@ -5,17 +5,17 @@
 import logging
 import sys
 from pathlib import Path
-from typing import Any, Tuple, TypeAlias
+from typing import Any, Optional, Tuple, TypeAlias
 
 import pandas as pd
 from pandas.core.frame import DataFrame
-from xlsxwriter.format import Format  # type: ignore
+from xlsxwriter.format import Format  # type:ignore[import-untyped]
 
 logging.basicConfig(level=logging.INFO, format="%(message)s", stream=sys.stdout)
 logger = logging.getLogger(__name__)
 
 SUBMODULE = Path("data/machine-readable_anwendungshandbuecher")
-OUTPUT_DIR = Path("data/output")
+DEFAULT_OUTPUT_DIR = Path("data/output")
 
 XlsxFormat: TypeAlias = Format
 
@@ -596,7 +596,9 @@ def export_to_excel(df: DataFrame, output_path_xlsx: str) -> None:
         logger.info("✅successfully exported XLSX file to: %s", {output_path_xlsx})
 
 
-def _process_files(previous_formatversion: str, subsequent_formatversion: str) -> None:
+def _process_files(
+    previous_formatversion: str, subsequent_formatversion: str, output_dir: Path = DEFAULT_OUTPUT_DIR
+) -> None:
     """
     process all matching ahb/<pruefid>.csv files between two <formatversion> directories.
     """
@@ -606,7 +608,7 @@ def _process_files(previous_formatversion: str, subsequent_formatversion: str) -
         logger.warning("No matching files found to compare")
         return
 
-    output_base = OUTPUT_DIR / f"{subsequent_formatversion}_{previous_formatversion}"
+    output_base = output_dir / f"{subsequent_formatversion}_{previous_formatversion}"
 
     for previous_pruefid, subsequent_pruefid, nachrichtentyp, pruefid in matching_files:
         logger.info("Processing %s - %s", nachrichtentyp, pruefid)
@@ -641,7 +643,7 @@ def _process_files(previous_formatversion: str, subsequent_formatversion: str) -
             logger.error("❌data processing error for %s/%s: %s", nachrichtentyp, pruefid, str(e))
 
 
-def _process_submodule() -> None:
+def _process_submodule(output_dir: Path = DEFAULT_OUTPUT_DIR) -> None:
     """
     processes all valid consecutive <formatversion> subdirectories.
     """
@@ -656,7 +658,7 @@ def _process_submodule() -> None:
             "⌛processing consecutive formatversions: %s -> %s", subsequent_formatversion, previous_formatversion
         )
         try:
-            _process_files(previous_formatversion, subsequent_formatversion)
+            _process_files(previous_formatversion, subsequent_formatversion, output_dir)
         except (OSError, pd.errors.EmptyDataError, ValueError) as e:
             logger.error(
                 "❌error processing formatversions %s -> %s: %s",
@@ -667,5 +669,17 @@ def _process_submodule() -> None:
             continue
 
 
+def main(output_dir: Optional[Path] = None) -> None:
+    """
+    main entrypoint for AHlBatross.
+    """
+    try:
+        _process_submodule(output_dir or DEFAULT_OUTPUT_DIR)
+    except (OSError, pd.errors.EmptyDataError, ValueError) as e:
+        logger.error("❌error processing AHB files: %s", str(e))
+        sys.exit(1)
+
+
+# run locally using $ PYTHONPATH=src python -m ahlbatross.main
 if __name__ == "__main__":
-    _process_submodule()
+    main()
diff --git a/src/ahlbatross/py.typed b/src/ahlbatross/py.typed
diff --git a/tox.ini b/tox.ini
@@ -16,7 +16,7 @@ commands = python -m pip install --upgrade pip
 deps =
     -r requirements.txt
     .[tests]
-setenv = PYTHONPATH = {toxinidir}/ahb_diff
+setenv = PYTHONPATH = {toxinidir}/src
 commands = python -m pytest --basetemp={envtmpdir} {posargs}
 
 [testenv:linting]
@@ -25,31 +25,31 @@ deps =
     {[testenv:tests]deps}
     .[linting]
     # add your fixtures like e.g. pytest_datafiles here
-setenv = PYTHONPATH = {toxinidir}/ahb_diff
+setenv = PYTHONPATH = {toxinidir}/src
 commands =
-    pylint ahb_diff
+    pylint ahlbatross
     pylint unittests --rcfile=unittests/.pylintrc
     # add single files (ending with .py) or packages here
 
 [testenv:type_check]
 # the type_check environment checks the type hints using mypy
-setenv = PYTHONPATH = {toxinidir}/ahb_diff
+setenv = PYTHONPATH = {toxinidir}/src
 deps =
     {[testenv:tests]deps}
     .[type_check]
 commands =
-    mypy --show-error-codes ahb_diff --strict
+    mypy --show-error-codes src/ahlbatross --strict
     mypy --show-error-codes unittests --strict
     # add single files (ending with .py) or packages here
 
 [testenv:spell_check]
 # the spellcheck environment checks the code for typos
-setenv = PYTHONPATH = {toxinidir}/ahb_diff
+setenv = PYTHONPATH = {toxinidir}/src
 deps =
     -r requirements.txt
     .[spell_check]
 commands =
-    codespell ahb_diff
+    codespell src
     codespell README.md
     # add single files (ending with .py) or packages here
 
@@ -59,7 +59,7 @@ changedir = unittests
 deps =
     {[testenv:tests]deps}
     .[coverage]
-setenv = PYTHONPATH = {toxinidir}/ahb_diff
+setenv = PYTHONPATH = {toxinidir}/src
 commands =
     coverage run -m pytest --basetemp={envtmpdir} {posargs}
     coverage html --omit .tox/*,unittests/*
@@ -71,6 +71,14 @@ deps =
 commands =
     pip-compile-multi -d dev_requirements --autoresolve
 
+[testenv:test_packaging]
+skip_install = true
+deps =
+    .[packaging]
+commands =
+    python -m build
+    twine check dist/*
+
 [testenv:dev]
 # the dev environment contains everything you need to start developing on your local machine.
 deps =
diff --git a/unittests/test_export.py b/unittests/test_export.py
@@ -5,7 +5,7 @@
 import pandas as pd
 from pandas.testing import assert_frame_equal
 
-from ahb_diff.main import align_columns, export_to_excel
+from ahlbatross.main import align_columns, export_to_excel
 
 
 def test_export() -> None:
diff --git a/unittests/test_parse_submodule.py b/unittests/test_parse_submodule.py
@@ -3,7 +3,7 @@
 import pytest
 from _pytest.monkeypatch import MonkeyPatch
 
-from ahb_diff.main import determine_consecutive_formatversions, get_matching_pruefid_files, parse_formatversions
+from ahlbatross.main import determine_consecutive_formatversions, get_matching_pruefid_files, parse_formatversions
 
 
 def test_parse_valid_formatversions() -> None:
@@ -37,7 +37,7 @@ def test_get_matching_files(tmp_path: Path, monkeypatch: MonkeyPatch) -> None:
     """
     test find matching files across formatversions.
     """
-    monkeypatch.setattr("ahb_diff.main.SUBMODULE", tmp_path)
+    monkeypatch.setattr("ahlbatross.main.SUBMODULE", tmp_path)
 
     submodule: dict[str, dict[str, dict[str, str]]] = {
         "FV2504": {
@@ -71,7 +71,7 @@ def test_determine_consecutive_formatversions(tmp_path: Path, monkeypatch: Monke
     """
     test successful determination of consecutive formatversions.
     """
-    monkeypatch.setattr("ahb_diff.main.SUBMODULE", tmp_path)
+    monkeypatch.setattr("ahlbatross.main.SUBMODULE", tmp_path)
 
     submodule: dict[str, dict[str, bool | dict[str, str]]] = {
         "FV2504": {"nachrichtenformat_1": True},
diff --git a/unittests/test_process_csv.py b/unittests/test_process_csv.py
@@ -3,7 +3,7 @@
 from pandas.core.frame import DataFrame
 from pandas.testing import assert_frame_equal
 
-from ahb_diff.main import align_columns
+from ahlbatross.main import align_columns
 from unittests.conftest import FormatVersions
 
 
