Merge pull request #96 from statisticsnorway/formatting-tools

arneso-ssb · web-flow · commit b41ddc557555 · 2024-06-25T11:16:43.000+02:00
Add formatting tools
diff --git a/{{cookiecutter.project_name}}/.github/workflows/tests.yml b/{{cookiecutter.project_name}}/.github/workflows/tests.yml
@@ -43,8 +43,8 @@ jobs:
 
       - name: SonarCloud Scan
         env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any
-          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
+          GITHUB_TOKEN: {{ "${{ secrets.GITHUB_TOKEN }}" }} # Needed to get PR information, if any
+          SONAR_TOKEN: {{ "${{ secrets.SONAR_TOKEN }}" }}
         # No need to run SonarCloud analysis if dependabot update or token not defined
         if: env.SONAR_TOKEN != '' && (github.actor != 'dependabot[bot]')
         uses: SonarSource/sonarcloud-github-action@v2.1.1
diff --git a/{{cookiecutter.project_name}}/.gitignore b/{{cookiecutter.project_name}}/.gitignore
@@ -15,6 +15,11 @@
 *.xlsx
 *.zip
 
+# Notebooks shall be stored in .py or .R-format.
+# See https://adr.ssb.no/0020-lagringsformat-for-jupyter-notebooks/
+*.ipynb
+
+
 # The section below is from the GitHub .gitignore template for Python:
 # https://raw.githubusercontent.com/github/gitignore/main/Python.gitignore
 
diff --git a/{{cookiecutter.project_name}}/.pre-commit-config.yaml b/{{cookiecutter.project_name}}/.pre-commit-config.yaml
@@ -0,0 +1,73 @@
+repos:
+  - repo: local
+    hooks:
+      # General file checkers
+      - id: check-added-large-files
+        name: Check for added large files
+        entry: check-added-large-files
+        language: system
+      - id: check-merge-conflict
+        name: Check for merge conflict strings
+        entry: check-merge-conflict
+        language: system
+      - id: check-case-conflict
+        name: Check for problematic file names
+        entry: check-case-conflict
+        language: system
+
+      # General text checkers
+      - id: end-of-file-fixer
+        name: Fix End of Files
+        entry: end-of-file-fixer
+        language: system
+        types: [text]
+        stages: [commit, push, manual]
+      - id: trailing-whitespace
+        name: Trim Trailing Whitespace
+        entry: trailing-whitespace-fixer
+        language: system
+        types: [text]
+        stages: [commit, push, manual]
+      - id: mixed-line-ending
+        name: Mixed Line Ending
+        entry: mixed-line-ending
+        language: system
+        types: [text]
+        stages: [commit, push, manual]
+
+      # Serialization format checkers
+      - id: check-yaml
+        name: Check Yaml
+        entry: check-yaml
+        language: system
+        types: [yaml]
+      - id: check-json
+        name: Check JSON
+        entry: check-json
+        language: system
+        types: [json]
+      - id: check-toml
+        name: Check Toml
+        entry: check-toml
+        language: system
+        types: [toml]
+
+      # Python tools
+      - id: check-ast
+        name: Check Python syntax
+        entry: check-ast
+        language: system
+        types: [python]
+      - id: isort
+        name: isort
+        entry: isort
+        require_serial: true
+        language: system
+        types_or: [cython, pyi, python]
+        args: ["--filter-files"]
+      - id: black
+        name: black
+        entry: black
+        language: system
+        types_or: [python, jupyter]
+        require_serial: true
diff --git a/{{cookiecutter.project_name}}/SECURITY.md b/{{cookiecutter.project_name}}/SECURITY.md
@@ -1,6 +1,6 @@
 # Security Policy
 
-SSB takes the security of our software products and services seriously, which 
+SSB takes the security of our software products and services seriously, which
 includes all source code repositories managed through our GitHub organization.
 
 We believe that responsible disclosure of security vulnerabilities helps us ensure
diff --git a/{{cookiecutter.project_name}}/poetry.lock b/{{cookiecutter.project_name}}/poetry.lock
diff --git a/{{cookiecutter.project_name}}/pyproject.toml b/{{cookiecutter.project_name}}/pyproject.toml
@@ -15,16 +15,28 @@ ipykernel = ">=6.15.3"
 [tool.poetry.group.dev.dependencies]
 pytest = ">=7.1.3"
 pytest-cov = ">=5.0.0"
+black = {extras = ["jupyter"], version = ">=24.4.2"}
+isort = ">=5.12.0"
+pre-commit = ">=3.3.1"
+pre-commit-hooks = ">=4.4.0"
 
 [build-system]
 requires = ["poetry-core>=1.0.0"]
 build-backend = "poetry.core.masonry.api"
 
+[tool.isort]
+profile = "black"
+force_single_line = true
+skip_gitignore = true
+lines_after_imports = 2
+# The line below is needed for jupyter notebooks stored as .py in percent format.
+# See https://github.com/PyCQA/isort/issues/1338 for details
+treat_comments_as_code = ["# %%"]
+
 [tool.jupytext]
 formats = "ipynb,auto:percent"
 notebook_metadata_filter = "jupytext.text_representation,-jupytext.text_representation.jupytext_version,-widgets,-varInspector"
 cell_metadata_filter = "-papermill,tags"
 
 [tool.pytest.ini_options]
 pythonpath = ["src"]
-
diff --git a/{{cookiecutter.project_name}}/src/functions/__init__.py b/{{cookiecutter.project_name}}/src/functions/__init__.py
@@ -1 +1 @@
-# This file is required to do local imports of the functions within this folder
+# This file is required to do local imports of the functions within this folder
diff --git a/{{cookiecutter.project_name}}/src/functions/fizzbuzz.py b/{{cookiecutter.project_name}}/src/functions/fizzbuzz.py
@@ -1,21 +1,24 @@
 #!/usr/bin/python3
 
-def fizz(x:int) -> str:
-    if x%3 == 0:
+
+def fizz(x: int) -> str:
+    if x % 3 == 0:
         return "fizz"
     return ""
 
-def buzz(x:int) -> str:
-    if x%5 == 0:
+
+def buzz(x: int) -> str:
+    if x % 5 == 0:
         return "buzz"
     return ""
-    
-def fizzbuzz(x:list) -> list:
+
+
+def fizzbuzz(x: list) -> list:
     result = []
     for y in x:
         r = ""
         r = fizz(y) + buzz(y)
         if not r:
             r = y
         result.append(r)
-    return result
+    return result
diff --git a/{{cookiecutter.project_name}}/src/notebooks/00_imports.ipynb b/{{cookiecutter.project_name}}/src/notebooks/00_imports.ipynb
diff --git a/{{cookiecutter.project_name}}/src/notebooks/README.md b/{{cookiecutter.project_name}}/src/notebooks/README.md
@@ -1,3 +1,3 @@
 # Organisering av notebooks
 
-### Om datatilstandene i SSB og reflektering av de i mappestrukturen
+### Om datatilstandene i SSB og reflektering av de i mappestrukturen
diff --git a/{{cookiecutter.project_name}}/src/notebooks/imports.py b/{{cookiecutter.project_name}}/src/notebooks/imports.py
@@ -0,0 +1,47 @@
+# ---
+# jupyter:
+#   jupytext:
+#     text_representation:
+#       extension: .py
+#       format_name: percent
+#       format_version: '1.3'
+# ---
+
+# %% [markdown]
+# # Lagring av notebooks som rene tekstfiler
+# Jupyter notebooks skal lagres `.py`- eller `.R`-format, i stedet for i
+# `.ipynb`-format. Se begrunnelse og beskrivelse i
+# [ADR0020](https://adr.ssb.no/0020-lagringsformat-for-jupyter-notebooks/).
+# Du kan fortsatt jobbe med filene som notebooks på vanlig måte.
+#
+# For å åpne en slik fil som Notebook i JupyterLab, høyreklikk på fila og velg
+# Open With &rarr; Notebook.
+
+# %%
+# Central imports (pandas as pd etc.)
+import os
+import pandas as pd
+
+# %% [markdown]
+# ### Demo av importer fra produksjonsnivå
+
+# %%
+# Change directory until find project root
+notebook_path = os.getcwd()
+for folder_level in range(50):
+    if "pyproject.toml" in os.listdir():
+        break
+    os.chdir("../")
+
+# %%
+# Do local imports here
+from src.functions.fizzbuzz import fizzbuzz
+
+# %%
+# Reset current working directory after local imports
+os.chdir(notebook_path)
+
+# %%
+# Example local function import
+for x in fizzbuzz(range(1, 26)):
+    print(x)
diff --git a/{{cookiecutter.project_name}}/tests/__init__.py b/{{cookiecutter.project_name}}/tests/__init__.py
@@ -1 +1 @@
-# This file is required by pytest
+# This file is required by pytest
diff --git a/{{cookiecutter.project_name}}/tests/test_fizzbuzz.py b/{{cookiecutter.project_name}}/tests/test_fizzbuzz.py
@@ -1,14 +1,18 @@
-from functions.fizzbuzz import fizz, buzz, fizzbuzz
+from functions.fizzbuzz import buzz
+from functions.fizzbuzz import fizz
+from functions.fizzbuzz import fizzbuzz
 
 
 def test_fizz():
     assert fizz(3) == "fizz"
     assert fizz(4) == ""
 
+
 def test_buzz():
     assert buzz(5) == "buzz"
     assert buzz(6) == ""
 
+
 def test_fizzbuzz():
     assert fizzbuzz([15]) == ["fizzbuzz"]
     assert fizzbuzz([3]) == ["fizz"]

Original file line number	Diff line number	Diff line change
`@@ -1 +1 @@`
`1`		`-# This file is required to do local imports of the functions within this folder`
	`1`	`+# This file is required to do local imports of the functions within this folder`
Original file line number	Diff line number	Diff line change
`@@ -1,3 +1,3 @@`
`1`	`1`	`# Organisering av notebooks`
`2`	`2`
`3`		`-### Om datatilstandene i SSB og reflektering av de i mappestrukturen`
	`3`	`+### Om datatilstandene i SSB og reflektering av de i mappestrukturen`