feat: linting, formating and CI updates

Author: kpiwko

.github/workflows/ci.yaml

@@ -4,7 +4,7 @@ on:
     branches:
       - main
     tags:
-      - '**'
+      - "**"
   release:
     types:
       - published

.github/workflows/python_lint_and_test.yaml

@@ -20,8 +20,8 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
-          cache: 'pip'
-          cache-dependency-path: '**/requirements.txt'
+          cache: "pip"
+          cache-dependency-path: "**/requirements.txt"
           # architecture: x64
       - name: Install dependencies
         run: |
@@ -32,16 +32,8 @@ jobs:
       - name: Run python style and linting tools
         run: |
           err=0
-          echo autoflake:
-          autoflake --check-diff . || err=$?
-          echo black:
-          black --check . || err=$?
-          echo docformatter:
-          docformatter . || err=$?
-          echo flake8:
-          flake8 . || err=$?
-          echo isort:
-          isort --check . || err=$?
+          pre-commit install
+          pre-commit run --all-files --hook-stage pre-commit || err=$?
           exit $err
       - name: Run tests with coverage
         run: |

.github/workflows/requirements.txt

@@ -1,6 +1,3 @@
-autoflake
-black
 coverage[toml]
-docformatter
-flake8
-isort
+pre-commit
+

.pre-commit-config.yaml

@@ -0,0 +1,50 @@
+# This file contains 2 stages
+# pre-commit stage - runs by default pre-commit
+# manual stage - runs the same tools but modifies files
+# run as: pre-commit run --all-files --hook-stage manual
+repos:
+  #  See README.md for more information on this pre-commit hook
+  #  Please uncomment this hook if you are a Red Hat employee, and make sure you don't commit it back upstream
+  # - repo: https://gitlab.cee.redhat.com/infosec-public/developer-workbench/tools.git
+  #   rev: rh-pre-commit-2.3.0
+  #   hooks:
+  #     # If you have not run this hook on your system before, it may prompt you to
+  #     # log in for patterns, and you will need to try again.
+  #     #
+  #     # Docs: https://source.redhat.com/departments/it/it-information-security/leaktk/leaktk_components/rh_pre_commit
+  #     - id: rh-pre-commit
+  #       name: Red Hat pre-commit (check)
+  #     # - id: rh-pre-commit.commit-msg # Optional for commit-msg attestation
+  - repo: https://github.com/PyCQA/docformatter
+    rev: v1.7.7
+    hooks:
+      - id: docformatter
+        name: docformatter (check)
+        additional_dependencies: [tomli]
+        args: [--check, --config, ./pyproject.toml]
+        stages: [pre-commit]
+      - id: docformatter
+        name: docformatter (fix)
+        additional_dependencies: [tomli]
+        args: [--in-place, --config, ./pyproject.toml]
+        stages: [manual]
+  - repo: https://github.com/psf/black
+    rev: 25.1.0
+    hooks:
+      - id: black
+        name: black (check)
+        args: [--check]
+        stages: [pre-commit]
+      - id: black
+        name: black (fix)
+        stages: [manual]
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.12.1
+    hooks:
+      - id: ruff-check
+        name: ruff linter (check)
+        stages: [pre-commit]
+      - id: ruff-check
+        name: ruff linter (fix)
+        args: [--fix]
+        stages: [manual]

README.md

@@ -1,18 +1,18 @@
 # `logilica-cli`
 
 `logilica-cli` is a command-line tool which provides CLI access to the Logilica
-UI.  The tool currently provides two subcommands.  The first exports charts and
+UI. The tool currently provides two subcommands. The first exports charts and
 text from Logilica reports and makes them available in a variety of formats,
-including HTML, Markdown, and as a Google Docs document.  The second is used to
+including HTML, Markdown, and as a Google Docs document. The second is used to
 configure Logilica integrations, supporting a "configuration as code" approach.
 
 The tool is configured via a YAML file which contains tool configuration
-options as well as options for exporting data or configuring Logilica.  By
+options as well as options for exporting data or configuring Logilica. By
 default, the file `logica-cli.yaml` in the current directory is used, but an
 alternate path can be specified on the command line.
 
-For export, the configuration includes a list of teams.  For each team, it
-specifies a list of Logilica dashboards and the team's Jira project.  For each
+For export, the configuration includes a list of teams. For each team, it
+specifies a list of Logilica dashboards and the team's Jira project. For each
 dashboard, it specifies the name of an output file (which must not conflict
 with any other team's dashboards' output files) and possibly other attributes.
 
@@ -29,32 +29,31 @@ the configuration file.
 
 Credentials for accessing Logilica are provided by command line options or by
 the environment variables, `LOGILICA_DOMAIN`, `LOGILICA_EMAIL`, and
-`LOGILICA_PASSWORD`.  (Note to the Developer Practices Team:  the appropriate
+`LOGILICA_PASSWORD`. (Note to the Developer Practices Team: the appropriate
 values for the bot accounts can be obtained from the Bitwarden vault; users
 with admin privileges can also create their own username/password credentials
-using the Logilica UI.)  For interactive use, OAuth or SSO credentials can be
+using the Logilica UI.) For interactive use, OAuth or SSO credentials can be
 used. The tool normally runs in "headless" mode; however, in order to enable
 the user to perform an SSO login, the tool opens a browser window during the
-run.  (You should not interact with this window unless it prompts for your SSO
+run. (You should not interact with this window unless it prompts for your SSO
 credentials; otherwise, you are likely to disrupt the functioning of the tool.)
 
-To run the tool, check out the Git repo and run:
-```pip install -r requirements.txt .```
+To install the tool, check out the Git repo and run:
+
+```
+pip install -r requirements.txt .
+playwright install chromium
+```
+
 We recommend doing this inside a [virtual environment](https://docs.python.org/3/library/venv.html),
-which uses Python 3.12+. (If you are doing development on the tool, you may want
-to specify `-e` in the `pip install` command.)  After installing the tool, you also
-need to install the browser which it uses to interact with the Logilica web UI:
-```playwright install chromium```  (For details on debugging the web interactions,
-see the `--pwdebug` option in the command help, below, and the
-[Playwright documentation](https://playwright.dev/python/docs/running-tests).)
+which uses Python 3.12+.
 
 The PDF files containing the downloaded Logilica reports are stored in a
 temporary directory which is created if it does not exist; if it was created by
-the tool, the directory is deleted after execution is complete.  By default,
-the directory is named `lwr_downloaded_pdfs`, and it is created in the current
-directory, but an alternate path can be specified on the command line.
+the tool, the directory is deleted after execution is complete.
 
 Help is available on the command line:
+
 ```text
 Usage: logilica-cli [OPTIONS] COMMAND [ARGS]...
 
@@ -80,6 +79,7 @@ Commands:
 ```
 
 ```text
+logilica-cli weekly-report --help
 Usage: logilica-cli weekly-report [OPTIONS]
 
   Downloads and processes weekly report for teams specified in the
@@ -90,9 +90,11 @@ Options:
                                   Name  [env var: LOGILICA_DOMAIN; required]
   -t, --downloads-temp-dir DIRECTORY
                                   Path to a directory to receive downloaded
-                                  files (will be created if it doesn't exist;
-                                  will be deleted if created)  [default:
-                                  ./lwr_downloaded_pdfs]
+                                  files (if unspecified, a temporary directory
+                                  will be used; otherwise, the specified
+                                  directory will be created if it doesn't
+                                  exist; if the directory is created by the
+                                  tool, it will be deleted after the run)
   -I, --input [logilica|local]    Input source -- download from Logilica or
                                   use pre-downloaded files  [default:
                                   logilica]
@@ -161,7 +163,35 @@ Options:
                                   var: LOGILICA_EMAIL]
   --help                          Show this message and exit.
 ```
+
 Some dashboards can be quite slow to load into the UI, so, if you are running
 the tool interactively and wish to track its progress, add a `-v` to the command
-line to see informational messages.  (If you are debugging or just nosy, adding
+line to see informational messages. (If you are debugging or just nosy, adding
 a _second_ `-v` will add debugging messages to the output.)
+
+## Development
+
+If you are doing development on the tool, you may want to specify `-e` in the `pip install` command to install it in editable mode.
+This means that you can make changes to the package's source code and those changes will be reflected in your project without having to reinstall the package.
+
+For details on debugging the web interactions, see the `--pwdebug` option in the command help and the [Playwright documentation](https://playwright.dev/python/docs/running-tests).
+
+### Linting and formatting rules
+
+This project uses `pre-commit` to handle linting and formatting. The `pre-commit` tool reads its configuration from [`.pre-commit-config.yaml`](./.pre-commit-config.yaml) and the configuration options for the individual linters and formatters can be found in the [`pyproject.toml`](./pyproject.toml) file.
+The same configuration is used in CI.
+
+To install and execute it locally, follow these steps:
+
+```
+pip install pre-commit
+pre-commit install
+pre-commit run --all-files --hook-stage [pre-commit|manual]
+```
+
+If you run `pre-commit` stage, you can check your files, if you run `manual` stage, it will modify your files to correct problems it finds.
+
+### Enabling Red Hat InfoSec plugin
+
+If you are developing locally within Red Hat, you should enable Red Hat InfoSec plugin. You can do that by uncommenting appropriate lines in `.pre-commit-config.yaml` file.
+Make sure that you don't commit that change back upstream, as it would break the CI.

logilica_cli/data_sources.py

@@ -10,11 +10,7 @@
 @common_options
 @click.pass_context
 def data_sources(
-    context: click.Context,
-    username: str,
-    password: str,
-    domain: str,
-    oauth: bool,
+    context: click.Context, username: str, password: str, domain: str, oauth: bool
 ) -> None:
     """Synchronizes configuration of integrations with the configuration file.
 

logilica_cli/page_dashboard.py

@@ -9,8 +9,9 @@ class DashboardPage:
     PDF_EXPORT_TIMEOUT = 120000
 
     def __init__(self, page: Page):
-        # PDF generation might take longer than default 30000 ms timeout, so modify for this page in global
-        # It can be export or download button that takes time to become visible
+        # PDF generation might take longer than default 30000 ms timeout,
+        # so modify for this page in global.
+        # It can be export or download button that takes time to become visible.
         page.set_default_timeout(self.PDF_EXPORT_TIMEOUT)
         self.page = page
         self.export_pdf_button = page.get_by_role("button", name="Export PDF")

logilica_cli/page_navigation.py

@@ -26,7 +26,8 @@ def navigate(self, *, menu_dropdown: Optional[str], link_name: str) -> None:
 
         link_locator = self.page.get_by_role("link", name=link_name)
 
-        # if dropdown argument was provided, check if dropdown is open as it hides the link
+        # If dropdown argument was provided, check if dropdown is open as
+        # it hides the link
         if menu_dropdown:
             for first_try in (True, False):
                 try:

logilica_cli/page_settings.py

@@ -26,7 +26,6 @@ class SettingsPage:
     AVAILABLE_LIST_TIMEOUT = 54000
 
     def __init__(self, page: Page):
-
         self.page = page
         # UI elements for public access
         self.add_public_repository_dialog_button = page.get_by_role(
@@ -62,7 +61,9 @@ def open_integration_configuration(
         """
 
         logging.debug(
-            "Opening integration '%s', connector type:'%s'", integration, connector
+            "Opening integration '%s', connector type:'%s'",
+            integration,
+            connector,
         )
         self.page.locator("div.items-center").filter(has_text=connector).filter(
             has_text=integration
@@ -78,7 +79,8 @@ def sync_integrations(self, integrations: dict[str, Any]) -> None:
         in integrations sections of the local configuration file.
 
         Raises:
-          RuntimeError: If any repositories could not have been added to the configuration.
+          RuntimeError:
+            If any repositories could not have been added to the configuration.
         """
 
         sync_failures: IntegrationSyncFailures = defaultdict(list)
@@ -278,7 +280,8 @@ def has_entity_imported(
     ) -> bool:
         search_field.fill(entity_id)
 
-        # here we need to find the innermost div element that exactly matches repository slug
+        # Here we need to find the innermost div element
+        # that exactly matches repository slug.
         found = self.page.get_by_text(text=entity_id, exact=True).nth(0).is_visible()
         if found:
             logging.debug("✅%s '%s' is imported", entity_type, entity_id)
@@ -322,8 +325,8 @@ def add_public_repository(
         self.add_public_repository_dialog_button.click()
         self.add_public_repository_input.fill(f"{host}/{entity_id}.git")
         self.add_public_repository_confirm_button.click()
-        # as UI refresh is triggered independently of the click and there is no guarantee the repository will be added
-        # at the top of the page, we don't validate the action success here but later
+        # UI refresh is triggered independently of the click and there is no guarantee
+        # the repository will be added at the top of the page, we'll validate it later
         return True
 
     def add_membership_entity(
@@ -373,8 +376,8 @@ def control_button(self, entity_id: str, *, order=0) -> Optional[Locator]:
             .get_by_role("button")
         )
 
-        # there might be the same slug in both imported repositories and available repositories
-        # in that case, we want to be able to select one specifically
+        # There might be the same slug in both imported repositories and available
+        # repositories, in that case, we want to be able to select one specifically.
         count = locator.count()
         if count == 1:
             return locator

logilica_cli/pdf_convert.py

@@ -92,7 +92,6 @@ def to_format_multiple(
         teams: dict[str, dict[str, Any]],
         embed_images: bool = True,
     ) -> int:
-
         total = 0
         for team, dashboards in teams.items():
             for dashboard, options in dashboards["team_dashboards"].items():