fix: pass process variables at runtime, and log errors

Author: samhotep

webapp/github.py

@@ -1,10 +1,19 @@
 import json
+import logging
 from pathlib import Path
 
 import flask
 import requests
 
-from webapp.tasks import save_github_file
+from webapp.settings import BASE_DIR, GH_TOKEN
+from webapp.tasks import register_task
+
+# Configure logger
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(levelname)s  [%(name)s] %(message)s",
+)
+logger = logging.getLogger(__name__)
 
 
 class Tree:
@@ -24,18 +33,14 @@ class GithubError(Exception):
 
 
 class GitHub:
-    def __init__(
-        self,
-        app: flask.Flask = None,
-    ):
+    logger = logger
+
+    def __init__(self) -> None:
         """
         Initialize the Github object.
-
-        Args:
-            app (Flask): The Flask application instance.
         """
-        self.REPOSITORY_PATH = Path(app.config["BASE_DIR"]) / "repositories"
-        token = app.config["GH_TOKEN"]
+        self.REPOSITORY_PATH = Path(BASE_DIR) / "repositories"
+        token = GH_TOKEN
         self.headers = {
             "Accept": "application/vnd.github.raw+json",
             "Authorization": f"Bearer {token}",
@@ -46,18 +51,19 @@ def __request__(
         self,
         method: str,
         url: str,
-        data: dict = {},
-        params: dict = {},
+        data: dict | None = None,
+        params: dict | None = None,
         blob=False,
-    ):
-        if data:
-            data = json.dumps(data)
+    ) -> bytes | dict:
+        req_data = json.dumps(data) if data else data
+
         response = requests.request(
             method,
             GITHUB_API_URL + url,
-            data=data,
+            data=req_data,
             headers=self.headers,
             params=params,
+            timeout=10,
         )
 
         if blob:
@@ -66,14 +72,12 @@ def __request__(
         if response.status_code == 200:
             return response.json()
 
-        raise GithubError(
+        err = (
             "Failed to make a request to GitHub. Status code:"
             f" {url} {method} {data} {params}"
             f" {response.status_code}. Response: {response.text}",
         )
-
-    def _url_path(self, url: str):
-        return url.split(self.URL)[-1]
+        raise GithubError(err)
 
     def get_file_content(self, repository: str, path: str) -> bytes:
         """
@@ -103,18 +107,53 @@ def get_repository_tree(self, repository: str, branch: str = "main"):
         """
         tree_file_path = self.REPOSITORY_PATH / repository
         tree_file_path.mkdir(parents=True, exist_ok=True)
-        data = self.__request__(
-            "GET",
-            f"repos/canonical/{repository}/git/trees/{branch}?recursive=1",
-        )
+
+        try:
+            data = self.__request__(
+                "GET",
+                f"repos/canonical/{repository}/git/trees/{branch}?recursive=1",
+            )
+        except Exception as e:
+            print(f"Failed to get repository tree: {e}")
+            raise
         for item in data["tree"]:
             if item["type"] == "blob" and item["path"].startswith("templates"):
-                save_github_file.delay(
+                task = async_save_file(
                     tree_file_path=str(tree_file_path),
                     repository=repository,
                     path=item["path"],
                 )
+                task()
+
+
+@register_task()
+def async_save_file(
+    repository: str = "",
+    path: str = "",
+    tree_file_path: str = "",
+) -> None:
+    """
+    Download a file to a given repository.
+
+    Args:
+        repository (str): The repository name.
+        path (str): The remote path to the file inside the repository.
+        tree_file_path (str): The local path where the file will be saved.
+    """
+    try:
+        github = GitHub()
+        github.logger.info(f"File path {path}")
+        content = github.get_file_content(repository, path)
+        github.logger.info(f"File {path} downloaded")
+
+        file_path = Path(tree_file_path) / path
+        file_path.parent.mkdir(parents=True, exist_ok=True)
+
+        with file_path.open("wb") as file:
+            file.write(content)
+    except Exception as e:
+        github.logger.exception(f"Failed to save file: {e}")
 
 
 def init_github(app: flask.Flask) -> None:
-    app.config["github"] = GitHub(app=app)
+    app.config["github"] = GitHub()

webapp/site_repository.py

@@ -2,7 +2,8 @@
 import re
 import subprocess
 import time
-from typing import Callable, TypedDict
+from collections.abc import Callable
+from typing import TypedDict
 
 from flask import Flask
 from flask_sqlalchemy import SQLAlchemy
@@ -84,7 +85,9 @@ def __exec__(self, command_str: str):
         """
         command = command_str.strip("").split(" ")
         process = subprocess.Popen(
-            command, stdout=subprocess.PIPE, stderr=subprocess.PIPE
+            command,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
         )
         # Wait for the process to finish
         process.wait()
@@ -141,7 +144,8 @@ def setup_site_repository(self):
         # Download files from the repository
         github = self.app.config["github"]
         github.get_repository_tree(
-            repository=self.repository_uri, branch=self.branch
+            repository=self.repository_uri,
+            branch=self.branch,
         )
 
     def repository_exists(self):
@@ -195,7 +199,7 @@ def get_tree_from_disk(self):
         if not folder_exists:
             raise SiteRepositoryError(
                 "Templates folder 'templates' not found for "
-                f"repository {self.repo_path}"
+                f"repository {self.repo_path}",
             )
 
         # Change directory to the templates folder
@@ -244,8 +248,8 @@ def get_tree_from_db(self):
         webpages = (
             self.db.session.execute(
                 select(Webpage).where(
-                    Webpage.project_id == get_project_id(self.repository_uri)
-                )
+                    Webpage.project_id == get_project_id(self.repository_uri),
+                ),
             )
             .scalars()
             .all()
@@ -259,7 +263,7 @@ def get_tree_from_db(self):
             # If the tree is empty, load from the repository
             if not tree.get("children") and not tree.get("parent_id"):
                 self.logger.info(
-                    f"Reloading incomplete tree root {self.repository_uri}."
+                    f"Reloading incomplete tree root {self.repository_uri}.",
                 )
                 tree = self.get_new_tree()
         return tree
@@ -319,22 +323,35 @@ def __create_webpage_for_node__(
         return {**node, **webpage_dict}
 
     def __create_webpages_for_children__(
-        self, db, children, project, owner, parent_id
+        self,
+        db,
+        children,
+        project,
+        owner,
+        parent_id,
     ):
         """
         Recursively create webpages for each child in the tree.
         """
         for child in children:
             # Create a webpage for the root node
             webpage_dict = self.__create_webpage_for_node__(
-                db, child, project, owner, parent_id
+                db,
+                child,
+                project,
+                owner,
+                parent_id,
             )
             # Update the child node with the webpage fields
             child.update(webpage_dict)
             if child.get("children"):
                 # Create webpages for the children recursively
                 self.__create_webpages_for_children__(
-                    db, child["children"], project, owner, webpage_dict["id"]
+                    db,
+                    child["children"],
+                    project,
+                    owner,
+                    webpage_dict["id"],
                 )
 
     def __remove_webpages_to_delete__(self, db, tree):
@@ -343,14 +360,14 @@ def __remove_webpages_to_delete__(self, db, tree):
         self.add_pages_to_list(tree, webpages)
 
         webpages_to_delete = db.session.execute(
-            select(Webpage).where(Webpage.status == WebpageStatus.TO_DELETE)
+            select(Webpage).where(Webpage.status == WebpageStatus.TO_DELETE),
         )
 
         for row in webpages_to_delete:
             page_to_delete = row[0]
             if page_to_delete.name not in webpages:
                 db.session.execute(
-                    delete(Webpage).where(Webpage.id == page_to_delete.id)
+                    delete(Webpage).where(Webpage.id == page_to_delete.id),
                 )
 
     def create_webpages_for_tree(self, db: SQLAlchemy, tree: Tree):
@@ -360,18 +377,28 @@ def create_webpages_for_tree(self, db: SQLAlchemy, tree: Tree):
         self.logger.info(f"Existing tree {tree}")
         # Get the default project and owner for new webpages
         project, _ = get_or_create(
-            db.session, Project, name=self.repository_uri
+            db.session,
+            Project,
+            name=self.repository_uri,
         )
         owner, _ = get_or_create(db.session, User, name="Default")
 
         # Create a webpage for the root node
         webpage_dict = self.__create_webpage_for_node__(
-            db, tree, project, owner, None
+            db,
+            tree,
+            project,
+            owner,
+            None,
         )
 
         # Create webpages for the children recursively
         self.__create_webpages_for_children__(
-            db, webpage_dict["children"], project, owner, webpage_dict["id"]
+            db,
+            webpage_dict["children"],
+            project,
+            owner,
+            webpage_dict["id"],
         )
 
         # Remove pages that don't exist in the repository anymore