Merge pull request #73 from filips123/performance-improvements

Author: filips123

.github/workflows/api.yaml

@@ -5,6 +5,8 @@ on:
     paths:
       - .github/workflows/api.yaml
       - API/**
+    branches:
+      - main
   pull_request:
     paths:
       - .github/workflows/api.yaml
@@ -28,12 +30,12 @@ jobs:
         uses: actions/cache@v3
         with:
           path: |
+            ~/.cache/black
             ~/.cache/pip
             ~/.cache/pypoetry
             ~/.local/share/pypoetry
             ~/.local/bin/poetry
-            ./.flakeheaven_cache
-            ./.mypy_cache
+            ./.ruff_cache
           key: ${{ runner.os }}-poetry-lint-${{ hashFiles('**/poetry.lock') }}
           restore-keys: ${{ runner.os }}-poetry-lint-
 
@@ -51,8 +53,11 @@ jobs:
       - name: Install dependencies
         run: poetry install --extras sentry
 
-      - name: Lint the project
-        run: flakeheaven lint
+      - name: Lint the project with ruff
+        run: ruff gimvicurnik
+
+      - name: Lint the project with black
+        run: black gimvicurnik --check
 
   typecheck:
     name: Typechecking
@@ -70,7 +75,6 @@ jobs:
             ~/.cache/pypoetry
             ~/.local/share/pypoetry
             ~/.local/bin/poetry
-            ./.flakeheaven_cache
             ./.mypy_cache
           key: ${{ runner.os }}-poetry-typecheck-${{ hashFiles('**/poetry.lock') }}
           restore-keys: ${{ runner.os }}-poetry-typecheck-

.github/workflows/deploy.yaml

@@ -28,8 +28,6 @@ jobs:
             ~/.cache/pypoetry
             ~/.local/share/pypoetry
             ~/.local/bin/poetry
-            ./.flakeheaven_cache
-            ./.mypy_cache
           key: ${{ runner.os }}-poetry-deploy-${{ hashFiles('**/poetry.lock') }}
           restore-keys: ${{ runner.os }}-poetry-
 

.github/workflows/website.yaml

@@ -5,6 +5,8 @@ on:
     paths:
       - .github/workflows/website.yaml
       - website/**
+    branches:
+      - main
   pull_request:
     paths:
       - .github/workflows/website.yaml

.gitignore

@@ -83,6 +83,7 @@ tmp/
 .node_repl_history
 .npm
 .nyc_output
+.ruff_cache
 .yarn-integrity
 *.tmp
 

API/README.md

@@ -57,10 +57,10 @@ You can retrieve all API routes using the `gimvicurnik routes` commands. The off
 
 ## Contributing
 
-The API uses FlakeHeaven, Black and mypy for linting the code. They are included in project's development dependencies.
+The API uses ruff, black and mypy for linting the code. They are included in project's development dependencies.
 
 Please make sure that your changes are formatted correctly according to the code style:
 
-* Linting: `flakeheaven lint`
+* Linting: `ruff gimvicurnik`
 * Typechecking: `mypy gimvicurnik`
 * Formatting: `black gimvicurnik`

API/gimvicurnik/blueprints/calendar.py

@@ -13,7 +13,8 @@
 from ..utils.sentry import start_span, with_span
 
 if typing.TYPE_CHECKING:
-    from typing import Any, Iterator
+    from typing import Any
+    from collections.abc import Iterator
     from flask import Blueprint, Response
     from sqlalchemy.orm.query import RowReturningQuery
     from ..config import Config, ConfigLessonTime

API/gimvicurnik/database/__init__.py

@@ -1,8 +1,9 @@
 from __future__ import annotations
 
 import enum
+from collections.abc import Iterator
 from datetime import date as date_, datetime, time as time_
-from typing import Annotated, Any, Iterator
+from typing import Annotated, Any
 
 from sqlalchemy import (
     Enum,
@@ -187,27 +188,20 @@ def get_empty(cls) -> Iterator[dict[str, Any]]:
         if times[0] is None or times[1] is None:
             return []
 
-        classrooms = list(Session.query(Classroom).order_by(Classroom.name))
-        lessons = list(Session.query(Lesson).join(Classroom))
+        classrooms = Session.query(Classroom.name).order_by(Classroom.name).distinct().all()
+        occupied = set(Session.query(Lesson.day, Lesson.time, Classroom.name).join(Classroom).distinct())
 
         for day in range(days[0], days[1] + 1):
             for time in range(times[0], times[1] + 1):
-                for classroom in classrooms:
-                    is_classroom_empty = True
-
-                    for lesson in lessons:
-                        if lesson.day == day and lesson.time == time and lesson.classroom == classroom:
-                            is_classroom_empty = False
-                            break
-
-                    if is_classroom_empty:
+                for (classroom,) in classrooms:
+                    if (day, time, classroom) not in occupied:
                         yield {
                             "day": day,
                             "time": time,
                             "subject": None,
                             "class": None,
                             "teacher": None,
-                            "classroom": classroom.name,
+                            "classroom": classroom,
                         }
 
 

API/gimvicurnik/updaters/base.py

@@ -12,7 +12,8 @@
 from ..utils.sentry import sentry_available, with_span
 
 if typing.TYPE_CHECKING:
-    from typing import ClassVar, Iterator
+    from typing import ClassVar
+    from collections.abc import Iterator
     from logging import Logger
     from sqlalchemy.orm import Session
     from sentry_sdk.tracing import Span
@@ -79,6 +80,15 @@ class BaseMultiUpdater(ABC):
     Must be set by subclasses before running `update`.
     """
 
+    requests: requests.Session
+    """
+    A requests session that the updater should use.
+    Will be set automatically by the base updater.
+    """
+
+    def __init__(self) -> None:
+        self.requests = requests.Session()
+
     def update(self) -> None:
         """Get all available documents and update them."""
 
@@ -347,14 +357,14 @@ def download_document(self, document: DocumentInfo) -> tuple[bytes, str]:
         """Download a document and return its content and hash"""
 
         try:
-            response = requests.get(self.tokenize_url(document.url))
+            response = self.requests.get(self.tokenize_url(document.url))
             response.raise_for_status()
 
             content = response.content
             sha = sha256(content).hexdigest()
             return content, sha
 
-        except IOError as error:
+        except OSError as error:
             raise self.error(f"Error while downloading a {document.type.value} document") from error
 
     # noinspection PyMethodMayBeStatic

API/gimvicurnik/updaters/eclassroom.py

@@ -10,7 +10,6 @@
 from datetime import date, datetime, timezone
 from itertools import product
 
-import requests
 from mammoth import convert_to_html  # type: ignore
 from sqlalchemy import insert
 
@@ -22,7 +21,8 @@
 from ..utils.sentry import with_span
 
 if typing.TYPE_CHECKING:
-    from typing import Any, Iterator
+    from typing import Any
+    from collections.abc import Iterator
     from mammoth.documents import Image  # type: ignore
     from sqlalchemy.orm import Session
     from sentry_sdk.tracing import Span
@@ -49,6 +49,8 @@ def __init__(self, config: ConfigSourcesEClassroom, session: Session) -> None:
         self.config = config
         self.session = session
 
+        super().__init__()
+
     def get_documents(self) -> Iterator[DocumentInfo]:
         """Get all documents from the e-classroom."""
 
@@ -70,9 +72,9 @@ def _mark_course_viewed(self) -> None:
         }
 
         try:
-            response = requests.post(self.config.webserviceUrl, params=params, data=data)
+            response = self.requests.post(self.config.webserviceUrl, params=params, data=data)
             response.raise_for_status()
-        except (IOError, ValueError) as error:
+        except (OSError, ValueError) as error:
             raise ClassroomApiError("Error while accessing e-classroom API") from error
 
     def _get_internal_urls(self) -> Iterator[DocumentInfo]:
@@ -86,11 +88,11 @@ def _get_internal_urls(self) -> Iterator[DocumentInfo]:
         }
 
         try:
-            response = requests.post(self.config.webserviceUrl, params=params, data=data)
+            response = self.requests.post(self.config.webserviceUrl, params=params, data=data)
             contents = response.json()
 
             response.raise_for_status()
-        except (IOError, ValueError) as error:
+        except (OSError, ValueError) as error:
             raise ClassroomApiError("Error while accessing e-classroom API") from error
 
         # Handle API errors
@@ -129,11 +131,11 @@ def _get_external_urls(self) -> Iterator[DocumentInfo]:
         }
 
         try:
-            response = requests.post(self.config.webserviceUrl, params=params, data=data)
+            response = self.requests.post(self.config.webserviceUrl, params=params, data=data)
             contents = response.json()
 
             response.raise_for_status()
-        except (IOError, ValueError) as error:
+        except (OSError, ValueError) as error:
             raise ClassroomApiError("Error while accessing e-classroom API") from error
 
         # Handle API errors

API/gimvicurnik/updaters/menu.py

@@ -7,18 +7,18 @@
 import tempfile
 import typing
 
-import requests
 from bs4 import BeautifulSoup, ParserRejectedMarkup
 from openpyxl import load_workbook
 
 from .base import BaseMultiUpdater, DocumentInfo
 from ..database import DocumentType, LunchMenu, SnackMenu
 from ..errors import MenuApiError, MenuDateError, MenuFormatError
-from ..utils.sentry import with_span
 from ..utils.pdf import extract_tables
+from ..utils.sentry import with_span
 
 if typing.TYPE_CHECKING:
-    from typing import Iterator, Any
+    from typing import Any
+    from collections.abc import Iterator
     from sqlalchemy.orm import Session
     from sentry_sdk.tracing import Span
     from ..config import ConfigSourcesMenu
@@ -33,15 +33,17 @@ def __init__(self, config: ConfigSourcesMenu, session: Session):
         self.config = config
         self.session = session
 
+        super().__init__()
+
     def get_documents(self) -> Iterator[DocumentInfo]:
         """Download and parse the website to retrieve all menu URLs."""
 
         try:
-            response = requests.get(self.config.url)
+            response = self.requests.get(self.config.url)
             response.raise_for_status()
 
             soup = with_span(op="soup")(BeautifulSoup)(response.text, features="lxml")
-        except (IOError, ParserRejectedMarkup) as error:
+        except (OSError, ParserRejectedMarkup) as error:
             raise MenuApiError("Error while downloading or parsing menu index") from error
 
         menus = soup.find_all("li", {"class": "jedilnik"})

API/gimvicurnik/updaters/timetable.py

@@ -56,7 +56,7 @@ def _download(self) -> tuple[str, str]:
             response = requests.get(self.config.url)
             content = response.content
             response.raise_for_status()
-        except IOError as error:
+        except OSError as error:
             raise TimetableApiError("Error while downloading the timetable") from error
 
         return content.decode("utf8"), sha256(content).hexdigest()

API/gimvicurnik/utils/errors.py

@@ -14,7 +14,7 @@
 from traceback import TracebackException as _TracebackException
 
 if typing.TYPE_CHECKING:
-    from typing import Iterable, Iterator
+    from collections.abc import Iterable, Iterator
 
 
 class TracebackException(_TracebackException):

API/gimvicurnik/utils/sentry.py

@@ -5,7 +5,8 @@
 
 if typing.TYPE_CHECKING:
     from types import TracebackType
-    from typing import Any, Callable, TypeVar, ParamSpec
+    from typing import Any, TypeVar, ParamSpec
+    from collections.abc import Callable
 
     TP = ParamSpec("TP")
     TR = TypeVar("TR")