Merge pull request #66 from filips123/sentry-and-performance-fixes

Author: filips123

.github/workflows/deploy.yaml

@@ -58,9 +58,6 @@ jobs:
         with:
           node-version: "18"
 
-      - name: Install Sentry CLI
-        run: npm install -g @sentry/cli
-
       - name: Prepare the SSH configuration
         run: |
           mkdir ~/.ssh
@@ -98,6 +95,11 @@ jobs:
           yarn version --no-git-tag-version --new-version "$VERSION"
           yarn install --frozen-lockfile
           yarn build
+        env:
+          SENTRY_UPLOAD_SOURCEMAPS: true
+          SENTRY_ORG: ${{ secrets.SENTRY_ORG }}
+          SENTRY_PROJECT: ${{ secrets.SENTRY_FRONTEND_PROJECT }}
+          SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
 
       - name: Deploy the backend
         run: |
@@ -117,13 +119,12 @@ jobs:
       - name: Reload the server
         run: ssh urnik "sudo systemctl reload apache2.service"
 
-      - name: Create the Sentry release
-        run: |
-          sentry-cli releases new "$VERSION" -p "${{ secrets.SENTRY_BACKEND_PROJECT }}" -p "${{ secrets.SENTRY_FRONTEND_PROJECT }}"
-          sentry-cli releases -p "${{ secrets.SENTRY_FRONTEND_PROJECT }}" files "$VERSION" upload-sourcemaps website/dist/
-          sentry-cli releases set-commits "$VERSION" --auto
-          sentry-cli releases finalize "$VERSION"
-          sentry-cli releases deploys "$VERSION" new -e "${{ secrets.SENTRY_ENVIRONMENT }}"
+      - name: Create a Sentry release
+        uses: getsentry/action-release@v1
         env:
           SENTRY_ORG: ${{ secrets.SENTRY_ORG }}
           SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
+        with:
+          projects: ${{ secrets.SENTRY_BACKEND_PROJECT }} ${{ secrets.SENTRY_FRONTEND_PROJECT }}
+          version: ${{ env.VERSION }}
+          environment: production

API/gimvicurnik/__init__.py

@@ -101,6 +101,7 @@ def configure_sentry(self) -> None:
             from sentry_sdk.integrations.flask import FlaskIntegration
             from sentry_sdk.integrations.sqlalchemy import SqlalchemyIntegration
             from sentry_sdk.integrations.pure_eval import PureEvalIntegration
+            from sentry_sdk.scrubber import EventScrubber, DEFAULT_DENYLIST
 
             sentry_config = self.config.sentry
 
@@ -112,6 +113,13 @@ def configure_sentry(self) -> None:
             environment = os.environ.get("GIMVICURNIK_ENV", "production")
             release = sentry_config.releasePrefix + version + sentry_config.releaseSuffix
 
+            # Remove IP-related elements from the default denylist
+            if sentry_config.collectIPs:
+                allowlist = {"ip_address", "remote_addr", "x_forwarded_for", "x_real_ip"}
+                denylist = list(set(DEFAULT_DENYLIST) - allowlist)
+            else:
+                denylist = DEFAULT_DENYLIST
+
             # Create custom traces sampler so different traces can be configured separately
             def _sentry_traces_sampler(context: dict[str, Any]) -> float | int | bool:
                 if context["transaction_context"]["op"] == "command":
@@ -136,6 +144,7 @@ def _sentry_profiler_sampler(context: dict[str, Any]) -> float | int | bool:
                 max_breadcrumbs=sentry_config.maxBreadcrumbs,
                 traces_sampler=_sentry_traces_sampler,
                 profiles_sampler=_sentry_profiler_sampler,
+                event_scrubber=EventScrubber(denylist=denylist),
                 integrations=[
                     FlaskIntegration(transaction_style="url"),
                     SqlalchemyIntegration(),

API/gimvicurnik/blueprints/calendar.py

@@ -15,7 +15,7 @@
 if typing.TYPE_CHECKING:
     from typing import Any, Iterator
     from flask import Blueprint, Response
-    from sqlalchemy.orm.query import Query
+    from sqlalchemy.orm.query import RowReturningQuery
     from ..config import Config, ConfigLessonTime
 
 
@@ -66,7 +66,7 @@ def create_school_calendar(
                 span.set_tag("event.time", subject["time"])
                 span.set_data("event.source", subject)
 
-                logger.info(
+                logger.debug(
                     "Preparing iCalendar event",
                     extra={"type": "timetable", "source": subject},
                 )
@@ -123,7 +123,7 @@ def create_school_calendar(
                 span.set_tag("event.time", subject["time"])
                 span.set_data("event.source", subject)
 
-                logger.info(
+                logger.debug(
                     "Preparing iCalendar event",
                     extra={"type": "substitution", "source": subject},
                 )
@@ -183,18 +183,22 @@ def create_school_calendar(
 
 
 @with_span(op="generate")
-def create_schedule_calendar(query: Query[LunchSchedule], name: str, url: str) -> Response:
+def create_schedule_calendar(
+    query: RowReturningQuery[tuple[LunchSchedule, str]],
+    name: str,
+    url: str,
+) -> Response:
     logger = logging.getLogger(__name__)
     calendar = create_calendar(name, url)
 
-    for model in query:
+    for model, classname in query:
         with start_span(op="event") as span:
             span.set_tag("event.type", "lunch-schedule")
             span.set_tag("event.date", model.date)
             span.set_tag("event.time", model.time)
             span.set_data("event.source", model)
 
-            logger.info(
+            logger.debug(
                 "Preparing iCalendar event",
                 extra={"type": "lunch-schedule", "source": model},
             )
@@ -215,7 +219,7 @@ def create_schedule_calendar(query: Query[LunchSchedule], name: str, url: str) -
                     (
                         str(model.date)
                         + str(model.time)
-                        + str(model.class_.name)
+                        + str(classname)
                         + str(model.location)
                         + str(model.notes)
                     ).encode("utf-8")
@@ -281,7 +285,7 @@ def get_substitutions_calendar_for_classes(classes: list[str]) -> Response:
         @bp.route("/calendar/schedules/<list:classes>")
         def get_schedules_calendar_for_classes(classes: list[str]) -> Response:
             return create_schedule_calendar(
-                Session.query(LunchSchedule)
+                Session.query(LunchSchedule, Class.name)
                 .join(Class)
                 .filter(Class.name.in_(classes))
                 .order_by(LunchSchedule.time, LunchSchedule.class_),

API/gimvicurnik/blueprints/schedule.py

@@ -21,14 +21,14 @@ def routes(cls, bp: Blueprint, config: Config) -> None:
         def get_lunch_schedule(date: datetime.date) -> list[dict[str, Any]]:
             return [
                 {
-                    "class": model.class_.name,
-                    "date": model.date.strftime("%Y-%m-%d"),
-                    "time": model.time.strftime("%H:%M"),
-                    "location": model.location,
-                    "notes": model.notes,
+                    "class": model[1],
+                    "date": model[0].date.strftime("%Y-%m-%d"),
+                    "time": model[0].time.strftime("%H:%M"),
+                    "location": model[0].location,
+                    "notes": model[0].notes,
                 }
                 for model in (
-                    Session.query(LunchSchedule)
+                    Session.query(LunchSchedule, Class.name)
                     .join(Class)
                     .filter(LunchSchedule.date == date)
                     .order_by(LunchSchedule.time, LunchSchedule.class_)
@@ -39,14 +39,14 @@ def get_lunch_schedule(date: datetime.date) -> list[dict[str, Any]]:
         def get_lunch_schedule_for_classes(date: datetime.date, classes: list[str]) -> list[dict[str, Any]]:
             return [
                 {
-                    "class": model.class_.name,
-                    "date": model.date.strftime("%Y-%m-%d"),
-                    "time": model.time.strftime("%H:%M"),
-                    "location": model.location,
-                    "notes": model.notes,
+                    "class": model[1],
+                    "date": model[0].date.strftime("%Y-%m-%d"),
+                    "time": model[0].time.strftime("%H:%M"),
+                    "location": model[0].location,
+                    "notes": model[0].notes,
                 }
                 for model in (
-                    Session.query(LunchSchedule)
+                    Session.query(LunchSchedule, Class.name)
                     .join(Class)
                     .filter(LunchSchedule.date == date, Class.name.in_(classes))
                     .order_by(LunchSchedule.time, LunchSchedule.class_)

API/gimvicurnik/database/__init__.py

@@ -100,7 +100,7 @@ def get_lessons(
         query = (
             Session.query(Lesson, Class.name, Teacher.name, Classroom.name)
             .join(Class)
-            .join(Teacher)
+            .join(Teacher, isouter=True)
             .join(Classroom, isouter=True)
             .order_by(Lesson.day, Lesson.time)
         )

API/gimvicurnik/updaters/eclassroom.py

@@ -399,6 +399,7 @@ def _parse_substitutions(self, tables: Tables, effective: date) -> None:
         # Parse tables into substitutions
         for table in tables:
             for row0 in table:
+                # We use different variable name here, otherwise mypy complains
                 row = [column.replace("\n", " ").strip() if column else "" for column in row0]
 
                 # Get parser type
@@ -569,9 +570,12 @@ def _parse_substitutions(self, tables: Tables, effective: date) -> None:
         # Deduplicate substitutions
         substitutions = [dict(subs2) for subs2 in {tuple(subs1.items()) for subs1 in substitutions}]
 
-        # Store substitutions to a database
+        # Remove old substitutions from a database
         self.session.query(Substitution).filter(Substitution.date == effective).delete()
-        self.session.execute(insert(Substitution), substitutions)
+
+        # Store new substitutions to a database
+        if substitutions:
+            self.session.execute(insert(Substitution), substitutions)
 
     def _parse_lunch_schedule(self, tables: Tables, effective: date) -> None:
         """Parse the lunch schedule document."""

website/package.json

@@ -37,6 +37,7 @@
   },
   "devDependencies": {
     "@rushstack/eslint-patch": "^1.2.0",
+    "@sentry/webpack-plugin": "^1.20.1",
     "@types/pulltorefreshjs": "^0.1.5",
     "@typescript-eslint/eslint-plugin": "^5.49.1",
     "@typescript-eslint/parser": "^5.49.1",

website/vue.config.js

@@ -1,5 +1,8 @@
 process.env.VUE_APP_VERSION = require('./package.json').version
 
+const path = require('path')
+
+const SentryWebpackPlugin = require('@sentry/webpack-plugin')
 const PreloadWebpackPlugin = require('@vue/preload-webpack-plugin')
 const { defineConfig } = require('@vue/cli-service')
 
@@ -77,5 +80,41 @@ module.exports = defineConfig({
         return 'script'
       }
     }])
+  },
+
+  configureWebpack: (config) => {
+    config.output.devtoolFallbackModuleFilenameTemplate = 'webpack:///[resource-path]?[hash]'
+
+    config.output.devtoolModuleFilenameTemplate = info => {
+      const isVue = info.resourcePath.match(/\.vue$/)
+      const isScript = info.query.match(/type=script/)
+      const hasModuleId = info.moduleId !== ''
+
+      const resourcePath = path.normalize(info.resourcePath).replaceAll('\\', '/')
+
+      if (isVue && (!isScript || hasModuleId)) {
+        // Detect generated files, filter as webpack-generated
+        return `webpack-generated:///${resourcePath}?${info.hash}`
+      } else {
+        // If not generated, filter as webpack
+        return `webpack:///${resourcePath}`
+      }
+    }
+
+    // Upload sourcemaps to Sentry if enabled
+    if (process.env.SENTRY_UPLOAD_SOURCEMAPS === 'true') {
+      const releasePrefix = process.env.VUE_APP_SENTRY_RELEASE_PREFIX || ''
+      const releaseSuffix = process.env.VUE_APP_SENTRY_RELEASE_SUFFIX || ''
+      const releaseVersion = releasePrefix + process.env.VUE_APP_VERSION + releaseSuffix
+
+      config.plugins.push(new SentryWebpackPlugin({
+        org: process.env.SENTRY_ORG,
+        project: process.env.SENTRY_PROJECT,
+        authToken: process.env.SENTRY_AUTH_TOKEN,
+        release: releaseVersion,
+        include: './dist',
+        finalize: false
+      }))
+    }
   }
 })