Merge pull request #1147 from nextcloud/cache-apps.json

`Last Modified` header support + prod setup optimisation

Author: bigcat88

.github/workflows/yarn.yml

@@ -33,7 +33,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: Install dependencies
-        run: sudo apt install gettext netcat-openbsd xvfb
+        run: sudo apt install gettext netcat-openbsd xvfb redis-server
 
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v4

.gitignore

@@ -38,3 +38,4 @@ newrelic.ini
 /config
 .DS_Store
 /nextcloudappstore/static
+/dev/

CHANGELOG.md

@@ -2,6 +2,17 @@
 
 ## [Unreleased]
 
+## [4.3.2]  - 2023-09-23
+
+### Added
+
+- Support of `Last-Modified`, e.g. `If-Modified-Since:` header for the Rest API endpoints. #1147
+- `redis` python dependency for `django-allauth`.
+
+### Changed
+
+- `/api/v1/apps.json` endpoint now ALWAYS return gzipped data. #1147
+
 ## [4.3.1]  - 2023-09-19
 
 ### Changed

nextcloudappstore/api/v1/urls.py

@@ -1,5 +1,5 @@
 from django.urls import re_path
-from django.views.decorators.http import etag
+from django.views.decorators.http import condition, etag
 
 from nextcloudappstore.api.v1.views import (
     AppRatingView,
@@ -15,8 +15,11 @@
 from nextcloudappstore.core.caching import (
     app_ratings_etag,
     apps_all_etag,
+    apps_all_last_modified,
     apps_etag,
+    apps_last_modified,
     categories_etag,
+    categories_last_modified,
     nextcloud_release_etag,
 )
 from nextcloudappstore.core.versioning import SEMVER_REGEX
@@ -25,8 +28,12 @@
 
 urlpatterns = [
     re_path(r"^platforms\.json$", etag(nextcloud_release_etag)(NextcloudReleaseView.as_view()), name="platforms"),
-    re_path(r"^platform/(?P<version>\d+\.\d+\.\d+)/apps\.json$", etag(apps_etag)(AppView.as_view()), name="app"),
-    re_path(r"^apps\.json$", etag(apps_all_etag)(AppsView.as_view()), name="apps"),
+    re_path(
+        r"^platform/(?P<version>\d+\.\d+\.\d+)/apps\.json$",
+        condition(apps_etag, apps_last_modified)(AppView.as_view()),
+        name="app",
+    ),
+    re_path(r"^apps\.json$", condition(apps_all_etag, apps_all_last_modified)(AppsView.as_view()), name="apps"),
     re_path(r"^apps/releases/?$", AppReleaseView.as_view(), name="app-release-create"),
     re_path(r"^apps/?$", AppRegisterView.as_view(), name="app-register"),
     re_path(r"^apps/(?P<pk>[a-z0-9_]+)/?$", AppView.as_view(), name="app-delete"),
@@ -38,5 +45,9 @@
     ),
     re_path(r"^token/?$", SessionObtainAuthToken.as_view(), name="user-token"),
     re_path(r"^token/new/?$", RegenerateAuthToken.as_view(), name="user-token-new"),
-    re_path(r"^categories.json$", etag(categories_etag)(CategoryView.as_view()), name="category"),
+    re_path(
+        r"^categories.json$",
+        condition(categories_etag, categories_last_modified)(CategoryView.as_view()),
+        name="category",
+    ),
 ]

nextcloudappstore/api/v1/views.py

@@ -2,6 +2,8 @@
 from django.conf import settings
 from django.db import transaction
 from django.http import Http404
+from django.utils.decorators import method_decorator
+from django.views.decorators.gzip import gzip_page
 from pymple import Container
 from requests import HTTPError
 from rest_framework import authentication, parsers, renderers  # type: ignore
@@ -66,6 +68,7 @@ class NextcloudReleaseView(ListAPIView):
     serializer_class = NextcloudReleaseSerializer
 
 
+@method_decorator(gzip_page, name="dispatch")
 class AppsView(ListAPIView):
     queryset = App.objects.prefetch_related(*APP_PREFETCH_LIST).all()
     serializer_class = AppSerializer

nextcloudappstore/core/caching.py

@@ -1,4 +1,5 @@
-from typing import Any, List, Tuple
+import datetime
+from typing import Any, List, Tuple, Union
 
 from django.db.models import Max, QuerySet
 from semantic_version import Version
@@ -24,6 +25,12 @@ def create_etag(pairs: List[Tuple[QuerySet, str]]) -> str:
     return str(max(result, default=""))
 
 
+def get_last_modified(pairs: List[Tuple[QuerySet, str]]) -> Union[datetime.datetime, None]:
+    result = map(lambda p: p[0].aggregate(m=Max(p[1]))["m"], pairs)
+    result = filter(lambda r: r is not None, result)
+    return max(result, default=None)
+
+
 def apps_etag(request: Any, version: str) -> str:
     return create_etag(
         [
@@ -33,6 +40,15 @@ def apps_etag(request: Any, version: str) -> str:
     )
 
 
+def apps_last_modified(request: Any, version: str) -> Union[datetime.datetime, None]:
+    return get_last_modified(
+        [
+            (App.objects.all(), "last_release"),
+            (AppReleaseDeleteLog.objects.all(), "last_modified"),
+        ]
+    )
+
+
 def apps_all_etag(request: Any) -> str:
     return create_etag(
         [
@@ -42,6 +58,15 @@ def apps_all_etag(request: Any) -> str:
     )
 
 
+def apps_all_last_modified(request: Any) -> Union[datetime.datetime, None]:
+    return get_last_modified(
+        [
+            (App.objects.all(), "last_release"),
+            (AppReleaseDeleteLog.objects.all(), "last_modified"),
+        ]
+    )
+
+
 def app_etag(request: Any, id: str) -> str:
     return str(App.objects.get(id=id).last_modified)
 
@@ -54,6 +79,10 @@ def categories_etag(request: Any) -> str:
     return create_etag([(Category.objects.all(), "last_modified")])
 
 
+def categories_last_modified(request: Any) -> Union[datetime.datetime, None]:
+    return get_last_modified([(Category.objects.all(), "last_modified")])
+
+
 def app_ratings_etag(request: Any) -> str:
     return create_etag([(AppRating.objects.all(), "rated_at")])