Import YouTube history into the rate later list

backend/tournesol/serializers/rate_later.py

@@ -1,4 +1,4 @@
-from django.db import IntegrityError
+from django.db import IntegrityError, transaction
 from django.utils.translation import gettext_lazy as _
 from rest_framework import serializers
 from rest_framework.serializers import ModelSerializer
@@ -16,6 +16,20 @@ class Meta:
         fields = ["created_at"]
 
 
+class RateLaterListSerializer(serializers.ListSerializer):
+    def create(self, validated_data):
+        result = []
+        for attrs in validated_data:
+            try:
+                with transaction.atomic():
+                    result.append(self.child.create(attrs))
+            except ConflictError:
+                # Ignore conflicts in bulk create
+                pass
+
+        return result
+
+
 class RateLaterSerializer(ModelSerializer):
     user = serializers.HiddenField(default=serializers.CurrentUserDefault())
     entity = RelatedEntitySerializer()
@@ -46,6 +60,7 @@ class Meta:
             "entity_contexts",
             "rate_later_metadata",
         ]
+        list_serializer_class = RateLaterListSerializer
 
     def create(self, validated_data):
         entity_id = validated_data.pop("entity")["pk"]

backend/tournesol/tests/test_api_rate_later.py

@@ -14,6 +14,7 @@
 from tournesol.tests.factories.entity_poll_rating import EntityPollRatingFactory
 from tournesol.tests.factories.poll import PollFactory
 from tournesol.tests.factories.ratings import ContributorRatingFactory
+from tournesol.utils.constants import RATE_LATER_BULK_MAX_SIZE
 
 
 class RateLaterCommonMixinTestCase:
@@ -35,6 +36,7 @@ def setUp(self) -> None:
 
         self.poll = PollFactory()
         self.rate_later_base_url = f"/users/me/rate_later/{self.poll.name}/"
+        self.rate_later_bulk_base_url = f"/users/me/rate_later/{self.poll.name}/_bulk_create"
 
         self.entity_in_ratelater = VideoFactory()
 
@@ -273,6 +275,177 @@ def test_auth_404_create_invalid_poll(self) -> None:
         self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND)
 
 
+class RateLaterBulkCreateTestCase(RateLaterCommonMixinTestCase, TestCase):
+    """
+    TestCase of the `RateLaterBulkCreate` API.
+
+    The `RateLaterBulkCreate` API provides an endpoint to add multiple entities to the rate-later list.
+    """
+
+    _other_uid_not_in_db = "yt:n-oujbO9fdQ"
+
+    def test_anon_401_create(self) -> None:
+        """
+        An anonymous user cannot add entities to its rate-later list, even if
+        the poll exists.
+        """
+        data = [
+            {"entity": {"uid": self._uid_not_in_db}},
+            {"entity": {"uid": self._other_uid_not_in_db}},
+        ]
+        response = self.client.post(self.rate_later_bulk_base_url, data, format="json")
+        self.assertEqual(response.status_code, status.HTTP_401_UNAUTHORIZED)
+
+    def test_anon_401_create_invalid_poll(self) -> None:
+        """
+        An anonymous user cannot add entities to its rate-later list, even if
+        the poll doesn't exist.
+        """
+        data = [
+            {"entity": {"uid": self._uid_not_in_db}},
+            {"entity": {"uid": self._other_uid_not_in_db}},
+        ]
+        response = self.client.post(
+            f"/users/me/rate_later/{self._invalid_poll_name}/_bulk_create", data, format="json"
+        )
+        self.assertEqual(response.status_code, status.HTTP_401_UNAUTHORIZED)
+
+    @override_settings(YOUTUBE_API_KEY=None)
+    def test_auth_201_create(self) -> None:
+        """
+        An authenticated user can add entities to its rate-later list from a
+        specific poll, even if the entity doesn't exist in the database yet.
+        """
+        # A second poll ensures the create operation is poll specific.
+        other_poll = PollFactory()
+        initial_nbr = RateLater.objects.filter(poll=self.poll, user=self.user).count()
+
+        self.client.force_authenticate(self.user)
+        data = [
+            {"entity": {"uid": self._uid_not_in_db}},
+            {"entity": {"uid": self._other_uid_not_in_db}},
+        ]
+        response = self.client.post(self.rate_later_bulk_base_url, data, format="json")
+
+        self.assertEqual(response.status_code, status.HTTP_201_CREATED)
+        self.assertListEqual(
+            response.data,
+            [
+                {
+                    "entity": {
+                        "uid": self._uid_not_in_db,
+                        "type": "video",
+                        "metadata": ANY,
+                    },
+                    "entity_contexts": [],
+                    "collective_rating": None,
+                    "individual_rating": None,
+                    "rate_later_metadata": {"created_at": ANY},
+                },
+                {
+                    "entity": {
+                        "uid": self._other_uid_not_in_db,
+                        "type": "video",
+                        "metadata": ANY,
+                    },
+                    "entity_contexts": [],
+                    "collective_rating": None,
+                    "individual_rating": None,
+                    "rate_later_metadata": {"created_at": ANY},
+                },
+            ],
+        )
+
+        self.assertEqual(
+            RateLater.objects.filter(poll=self.poll, user=self.user).count(),
+            initial_nbr + 2,
+        )
+
+        # Rate-later items are saved per poll.
+        self.assertEqual(RateLater.objects.filter(poll=other_poll, user=self.user).count(), 0)
+
+    @override_settings(YOUTUBE_API_KEY=None)
+    def test_auth_201_create_two_times(self) -> None:
+        """
+        An authenticated user can request to add two times the same entity to a
+        rate-later list of a specific poll. In this case the duplicates are ignored.
+        """
+        other_poll = PollFactory()
+        initial_nbr = RateLater.objects.filter(poll=self.poll, user=self.user).count()
+
+        self.client.force_authenticate(self.user)
+        data = [
+            {"entity": {"uid": self.entity_in_ratelater.uid}},
+            {"entity": {"uid": self._uid_not_in_db}},
+        ]
+
+        response = self.client.post(self.rate_later_bulk_base_url, data, format="json")
+        self.assertEqual(response.status_code, status.HTTP_201_CREATED)
+
+        self.assertListEqual(
+            response.data,
+            [
+                {
+                    "entity": {
+                        "uid": self._uid_not_in_db,
+                        "type": "video",
+                        "metadata": ANY,
+                    },
+                    "entity_contexts": [],
+                    "collective_rating": None,
+                    "individual_rating": None,
+                    "rate_later_metadata": {"created_at": ANY},
+                },
+            ],
+        )
+        self.assertEqual(
+            RateLater.objects.filter(poll=self.poll, user=self.user).count(),
+            initial_nbr + 1,
+        )
+
+    @override_settings(YOUTUBE_API_KEY=None)
+    def test_create_size_limit(self) -> None:
+        """
+        An authenticated user can only add a limited number of entities.
+        """
+        self.client.force_authenticate(self.user)
+
+        limit = RATE_LATER_BULK_MAX_SIZE
+
+        initial_nbr = RateLater.objects.filter(poll=self.poll, user=self.user).count()
+        data = [{"entity": {"uid": "yt:entity1-{:03d}".format(n)}} for n in range(limit)]
+        response = self.client.post(self.rate_later_bulk_base_url, data, format="json")
+        self.assertEqual(response.status_code, status.HTTP_201_CREATED)
+        self.assertEqual(
+            RateLater.objects.filter(poll=self.poll, user=self.user).count(),
+            initial_nbr + 20,
+        )
+
+        initial_nbr = RateLater.objects.filter(poll=self.poll, user=self.user).count()
+        data = [{"entity": {"uid": "yt:entity2-{:03d}".format(n)}} for n in range(limit + 1)]
+        response = self.client.post(self.rate_later_bulk_base_url, data, format="json")
+        self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
+        self.assertEqual(
+            RateLater.objects.filter(poll=self.poll, user=self.user).count(),
+            initial_nbr,
+        )
+
+    def test_auth_404_create_invalid_poll(self) -> None:
+        """
+        An authenticated user cannot add entities in a rate-later list from a
+        non-existing poll.
+        """
+        self.client.force_authenticate(self.user)
+        data = [
+            {"entity": {"uid": self._uid_not_in_db}},
+            {"entity": {"uid": self._other_uid_not_in_db}},
+        ]
+        response = self.client.post(
+            f"/users/me/rate_later/{self._invalid_poll_name}/_bulk_create", data, format="json"
+        )
+        self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND)
+
+
 class RateLaterDetailTestCase(RateLaterCommonMixinTestCase, TestCase):
     """
     TestCase of the `RateLaterDetail` API.

backend/tournesol/urls.py

@@ -38,7 +38,7 @@
 )
 from .views.previews.recommendations import get_preview_recommendations_redirect_params
 from .views.proof import ProofView
-from .views.rate_later import RateLaterDetail, RateLaterList
+from .views.rate_later import RateLaterBulkCreate, RateLaterDetail, RateLaterList
 from .views.ratings import (
     ContributorRatingDetail,
     ContributorRatingList,
@@ -113,6 +113,11 @@
         RateLaterList.as_view(),
         name="usersme_ratelater_list",
     ),
+    path(
+        "users/me/rate_later/<str:poll_name>/_bulk_create",
+        RateLaterBulkCreate.as_view(),
+        name="usersme_ratelater_bulk_create",
+    ),
     path(
         "users/me/rate_later/<str:poll_name>/<str:uid>/",
         RateLaterDetail.as_view(),

backend/tournesol/utils/constants.py

@@ -17,3 +17,6 @@
 # Default weight for a criteria in the recommendations
 # FIXME: the default weight used by the front end is 50, not 10
 CRITERIA_DEFAULT_WEIGHT = 10
+
+# Maximum number of "rate later" entries that can be created in a bulk create request
+RATE_LATER_BULK_MAX_SIZE = 20

backend/tournesol/views/rate_later.py

@@ -9,6 +9,7 @@
 
 from tournesol.models import Entity, RateLater
 from tournesol.serializers.rate_later import RateLaterSerializer
+from tournesol.utils.constants import RATE_LATER_BULK_MAX_SIZE
 from tournesol.views.mixins.poll import PollScopedViewMixin
 
 
@@ -64,6 +65,36 @@ def perform_create(self, serializer):
         self.prefetch_entity(rate_later)
 
 
+@extend_schema_view(
+    post=extend_schema(
+        description="Add a multiple new entities to the logged user's rate-later list,"
+        " for a given poll.",
+        request=RateLaterSerializer(many=True),
+        responses={
+            201: RateLaterSerializer,
+        },
+    ),
+)
+class RateLaterBulkCreate(RateLaterQuerysetMixin, generics.CreateAPIView):
+    """
+    Create multiple rate-later entries at once.
+    Accepts an array of entities to be added to the rate-later list.
+    """
+
+    permission_classes = [IsAuthenticated]
+    serializer_class = RateLaterSerializer
+
+    def get_serializer(self, *args, **kwargs):
+        kwargs["many"] = True
+        kwargs["max_length"] = RATE_LATER_BULK_MAX_SIZE
+        return super().get_serializer(*args, **kwargs)
+
+    def perform_create(self, serializer):
+        rate_later_instances = serializer.save()
+        for rate_later in rate_later_instances:
+            self.prefetch_entity(rate_later)
+
+
 @extend_schema_view(
     get=extend_schema(description="Get an entity from the logged user's rate-later list."),
     delete=extend_schema(description="Delete an entity from the logged user's rate-later list."),

browser-extension/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tournesol-extension",
-  "version": "3.7.0",
+  "version": "3.8.0",
   "license": "AGPL-3.0-or-later",
   "type": "module",
   "scripts": {

browser-extension/prepareExtension.js

@@ -119,10 +119,17 @@ const manifest = {
       run_at: 'document_end',
       all_frames: true,
     },
+    {
+      matches: ['https://*.youtube.com/feed/history'],
+      js: ['rateLaterHistory.js'],
+      css: ['rateLaterHistory.css'],
+      run_at: 'document_end',
+      all_frames: true,
+    },
     {
       matches: selectValue(env, {
         production: ['https://tournesol.app/*'],
-        'dev-env': ['http://localhost:3000/*'],
+        'dev-env': ['http://localhost/*'],
       }),
       js: ['fetchTournesolToken.js'],
       run_at: 'document_end',

browser-extension/src/_locales/en/messages.json

@@ -14,6 +14,9 @@
   "menuPreferences": {
     "message": "Preferences"
   },
+  "menuRateLaterHistory": {
+    "message": "Import YouTube watch history"
+  },
   "recommendedByTournesol": {
     "message": "Recommended by Tournesol",
     "description": "Title of the main Tournesol container."
@@ -149,5 +152,33 @@
   "watchOnTournesolButtonLabel": {
     "message": "Watch on Tournesol",
     "description": "Label of the button displayed on video pages."
+  },
+  "rateLaterHistoryImportButton": {
+    "message": "Import history to Tournesol",
+    "description": "Label of the button displayed on the YouTube history page to import the history into your 'rate later' list."
+  },
+  "rateLaterHistoryStatusBoxTitle": {
+    "message": "Videos are being imported, please wait.",
+    "description": "Title of the the rate later history import status box."
+  },
+  "rateLaterHistoryStatusBoxMessage": {
+    "message": "Press Stop when the visible videos count doesn't increase anymore or when you think enough videos have been imported.",
+    "description": "Instructions displayed on the rate later history import status box."
+  },
+  "rateLaterHistoryVideoCount": {
+    "message": "Videos visible in history:",
+    "description": "Label showing the count of videos found in the YouTube history during import."
+  },
+  "rateLaterHistoryAddedCount": {
+    "message": "Videos added to rate later:",
+    "description": "Label showing the count of videos successfully added to the rate later list."
+  },
+  "rateLaterHistoryFailureCount": {
+    "message": "Failures:",
+    "description": "Label showing the count of videos that failed to be added to the rate later list."
+  },
+  "rateLaterHistoryStopButtonLabel": {
+    "message": "Stop",
+    "description": "Label of the button to stop the YouTube history import process."
   }
 }