feat: Upload captions (#483)

* feat: Upload captions

* iterate

- Trying to fetch directly from the API is too dangerous at this point, since the API is blocked
- I've chosen to iterate on a GraphQL subtitles upload API
- Trying the GraphQL mutation with Absinthe, I got a "No query document supplied" error
	-
	  ```gql
	  mutation SetVideoCaptions($captions: Upload!) {
	    setVideoCaptions(videoId: 1, captions: $captions) {
	      captions {
	       text
	      }
	    }
	  }
	  ```
- Decided to upgrade Absinthe to see how that goes
- [absinthe_ecto](https://github.com/absinthe-graphql/absinthe_ecto) is deprecated, so I'm looking to move to [dataloader](https://github.com/absinthe-graphql/dataloader)
- Updated the dependencies, now looking to migrate the actual `assoc` calls

* misc fixes

Author: Betree

apps/cf/lib/actions/action_creator.ex

@@ -10,6 +10,7 @@ defmodule CF.Actions.ActionCreator do
   alias DB.Schema.Speaker
   alias DB.Schema.Statement
   alias DB.Schema.Comment
+  alias DB.Schema.VideoCaption
   alias CF.Actions.ReputationChange
 
   # Create
@@ -175,6 +176,19 @@ defmodule CF.Actions.ActionCreator do
     )
   end
 
+  def action_upload_video_captions(user_id, video_id, caption = %VideoCaption{}) do
+    action(
+      user_id,
+      :video_caption,
+      :upload,
+      video_id: video_id,
+      changes: %{
+        "format" => caption.format,
+        "parsed" => caption.parsed
+      }
+    )
+  end
+
   def action_revert_vote(user_id, video_id, vote_type, comment = %Comment{})
       when vote_type in [:revert_vote_up, :revert_vote_down, :revert_self_vote] do
     action(

apps/cf/lib/videos/captions_fetcher_youtube.ex

@@ -69,30 +69,7 @@ defmodule CF.Videos.CaptionsFetcherYoutube do
   end
 
   defp process_transcript(transcript) do
-    transcript
-    |> SweetXml.xpath(
-      ~x"//transcript/text"l,
-      text: ~x"./text()"s |> transform_by(&clean_text/1),
-      start: ~x"./@start"s |> transform_by(&parse_float/1),
-      duration: ~x"./@dur"os |> transform_by(&parse_float/1)
-    )
-    |> Enum.filter(fn %{text: text, start: start} ->
-      start != nil and text != nil and text != ""
-    end)
-  end
-
-  defp clean_text(text) do
-    text
-    |> String.replace("&", "&")
-    |> HtmlEntities.decode()
-    |> String.trim()
-  end
-
-  defp parse_float(val) do
-    case Float.parse(val) do
-      {num, _} -> num
-      _ -> nil
-    end
+    CF.Videos.CaptionsSrv1Parser.parse_file(transcript)
   end
 
   # Below is an implementation using the official YouTube API, but it requires OAuth2 authentication.

apps/cf/lib/videos/captions_srv1_parser.ex

@@ -0,0 +1,37 @@
+defmodule CF.Videos.CaptionsSrv1Parser do
+  @moduledoc """
+  A captions parser for the srv1 format.
+  """
+
+  require Logger
+  import SweetXml
+
+  def parse_file(content) do
+    content
+    |> SweetXml.xpath(
+      ~x"//transcript/text"l,
+      text: ~x"./text()"s |> transform_by(&clean_text/1),
+      start: ~x"./@start"s |> transform_by(&parse_float/1),
+      duration: ~x"./@dur"os |> transform_by(&parse_float/1)
+    )
+    |> Enum.filter(fn %{text: text, start: start} ->
+      # Filter out text in brackets, like "[Music]"
+      start != nil and text != nil and text != "" and
+        String.match?(text, ~r/^\[.*\]$/) == false
+    end)
+  end
+
+  defp clean_text(text) do
+    text
+    |> String.replace("&", "&")
+    |> HtmlEntities.decode()
+    |> String.trim()
+  end
+
+  defp parse_float(val) do
+    case Float.parse(val) do
+      {num, _} -> num
+      _ -> nil
+    end
+  end
+end

apps/cf/test/comments/comments_test.exs

@@ -141,7 +141,7 @@ defmodule CF.Comments.CommentsTest do
         Comments.delete_comment(random_user, comment.statement.video_id, comment)
       end
 
-      refute_deleted(comment)
+      assert_not_deleted(comment)
 
       Comments.delete_comment(comment.user, comment.statement.video_id, comment)
       assert_deleted(comment)
@@ -168,7 +168,7 @@ defmodule CF.Comments.CommentsTest do
         Comments.delete_comment(comment.user, comment.statement.video_id, comment)
       end
 
-      refute_deleted(comment)
+      assert_not_deleted(comment)
     end
 
     test "but an admin can" do

apps/cf/test/support/test_utils.ex

@@ -36,7 +36,7 @@ defmodule CF.TestUtils do
     end
   end
 
-  def refute_deleted(%Comment{id: id}) do
+  def assert_not_deleted(%Comment{id: id}) do
     {comment, _} = get_comment_and_actions(id)
     assert comment != nil
 

apps/cf_graphql/lib/endpoint.ex

@@ -13,9 +13,9 @@ defmodule CF.GraphQLWeb.Endpoint do
 
   plug(
     Plug.Parsers,
-    parsers: [:urlencoded, :multipart, :json],
+    parsers: [:urlencoded, :multipart, :json, Absinthe.Plug.Parser],
     pass: ["*/*"],
-    json_decoder: Poison
+    json_decoder: Jason
   )
 
   plug(Plug.MethodOverride)

apps/cf_graphql/lib/resolvers/videos.ex

@@ -8,6 +8,7 @@ defmodule CF.Graphql.Resolvers.Videos do
   import Ecto.Query
   import Absinthe.Resolution.Helpers, only: [batch: 3]
 
+  alias Ecto.Multi
   alias DB.Repo
   alias DB.Schema.Video
   alias DB.Schema.VideoCaption
@@ -137,4 +138,48 @@ defmodule CF.Graphql.Resolvers.Videos do
         {:error, "Failed to update video"}
     end
   end
+
+  def set_captions(
+        _root,
+        %{video_id: video_id, captions: %{path: path, content_type: content_type}},
+        %{
+          context: %{user: user}
+        }
+      ) do
+    cond do
+      content_type != "text/xml" ->
+        {:error, "Unsupported content type: #{content_type}"}
+
+      File.stat!(path).size > 10_000_000 ->
+        {:error, "File must be < 10MB"}
+
+      true ->
+        video = DB.Repo.get!(DB.Schema.Video, video_id)
+        captions_file_content = File.read!(path)
+        parsed = CF.Videos.CaptionsSrv1Parser.parse_file(captions_file_content)
+
+        Multi.new()
+        |> Multi.insert(
+          :caption,
+          VideoCaption.changeset(%VideoCaption{
+            video_id: video.id,
+            raw: captions_file_content,
+            parsed: parsed,
+            format: "xml"
+          })
+        )
+        |> Multi.run(
+          :action,
+          fn _repo, %{caption: caption} ->
+            CF.Actions.ActionCreator.action_upload_video_captions(user.id, video.id, caption)
+            |> DB.Repo.insert!()
+
+            {:ok, caption}
+          end
+        )
+        |> DB.Repo.transaction()
+
+        {:ok, video}
+    end
+  end
 end

apps/cf_graphql/lib/schema/input_objects.ex

@@ -4,7 +4,6 @@ defmodule CF.Graphql.Schema.InputObjects.StatementFilter do
   """
 
   use Absinthe.Schema.Notation
-  use Absinthe.Ecto, repo: DB.Repo
 
   @desc "Props to filter statements on"
   input_object :statement_filter do

apps/cf_graphql/lib/schema/input_objects/statement_filter.ex

@@ -4,7 +4,6 @@ defmodule CF.Graphql.Schema.InputObjects.VideoFilter do
   """
 
   use Absinthe.Schema.Notation
-  use Absinthe.Ecto, repo: DB.Repo
 
   @desc "Props to filter videos on"
   input_object :video_filter do

apps/cf_graphql/lib/schema/input_objects/video_filter.ex

@@ -2,9 +2,42 @@ defmodule CF.Graphql.Schema do
   use Absinthe.Schema
   alias CF.Graphql.Resolvers
   alias CF.Graphql.Schema.Middleware
+  import Absinthe.Resolution.Helpers, only: [dataloader: 1]
+
+  import_types(Absinthe.Plug.Types)
+
+  import_types(CF.Graphql.Schema.Types.{
+    AppInfo,
+    Comment,
+    Notification,
+    Paginated,
+    Source,
+    Speaker,
+    Statement,
+    Statistics,
+    Subscription,
+    UserAction,
+    User,
+    Video,
+    VideoCaption
+  })
+
+  import_types(CF.Graphql.Schema.InputObjects.{
+    VideoFilter,
+    StatementFilter
+  })
+
+  def context(ctx) do
+    loader =
+      Dataloader.new()
+      |> Dataloader.add_source(DB.Repo, Dataloader.Ecto.new(DB.Repo))
+
+    Map.put(ctx, :loader, loader)
+  end
 
-  import_types(CF.Graphql.Schema.Types)
-  import_types(CF.Graphql.Schema.InputObjects)
+  def plugins do
+    [Absinthe.Middleware.Dataloader | Absinthe.Plugin.defaults()]
+  end
 
   # Query API
 
@@ -110,5 +143,15 @@ defmodule CF.Graphql.Schema do
 
       resolve(&Resolvers.Videos.edit/3)
     end
+
+    field :set_video_captions, :video do
+      middleware(Middleware.RequireAuthentication)
+      middleware(Middleware.RequireReputation, 450)
+
+      arg(:video_id, non_null(:id))
+      arg(:captions, non_null(:upload))
+
+      resolve(&Resolvers.Videos.set_captions/3)
+    end
   end
 end

apps/cf_graphql/lib/schema/schema.ex

@@ -4,7 +4,6 @@ defmodule CF.Graphql.Schema.Types.AppInfo do
   """
 
   use Absinthe.Schema.Notation
-  use Absinthe.Ecto, repo: DB.Repo
 
   @desc "Information about the application"
   object :app_info do

apps/cf_graphql/lib/schema/types.ex

@@ -4,7 +4,8 @@ defmodule CF.Graphql.Schema.Types.Comment do
   """
 
   use Absinthe.Schema.Notation
-  use Absinthe.Ecto, repo: DB.Repo
+
+  import Absinthe.Resolution.Helpers, only: [dataloader: 1]
   import CF.Graphql.Schema.Utils
   alias CF.Graphql.Resolvers
 
@@ -13,7 +14,7 @@ defmodule CF.Graphql.Schema.Types.Comment do
     field(:id, non_null(:id))
     @desc "User who made the comment"
     field :user, :user do
-      resolve(assoc(:user))
+      resolve(dataloader(DB.Repo))
       complexity(join_complexity())
     end
 
@@ -31,7 +32,7 @@ defmodule CF.Graphql.Schema.Types.Comment do
 
     @desc "Source of the scomment. If null, a text must be set"
     field :source, :source do
-      resolve(assoc(:source))
+      resolve(dataloader(DB.Repo))
       complexity(join_complexity())
     end
 

apps/cf_graphql/lib/schema/types/app_info.ex

@@ -4,10 +4,9 @@ defmodule CF.Graphql.Schema.Types.Notification do
   """
 
   use Absinthe.Schema.Notation
-  use Absinthe.Ecto, repo: DB.Repo
-  import CF.Graphql.Schema.Utils
 
-  import_types(CF.Graphql.Schema.Types.Paginated)
+  import Absinthe.Resolution.Helpers, only: [dataloader: 1]
+  import CF.Graphql.Schema.Utils
 
   @desc "A user notification"
   object :notification do
@@ -20,7 +19,7 @@ defmodule CF.Graphql.Schema.Types.Notification do
     field(:seen_at, :string)
     @desc "Action the notification is referencing"
     field :action, :user_action do
-      resolve(assoc(:action))
+      resolve(dataloader(DB.Repo))
       complexity(join_complexity())
     end
   end

apps/cf_graphql/lib/schema/types/comment.ex

@@ -4,7 +4,6 @@ defmodule CF.Graphql.Schema.Types.Paginated do
   """
 
   use Absinthe.Schema.Notation
-  use Absinthe.Ecto, repo: DB.Repo
 
   object :paginated do
     field(:page_number, :integer)

apps/cf_graphql/lib/schema/types/notification.ex

@@ -4,7 +4,6 @@ defmodule CF.Graphql.Schema.Types.Source do
   """
 
   use Absinthe.Schema.Notation
-  use Absinthe.Ecto, repo: DB.Repo
 
   @desc "An URL pointing toward a source (article, video, pdf...)"
   object :source do

apps/cf_graphql/lib/schema/types/paginated.ex

@@ -4,7 +4,9 @@ defmodule CF.Graphql.Schema.Types.Speaker do
   """
 
   use Absinthe.Schema.Notation
-  use Absinthe.Ecto, repo: DB.Repo
+
+  import Absinthe.Resolution.Helpers, only: [dataloader: 1]
+  import Absinthe.Resolution.Helpers, only: [dataloader: 1]
   import CF.Graphql.Schema.Utils
   alias CF.Graphql.Resolvers
 
@@ -26,7 +28,7 @@ defmodule CF.Graphql.Schema.Types.Speaker do
     field(:picture, :string, do: resolve(&Resolvers.Speakers.picture/3))
     @desc "List of speaker's videos"
     field :videos, list_of(:video) do
-      resolve(assoc(:videos))
+      resolve(dataloader(DB.Repo))
       complexity(join_complexity())
     end
   end