separate invidious_companion logic + better config.yaml config

Author: unixfox

src/invidious/config.cr

@@ -67,6 +67,28 @@ end
 class Config
   include YAML::Serializable
 
+  module URIArrayConverter
+    def self.to_yaml(values : Array(URI), yaml : YAML::Nodes::Builder)
+      yaml.sequence do
+        values.each { |v| yaml.scalar v.to_s }
+      end
+    end
+
+    def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : Array(URI)
+      if node.is_a?(YAML::Nodes::Sequence)
+        node.map do |child|
+          unless child.is_a?(YAML::Nodes::Scalar)
+            node.raise "Expected scalar, not #{child.class}"
+          end
+
+          URI.parse(child.value)
+        end
+      else
+        node.raise "Expected sequence, not #{node.class}"
+      end
+    end
+  end
+
   # Number of threads to use for crawling videos from channels (for updating subscriptions)
   property channel_threads : Int32 = 1
   # Time interval between two executions of the job that crawls channel videos (subscriptions update).
@@ -152,10 +174,11 @@ class Config
   property po_token : String? = nil
 
   # Invidious companion
-  property invidious_companion : Array(String)? = nil
+  @[YAML::Field(converter: Config::URIArrayConverter)]
+  property invidious_companion : Array(URI) = [] of URI
 
   # Invidious companion API key
-  property invidious_companion_key : String? = nil
+  property invidious_companion_key : String = ""
 
   # Saved cookies in "name1=value1; name2=value2..." format
   @[YAML::Field(converter: Preferences::StringToCookies)]
@@ -228,7 +251,7 @@ class Config
         end
     {% end %}
 
-    if CONFIG.invidious_companion
+    if !CONFIG.invidious_companion.empty?
       # invidious_companion and signature_server can't work together
       if CONFIG.signature_server
         puts "Config: You can not run inv_sig_helper and invidious_companion at the same time."

src/invidious/routes/api/manifest.cr

@@ -1,6 +1,10 @@
 module Invidious::Routes::API::Manifest
   # /api/manifest/dash/id/:id
   def self.get_dash_video_id(env)
+    if !CONFIG.invidious_companion.empty?
+      return error_template(403, "This endpoint is not permitted because it is handled by Invidious companion.")
+    end
+
     env.response.headers.add("Access-Control-Allow-Origin", "*")
     env.response.content_type = "application/dash+xml"
 
@@ -20,10 +24,6 @@ module Invidious::Routes::API::Manifest
       haltf env, status_code: 403
     end
 
-    if local && CONFIG.invidious_companion
-      return env.redirect "#{video.invidious_companion["baseUrl"].as_s}#{env.request.path}?#{env.request.query}"
-    end
-
     if dashmpd = video.dash_manifest_url
       response = YT_POOL.client &.get(URI.parse(dashmpd).request_target)
 

src/invidious/routes/embed.cr

@@ -201,6 +201,13 @@ module Invidious::Routes::Embed
       return env.redirect url
     end
 
+    if (!CONFIG.invidious_companion.empty? && (preferences.local || preferences.quality == "dash"))
+      env.response.headers["Content-Security-Policy"] =
+        env.response.headers["Content-Security-Policy"]
+          .gsub("media-src", "media-src " + video.invidious_companion.not_nil!["baseUrl"].as_s)
+          .gsub("connect-src", "connect-src " + video.invidious_companion.not_nil!["baseUrl"].as_s)
+    end
+
     rendered "embed"
   end
 end

src/invidious/routes/video_playback.cr

@@ -253,6 +253,10 @@ module Invidious::Routes::VideoPlayback
   # YouTube /videoplayback links expire after 6 hours,
   # so we have a mechanism here to redirect to the latest version
   def self.latest_version(env)
+    if !CONFIG.invidious_companion.empty? && CONFIG.disabled?("downloads")
+      return error_template(403, "This endpoint is not permitted because it is handled by Invidious companion.")
+    end
+
     id = env.params.query["id"]?
     itag = env.params.query["itag"]?.try &.to_i?
 
@@ -294,8 +298,8 @@ module Invidious::Routes::VideoPlayback
     end
 
     if local
-      if (CONFIG.invidious_companion)
-        return env.redirect "#{video.invidious_companion["baseUrl"].as_s}#{env.request.path}?#{env.request.query}"
+      if (!CONFIG.invidious_companion.empty?)
+        return env.redirect "#{video.invidious_companion.not_nil!["baseUrl"].as_s}#{env.request.path}?#{env.request.query}"
       end
       url = URI.parse(url).request_target.not_nil!
       url += "&title=#{URI.encode_www_form(title, space_to_plus: false)}" if title

src/invidious/routes/watch.cr

@@ -190,11 +190,11 @@ module Invidious::Routes::Watch
       captions: video.captions
     )
 
-    if (CONFIG.invidious_companion && (preferences.local || preferences.quality == "dash"))
+    if (!CONFIG.invidious_companion.empty? && (preferences.local || preferences.quality == "dash"))
       env.response.headers["Content-Security-Policy"] =
         env.response.headers["Content-Security-Policy"]
-          .gsub("media-src", "media-src " + video.invidious_companion["baseUrl"].as_s)
-          .gsub("connect-src", "connect-src " + video.invidious_companion["baseUrl"].as_s)
+          .gsub("media-src", "media-src " + video.invidious_companion.not_nil!["baseUrl"].as_s)
+          .gsub("connect-src", "connect-src " + video.invidious_companion.not_nil!["baseUrl"].as_s)
     end
 
     templated "watch"

src/invidious/videos.cr

@@ -15,7 +15,7 @@ struct Video
   # NOTE: don't forget to bump this number if any change is made to
   # the `params` structure in videos/parser.cr!!!
   #
-  SCHEMA_VERSION = 2
+  SCHEMA_VERSION = 3
 
   property id : String
 
@@ -192,8 +192,8 @@ struct Video
     }
   end
 
-  def invidious_companion : Hash(String, JSON::Any)
-    info["invidiousCompanion"].try &.as_h
+  def invidious_companion : Hash(String, JSON::Any)?
+    info["invidiousCompanion"]?.try &.as_h
   end
 
   # Macros defining getters/setters for various types of data

src/invidious/videos/parser.cr

@@ -100,7 +100,7 @@ def extract_video_info(video_id : String)
   params = parse_video_info(video_id, player_response)
   params["reason"] = JSON::Any.new(reason) if reason
 
-  if CONFIG.invidious_companion.nil?
+  if !CONFIG.invidious_companion.empty?
     new_player_response = nil
 
     # Don't use Android test suite client if po_token is passed because po_token doesn't
@@ -126,7 +126,7 @@ def extract_video_info(video_id : String)
     end
   end
 
-  {"captions", "playabilityStatus", "playerConfig", "storyboards"}.each do |f|
+  {"captions", "playabilityStatus", "playerConfig", "storyboards", "invidiousCompanion"}.each do |f|
     params[f] = player_response[f] if player_response[f]?
   end
 
@@ -141,10 +141,6 @@ def extract_video_info(video_id : String)
     params["streamingData"] = streaming_data
   end
 
-  if CONFIG.invidious_companion
-    params["invidiousCompanion"] = player_response["invidiousCompanion"]
-  end
-
   # Data structure version, for cache control
   params["version"] = JSON::Any.new(Video::SCHEMA_VERSION.to_i64)
 
@@ -453,12 +449,11 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any
     # Music section
     "music" => JSON.parse(music_list.to_json),
     # Author infos
-    "author"             => JSON::Any.new(author || ""),
-    "ucid"               => JSON::Any.new(ucid || ""),
-    "authorThumbnail"    => JSON::Any.new(author_thumbnail.try &.as_s || ""),
-    "authorVerified"     => JSON::Any.new(author_verified || false),
-    "subCountText"       => JSON::Any.new(subs_text || "-"),
-    "invidiousCompanion" => JSON::Any.new(subs_text),
+    "author"          => JSON::Any.new(author || ""),
+    "ucid"            => JSON::Any.new(ucid || ""),
+    "authorThumbnail" => JSON::Any.new(author_thumbnail.try &.as_s || ""),
+    "authorVerified"  => JSON::Any.new(author_verified || false),
+    "subCountText"    => JSON::Any.new(subs_text || "-"),
   }
 
   return params

src/invidious/views/components/player.ecr

@@ -22,7 +22,7 @@
                audio_streams.each_with_index do |fmt, i|
                 src_url  = "/latest_version?id=#{video.id}&itag=#{fmt["itag"]}"
                 src_url += "&local=true" if params.local
-                src_url = video.invidious_companion["baseUrl"].as_s + src_url if (CONFIG.invidious_companion && params.local)
+                src_url = video.invidious_companion.not_nil!["baseUrl"].as_s + src_url if (!CONFIG.invidious_companion.empty? && params.local)
 
                 bitrate = fmt["bitrate"]
                 mimetype = HTML.escape(fmt["mimeType"].as_s)
@@ -37,7 +37,7 @@
         <% else %>
             <% if params.quality == "dash"
                src_url = "/api/manifest/dash/id/" + video.id + "?local=true&unique_res=1"
-               src_url = video.invidious_companion["baseUrl"].as_s + src_url if (CONFIG.invidious_companion)
+               src_url = video.invidious_companion.not_nil!["baseUrl"].as_s + src_url if (!CONFIG.invidious_companion.empty?)
             %>
                 <source src="<%= src_url %>" type='application/dash+xml' label="dash">
             <% end %>
@@ -48,7 +48,7 @@
             fmt_stream.each_with_index do |fmt, i|
                 src_url  = "/latest_version?id=#{video.id}&itag=#{fmt["itag"]}"
                 src_url += "&local=true" if params.local
-                src_url = video.invidious_companion["baseUrl"].as_s + src_url if (CONFIG.invidious_companion && params.local)
+                src_url = video.invidious_companion.not_nil!["baseUrl"].as_s + src_url if (!CONFIG.invidious_companion.empty? && params.local)
 
                 quality = fmt["quality"]
                 mimetype = HTML.escape(fmt["mimeType"].as_s)

src/invidious/yt_backend/youtube_api.cr

@@ -500,7 +500,11 @@ module YoutubeAPI
       data["params"] = params
     end
 
-    return self._post_json("/youtubei/v1/player", data, client_config)
+    if !CONFIG.invidious_companion.empty?
+      return self._post_invidious_companion("/youtubei/v1/player", data)
+    else
+      return self._post_json("/youtubei/v1/player", data, client_config)
+    end
   end
 
   ####################################################################
@@ -615,19 +619,12 @@ module YoutubeAPI
 
     headers = HTTP::Headers{
       "Content-Type"              => "application/json; charset=UTF-8",
+      "Accept-Encoding"           => "gzip, deflate",
       "x-goog-api-format-version" => "2",
       "x-youtube-client-name"     => client_config.name_proto,
       "x-youtube-client-version"  => client_config.version,
     }
 
-    if CONFIG.invidious_companion && endpoint == "/youtubei/v1/player"
-      headers["Authorization"] = "Bearer " + CONFIG.hmac_key
-    end
-
-    if !CONFIG.invidious_companion
-      headers["Accept-Encoding"] = "gzip, deflate"
-    end
-
     if user_agent = client_config.user_agent
       headers["User-Agent"] = user_agent
     end
@@ -641,37 +638,18 @@ module YoutubeAPI
     LOGGER.trace("YoutubeAPI: ClientConfig: #{client_config}")
     LOGGER.trace("YoutubeAPI: POST data: #{data}")
 
-    invidious_companion_urls = CONFIG.invidious_companion
-
     # Send the POST request
-    if invidious_companion_urls && endpoint == "/youtubei/v1/player"
-      begin
-        invidious_companion_response = make_client(URI.parse(invidious_companion_urls.sample),
-          &.post(endpoint, headers: headers, body: data.to_json))
-        body = invidious_companion_response.body
-        if (invidious_companion_response.status_code != 200)
-          raise Exception.new("status code: " + invidious_companion_response.status_code.to_s + " and body: " + body)
-        end
-      rescue ex
-        raise InfoException.new("Error while communicating with Invidious companion: " + (ex.message || "no extra info found"))
-      end
-    else
-      body = YT_POOL.client() do |client|
-        client.post(url, headers: headers, body: data.to_json) do |response|
-          if response.status_code != 200
-            raise InfoException.new("Error: non 200 status code. Youtube API returned \
-              status code #{response.status_code}. See <a href=\"https://docs.invidious.io/youtube-errors-explained/\"> \
-              https://docs.invidious.io/youtube-errors-explained/</a> for troubleshooting.")
-          end
-          self._decompress(response.body_io, response.headers["Content-Encoding"]?)
+    body = YT_POOL.client() do |client|
+      client.post(url, headers: headers, body: data.to_json) do |response|
+        if response.status_code != 200
+          raise InfoException.new("Error: non 200 status code. Youtube API returned \
+            status code #{response.status_code}. See <a href=\"https://docs.invidious.io/youtube-errors-explained/\"> \
+            https://docs.invidious.io/youtube-errors-explained/</a> for troubleshooting.")
         end
+        self._decompress(response.body_io, response.headers["Content-Encoding"]?)
       end
     end
 
-    if body.nil? && CONFIG.invidious_companion
-      raise InfoException.new("Error while communicating with Invidious companion: no response data.")
-    end
-
     # Convert result to Hash
     initial_data = JSON.parse(body).as_h
 
@@ -692,6 +670,53 @@ module YoutubeAPI
     return initial_data
   end
 
+  ####################################################################
+  # _post_invidious_companion(endpoint, data)
+  #
+  # Internal function that does the actual request to Invidious companion
+  # and handles errors.
+  #
+  # The requested data is an endpoint (URL without the domain part)
+  # and the data as a Hash object.
+  #
+  def _post_invidious_companion(
+    endpoint : String,
+    data : Hash
+  ) : Hash(String, JSON::Any)
+    headers = HTTP::Headers{
+      "Content-Type"    => "application/json; charset=UTF-8",
+      "Accept-Encoding" => "gzip",
+      "Authorization"   => "Bearer " + CONFIG.invidious_companion_key,
+    }
+
+    # Logging
+    LOGGER.debug("Invidious companion: Using endpoint: \"#{endpoint}\"")
+    LOGGER.trace("Invidious companion: POST data: #{data}")
+
+    # Send the POST request
+
+    begin
+      response = make_client(CONFIG.invidious_companion.sample,
+        &.post(endpoint, headers: headers, body: data.to_json))
+      body = self._decompress(response.body_io, response.headers["Content-Encoding"]?)
+      if (response.status_code != 200)
+        raise Exception.new("Error while communicating with Invidious companion: \
+                              status code: " + response.status_code.to_s + " and body: " + body)
+      end
+    rescue ex
+      raise InfoException.new("Error while communicating with Invidious companion: " + (ex.message || "no extra info found"))
+    end
+
+    if body.nil?
+      raise InfoException.new("Error while communicating with Invidious companion: no response data.")
+    end
+
+    # Convert result to Hash
+    initial_data = JSON.parse(body).as_h
+
+    return initial_data
+  end
+
   ####################################################################
   # _decompress(body_io, headers)
   #