Towards using only next endpoint for the major data. (Partial solution to #5003)

locales/en-US.json

@@ -501,5 +501,8 @@
     "toggle_theme": "Toggle Theme",
     "carousel_slide": "Slide {{current}} of {{total}}",
     "carousel_skip": "Skip the Carousel",
-    "carousel_go_to": "Go to slide `x`"
+    "carousel_go_to": "Go to slide `x`",
+    "error_from_youtube_unplayable": "Video unplayable due to an error from YouTube:",
+    "error_processing_data_youtube": "Error while processing the data sent by YouTube",
+    "refresh_page": "Refresh the page"
 }

src/invidious/helpers/errors.cr

@@ -73,10 +73,6 @@ def error_template_helper(env : HTTP::Server::Context, status_code : Int32, exce
     </div>
   END_HTML
 
-  # Don't show the usual "next steps" widget. The same options are
-  # proposed above the error message, just worded differently.
-  next_steps = ""
-
   return templated "error"
 end
 
@@ -86,8 +82,13 @@ def error_template_helper(env : HTTP::Server::Context, status_code : Int32, mess
 
   locale = env.get("preferences").as(Preferences).locale
 
-  error_message = translate(locale, message)
-  next_steps = error_redirect_helper(env)
+  error_message = <<-END_HTML
+    <div class="error_message">
+      <h2>#{translate(locale, "error_processing_data_youtube")}</h2>
+      <p>#{translate(locale, message)}</p>
+      #{error_redirect_helper(env)}
+    </div>
+  END_HTML
 
   return templated "error"
 end

src/invidious/videos.cr

@@ -313,7 +313,7 @@ def get_video(id, refresh = true, region = nil, force_refresh = false)
     end
   else
     video = fetch_video(id, region)
-    Invidious::Database::Videos.insert(video) if !region
+    Invidious::Database::Videos.insert(video) if !region && !video.info.dig?("reason")
   end
 
   return video
@@ -326,13 +326,13 @@ end
 def fetch_video(id, region)
   info = extract_video_info(video_id: id)
 
-  if reason = info["reason"]?
+  if info["reason"]? && info["subreason"]?
+    reason = info["reason"].as_s
+    subreason = info["subreason"].as_s
     if reason == "Video unavailable"
-      raise NotFoundException.new(reason.as_s || "")
-    elsif !reason.as_s.starts_with? "Premieres"
-      # dont error when it's a premiere.
-      # we already parsed most of the data and display the premiere date
-      raise InfoException.new(reason.as_s || "")
+      raise NotFoundException.new(reason + ": Video not found" || "")
+    elsif {"Private video"}.any?(reason)
+      raise InfoException.new(reason + ": " + subreason || "")
     end
   end
 

src/invidious/videos/parser.cr

@@ -68,18 +68,20 @@ def extract_video_info(video_id : String)
   playability_status = player_response.dig?("playabilityStatus", "status").try &.as_s
 
   if playability_status != "OK"
-    subreason = player_response.dig?("playabilityStatus", "errorScreen", "playerErrorMessageRenderer", "subreason")
-    reason = subreason.try &.[]?("simpleText").try &.as_s
-    reason ||= subreason.try &.[]("runs").as_a.map(&.[]("text")).join("")
-    reason ||= player_response.dig("playabilityStatus", "reason").as_s
-
-    # Stop here if video is not a scheduled livestream or
-    # for LOGIN_REQUIRED when videoDetails element is not found because retrying won't help
-    if !{"LIVE_STREAM_OFFLINE", "LOGIN_REQUIRED"}.any?(playability_status) ||
-       playability_status == "LOGIN_REQUIRED" && !player_response.dig?("videoDetails")
+    reason = player_response.dig?("playabilityStatus", "reason").try &.as_s
+    reason ||= player_response.dig("playabilityStatus", "errorScreen", "playerErrorMessageRenderer", "reason", "simpleText").as_s
+    subreason_main = player_response.dig?("playabilityStatus", "errorScreen", "playerErrorMessageRenderer", "subreason")
+    subreason = subreason_main.try &.[]?("simpleText").try &.as_s
+    subreason ||= subreason_main.try &.[]("runs").as_a.map(&.[]("text")).join("")
+
+    # Stop if private video or video not found.
+    # But for video unavailable, only stop if playability_status is ERROR because playability_status UNPLAYABLE
+    # still gives all the necessary info for displaying the video page (title, description and more)
+    if {"Private video", "Video unavailable"}.any?(reason) && !{"UNPLAYABLE"}.any?(playability_status)
       return {
-        "version" => JSON::Any.new(Video::SCHEMA_VERSION.to_i64),
-        "reason"  => JSON::Any.new(reason),
+        "version"   => JSON::Any.new(Video::SCHEMA_VERSION.to_i64),
+        "reason"    => JSON::Any.new(reason),
+        "subreason" => JSON::Any.new(subreason),
       }
     end
   elsif video_id != player_response.dig("videoDetails", "videoId")
@@ -99,11 +101,8 @@ def extract_video_info(video_id : String)
     reason = nil
   end
 
-  # Don't fetch the next endpoint if the video is unavailable.
-  if {"OK", "LIVE_STREAM_OFFLINE", "LOGIN_REQUIRED"}.any?(playability_status)
-    next_response = YoutubeAPI.next({"videoId": video_id, "params": ""})
-    player_response = player_response.merge(next_response)
-  end
+  next_response = YoutubeAPI.next({"videoId": video_id, "params": ""})
+  player_response = player_response.merge(next_response)
 
   params = parse_video_info(video_id, player_response)
   params["reason"] = JSON::Any.new(reason) if reason
@@ -197,16 +196,20 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any
   end
 
   video_details = player_response.dig?("videoDetails")
+  video_details ||= {} of String => JSON::Any
   if !(microformat = player_response.dig?("microformat", "playerMicroformatRenderer"))
     microformat = {} of String => JSON::Any
   end
 
-  raise BrokenTubeException.new("videoDetails") if !video_details
-
   # Basic video infos
 
   title = video_details["title"]?.try &.as_s
 
+  title ||= extract_text(
+    video_primary_renderer
+      .try &.dig?("title")
+  )
+
   # We have to try to extract viewCount from videoPrimaryInfoRenderer first,
   # then from videoDetails, as the latter is "0" for livestreams (we want
   # to get the amount of viewers watching).
@@ -217,12 +220,25 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any
   views_txt ||= video_details["viewCount"]?.try &.as_s || ""
   views = views_txt.gsub(/\D/, "").to_i64?
 
-  length_txt = (microformat["lengthSeconds"]? || video_details["lengthSeconds"])
+  length_txt = (microformat["lengthSeconds"]? || video_details["lengthSeconds"]?)
     .try &.as_s.to_i64
 
   published = microformat["publishDate"]?
     .try { |t| Time.parse(t.as_s, "%Y-%m-%d", Time::Location::UTC) } || Time.utc
 
+  if published.nil?
+    published_txt = video_primary_renderer
+      .try &.dig?("dateText", "simpleText")
+
+    if published_txt.try &.as_s.includes?("ago") && !published_txt.nil?
+      published = decode_date(published_txt.as_s.lchop("Started streaming "))
+    elsif published_txt && published_txt.try &.as_s.matches?(/(\w{3} \d{1,2}, \d{4})$/)
+      published = Time.parse(published_txt.as_s.match!(/(\w{3} \d{1,2}, \d{4})$/)[0], "%b %-d, %Y", Time::Location::UTC)
+    else
+      published = Time.utc
+    end
+  end
+
   premiere_timestamp = microformat.dig?("liveBroadcastDetails", "startTimestamp")
     .try { |t| Time.parse_rfc3339(t.as_s) }
 
@@ -236,6 +252,10 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any
 
   live_now = microformat.dig?("liveBroadcastDetails", "isLiveNow")
     .try &.as_bool
+  if live_now.nil?
+    live_now = video_primary_renderer
+      .try &.dig?("viewCount", "videoViewCountRenderer", "isLive").try &.as_bool
+  end
   live_now ||= video_details.dig?("isLive").try &.as_bool || false
 
   post_live_dvr = video_details.dig?("isPostLiveDvr")
@@ -247,8 +267,23 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any
     .try &.as_a.map &.as_s || [] of String
 
   allow_ratings = video_details["allowRatings"]?.try &.as_bool
+
   family_friendly = microformat["isFamilySafe"]?.try &.as_bool
+  if family_friendly.nil?
+    family_friendly = true # if isFamilySafe not found then assume is safe
+  end
+
   is_listed = video_details["isCrawlable"]?.try &.as_bool
+  if is_listed.nil?
+    if video_badges = video_primary_renderer.try &.dig?("badges")
+      is_listed = !has_unlisted_badge?(video_badges)
+    else
+      # If video has no badges and videoDetails is not
+      # available, then assume isListed
+      is_listed = true
+    end
+  end
+
   is_upcoming = video_details["isUpcoming"]?.try &.as_bool
 
   keywords = video_details["keywords"]?
@@ -414,6 +449,9 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any
     subs_text = author_info["subscriberCountText"]?
       .try { |t| t["simpleText"]? || t.dig?("runs", 0, "text") }
       .try &.as_s.split(" ", 2)[0]
+
+    author ||= author_info.dig?("title", "runs", 0, "text").try &.as_s
+    ucid ||= author_info.dig?("title", "runs", 0, "navigationEndpoint", "browseEndpoint", "browseId").try &.as_s
   end
 
   # Return data
@@ -438,8 +476,8 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any
     # Extra video infos
     "allowedRegions"   => JSON::Any.new(allowed_regions.map { |v| JSON::Any.new(v) }),
     "allowRatings"     => JSON::Any.new(allow_ratings || false),
-    "isFamilyFriendly" => JSON::Any.new(family_friendly || false),
-    "isListed"         => JSON::Any.new(is_listed || false),
+    "isFamilyFriendly" => JSON::Any.new(family_friendly),
+    "isListed"         => JSON::Any.new(is_listed),
     "isUpcoming"       => JSON::Any.new(is_upcoming || false),
     "keywords"         => JSON::Any.new(keywords.map { |v| JSON::Any.new(v) }),
     "isPostLiveDvr"    => JSON::Any.new(post_live_dvr),
@@ -448,7 +486,7 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any
     # Description
     "description"      => JSON::Any.new(description || ""),
     "descriptionHtml"  => JSON::Any.new(description_html || "<p></p>"),
-    "shortDescription" => JSON::Any.new(short_description.try &.as_s || nil),
+    "shortDescription" => JSON::Any.new(short_description.try &.as_s || ""),
     # Video metadata
     "genre"     => JSON::Any.new(genre.try &.as_s || ""),
     "genreUcid" => JSON::Any.new(genre_ucid.try &.as_s?),

src/invidious/views/components/player.ecr

@@ -1,3 +1,4 @@
+<% if audio_streams && fmt_stream && preferred_captions && captions %>
 <video style="outline:none;width:100%;background-color:#000" playsinline poster="<%= thumbnail %>"
     id="player" class="on-video_player video-js player-style-<%= params.player_style %>"
     preload="<% if params.preload %>auto<% else %>none<% end %>"
@@ -87,3 +88,4 @@
 %>
 </script>
 <script src="/js/player.js?v=<%= ASSET_COMMIT %>"></script>
+<% end %>

src/invidious/views/embed.ecr

@@ -31,7 +31,17 @@
 %>
 </script>
 
-<%= rendered "components/player" %>
+<% if video.reason.nil? %>
+<div id="player-container" class="h-box">
+    <%= rendered "components/player" %>
+</div>
+<% else %>
+<div id="player-error-container" class="h-box">
+    <h3>
+        <%= video.reason %>
+    </h3>
+</div>
+<% end %>
 <script src="/js/embed.js?v=<%= ASSET_COMMIT %>"></script>
 </body>
 </html>

src/invidious/views/error.ecr

@@ -4,5 +4,4 @@
 
 <div class="h-box">
     <%= error_message %>
-    <%= next_steps %>
 </div>

src/invidious/views/watch.ecr

@@ -70,9 +70,11 @@ we're going to need to do it here in order to allow for translations.
 %>
 </script>
 
+<% if video.reason.nil? %>
 <div id="player-container" class="h-box">
     <%= rendered "components/player" %>
 </div>
+<% end %>
 
 <div class="h-box">
     <h1>
@@ -96,7 +98,10 @@ we're going to need to do it here in order to allow for translations.
 
     <% if video.reason %>
         <h3>
-            <%= video.reason %>
+            <%= translate(locale, "error_from_youtube_unplayable") %> <%= video.reason %>
+        </h3>
+        <h3>
+            <%= translate(locale, "next_steps_error_message") %>
         </h3>
     <% elsif video.premiere_timestamp.try &.> Time.utc %>
         <h3>
@@ -112,7 +117,7 @@ we're going to need to do it here in order to allow for translations.
 <div class="pure-g">
     <div class="pure-u-1 pure-u-lg-1-5">
         <div class="h-box">
-            <span id="watch-on-youtube">
+            <p id="watch-on-youtube">
                 <%-
                     link_yt_watch = URI.new(scheme: "https", host: "www.youtube.com", path: "/watch", query: "v=#{video.id}")
                     link_yt_embed = URI.new(scheme: "https", host: "www.youtube.com", path: "/embed/#{video.id}")
@@ -125,7 +130,7 @@ we're going to need to do it here in order to allow for translations.
                 -%>
                 <a id="link-yt-watch" rel="noreferrer noopener" data-base-url="<%= link_yt_watch %>" href="<%= link_yt_watch %>"><%= translate(locale, "videoinfo_watch_on_youTube") %></a>
                 (<a id="link-yt-embed" rel="noreferrer noopener" data-base-url="<%= link_yt_embed %>" href="<%= link_yt_embed %>"><%= translate(locale, "videoinfo_youTube_embed_link") %></a>)
-            </span>
+            </p>
 
             <p id="watch-on-another-invidious-instance">
                 <%- link_iv_other = IV::Frontend::Misc.redirect_url(env) -%>
@@ -185,18 +190,22 @@ we're going to need to do it here in order to allow for translations.
                 <% end %>
             <% end %>
 
+            <% if video_assets %>
             <%= Invidious::Frontend::WatchPage.download_widget(locale, video, video_assets) %>
+            <% end %>
 
             <p id="views"><i class="icon ion-ios-eye"></i> <%= number_with_separator(video.views) %></p>
             <p id="likes"><i class="icon ion-ios-thumbs-up"></i> <%= number_with_separator(video.likes) %></p>
             <p id="dislikes" style="display: none; visibility: hidden;"></p>
+            <% if video.genre %>
             <p id="genre"><%= translate(locale, "Genre: ") %>
                 <% if !video.genre_url %>
                     <%= video.genre %>
                 <% else %>
                     <a href="<%= video.genre_url %>"><%= video.genre %></a>
                 <% end %>
             </p>
+            <% end %>
             <% if video.license %>
                 <% if video.license.empty? %>
                     <p id="license"><%= translate(locale, "License: ") %><%= translate(locale, "Standard YouTube license") %></p>

src/invidious/yt_backend/extractors_utils.cr

@@ -68,6 +68,23 @@ rescue ex
   return false
 end
 
+def has_unlisted_badge?(badges : JSON::Any?)
+  return false if badges.nil?
+
+  badges.as_a.each do |badge|
+    icon_type = badge.dig("metadataBadgeRenderer", "icon", "iconType").as_s
+
+    return true if icon_type == "PRIVACY_UNLISTED"
+  end
+
+  return false
+rescue ex
+  LOGGER.debug("Unable to parse owner badges. Got exception: #{ex.message}")
+  LOGGER.trace("Owner badges data: #{badges.to_json}")
+
+  return false
+end
+
 # This function extracts SearchVideo items from a Category.
 # Categories are commonly returned in search results and trending pages.
 def extract_category(category : Category) : Array(SearchVideo)