Re-introduce content language for YouTube

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java

@@ -166,14 +166,13 @@ public CommentsExtractor getCommentsExtractor(ListLinkHandler urlIdHandler)
 
     // https://www.youtube.com/picker_ajax?action_language_json=1
     private static final List<Localization> SUPPORTED_LANGUAGES = Localization.listFrom(
-            "en-GB"
-            /*"af", "am", "ar", "az", "be", "bg", "bn", "bs", "ca", "cs", "da", "de",
+            "af", "am", "ar", "az", "be", "bg", "bn", "bs", "ca", "cs", "da", "de",
             "el", "en", "en-GB", "es", "es-419", "es-US", "et", "eu", "fa", "fi", "fil", "fr",
             "fr-CA", "gl", "gu", "hi", "hr", "hu", "hy", "id", "is", "it", "iw", "ja",
             "ka", "kk", "km", "kn", "ko", "ky", "lo", "lt", "lv", "mk", "ml", "mn",
             "mr", "ms", "my", "ne", "nl", "no", "pa", "pl", "pt", "pt-PT", "ro", "ru",
             "si", "sk", "sl", "sq", "sr", "sr-Latn", "sv", "sw", "ta", "te", "th", "tr",
-            "uk", "ur", "uz", "vi", "zh-CN", "zh-HK", "zh-TW", "zu"*/
+            "uk", "ur", "uz", "vi", "zh-CN", "zh-HK", "zh-TW", "zu"
     );
 
     // https://www.youtube.com/picker_ajax?action_country_json=1

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java

@@ -6,23 +6,27 @@
 import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
+import org.schabi.newpipe.extractor.NewPipe;
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.channel.ChannelExtractor;
 import org.schabi.newpipe.extractor.downloader.Downloader;
 import org.schabi.newpipe.extractor.downloader.Response;
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
 import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
+import org.schabi.newpipe.extractor.localization.Localization;
 import org.schabi.newpipe.extractor.localization.TimeAgoParser;
 import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeParsingHelper;
 import org.schabi.newpipe.extractor.stream.StreamInfoItem;
 import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
 import org.schabi.newpipe.extractor.utils.Parser;
-import org.schabi.newpipe.extractor.utils.Utils;
 
 import javax.annotation.Nonnull;
 import java.io.IOException;
 
+import static org.schabi.newpipe.extractor.utils.Utils.mixedNumberWordToLong;
+
 /*
  * Created by Christian Schabesberger on 25.07.16.
  *
@@ -81,7 +85,8 @@ public String getUrl() throws ParsingException {
     public String getId() throws ParsingException {
         try {
             return doc.select("meta[itemprop=\"channelId\"]").first().attr("content");
-        } catch (Exception ignored) {}
+        } catch (Exception ignored) {
+        }
 
         // fallback method; does not work with channels that have no "Subscribe" button (e.g. EminemVEVO)
         try {
@@ -137,19 +142,18 @@ public String getFeedUrl() throws ParsingException {
 
     @Override
     public long getSubscriberCount() throws ParsingException {
-
+        long subCount = -1;
         final Element el = doc.select("span[class*=\"yt-subscription-button-subscriber-count\"]").first();
         if (el != null) {
+            // If the element is null, the channel have the subscriber count disabled
             String elTitle = el.attr("title");
             try {
-                return Utils.mixedNumberWordToLong(elTitle);
+                subCount = mixedNumberWordToLong(elTitle, getExtractorLocalization());
             } catch (NumberFormatException e) {
                 throw new ParsingException("Could not get subscriber count", e);
             }
-        } else {
-            // If the element is null, the channel have the subscriber count disabled
-            return -1;
         }
+        return subCount;
     }
 
     @Override
@@ -196,7 +200,7 @@ public InfoItemsPage<StreamInfoItem> getPage(String pageUrl) throws IOException,
     }
 
     private String getNextPageUrlFromAjaxPage(final JsonObject ajaxJson, final String pageUrl)
-        throws ParsingException {
+            throws ParsingException {
         String loadMoreHtmlDataRaw = ajaxJson.getString("load_more_widget_html");
         if (!loadMoreHtmlDataRaw.isEmpty()) {
             return getNextPageUrlFrom(Jsoup.parse(loadMoreHtmlDataRaw, pageUrl));

extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java

@@ -41,6 +41,8 @@
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
+import static org.schabi.newpipe.extractor.utils.JsonUtils.getString;
+
 /*
  * Created by Christian Schabesberger on 06.08.15.
  *
@@ -107,8 +109,7 @@ public YoutubeStreamExtractor(StreamingService service, LinkHandler linkHandler)
     public String getName() throws ParsingException {
         assertPageFetched();
         try {
-            return playerResponse.getObject("videoDetails").getString("title");
-
+            return getString(playerResponse.getObject("microformat").getObject("playerMicroformatRenderer").getObject("title"), "simpleText");
         } catch (Exception e) {
             // fallback HTML method
             String name = null;
@@ -183,12 +184,11 @@ public String getThumbnailUrl() throws ParsingException {
     public Description getDescription() throws ParsingException {
         assertPageFetched();
         try {
-            // first try to get html-formatted description
-            return new Description(parseHtmlAndGetFullLinks(doc.select("p[id=\"eow-description\"]").first().html()), Description.HTML);
+            //JSON first because formatting is better, see https://github.com/TeamNewPipe/NewPipeExtractor/pull/257#discussion_r379828770
+            return new Description(getString(playerResponse.getObject("microformat").getObject("playerMicroformatRenderer").getObject("description"), "simpleText"), Description.PLAIN_TEXT);
         } catch (Exception e) {
             try {
-                // fallback to raw non-html description
-                return new Description(playerResponse.getObject("videoDetails").getString("shortDescription"), Description.PLAIN_TEXT);
+                return new Description(parseHtmlAndGetFullLinks(doc.select("p[id=\"eow-description\"]").first().html()), Description.HTML);
             } catch (Exception ignored) {
                 throw new ParsingException("Could not get the description", e);
             }

extractor/src/main/java/org/schabi/newpipe/extractor/utils/AbbreviationHashMap.java

@@ -0,0 +1,156 @@
+package org.schabi.newpipe.extractor.utils;
+
+import java.util.HashMap;
+
+
+/**
+ * Map matching abbreviations with their English equivalents
+ *
+ * @author B0pol
+ * <p>
+ * By using this map, you can replace the abbreviations used for numbers in the 80 languages supported by YouTube
+ * With their English equivalent.
+ * </p>
+ * <p>
+ * Some language use more abbreviations for numbers: east-asian languages have abbreviations for ten thousand,
+ * and hundred million, indo-arabic languages have abbreviations for a hundred thousand and ten million,
+ * then we replace ten thousand by {@link #tenThousandAbbreviation},
+ * hundred thousand by {@link #hundredThousandAbbreviation},
+ * ten million by {@link #tenMillionAbbreviation},
+ * hundred million by {@link #hundredMillionAbbreviation}.
+ * </p>
+ * <p>
+ * The languages using the abbreviation is commented with the language code at the left.
+ * @see <a href="https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes">Wikipedia page of language codes</a>
+ * </p>
+ */
+public class AbbreviationHashMap {
+
+    //should be safe until someone has 1 billion subscribers on YouTube
+    public static final HashMap<String, String> abbreviationSubscribersCount = new HashMap<>();
+    public static final String englishMillionAbbreviation = "M";
+    public static final String englishThousandAbbreviation = "K";
+
+    public static final String tenThousandAbbreviation = "万";
+    public static final String hundredThousandAbbreviation = "ল";
+    public static final String tenMillionAbbreviation = "ক";
+    public static final String hundredMillionAbbreviation = "億";
+
+    static {
+        abbreviationSubscribersCount.put(englishThousandAbbreviation, englishThousandAbbreviation); //az, iw, en
+        abbreviationSubscribersCount.put(englishMillionAbbreviation, englishMillionAbbreviation); //iw, en
+        abbreviationSubscribersCount.put(tenMillionAbbreviation, tenMillionAbbreviation);
+
+        abbreviationSubscribersCount.put(" k", englishThousandAbbreviation); //af
+        abbreviationSubscribersCount.put(" ሺ", englishThousandAbbreviation); //am
+        abbreviationSubscribersCount.put(" ألف", englishThousandAbbreviation); //ar
+        abbreviationSubscribersCount.put(" тыс", englishThousandAbbreviation); //be, ru
+        abbreviationSubscribersCount.put(" хил", englishThousandAbbreviation); //bg
+        abbreviationSubscribersCount.put(" হা", englishThousandAbbreviation); //bn
+        abbreviationSubscribersCount.put(" hilj", englishThousandAbbreviation); //bs, sr
+        abbreviationSubscribersCount.put("m", englishThousandAbbreviation); //ca
+        abbreviationSubscribersCount.put(" tis", englishThousandAbbreviation); //cs, hr, sk, sl
+        abbreviationSubscribersCount.put(" χιλ", englishThousandAbbreviation); //el
+        abbreviationSubscribersCount.put(" tuh", englishThousandAbbreviation); //et
+        abbreviationSubscribersCount.put(" هزار", englishThousandAbbreviation); //fa
+        abbreviationSubscribersCount.put(" t", englishThousandAbbreviation); //fi
+        abbreviationSubscribersCount.put(" હજાર", englishThousandAbbreviation); //gu
+        abbreviationSubscribersCount.put(" हज़ार", englishThousandAbbreviation); //hi
+        abbreviationSubscribersCount.put(" E", englishThousandAbbreviation); //hu
+        abbreviationSubscribersCount.put(" հզր", englishThousandAbbreviation); //hy
+        abbreviationSubscribersCount.put(" rb", englishThousandAbbreviation); //id
+        abbreviationSubscribersCount.put(" þ", englishThousandAbbreviation); //is
+        abbreviationSubscribersCount.put(" ათ", englishThousandAbbreviation); //ka
+        abbreviationSubscribersCount.put(" мың", englishThousandAbbreviation); //kk
+        abbreviationSubscribersCount.put("ពាន់", englishThousandAbbreviation); //km
+        abbreviationSubscribersCount.put(" ពាន់", englishThousandAbbreviation); //km
+        abbreviationSubscribersCount.put("ಸಾ", englishThousandAbbreviation); //kn
+        abbreviationSubscribersCount.put("천", englishThousandAbbreviation); //ko
+        abbreviationSubscribersCount.put(" миң", englishThousandAbbreviation); //ky
+        abbreviationSubscribersCount.put(" ກີບ", englishThousandAbbreviation); //lo
+        abbreviationSubscribersCount.put(" ພັນ", englishThousandAbbreviation); //lo
+        abbreviationSubscribersCount.put(" tūkst", englishThousandAbbreviation); //lt, lv
+        abbreviationSubscribersCount.put(" илј", englishThousandAbbreviation); //mk
+        abbreviationSubscribersCount.put(" мянга", englishThousandAbbreviation); //mn
+        abbreviationSubscribersCount.put(" ह", englishThousandAbbreviation); //mr
+        abbreviationSubscribersCount.put("ထောင်", englishThousandAbbreviation); //my
+        abbreviationSubscribersCount.put(" हजार", englishThousandAbbreviation); //ne
+        abbreviationSubscribersCount.put("k", englishThousandAbbreviation); //no
+        abbreviationSubscribersCount.put(" ਹਜ਼ਾਰ", englishThousandAbbreviation); //pa
+        abbreviationSubscribersCount.put(" tys", englishThousandAbbreviation); //pl
+        abbreviationSubscribersCount.put(" K", englishThousandAbbreviation); //ro
+        abbreviationSubscribersCount.put("ද", englishThousandAbbreviation); //si
+        abbreviationSubscribersCount.put(" mijë", englishThousandAbbreviation); //sq
+        abbreviationSubscribersCount.put(" хиљ", englishThousandAbbreviation); //sr-Latn
+        abbreviationSubscribersCount.put("elfu ", englishThousandAbbreviation); //sw
+        abbreviationSubscribersCount.put("ஆ", englishThousandAbbreviation); //ta
+        abbreviationSubscribersCount.put("వే", englishThousandAbbreviation); //te
+        abbreviationSubscribersCount.put(" พัน", englishThousandAbbreviation); //th
+        abbreviationSubscribersCount.put(" B", englishThousandAbbreviation); //tr
+        abbreviationSubscribersCount.put(" тис", englishThousandAbbreviation); //uk
+        abbreviationSubscribersCount.put(" ہزار", englishThousandAbbreviation); //ur
+        abbreviationSubscribersCount.put(" ming", englishThousandAbbreviation); //uz
+        abbreviationSubscribersCount.put(" N", englishThousandAbbreviation); //vi
+
+        abbreviationSubscribersCount.put(" m", englishMillionAbbreviation); //af, is
+        abbreviationSubscribersCount.put(" م", englishMillionAbbreviation); //an
+        abbreviationSubscribersCount.put("ሜ", englishMillionAbbreviation); //am
+        abbreviationSubscribersCount.put(" ሜትር", englishMillionAbbreviation); //am
+        abbreviationSubscribersCount.put(" مليون", englishMillionAbbreviation); //ar
+        abbreviationSubscribersCount.put(" mln", englishMillionAbbreviation); //az, et, lt, nl, pl, sq, uz
+        abbreviationSubscribersCount.put(" млн", englishMillionAbbreviation); //be, bg, kk, ky, ru, uk
+        abbreviationSubscribersCount.put(" mil", englishMillionAbbreviation); //bs, cs, hr, ro, sk, sr-Latn
+        abbreviationSubscribersCount.put(" M", englishMillionAbbreviation); //ca, es, eu, and many more
+        abbreviationSubscribersCount.put(" mio", englishMillionAbbreviation); //da, sl
+        abbreviationSubscribersCount.put(" Mio", englishMillionAbbreviation); //de
+        abbreviationSubscribersCount.put(" εκ", englishMillionAbbreviation); //el
+        abbreviationSubscribersCount.put(" میلیون", englishMillionAbbreviation); //fa
+        abbreviationSubscribersCount.put(" milj", englishMillionAbbreviation); //fi, lv
+        abbreviationSubscribersCount.put(" մլն", englishMillionAbbreviation); //hy
+        abbreviationSubscribersCount.put(" jt", englishMillionAbbreviation); //id
+        abbreviationSubscribersCount.put(" Mln", englishMillionAbbreviation); //it
+        abbreviationSubscribersCount.put(" მლნ", englishMillionAbbreviation); //ka
+        abbreviationSubscribersCount.put(" លាន", englishMillionAbbreviation); //km
+        abbreviationSubscribersCount.put("ಮಿ", englishMillionAbbreviation); //kn
+        abbreviationSubscribersCount.put(" ລ້ານ", englishMillionAbbreviation); //lo
+        abbreviationSubscribersCount.put(" М", englishMillionAbbreviation); //mk. It isn't a space but a
+        // narrow non-breaking space, ie U+202F
+        abbreviationSubscribersCount.put(" мил", englishMillionAbbreviation); //mk, sr
+        abbreviationSubscribersCount.put(" сая", englishMillionAbbreviation); //mn
+        abbreviationSubscribersCount.put("သန်း", englishMillionAbbreviation); //my
+        abbreviationSubscribersCount.put(" mill", englishMillionAbbreviation); //no
+        abbreviationSubscribersCount.put(" mi", englishMillionAbbreviation); //pt
+        abbreviationSubscribersCount.put("මි", englishMillionAbbreviation); //si
+        abbreviationSubscribersCount.put(" mn", englishMillionAbbreviation); //sv
+        abbreviationSubscribersCount.put("మి", englishMillionAbbreviation); //te
+        abbreviationSubscribersCount.put("மி", englishMillionAbbreviation); //ta
+        abbreviationSubscribersCount.put(" ล้าน", englishMillionAbbreviation); //th
+        abbreviationSubscribersCount.put(" Mn", englishMillionAbbreviation); //tr
+        abbreviationSubscribersCount.put(" Tr", englishMillionAbbreviation); //vi
+
+        abbreviationSubscribersCount.put("만", tenThousandAbbreviation); //ko
+        abbreviationSubscribersCount.put("万", tenThousandAbbreviation); //ja, zh-CN
+        abbreviationSubscribersCount.put("萬", tenThousandAbbreviation); //zh-TW
+
+        abbreviationSubscribersCount.put(" লা", hundredThousandAbbreviation); //bn
+        abbreviationSubscribersCount.put(" લાખ", hundredThousandAbbreviation); //gu
+        abbreviationSubscribersCount.put(" लाख", hundredThousandAbbreviation); //hi, mr, ne
+        abbreviationSubscribersCount.put(" ਲੱਖ", hundredThousandAbbreviation); //pa
+        abbreviationSubscribersCount.put(" لاکھ", hundredThousandAbbreviation); //ur
+        abbreviationSubscribersCount.put("သိန်း", hundredThousandAbbreviation); //my
+        abbreviationSubscribersCount.put(" แสน", hundredThousandAbbreviation); //th
+
+        abbreviationSubscribersCount.put(" কো", tenMillionAbbreviation); //bn
+        abbreviationSubscribersCount.put(" કરોડ", tenMillionAbbreviation); //gu
+        abbreviationSubscribersCount.put(" क॰", tenMillionAbbreviation); //hi
+        abbreviationSubscribersCount.put(" कोटी", tenMillionAbbreviation); //mr
+        abbreviationSubscribersCount.put("ကုဋေ", tenMillionAbbreviation); //my
+        abbreviationSubscribersCount.put(" करोड", tenMillionAbbreviation); //ne
+        abbreviationSubscribersCount.put(" ਕਰੋੜ", tenMillionAbbreviation); //pa
+        abbreviationSubscribersCount.put(" کروڑ", tenMillionAbbreviation); //ur
+
+        abbreviationSubscribersCount.put("億", hundredMillionAbbreviation); //ja, zh-TW
+        abbreviationSubscribersCount.put("억", hundredMillionAbbreviation); //ko
+        abbreviationSubscribersCount.put("亿", hundredMillionAbbreviation); //zh-CN
+    }
+}