From f4e7abf420f3a45e6cb65c1df770f48d837b9bc4 Mon Sep 17 00:00:00 2001 From: reapier Date: Mon, 21 Jan 2019 23:46:30 -0800 Subject: [PATCH 1/3] Fix image download redirect -> follow 30* from http to https --- .../specific/OnePondoParsingProfile.java | 7 ++- .../doctord/model/dataitem/Thumb.java | 0 .../customcomponents/AsyncImageComponent.java | 45 ++++++++++++++++--- 3 files changed, 44 insertions(+), 8 deletions(-) mode change 100644 => 100755 src/main/java/moviescraper/doctord/controller/siteparsingprofile/specific/OnePondoParsingProfile.java mode change 100644 => 100755 src/main/java/moviescraper/doctord/model/dataitem/Thumb.java mode change 100644 => 100755 src/main/java/moviescraper/doctord/view/customcomponents/AsyncImageComponent.java diff --git a/src/main/java/moviescraper/doctord/controller/siteparsingprofile/specific/OnePondoParsingProfile.java b/src/main/java/moviescraper/doctord/controller/siteparsingprofile/specific/OnePondoParsingProfile.java old mode 100644 new mode 100755 index 5a60bf3b..a9a6193d --- a/src/main/java/moviescraper/doctord/controller/siteparsingprofile/specific/OnePondoParsingProfile.java +++ b/src/main/java/moviescraper/doctord/controller/siteparsingprofile/specific/OnePondoParsingProfile.java @@ -188,8 +188,13 @@ public Thumb[] scrapeFanart() { thumbList.add(new Thumb(backgroundURLTwo)); if (SiteParsingProfile.fileExistsAtURL(popupFourURL)) thumbList.add(new Thumb(popupFourURL)); - return thumbList.toArray(new Thumb[thumbList.size()]); + // Return thumbs if posters are not found + if (thumbList.size() == 0) { + return scrapePosters(); + } + + return thumbList.toArray(new Thumb[thumbList.size()]); } catch (MalformedURLException e) { // TODO Auto-generated catch block e.printStackTrace(); diff --git a/src/main/java/moviescraper/doctord/model/dataitem/Thumb.java b/src/main/java/moviescraper/doctord/model/dataitem/Thumb.java old mode 100644 new mode 100755 diff --git a/src/main/java/moviescraper/doctord/view/customcomponents/AsyncImageComponent.java b/src/main/java/moviescraper/doctord/view/customcomponents/AsyncImageComponent.java old mode 100644 new mode 100755 index 45ebfb53..69d81cce --- a/src/main/java/moviescraper/doctord/view/customcomponents/AsyncImageComponent.java +++ b/src/main/java/moviescraper/doctord/view/customcomponents/AsyncImageComponent.java @@ -7,6 +7,8 @@ import java.awt.event.MouseListener; import java.awt.image.BufferedImage; import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.MalformedURLException; import java.net.URL; import java.net.URLConnection; import javax.imageio.ImageIO; @@ -227,6 +229,39 @@ public ImageLoader(ImageConsumer consumer, URL url, URL referrerURL, boolean isI } } + // Auto follow url - from http to https + protected URLConnection autoFollow(URL url) throws IOException { + URL resourceUrl, base, next = url; + URLConnection conn; + String location; + + while (true) + { + conn = next.openConnection(); + if (referrerURL != null) { + conn.setRequestProperty("Referer", referrerURL.toString()); + } + conn.setConnectTimeout(15000); + conn.setReadTimeout(15000); + conn.setRequestProperty("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_5) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/71.0.1410.65 Safari/537.31"); + + if (conn instanceof HttpURLConnection) { + switch (((HttpURLConnection) conn).getResponseCode()) { + case 301: + case 302: + location = conn.getHeaderField("Location"); + base = new URL(next.toString()); + next = new URL(base, location); // Deal with relative URLs + continue; + } + } + + break; + } + + return conn; + } + @Override protected BufferedImage doInBackground() throws IOException { @@ -235,14 +270,10 @@ protected BufferedImage doInBackground() throws IOException { } if (ImageCache.isImageCached(url, isImageModified)) { pictureLoaded = Thumb.convertToBufferedImage(ImageCache.getImageFromCache(url, isImageModified, referrerURL)); - } else { - + } + else { try { - URLConnection imageConnection = url.openConnection(); - imageConnection.setRequestProperty("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_5) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.65 Safari/537.31"); - if (referrerURL != null) { - imageConnection.setRequestProperty("Referer", referrerURL.toString()); - } + URLConnection imageConnection = autoFollow(url); pictureLoaded = ImageIO.read(imageConnection.getInputStream()); } catch (Throwable t) { From 26ee22904648506f4e7dea5ef328066f9cd5d493 Mon Sep 17 00:00:00 2001 From: reapier Date: Tue, 22 Jan 2019 00:03:16 -0800 Subject: [PATCH 2/3] 1Pondo - parse genres as well --- .../siteparsingprofile/specific/OnePondoParsingProfile.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/main/java/moviescraper/doctord/controller/siteparsingprofile/specific/OnePondoParsingProfile.java b/src/main/java/moviescraper/doctord/controller/siteparsingprofile/specific/OnePondoParsingProfile.java index a9a6193d..29b1f2dd 100755 --- a/src/main/java/moviescraper/doctord/controller/siteparsingprofile/specific/OnePondoParsingProfile.java +++ b/src/main/java/moviescraper/doctord/controller/siteparsingprofile/specific/OnePondoParsingProfile.java @@ -223,6 +223,12 @@ public ID scrapeID() { public ArrayList scrapeGenres() { //For now, I wasn't able to find any genres on the page ArrayList genreList = new ArrayList<>(); + JSONObject pageJSON = getMovieJSON(); + JSONArray genres = pageJSON.getJSONArray("UCNAMEEn"); + for (Object genre : genres) { + genreList.add(new Genre((String) genre)); + } + return genreList; } From 4d782cffabee4edb5de8d2fd68c2ab483a842035 Mon Sep 17 00:00:00 2001 From: reapier Date: Tue, 22 Jan 2019 01:15:23 -0800 Subject: [PATCH 3/3] Save to file needs to follow http -> https redirect as well --- .../controller/FileDownloaderUtilities.java | 35 +++++++++++++++-- .../customcomponents/AsyncImageComponent.java | 39 ++----------------- 2 files changed, 35 insertions(+), 39 deletions(-) mode change 100644 => 100755 src/main/java/moviescraper/doctord/controller/FileDownloaderUtilities.java diff --git a/src/main/java/moviescraper/doctord/controller/FileDownloaderUtilities.java b/src/main/java/moviescraper/doctord/controller/FileDownloaderUtilities.java old mode 100644 new mode 100755 index b5feff8f..58b31861 --- a/src/main/java/moviescraper/doctord/controller/FileDownloaderUtilities.java +++ b/src/main/java/moviescraper/doctord/controller/FileDownloaderUtilities.java @@ -4,6 +4,7 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.net.HttpURLConnection; import java.net.URL; import java.net.URLConnection; import java.awt.image.BufferedImage; @@ -59,10 +60,39 @@ public static void writeURLToFile(URL url, File file) throws IOException { FileUtils.copyInputStreamToFile(FileDownloaderUtilities.getDefaultUrlConnection(url).getInputStream(), file); } + // Auto follow url - from http to https + public static URLConnection autoFollow(URL url) throws IOException { + URL resourceUrl, base, next = url; + URLConnection conn; + String location; + + while (true) + { + conn = next.openConnection(); + conn.setConnectTimeout(15000); + conn.setReadTimeout(15000); + conn.setRequestProperty("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_5) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/71.0.1410.65 Safari/537.31"); + + if (conn instanceof HttpURLConnection) { + switch (((HttpURLConnection) conn).getResponseCode()) { + case 301: + case 302: + location = conn.getHeaderField("Location"); + base = new URL(next.toString()); + next = new URL(base, location); // Deal with relative URLs + continue; + } + } + + break; + } + + return conn; + } + public static void writeURLToFile(URL url, File file, URL viewerUrl) throws IOException { try { - URLConnection imageConnection = url.openConnection(); - imageConnection.setRequestProperty("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_5) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.65 Safari/537.31"); + URLConnection imageConnection = autoFollow(url); if (viewerUrl != null) { imageConnection.setRequestProperty("Referer", viewerUrl.toString()); } @@ -72,6 +102,5 @@ public static void writeURLToFile(URL url, File file, URL viewerUrl) throws IOEx } catch (Throwable t) { System.out.println("Cannot write file: " + t.getMessage()); } - } } diff --git a/src/main/java/moviescraper/doctord/view/customcomponents/AsyncImageComponent.java b/src/main/java/moviescraper/doctord/view/customcomponents/AsyncImageComponent.java index 69d81cce..6be870c4 100755 --- a/src/main/java/moviescraper/doctord/view/customcomponents/AsyncImageComponent.java +++ b/src/main/java/moviescraper/doctord/view/customcomponents/AsyncImageComponent.java @@ -17,6 +17,7 @@ import javax.swing.SwingWorker; import javax.swing.border.Border; +import moviescraper.doctord.controller.FileDownloaderUtilities; import org.imgscalr.Scalr; import org.imgscalr.Scalr.Method; @@ -229,53 +230,19 @@ public ImageLoader(ImageConsumer consumer, URL url, URL referrerURL, boolean isI } } - // Auto follow url - from http to https - protected URLConnection autoFollow(URL url) throws IOException { - URL resourceUrl, base, next = url; - URLConnection conn; - String location; - - while (true) - { - conn = next.openConnection(); - if (referrerURL != null) { - conn.setRequestProperty("Referer", referrerURL.toString()); - } - conn.setConnectTimeout(15000); - conn.setReadTimeout(15000); - conn.setRequestProperty("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_5) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/71.0.1410.65 Safari/537.31"); - - if (conn instanceof HttpURLConnection) { - switch (((HttpURLConnection) conn).getResponseCode()) { - case 301: - case 302: - location = conn.getHeaderField("Location"); - base = new URL(next.toString()); - next = new URL(base, location); // Deal with relative URLs - continue; - } - } - - break; - } - - return conn; - } - @Override protected BufferedImage doInBackground() throws IOException { - if (url == null) { return null; } + if (ImageCache.isImageCached(url, isImageModified)) { pictureLoaded = Thumb.convertToBufferedImage(ImageCache.getImageFromCache(url, isImageModified, referrerURL)); } else { try { - URLConnection imageConnection = autoFollow(url); + URLConnection imageConnection = FileDownloaderUtilities.autoFollow(url); pictureLoaded = ImageIO.read(imageConnection.getInputStream()); - } catch (Throwable t) { System.out.println("Cannot load image: " + t.getMessage()); }