diff --git a/src/main/java/moviescraper/doctord/controller/FileDownloaderUtilities.java b/src/main/java/moviescraper/doctord/controller/FileDownloaderUtilities.java old mode 100644 new mode 100755 index b5feff8f..58b31861 --- a/src/main/java/moviescraper/doctord/controller/FileDownloaderUtilities.java +++ b/src/main/java/moviescraper/doctord/controller/FileDownloaderUtilities.java @@ -4,6 +4,7 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.net.HttpURLConnection; import java.net.URL; import java.net.URLConnection; import java.awt.image.BufferedImage; @@ -59,10 +60,39 @@ public static void writeURLToFile(URL url, File file) throws IOException { FileUtils.copyInputStreamToFile(FileDownloaderUtilities.getDefaultUrlConnection(url).getInputStream(), file); } + // Auto follow url - from http to https + public static URLConnection autoFollow(URL url) throws IOException { + URL resourceUrl, base, next = url; + URLConnection conn; + String location; + + while (true) + { + conn = next.openConnection(); + conn.setConnectTimeout(15000); + conn.setReadTimeout(15000); + conn.setRequestProperty("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_5) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/71.0.1410.65 Safari/537.31"); + + if (conn instanceof HttpURLConnection) { + switch (((HttpURLConnection) conn).getResponseCode()) { + case 301: + case 302: + location = conn.getHeaderField("Location"); + base = new URL(next.toString()); + next = new URL(base, location); // Deal with relative URLs + continue; + } + } + + break; + } + + return conn; + } + public static void writeURLToFile(URL url, File file, URL viewerUrl) throws IOException { try { - URLConnection imageConnection = url.openConnection(); - imageConnection.setRequestProperty("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_5) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.65 Safari/537.31"); + URLConnection imageConnection = autoFollow(url); if (viewerUrl != null) { imageConnection.setRequestProperty("Referer", viewerUrl.toString()); } @@ -72,6 +102,5 @@ public static void writeURLToFile(URL url, File file, URL viewerUrl) throws IOEx } catch (Throwable t) { System.out.println("Cannot write file: " + t.getMessage()); } - } } diff --git a/src/main/java/moviescraper/doctord/controller/siteparsingprofile/specific/OnePondoParsingProfile.java b/src/main/java/moviescraper/doctord/controller/siteparsingprofile/specific/OnePondoParsingProfile.java old mode 100644 new mode 100755 index 5a60bf3b..29b1f2dd --- a/src/main/java/moviescraper/doctord/controller/siteparsingprofile/specific/OnePondoParsingProfile.java +++ b/src/main/java/moviescraper/doctord/controller/siteparsingprofile/specific/OnePondoParsingProfile.java @@ -188,8 +188,13 @@ public Thumb[] scrapeFanart() { thumbList.add(new Thumb(backgroundURLTwo)); if (SiteParsingProfile.fileExistsAtURL(popupFourURL)) thumbList.add(new Thumb(popupFourURL)); - return thumbList.toArray(new Thumb[thumbList.size()]); + // Return thumbs if posters are not found + if (thumbList.size() == 0) { + return scrapePosters(); + } + + return thumbList.toArray(new Thumb[thumbList.size()]); } catch (MalformedURLException e) { // TODO Auto-generated catch block e.printStackTrace(); @@ -218,6 +223,12 @@ public ID scrapeID() { public ArrayList scrapeGenres() { //For now, I wasn't able to find any genres on the page ArrayList genreList = new ArrayList<>(); + JSONObject pageJSON = getMovieJSON(); + JSONArray genres = pageJSON.getJSONArray("UCNAMEEn"); + for (Object genre : genres) { + genreList.add(new Genre((String) genre)); + } + return genreList; } diff --git a/src/main/java/moviescraper/doctord/model/dataitem/Thumb.java b/src/main/java/moviescraper/doctord/model/dataitem/Thumb.java old mode 100644 new mode 100755 diff --git a/src/main/java/moviescraper/doctord/view/customcomponents/AsyncImageComponent.java b/src/main/java/moviescraper/doctord/view/customcomponents/AsyncImageComponent.java old mode 100644 new mode 100755 index 45ebfb53..6be870c4 --- a/src/main/java/moviescraper/doctord/view/customcomponents/AsyncImageComponent.java +++ b/src/main/java/moviescraper/doctord/view/customcomponents/AsyncImageComponent.java @@ -7,6 +7,8 @@ import java.awt.event.MouseListener; import java.awt.image.BufferedImage; import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.MalformedURLException; import java.net.URL; import java.net.URLConnection; import javax.imageio.ImageIO; @@ -15,6 +17,7 @@ import javax.swing.SwingWorker; import javax.swing.border.Border; +import moviescraper.doctord.controller.FileDownloaderUtilities; import org.imgscalr.Scalr; import org.imgscalr.Scalr.Method; @@ -229,22 +232,17 @@ public ImageLoader(ImageConsumer consumer, URL url, URL referrerURL, boolean isI @Override protected BufferedImage doInBackground() throws IOException { - if (url == null) { return null; } + if (ImageCache.isImageCached(url, isImageModified)) { pictureLoaded = Thumb.convertToBufferedImage(ImageCache.getImageFromCache(url, isImageModified, referrerURL)); - } else { - + } + else { try { - URLConnection imageConnection = url.openConnection(); - imageConnection.setRequestProperty("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_5) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.65 Safari/537.31"); - if (referrerURL != null) { - imageConnection.setRequestProperty("Referer", referrerURL.toString()); - } + URLConnection imageConnection = FileDownloaderUtilities.autoFollow(url); pictureLoaded = ImageIO.read(imageConnection.getInputStream()); - } catch (Throwable t) { System.out.println("Cannot load image: " + t.getMessage()); }