diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java index 5b2698085..2d6143752 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java @@ -106,23 +106,9 @@ public boolean addURLToDownload(URL url, String prefix, String subdirectory, Str return false; } logger.debug("url: " + url + ", prefix: " + prefix + ", subdirectory" + subdirectory + ", referrer: " + referrer + ", cookies: " + cookies); - String saveAs = url.toExternalForm(); - saveAs = saveAs.substring(saveAs.lastIndexOf('/')+1); - if (saveAs.indexOf('?') >= 0) { saveAs = saveAs.substring(0, saveAs.indexOf('?')); } - if (saveAs.indexOf('#') >= 0) { saveAs = saveAs.substring(0, saveAs.indexOf('#')); } - if (saveAs.indexOf('&') >= 0) { saveAs = saveAs.substring(0, saveAs.indexOf('&')); } - if (saveAs.indexOf(':') >= 0) { saveAs = saveAs.substring(0, saveAs.indexOf(':')); } File saveFileAs; try { - if (!subdirectory.equals("")) { - subdirectory = File.separator + subdirectory; - } - saveFileAs = new File( - workingDir.getCanonicalPath() - + subdirectory - + File.separator - + prefix - + saveAs); + saveFileAs = getSaveAsFile(url, prefix, subdirectory); } catch (IOException e) { logger.error("[!] Error creating save file path for URL '" + url + "':", e); return false; @@ -134,7 +120,26 @@ public boolean addURLToDownload(URL url, String prefix, String subdirectory, Str } return addURLToDownload(url, saveFileAs, referrer, cookies); } - + + protected File getSaveAsFile(URL url, String prefix, String subdirectory) throws IOException { + String saveAs = url.toExternalForm(); + saveAs = saveAs.substring(saveAs.lastIndexOf('/')+1); + if (saveAs.indexOf('?') >= 0) { saveAs = saveAs.substring(0, saveAs.indexOf('?')); } + if (saveAs.indexOf('#') >= 0) { saveAs = saveAs.substring(0, saveAs.indexOf('#')); } + if (saveAs.indexOf('&') >= 0) { saveAs = saveAs.substring(0, saveAs.indexOf('&')); } + if (saveAs.indexOf(':') >= 0) { saveAs = saveAs.substring(0, saveAs.indexOf(':')); } + File saveFileAs; + if (!subdirectory.equals("")) { + subdirectory = File.separator + subdirectory; + } + saveFileAs = new File( + workingDir.getCanonicalPath() + + subdirectory + + File.separator + + prefix + + saveAs); + return saveFileAs; + } /** * Queues file to be downloaded and saved. With options. diff --git a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java index e8dbc6e32..470d0aa6f 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java @@ -82,6 +82,18 @@ public boolean addURLToDownload(URL url, File saveAs, String referrer, Map cookies) { + File saveFileAs; + try { + saveFileAs = getSaveAsFile(url, prefix, subdirectory); + } catch (IOException e) { + logger.error("[!] Error creating save file path for URL '" + url + "':", e); + return false; + } + return addURLToDownload(url, saveFileAs, referrer, cookies); + } + @Override public boolean addURLToDownload(URL url, File saveAs) { return addURLToDownload(url, saveAs, null, null); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java index c6f9cab54..f96bc01bc 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java @@ -3,6 +3,8 @@ import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; +import java.util.HashMap; +import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -17,6 +19,8 @@ public class XhamsterRipper extends AlbumRipper { private static final String HOST = "xhamster"; + private HashMap docs = new HashMap(); + public XhamsterRipper(URL url) throws IOException { super(url); } @@ -39,7 +43,7 @@ public void rip() throws IOException { String nextURL = this.url.toExternalForm(); while (nextURL != null) { logger.info(" Retrieving " + nextURL); - Document doc = Http.url(nextURL).get(); + Document doc = Http.url(nextURL).header("User-Agent", USER_AGENT).referrer("http://" + HOST + ".com/").cookies(Utils.getCookies(HOST)).get(); for (Element thumb : doc.select("table.iListing div.img img")) { if (!thumb.hasAttr("src")) { continue; @@ -73,6 +77,36 @@ public void rip() throws IOException { waitForThreads(); } + /** + * @todo prefix with uploader username + */ + @Override + public String getAlbumTitle(URL url) throws MalformedURLException { + String title = HOST + "_"; + Document doc = docs.get(url);; + if (doc == null) { + try { + doc = Http.url(url).header("User-Agent", USER_AGENT).referrer(url).cookies(Utils.getCookies(HOST)).get(); + docs.put(url.toString(), doc); + } catch (IOException e) { + logger.error("Failed to download url=" + url + ": " + e.getMessage()); + } + } + // Find username. + if (doc != null) { + for (Element link : doc.select("#galleryUser .item a")) { + title += link.text() + "_"; + break; + } + } + String galleryLink = url.toExternalForm(); + title += galleryLink + .replaceFirst("^http.*/photos/(?:gallery/([^?#:&]+)|view/([^-]+)-).*$", "$1$2") + .replace('/', '-') + .replace(".html", ""); + return title; + } + @Override public String getHost() { return HOST; @@ -80,15 +114,14 @@ public String getHost() { @Override public String getGID(URL url) throws MalformedURLException { - Pattern p = Pattern.compile("^https?://([a-z0-9.]*?)xhamster\\.com/photos/gallery/([0-9]{1,})/.*\\.html"); - Matcher m = p.matcher(url.toExternalForm()); - if (m.matches()) { - return m.group(2); - } - throw new MalformedURLException( + String gid = url.toExternalForm().replaceFirst("^https?://(?:[a-z0-9.]*?)" + HOST + "\\.com/photos/(?:gallery/([0-9]{1,})/.*\\.html|view/([^-]+)-).*$", "$1$2"); + if (gid.length() == 0) { + throw new MalformedURLException( "Expected xhamster.com gallery formats: " - + "xhamster.com/photos/gallery/#####/xxxxx..html" - + " Got: " + url); + + "http://xhamster.com/photos/gallery/#####/xxxxx..html or http://xhamster.com/photos/view/####-####.html" + + " Got: " + url); + } + return gid; } } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/XhamsterRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/video/XhamsterRipper.java index cb6d57257..f184d1681 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/video/XhamsterRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/video/XhamsterRipper.java @@ -6,6 +6,7 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import com.rarchives.ripme.utils.Utils; import org.jsoup.nodes.Document; import org.jsoup.select.Elements; @@ -39,28 +40,27 @@ public URL sanitizeURL(URL url) throws MalformedURLException { @Override public String getGID(URL url) throws MalformedURLException { - Pattern p = Pattern.compile("^https?://.*xhamster\\.com/movies/([0-9]+).*$"); - Matcher m = p.matcher(url.toExternalForm()); - if (m.matches()) { - return m.group(1); - } - - throw new MalformedURLException( + String gid = url.toExternalForm().replaceFirst("^https?://.*" + HOST + "\\.com/movies/([0-9]+)/.*$", "$1"); + if (gid.length() == 0) { + throw new MalformedURLException( "Expected xhamster format:" - + "xhamster.com/movies/####" - + " Got: " + url); + + "xhamster.com/movies/####" + + " Got: " + url); + + } + return gid; } @Override public void rip() throws IOException { - logger.info("Retrieving " + this.url); - Document doc = Http.url(url).get(); + logger.info("Retrieving " + url); + Document doc = Http.url(url).header("User-Agent", USER_AGENT).referrer("http://" + HOST + ".com/").cookies(Utils.getCookies(HOST)).get(); Elements videos = doc.select("a.mp4Thumb"); if (videos.size() == 0) { throw new IOException("Could not find Embed code at " + url); } String vidUrl = videos.attr("href"); - addURLToDownload(new URL(vidUrl), HOST + "_" + getGID(this.url)); + addURLToDownload(new URL(vidUrl), HOST + "_" + getGID(this.url), "", url.toExternalForm(), Utils.getCookies(HOST)); waitForThreads(); } -} \ No newline at end of file +} diff --git a/src/main/java/com/rarchives/ripme/utils/Utils.java b/src/main/java/com/rarchives/ripme/utils/Utils.java index 946fce549..b223d194a 100644 --- a/src/main/java/com/rarchives/ripme/utils/Utils.java +++ b/src/main/java/com/rarchives/ripme/utils/Utils.java @@ -9,7 +9,9 @@ import java.net.URLDecoder; import java.util.ArrayList; import java.util.Enumeration; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.jar.JarEntry; import java.util.jar.JarFile; @@ -71,6 +73,11 @@ public class Utils { } } + private static HashMap> cookieCache; + static { + cookieCache = new HashMap>(); + } + /** * Get the root rips directory. * @return @@ -387,4 +394,21 @@ public static List between(String fullText, String start, String finish) } return result; } + + public static Map getCookies(String host) { + HashMap domainCookies = cookieCache.get(host); + if (domainCookies == null) { + domainCookies = new HashMap(); + String cookiesConfig = getConfigString("cookies." + host, ""); + for (String pair : cookiesConfig.split(" ")) { + pair = pair.trim(); + if (pair.contains("=")) { + String[] pieces = pair.split("=", 2); + domainCookies.put(pieces[0], pieces[1]); + } + } + cookieCache.put(host, domainCookies); + } + return domainCookies; + } } diff --git a/src/main/resources/rip.properties b/src/main/resources/rip.properties index 1a0ff01c7..a92d99f75 100644 --- a/src/main/resources/rip.properties +++ b/src/main/resources/rip.properties @@ -30,3 +30,7 @@ twitter.max_requests = 10 clipboard.autorip = false download.save_order = true + +cookies.xhamster = +# e.g. cookies.xhamster = USERNAME=sleaze UID=69696969 PWD=144354bc90792a91957df1ef962908c1 fingerprint=d65f704a8fef31b5327175e00f1eeb85 +