Kemono ripper and danbooru fix

src/main/java/com/rarchives/ripme/App.java

@@ -1,5 +1,6 @@
 package com.rarchives.ripme;
 
+import com.oracle.truffle.regex.tregex.util.json.JsonArray;
 import com.rarchives.ripme.ripper.AbstractRipper;
 import com.rarchives.ripme.ui.History;
 import com.rarchives.ripme.ui.HistoryEntry;
@@ -17,6 +18,8 @@
 import org.apache.commons.lang.SystemUtils;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.json.JSONArray;
+import org.json.JSONObject;
 
 import javax.swing.*;
 import java.awt.*;
@@ -354,8 +357,23 @@ private static void loadHistory() throws IOException {
         HISTORY.clear();
         if (Files.exists(historyFile)) {
             try {
-                logger.info("Loading history from " + historyFile);
+                logger.info("Loading history from file: " + historyFile);
                 HISTORY.fromFile(historyFile.toString());
+//                JSONArray json = HISTORY.toJSON();
+//                JSONArray rebuildArray = new JSONArray();
+//                for (Object currentJsonItem : json) {
+//                    JSONObject jsonData = (JSONObject) currentJsonItem;
+//                    if(jsonData.has("url")){
+//                        String url = jsonData.getString("url");
+//                        if(url.contains("reddit")){
+//                            rebuildArray.put(jsonData);
+//                        }
+//                    }
+//                }
+//                HISTORY.clear();
+//                HISTORY.fromJSON(rebuildArray);
+//                  HISTORY.toFile(historyFile.toString());
+
             } catch (IOException e) {
                 logger.error("Failed to load history from file " + historyFile, e);
                 logger.warn(

src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java

@@ -354,16 +354,18 @@ protected boolean addURLToDownload(URL url, String subdirectory, String referrer
             return false;
         }
 
-        if (Utils.getConfigBoolean("remember.url_history", true) && !isThisATest()) {
-            logger.info("Writing " + url.toExternalForm() + " to file");
-            try {
-                writeDownloadedURL(url.toExternalForm() + "\n");
-            } catch (IOException e) {
-                logger.debug("Unable to write URL history file");
+        boolean fileDownloadStatus = addURLToDownload(url, saveAs, referrer, cookies, getFileExtFromMIME);
+        if(fileDownloadStatus){
+            if (Utils.getConfigBoolean("remember.url_history", true) && !isThisATest()) {
+                logger.info("Writing " + url.toExternalForm() + " to file");
+                try {
+                    writeDownloadedURL(url.toExternalForm() + "\n");
+                } catch (IOException e) {
+                    logger.debug("Unable to write URL history file");
+                }
             }
         }
-
-        return addURLToDownload(url, saveAs, referrer, cookies, getFileExtFromMIME);
+        return fileDownloadStatus;
     }
 
     protected boolean addURLToDownload(URL url, String prefix, String subdirectory, String referrer,

src/main/java/com/rarchives/ripme/ripper/rippers/DanbooruRipper.java

@@ -6,7 +6,9 @@
 import java.net.URISyntaxException;
 import java.net.URL;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.concurrent.TimeUnit;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
@@ -24,6 +26,8 @@
 import okhttp3.Request;
 import okhttp3.Response;
 
+import static com.rarchives.ripme.utils.RipUtils.getCookiesFromString;
+
 public class DanbooruRipper extends AbstractJSONRipper {
 
     private static final Logger logger = LogManager.getLogger(DanbooruRipper.class);
@@ -35,6 +39,8 @@ public class DanbooruRipper extends AbstractJSONRipper {
     private Pattern gidPattern = null;
 
     private int currentPageNum = 1;
+    private Map<String,String> cookies = new HashMap<>();
+
 
     public DanbooruRipper(URL url) throws IOException {
         super(url);
@@ -44,6 +50,15 @@ public DanbooruRipper(URL url) throws IOException {
                 .build();
     }
 
+    private void setCookies(String currentCookie) {
+        if (Utils.getConfigBoolean("danbooru.login", true)) {
+            logger.info("Logging in using cookies");
+//            String faCookies = Utils.getConfigString("danbooru.cookies", currentCookie);
+            String[] cookieInfo = currentCookie.split("=");
+            cookies.put(cookieInfo[0], cookieInfo[1]);
+        }
+    }
+
     @Override
     protected String getDomain() {
         return DOMAIN;
@@ -68,6 +83,43 @@ protected JSONObject getNextPage(JSONObject doc) throws IOException {
         return getCurrentPage();
     }
 
+    private String getSubUrlFromUrl(String url){
+        Request request = new Request.Builder()
+                .url(url)
+                .header("User-Agent", "Mozilla/5.0 (iPhone; CPU iPhone OS 15_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.0 Mobile/15E148 Safari/604.1")
+                .header("Accept", "application/json,text/javascript,*/*;q=0.01")
+                .header("Accept-Language", "en-US,en;q=0.9")
+                .header("Sec-Fetch-Dest", "empty")
+                .header("Sec-Fetch-Mode", "cors")
+                .header("Sec-Fetch-Site", "same-origin")
+                .header("Referer", "https://danbooru.donmai.us/")
+                .header("X-Requested-With", "XMLHttpRequest")
+                .header("Connection", "keep-alive")
+                .build();
+        Response response = null;
+        try {
+            response = client.newCall(request).execute();
+            if (!response.isSuccessful()) throw new IOException("Unexpected code " + response);
+
+            String responseData = response.body().string();
+
+            JSONObject json = new JSONObject(responseData);
+            if(json.has("file_url")){
+                return json.getString("file_url");
+            }else{
+                return null;
+            }
+
+        } catch (Exception e) {
+            e.printStackTrace();
+        } finally {
+            if(response !=null) {
+                response.body().close();
+            }
+        }
+        return null;
+    }
+
     @Nullable
     private JSONObject getCurrentPage() throws MalformedURLException {
         Request request = new Request.Builder()
@@ -87,14 +139,27 @@ private JSONObject getCurrentPage() throws MalformedURLException {
         try {
             response = client.newCall(request).execute();
             if (!response.isSuccessful()) throw new IOException("Unexpected code " + response);
-
+            if(response.headers().get("set-cookie") != null) {
+                setCookies(response.headers().get("set-cookie"));
+            }
             String responseData = response.body().string();
             JSONArray jsonArray = new JSONArray(responseData);
             if(!jsonArray.isEmpty()){
-                String newCompatibleJSON = "{ \"resources\":" + jsonArray + " }";
+                JSONArray resultsToUse = new JSONArray();
+                for(Object currentObject : jsonArray){
+                    JSONObject currentJSONObject = (JSONObject) currentObject;
+                    //tag_string_artist
+                    //https://danbooru.donmai.us/posts/6677739?q=jagerctm1
+                    JSONObject objectToAdd = new JSONObject();
+                    objectToAdd.put("file_url", getSubUrlFromUrl("https://danbooru.donmai.us/posts/" + currentJSONObject.get("id") + "?q="+ currentJSONObject.getString("tag_string_artist") + "&z=1"));
+                    //"https://danbooru.donmai.us/posts/" + currentJSONObject.get("id") + "?q="+ currentJSONObject.getString("tag_string_artist") + "&z=1"
+
+                    resultsToUse.put(objectToAdd);
+                }
+                String newCompatibleJSON = "{ \"resources\":" + resultsToUse + " }";
                 return new JSONObject(newCompatibleJSON);
             }
-        } catch (IOException e) {
+        } catch (Exception e) {
             e.printStackTrace();
         } finally {
             if(response !=null) {
@@ -129,7 +194,8 @@ public String getGID(URL url) throws MalformedURLException {
 
     @Override
     protected void downloadURL(URL url, int index) {
-        addURLToDownload(url, getPrefix(index));
+//        addURLToDownload(url, getPrefix(index));
+        addURLToDownload(url, getPrefix(index), "", "https://danbooru.donmai.us", cookies);
     }
 
     private String getTag(URL url) throws MalformedURLException {

src/main/java/com/rarchives/ripme/ripper/rippers/FileFinishCallBack.java

@@ -0,0 +1,9 @@
+package com.rarchives.ripme.ripper.rippers;
+
+import java.net.URL;
+
+public interface FileFinishCallBack {
+    void onComplete(String result, URL currentUrl);
+
+    void onFail(String result, URL currentUrl);
+}