Merge pull request #297 from s17t/ioc_modularization

#145, see details

Author: s17t

crawler4j/src/main/java/edu/uci/ics/crawler4j/crawler/Configurable.java

@@ -21,8 +21,10 @@
  * Several core components of crawler4j extend this class
  * to make them configurable.
  *
+ * @deprecated This will removed without notice.
  * @author Yasser Ganjisaffar
  */
+@Deprecated
 public abstract class Configurable {
 
     protected CrawlConfig config;

crawler4j/src/main/java/edu/uci/ics/crawler4j/crawler/CrawlController.java

@@ -43,9 +43,10 @@
  *
  * @author Yasser Ganjisaffar
  */
-public class CrawlController extends Configurable {
+public class CrawlController {
 
     static final Logger logger = LoggerFactory.getLogger(CrawlController.class);
+    private final CrawlConfig config;
 
     /**
      * The 'customData' object can be used for passing custom crawl-related
@@ -80,9 +81,9 @@ public class CrawlController extends Configurable {
 
     public CrawlController(CrawlConfig config, PageFetcher pageFetcher,
                            RobotstxtServer robotstxtServer) throws Exception {
-        super(config);
-
         config.validate();
+        this.config = config;
+
         File folder = new File(config.getCrawlStorageFolder());
         if (!folder.exists()) {
             if (folder.mkdirs()) {
@@ -233,8 +234,6 @@ protected <T extends WebCrawler> void start(final WebCrawlerFactory<T> crawlerFa
             }
 
             final CrawlController controller = this;
-            final CrawlConfig config = this.getConfig();
-
             Thread monitorThread = new Thread(new Runnable() {
 
                 @Override
@@ -547,4 +546,8 @@ public void shutdown() {
         pageFetcher.shutDown();
         frontier.finish();
     }
+
+    public CrawlConfig getConfig() {
+        return config;
+    }
 }

crawler4j/src/main/java/edu/uci/ics/crawler4j/fetcher/PageFetcher.java

@@ -61,7 +61,6 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import edu.uci.ics.crawler4j.crawler.Configurable;
 import edu.uci.ics.crawler4j.crawler.CrawlConfig;
 import edu.uci.ics.crawler4j.crawler.authentication.AuthInfo;
 import edu.uci.ics.crawler4j.crawler.authentication.BasicAuthInfo;
@@ -74,16 +73,21 @@
 /**
  * @author Yasser Ganjisaffar
  */
-public class PageFetcher extends Configurable {
+public class PageFetcher {
     protected static final Logger logger = LoggerFactory.getLogger(PageFetcher.class);
     protected final Object mutex = new Object();
+    /**
+     * This field is protected for retro compatibility. Please use the getter method: getConfig() to
+     * read this field;
+     */
+    protected final CrawlConfig config;
     protected PoolingHttpClientConnectionManager connectionManager;
     protected CloseableHttpClient httpClient;
     protected long lastFetchTime = 0;
     protected IdleConnectionMonitorThread connectionMonitorThread = null;
 
     public PageFetcher(CrawlConfig config) {
-        super(config);
+        this.config = config;
 
         RequestConfig requestConfig = RequestConfig.custom()
                 .setExpectContinueEnabled(false)
@@ -336,4 +340,7 @@ protected HttpUriRequest newHttpUriRequest(String url) {
         return new HttpGet(url);
     }
 
+    protected CrawlConfig getConfig() {
+        return config;
+    }
 }

crawler4j/src/main/java/edu/uci/ics/crawler4j/frontier/Counters.java

@@ -32,14 +32,13 @@
 import com.sleepycat.je.OperationStatus;
 import com.sleepycat.je.Transaction;
 
-import edu.uci.ics.crawler4j.crawler.Configurable;
 import edu.uci.ics.crawler4j.crawler.CrawlConfig;
 import edu.uci.ics.crawler4j.util.Util;
 
 /**
  * @author Yasser Ganjisaffar
  */
-public class Counters extends Configurable {
+public class Counters {
     private static final Logger logger = LoggerFactory.getLogger(Counters.class);
 
     public static class ReservedCounterNames {
@@ -56,8 +55,6 @@ public static class ReservedCounterNames {
     protected Map<String, Long> counterValues;
 
     public Counters(Environment env, CrawlConfig config) {
-        super(config);
-
         this.env = env;
         this.counterValues = new HashMap<>();
 

crawler4j/src/main/java/edu/uci/ics/crawler4j/frontier/DocIDServer.java

@@ -27,15 +27,14 @@
 import com.sleepycat.je.Environment;
 import com.sleepycat.je.OperationStatus;
 
-import edu.uci.ics.crawler4j.crawler.Configurable;
 import edu.uci.ics.crawler4j.crawler.CrawlConfig;
 import edu.uci.ics.crawler4j.util.Util;
 
 /**
  * @author Yasser Ganjisaffar
  */
 
-public class DocIDServer extends Configurable {
+public class DocIDServer {
     private static final Logger logger = LoggerFactory.getLogger(DocIDServer.class);
 
     private final Database docIDsDB;
@@ -46,7 +45,6 @@ public class DocIDServer extends Configurable {
     private int lastDocID;
 
     public DocIDServer(Environment env, CrawlConfig config) {
-        super(config);
         DatabaseConfig dbConfig = new DatabaseConfig();
         dbConfig.setAllowCreate(true);
         dbConfig.setTransactional(config.isResumableCrawling());

crawler4j/src/main/java/edu/uci/ics/crawler4j/frontier/Frontier.java

@@ -25,19 +25,19 @@
 import com.sleepycat.je.DatabaseException;
 import com.sleepycat.je.Environment;
 
-import edu.uci.ics.crawler4j.crawler.Configurable;
 import edu.uci.ics.crawler4j.crawler.CrawlConfig;
 import edu.uci.ics.crawler4j.url.WebURL;
 
 /**
  * @author Yasser Ganjisaffar
  */
 
-public class Frontier extends Configurable {
+public class Frontier {
     protected static final Logger logger = LoggerFactory.getLogger(Frontier.class);
 
     private static final String DATABASE_NAME = "PendingURLsDB";
     private static final int IN_PROCESS_RESCHEDULE_BATCH_SIZE = 100;
+    private final CrawlConfig config;
     protected WorkQueues workQueues;
 
     protected InProcessPagesDB inProcessPages;
@@ -52,7 +52,7 @@ public class Frontier extends Configurable {
     protected Counters counters;
 
     public Frontier(Environment env, CrawlConfig config) {
-        super(config);
+        this.config = config;
         this.counters = new Counters(env, config);
         try {
             workQueues = new WorkQueues(env, DATABASE_NAME, config.isResumableCrawling());

crawler4j/src/main/java/edu/uci/ics/crawler4j/parser/Parser.java

@@ -35,7 +35,6 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import edu.uci.ics.crawler4j.crawler.Configurable;
 import edu.uci.ics.crawler4j.crawler.CrawlConfig;
 import edu.uci.ics.crawler4j.crawler.Page;
 import edu.uci.ics.crawler4j.crawler.exceptions.ParseException;
@@ -47,15 +46,16 @@
 /**
  * @author Yasser Ganjisaffar
  */
-public class Parser extends Configurable {
+public class Parser {
 
     protected static final Logger logger = LoggerFactory.getLogger(Parser.class);
 
     private final HtmlParser htmlParser;
     private final ParseContext parseContext;
+    private final CrawlConfig config;
 
     public Parser(CrawlConfig config) throws InstantiationException, IllegalAccessException {
-        super(config);
+        this.config = config;
         htmlParser = new HtmlParser();
         parseContext = new ParseContext();
         parseContext.set(HtmlMapper.class, AllTagMapper.class.newInstance());

crawler4j/src/test/java/edu/uci/ics/crawler4j/tests/fetcher/PageFetcherHtmlOnly.java

@@ -28,8 +28,8 @@ public PageFetchResult fetchPage(WebURL webUrl)
 
             synchronized (mutex) {
                 long now = new Date().getTime();
-                if (now - this.lastFetchTime < this.config.getPolitenessDelay()) {
-                    Thread.sleep(this.config.getPolitenessDelay() - (now - this.lastFetchTime));
+                if (now - this.lastFetchTime < getConfig().getPolitenessDelay()) {
+                    Thread.sleep(getConfig().getPolitenessDelay() - (now - this.lastFetchTime));
                 }
                 this.lastFetchTime = new Date().getTime();
             }