Merge yasserg#155 into refactored codebase

Author: pgalbraith

crawler4j/src/main/java/edu/uci/ics/crawler4j/crawler/CrawlController.java

@@ -298,7 +298,7 @@ public void run() {
                                     if (!someoneIsWorking) {
                                         if (!shuttingDown) {
                                             long queueLength = frontier.getQueueLength();
-                                            if (queueLength > 0) {
+                                            if (! frontier.isFinished() && queueLength > 0) {
                                                 continue;
                                             }
                                             logger.info(
@@ -308,7 +308,7 @@ public void run() {
                                                 " seconds to make sure...");
                                             sleep(config.getThreadShutdownDelaySeconds());
                                             queueLength = frontier.getQueueLength();
-                                            if (queueLength > 0) {
+                                            if (! frontier.isFinished() && queueLength > 0) {
                                                 continue;
                                             }
                                         }

crawler4j/src/main/java/edu/uci/ics/crawler4j/frontier/Frontier.java

@@ -48,6 +48,7 @@ public class Frontier {
     protected boolean isFinished = false;
 
     protected long scheduledPages;
+    protected long fetchedPages = 0;
 
     protected Counters counters;
 
@@ -83,7 +84,7 @@ public Frontier(Environment env, CrawlConfig config) {
     }
 
     public void scheduleAll(List<WebURL> urls) {
-        int maxPagesToFetch = config.getMaxPagesToFetch();
+        int maxPagesToFetch = -1; //config.getMaxPagesToFetch();
         synchronized (mutex) {
             int newScheduledPage = 0;
             for (WebURL url : urls) {
@@ -110,7 +111,7 @@ public void scheduleAll(List<WebURL> urls) {
     }
 
     public void schedule(WebURL url) {
-        int maxPagesToFetch = config.getMaxPagesToFetch();
+        int maxPagesToFetch = -1; //config.getMaxPagesToFetch();
         synchronized (mutex) {
             try {
                 if (maxPagesToFetch < 0 || scheduledPages < maxPagesToFetch) {
@@ -125,8 +126,11 @@ public void schedule(WebURL url) {
     }
 
     public void getNextURLs(int max, List<WebURL> result) {
+        int maxPagesToFetch = config.getMaxPagesToFetch();
         while (true) {
             synchronized (mutex) {
+                if (maxPagesToFetch > 0 && fetchedPages > maxPagesToFetch)
+                    finish();
                 if (isFinished) {
                     return;
                 }
@@ -162,6 +166,7 @@ public void getNextURLs(int max, List<WebURL> result) {
     }
 
     public void setProcessed(WebURL webURL) {
+        fetchedPages++;
         counters.increment(Counters.ReservedCounterNames.PROCESSED_PAGES);
         if (inProcessPages != null) {
             if (!inProcessPages.removeURL(webURL)) {