Handle failed navigation

Author: karust

baidu/search.go

@@ -74,7 +74,10 @@ func (baid *Baidu) Search(query core.Query) ([]core.SearchResult, error) {
 		return nil, err
 	}
 
-	page := baid.Navigate(url)
+	page, err := baid.Navigate(url)
+	if err != nil {
+		return nil, err
+	}
 
 	results, err := page.Timeout(baid.Timeout).Search("div.c-container.new-pmd")
 	if err != nil {
@@ -154,7 +157,11 @@ func (baid *Baidu) SearchImage(query core.Query) ([]core.SearchResult, error) {
 		}
 
 		// Get anti-crawler cookies first, then reload page
-		page := baid.Navigate(url)
+		page, err := baid.Navigate(url)
+		if err != nil {
+			return nil, err
+		}
+
 		if !baid.LeavePageOpen {
 			defer page.Close()
 		}

core/browser.go

@@ -97,7 +97,7 @@ func (b *Browser) IsInitialized() bool {
 }
 
 // Open URL
-func (b *Browser) Navigate(URL string) *rod.Page {
+func (b *Browser) Navigate(URL string) (*rod.Page, error) {
 	logrus.Debug("Navigate to: ", URL)
 
 	b.browser = rod.New().ControlURL(b.browserAddr)
@@ -127,15 +127,19 @@ func (b *Browser) Navigate(URL string) *rod.Page {
 	page.MustEmulate(devices.Device{
 		AcceptLanguage: b.LanguageCode,
 	})
-	page.MustNavigate(URL)
+
+	err := page.Navigate(URL)
+	if err != nil {
+		return nil, err
+	}
 
 	wait := page.MustWaitRequestIdle()
 	// may cause bugs with google
 	if b.WaitRequests {
 		wait()
 	}
 
-	return page
+	return page, nil
 }
 
 func (b *Browser) Close() error {

core/browser_test.go

@@ -38,13 +38,12 @@ func TestBot(t *testing.T) {
 		t.Fatalf("Error failed initializing browser: %s", err)
 	}
 
-	page := browser.Navigate("https://bot.sannysoft.com")
+	page, _ := browser.Navigate("https://bot.sannysoft.com")
 	page.MustScreenshotFullPage("./test/screenshot_bot.png")
 
-	page = browser.Navigate("https://www.whatismybrowser.com/")
+	page, _ = browser.Navigate("https://www.whatismybrowser.com/")
 	page.MustScreenshotFullPage("./test/screenshot_browser.png")
 
-	page = browser.Navigate("https://abrahamjuliot.github.io/creepjs/")
+	page, _ = browser.Navigate("https://abrahamjuliot.github.io/creepjs/")
 	page.MustScreenshotFullPage("./test/screenshot_creep.png")
-
 }

google/search.go

@@ -70,7 +70,7 @@ func (gogl *Google) getTotalResults(page *rod.Page) (int, error) {
 func (gogl *Google) solveCaptcha(page *rod.Page, sitekey, datas string) bool {
 	logrus.Debugf("Solve google Captcha: sitekey=%s, datas=%s, url=%s", sitekey, datas, page.MustInfo().URL)
 
-	resp, err := gogl.CaptchaSolver.SolveReCaptcha2(sitekey, page.MustInfo().URL, datas)
+	resp, err := gogl.CaptchaSolver.SolveReCaptcha2(sitekey, page.MustInfo().URL)
 	if err != nil {
 		logrus.Errorf("Error solving google captcha: %s", err)
 		return false
@@ -143,7 +143,11 @@ func (gogl *Google) Search(query core.Query) ([]core.SearchResult, error) {
 	if err != nil {
 		return nil, err
 	}
-	page := gogl.Navigate(url)
+	page, err := gogl.Navigate(url)
+	if err != nil {
+		return nil, err
+	}
+
 	defer gogl.close(page)
 	gogl.preparePage(page)
 
@@ -338,7 +342,11 @@ func (gogl *Google) SearchImage(query core.Query) ([]core.SearchResult, error) {
 		return nil, err
 	}
 
-	page := gogl.Navigate(url)
+	page, err := gogl.Navigate(url)
+	if err != nil {
+		return nil, err
+	}
+
 	defer gogl.close(page)
 
 	for len(searchResultsMap) < query.Limit {

yandex/search.go

@@ -124,7 +124,10 @@ func (yand *Yandex) Search(query core.Query) ([]core.SearchResult, error) {
 			return nil, err
 		}
 
-		page := yand.Navigate(url)
+		page, err := yand.Navigate(url)
+		if err != nil {
+			return nil, err
+		}
 
 		// Get all search results in page
 		searchRes, err := page.Timeout(yand.Timeout).Search("li.serp-item")
@@ -185,7 +188,10 @@ func (yand *Yandex) SearchImage(query core.Query) ([]core.SearchResult, error) {
 		}
 		searchPage += 1
 
-		page := yand.Navigate(url)
+		page, err := yand.Navigate(url)
+		if err != nil {
+			return nil, err
+		}
 
 		if !yand.LeavePageOpen {
 			defer page.Close()