Merge pull request gocolly#826 from Shinku-Chen/retry

fix : retry redirect to AlreadyVisitedUrl will  error, 修正Redirect的retry报AlreadyVisitedError错误

Author: asciimoo

colly.go

@@ -209,7 +209,10 @@ var collectorCounter uint32
 type key int
 
 // ProxyURLKey is the context key for the request proxy address.
-const ProxyURLKey key = iota
+const (
+	ProxyURLKey key = iota
+	CheckRevisitKey
+)
 
 var (
 	// ErrForbiddenDomain is the error thrown if visiting
@@ -667,7 +670,8 @@ func (c *Collector) scrape(u, method string, depth int, requestData io.Reader, c
 	}
 	// note: once 1.13 is minimum supported Go version,
 	// replace this with http.NewRequestWithContext
-	req = req.WithContext(c.Context)
+	req = req.WithContext(context.WithValue(c.Context, CheckRevisitKey, checkRevisit))
+
 	if err := c.requestCheck(parsedURL, method, req.GetBody, depth, checkRevisit); err != nil {
 		return err
 	}
@@ -1477,7 +1481,9 @@ func (c *Collector) checkRedirectFunc() func(req *http.Request, via []*http.Requ
 				return err
 			}
 			if visited {
-				return &AlreadyVisitedError{req.URL}
+				if checkRevisit, ok := req.Context().Value(CheckRevisitKey).(bool); !ok || checkRevisit {
+					return &AlreadyVisitedError{req.URL}
+				}
 			}
 			err = c.store.Visited(uHash)
 			if err != nil {

colly_test.go

@@ -1885,6 +1885,28 @@ func TestCollectorPostRetryUnseekable(t *testing.T) {
 	}
 }
 
+func TestRedirectErrorRetry(t *testing.T) {
+	ts := newTestServer()
+	defer ts.Close()
+	c := NewCollector()
+	c.OnError(func(r *Response, err error) {
+		if r.Ctx.Get("notFirst") == "" {
+			r.Ctx.Put("notFirst", "first")
+			_ = r.Request.Retry()
+			return
+		}
+		if e := (&AlreadyVisitedError{}); errors.As(err, &e) {
+			t.Error("loop AlreadyVisitedError")
+		}
+
+	})
+	c.OnResponse(func(response *Response) {
+		//println(1)
+	})
+	c.Visit(ts.URL + "/redirected/")
+	c.Visit(ts.URL + "/redirect")
+}
+
 func TestCheckRequestHeadersFunc(t *testing.T) {
 	ts := newTestServer()
 	defer ts.Close()