Merge pull request #29 from planetlabs/retry-more

Retry on connection reset errors

Author: tschaub

crawler/crawler.go

@@ -4,16 +4,22 @@ package crawler
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io/ioutil"
+	"math"
+	"math/rand"
 	"net/http"
 	"net/url"
 	"path/filepath"
 	"runtime"
 	"strings"
 	"sync"
+	"syscall"
+	"time"
 
 	"github.com/hashicorp/go-retryablehttp"
+	"github.com/tschaub/retry"
 	"github.com/tschaub/workgroup"
 )
 
@@ -71,6 +77,29 @@ func loadFile(resourcePath string, value any) error {
 }
 
 func loadUrl(resourceUrl string, value any) error {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Minute*30)
+	defer cancel()
+
+	retries := 5
+
+	return retry.Limit(ctx, retries, func(ctx context.Context, attempt int) error {
+		err := tryLoadUrl(resourceUrl, value)
+		if err == nil {
+			return nil
+		}
+
+		// these come when parsing the response body
+		if !errors.Is(err, syscall.ECONNRESET) {
+			return retry.Stop(err)
+		}
+
+		jitter := time.Duration(rand.Float64()) * time.Second
+		time.Sleep(time.Second*time.Duration(math.Pow(2, float64(attempt))) + jitter)
+		return err
+	})
+}
+
+func tryLoadUrl(resourceUrl string, value any) error {
 	resp, err := httpClient.Get(resourceUrl)
 	if err != nil {
 		return err

go.mod

@@ -10,6 +10,7 @@ require (
 	github.com/santhosh-tekuri/jsonschema/v5 v5.0.0
 	github.com/schollz/progressbar/v3 v3.8.6
 	github.com/stretchr/testify v1.7.1
+	github.com/tschaub/retry v1.0.0
 	github.com/tschaub/workgroup v0.2.0
 	github.com/urfave/cli/v2 v2.4.0
 	go.uber.org/zap v1.21.0

go.sum

@@ -45,13 +45,16 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/tschaub/jsonschema/v5 v5.1.0-beta.1 h1:yuHeTkhFAApxLRdWXFEeArU0iMIBJm27f1/9t41Gf5A=
 github.com/tschaub/jsonschema/v5 v5.1.0-beta.1/go.mod h1:FKdcjfQW6rpZSnxxUvEA5H/cDPdvJ/SZJQLWWXWGrZ0=
 github.com/tschaub/limited v0.2.0 h1:XBjhp4JNhRruzRlU4/RCLGxxSmRPO+rJelpB5EVZYJU=
 github.com/tschaub/limited v0.2.0/go.mod h1:MpOH3bHLpfI+6MO7P1DulWY4aXXqKdkUn8v4eMm4xOI=
+github.com/tschaub/retry v1.0.0 h1:zY9jeVNPxv2UovkFikgYOpvT5q52/3KQuJrY70MK+ws=
+github.com/tschaub/retry v1.0.0/go.mod h1:yBQOt5++ZYLg8LqzLICIFMmiG7WM45CYCnemriP95T8=
 github.com/tschaub/workgroup v0.2.0 h1:llc29/O3boFokyJkPWL3SFifcnQeD0kIcPtrmtsM0Nw=
 github.com/tschaub/workgroup v0.2.0/go.mod h1:UEtjKJmK1+hsGa6zC1sSHC6iZh8GmzesqybVruhdGX8=
 github.com/urfave/cli/v2 v2.4.0 h1:m2pxjjDFgDxSPtO8WSdbndj17Wu2y8vOT86wE/tjr+I=