@@ -3,12 +3,16 @@ package crawler_test
33import (
44 "context"
55 "fmt"
6+ "net/http"
7+ "net/http/httptest"
8+ "strings"
69 "sync"
710 "sync/atomic"
811 "testing"
912
1013 "github.com/planetlabs/go-stac/crawler"
1114 "github.com/stretchr/testify/assert"
15+ "github.com/stretchr/testify/require"
1216)
1317
1418func TestCrawler (t * testing.T ) {
@@ -31,6 +35,71 @@ func TestCrawler(t *testing.T) {
3135 assert .Equal (t , uint64 (3 ), count )
3236}
3337
38+ func TestCrawlerHTTP (t * testing.T ) {
39+ server := httptest .NewServer (http .FileServer (http .Dir ("testdata" )))
40+ defer server .Close ()
41+
42+ count := uint64 (0 )
43+ visited := & sync.Map {}
44+
45+ visitor := func (location string , resource crawler.Resource ) error {
46+ atomic .AddUint64 (& count , 1 )
47+ _ , loaded := visited .LoadOrStore (location , true )
48+ if loaded {
49+ return fmt .Errorf ("already visited %s" , location )
50+ }
51+ return nil
52+ }
53+ c := crawler .New (visitor )
54+
55+ err := c .Crawl (context .Background (), server .URL + "/v1.0.0/catalog-with-collection-of-items.json" )
56+ assert .NoError (t , err )
57+
58+ assert .Equal (t , uint64 (3 ), count )
59+
60+ _ , visitedCatalog := visited .Load (server .URL + "/v1.0.0/catalog-with-collection-of-items.json" )
61+ assert .True (t , visitedCatalog )
62+
63+ _ , visitedCollection := visited .Load (server .URL + "/v1.0.0/collection-with-items.json" )
64+ assert .True (t , visitedCollection )
65+
66+ _ , visitedItem := visited .Load (server .URL + "/v1.0.0/item-in-collection.json" )
67+ assert .True (t , visitedItem )
68+ }
69+
70+ func TestCrawlerHTTPRetry (t * testing.T ) {
71+
72+ tried := false
73+ server := httptest .NewServer (http .HandlerFunc (func (w http.ResponseWriter , r * http.Request ) {
74+ if ! tried {
75+ tried = true
76+ w .WriteHeader (http .StatusInternalServerError )
77+ return
78+ }
79+ w .WriteHeader (http .StatusNotImplemented ) // stop trying
80+ }))
81+ defer server .Close ()
82+
83+ count := uint64 (0 )
84+ visited := & sync.Map {}
85+ visitor := func (location string , resource crawler.Resource ) error {
86+ atomic .AddUint64 (& count , 1 )
87+ _ , loaded := visited .LoadOrStore (location , true )
88+ if loaded {
89+ return fmt .Errorf ("already visited %s" , location )
90+ }
91+ return nil
92+ }
93+ c := crawler .New (visitor )
94+
95+ err := c .Crawl (context .Background (), server .URL + "/not-found" )
96+ require .Error (t , err )
97+
98+ assert .True (t , strings .HasPrefix (err .Error (), "unexpected response" ))
99+ assert .Equal (t , uint64 (0 ), count )
100+ assert .True (t , tried )
101+ }
102+
34103func TestCrawlerSingle (t * testing.T ) {
35104 count := uint64 (0 )
36105 visited := & sync.Map {}
0 commit comments