Merge pull request #41 from planetlabs/bigger-pages

Bigger pages

Author: tschaub

crawler/crawler.go

@@ -353,6 +353,7 @@ func (c *crawler) crawlResource(worker *workgroup.Worker[*Task], resourceUrl str
 			if err != nil {
 				return c.errorHandler(err)
 			}
+			linkLoc.SetQueryParam("limit", "250")
 			return worker.Add(&Task{Url: linkLoc.String(), Type: featuresTask})
 		}
 	}
@@ -413,6 +414,7 @@ func (c *crawler) crawlCollections(worker *workgroup.Worker[*Task], collectionsU
 		if itemsLinkErr != nil {
 			return c.errorHandler(itemsLinkErr)
 		}
+		itemsLinkLoc.SetQueryParam("limit", "250")
 		addErr := worker.Add(&Task{Url: itemsLinkLoc.String(), Type: featuresTask})
 		if addErr != nil {
 			return addErr

internal/normurl/normurl.go

@@ -17,6 +17,19 @@ func (l *Locator) String() string {
 	return l.url.String()
 }
 
+func (l *Locator) SetQueryParam(param string, value string) {
+	if l.isFilepath {
+		return
+	}
+	query := l.url.Query()
+	if value != "" {
+		query.Set(param, value)
+	} else {
+		query.Del(param)
+	}
+	l.url.RawQuery = query.Encode()
+}
+
 func (l *Locator) IsFilepath() bool {
 	return l.isFilepath
 }

internal/normurl/normurl_test.go

@@ -3,6 +3,7 @@ package normurl_test
 import (
 	"errors"
 	"fmt"
+	"net/url"
 	"testing"
 
 	"github.com/planetlabs/go-stac/internal/normurl"
@@ -63,6 +64,71 @@ func TestNew(t *testing.T) {
 	}
 }
 
+func TestSetQueryParam(t *testing.T) {
+	cases := []struct {
+		input    string
+		key      string
+		value    string
+		expected string
+	}{
+		{
+			input:    "https://example.com",
+			key:      "foo",
+			value:    "bar",
+			expected: "https://example.com?foo=bar",
+		},
+		{
+			input:    "https://example.com?foo=bar",
+			key:      "baz",
+			value:    "qux",
+			expected: "https://example.com?foo=bar&baz=qux",
+		},
+		{
+			input:    "https://example.com?foo=bar&baz=qux",
+			key:      "baz",
+			value:    "bam",
+			expected: "https://example.com?foo=bar&baz=bam",
+		},
+		{
+			input:    "https://example.com?foo=bar&baz=qux",
+			key:      "baz",
+			value:    "",
+			expected: "https://example.com?foo=bar",
+		},
+		{
+			input:    "https://example.com?foo=bar",
+			key:      "baz",
+			value:    "",
+			expected: "https://example.com?foo=bar",
+		},
+		{
+			input:    "/path/to/file",
+			key:      "foo",
+			value:    "bar",
+			expected: "/path/to/file",
+		},
+	}
+
+	for i, c := range cases {
+		t.Run(fmt.Sprintf("case %d", i), func(t *testing.T) {
+			l, err := normurl.New(c.input)
+			require.NoError(t, err)
+
+			l.SetQueryParam(c.key, c.value)
+
+			expUrl, err := url.Parse(c.expected)
+			require.NoError(t, err)
+			expQuery := expUrl.Query()
+
+			gotUrl, err := url.Parse(l.String())
+			require.NoError(t, err)
+			gotQuery := gotUrl.Query()
+
+			assert.Equal(t, expQuery, gotQuery)
+		})
+	}
+}
+
 func TestResolve(t *testing.T) {
 	cases := []struct {
 		base     string