Skip to content

Commit 65c4006

Browse files
tschaubtschaub
authored
Merge pull request #40 from planetlabs/children
Support the STAC API - Children conformance class
2 parents 60b52ba + 820d8f4 commit 65c4006

File tree

6 files changed

+213
-25
lines changed

6 files changed

+213
-25
lines changed

‎crawler/crawler.go

+61
Original file line numberDiff line numberDiff line change
@@ -273,6 +273,7 @@ func newCrawler(resource string, visitor Visitor, options ...*Options) (*crawler
273273
const (
274274
resourceTask = "resource"
275275
collectionsTask = "collections"
276+
childrenTask = "children"
276277
featuresTask = "features"
277278
)
278279

@@ -290,6 +291,8 @@ func (c *crawler) crawl(worker *workgroup.Worker[*Task], t *Task) error {
290291
return c.crawlResource(worker, t.Url)
291292
case collectionsTask:
292293
return c.crawlCollections(worker, t.Url)
294+
case childrenTask:
295+
return c.crawlChildren(worker, t.Url)
293296
case featuresTask:
294297
return c.crawlFeatures(worker, t.Url)
295298
default:
@@ -330,6 +333,18 @@ func (c *crawler) crawlResource(worker *workgroup.Worker[*Task], resourceUrl str
330333
}
331334
}
332335

336+
// check if this looks like a STAC API root catalog that implements STAC API - Children
337+
if resource.Type() == Catalog && len(resource.ConformsTo()) > 1 {
338+
childrenLink := links.Rel("children", LinkTypeApplicationJSON, LinkTypeAnyJSON, LinkTypeNone)
339+
if childrenLink != nil {
340+
linkLoc, err := loc.Resolve(childrenLink["href"])
341+
if err != nil {
342+
return c.errorHandler(err)
343+
}
344+
return worker.Add(&Task{Url: linkLoc.String(), Type: childrenTask})
345+
}
346+
}
347+
333348
if resource.Type() == Collection {
334349
// shortcut for "items" link
335350
itemsLink := links.Rel("items", LinkTypeGeoJSON, LinkTypeApplicationJSON, LinkTypeAnyJSON, LinkTypeNone)
@@ -419,6 +434,52 @@ func (c *crawler) crawlCollections(worker *workgroup.Worker[*Task], collectionsU
419434
return nil
420435
}
421436

437+
func (c *crawler) crawlChildren(worker *workgroup.Worker[*Task], childrenUrl string) error {
438+
loc, locErr := normurl.New(childrenUrl)
439+
if locErr != nil {
440+
return locErr
441+
}
442+
response := &childrenResponse{}
443+
loadErr := load(c.entry, loc, response)
444+
if loadErr != nil {
445+
return c.errorHandler(loadErr)
446+
}
447+
448+
for i, resource := range response.Children {
449+
if resource.Type() != Catalog && resource.Type() != Collection {
450+
return c.errorHandler(fmt.Errorf("expected catalog or collection at index %d, got %s", i, resource.Type()))
451+
}
452+
links := resource.Links()
453+
454+
selfLink := links.Rel("self", LinkTypeApplicationJSON, LinkTypeAnyJSON, LinkTypeNone)
455+
if selfLink == nil {
456+
return c.errorHandler(fmt.Errorf("missing self link for %s %d in %s", resource.Type(), i, childrenUrl))
457+
}
458+
selfLinkLoc, selfLinkErr := loc.Resolve(selfLink["href"])
459+
if selfLinkErr != nil {
460+
return c.errorHandler(selfLinkErr)
461+
}
462+
addErr := worker.Add(&Task{Url: selfLinkLoc.String(), Type: resourceTask})
463+
if addErr != nil {
464+
return addErr
465+
}
466+
}
467+
468+
nextLink := response.Links.Rel("next", LinkTypeApplicationJSON, LinkTypeAnyJSON, LinkTypeNone)
469+
if nextLink != nil {
470+
linkLoc, err := loc.Resolve(nextLink["href"])
471+
if err != nil {
472+
return c.errorHandler(err)
473+
}
474+
addErr := worker.Add(&Task{Url: linkLoc.String(), Type: childrenTask})
475+
if addErr != nil {
476+
return addErr
477+
}
478+
}
479+
480+
return nil
481+
}
482+
422483
func (c *crawler) crawlFeatures(worker *workgroup.Worker[*Task], featuresUrl string) error {
423484
loc, locErr := normurl.New(featuresUrl)
424485
if locErr != nil {

‎crawler/crawler_test.go

+38
Original file line numberDiff line numberDiff line change
@@ -414,3 +414,41 @@ func TestCrawlerAPICollection(t *testing.T) {
414414
_, visitedItem := visited.Load(filepath.Join(wd, "testdata/v1.0.0/item-in-collection.json"))
415415
assert.True(t, visitedItem)
416416
}
417+
418+
func TestCrawlerAPIChildren(t *testing.T) {
419+
count := uint64(0)
420+
visited := &sync.Map{}
421+
422+
visitor := func(location string, resource crawler.Resource) error {
423+
atomic.AddUint64(&count, 1)
424+
_, loaded := visited.LoadOrStore(location, true)
425+
if loaded {
426+
return fmt.Errorf("already visited %s", location)
427+
}
428+
return nil
429+
}
430+
entry := "testdata/v1.0.0/api-catalog-with-children.json"
431+
432+
err := crawler.Crawl(entry, visitor)
433+
assert.NoError(t, err)
434+
435+
assert.Equal(t, uint64(5), count)
436+
437+
wd, wdErr := os.Getwd()
438+
require.NoError(t, wdErr)
439+
440+
_, visitedCatalog := visited.Load(filepath.Join(wd, entry))
441+
assert.True(t, visitedCatalog)
442+
443+
_, visitedOneCatalog := visited.Load(filepath.Join(wd, "testdata/v1.0.0/catalog.json"))
444+
assert.True(t, visitedOneCatalog)
445+
446+
_, visitedOtherCatalog := visited.Load(filepath.Join(wd, "testdata/v1.0.0/catalog-with-collection-of-items.json"))
447+
assert.True(t, visitedOtherCatalog)
448+
449+
_, visitedCollection := visited.Load(filepath.Join(wd, "testdata/v1.0.0/collection-with-items.json"))
450+
assert.True(t, visitedCollection)
451+
452+
_, visitedItem := visited.Load(filepath.Join(wd, "testdata/v1.0.0/item-in-collection.json"))
453+
assert.True(t, visitedItem)
454+
}

‎crawler/resources.go

+5
Original file line numberDiff line numberDiff line change
@@ -159,3 +159,8 @@ type featureCollectionResponse struct {
159159
Features []Resource `json:"features"`
160160
Links Links `json:"links"`
161161
}
162+
163+
type childrenResponse struct {
164+
Children []Resource `json:"children"`
165+
Links Links `json:"links"`
166+
}

‎crawler/testdata/v1.0.0/api-catalog-with-children.json

+38
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
{
2+
"stac_version": "1.0.0",
3+
"id": "example-stac",
4+
"title": "A simple STAC API Example, implementing STAC API - Children",
5+
"description": "This Catalog aims to demonstrate the a simple landing page",
6+
"type": "Catalog",
7+
"conformsTo": [
8+
"https://api.stacspec.org/v1.0.0-rc.1/core",
9+
"https://api.stacspec.org/v1.0.0-rc.1/children"
10+
],
11+
"links": [
12+
{
13+
"rel": "self",
14+
"type": "application/json",
15+
"href": "./api-catalog-with-children.com"
16+
},
17+
{
18+
"rel": "root",
19+
"type": "application/json",
20+
"href": "./api-catalog-with-children.com"
21+
},
22+
{
23+
"rel": "service-desc",
24+
"type": "application/vnd.oai.openapi+json;version=3.0",
25+
"href": "https://stac-api.example.com/api"
26+
},
27+
{
28+
"rel": "service-doc",
29+
"type": "text/html",
30+
"href": "https://stac-api.example.com/api.html"
31+
},
32+
{
33+
"rel": "children",
34+
"type": "application/json",
35+
"href": "./api-children.json"
36+
}
37+
]
38+
}

‎crawler/testdata/v1.0.0/api-children.json

+62
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
{
2+
"children": [
3+
{
4+
"stac_version": "1.0.0",
5+
"stac_extensions": [],
6+
"id": "one-catalog",
7+
"type": "Catalog",
8+
"links": [
9+
{
10+
"rel": "root",
11+
"type": "application/json",
12+
"href": "./api-catalog-with-children.json"
13+
},
14+
{
15+
"rel": "parent",
16+
"type": "application/json",
17+
"href": "./api-catalog-with-children.json"
18+
},
19+
{
20+
"rel": "self",
21+
"type": "application/json",
22+
"href": "./catalog-with-collection-of-items.json"
23+
}
24+
]
25+
},
26+
{
27+
"stac_version": "1.0.0",
28+
"stac_extensions": [],
29+
"id": "another-catalog",
30+
"type": "Catalog",
31+
"links": [
32+
{
33+
"rel": "root",
34+
"type": "application/json",
35+
"href": "./api-catalog-with-children.json"
36+
},
37+
{
38+
"rel": "parent",
39+
"type": "application/json",
40+
"href": "./api-catalog-with-children.json"
41+
},
42+
{
43+
"rel": "self",
44+
"type": "application/json",
45+
"href": "./catalog.json"
46+
}
47+
]
48+
}
49+
],
50+
"links": [
51+
{
52+
"rel": "root",
53+
"type": "application/json",
54+
"href": "https://stac-api.example.com"
55+
},
56+
{
57+
"rel": "self",
58+
"type": "application/json",
59+
"href": "https://stac-api.example.com/children"
60+
}
61+
]
62+
}

‎validator/validator_test.go

+9-25
Original file line numberDiff line numberDiff line change
@@ -83,38 +83,22 @@ func (s *Suite) TestSchemaMap() {
8383
s.Assert().NoError(err)
8484
}
8585

86-
func TestSuite(t *testing.T) {
87-
suite.Run(t, &Suite{})
88-
}
89-
90-
func ExampleValidator_Validate_children() {
86+
func (s *Suite) TestCatalogWithInvalidItem() {
9187
v := validator.New()
9288

9389
err := v.Validate(context.Background(), "testdata/cases/v1.0.0/catalog-with-item-missing-id.json")
94-
95-
workdir, _ := os.Getwd()
96-
fmt.Println(strings.Replace(fmt.Sprintf("%#v\n", err), workdir, "/path/to", 1))
97-
// Output:
98-
// invalid item: /path/to/testdata/cases/v1.0.0/item-missing-id.json
99-
// [I#] [S#] doesn't validate with https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#
100-
// [I#] [S#/allOf/0] allOf failed
101-
// [I#] [S#/allOf/0/$ref] doesn't validate with '/definitions/core'
102-
// [I#] [S#/definitions/core/allOf/2] allOf failed
103-
// [I#] [S#/definitions/core/allOf/2/required] missing properties: 'id'
90+
s.Require().Error(err)
91+
s.Assert().True(strings.HasSuffix(fmt.Sprintf("%#v", err), "missing properties: 'id'"))
10492
}
10593

106-
func ExampleValidator_Validate_single() {
94+
func (s *Suite) TestInvalidItem() {
10795
v := validator.New()
10896

10997
err := v.Validate(context.Background(), "testdata/cases/v1.0.0/item-missing-id.json")
98+
s.Require().Error(err)
99+
s.Assert().True(strings.HasSuffix(fmt.Sprintf("%#v", err), "missing properties: 'id'"))
100+
}
110101

111-
workdir, _ := os.Getwd()
112-
fmt.Println(strings.Replace(fmt.Sprintf("%#v\n", err), workdir, "/path/to", 1))
113-
// Output:
114-
// invalid item: /path/to/testdata/cases/v1.0.0/item-missing-id.json
115-
// [I#] [S#] doesn't validate with https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#
116-
// [I#] [S#/allOf/0] allOf failed
117-
// [I#] [S#/allOf/0/$ref] doesn't validate with '/definitions/core'
118-
// [I#] [S#/definitions/core/allOf/2] allOf failed
119-
// [I#] [S#/definitions/core/allOf/2/required] missing properties: 'id'
102+
func TestSuite(t *testing.T) {
103+
suite.Run(t, &Suite{})
120104
}

0 commit comments

Comments
 (0)