Bug fix: Refactor multi:// scheme to use correct URI template (#4)

* snapshot: block out code to support multiple resolver+template findingaids

* refactor the code to handle multi:// URIs

* Increaded (debug) logging

---------

Co-authored-by: thisisaaronland <thisisaaronland@localhost>

Author: thisisaaronland

reader.go

@@ -2,10 +2,9 @@ package findingaid
 
 import (
 	"context"
-	"database/sql"
 	"fmt"
 	"io"
-	_ "log"
+	"log/slog"
 	"net/url"
 	"strings"
 
@@ -18,15 +17,18 @@ import (
 // WHOSONFIRST_DATA_TEMPLATE is a URL template for the root `data` directory in Who's On First data repositories.
 const WHOSONFIRST_DATA_TEMPLATE string = "https://raw.githubusercontent.com/whosonfirst-data/{repo}/master/data/"
 
+// findingaid is a struct defining a resolver.Resolver and *uritemplates.UriTemplate pair
+type findingaid struct {
+	// A resolver.Resolver instance used to derive the Who's On First repository name for an ID.
+	resolver resolver.Resolver
+	// A compiled `uritemplates.UriTemplate` to use resolving Who's On First finding aid URIs.
+	template *uritemplates.UriTemplate
+}
+
 // type FindingAidReader implements the `whosonfirst/go-reader` interface for use with Who's On First finding aids.
 type FindingAidReader struct {
 	wof_reader.Reader
-	// A SQLite `sql.DB` instance containing Who's On First finding aid data. (Optional)
-	db *sql.DB
-	// A compiled `uritemplates.UriTemplate` to use resolving Who's On First finding aid URIs.
-	template *uritemplates.UriTemplate
-	// A resolver.Resolver instance used to derive the Who's On First repository name for an ID.
-	resolver resolver.Resolver
+	findingaids []*findingaid
 }
 
 func init() {
@@ -46,82 +48,65 @@ func NewFindingAidReader(ctx context.Context, uri string) (wof_reader.Reader, er
 
 	q := u.Query()
 
-	uri_template := WHOSONFIRST_DATA_TEMPLATE
-
-	if q.Get("template") != "" {
-		uri_template = q.Get("template")
-	}
+	findingaids := make([]*findingaid, 0)
 
-	uri_template, err = url.QueryUnescape(uri_template)
-
-	if err != nil {
-		return nil, fmt.Errorf("Failed to unescape ?template= parameter, %w", err)
-	}
+	logger := slog.Default()
+	logger = logger.With("scheme", u.Host)
+	
+	switch u.Host {
+	case "multi":
 
-	t, err := uritemplates.Parse(uri_template)
+		for _, rt_uri := range q["resolver"] {
 
-	if err != nil {
-		return nil, fmt.Errorf("Failed to parse URI template, %w", err)
-	}
+			rt_u, err := url.Parse(rt_uri)
 
-	q.Del("template")
-	u.RawQuery = q.Encode()
-
-	// findingaid://sqlite?dsn={DSN}
-	// findingaid://awsdynamo/{TABLENAME}
-	// findingaid://http(s)/{HOST}/{PATH}
+			if err != nil {
+				return nil, fmt.Errorf("Failed to parse resolver URI, %w", err)
+			}
 
-	// Set up resolver
+			fa_u := url.URL{}
+			fa_u.Scheme = "findingaid"
+			fa_u.Host = rt_u.Scheme
+			fa_u.Path = rt_u.Host + rt_u.Path
+			fa_u.RawQuery = rt_u.RawQuery
 
-	var ru *url.URL
+			fa_uri := fa_u.String()
 
-	switch u.Host {
-	case "http", "https":
+			r, t, err := deriveResolverAndTemplate(ctx, fa_uri)
 
-		path := u.Path
-		path = strings.TrimLeft(path, "/")
+			if err != nil {
+				return nil, fmt.Errorf("Failed to derive resolver and template from ?resolver= URI, %w", err)
+			}
 
-		parts := strings.Split(path, "/")
+			fa := &findingaid{
+				resolver: r,
+				template: t,
+			}
 
-		ru = &url.URL{}
-		ru.Scheme = u.Host
-		ru.Host = parts[0]
-
-		if len(parts) > 1 {
-			path = strings.Join(parts[1:], "/")
-			ru.Path = fmt.Sprintf("/%s", path)
+			logger.Debug("Add findingaid reader", "uri", fa_uri)
+			findingaids = append(findingaids, fa)
 		}
 
-		ru.RawQuery = u.RawQuery
-
-	case "multi":
-
-		ru = &url.URL{}
-		ru.Scheme = u.Host
-		ru.RawQuery = u.RawQuery
-
 	default:
 
-		path := u.Path
-		path = strings.TrimLeft(path, "/")
-
-		ru = &url.URL{}
-		ru.Scheme = u.Host
-		ru.Host = path
-		ru.RawQuery = u.RawQuery
-	}
+		r, t, err := deriveResolverAndTemplate(ctx, uri)
 
-	r_uri := ru.String()
-
-	f, err := resolver.NewResolver(ctx, r_uri)
+		if err != nil {
+			return nil, err
+		}
 
-	if err != nil {
-		return nil, fmt.Errorf("Failed to create resolver, %w", err)
+		logger.Debug("Add findingaid reader", "uri", uri)
+		
+		findingaids = []*findingaid{
+			&findingaid{
+				resolver: r,
+				template: t,
+			},
+		}
 	}
 
 	r := &FindingAidReader{
-		resolver: f,
-		template: t,
+		findingaids: findingaids,
 	}
 
 	return r, nil
@@ -186,33 +171,133 @@ func (r *FindingAidReader) getReaderURIAndPath(ctx context.Context, uri string)
 
 	// TBD: cache this?
 
+	logger := slog.Default()
+	logger = logger.With("uri", uri)
+
 	id, uri_args, err := wof_uri.ParseURI(uri)
 
 	if err != nil {
+		logger.Error("Failed to parse URI", "error", err)
 		return "", "", fmt.Errorf("Failed to parse URI, %w", err)
 	}
 
-	repo, err := r.resolver.GetRepo(ctx, id)
+	rel_path, err := wof_uri.Id2RelPath(id, uri_args)
 
 	if err != nil {
-		return "", "", fmt.Errorf("Failed to derive repo, %w", err)
+		logger.Error("Failed to derive relative path for ID", "id", id, "error", err)
+		return "", "", fmt.Errorf("Failed to derive path, %w", err)
 	}
 
-	rel_path, err := wof_uri.Id2RelPath(id, uri_args)
+	for idx, fa := range r.findingaids {
+
+		logger.Debug("Get repo", "findingaid", idx, "id", id)
+		repo, err := fa.resolver.GetRepo(ctx, id)
+
+		if err != nil {
+
+			if err == resolver.ErrNotFound {
+				logger.Debug("Failed to derive repo with resolver, not found", "id", id)
+				continue
+			}
+
+			logger.Error("Failed to derive repo with resolver", "id", id, "error", err)
+			return "", "", fmt.Errorf("Failed to derive repo, %w", err)
+		}
+
+		values := map[string]interface{}{
+			"repo": repo,
+		}
+
+		reader_uri, err := fa.template.Expand(values)
+
+		if err != nil {
+			logger.Error("Failed to expand template for resolver", "repo", repo, "template", fa.template, "error", err)
+			return "", "", fmt.Errorf("Failed to derive reader URI, %w", err)
+		}
+
+		logger.Debug("Return reader URI for resolver", "reader_uri", reader_uri, "rel_path", rel_path)
+		return reader_uri, rel_path, nil
+	}
+
+	return "", "", fmt.Errorf("Failed to derive repo, no findingaid matches")
+}
+
+func deriveResolverAndTemplate(ctx context.Context, uri string) (resolver.Resolver, *uritemplates.UriTemplate, error) {
+
+	u, err := url.Parse(uri)
 
 	if err != nil {
-		return "", "", fmt.Errorf("Failed to derive path, %w", err)
+		return nil, nil, err
 	}
 
-	values := map[string]interface{}{
-		"repo": repo,
+	q := u.Query()
+
+	uri_template := WHOSONFIRST_DATA_TEMPLATE
+
+	if q.Get("template") != "" {
+		uri_template = q.Get("template")
+	}
+
+	uri_template, err = url.QueryUnescape(uri_template)
+
+	if err != nil {
+		return nil, nil, fmt.Errorf("Failed to unescape ?template= parameter, %w", err)
+	}
+
+	t, err := uritemplates.Parse(uri_template)
+
+	if err != nil {
+		return nil, nil, fmt.Errorf("Failed to parse URI template, %w", err)
 	}
 
-	reader_uri, err := r.template.Expand(values)
+	q.Del("template")
+	u.RawQuery = q.Encode()
+
+	// findingaid://sqlite?dsn={DSN}
+	// findingaid://awsdynamo/{TABLENAME}
+	// findingaid://http(s)/{HOST}/{PATH}
+
+	// Set up resolver
+
+	var ru *url.URL
+
+	switch u.Host {
+	case "http", "https":
+
+		path := u.Path
+		path = strings.TrimLeft(path, "/")
+
+		parts := strings.Split(path, "/")
+
+		ru = &url.URL{}
+		ru.Scheme = u.Host
+		ru.Host = parts[0]
+
+		if len(parts) > 1 {
+			path = strings.Join(parts[1:], "/")
+			ru.Path = fmt.Sprintf("/%s", path)
+		}
+
+		ru.RawQuery = u.RawQuery
+
+	default:
+
+		path := u.Path
+		path = strings.TrimLeft(path, "/")
+
+		ru = &url.URL{}
+		ru.Scheme = u.Host
+		ru.Host = path
+		ru.RawQuery = u.RawQuery
+	}
+
+	r_uri := ru.String()
+
+	r, err := resolver.NewResolver(ctx, r_uri)
 
 	if err != nil {
-		return "", "", fmt.Errorf("Failed to derive reader URI, %w", err)
+		return nil, nil, fmt.Errorf("Failed to create resolver, %w", err)
 	}
 
-	return reader_uri, rel_path, nil
+	return r, t, nil
 }

reader_test.go

@@ -3,14 +3,14 @@ package findingaid
 import (
 	"context"
 	"fmt"
+	"log/slog"
+	"net/url"
 	"os"
 	"testing"
 
 	"github.com/whosonfirst/go-reader/v2"
 )
 
-//
-
 func TestSQLiteFindingAid(t *testing.T) {
 
 	ctx := context.Background()
@@ -87,33 +87,52 @@ func TestHTTPFindingAid(t *testing.T) {
 
 func TestMultiFindingAid(t *testing.T) {
 
+	slog.SetLogLoggerLevel(slog.LevelDebug)
+	slog.Debug("Verbose logging enabled")
+
 	ctx := context.Background()
 
-	reader_uri := "findingaid://multi?resolver=https%3A%2F%2Fstatic.sfomuseum.org%2Ffindingaid%3Ftemplate%3Dhttps%3A%2F%2Fraw.githubusercontent.com%2Fsfomuseum-data%2F%7Brepo%7D%2Fmain%2Fdata%2F&resolver=https%3A%2F%2Fdata.whosonfirst.org%2Ffindingaid%3Ftemplate%3Dhttps%3A%2F%2Fraw.githubusercontent.com%2Fwhosonfirst-data%2F%7Brepo%7D%2Fmaster%2Fdata%2F"
+	reader_q := url.Values{}
+	reader_q.Add("resolver", "https://static.sfomuseum.org/findingaid?template=https://raw.githubusercontent.com/sfomuseum-data/{repo}/main/data/")
+	reader_q.Add("resolver", "https://data.whosonfirst.org/findingaid?template=https://raw.githubusercontent.com/whosonfirst-data/{repo}/master/data/")
+
+	reader_u := url.URL{}
+	reader_u.Scheme = "findingaid"
+	reader_u.Host = "multi"
+	reader_u.RawQuery = reader_q.Encode()
+
+	reader_uri := reader_u.String()
+	slog.Debug("Create reader", "uri", reader_uri)
 
 	r, err := reader.NewReader(ctx, reader_uri)
 
 	if err != nil {
 		t.Fatalf("Failed to create new reader, %v", err)
 	}
 
-	uri := "85865975"
+	tests := []string{
+		"85865975",
+		"1159396133",
+	}
 
-	fh, err := r.Read(ctx, uri)
+	for _, uri := range tests {
 
-	if err != nil {
-		t.Fatalf("Failed to read %s, %v", uri, err)
-	}
+		fh, err := r.Read(ctx, uri)
 
-	fh.Close()
+		if err != nil {
+			t.Fatalf("Failed to read %s, %v", uri, err)
+		}
 
-	exists, err := r.Exists(ctx, uri)
+		fh.Close()
 
-	if err != nil {
-		t.Fatalf("Failed to determine if %s exists, %v", uri, err)
-	}
+		exists, err := r.Exists(ctx, uri)
 
-	if !exists {
-		t.Fatalf("Expected %s to exists", uri)
+		if err != nil {
+			t.Fatalf("Failed to determine if %s exists, %v", uri, err)
+		}
+
+		if !exists {
+			t.Fatalf("Expected %s to exists", uri)
+		}
 	}
 }