Parameterize by CDN source; add R2 source support (#544)

Author: wmitsuda

embed.go

@@ -31,59 +31,72 @@ var Chiado []byte
 //go:embed holesky.toml
 var Holesky []byte
 
-func getURLByChain(chain, branch string) string {
-	return fmt.Sprintf("https://raw.githubusercontent.com/erigontech/erigon-snapshot/%s/%s.toml", branch, chain)
+type SnapshotSource int
+
+const (
+	Github SnapshotSource = 0
+	R2     SnapshotSource = 1
+)
+
+func getURLByChain(source SnapshotSource, chain, branch string) string {
+	if source == Github {
+		return fmt.Sprintf("https://raw.githubusercontent.com/erigontech/erigon-snapshot/%s/%s.toml", branch, chain)
+	} else if source == R2 {
+		return fmt.Sprintf("https://erigon-snapshots.erigon.network/%s/%s.toml", branch, chain)
+	}
+
+	panic(fmt.Sprintf("unknown snapshot source: %d", source))
 }
 
-func LoadSnapshots(ctx context.Context, branch string) (fetched bool, err error) {
+func LoadSnapshots(ctx context.Context, source SnapshotSource, branch string) (fetched bool, err error) {
 	var (
-		mainnetUrl    = getURLByChain("mainnet", branch)
-		sepoliaUrl    = getURLByChain("sepolia", branch)
-		amoyUrl       = getURLByChain("amoy", branch)
-		borMainnetUrl = getURLByChain("bor-mainnet", branch)
-		gnosisUrl     = getURLByChain("gnosis", branch)
-		chiadoUrl     = getURLByChain("chiado", branch)
-		holeskyUrl    = getURLByChain("holesky", branch)
+		mainnetUrl    = getURLByChain(source, "mainnet", branch)
+		sepoliaUrl    = getURLByChain(source, "sepolia", branch)
+		amoyUrl       = getURLByChain(source, "amoy", branch)
+		borMainnetUrl = getURLByChain(source, "bor-mainnet", branch)
+		gnosisUrl     = getURLByChain(source, "gnosis", branch)
+		chiadoUrl     = getURLByChain(source, "chiado", branch)
+		holeskyUrl    = getURLByChain(source, "holesky", branch)
 	)
 	var hashes []byte
 	// Try to fetch the latest snapshot hashes from the web
-	if hashes, err = fetchSnapshotHashes(ctx, mainnetUrl); err != nil {
+	if hashes, err = fetchSnapshotHashes(ctx, source, mainnetUrl); err != nil {
 		fetched = false
 		return
 	}
 	Mainnet = hashes
 
-	if hashes, err = fetchSnapshotHashes(ctx, sepoliaUrl); err != nil {
+	if hashes, err = fetchSnapshotHashes(ctx, source, sepoliaUrl); err != nil {
 		fetched = false
 		return
 	}
 	Sepolia = hashes
 
-	if hashes, err = fetchSnapshotHashes(ctx, amoyUrl); err != nil {
+	if hashes, err = fetchSnapshotHashes(ctx, source, amoyUrl); err != nil {
 		fetched = false
 		return
 	}
 	Amoy = hashes
 
-	if hashes, err = fetchSnapshotHashes(ctx, borMainnetUrl); err != nil {
+	if hashes, err = fetchSnapshotHashes(ctx, source, borMainnetUrl); err != nil {
 		fetched = false
 		return
 	}
 	BorMainnet = hashes
 
-	if hashes, err = fetchSnapshotHashes(ctx, gnosisUrl); err != nil {
+	if hashes, err = fetchSnapshotHashes(ctx, source, gnosisUrl); err != nil {
 		fetched = false
 		return
 	}
 	Gnosis = hashes
 
-	if hashes, err = fetchSnapshotHashes(ctx, chiadoUrl); err != nil {
+	if hashes, err = fetchSnapshotHashes(ctx, source, chiadoUrl); err != nil {
 		fetched = false
 		return
 	}
 	Chiado = hashes
 
-	if hashes, err = fetchSnapshotHashes(ctx, holeskyUrl); err != nil {
+	if hashes, err = fetchSnapshotHashes(ctx, source, holeskyUrl); err != nil {
 		fetched = false
 		return
 	}
@@ -93,11 +106,14 @@ func LoadSnapshots(ctx context.Context, branch string) (fetched bool, err error)
 	return fetched, nil
 }
 
-func fetchSnapshotHashes(ctx context.Context, url string) ([]byte, error) {
+func fetchSnapshotHashes(ctx context.Context, source SnapshotSource, url string) ([]byte, error) {
 	req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
 	if err != nil {
 		return nil, err
 	}
+	if source == R2 {
+		insertCloudflareHeaders(req)
+	}
 	resp, err := http.DefaultClient.Do(req)
 	if err != nil {
 		return nil, err
@@ -112,3 +128,14 @@ func fetchSnapshotHashes(ctx context.Context, url string) ([]byte, error) {
 	}
 	return res, err
 }
+
+// TODO: this was taken originally from erigon repo, downloader.go; we need to decide on a unique place to store such headers
+var cloudflareHeaders = http.Header{
+	"lsjdjwcush6jbnjj3jnjscoscisoc5s": []string{"I%OSJDNFKE783DDHHJD873EFSIVNI7384R78SSJBJBCCJBC32JABBJCBJK45"},
+}
+
+func insertCloudflareHeaders(req *http.Request) {
+	for key, value := range cloudflareHeaders {
+		req.Header[key] = value
+	}
+}

embed_test.go

@@ -6,7 +6,7 @@ import (
 )
 
 func TestFetchSnapshotHashes(t *testing.T) {
-	dat, err := fetchSnapshotHashes(context.Background(), "https://raw.githubusercontent.com/erigontech/erigon-snapshot/main/mainnet.toml")
+	dat, err := fetchSnapshotHashes(context.Background(), Github, "https://raw.githubusercontent.com/erigontech/erigon-snapshot/main/mainnet.toml")
 	if err != nil {
 		t.Errorf("fetchSnapshotHashes() failed: %v", err)
 	}
@@ -16,7 +16,7 @@ func TestFetchSnapshotHashes(t *testing.T) {
 }
 
 func TestFetchSnapshotHashesAll(t *testing.T) {
-	ok, err := LoadSnapshots(context.Background(), "main")
+	ok, err := LoadSnapshots(context.Background(), Github, "main")
 	if err != nil {
 		t.Errorf("LoadSnapshots() failed %s", err)
 	}