diff --git a/.omp/SYSTEM.md b/.omp/SYSTEM.md index 77d92af..349204c 100644 --- a/.omp/SYSTEM.md +++ b/.omp/SYSTEM.md @@ -354,7 +354,7 @@ Test files: ## Koios REST API -Direct REST calls use `koiosRestBase` constant (`https://koios.tosidrop.me/api/v1`) with RPC parameter syntax (`_epoch_no=N`, `_pool_bech32=ID`). All calls go through `koiosGetWithRetry` (max 5 retries, exponential backoff on 429/503) using `koiosHTTPClient` (shared `http.Client` with 30s timeout). The Go client (`koios-go-client/v3`) is still used for startup pool info and some live queries. +Direct REST calls use `koiosRESTBase(networkMagic)` helper (defaults to public koios.rest endpoints, overridable via `koios.url` config) with RPC parameter syntax (`_epoch_no=N`, `_pool_bech32=ID`). All calls go through `koiosGetWithRetry` (max 5 retries, exponential backoff on 429/503) using `koiosHTTPClient` (shared `http.Client` with 30s timeout). The Go client (`koios-go-client/v3`) is still used for startup pool info and some live queries. ## History Classification @@ -366,7 +366,7 @@ Background job (`buildLeaderlogHistory`) that runs after nonce backfill in full - **Slot classification**: forged (in Koios pool_blocks for our pool), battle (different pool's block exists at slot via `HasBlockAtSlot`), missed (no block at slot from any pool) - **Own context**: 12-hour timeout (daemon mode), independent from the nonce backfill context - **CLI mode**: shows estimated time before starting (~5s/epoch), no timeout -- **Rate**: ~75s/epoch in daemon (Koios rate limiting on tosidrop), ~5s/epoch in CLI (direct Koios API) +- **Rate**: ~75s/epoch in daemon (Koios rate limiting on api.koios.rest), ~5s/epoch in CLI (direct Koios API) ### Classification Limitations diff --git a/.omp/skills/history-debug/SKILL.md b/.omp/skills/history-debug/SKILL.md index 81b4ee1..c403f17 100644 --- a/.omp/skills/history-debug/SKILL.md +++ b/.omp/skills/history-debug/SKILL.md @@ -163,7 +163,7 @@ go func() { ```bash # Direct Koios query for pool-specific forged slots -curl -s "https://koios.tosidrop.me/api/v1/pool_blocks?_pool_bech32=pool1eq33dptvch4lstwuwkzt6atuutp3urh7pry7laudal54y2lsw0x&_epoch_no=612" | \ +curl -s "https://api.koios.rest/api/v1/pool_blocks?_pool_bech32=pool1eq33dptvch4lstwuwkzt6atuutp3urh7pry7laudal54y2lsw0x&_epoch_no=612" | \ jq -r '.[] | .abs_slot' # Returns array of slots where pool forged blocks @@ -221,8 +221,8 @@ kubectl -n cardano rollout restart deployment/goduckbot ### Koios API Errors ```bash -# Check tosidrop Koios status -curl -s https://koios.tosidrop.me/api/v1/tip | jq +# Check Koios status +curl -s https://api.koios.rest/api/v1/tip | jq # Switch to official Koios (change koiosRestBase in code) # Or add retry logic with longer backoff diff --git a/.omp/skills/verify-nonces/SKILL.md b/.omp/skills/verify-nonces/SKILL.md index 4a2e888..d80740e 100644 --- a/.omp/skills/verify-nonces/SKILL.md +++ b/.omp/skills/verify-nonces/SKILL.md @@ -24,10 +24,6 @@ kubectl -n postgres exec k3s-postgres-1 -- psql -U postgres -d goduckbot_v2 -c \ curl -s "https://api.koios.rest/api/v1/epoch_params?_epoch_no=530" | jq '.[0].nonce' ``` -Or via tosidrop (higher rate limits for bulk): -```bash -curl -s "https://koios.tosidrop.me/api/v1/epoch_params?_epoch_no=530" | jq '.[0].nonce' -``` ## 3. Batch verification script @@ -84,7 +80,7 @@ See `nonce-debug` skill for deeper investigation. ## 6. Known caveats -- **Koios rate limits:** api.koios.rest limits to ~100 req/min. Use tosidrop endpoint for bulk or add `sleep 1` between calls. +- **Koios rate limits:** api.koios.rest limits to ~100 req/min. Add `sleep 1` between calls for bulk verification. - **Rolling accumulation:** Nonce evolution is cumulative. A single wrong VRF output corrupts ALL subsequent epoch nonces. If you find a mismatch, check the earliest mismatched epoch first. - **Source column:** `tickn` = computed locally via TICKN rule, `koios` = fetched from Koios as fallback. TICKN-sourced nonces that match Koios confirm the entire local chain is correct. - **Lite mode:** Only has Koios-sourced nonces. Verification is trivial (always matches). Full mode is where TICKN verification matters. diff --git a/cli.go b/cli.go index 9b0b148..27000b4 100644 --- a/cli.go +++ b/cli.go @@ -123,15 +123,8 @@ func cliInit() (*cliContext, func(), error) { } cc.store = store - // Koios client - switch cc.networkMagic { - case PreprodNetworkMagic: - cc.koios, err = koios.New(koios.Host(koios.PreProdHost), koios.APIVersion("v1")) - case PreviewNetworkMagic: - cc.koios, err = koios.New(koios.Host(koios.PreviewHost), koios.APIVersion("v1")) - default: - cc.koios, err = koios.New(koios.Host(koios.MainnetHost), koios.APIVersion("v1")) - } + // Koios client; host defaults to public koios.rest, overridable via koios.url config + cc.koios, err = koios.New(koios.Host(koiosClientHost(cc.networkMagic)), koios.APIVersion("v1")) if err != nil { store.Close() if cc.vrfKey != nil { diff --git a/comprehensive_test.go b/comprehensive_test.go index 1517a10..9f1029a 100644 --- a/comprehensive_test.go +++ b/comprehensive_test.go @@ -949,7 +949,7 @@ func TestKoiosStakeDataAvailable(t *testing.T) { for _, epoch := range epochs { // Fetch pool stake - url := fmt.Sprintf("https://api.koios.rest/api/v1/pool_history?_pool_bech32=%s&_epoch_no=%d", poolBech32, epoch) + url := fmt.Sprintf(koiosRESTBase(MainnetNetworkMagic)+"/pool_history?_pool_bech32=%s&_epoch_no=%d", poolBech32, epoch) resp, err := http.Get(url) if err != nil { t.Logf("Epoch %d pool_history: NETWORK ERROR: %v", epoch, err) @@ -992,7 +992,7 @@ func TestKoiosLastBlockHash(t *testing.T) { epochs := []int{400, 450, 500, 550, 600} for _, epoch := range epochs { - url := fmt.Sprintf("https://api.koios.rest/api/v1/blocks?epoch_no=eq.%d&order=block_height.desc&limit=1&select=hash,epoch_no,block_height", epoch) + url := fmt.Sprintf(koiosRESTBase(MainnetNetworkMagic)+"/blocks?epoch_no=eq.%d&order=block_height.desc&limit=1&select=hash,epoch_no,block_height", epoch) resp, err := http.Get(url) if err != nil { t.Logf("Epoch %d: NETWORK ERROR: %v", epoch, err) diff --git a/config.yaml.example b/config.yaml.example index 6cee67b..056a255 100644 --- a/config.yaml.example +++ b/config.yaml.example @@ -54,6 +54,15 @@ leaderlog: # backfillSchedules: false # After nonce backfill, calculate and store leader schedules # # for all historical epochs where nonce data is available. +# Koios API configuration +# By default, goduckbot uses the public koios.rest endpoints: +# mainnet: https://api.koios.rest/api/v1 +# preprod: https://preprod.koios.rest/api/v1 +# preview: https://preview.koios.rest/api/v1 +# Set koios.url to point at a private Koios mirror (applies to all networks). +#koios: +# url: "https://your-private-koios-mirror.example.com/api/v1" + # Database configuration (only used when leaderlog.enabled is true) database: driver: "sqlite" # "sqlite" (default) or "postgres" diff --git a/main.go b/main.go index 9f872e1..74c0c76 100644 --- a/main.go +++ b/main.go @@ -63,15 +63,37 @@ const ( ) // koiosRESTBase returns the Koios REST API base URL for the given network magic. -// Used by history/backfill helpers that call Koios REST directly instead of via the Go client. +// Operators running a private Koios mirror can override via koios.url in config. func koiosRESTBase(networkMagic int) string { + if override := viper.GetString("koios.url"); override != "" { + return override + } switch networkMagic { case PreprodNetworkMagic: return "https://preprod.koios.rest/api/v1" case PreviewNetworkMagic: return "https://preview.koios.rest/api/v1" default: - return "https://koios.tosidrop.me/api/v1" + return "https://api.koios.rest/api/v1" + } +} + +// koiosClientHost returns the hostname for the Koios Go client. +// Extracts the host from koios.url config override if set, otherwise uses the +// public koios.rest endpoint for the network. +func koiosClientHost(networkMagic int) string { + if rawURL := viper.GetString("koios.url"); rawURL != "" { + if parsed, err := url.Parse(rawURL); err == nil && parsed.Host != "" { + return parsed.Host + } + } + switch networkMagic { + case PreprodNetworkMagic: + return koios.PreProdHost + case PreviewNetworkMagic: + return koios.PreviewHost + default: + return koios.MainnetHost } } @@ -381,31 +403,10 @@ func (i *Indexer) Start() error { log.Println("Twitter notifications disabled via config") } - /// Initialize the koios client based on networkMagic number - if i.networkMagic == PreprodNetworkMagic { - i.koios, e = koios.New( - koios.Host(koios.PreProdHost), - koios.APIVersion("v1"), - ) - if e != nil { - log.Fatal(e) - } - } else if i.networkMagic == PreviewNetworkMagic { - i.koios, e = koios.New( - koios.Host(koios.PreviewHost), - koios.APIVersion("v1"), - ) - if e != nil { - log.Fatal(e) - } - } else { - i.koios, e = koios.New( - koios.Host(koios.MainnetHost), - koios.APIVersion("v1"), - ) - if e != nil { - log.Fatal(e) - } + // Initialize Koios client; host defaults to public koios.rest, overridable via koios.url + i.koios, e = koios.New(koios.Host(koiosClientHost(i.networkMagic)), koios.APIVersion("v1")) + if e != nil { + log.Fatal(e) } i.epoch = i.getCurrentEpoch() diff --git a/nonce_koios_test.go b/nonce_koios_test.go index 02939ed..b75298a 100644 --- a/nonce_koios_test.go +++ b/nonce_koios_test.go @@ -122,7 +122,7 @@ func TestNonceVsKoios(t *testing.T) { } func fetchKoiosNonce(epoch int) (string, error) { - url := fmt.Sprintf("https://api.koios.rest/api/v1/epoch_params?_epoch_no=%d&select=nonce", epoch) + url := fmt.Sprintf(koiosRESTBase(MainnetNetworkMagic)+"/epoch_params?_epoch_no=%d&select=nonce", epoch) req, err := http.NewRequest("GET", url, nil) if err != nil { return "", err