Skip to content

Commit 4e66f90

Browse files
jterzisjterzis
authored
subgraph: fallback to appRegistry on failures during indexing (#4568)
### Description - [x] better error logging on agentData fetch - [ ] fallback to appRegistry on failures during indexing ### Changes <!-- List the specific changes made in this PR, for example: - Added/modified feature X - Fixed bug in component Y - Refactored module Z - Updated documentation --> ### Checklist - [ ] Tests added where required - [ ] Documentation updated where applicable - [ ] Changes adhere to the repository's contribution guidelines
1 parent 34d3525 commit 4e66f90

File tree

4 files changed

+320
-23
lines changed

4 files changed

+320
-23
lines changed

‎packages/subgraph/src/agentData.ts‎

Lines changed: 28 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
1-
import axios from 'axios'
1+
import { fetchJson } from './httpClient'
2+
import { fetchAgentDataFromAppRegistry } from './appRegistryFallback'
23

34
export type AgentData = {
45
type?: string
@@ -21,27 +22,35 @@ export async function fetchAgentData(
2122
uri: string,
2223
maxRetries: number = 3,
2324
retryDelayMs: number = 1000,
25+
appAddress?: string,
26+
environment?: string,
2427
): Promise<AgentData | null> {
25-
if (!uri.startsWith('https://')) return null
28+
if (!uri.startsWith('https://')) {
29+
console.warn(`[AgentData] Skipping non-HTTPS URI: ${uri}`)
30+
return null
31+
}
32+
33+
// Try HTTP fetch with retries
34+
const httpResult = await fetchJson<AgentData>(uri, {
35+
maxRetries,
36+
retryDelayMs,
37+
logPrefix: '[AgentData]',
38+
})
39+
40+
// If HTTP succeeds, return immediately
41+
if (httpResult) {
42+
return httpResult
43+
}
2644

27-
for (let attempt = 1; attempt <= maxRetries; attempt++) {
28-
try {
29-
const response = await axios.get<AgentData>(uri, { timeout: 5000 })
30-
if (response.status !== 200) {
31-
console.warn(`Warning: Received status ${response.status} from ${uri}`)
32-
}
33-
return response.data
34-
} catch (error) {
35-
if (attempt < maxRetries) {
36-
console.warn(
37-
`Fetch attempt ${attempt} failed for ${uri}, retrying in ${retryDelayMs}ms...`,
38-
)
39-
await new Promise((resolve) => setTimeout(resolve, retryDelayMs))
40-
} else {
41-
console.error(`All ${maxRetries} fetch attempts failed for ${uri}`)
42-
}
43-
}
45+
// Fallback: Try App Registry if we have the necessary parameters
46+
if (appAddress && environment) {
47+
console.warn(
48+
`[AgentData] HTTP fetch exhausted all retries, trying App Registry fallback: ` +
49+
`uri=${uri}, app=${appAddress}`,
50+
)
51+
return fetchAgentDataFromAppRegistry(appAddress, environment)
4452
}
4553

54+
// No fallback possible
4655
return null
4756
}

‎packages/subgraph/src/appRegistryFallback.ts‎

Lines changed: 161 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,161 @@
1+
import axios from 'axios'
2+
import { AgentData } from './agentData'
3+
4+
/**
5+
* Convert hex string to base64 (for gRPC bytes fields)
6+
*/
7+
function hexToBase64(hex: string): string {
8+
const hexString = hex.startsWith('0x') ? hex.slice(2) : hex
9+
return Buffer.from(hexString, 'hex').toString('base64')
10+
}
11+
12+
/**
13+
* Get the App Registry service URL for the given environment
14+
*/
15+
function getAppRegistryUrl(environment: string): string {
16+
switch (environment) {
17+
case 'local_dev':
18+
case 'development':
19+
return 'http://localhost:6170'
20+
case 'test':
21+
return 'http://localhost:6170'
22+
case 'alpha':
23+
return 'https://app-registry.alpha.towns.com'
24+
case 'beta':
25+
return 'https://app-registry.beta.towns.com'
26+
case 'gamma':
27+
case 'test-beta':
28+
return 'https://app-registry.gamma.towns.com'
29+
case 'omega':
30+
return 'https://app-registry.omega.towns.com'
31+
case 'delta':
32+
return 'https://app-registry.delta.towns.com'
33+
default:
34+
console.warn(`[AppRegistry] No app registry url for environment ${environment}`)
35+
return ''
36+
}
37+
}
38+
39+
/**
40+
* Fetch bot metadata from the App Registry service
41+
* Uses gRPC-web protocol to communicate with the service
42+
* See harmony/servers/workers/gateway-worker/src/bots/fetchBotMetadata.ts
43+
*/
44+
async function fetchBotMetadata(
45+
clientAddress: string,
46+
environment: string,
47+
): Promise<{
48+
username?: string
49+
displayName?: string
50+
imageUrl?: string
51+
avatarUrl?: string
52+
motto?: string
53+
description?: string
54+
externalUrl?: string
55+
} | null> {
56+
try {
57+
const appRegistryUrl = getAppRegistryUrl(environment)
58+
if (!appRegistryUrl) {
59+
return null
60+
}
61+
62+
// Convert hex address to base64 for gRPC bytes field
63+
const appIdBase64 = hexToBase64(clientAddress)
64+
65+
// Create the request body for GetAppMetadata
66+
// Using JSON format for gRPC-web (bytes must be base64 encoded)
67+
const requestBody = {
68+
appId: appIdBase64,
69+
}
70+
71+
const response = await axios.post<{
72+
metadata?: {
73+
username?: string
74+
displayName?: string
75+
imageUrl?: string
76+
avatarUrl?: string
77+
motto?: string
78+
description?: string
79+
externalUrl?: string
80+
}
81+
}>(`${appRegistryUrl}/river.AppRegistryService/GetAppMetadata`, requestBody, {
82+
headers: {
83+
'Content-Type': 'application/json',
84+
},
85+
timeout: 10000,
86+
})
87+
88+
const data = response.data
89+
90+
if (!data.metadata) {
91+
console.warn(`[AppRegistry] No metadata in response for app=${clientAddress}`)
92+
return null
93+
}
94+
95+
return data.metadata
96+
} catch (error) {
97+
if (axios.isAxiosError(error)) {
98+
const status = error.response?.status
99+
console.error(
100+
`[AppRegistry] Failed to fetch metadata: ` +
101+
`app=${clientAddress}, status=${status || 'unknown'}, message="${error.message}"`,
102+
)
103+
} else {
104+
console.error(
105+
`[AppRegistry] Error fetching metadata for app=${clientAddress}:`,
106+
error instanceof Error ? error.message : String(error),
107+
)
108+
}
109+
return null
110+
}
111+
}
112+
113+
/**
114+
* Convert App Registry metadata to AgentData format (EIP-8004 compliant)
115+
* Maps gateway-worker's BotMetadata structure to subgraph's AgentData type
116+
*/
117+
function convertToAgentData(metadata: {
118+
username?: string
119+
displayName?: string
120+
imageUrl?: string
121+
avatarUrl?: string
122+
motto?: string
123+
description?: string
124+
externalUrl?: string
125+
}): AgentData {
126+
return {
127+
type: 'https://eips.ethereum.org/EIPS/eip-8004#registration-v1',
128+
name: metadata.displayName || metadata.username,
129+
description: metadata.description,
130+
image: metadata.avatarUrl || metadata.imageUrl,
131+
// Note: App Registry doesn't provide endpoints/registrations/supportedTrust
132+
// These fields will be undefined, which is valid for optional fields
133+
}
134+
}
135+
136+
/**
137+
* Fetch agent data from App Registry service as fallback
138+
* This is called when HTTP fetch of agentUri fails after all retries
139+
*/
140+
export async function fetchAgentDataFromAppRegistry(
141+
appAddress: string,
142+
environment: string,
143+
): Promise<AgentData | null> {
144+
console.info(
145+
`[AppRegistry] Attempting fallback fetch: app=${appAddress}, environment=${environment}`,
146+
)
147+
148+
const metadata = await fetchBotMetadata(appAddress, environment)
149+
150+
if (!metadata) {
151+
console.warn(`[AppRegistry] Fallback fetch failed: app=${appAddress}`)
152+
return null
153+
}
154+
155+
const agentData = convertToAgentData(metadata)
156+
console.info(
157+
`[AppRegistry] Fallback fetch succeeded: app=${appAddress}, name=${agentData.name}`,
158+
)
159+
160+
return agentData
161+
}

‎packages/subgraph/src/httpClient.ts‎

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
import axios, { AxiosError } from 'axios'
2+
3+
export type HttpErrorType = 'TIMEOUT' | `HTTP_${number}` | 'NETWORK_ERROR' | 'UNKNOWN_ERROR'
4+
5+
export interface HttpError {
6+
type: HttpErrorType
7+
message: string
8+
originalError: Error
9+
}
10+
11+
/**
12+
* Classifies an error from an HTTP request into a diagnostic type
13+
*/
14+
export function classifyHttpError(error: unknown): HttpError {
15+
const err = error as Error
16+
17+
if (axios.isAxiosError(error)) {
18+
const axiosError = error as AxiosError
19+
20+
// Timeout errors
21+
if (axiosError.code === 'ECONNABORTED' || axiosError.code === 'ETIMEDOUT') {
22+
return {
23+
type: 'TIMEOUT',
24+
message: axiosError.message,
25+
originalError: err,
26+
}
27+
}
28+
29+
// HTTP status errors
30+
if (axiosError.response) {
31+
return {
32+
type: `HTTP_${axiosError.response.status}`,
33+
message: axiosError.message,
34+
originalError: err,
35+
}
36+
}
37+
38+
// Network errors (DNS, connection refused, etc.)
39+
return {
40+
type: 'NETWORK_ERROR',
41+
message: axiosError.code || axiosError.message,
42+
originalError: err,
43+
}
44+
}
45+
46+
// Unknown error type
47+
return {
48+
type: 'UNKNOWN_ERROR',
49+
message: err instanceof Error ? err.message : String(error),
50+
originalError: err,
51+
}
52+
}
53+
54+
export interface FetchOptions {
55+
timeout?: number
56+
maxRetries?: number
57+
retryDelayMs?: number
58+
logPrefix?: string
59+
}
60+
61+
/**
62+
* Fetch JSON data with automatic retries and enhanced error logging
63+
*/
64+
export async function fetchJson<T>(uri: string, options: FetchOptions = {}): Promise<T | null> {
65+
const {
66+
timeout = 5000,
67+
maxRetries = 3,
68+
retryDelayMs = 1000,
69+
logPrefix = '[HttpClient]',
70+
} = options
71+
72+
for (let attempt = 1; attempt <= maxRetries; attempt++) {
73+
try {
74+
const response = await axios.get<T>(uri, { timeout })
75+
76+
if (response.status !== 200) {
77+
console.warn(
78+
`${logPrefix} Non-200 status: ` +
79+
`uri=${uri}, status=${response.status}, attempt=${attempt}/${maxRetries}`,
80+
)
81+
}
82+
83+
// Success - log if this was a retry
84+
if (attempt > 1) {
85+
console.info(`${logPrefix} Fetch succeeded after ${attempt} attempts: uri=${uri}`)
86+
}
87+
88+
return response.data
89+
} catch (error) {
90+
const httpError = classifyHttpError(error)
91+
92+
if (attempt < maxRetries) {
93+
console.warn(
94+
`${logPrefix} Fetch attempt ${attempt}/${maxRetries} failed: ` +
95+
`uri=${uri}, errorType=${httpError.type}, message="${httpError.message}", ` +
96+
`retryingIn=${retryDelayMs}ms`,
97+
)
98+
await new Promise((resolve) => setTimeout(resolve, retryDelayMs))
99+
} else {
100+
console.error(
101+
`${logPrefix} All ${maxRetries} fetch attempts exhausted: ` +
102+
`uri=${uri}, errorType=${httpError.type}, message="${httpError.message}"`,
103+
)
104+
}
105+
}
106+
}
107+
108+
return null
109+
}

‎packages/subgraph/src/index.ts‎

Lines changed: 22 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import {
1010
import { fetchAgentData } from './agentData'
1111

1212
const ETH_ADDRESS = '0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee' as const
13+
const ENVIRONMENT = process.env.PONDER_ENVIRONMENT || 'alpha'
1314

1415
// Setup hook: Create critical indexes before indexing starts
1516
// These indexes are needed during historic sync for performance
@@ -928,7 +929,10 @@ ponder.on('AppRegistry:Registered', async ({ event, context }) => {
928929

929930
// Fetch and store agent data if URI is provided
930931
if (agentUri) {
931-
const agentData = await fetchAgentData(agentUri)
932+
console.info(
933+
`[AgentRegistered] Fetching agent data: agentId=${agentId}, app=${owner}, uri=${agentUri}`,
934+
)
935+
const agentData = await fetchAgentData(agentUri, 3, 1000, owner, ENVIRONMENT)
932936
if (agentData) {
933937
await context.db.sql
934938
.update(schema.agentIdentity)
@@ -939,6 +943,13 @@ ponder.on('AppRegistry:Registered', async ({ event, context }) => {
939943
eq(schema.agentIdentity.agentId, agentId),
940944
),
941945
)
946+
console.info(
947+
`[AgentRegistered] Successfully stored agent data: agentId=${agentId}, app=${owner}`,
948+
)
949+
} else {
950+
console.warn(
951+
`[AgentRegistered] Failed to fetch agent data: agentId=${agentId}, app=${owner}, uri=${agentUri}`,
952+
)
942953
}
943954
}
944955

@@ -982,7 +993,11 @@ ponder.on('AppRegistry:UriUpdated', async ({ event, context }) => {
982993
}
983994

984995
// Fetch agent data from new URI with retries
985-
const agentData = await fetchAgentData(agentUri)
996+
console.info(
997+
`[AgentUriUpdated] Updating URI: agentId=${agentId}, app=${agent.app}, ` +
998+
`oldUri=${agent.agentUri}, newUri=${agentUri}`,
999+
)
1000+
const agentData = await fetchAgentData(agentUri, 3, 1000, agent.app, ENVIRONMENT)
9861001

9871002
if (agentData !== null) {
9881003
// Only update if fetch succeeds - keeps URI and data in sync
@@ -999,10 +1014,13 @@ ponder.on('AppRegistry:UriUpdated', async ({ event, context }) => {
9991014
eq(schema.agentIdentity.agentId, agentId),
10001015
),
10011016
)
1017+
console.info(
1018+
`[AgentUriUpdated] Successfully updated agent data: agentId=${agentId}, app=${agent.app}`,
1019+
)
10021020
} else {
10031021
console.warn(
1004-
`Skipping URI update for agentId ${agentId}, fetch failed after retries. ` +
1005-
`Keeping existing URI: ${agent.agentUri}`,
1022+
`[AgentUriUpdated] Skipping URI update due to fetch failure: ` +
1023+
`agentId=${agentId}, app=${agent.app}, attemptedUri=${agentUri}, keepingUri=${agent.agentUri}`,
10061024
)
10071025
}
10081026
} catch (error) {

0 commit comments

Comments
 (0)