|
| 1 | +--- |
| 2 | +title: Go Example |
| 3 | +description: Complete Go example for CoreClaw API integration |
| 4 | +sidebar: |
| 5 | + order: 5 |
| 6 | +--- |
| 7 | + |
| 8 | +Complete Go example showing how to run a Worker and retrieve results. |
| 9 | + |
| 10 | +## Prerequisites |
| 11 | + |
| 12 | +No external dependencies required. Uses Go's built-in `net/http` package. |
| 13 | + |
| 14 | +## Complete Example |
| 15 | + |
| 16 | +```go |
| 17 | +package main |
| 18 | + |
| 19 | +import ( |
| 20 | + "bytes" |
| 21 | + "encoding/json" |
| 22 | + "fmt" |
| 23 | + "io" |
| 24 | + "net/http" |
| 25 | + "time" |
| 26 | +) |
| 27 | + |
| 28 | +// API Configuration |
| 29 | +const API_BASE_URL = "https://openapi.coreclaw.com" |
| 30 | +const API_KEY = "YOUR_API_KEY" |
| 31 | +const TIMEOUT = 30 |
| 32 | + |
| 33 | +// Response structures |
| 34 | +type ApiResponse struct { |
| 35 | + Code int `json:"code"` |
| 36 | + Message string `json:"message"` |
| 37 | + Data json.RawMessage `json:"data"` |
| 38 | +} |
| 39 | + |
| 40 | +type RunData struct { |
| 41 | + RunSlug string `json:"run_slug"` |
| 42 | +} |
| 43 | + |
| 44 | +type StatusData struct { |
| 45 | + Status int `json:"status"` |
| 46 | + Results int `json:"results"` |
| 47 | + Duration int `json:"duration"` |
| 48 | +} |
| 49 | + |
| 50 | +type ResultData struct { |
| 51 | + Count int `json:"count"` |
| 52 | + List []map[string]interface{} `json:"list"` |
| 53 | +} |
| 54 | + |
| 55 | +func main() { |
| 56 | + // Build request params |
| 57 | + requestParams := map[string]interface{}{ |
| 58 | + "scraper_slug": "YOUR_SCRAPER_SLUG", |
| 59 | + "version": "v1.0.0", // Get from /api/scraper |
| 60 | + "is_async": true, |
| 61 | + "input": map[string]interface{}{ |
| 62 | + "parameters": map[string]interface{}{ |
| 63 | + "system": map[string]interface{}{ |
| 64 | + "cpus": 0.125, |
| 65 | + "memory": 512, |
| 66 | + "execute_limit_time_seconds": 1800, |
| 67 | + "max_total_charge": 0, |
| 68 | + "max_total_traffic": 0, |
| 69 | + }, |
| 70 | + "custom": map[string]interface{}{ |
| 71 | + // Build from /api/scraper response |
| 72 | + }, |
| 73 | + }, |
| 74 | + }, |
| 75 | + } |
| 76 | + |
| 77 | + // Step 1: Start Worker |
| 78 | + fmt.Println("Starting scraper...") |
| 79 | + runSlug, err := runScraperAsync(requestParams) |
| 80 | + if err != nil { |
| 81 | + fmt.Printf("Failed to start: %v\n", err) |
| 82 | + return |
| 83 | + } |
| 84 | + |
| 85 | + fmt.Printf("Started! Run ID: %s\n", runSlug) |
| 86 | + |
| 87 | + // Step 2: Poll status |
| 88 | + fmt.Println("Polling status...") |
| 89 | + status, statusData, err := pollUntilComplete(runSlug) |
| 90 | + if err != nil { |
| 91 | + fmt.Printf("Polling failed: %v\n", err) |
| 92 | + return |
| 93 | + } |
| 94 | + |
| 95 | + // Status: 1=Ready, 2=Running, 3=Succeeded, 4=Failed, 5=Aborting |
| 96 | + if status == 3 { |
| 97 | + fmt.Printf("Completed! Results: %d, Duration: %ds\n", statusData.Results, statusData.Duration) |
| 98 | + |
| 99 | + // Step 3: Get results |
| 100 | + results, err := getResults(runSlug) |
| 101 | + if err != nil { |
| 102 | + fmt.Printf("Failed to get results: %v\n", err) |
| 103 | + return |
| 104 | + } |
| 105 | + |
| 106 | + fmt.Printf("Got %d records\n", results.Count) |
| 107 | + } else if status == 4 { |
| 108 | + fmt.Println("Run failed!") |
| 109 | + } |
| 110 | +} |
| 111 | + |
| 112 | +func runScraperAsync(params map[string]interface{}) (string, error) { |
| 113 | + body, _ := json.Marshal(params) |
| 114 | + |
| 115 | + client := &http.Client{Timeout: time.Duration(TIMEOUT) * time.Second} |
| 116 | + req, _ := http.NewRequest("POST", API_BASE_URL+"/api/v1/scraper/run", bytes.NewBuffer(body)) |
| 117 | + req.Header.Set("api-key", API_KEY) |
| 118 | + req.Header.Set("Content-Type", "application/json") |
| 119 | + |
| 120 | + resp, err := client.Do(req) |
| 121 | + if err != nil { |
| 122 | + return "", err |
| 123 | + } |
| 124 | + defer resp.Body.Close() |
| 125 | + |
| 126 | + respBody, _ := io.ReadAll(resp.Body) |
| 127 | + |
| 128 | + if resp.StatusCode != 200 { |
| 129 | + return "", fmt.Errorf("HTTP %d", resp.StatusCode) |
| 130 | + } |
| 131 | + |
| 132 | + var result ApiResponse |
| 133 | + json.Unmarshal(respBody, &result) |
| 134 | + |
| 135 | + if result.Code != 0 { |
| 136 | + return "", fmt.Errorf("%s (code: %d)", result.Message, result.Code) |
| 137 | + } |
| 138 | + |
| 139 | + var runData RunData |
| 140 | + json.Unmarshal(result.Data, &runData) |
| 141 | + |
| 142 | + return runData.RunSlug, nil |
| 143 | +} |
| 144 | + |
| 145 | +func getRunStatus(runSlug string) (int, *StatusData, error) { |
| 146 | + body, _ := json.Marshal(map[string]string{"run_slug": runSlug}) |
| 147 | + |
| 148 | + client := &http.Client{Timeout: time.Duration(TIMEOUT) * time.Second} |
| 149 | + req, _ := http.NewRequest("POST", API_BASE_URL+"/api/v1/run/detail", bytes.NewBuffer(body)) |
| 150 | + req.Header.Set("api-key", API_KEY) |
| 151 | + req.Header.Set("Content-Type", "application/json") |
| 152 | + |
| 153 | + resp, err := client.Do(req) |
| 154 | + if err != nil { |
| 155 | + return -1, nil, err |
| 156 | + } |
| 157 | + defer resp.Body.Close() |
| 158 | + |
| 159 | + respBody, _ := io.ReadAll(resp.Body) |
| 160 | + |
| 161 | + var result ApiResponse |
| 162 | + json.Unmarshal(respBody, &result) |
| 163 | + |
| 164 | + if result.Code != 0 { |
| 165 | + return -1, nil, fmt.Errorf("%s", result.Message) |
| 166 | + } |
| 167 | + |
| 168 | + var statusData StatusData |
| 169 | + json.Unmarshal(result.Data, &statusData) |
| 170 | + |
| 171 | + return statusData.Status, &statusData, nil |
| 172 | +} |
| 173 | + |
| 174 | +func pollUntilComplete(runSlug string) (int, *StatusData, error) { |
| 175 | + terminalStates := []int{3, 4, 5} |
| 176 | + maxWait := 300 * time.Second |
| 177 | + startTime := time.Now() |
| 178 | + |
| 179 | + for time.Since(startTime) < maxWait { |
| 180 | + status, statusData, err := getRunStatus(runSlug) |
| 181 | + if err != nil { |
| 182 | + return -1, nil, err |
| 183 | + } |
| 184 | + |
| 185 | + for _, terminal := range terminalStates { |
| 186 | + if status == terminal { |
| 187 | + return status, statusData, nil |
| 188 | + } |
| 189 | + } |
| 190 | + |
| 191 | + fmt.Printf("Status: %d (Running...)\n", status) |
| 192 | + time.Sleep(5 * time.Second) |
| 193 | + } |
| 194 | + |
| 195 | + return -1, nil, fmt.Errorf("timeout") |
| 196 | +} |
| 197 | + |
| 198 | +func getResults(runSlug string) (*ResultData, error) { |
| 199 | + body, _ := json.Marshal(map[string]interface{}{ |
| 200 | + "run_slug": runSlug, |
| 201 | + "page_index": 1, |
| 202 | + "page_size": 20, |
| 203 | + }) |
| 204 | + |
| 205 | + client := &http.Client{Timeout: time.Duration(TIMEOUT) * time.Second} |
| 206 | + req, _ := http.NewRequest("POST", API_BASE_URL+"/api/v1/run/result/list", bytes.NewBuffer(body)) |
| 207 | + req.Header.Set("api-key", API_KEY) |
| 208 | + req.Header.Set("Content-Type", "application/json") |
| 209 | + |
| 210 | + resp, err := client.Do(req) |
| 211 | + if err != nil { |
| 212 | + return nil, err |
| 213 | + } |
| 214 | + defer resp.Body.Close() |
| 215 | + |
| 216 | + respBody, _ := io.ReadAll(resp.Body) |
| 217 | + |
| 218 | + var result ApiResponse |
| 219 | + json.Unmarshal(respBody, &result) |
| 220 | + |
| 221 | + if result.Code != 0 { |
| 222 | + return nil, fmt.Errorf("%s", result.Message) |
| 223 | + } |
| 224 | + |
| 225 | + var resultData ResultData |
| 226 | + json.Unmarshal(result.Data, &resultData) |
| 227 | + |
| 228 | + return &resultData, nil |
| 229 | +} |
| 230 | +``` |
| 231 | + |
| 232 | +## Key Functions |
| 233 | + |
| 234 | +| Function | Purpose | |
| 235 | +|----------|---------| |
| 236 | +| `runScraperAsync()` | Start an async Worker run | |
| 237 | +| `getRunStatus()` | Get current run status | |
| 238 | +| `pollUntilComplete()` | Poll until terminal state (success/failure) | |
| 239 | +| `getResults()` | Retrieve result data with pagination | |
| 240 | + |
| 241 | +## Status Codes |
| 242 | + |
| 243 | +| Code | Status | |
| 244 | +|------|--------| |
| 245 | +| 1 | Ready | |
| 246 | +| 2 | Running | |
| 247 | +| 3 | Succeeded | |
| 248 | +| 4 | Failed | |
| 249 | +| 5 | Aborting | |
| 250 | + |
| 251 | +## Next Steps |
| 252 | + |
| 253 | +- [Back to Integration Guide](/api/integration/) |
0 commit comments