Skip to content

Commit c645236

Browse files
github-actions[bot]abeizn
github-actions[bot]
and
abeizn
authored
fix: github run api page<35 limit (#6876) (#6877)
* fix: github run api page<35 limit * fix: some tips * fix: update note Co-authored-by: abeizn <zikuan.an@merico.dev>
1 parent 19c6ec0 commit c645236

File tree

1 file changed

+262
-8
lines changed

1 file changed

+262
-8
lines changed

‎backend/plugins/github/tasks/cicd_run_collector.go‎

Lines changed: 262 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -41,11 +41,6 @@ const RAW_RUN_TABLE = "github_api_runs"
4141
// https://github.com/apache/incubator-devlake/issues/3199
4242
const PAGE_SIZE = 30
4343

44-
type GithubRawRunsResult struct {
45-
TotalCount int64 `json:"total_count"`
46-
GithubWorkflowRuns []json.RawMessage `json:"workflow_runs"`
47-
}
48-
4944
type SimpleGithubApiJob struct {
5045
ID int64
5146
CreatedAt common.Iso8601Time `json:"created_at"`
@@ -80,7 +75,9 @@ func CollectRuns(taskCtx plugin.SubTaskContext) errors.Error {
8075
UrlTemplate: "repos/{{ .Params.Name }}/actions/runs",
8176
Query: func(reqData *helper.RequestData, createdAfter *time.Time) (url.Values, errors.Error) {
8277
query := url.Values{}
83-
query.Set("status", "completed")
78+
// GitHub API returns only the first 34 pages (with a size of 30) when specifying status=compleleted, try the following API request to verify the problem.
79+
// https://api.github.com/repos/apache/incubator-devlake/actions/runs?per_page=30&page=35&status=completed
80+
// query.Set("status", "completed")
8481
query.Set("page", fmt.Sprintf("%v", reqData.Pager.Page))
8582
query.Set("per_page", fmt.Sprintf("%v", reqData.Pager.Size))
8683
return query, nil
@@ -91,10 +88,21 @@ func CollectRuns(taskCtx plugin.SubTaskContext) errors.Error {
9188
if err != nil {
9289
return nil, err
9390
}
94-
if len(body.GithubWorkflowRuns) == 0 {
91+
if len(body.WorkflowRuns) == 0 {
9592
return nil, nil
9693
}
97-
return body.GithubWorkflowRuns, nil
94+
// filter out the runs that are not completed
95+
filteredRuns := make([]json.RawMessage, 0)
96+
for _, run := range body.WorkflowRuns {
97+
if run.Status == "completed" {
98+
runJSON, err := json.Marshal(run)
99+
if err != nil {
100+
return nil, errors.Convert(err)
101+
}
102+
filteredRuns = append(filteredRuns, json.RawMessage(runJSON))
103+
}
104+
}
105+
return filteredRuns, nil
98106
},
99107
},
100108
GetCreated: func(item json.RawMessage) (time.Time, errors.Error) {
@@ -115,3 +123,249 @@ func CollectRuns(taskCtx plugin.SubTaskContext) errors.Error {
115123
return collector.Execute()
116124

117125
}
126+
127+
type GithubRawRunsResult struct {
128+
TotalCount int `json:"total_count"`
129+
WorkflowRuns []struct {
130+
ID int64 `json:"id"`
131+
Name string `json:"name"`
132+
NodeID string `json:"node_id"`
133+
HeadBranch string `json:"head_branch"`
134+
HeadSha string `json:"head_sha"`
135+
Path string `json:"path"`
136+
DisplayTitle string `json:"display_title"`
137+
RunNumber int `json:"run_number"`
138+
Event string `json:"event"`
139+
Status string `json:"status"`
140+
Conclusion string `json:"conclusion"`
141+
WorkflowID int `json:"workflow_id"`
142+
CheckSuiteID int64 `json:"check_suite_id"`
143+
CheckSuiteNodeID string `json:"check_suite_node_id"`
144+
URL string `json:"url"`
145+
HTMLURL string `json:"html_url"`
146+
PullRequests []struct {
147+
URL string `json:"url"`
148+
ID int `json:"id"`
149+
Number int `json:"number"`
150+
Head struct {
151+
Ref string `json:"ref"`
152+
Sha string `json:"sha"`
153+
Repo struct {
154+
ID int `json:"id"`
155+
URL string `json:"url"`
156+
Name string `json:"name"`
157+
} `json:"repo"`
158+
} `json:"head"`
159+
Base struct {
160+
Ref string `json:"ref"`
161+
Sha string `json:"sha"`
162+
Repo struct {
163+
ID int `json:"id"`
164+
URL string `json:"url"`
165+
Name string `json:"name"`
166+
} `json:"repo"`
167+
} `json:"base"`
168+
} `json:"pull_requests"`
169+
CreatedAt time.Time `json:"created_at"`
170+
UpdatedAt time.Time `json:"updated_at"`
171+
Actor struct {
172+
Login string `json:"login"`
173+
ID int `json:"id"`
174+
NodeID string `json:"node_id"`
175+
AvatarURL string `json:"avatar_url"`
176+
GravatarID string `json:"gravatar_id"`
177+
URL string `json:"url"`
178+
HTMLURL string `json:"html_url"`
179+
FollowersURL string `json:"followers_url"`
180+
FollowingURL string `json:"following_url"`
181+
GistsURL string `json:"gists_url"`
182+
StarredURL string `json:"starred_url"`
183+
SubscriptionsURL string `json:"subscriptions_url"`
184+
OrganizationsURL string `json:"organizations_url"`
185+
ReposURL string `json:"repos_url"`
186+
EventsURL string `json:"events_url"`
187+
ReceivedEventsURL string `json:"received_events_url"`
188+
Type string `json:"type"`
189+
SiteAdmin bool `json:"site_admin"`
190+
} `json:"actor"`
191+
RunAttempt int `json:"run_attempt"`
192+
ReferencedWorkflows []any `json:"referenced_workflows"`
193+
RunStartedAt time.Time `json:"run_started_at"`
194+
TriggeringActor struct {
195+
Login string `json:"login"`
196+
ID int `json:"id"`
197+
NodeID string `json:"node_id"`
198+
AvatarURL string `json:"avatar_url"`
199+
GravatarID string `json:"gravatar_id"`
200+
URL string `json:"url"`
201+
HTMLURL string `json:"html_url"`
202+
FollowersURL string `json:"followers_url"`
203+
FollowingURL string `json:"following_url"`
204+
GistsURL string `json:"gists_url"`
205+
StarredURL string `json:"starred_url"`
206+
SubscriptionsURL string `json:"subscriptions_url"`
207+
OrganizationsURL string `json:"organizations_url"`
208+
ReposURL string `json:"repos_url"`
209+
EventsURL string `json:"events_url"`
210+
ReceivedEventsURL string `json:"received_events_url"`
211+
Type string `json:"type"`
212+
SiteAdmin bool `json:"site_admin"`
213+
} `json:"triggering_actor"`
214+
JobsURL string `json:"jobs_url"`
215+
LogsURL string `json:"logs_url"`
216+
CheckSuiteURL string `json:"check_suite_url"`
217+
ArtifactsURL string `json:"artifacts_url"`
218+
CancelURL string `json:"cancel_url"`
219+
RerunURL string `json:"rerun_url"`
220+
PreviousAttemptURL any `json:"previous_attempt_url"`
221+
WorkflowURL string `json:"workflow_url"`
222+
HeadCommit struct {
223+
ID string `json:"id"`
224+
TreeID string `json:"tree_id"`
225+
Message string `json:"message"`
226+
Timestamp time.Time `json:"timestamp"`
227+
Author struct {
228+
Name string `json:"name"`
229+
Email string `json:"email"`
230+
} `json:"author"`
231+
Committer struct {
232+
Name string `json:"name"`
233+
Email string `json:"email"`
234+
} `json:"committer"`
235+
} `json:"head_commit"`
236+
Repository struct {
237+
ID int `json:"id"`
238+
NodeID string `json:"node_id"`
239+
Name string `json:"name"`
240+
FullName string `json:"full_name"`
241+
Private bool `json:"private"`
242+
Owner struct {
243+
Login string `json:"login"`
244+
ID int `json:"id"`
245+
NodeID string `json:"node_id"`
246+
AvatarURL string `json:"avatar_url"`
247+
GravatarID string `json:"gravatar_id"`
248+
URL string `json:"url"`
249+
HTMLURL string `json:"html_url"`
250+
FollowersURL string `json:"followers_url"`
251+
FollowingURL string `json:"following_url"`
252+
GistsURL string `json:"gists_url"`
253+
StarredURL string `json:"starred_url"`
254+
SubscriptionsURL string `json:"subscriptions_url"`
255+
OrganizationsURL string `json:"organizations_url"`
256+
ReposURL string `json:"repos_url"`
257+
EventsURL string `json:"events_url"`
258+
ReceivedEventsURL string `json:"received_events_url"`
259+
Type string `json:"type"`
260+
SiteAdmin bool `json:"site_admin"`
261+
} `json:"owner"`
262+
HTMLURL string `json:"html_url"`
263+
Description string `json:"description"`
264+
Fork bool `json:"fork"`
265+
URL string `json:"url"`
266+
ForksURL string `json:"forks_url"`
267+
KeysURL string `json:"keys_url"`
268+
CollaboratorsURL string `json:"collaborators_url"`
269+
TeamsURL string `json:"teams_url"`
270+
HooksURL string `json:"hooks_url"`
271+
IssueEventsURL string `json:"issue_events_url"`
272+
EventsURL string `json:"events_url"`
273+
AssigneesURL string `json:"assignees_url"`
274+
BranchesURL string `json:"branches_url"`
275+
TagsURL string `json:"tags_url"`
276+
BlobsURL string `json:"blobs_url"`
277+
GitTagsURL string `json:"git_tags_url"`
278+
GitRefsURL string `json:"git_refs_url"`
279+
TreesURL string `json:"trees_url"`
280+
StatusesURL string `json:"statuses_url"`
281+
LanguagesURL string `json:"languages_url"`
282+
StargazersURL string `json:"stargazers_url"`
283+
ContributorsURL string `json:"contributors_url"`
284+
SubscribersURL string `json:"subscribers_url"`
285+
SubscriptionURL string `json:"subscription_url"`
286+
CommitsURL string `json:"commits_url"`
287+
GitCommitsURL string `json:"git_commits_url"`
288+
CommentsURL string `json:"comments_url"`
289+
IssueCommentURL string `json:"issue_comment_url"`
290+
ContentsURL string `json:"contents_url"`
291+
CompareURL string `json:"compare_url"`
292+
MergesURL string `json:"merges_url"`
293+
ArchiveURL string `json:"archive_url"`
294+
DownloadsURL string `json:"downloads_url"`
295+
IssuesURL string `json:"issues_url"`
296+
PullsURL string `json:"pulls_url"`
297+
MilestonesURL string `json:"milestones_url"`
298+
NotificationsURL string `json:"notifications_url"`
299+
LabelsURL string `json:"labels_url"`
300+
ReleasesURL string `json:"releases_url"`
301+
DeploymentsURL string `json:"deployments_url"`
302+
} `json:"repository"`
303+
HeadRepository struct {
304+
ID int `json:"id"`
305+
NodeID string `json:"node_id"`
306+
Name string `json:"name"`
307+
FullName string `json:"full_name"`
308+
Private bool `json:"private"`
309+
Owner struct {
310+
Login string `json:"login"`
311+
ID int `json:"id"`
312+
NodeID string `json:"node_id"`
313+
AvatarURL string `json:"avatar_url"`
314+
GravatarID string `json:"gravatar_id"`
315+
URL string `json:"url"`
316+
HTMLURL string `json:"html_url"`
317+
FollowersURL string `json:"followers_url"`
318+
FollowingURL string `json:"following_url"`
319+
GistsURL string `json:"gists_url"`
320+
StarredURL string `json:"starred_url"`
321+
SubscriptionsURL string `json:"subscriptions_url"`
322+
OrganizationsURL string `json:"organizations_url"`
323+
ReposURL string `json:"repos_url"`
324+
EventsURL string `json:"events_url"`
325+
ReceivedEventsURL string `json:"received_events_url"`
326+
Type string `json:"type"`
327+
SiteAdmin bool `json:"site_admin"`
328+
} `json:"owner"`
329+
HTMLURL string `json:"html_url"`
330+
Description string `json:"description"`
331+
Fork bool `json:"fork"`
332+
URL string `json:"url"`
333+
ForksURL string `json:"forks_url"`
334+
KeysURL string `json:"keys_url"`
335+
CollaboratorsURL string `json:"collaborators_url"`
336+
TeamsURL string `json:"teams_url"`
337+
HooksURL string `json:"hooks_url"`
338+
IssueEventsURL string `json:"issue_events_url"`
339+
EventsURL string `json:"events_url"`
340+
AssigneesURL string `json:"assignees_url"`
341+
BranchesURL string `json:"branches_url"`
342+
TagsURL string `json:"tags_url"`
343+
BlobsURL string `json:"blobs_url"`
344+
GitTagsURL string `json:"git_tags_url"`
345+
GitRefsURL string `json:"git_refs_url"`
346+
TreesURL string `json:"trees_url"`
347+
StatusesURL string `json:"statuses_url"`
348+
LanguagesURL string `json:"languages_url"`
349+
StargazersURL string `json:"stargazers_url"`
350+
ContributorsURL string `json:"contributors_url"`
351+
SubscribersURL string `json:"subscribers_url"`
352+
SubscriptionURL string `json:"subscription_url"`
353+
CommitsURL string `json:"commits_url"`
354+
GitCommitsURL string `json:"git_commits_url"`
355+
CommentsURL string `json:"comments_url"`
356+
IssueCommentURL string `json:"issue_comment_url"`
357+
ContentsURL string `json:"contents_url"`
358+
CompareURL string `json:"compare_url"`
359+
MergesURL string `json:"merges_url"`
360+
ArchiveURL string `json:"archive_url"`
361+
DownloadsURL string `json:"downloads_url"`
362+
IssuesURL string `json:"issues_url"`
363+
PullsURL string `json:"pulls_url"`
364+
MilestonesURL string `json:"milestones_url"`
365+
NotificationsURL string `json:"notifications_url"`
366+
LabelsURL string `json:"labels_url"`
367+
ReleasesURL string `json:"releases_url"`
368+
DeploymentsURL string `json:"deployments_url"`
369+
} `json:"head_repository"`
370+
} `json:"workflow_runs"`
371+
}

0 commit comments

Comments
 (0)