Skip to content

Commit 16a1d63

Browse files
motatoesmotatoes
authored
implement caching layer for digger config (#1825)
* implement caching layer for digger config
1 parent 68e3850 commit 16a1d63

File tree

14 files changed

+735
-257
lines changed

14 files changed

+735
-257
lines changed

‎.github/workflows/backend_test.yml

+3-1
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,9 @@ jobs:
3535
working-directory: backend
3636

3737
- name: Test
38-
run: go test -v ./...
38+
run: |
39+
go install github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@latest
40+
go test -json ./... | gotestfmt
3941
env:
4042
GITHUB_PAT_TOKEN: ${{ secrets.TOKEN_GITHUB }}
4143
working-directory: backend

‎backend/bootstrap/main.go

+3
Original file line numberDiff line numberDiff line change
@@ -221,6 +221,9 @@ func Bootstrap(templates embed.FS, diggerController controllers.DiggerController
221221
runsApiGroup.GET("/:run_id", controllers.RunDetails)
222222
runsApiGroup.POST("/:run_id/approve", controllers.ApproveRun)
223223

224+
// internal endpoints not meant to be exposed to public and protected behing webhook secret
225+
r.POST("_internal/update_repo_cache", middleware.WebhookAuth(), diggerController.UpdateRepoCache)
226+
224227
fronteggWebhookProcessor.POST("/create-org-from-frontegg", controllers.CreateFronteggOrgFromWebhook)
225228

226229
return r

‎backend/controllers/cache.go

+101
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
package controllers
2+
3+
import (
4+
"fmt"
5+
"github.com/diggerhq/digger/backend/models"
6+
"github.com/diggerhq/digger/backend/utils"
7+
dg_configuration "github.com/diggerhq/digger/libs/digger_config"
8+
"github.com/gin-gonic/gin"
9+
"log"
10+
"net/http"
11+
"os"
12+
"path"
13+
"strings"
14+
)
15+
16+
func (d DiggerController) UpdateRepoCache(c *gin.Context) {
17+
type UpdateCacheRequest struct {
18+
RepoFullName string `json:"repo_full_name"`
19+
Branch string `json:"branch"`
20+
OrgId uint `json:"org_id"`
21+
InstallationId int64 `json:"installation_id"`
22+
}
23+
24+
var request UpdateCacheRequest
25+
err := c.BindJSON(&request)
26+
if err != nil {
27+
log.Printf("Error binding JSON: %v", err)
28+
c.JSON(http.StatusInternalServerError, gin.H{"error": "Error binding JSON"})
29+
return
30+
}
31+
32+
repoFullName := request.RepoFullName
33+
installationId := request.InstallationId
34+
link, err := models.DB.GetGithubAppInstallationLink(installationId)
35+
if err != nil {
36+
log.Printf("could not installation link: %v", err)
37+
c.String(500, fmt.Sprintf("coulnt not find installation link %v %v", repoFullName, installationId))
38+
return
39+
40+
}
41+
orgId := link.OrganisationId
42+
43+
log.Printf("the org id is %v", orgId)
44+
45+
repoOwner, repoName, _ := strings.Cut(repoFullName, "/")
46+
repoDiggerName := strings.ReplaceAll(repoFullName, "/", "-")
47+
48+
repo, err := models.DB.GetRepo(orgId, repoDiggerName)
49+
if err != nil {
50+
log.Printf("could not get repo: %v", err)
51+
c.String(500, fmt.Sprintf("coulnt not get repository %v %v", repoFullName, orgId))
52+
return
53+
}
54+
55+
cloneUrl := repo.RepoUrl
56+
branch := request.Branch
57+
58+
//ghInstallation, err := models.DB.GetInstallationForRepo(repoFullName)
59+
//if err != nil {
60+
// log.Printf("could not get repo: %v", err)
61+
// c.String(500, fmt.Sprintf("coulnt not get repository %v %v", repoFullName, orgId))
62+
// return
63+
//}
64+
65+
_, token, err := utils.GetGithubService(d.GithubClientProvider, installationId, repoFullName, repoOwner, repoName)
66+
if err != nil {
67+
log.Printf("could not get github service :%v", err)
68+
c.String(500, fmt.Sprintf("could not get github service %v %v", repoFullName, orgId))
69+
return
70+
}
71+
72+
var diggerYmlStr string
73+
var config *dg_configuration.DiggerConfig
74+
75+
// update the cache here, do it async for immediate response
76+
go func() {
77+
err = utils.CloneGitRepoAndDoAction(cloneUrl, branch, *token, func(dir string) error {
78+
diggerYmlBytes, err := os.ReadFile(path.Join(dir, "digger.yml"))
79+
diggerYmlStr = string(diggerYmlBytes)
80+
config, _, _, err = dg_configuration.LoadDiggerConfig(dir, true, nil)
81+
if err != nil {
82+
log.Printf("Error loading digger config: %v", err)
83+
return err
84+
}
85+
return nil
86+
})
87+
88+
if err != nil {
89+
log.Printf("could not load digger config :%v", err)
90+
return
91+
}
92+
_, err = models.DB.UpsertRepoCache(orgId, repoFullName, diggerYmlStr, *config)
93+
if err != nil {
94+
log.Printf("could upadate repo cache :%v", err)
95+
return
96+
97+
}
98+
}()
99+
100+
c.String(200, "successfully submitted cache for processing, check backend logs for progress")
101+
}

‎backend/controllers/github.go

+46-4
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import (
2424
"os"
2525
"path"
2626
"reflect"
27+
"slices"
2728
"strconv"
2829
"strings"
2930

@@ -53,7 +54,7 @@ func (d DiggerController) GithubAppWebHook(c *gin.Context) {
5354
log.Printf("GithubAppWebHook")
5455

5556
appID := c.GetHeader("X-GitHub-Hook-Installation-Target-ID")
56-
log.Printf("app id from header is: %v", appID)
57+
5758
_, _, webhookSecret, _, err := d.GithubClientProvider.FetchCredentials(appID)
5859

5960
payload, err := github.ValidatePayload(c.Request, []byte(webhookSecret))
@@ -316,6 +317,10 @@ func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullR
316317
commitSha := payload.PullRequest.Head.GetSHA()
317318
branch := payload.PullRequest.Head.GetRef()
318319
action := *payload.Action
320+
labels := payload.PullRequest.Labels
321+
prLabelsStr := lo.Map(labels, func(label *github.Label, i int) string {
322+
return *label.Name
323+
})
319324

320325
link, err := models.DB.GetGithubAppInstallationLink(installationId)
321326
if err != nil {
@@ -361,7 +366,7 @@ func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullR
361366
}
362367
}
363368

364-
diggerYmlStr, ghService, config, projectsGraph, _, _, err := getDiggerConfigForPR(gh, installationId, repoFullName, repoOwner, repoName, cloneURL, prNumber)
369+
diggerYmlStr, ghService, config, projectsGraph, _, _, err := getDiggerConfigForPR(gh, organisationId, prLabelsStr, installationId, repoFullName, repoOwner, repoName, cloneURL, prNumber)
365370
if err != nil {
366371
log.Printf("getDiggerConfigForPR error: %v", err)
367372
return fmt.Errorf("error getting digger config")
@@ -563,7 +568,7 @@ func GetDiggerConfigForBranch(gh utils.GithubClientProvider, installationId int6
563568
}
564569

565570
// TODO: Refactor this func to receive ghService as input
566-
func getDiggerConfigForPR(gh utils.GithubClientProvider, installationId int64, repoFullName string, repoOwner string, repoName string, cloneUrl string, prNumber int) (string, *dg_github.GithubService, *dg_configuration.DiggerConfig, graph.Graph[string, dg_configuration.Project], *string, *string, error) {
571+
func getDiggerConfigForPR(gh utils.GithubClientProvider, orgId uint, prLabels []string, installationId int64, repoFullName string, repoOwner string, repoName string, cloneUrl string, prNumber int) (string, *dg_github.GithubService, *dg_configuration.DiggerConfig, graph.Graph[string, dg_configuration.Project], *string, *string, error) {
567572
ghService, _, err := utils.GetGithubService(gh, installationId, repoFullName, repoOwner, repoName)
568573
if err != nil {
569574
log.Printf("Error getting github service: %v", err)
@@ -583,6 +588,17 @@ func getDiggerConfigForPR(gh utils.GithubClientProvider, installationId int64, r
583588
return "", nil, nil, nil, nil, nil, fmt.Errorf("error getting changed files")
584589
}
585590

591+
// check if items should be loaded from cache
592+
if val, _ := os.LookupEnv("DIGGER_CONFIG_REPO_CACHE_ENABLED"); val == "1" && !slices.Contains(prLabels, "digger:no-cache") {
593+
diggerYmlStr, config, dependencyGraph, err := retrieveConfigFromCache(orgId, repoFullName)
594+
if err != nil {
595+
log.Printf("could not load from cache")
596+
} else {
597+
log.Printf("successfully loaded from cache")
598+
return diggerYmlStr, ghService, config, *dependencyGraph, &prBranch, &prCommitSha, nil
599+
}
600+
}
601+
586602
diggerYmlStr, ghService, config, dependencyGraph, err := GetDiggerConfigForBranch(gh, installationId, repoFullName, repoOwner, repoName, cloneUrl, prBranch, changedFiles)
587603
if err != nil {
588604
log.Printf("Error loading digger.yml: %v", err)
@@ -593,6 +609,28 @@ func getDiggerConfigForPR(gh utils.GithubClientProvider, installationId int64, r
593609
return diggerYmlStr, ghService, config, dependencyGraph, &prBranch, &prCommitSha, nil
594610
}
595611

612+
func retrieveConfigFromCache(orgId uint, repoFullName string) (string, *dg_configuration.DiggerConfig, *graph.Graph[string, dg_configuration.Project], error) {
613+
repoCache, err := models.DB.GetRepoCache(orgId, repoFullName)
614+
if err != nil {
615+
log.Printf("Error: failed to load repoCache, going to try live load %v", err)
616+
return "", nil, nil, fmt.Errorf("")
617+
}
618+
var config dg_configuration.DiggerConfig
619+
err = json.Unmarshal(repoCache.DiggerConfig, &config)
620+
if err != nil {
621+
log.Printf("Error: failed to load repoCache unmarshall config %v", err)
622+
return "", nil, nil, fmt.Errorf("failed to load repoCache unmarshall config %v", err)
623+
}
624+
625+
projectsGraph, err := dg_configuration.CreateProjectDependencyGraph(config.Projects)
626+
if err != nil {
627+
log.Printf("error retrieving graph of dependencies: %v", err)
628+
return "", nil, nil, fmt.Errorf("error retrieving graph of dependencies: %v", err)
629+
}
630+
631+
return repoCache.DiggerYmlStr, &config, &projectsGraph, nil
632+
}
633+
596634
func GetRepoByInstllationId(installationId int64, repoOwner string, repoName string) (*models.Repo, error) {
597635
link, err := models.DB.GetGithubAppInstallationLink(installationId)
598636
if err != nil {
@@ -634,6 +672,10 @@ func handleIssueCommentEvent(gh utils.GithubClientProvider, payload *github.Issu
634672
commentBody := *payload.Comment.Body
635673
defaultBranch := *payload.Repo.DefaultBranch
636674
isPullRequest := payload.Issue.IsPullRequest()
675+
labels := payload.Issue.Labels
676+
prLabelsStr := lo.Map(labels, func(label *github.Label, i int) string {
677+
return *label.Name
678+
})
637679

638680
if !isPullRequest {
639681
log.Printf("comment not on pullrequest, ignroning")
@@ -671,7 +713,7 @@ func handleIssueCommentEvent(gh utils.GithubClientProvider, payload *github.Issu
671713
}
672714
}
673715

674-
diggerYmlStr, ghService, config, projectsGraph, branch, commitSha, err := getDiggerConfigForPR(gh, installationId, repoFullName, repoOwner, repoName, cloneURL, issueNumber)
716+
diggerYmlStr, ghService, config, projectsGraph, branch, commitSha, err := getDiggerConfigForPR(gh, orgId, prLabelsStr, installationId, repoFullName, repoOwner, repoName, cloneURL, issueNumber)
675717
if err != nil {
676718
commentReporterManager.UpdateComment(fmt.Sprintf(":x: Could not load digger config, error: %v", err))
677719
log.Printf("getDiggerConfigForPR error: %v", err)

0 commit comments

Comments
 (0)