Skip to content

Commit

Permalink
merge develop into comment refactors (#1867)
Browse files Browse the repository at this point in the history 
* add flag to ignore all external directories per project (#1851)

* add flag to ignore all external directories per project

* revert includeparentblocks flag (#1852)

* improve efficiency in terragrunt generation (#1854)

* improve efficiency in terragrunt generation

* Update action.yml (#1856)

* handle crashes in goroutine events (#1857)

* fix/recover from webhook goroutines (#1858)

* handle crashes in goroutine events

* include stacktrace in errors

* wip generation of terraform code from application code (#1855)

* terraform code generation demo


---------

Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>

* fix: incorrect comment in backend/migrations (#1860)

* Update setup-opentofu to fix issues with 1.6.x downloads (#1861)

* restructure docs to have no columns (#1862)

* Create curl_bootstrap.sh

* refactor codegen parts (#1863)

* refactor codegen parts

* publish ai summaries (#1864)

* publish ai summaries

* add a header for summary comment (#1865)

* fix heading (#1866)

---------

Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>
Co-authored-by: Aldo <[email protected]>
Co-authored-by: Hugo Samayoa <[email protected]>
  • Loading branch information
@motatoes @coderabbitai
@aldoborrero @htplbc
4 people authored Jan 2, 2025
1 parent 1a48f98 commit 60dbc32
Show file tree
Hide file tree
Showing 31 changed files with 949 additions and 784 deletions.
8 changes: 3 additions & 5 deletions action.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -428,11 +428,9 @@ runs:
name: cache-save
if: ${{ always() && inputs.cache-dependencies == 'true' && steps.restore_cache.outputs.cache-hit != 'true' }}
with:
path: |
${{ steps.golang-env.outputs.build-cache-path }}
${{ steps.golang-env.outputs.module-cache-path }}
key: digger-cli-cache-${{ hashFiles('.digger.go.sum') }}

path: ${{ github.workspace }}/cache
key: digger-cache-${{ hashFiles('**/cache') }}

branding:
icon: globe
color: purple
246 changes: 232 additions & 14 deletions backend/controllers/github.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,36 +10,36 @@ import (
"github.com/diggerhq/digger/backend/ci_backends"
config2 "github.com/diggerhq/digger/backend/config"
"github.com/diggerhq/digger/backend/locking"
"github.com/diggerhq/digger/backend/middleware"
"github.com/diggerhq/digger/backend/models"
"github.com/diggerhq/digger/backend/segment"
"github.com/diggerhq/digger/backend/services"
"github.com/diggerhq/digger/backend/utils"
"github.com/diggerhq/digger/libs/ci"
"github.com/diggerhq/digger/libs/ci/generic"
dg_github "github.com/diggerhq/digger/libs/ci/github"
comment_updater "github.com/diggerhq/digger/libs/comment_utils/reporting"
dg_configuration "github.com/diggerhq/digger/libs/digger_config"
dg_locking "github.com/diggerhq/digger/libs/locking"
orchestrator_scheduler "github.com/diggerhq/digger/libs/scheduler"
"github.com/dominikbraun/graph"
"github.com/gin-gonic/gin"
"github.com/google/go-github/v61/github"
"github.com/google/uuid"
"github.com/samber/lo"
"golang.org/x/oauth2"
"gorm.io/gorm"
"log"
"math/rand"
"net/http"
"net/url"
"os"
"path"
"path/filepath"
"reflect"
"runtime/debug"
"slices"
"strconv"
"strings"

"github.com/diggerhq/digger/backend/middleware"
"github.com/diggerhq/digger/backend/models"
"github.com/diggerhq/digger/backend/utils"
dg_github "github.com/diggerhq/digger/libs/ci/github"
dg_configuration "github.com/diggerhq/digger/libs/digger_config"
"github.com/dominikbraun/graph"
"github.com/gin-gonic/gin"
"github.com/google/go-github/v61/github"
"github.com/samber/lo"
"golang.org/x/oauth2"
)

type IssueCommentHook func(gh utils.GithubClientProvider, payload *github.IssueCommentEvent, ciBackendProvider ci_backends.CiBackendProvider) error
Expand Down Expand Up @@ -309,6 +309,16 @@ func handleInstallationDeletedEvent(installation *github.InstallationEvent, appI
}

func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullRequestEvent, ciBackendProvider ci_backends.CiBackendProvider, appId int64) error {
defer func() {
if r := recover(); r != nil {
log.Printf("Recovered from panic in handlePullRequestEvent handler: %v", r)
log.Printf("\n=== PANIC RECOVERED ===\n")
log.Printf("Error: %v\n", r)
log.Printf("Stack Trace:\n%s", string(debug.Stack()))
log.Printf("=== END PANIC ===\n")
}
}()

installationId := *payload.Installation.ID
repoName := *payload.Repo.Name
repoOwner := *payload.Repo.Owner.Login
Expand Down Expand Up @@ -376,6 +386,7 @@ func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullR
diggerYmlStr, ghService, config, projectsGraph, _, _, changedFiles, err := getDiggerConfigForPR(gh, organisationId, prLabelsStr, installationId, repoFullName, repoOwner, repoName, cloneURL, prNumber)
if err != nil {
log.Printf("getDiggerConfigForPR error: %v", err)
commentReporterManager.UpdateComment(fmt.Sprintf(":x: Error loading digger config: %v", err))
return fmt.Errorf("error getting digger config")
}

Expand Down Expand Up @@ -501,6 +512,22 @@ func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullR
commentReporterManager.UpdateComment(fmt.Sprintf(":x: could not handle commentId: %v", err))
}



var aiSummaryCommentId = ""
if config.Reporting.AiSummary {
aiSummaryComment, err := ghService.PublishComment(prNumber, "AI Summary will be posted here after completion")
if err != nil {
log.Printf("could not post ai summary comment: %v", err)
commentReporterManager.UpdateComment(fmt.Sprintf(":x: could not post ai comment summary comment id: %v", err))
return fmt.Errorf("could not post ai summary comment: %v", err)
}
aiSummaryCommentId = aiSummaryComment.Id
}

batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, models.DiggerVCSGithub, organisationId, impactedJobsMap, impactedProjectsMap, projectsGraph, installationId, branch, prNumber, repoOwner, repoName, repoFullName, commitSha, commentId, diggerYmlStr, 0, aiSummaryCommentId, config.ReportTerraformOutputs)


placeholderComment, err := ghService.PublishComment(prNumber, "digger report placehoder")
if err != nil {
log.Printf("strconv.ParseInt error: %v", err)
Expand All @@ -509,6 +536,8 @@ func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullR
}

batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, models.DiggerVCSGithub, organisationId, impactedJobsMap, impactedProjectsMap, projectsGraph, installationId, branch, prNumber, repoOwner, repoName, repoFullName, commitSha, commentId, &placeholderComment.Id, diggerYmlStr, 0)


if err != nil {
log.Printf("ConvertJobsToDiggerJobs error: %v", err)
commentReporterManager.UpdateComment(fmt.Sprintf(":x: ConvertJobsToDiggerJobs error: %v", err))
Expand Down Expand Up @@ -582,8 +611,11 @@ func GetDiggerConfigForBranch(gh utils.GithubClientProvider, installationId int6
var dependencyGraph graph.Graph[string, dg_configuration.Project]

err = utils.CloneGitRepoAndDoAction(cloneUrl, branch, "", *token, func(dir string) error {
diggerYmlBytes, err := os.ReadFile(path.Join(dir, "digger.yml"))
diggerYmlStr = string(diggerYmlBytes)
diggerYmlStr, err = dg_configuration.ReadDiggerYmlFileContents(dir)
if err != nil {
log.Printf("could not load digger config: %v", err)
return err
}
config, _, dependencyGraph, err = dg_configuration.LoadDiggerConfig(dir, true, changedFiles)
if err != nil {
log.Printf("Error loading digger config: %v", err)
Expand Down Expand Up @@ -692,6 +724,16 @@ func getBatchType(jobs []orchestrator_scheduler.Job) orchestrator_scheduler.Digg
}

func handleIssueCommentEvent(gh utils.GithubClientProvider, payload *github.IssueCommentEvent, ciBackendProvider ci_backends.CiBackendProvider, appId int64, postCommentHooks []IssueCommentHook) error {
defer func() {
if r := recover(); r != nil {
log.Printf("Recovered from panic in handleIssueCommentEvent handler: %v", r)
log.Printf("\n=== PANIC RECOVERED ===\n")
log.Printf("Error: %v\n", r)
log.Printf("Stack Trace:\n%s", string(debug.Stack()))
log.Printf("=== END PANIC ===\n")
}
}()

installationId := *payload.Installation.ID
repoName := *payload.Repo.Name
repoOwner := *payload.Repo.Owner.Login
Expand Down Expand Up @@ -752,6 +794,15 @@ func handleIssueCommentEvent(gh utils.GithubClientProvider, payload *github.Issu
return fmt.Errorf("error getting digger config")
}

// terraform code generator
if os.Getenv("DIGGER_GENERATION_ENABLED") == "1" {
err = GenerateTerraformFromCode(payload, commentReporterManager, config, defaultBranch, ghService, repoOwner, repoName, commitSha, issueNumber, branch)
if err != nil {
log.Printf("terraform generation failed: %v", err)
return err
}
}

commentIdStr := strconv.FormatInt(userCommentId, 10)
err = ghService.CreateCommentReaction(commentIdStr, string(dg_github.GithubCommentEyesReaction))
if err != nil {
Expand Down Expand Up @@ -883,6 +934,20 @@ func handleIssueCommentEvent(gh utils.GithubClientProvider, payload *github.Issu
return fmt.Errorf("comment reporter error: %v", err)
}


var aiSummaryCommentId = ""
if config.Reporting.AiSummary {
aiSummaryComment, err := ghService.PublishComment(issueNumber, "AI Summary will be posted here after completion")
if err != nil {
log.Printf("could not post ai summary comment: %v", err)
commentReporterManager.UpdateComment(fmt.Sprintf(":x: could not post ai comment summary comment id: %v", err))
return fmt.Errorf("could not post ai summary comment: %v", err)
}
aiSummaryCommentId = aiSummaryComment.Id
}

batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, "github", orgId, impactedProjectsJobMap, impactedProjectsMap, projectsGraph, installationId, *branch, issueNumber, repoOwner, repoName, repoFullName, *commitSha, reporterCommentId, diggerYmlStr, 0, aiSummaryCommentId, config.ReportTerraformOutputs)

placeholderComment, err := ghService.PublishComment(issueNumber, "digger report placehoder")
if err != nil {
log.Printf("strconv.ParseInt error: %v", err)
Expand All @@ -891,6 +956,7 @@ func handleIssueCommentEvent(gh utils.GithubClientProvider, payload *github.Issu
}

batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, "github", orgId, impactedProjectsJobMap, impactedProjectsMap, projectsGraph, installationId, *branch, issueNumber, repoOwner, repoName, repoFullName, *commitSha, reporterCommentId, &placeholderComment.Id, diggerYmlStr, 0)

if err != nil {
log.Printf("ConvertJobsToDiggerJobs error: %v", err)
commentReporterManager.UpdateComment(fmt.Sprintf(":x: ConvertJobsToDiggerJobs error: %v", err))
Expand Down Expand Up @@ -962,6 +1028,158 @@ func handleIssueCommentEvent(gh utils.GithubClientProvider, payload *github.Issu
return nil
}

func GenerateTerraformFromCode(payload *github.IssueCommentEvent, commentReporterManager utils.CommentReporterManager, config *dg_configuration.DiggerConfig, defaultBranch string, ghService *dg_github.GithubService, repoOwner string, repoName string, commitSha *string, issueNumber int, branch *string) error {
if strings.HasPrefix(*payload.Comment.Body, "digger generate") {
projectName := ci.ParseProjectName(*payload.Comment.Body)
if projectName == "" {
commentReporterManager.UpdateComment(fmt.Sprintf(":x: generate requires argument -p <project_name>"))
log.Printf("missing project in command: %v", *payload.Comment.Body)
return fmt.Errorf("generate requires argument -p <project_name>")
}

project := config.GetProject(projectName)
if project == nil {
commentReporterManager.UpdateComment(fmt.Sprintf("could not find project %v in digger.yml", projectName))
log.Printf("could not find project %v in digger.yml", projectName)
return fmt.Errorf("could not find project %v in digger.yml", projectName)
}

commentReporterManager.UpdateComment(fmt.Sprintf(":white_check_mark: Successfully loaded project"))

generationEndpoint := os.Getenv("DIGGER_GENERATION_ENDPOINT")
if generationEndpoint == "" {
commentReporterManager.UpdateComment(fmt.Sprintf(":x: server does not have generation endpoint configured, please verify"))
log.Printf("server does not have generation endpoint configured, please verify")
return fmt.Errorf("server does not have generation endpoint configured, please verify")
}
apiToken := os.Getenv("DIGGER_GENERATION_API_TOKEN")

// Get all code content from the repository at a specific commit
getCodeFromCommit := func(ghService *dg_github.GithubService, repoOwner, repoName string, commitSha *string, projectDir string) (string, error) {
const MaxPatchSize = 1024 * 1024 // 1MB limit

// Get the commit's changes compared to default branch
comparison, _, err := ghService.Client.Repositories.CompareCommits(
context.Background(),
repoOwner,
repoName,
defaultBranch,
*commitSha,
nil,
)
if err != nil {
return "", fmt.Errorf("error comparing commits: %v", err)
}

var appCode strings.Builder
for _, file := range comparison.Files {
if file.Patch == nil {
continue // Skip files without patches
}
log.Printf("Processing patch for file: %s", *file.Filename)
if *file.Additions > 0 {
lines := strings.Split(*file.Patch, "\n")
for _, line := range lines {
if strings.HasPrefix(line, "+") && !strings.HasPrefix(line, "+++") {
appCode.WriteString(strings.TrimPrefix(line, "+"))
appCode.WriteString("\n")
}
}
}
appCode.WriteString("\n")
}

if appCode.Len() == 0 {
return "", fmt.Errorf("no code changes found in commit %s. Please ensure the PR contains added or modified code", *commitSha)
}

return appCode.String(), nil
}

appCode, err := getCodeFromCommit(ghService, repoOwner, repoName, commitSha, project.Dir)
if err != nil {
commentReporterManager.UpdateComment(fmt.Sprintf(":x: Failed to get code content: %v", err))
log.Printf("Error getting code content: %v", err)
return fmt.Errorf("error getting code content: %v", err)
}

commentReporterManager.UpdateComment(fmt.Sprintf(":white_check_mark: Successfully loaded code from commit"))

log.Printf("the app code is: %v", appCode)

commentReporterManager.UpdateComment(fmt.Sprintf("Generating terraform..."))
terraformCode, err := utils.GenerateTerraformCode(appCode, generationEndpoint, apiToken)
if err != nil {
commentReporterManager.UpdateComment(fmt.Sprintf(":x: could not generate terraform code: %v", err))
log.Printf("could not generate terraform code: %v", err)
return fmt.Errorf("could not generate terraform code: %v", err)
}

commentReporterManager.UpdateComment(fmt.Sprintf(":white_check_mark: Generated terraform"))

// comment terraform code to project dir
//project.Dir
log.Printf("terraform code is %v", terraformCode)

baseTree, _, err := ghService.Client.Git.GetTree(context.Background(), repoOwner, repoName, *commitSha, false)
if err != nil {
commentReporterManager.UpdateComment(fmt.Sprintf(":x: Failed to get base tree: %v", err))
log.Printf("Error getting base tree: %v", err)
return fmt.Errorf("error getting base tree: %v", err)
}

// Create a new tree with the new file
treeEntries := []*github.TreeEntry{
{
Path: github.String(filepath.Join(project.Dir, fmt.Sprintf("generated_%v.tf", issueNumber))),
Mode: github.String("100644"),
Type: github.String("blob"),
Content: github.String(terraformCode),
},
}

newTree, _, err := ghService.Client.Git.CreateTree(context.Background(), repoOwner, repoName, *baseTree.SHA, treeEntries)
if err != nil {
commentReporterManager.UpdateComment(fmt.Sprintf(":x: Failed to create new tree: %v", err))
log.Printf("Error creating new tree: %v", err)
return fmt.Errorf("error creating new tree: %v", err)
}

// Create the commit
commitMsg := fmt.Sprintf("Add generated Terraform code for %v", projectName)
commit := &github.Commit{
Message: &commitMsg,
Tree: newTree,
Parents: []*github.Commit{{SHA: commitSha}},
}

newCommit, _, err := ghService.Client.Git.CreateCommit(context.Background(), repoOwner, repoName, commit, nil)
if err != nil {
commentReporterManager.UpdateComment(fmt.Sprintf(":x: Failed to commit Terraform file: %v", err))
log.Printf("Error committing Terraform file: %v", err)
return fmt.Errorf("error committing Terraform file: %v", err)
}

// Update the reference to point to the new commit
ref := &github.Reference{
Ref: github.String(fmt.Sprintf("refs/heads/%s", *branch)),
Object: &github.GitObject{
SHA: newCommit.SHA,
},
}
_, _, err = ghService.Client.Git.UpdateRef(context.Background(), repoOwner, repoName, ref, false)
if err != nil {
commentReporterManager.UpdateComment(fmt.Sprintf(":x: Failed to update branch reference: %v", err))
log.Printf("Error updating branch reference: %v", err)
return fmt.Errorf("error updating branch reference: %v", err)
}

commentReporterManager.UpdateComment(":white_check_mark: Successfully generated and committed Terraform code")
return nil
}
return nil
}

func TriggerDiggerJobs(ciBackend ci_backends.CiBackend, repoFullName string, repoOwner string, repoName string, batchId *uuid.UUID, prNumber int, prService ci.PullRequestService, gh utils.GithubClientProvider) error {
_, err := models.DB.GetDiggerBatch(batchId)
if err != nil {
Expand Down
8 changes: 4 additions & 4 deletions backend/controllers/github_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -731,7 +731,7 @@ func TestJobsTreeWithOneJobsAndTwoProjects(t *testing.T) {
graph, err := configuration.CreateProjectDependencyGraph(projects)
assert.NoError(t, err)

_, result, err := utils.ConvertJobsToDiggerJobs("", "github", 1, jobs, projectMap, graph, 41584295, "", 2, "diggerhq", "parallel_jobs_demo", "diggerhq/parallel_jobs_demo", "", 123, "test", 0)
_, result, err := utils.ConvertJobsToDiggerJobs("", "github", 1, jobs, projectMap, graph, 41584295, "", 2, "diggerhq", "parallel_jobs_demo", "diggerhq/parallel_jobs_demo", "", 123, "test", 0, "", false)
assert.NoError(t, err)
assert.Equal(t, 1, len(result))
parentLinks, err := models.DB.GetDiggerJobParentLinksChildId(&result["dev"].DiggerJobID)
Expand Down Expand Up @@ -760,7 +760,7 @@ func TestJobsTreeWithTwoDependantJobs(t *testing.T) {
projectMap["dev"] = project1
projectMap["prod"] = project2

_, result, err := utils.ConvertJobsToDiggerJobs("", "github", 1, jobs, projectMap, graph, 123, "", 2, "", "", "test", "", 123, "test", 0)
_, result, err := utils.ConvertJobsToDiggerJobs("", "github", 1, jobs, projectMap, graph, 123, "", 2, "", "", "test", "", 123, "test", 0, "", false)
assert.NoError(t, err)
assert.Equal(t, 2, len(result))

Expand Down Expand Up @@ -793,7 +793,7 @@ func TestJobsTreeWithTwoIndependentJobs(t *testing.T) {
projectMap["dev"] = project1
projectMap["prod"] = project2

_, result, err := utils.ConvertJobsToDiggerJobs("", "github", 1, jobs, projectMap, graph, 123, "", 2, "", "", "test", "", 123, "test", 0)
_, result, err := utils.ConvertJobsToDiggerJobs("", "github", 1, jobs, projectMap, graph, 123, "", 2, "", "", "test", "", 123, "test", 0, "", false)
assert.NoError(t, err)
assert.Equal(t, 2, len(result))
parentLinks, err := models.DB.GetDiggerJobParentLinksChildId(&result["dev"].DiggerJobID)
Expand Down Expand Up @@ -838,7 +838,7 @@ func TestJobsTreeWithThreeLevels(t *testing.T) {
projectMap["555"] = project5
projectMap["666"] = project6

_, result, err := utils.ConvertJobsToDiggerJobs("", "github", 1, jobs, projectMap, graph, 123, "", 2, "", "", "test", "", 123, "test", 0)
_, result, err := utils.ConvertJobsToDiggerJobs("", "github", 1, jobs, projectMap, graph, 123, "", 2, "", "", "test", "", 123, "test", 0, "", false)
assert.NoError(t, err)
assert.Equal(t, 6, len(result))
parentLinks, err := models.DB.GetDiggerJobParentLinksChildId(&result["111"].DiggerJobID)
Expand Down
Loading

0 comments on commit 60dbc32

Please sign in to comment.