Skip to content

Commit

Permalink
feat/blocks include patterns (#1830)
Browse files Browse the repository at this point in the history 
* dd missing include_patterns, exclude_patterns, workspace to blocks
  • Loading branch information
@motatoes
motatoes authored Nov 18, 2024
1 parent 354034b commit 9a2b940
Show file tree
Hide file tree
Showing 5 changed files with 98 additions and 59 deletions.
11 changes: 10 additions & 1 deletion docs/ce/howto/generate-projects.mdx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,19 @@ generate_projects:
workflow: default_work_flow
- include: "environments/dev/**"
workflow: default_work_flow
- include: "environments/qa/**"
- include: "environments/dev/**"
exclude: "environments/dev/local-tests/**"
workspace: "qa"
workflow: default_work_flow
include_patterns: ["modules/**"]
exclude_patterns: ["modules/dontincludeme/**"]
```

Note that there is a distinction between `include_patterns`, `exclude_patterns` and `include`, `exclude`.
`include` and `exclude` are used to decide which project get generated in the first place
whereas `include_patterns` and `exclude_patterns` are used to decide which additional folders should cause the generated project to trigger


[Demo repo](https://github.com/diggerhq/generate_projects_demo)

# Traversing nested directories
Expand Down
65 changes: 36 additions & 29 deletions libs/digger_config/digger_config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -243,12 +243,12 @@ func HandleYamlProjectGeneration(config *DiggerConfigYaml, terraformDir string,
if MatchIncludeExcludePatternsToFile(dir, includePatterns, excludePatterns) {
projectName := strings.ReplaceAll(dir, "/", "_")
project := ProjectYaml{
Name: projectName,
Dir: dir,
Workflow: defaultWorkflowName,
Workspace: "default",
AwsRoleToAssume: config.GenerateProjectsConfig.AwsRoleToAssume,
Generated: true,
Name: projectName,
Dir: dir,
Workflow: defaultWorkflowName,
Workspace: "default",
AwsRoleToAssume: config.GenerateProjectsConfig.AwsRoleToAssume,
Generated: true,
AwsCognitoOidcConfig: config.GenerateProjectsConfig.AwsCognitoOidcConfig,
}
config.Projects = append(config.Projects, &project)
Expand All @@ -272,13 +272,13 @@ func HandleYamlProjectGeneration(config *DiggerConfigYaml, terraformDir string,
}

tgParsingConfig := TerragruntParsingConfig{
CreateProjectName: true,
DefaultWorkflow: workflow,
WorkflowFile: b.WorkflowFile,
FilterPath: path.Join(terraformDir, *b.RootDir),
AwsRoleToAssume: b.AwsRoleToAssume,
CreateProjectName: true,
DefaultWorkflow: workflow,
WorkflowFile: b.WorkflowFile,
FilterPath: path.Join(terraformDir, *b.RootDir),
AwsRoleToAssume: b.AwsRoleToAssume,
AwsCognitoOidcConfig: b.AwsCognitoOidcConfig,
};
}

err := hydrateDiggerConfigYamlWithTerragrunt(config, tgParsingConfig, terraformDir)
if err != nil {
Expand All @@ -294,19 +294,26 @@ func HandleYamlProjectGeneration(config *DiggerConfigYaml, terraformDir string,
workflow = b.Workflow
}

workspace := "default"
if b.Workspace != "" {
workspace = b.Workspace
}

for _, dir := range dirs {
if MatchIncludeExcludePatternsToFile(dir, includePatterns, excludePatterns) {
projectName := strings.ReplaceAll(dir, "/", "_")
project := ProjectYaml{
Name: projectName,
Dir: dir,
Workflow: workflow,
Workspace: "default",
OpenTofu: b.OpenTofu,
AwsRoleToAssume: b.AwsRoleToAssume,
Generated: true,
Name: projectName,
Dir: dir,
Workflow: workflow,
Workspace: workspace,
OpenTofu: b.OpenTofu,
AwsRoleToAssume: b.AwsRoleToAssume,
Generated: true,
AwsCognitoOidcConfig: b.AwsCognitoOidcConfig,
WorkflowFile: b.WorkflowFile,
WorkflowFile: b.WorkflowFile,
IncludePatterns: b.IncludePatterns,
ExcludePatterns: b.ExcludePatterns,
}
config.Projects = append(config.Projects, &project)
}
Expand Down Expand Up @@ -538,15 +545,15 @@ func hydrateDiggerConfigYamlWithTerragrunt(configYaml *DiggerConfigYaml, parsing
}

configYaml.Projects = append(configYaml.Projects, &ProjectYaml{
Name: atlantisProject.Name,
Dir: projectDir,
Workspace: atlantisProject.Workspace,
Terragrunt: true,
Workflow: atlantisProject.Workflow,
WorkflowFile: workflowFile,
IncludePatterns: atlantisProject.Autoplan.WhenModified,
Generated: true,
AwsRoleToAssume: parsingConfig.AwsRoleToAssume,
Name: atlantisProject.Name,
Dir: projectDir,
Workspace: atlantisProject.Workspace,
Terragrunt: true,
Workflow: atlantisProject.Workflow,
WorkflowFile: workflowFile,
IncludePatterns: atlantisProject.Autoplan.WhenModified,
Generated: true,
AwsRoleToAssume: parsingConfig.AwsRoleToAssume,
AwsCognitoOidcConfig: parsingConfig.AwsCognitoOidcConfig,
})
}
Expand Down
15 changes: 14 additions & 1 deletion libs/digger_config/digger_config_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -843,7 +843,10 @@ generate_projects:
blocks:
- include: dev/*
exclude: dev/project
include_patterns: ["modules/**"]
exclude_patterns: ["modules/dontincludeme/**"]
workflow: dev_workflow
workspace: devdev
- include: prod/*
exclude: prod/project
workflow: prod_workflow
Expand Down Expand Up @@ -878,9 +881,19 @@ workflows:
assert.Equal(t, "dev_workflow", dg.Projects[0].Workflow)
assert.Equal(t, "dev_workflow", dg.Projects[1].Workflow)
assert.Equal(t, "prod_workflow", dg.Projects[2].Workflow)
assert.Equal(t, "devdev", dg.Projects[0].Workspace)
assert.Equal(t, "devdev", dg.Projects[1].Workspace)
assert.Equal(t, "default", dg.Projects[2].Workspace)
assert.Equal(t, "dev/test1", dg.Projects[0].Dir)
assert.Equal(t, "dev/test2", dg.Projects[1].Dir)
assert.Equal(t, "prod/one", dg.Projects[2].Dir)
assert.Equal(t, []string{"modules/**"}, dg.Projects[0].IncludePatterns)
assert.Equal(t, []string{"modules/**"}, dg.Projects[1].IncludePatterns)
assert.Nil(t, dg.Projects[2].IncludePatterns)
assert.Equal(t, []string{"modules/dontincludeme/**"}, dg.Projects[0].ExcludePatterns)
assert.Equal(t, []string{"modules/dontincludeme/**"}, dg.Projects[1].ExcludePatterns)
assert.Nil(t, dg.Projects[2].ExcludePatterns)

assert.Equal(t, 3, len(dg.Projects))
}

Expand Down Expand Up @@ -1221,4 +1234,4 @@ projects:

assert.Equal(t, dg.Projects[0].AwsCognitoOidcConfig.CognitoPoolId, "us-east-1:00000000-0000-0000-0000-000000000000")
assert.Equal(t, dg.Projects[0].AwsCognitoOidcConfig.AwsAccountId, "000000000000")
}
}
7 changes: 7 additions & 0 deletions libs/digger_config/utils_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,13 @@ func TestMatchIncludeExcludePatternsToFile(t *testing.T) {

result = MatchIncludeExcludePatternsToFile("projects/dev/project", includePatterns, excludePatterns)
assert.Equal(t, false, result)

// also checking for uninitialized case which is going to be the scenario when not specified in yaml file
var ip []string
var ep []string
result = MatchIncludeExcludePatternsToFile("/projects/dev/test1", ip, ep)
assert.Equal(t, false, result)

}

func TestGetPatternsRelativeToRepo(t *testing.T) {
Expand Down
59 changes: 31 additions & 28 deletions libs/digger_config/yaml.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,22 +26,22 @@ type DependencyConfigurationYaml struct {
}

type ProjectYaml struct {
Name string `yaml:"name"`
Dir string `yaml:"dir"`
Workspace string `yaml:"workspace"`
Terragrunt bool `yaml:"terragrunt"`
OpenTofu bool `yaml:"opentofu"`
Pulumi bool `yaml:"pulumi"`
Workflow string `yaml:"workflow"`
WorkflowFile string `yaml:"workflow_file"`
IncludePatterns []string `yaml:"include_patterns,omitempty"`
ExcludePatterns []string `yaml:"exclude_patterns,omitempty"`
DependencyProjects []string `yaml:"depends_on,omitempty"`
DriftDetection *bool `yaml:"drift_detection,omitempty"`
AwsRoleToAssume *AssumeRoleForProjectConfig `yaml:"aws_role_to_assume,omitempty"`
Generated bool `yaml:"generated"`
AwsCognitoOidcConfig *AwsCognitoOidcConfig `yaml:"aws_cognito_oidc,omitempty"`
PulumiStack string `yaml:"pulumi_stack"`
Name string `yaml:"name"`
Dir string `yaml:"dir"`
Workspace string `yaml:"workspace"`
Terragrunt bool `yaml:"terragrunt"`
OpenTofu bool `yaml:"opentofu"`
Pulumi bool `yaml:"pulumi"`
Workflow string `yaml:"workflow"`
WorkflowFile string `yaml:"workflow_file"`
IncludePatterns []string `yaml:"include_patterns,omitempty"`
ExcludePatterns []string `yaml:"exclude_patterns,omitempty"`
DependencyProjects []string `yaml:"depends_on,omitempty"`
DriftDetection *bool `yaml:"drift_detection,omitempty"`
AwsRoleToAssume *AssumeRoleForProjectConfig `yaml:"aws_role_to_assume,omitempty"`
Generated bool `yaml:"generated"`
AwsCognitoOidcConfig *AwsCognitoOidcConfig `yaml:"aws_cognito_oidc,omitempty"`
PulumiStack string `yaml:"pulumi_stack"`
}

type WorkflowYaml struct {
Expand Down Expand Up @@ -92,8 +92,10 @@ type EnvVarYaml struct {

type BlockYaml struct {
// these flags are only for terraform and opentofu
Include string `yaml:"include"`
Exclude string `yaml:"exclude"`
Include string `yaml:"include"`
Exclude string `yaml:"exclude"`
IncludePatterns []string `yaml:"include_patterns,omitempty"`
ExcludePatterns []string `yaml:"exclude_patterns,omitempty"`

// these flags are only for terragrunt
Terragrunt bool `yaml:"terragrunt"`
Expand All @@ -103,11 +105,12 @@ type BlockYaml struct {
OpenTofu bool `yaml:"opentofu"`

// common flags
BlockName string `yaml:"block_name"`
Workflow string `yaml:"workflow"`
WorkflowFile string `yaml:"workflow_file"`
AwsRoleToAssume *AssumeRoleForProjectConfig `yaml:"aws_role_to_assume,omitempty"`
AwsCognitoOidcConfig *AwsCognitoOidcConfig `yaml:"aws_cognito_oidc,omitempty"`
Workspace string `yaml:"workspace"`
BlockName string `yaml:"block_name"`
Workflow string `yaml:"workflow"`
WorkflowFile string `yaml:"workflow_file"`
AwsRoleToAssume *AssumeRoleForProjectConfig `yaml:"aws_role_to_assume,omitempty"`
AwsCognitoOidcConfig *AwsCognitoOidcConfig `yaml:"aws_cognito_oidc,omitempty"`
}

type AssumeRoleForProjectConfig struct {
Expand All @@ -117,9 +120,9 @@ type AssumeRoleForProjectConfig struct {
}

type AwsCognitoOidcConfig struct {
AwsAccountId string `yaml:"aws_account_id"`
AwsRegion string `yaml:"aws_region,omitempty"`
CognitoPoolId string `yaml:"cognito_identity_pool_id"`
AwsAccountId string `yaml:"aws_account_id"`
AwsRegion string `yaml:"aws_region,omitempty"`
CognitoPoolId string `yaml:"cognito_identity_pool_id"`
SessionDuration int `yaml:"session_duration"`
}

Expand All @@ -130,7 +133,7 @@ type GenerateProjectsConfigYaml struct {
Blocks []BlockYaml `yaml:"blocks"`
TerragruntParsingConfig *TerragruntParsingConfig `yaml:"terragrunt_parsing,omitempty"`
AwsRoleToAssume *AssumeRoleForProjectConfig `yaml:"aws_role_to_assume,omitempty"`
AwsCognitoOidcConfig *AwsCognitoOidcConfig `yaml:"aws_cognito_oidc,omitempty"`
AwsCognitoOidcConfig *AwsCognitoOidcConfig `yaml:"aws_cognito_oidc,omitempty"`
}

type TerragruntParsingConfig struct {
Expand Down Expand Up @@ -159,7 +162,7 @@ type TerragruntParsingConfig struct {
ExecutionOrderGroups *bool `yaml:"executionOrderGroups"`
WorkflowFile string `yaml:"workflow_file"`
AwsRoleToAssume *AssumeRoleForProjectConfig `yaml:"aws_role_to_assume,omitempty"`
AwsCognitoOidcConfig *AwsCognitoOidcConfig `yaml:"aws_cognito_oidc,omitempty"`
AwsCognitoOidcConfig *AwsCognitoOidcConfig `yaml:"aws_cognito_oidc,omitempty"`
}

func (p *ProjectYaml) UnmarshalYAML(unmarshal func(interface{}) error) error {
Expand Down

0 comments on commit 9a2b940

Please sign in to comment.