Skip to content

Commit 9a2b940

Browse files
motatoesmotatoes
authored
feat/blocks include patterns (#1830)
* dd missing include_patterns, exclude_patterns, workspace to blocks
1 parent 354034b commit 9a2b940

File tree

5 files changed

+98
-59
lines changed

5 files changed

+98
-59
lines changed

‎docs/ce/howto/generate-projects.mdx

+10-1
Original file line numberDiff line numberDiff line change
@@ -24,10 +24,19 @@ generate_projects:
2424
workflow: default_work_flow
2525
- include: "environments/dev/**"
2626
workflow: default_work_flow
27-
- include: "environments/qa/**"
27+
- include: "environments/dev/**"
28+
exclude: "environments/dev/local-tests/**"
29+
workspace: "qa"
2830
workflow: default_work_flow
31+
include_patterns: ["modules/**"]
32+
exclude_patterns: ["modules/dontincludeme/**"]
2933
```
3034

35+
Note that there is a distinction between `include_patterns`, `exclude_patterns` and `include`, `exclude`.
36+
`include` and `exclude` are used to decide which project get generated in the first place
37+
whereas `include_patterns` and `exclude_patterns` are used to decide which additional folders should cause the generated project to trigger
38+
39+
3140
[Demo repo](https://github.com/diggerhq/generate_projects_demo)
3241

3342
# Traversing nested directories

‎libs/digger_config/digger_config.go

+36-29
Original file line numberDiff line numberDiff line change
@@ -243,12 +243,12 @@ func HandleYamlProjectGeneration(config *DiggerConfigYaml, terraformDir string,
243243
if MatchIncludeExcludePatternsToFile(dir, includePatterns, excludePatterns) {
244244
projectName := strings.ReplaceAll(dir, "/", "_")
245245
project := ProjectYaml{
246-
Name: projectName,
247-
Dir: dir,
248-
Workflow: defaultWorkflowName,
249-
Workspace: "default",
250-
AwsRoleToAssume: config.GenerateProjectsConfig.AwsRoleToAssume,
251-
Generated: true,
246+
Name: projectName,
247+
Dir: dir,
248+
Workflow: defaultWorkflowName,
249+
Workspace: "default",
250+
AwsRoleToAssume: config.GenerateProjectsConfig.AwsRoleToAssume,
251+
Generated: true,
252252
AwsCognitoOidcConfig: config.GenerateProjectsConfig.AwsCognitoOidcConfig,
253253
}
254254
config.Projects = append(config.Projects, &project)
@@ -272,13 +272,13 @@ func HandleYamlProjectGeneration(config *DiggerConfigYaml, terraformDir string,
272272
}
273273

274274
tgParsingConfig := TerragruntParsingConfig{
275-
CreateProjectName: true,
276-
DefaultWorkflow: workflow,
277-
WorkflowFile: b.WorkflowFile,
278-
FilterPath: path.Join(terraformDir, *b.RootDir),
279-
AwsRoleToAssume: b.AwsRoleToAssume,
275+
CreateProjectName: true,
276+
DefaultWorkflow: workflow,
277+
WorkflowFile: b.WorkflowFile,
278+
FilterPath: path.Join(terraformDir, *b.RootDir),
279+
AwsRoleToAssume: b.AwsRoleToAssume,
280280
AwsCognitoOidcConfig: b.AwsCognitoOidcConfig,
281-
};
281+
}
282282

283283
err := hydrateDiggerConfigYamlWithTerragrunt(config, tgParsingConfig, terraformDir)
284284
if err != nil {
@@ -294,19 +294,26 @@ func HandleYamlProjectGeneration(config *DiggerConfigYaml, terraformDir string,
294294
workflow = b.Workflow
295295
}
296296

297+
workspace := "default"
298+
if b.Workspace != "" {
299+
workspace = b.Workspace
300+
}
301+
297302
for _, dir := range dirs {
298303
if MatchIncludeExcludePatternsToFile(dir, includePatterns, excludePatterns) {
299304
projectName := strings.ReplaceAll(dir, "/", "_")
300305
project := ProjectYaml{
301-
Name: projectName,
302-
Dir: dir,
303-
Workflow: workflow,
304-
Workspace: "default",
305-
OpenTofu: b.OpenTofu,
306-
AwsRoleToAssume: b.AwsRoleToAssume,
307-
Generated: true,
306+
Name: projectName,
307+
Dir: dir,
308+
Workflow: workflow,
309+
Workspace: workspace,
310+
OpenTofu: b.OpenTofu,
311+
AwsRoleToAssume: b.AwsRoleToAssume,
312+
Generated: true,
308313
AwsCognitoOidcConfig: b.AwsCognitoOidcConfig,
309-
WorkflowFile: b.WorkflowFile,
314+
WorkflowFile: b.WorkflowFile,
315+
IncludePatterns: b.IncludePatterns,
316+
ExcludePatterns: b.ExcludePatterns,
310317
}
311318
config.Projects = append(config.Projects, &project)
312319
}
@@ -538,15 +545,15 @@ func hydrateDiggerConfigYamlWithTerragrunt(configYaml *DiggerConfigYaml, parsing
538545
}
539546

540547
configYaml.Projects = append(configYaml.Projects, &ProjectYaml{
541-
Name: atlantisProject.Name,
542-
Dir: projectDir,
543-
Workspace: atlantisProject.Workspace,
544-
Terragrunt: true,
545-
Workflow: atlantisProject.Workflow,
546-
WorkflowFile: workflowFile,
547-
IncludePatterns: atlantisProject.Autoplan.WhenModified,
548-
Generated: true,
549-
AwsRoleToAssume: parsingConfig.AwsRoleToAssume,
548+
Name: atlantisProject.Name,
549+
Dir: projectDir,
550+
Workspace: atlantisProject.Workspace,
551+
Terragrunt: true,
552+
Workflow: atlantisProject.Workflow,
553+
WorkflowFile: workflowFile,
554+
IncludePatterns: atlantisProject.Autoplan.WhenModified,
555+
Generated: true,
556+
AwsRoleToAssume: parsingConfig.AwsRoleToAssume,
550557
AwsCognitoOidcConfig: parsingConfig.AwsCognitoOidcConfig,
551558
})
552559
}

‎libs/digger_config/digger_config_test.go

+14-1
Original file line numberDiff line numberDiff line change
@@ -843,7 +843,10 @@ generate_projects:
843843
blocks:
844844
- include: dev/*
845845
exclude: dev/project
846+
include_patterns: ["modules/**"]
847+
exclude_patterns: ["modules/dontincludeme/**"]
846848
workflow: dev_workflow
849+
workspace: devdev
847850
- include: prod/*
848851
exclude: prod/project
849852
workflow: prod_workflow
@@ -878,9 +881,19 @@ workflows:
878881
assert.Equal(t, "dev_workflow", dg.Projects[0].Workflow)
879882
assert.Equal(t, "dev_workflow", dg.Projects[1].Workflow)
880883
assert.Equal(t, "prod_workflow", dg.Projects[2].Workflow)
884+
assert.Equal(t, "devdev", dg.Projects[0].Workspace)
885+
assert.Equal(t, "devdev", dg.Projects[1].Workspace)
886+
assert.Equal(t, "default", dg.Projects[2].Workspace)
881887
assert.Equal(t, "dev/test1", dg.Projects[0].Dir)
882888
assert.Equal(t, "dev/test2", dg.Projects[1].Dir)
883889
assert.Equal(t, "prod/one", dg.Projects[2].Dir)
890+
assert.Equal(t, []string{"modules/**"}, dg.Projects[0].IncludePatterns)
891+
assert.Equal(t, []string{"modules/**"}, dg.Projects[1].IncludePatterns)
892+
assert.Nil(t, dg.Projects[2].IncludePatterns)
893+
assert.Equal(t, []string{"modules/dontincludeme/**"}, dg.Projects[0].ExcludePatterns)
894+
assert.Equal(t, []string{"modules/dontincludeme/**"}, dg.Projects[1].ExcludePatterns)
895+
assert.Nil(t, dg.Projects[2].ExcludePatterns)
896+
884897
assert.Equal(t, 3, len(dg.Projects))
885898
}
886899

@@ -1221,4 +1234,4 @@ projects:
12211234

12221235
assert.Equal(t, dg.Projects[0].AwsCognitoOidcConfig.CognitoPoolId, "us-east-1:00000000-0000-0000-0000-000000000000")
12231236
assert.Equal(t, dg.Projects[0].AwsCognitoOidcConfig.AwsAccountId, "000000000000")
1224-
}
1237+
}

‎libs/digger_config/utils_test.go

+7
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,13 @@ func TestMatchIncludeExcludePatternsToFile(t *testing.T) {
1919

2020
result = MatchIncludeExcludePatternsToFile("projects/dev/project", includePatterns, excludePatterns)
2121
assert.Equal(t, false, result)
22+
23+
// also checking for uninitialized case which is going to be the scenario when not specified in yaml file
24+
var ip []string
25+
var ep []string
26+
result = MatchIncludeExcludePatternsToFile("/projects/dev/test1", ip, ep)
27+
assert.Equal(t, false, result)
28+
2229
}
2330

2431
func TestGetPatternsRelativeToRepo(t *testing.T) {

‎libs/digger_config/yaml.go

+31-28
Original file line numberDiff line numberDiff line change
@@ -26,22 +26,22 @@ type DependencyConfigurationYaml struct {
2626
}
2727

2828
type ProjectYaml struct {
29-
Name string `yaml:"name"`
30-
Dir string `yaml:"dir"`
31-
Workspace string `yaml:"workspace"`
32-
Terragrunt bool `yaml:"terragrunt"`
33-
OpenTofu bool `yaml:"opentofu"`
34-
Pulumi bool `yaml:"pulumi"`
35-
Workflow string `yaml:"workflow"`
36-
WorkflowFile string `yaml:"workflow_file"`
37-
IncludePatterns []string `yaml:"include_patterns,omitempty"`
38-
ExcludePatterns []string `yaml:"exclude_patterns,omitempty"`
39-
DependencyProjects []string `yaml:"depends_on,omitempty"`
40-
DriftDetection *bool `yaml:"drift_detection,omitempty"`
41-
AwsRoleToAssume *AssumeRoleForProjectConfig `yaml:"aws_role_to_assume,omitempty"`
42-
Generated bool `yaml:"generated"`
43-
AwsCognitoOidcConfig *AwsCognitoOidcConfig `yaml:"aws_cognito_oidc,omitempty"`
44-
PulumiStack string `yaml:"pulumi_stack"`
29+
Name string `yaml:"name"`
30+
Dir string `yaml:"dir"`
31+
Workspace string `yaml:"workspace"`
32+
Terragrunt bool `yaml:"terragrunt"`
33+
OpenTofu bool `yaml:"opentofu"`
34+
Pulumi bool `yaml:"pulumi"`
35+
Workflow string `yaml:"workflow"`
36+
WorkflowFile string `yaml:"workflow_file"`
37+
IncludePatterns []string `yaml:"include_patterns,omitempty"`
38+
ExcludePatterns []string `yaml:"exclude_patterns,omitempty"`
39+
DependencyProjects []string `yaml:"depends_on,omitempty"`
40+
DriftDetection *bool `yaml:"drift_detection,omitempty"`
41+
AwsRoleToAssume *AssumeRoleForProjectConfig `yaml:"aws_role_to_assume,omitempty"`
42+
Generated bool `yaml:"generated"`
43+
AwsCognitoOidcConfig *AwsCognitoOidcConfig `yaml:"aws_cognito_oidc,omitempty"`
44+
PulumiStack string `yaml:"pulumi_stack"`
4545
}
4646

4747
type WorkflowYaml struct {
@@ -92,8 +92,10 @@ type EnvVarYaml struct {
9292

9393
type BlockYaml struct {
9494
// these flags are only for terraform and opentofu
95-
Include string `yaml:"include"`
96-
Exclude string `yaml:"exclude"`
95+
Include string `yaml:"include"`
96+
Exclude string `yaml:"exclude"`
97+
IncludePatterns []string `yaml:"include_patterns,omitempty"`
98+
ExcludePatterns []string `yaml:"exclude_patterns,omitempty"`
9799

98100
// these flags are only for terragrunt
99101
Terragrunt bool `yaml:"terragrunt"`
@@ -103,11 +105,12 @@ type BlockYaml struct {
103105
OpenTofu bool `yaml:"opentofu"`
104106

105107
// common flags
106-
BlockName string `yaml:"block_name"`
107-
Workflow string `yaml:"workflow"`
108-
WorkflowFile string `yaml:"workflow_file"`
109-
AwsRoleToAssume *AssumeRoleForProjectConfig `yaml:"aws_role_to_assume,omitempty"`
110-
AwsCognitoOidcConfig *AwsCognitoOidcConfig `yaml:"aws_cognito_oidc,omitempty"`
108+
Workspace string `yaml:"workspace"`
109+
BlockName string `yaml:"block_name"`
110+
Workflow string `yaml:"workflow"`
111+
WorkflowFile string `yaml:"workflow_file"`
112+
AwsRoleToAssume *AssumeRoleForProjectConfig `yaml:"aws_role_to_assume,omitempty"`
113+
AwsCognitoOidcConfig *AwsCognitoOidcConfig `yaml:"aws_cognito_oidc,omitempty"`
111114
}
112115

113116
type AssumeRoleForProjectConfig struct {
@@ -117,9 +120,9 @@ type AssumeRoleForProjectConfig struct {
117120
}
118121

119122
type AwsCognitoOidcConfig struct {
120-
AwsAccountId string `yaml:"aws_account_id"`
121-
AwsRegion string `yaml:"aws_region,omitempty"`
122-
CognitoPoolId string `yaml:"cognito_identity_pool_id"`
123+
AwsAccountId string `yaml:"aws_account_id"`
124+
AwsRegion string `yaml:"aws_region,omitempty"`
125+
CognitoPoolId string `yaml:"cognito_identity_pool_id"`
123126
SessionDuration int `yaml:"session_duration"`
124127
}
125128

@@ -130,7 +133,7 @@ type GenerateProjectsConfigYaml struct {
130133
Blocks []BlockYaml `yaml:"blocks"`
131134
TerragruntParsingConfig *TerragruntParsingConfig `yaml:"terragrunt_parsing,omitempty"`
132135
AwsRoleToAssume *AssumeRoleForProjectConfig `yaml:"aws_role_to_assume,omitempty"`
133-
AwsCognitoOidcConfig *AwsCognitoOidcConfig `yaml:"aws_cognito_oidc,omitempty"`
136+
AwsCognitoOidcConfig *AwsCognitoOidcConfig `yaml:"aws_cognito_oidc,omitempty"`
134137
}
135138

136139
type TerragruntParsingConfig struct {
@@ -159,7 +162,7 @@ type TerragruntParsingConfig struct {
159162
ExecutionOrderGroups *bool `yaml:"executionOrderGroups"`
160163
WorkflowFile string `yaml:"workflow_file"`
161164
AwsRoleToAssume *AssumeRoleForProjectConfig `yaml:"aws_role_to_assume,omitempty"`
162-
AwsCognitoOidcConfig *AwsCognitoOidcConfig `yaml:"aws_cognito_oidc,omitempty"`
165+
AwsCognitoOidcConfig *AwsCognitoOidcConfig `yaml:"aws_cognito_oidc,omitempty"`
163166
}
164167

165168
func (p *ProjectYaml) UnmarshalYAML(unmarshal func(interface{}) error) error {

0 commit comments

Comments
 (0)