Skip to content

Commit c103ae1

Browse files
b-perb-per
authored
Merge pull request #200 from dbt-labs/release-0.2.10
Release 0.2.10
2 parents 31a4bfa + cf777ab commit c103ae1

File tree

11 files changed

+198
-60
lines changed

11 files changed

+198
-60
lines changed

‎CHANGELOG.md

+14-1
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,20 @@
22

33
All notable changes to this project will be documented in this file.
44

5-
## [Unreleased](https://github.com/dbt-labs/terraform-provider-dbtcloud/compare/v0.2.9...HEAD)
5+
## [Unreleased](https://github.com/dbt-labs/terraform-provider-dbtcloud/compare/v0.2.10...HEAD)
6+
7+
8+
## [0.2.10](https://github.com/dbt-labs/terraform-provider-dbtcloud/compare/v0.2.9...v0.2.10)
9+
10+
## Fix
11+
12+
- [#197](https://github.com/dbt-labs/terraform-provider-dbtcloud/issues/197) - Community contribution to handle cases where more than 100 groups are created in dbt Cloud
13+
- [#199](https://github.com/dbt-labs/terraform-provider-dbtcloud/issues/199) - Update logic to allow finding users by their email addresses in a cases insensitive way
14+
- [#198](https://github.com/dbt-labs/terraform-provider-dbtcloud/issues/198) - Update some internal logic to call endpoints by their unique IDs instead of looping through answers to avoid issues like #199 and paginate through results for endpoints where we can't query the ID directly
15+
16+
## Changes
17+
18+
- [#189](https://github.com/dbt-labs/terraform-provider-dbtcloud/issues/189) - Allow users to retrieve project data sources by providing project names instead of project IDs. This will return an error if more than 1 project has the given name and takes care of the pagination required for handling more than 100 projects
619

720
## [0.2.9](https://github.com/dbt-labs/terraform-provider-dbtcloud/compare/v0.2.8...v0.2.9)
821

‎examples/data-sources/dbtcloud_project/data-source.tf

+7
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,13 @@
11
// use dbt_cloud_project instead of dbtcloud_project for the legacy resource names
22
// legacy names will be removed from 0.3 onwards
33

4+
// projects data sources can use the project_id parameter (preferred uniqueness is ensured)
45
data "dbtcloud_project" "test_project" {
56
project_id = var.dbt_cloud_project_id
67
}
8+
9+
// or they can use project names
10+
// the provider will raise an error if more than one project is found with the same name
11+
data "dbtcloud_project" "test_project" {
12+
name = "My project name"
13+
}

‎pkg/data_sources/project.go

+28-6
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package data_sources
22

33
import (
44
"context"
5+
"fmt"
56
"strconv"
67

78
"github.com/dbt-labs/terraform-provider-dbtcloud/pkg/dbt_cloud"
@@ -12,12 +13,13 @@ import (
1213
var projectSchema = map[string]*schema.Schema{
1314
"project_id": &schema.Schema{
1415
Type: schema.TypeInt,
15-
Required: true,
16+
Optional: true,
1617
Description: "ID of the project to represent",
1718
},
1819
"name": &schema.Schema{
1920
Type: schema.TypeString,
2021
Computed: true,
22+
Optional: true,
2123
Description: "Given name for project",
2224
},
2325
"connection_id": &schema.Schema{
@@ -58,12 +60,32 @@ func datasourceProjectRead(ctx context.Context, d *schema.ResourceData, m interf
5860
c := m.(*dbt_cloud.Client)
5961

6062
var diags diag.Diagnostics
63+
var project *dbt_cloud.Project
6164

62-
projectId := strconv.Itoa(d.Get("project_id").(int))
65+
if _, ok := d.GetOk("project_id"); ok {
66+
projectId := strconv.Itoa(d.Get("project_id").(int))
6367

64-
project, err := c.GetProject(projectId)
65-
if err != nil {
66-
return diag.FromErr(err)
68+
if _, ok := d.GetOk("name"); ok {
69+
return diag.FromErr(fmt.Errorf("Both project_id and name were provided, only one is allowed"))
70+
}
71+
72+
var err error
73+
project, err = c.GetProject(projectId)
74+
if err != nil {
75+
return diag.FromErr(err)
76+
}
77+
78+
} else if _, ok := d.GetOk("name"); ok {
79+
projectName := d.Get("name").(string)
80+
81+
var err error
82+
project, err = c.GetProjectByName(projectName)
83+
if err != nil {
84+
return diag.FromErr(err)
85+
}
86+
87+
} else {
88+
return diag.FromErr(fmt.Errorf("Either project_id or name must be provided"))
6789
}
6890

6991
if err := d.Set("project_id", project.ID); err != nil {
@@ -88,7 +110,7 @@ func datasourceProjectRead(ctx context.Context, d *schema.ResourceData, m interf
88110
return diag.FromErr(err)
89111
}
90112

91-
d.SetId(projectId)
113+
d.SetId(strconv.Itoa(*project.ID))
92114

93115
return diags
94116
}

‎pkg/data_sources/project_acceptance_test.go

+10
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,12 @@ func TestAccDbtCloudProjectDataSource(t *testing.T) {
2020
resource.TestCheckResourceAttrSet("data.dbtcloud_project.test", "connection_id"),
2121
resource.TestCheckResourceAttrSet("data.dbtcloud_project.test", "repository_id"),
2222
resource.TestCheckResourceAttrSet("data.dbtcloud_project.test", "state"),
23+
24+
resource.TestCheckResourceAttrSet("data.dbtcloud_project.test_with_name", "project_id"),
25+
resource.TestCheckResourceAttr("data.dbtcloud_project.test_with_name", "name", randomProjectName),
26+
resource.TestCheckResourceAttrSet("data.dbtcloud_project.test_with_name", "connection_id"),
27+
resource.TestCheckResourceAttrSet("data.dbtcloud_project.test_with_name", "repository_id"),
28+
resource.TestCheckResourceAttrSet("data.dbtcloud_project.test_with_name", "state"),
2329
)
2430

2531
resource.ParallelTest(t, resource.TestCase{
@@ -43,5 +49,9 @@ func project(projectName string) string {
4349
data "dbtcloud_project" "test" {
4450
project_id = dbtcloud_project.test.id
4551
}
52+
53+
data "dbtcloud_project" "test_with_name" {
54+
name = dbtcloud_project.test.name
55+
}
4656
`, projectName)
4757
}

‎pkg/dbt_cloud/bigquery_credential.go

+4-15
Original file line numberDiff line numberDiff line change
@@ -7,11 +7,6 @@ import (
77
"strings"
88
)
99

10-
type BigQueryCredentialListResponse struct {
11-
Data []BigQueryCredential `json:"data"`
12-
Status ResponseStatus `json:"status"`
13-
}
14-
1510
type BigQueryCredentialResponse struct {
1611
Data BigQueryCredential `json:"data"`
1712
Status ResponseStatus `json:"status"`
@@ -28,7 +23,7 @@ type BigQueryCredential struct {
2823
}
2924

3025
func (c *Client) GetBigQueryCredential(projectId int, credentialId int) (*BigQueryCredential, error) {
31-
req, err := http.NewRequest("GET", fmt.Sprintf("%s/v3/accounts/%d/projects/%d/credentials/", c.HostURL, c.AccountID, projectId), nil)
26+
req, err := http.NewRequest("GET", fmt.Sprintf("%s/v3/accounts/%d/projects/%d/credentials/%d/", c.HostURL, c.AccountID, projectId, credentialId), nil)
3227
if err != nil {
3328
return nil, err
3429
}
@@ -38,19 +33,13 @@ func (c *Client) GetBigQueryCredential(projectId int, credentialId int) (*BigQue
3833
return nil, err
3934
}
4035

41-
BigQueryCredentialListResponse := BigQueryCredentialListResponse{}
42-
err = json.Unmarshal(body, &BigQueryCredentialListResponse)
36+
BigQueryCredentialResponse := BigQueryCredentialResponse{}
37+
err = json.Unmarshal(body, &BigQueryCredentialResponse)
4338
if err != nil {
4439
return nil, err
4540
}
4641

47-
for i, credential := range BigQueryCredentialListResponse.Data {
48-
if *credential.ID == credentialId {
49-
return &BigQueryCredentialListResponse.Data[i], nil
50-
}
51-
}
52-
53-
return nil, fmt.Errorf("resource-not-found: did not find credential ID %d in project ID %d", credentialId, projectId)
42+
return &BigQueryCredentialResponse.Data, nil
5443
}
5544

5645
func (c *Client) CreateBigQueryCredential(projectId int, type_ string, isActive bool, dataset string, numThreads int) (*BigQueryCredential, error) {

‎pkg/dbt_cloud/client.go

+15
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,21 @@ type ResponseStatus struct {
2424
Developer_Message string `json:"developer_message"`
2525
}
2626

27+
type ResponseExtraFilters struct {
28+
Limit int `json:"limit"`
29+
Offset int `json:"offset"`
30+
}
31+
32+
type ResponseExtraPagination struct {
33+
Count int `json:"count"`
34+
TotalCount int `json:"total_count"`
35+
}
36+
37+
type ResponseExtra struct {
38+
Pagination ResponseExtraPagination `json:"pagination"`
39+
Filters ResponseExtraFilters `json:"filters"`
40+
}
41+
2742
type AuthResponseData struct {
2843
DocsJobId int `json:"docs_job_id"`
2944
FreshnessJobId int `json:"freshness_job_id"`

‎pkg/dbt_cloud/group.go

-5
Original file line numberDiff line numberDiff line change
@@ -34,11 +34,6 @@ type GroupResponse struct {
3434
Status ResponseStatus `json:"status"`
3535
}
3636

37-
type GroupListResponse struct {
38-
Data []Group `json:"data"`
39-
Status ResponseStatus `json:"status"`
40-
}
41-
4237
type GroupPermissionListResponse struct {
4338
Data []GroupPermission `json:"data"`
4439
Status ResponseStatus `json:"status"`

‎pkg/dbt_cloud/postgres_credential.go

+4-15
Original file line numberDiff line numberDiff line change
@@ -20,19 +20,14 @@ type PostgresCredential struct {
2020
Password string `json:"password,omitempty"`
2121
}
2222

23-
type PostgresCredentialListResponse struct {
24-
Data []PostgresCredential `json:"data"`
25-
Status ResponseStatus `json:"status"`
26-
}
27-
2823
type PostgresCredentialResponse struct {
2924
Data PostgresCredential `json:"data"`
3025
Status ResponseStatus `json:"status"`
3126
}
3227

3328
// GetPostgresCredential retrieves a specific Postgres credential by its ID
3429
func (c *Client) GetPostgresCredential(projectId int, credentialId int) (*PostgresCredential, error) {
35-
req, err := http.NewRequest("GET", fmt.Sprintf("%s/v3/accounts/%d/projects/%d/credentials/", c.HostURL, c.AccountID, projectId), nil)
30+
req, err := http.NewRequest("GET", fmt.Sprintf("%s/v3/accounts/%d/projects/%d/credentials/%d/", c.HostURL, c.AccountID, projectId, credentialId), nil)
3631
if err != nil {
3732
return nil, err
3833
}
@@ -42,19 +37,13 @@ func (c *Client) GetPostgresCredential(projectId int, credentialId int) (*Postgr
4237
return nil, err
4338
}
4439

45-
PostgresCredentialListResponse := PostgresCredentialListResponse{}
46-
err = json.Unmarshal(body, &PostgresCredentialListResponse)
40+
PostgresCredentialResponse := PostgresCredentialResponse{}
41+
err = json.Unmarshal(body, &PostgresCredentialResponse)
4742
if err != nil {
4843
return nil, err
4944
}
5045

51-
for i, credential := range PostgresCredentialListResponse.Data {
52-
if *credential.ID == credentialId {
53-
return &PostgresCredentialListResponse.Data[i], nil
54-
}
55-
}
56-
57-
return nil, fmt.Errorf("resource-not-found: did not find credential ID %d in project ID %d", credentialId, projectId)
46+
return &PostgresCredentialResponse.Data, nil
5847
}
5948

6049
// CreatePostgresCredential creates a new Postgres credential

‎pkg/dbt_cloud/project.go

+71
Original file line numberDiff line numberDiff line change
@@ -23,13 +23,84 @@ type Project struct {
2323
type ProjectListResponse struct {
2424
Data []Project `json:"data"`
2525
Status ResponseStatus `json:"status"`
26+
Extra ResponseExtra `json:"extra"`
2627
}
2728

2829
type ProjectResponse struct {
2930
Data Project `json:"data"`
3031
Status ResponseStatus `json:"status"`
3132
}
3233

34+
func (c *Client) GetProjectByName(projectName string) (*Project, error) {
35+
req, err := http.NewRequest("GET", fmt.Sprintf("%s/v3/accounts/%s/projects/?include_related=[freshness_job_id,docs_job_id]", c.HostURL, strconv.Itoa(c.AccountID)), nil)
36+
if err != nil {
37+
return nil, err
38+
}
39+
40+
body, err := c.doRequest(req)
41+
if err != nil {
42+
return nil, err
43+
}
44+
45+
projectListResponse := ProjectListResponse{}
46+
err = json.Unmarshal(body, &projectListResponse)
47+
if err != nil {
48+
return nil, err
49+
}
50+
51+
listAllProjects := projectListResponse.Data
52+
53+
// if there are more than the limit, we need to paginate
54+
if projectListResponse.Extra.Pagination.TotalCount > projectListResponse.Extra.Filters.Limit {
55+
numProjects := projectListResponse.Extra.Pagination.Count
56+
for numProjects < projectListResponse.Extra.Pagination.TotalCount {
57+
58+
req, err := http.NewRequest("GET", fmt.Sprintf("%s/v3/accounts/%s/projects/?include_related=[freshness_job_id,docs_job_id]&offset=%d", c.HostURL, strconv.Itoa(c.AccountID), numProjects), nil)
59+
if err != nil {
60+
return nil, err
61+
}
62+
63+
body, err := c.doRequest(req)
64+
if err != nil {
65+
return nil, err
66+
}
67+
68+
projectListResponse := ProjectListResponse{}
69+
err = json.Unmarshal(body, &projectListResponse)
70+
if err != nil {
71+
return nil, err
72+
}
73+
74+
numProjectsLastCall := projectListResponse.Extra.Pagination.Count
75+
if numProjectsLastCall > 0 {
76+
listAllProjects = append(listAllProjects, projectListResponse.Data...)
77+
numProjects += projectListResponse.Extra.Pagination.Count
78+
} else {
79+
// this means that most likely one item was deleted since the first call
80+
// so the number of items is less than the initial total, we can break the loop
81+
break
82+
}
83+
84+
}
85+
}
86+
87+
// we now loop though the projects to find the ones with the name we are looking for
88+
matchingProjects := []Project{}
89+
for _, project := range listAllProjects {
90+
if strings.EqualFold(project.Name, projectName) {
91+
matchingProjects = append(matchingProjects, project)
92+
}
93+
}
94+
95+
if len(matchingProjects) == 0 {
96+
return nil, fmt.Errorf("Did not find any project with the name: %s", projectName)
97+
} else if len(matchingProjects) > 1 {
98+
return nil, fmt.Errorf("Found more than one project with the name: %s", projectName)
99+
}
100+
101+
return &matchingProjects[0], nil
102+
}
103+
33104
func (c *Client) GetProject(projectID string) (*Project, error) {
34105
req, err := http.NewRequest("GET", fmt.Sprintf("%s/v3/accounts/%s/projects/%s/?include_related=[freshness_job_id,docs_job_id]", c.HostURL, strconv.Itoa(c.AccountID), projectID), nil)
35106
if err != nil {

‎pkg/dbt_cloud/snowflake_credential.go

+4-15
Original file line numberDiff line numberDiff line change
@@ -7,11 +7,6 @@ import (
77
"strings"
88
)
99

10-
type SnowflakeCredentialListResponse struct {
11-
Data []SnowflakeCredential `json:"data"`
12-
Status ResponseStatus `json:"status"`
13-
}
14-
1510
type SnowflakeCredentialResponse struct {
1611
Data SnowflakeCredential `json:"data"`
1712
Status ResponseStatus `json:"status"`
@@ -36,7 +31,7 @@ type SnowflakeCredential struct {
3631
}
3732

3833
func (c *Client) GetSnowflakeCredential(projectId int, credentialId int) (*SnowflakeCredential, error) {
39-
req, err := http.NewRequest("GET", fmt.Sprintf("%s/v3/accounts/%d/projects/%d/credentials/", c.HostURL, c.AccountID, projectId), nil)
34+
req, err := http.NewRequest("GET", fmt.Sprintf("%s/v3/accounts/%d/projects/%d/credentials/%d/", c.HostURL, c.AccountID, projectId, credentialId), nil)
4035
if err != nil {
4136
return nil, err
4237
}
@@ -46,19 +41,13 @@ func (c *Client) GetSnowflakeCredential(projectId int, credentialId int) (*Snowf
4641
return nil, err
4742
}
4843

49-
snowflakeCredentialListResponse := SnowflakeCredentialListResponse{}
50-
err = json.Unmarshal(body, &snowflakeCredentialListResponse)
44+
snowflakeCredentialResponse := SnowflakeCredentialResponse{}
45+
err = json.Unmarshal(body, &snowflakeCredentialResponse)
5146
if err != nil {
5247
return nil, err
5348
}
5449

55-
for i, credential := range snowflakeCredentialListResponse.Data {
56-
if *credential.ID == credentialId {
57-
return &snowflakeCredentialListResponse.Data[i], nil
58-
}
59-
}
60-
61-
return nil, fmt.Errorf("resource-not-found: did not find credential ID %d in project ID %d", credentialId, projectId)
50+
return &snowflakeCredentialResponse.Data, nil
6251
}
6352

6453
func (c *Client) CreateSnowflakeCredential(projectId int, type_ string, isActive bool, database string, role string, warehouse string, schema string, user string, password string, privateKey string, privateKeyPassphrase string, authType string, numThreads int) (*SnowflakeCredential, error) {

0 commit comments

Comments
 (0)