Skip to content

Commit e7803f6

Browse files
devops-robdevops-rob
authored
Merge pull request #169 from devops-rob/feat/62-binary-response-base64
Add base64 response attributes to the terracurl_request data source for lossless binary downloads. Closes #62
2 parents 69d6289 + 951312f commit e7803f6

9 files changed

Lines changed: 265 additions & 36 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,9 @@
1+
## 2.11.0
2+
3+
ENHANCEMENTS:
4+
5+
- Add `response_base64` and `sensitive_response_base64` computed attributes to the `terracurl_request` data source for lossless binary response download via RFC 4648 base64 encoding. Closes #62.
6+
17
## 2.10.0
28

39
ENHANCEMENTS:

‎docs/data-sources/request.md‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,8 +44,10 @@ TerraCurl request data source
4444

4545
- `id` (String) Example identifier
4646
- `request_url_string` (String) Request URL includes parameters if request specified
47-
- `response` (String) JSON response received from request. Empty when `response_sensitive` is `true`; use `sensitive_response` instead.
48-
- `sensitive_response` (String, Sensitive) JSON response received from request, marked as sensitive so it is not displayed in plan output. Populated only when `response_sensitive` is `true`.
47+
- `response` (String) JSON response received from request. Empty when `response_sensitive` is `true`; use `sensitive_response` instead. For binary responses, use `response_base64` instead.
48+
- `response_base64` (String) Response body encoded as base64 (standard) as defined in RFC 4648. Use this for binary content. Empty when `response_sensitive` is `true`; use `sensitive_response_base64` instead.
49+
- `sensitive_response` (String, Sensitive) JSON response received from request, marked as sensitive so it is not displayed in plan output. Populated only when `response_sensitive` is `true`. For binary responses, use `sensitive_response_base64` instead.
50+
- `sensitive_response_base64` (String, Sensitive) Response body encoded as base64 (standard) as defined in RFC 4648, marked as sensitive. Populated only when `response_sensitive` is `true`.
4951
- `status_code` (String) Response status code received from request
5052

5153
<a id="nestedatt--digest_auth"></a>

‎docs/guides/file_and_multipart_bodies.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -170,6 +170,12 @@ Write-only string bodies (`*_request_body_wo`) remain available for inline secre
170170

171171
See the [Destroy Response Templating guide](destroy_templating) for placeholder syntax.
172172

173+
## Binary response bodies (data source)
174+
175+
To download binary files without UTF-8 corruption, use the data source computed attributes `response_base64` and `sensitive_response_base64`. These encode the raw response bytes with RFC 4648 standard base64. Decode in Terraform with `base64decode()` (for example when writing to disk with `local_file`).
176+
177+
See [`examples/data-sources/binary_response_example`](../../examples/data-sources/binary_response_example/data-source.tf).
178+
173179
## Limitations
174180

175181
- File contents are loaded fully into memory at request time (same as embedding bytes in configuration).

‎examples/data-sources/binary_response_example/data-source.tf‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
data "terracurl_request" "binary_download" {
2+
name = "binary-download"
3+
url = "https://api.example.com/files/report.pdf"
4+
method = "GET"
5+
response_codes = ["200"]
6+
}
7+
8+
# Decode the base64 response and write to disk with the local provider.
9+
resource "local_file" "report" {
10+
filename = "${path.module}/report.pdf"
11+
content = base64decode(data.terracurl_request.binary_download.response_base64)
12+
}

‎internal/provider/curl_data_source.go‎

Lines changed: 41 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -30,30 +30,32 @@ func NewCurlDataSource() datasource.DataSource {
3030
}
3131

3232
type CurlDataSourceModel struct {
33-
ID types.String `tfsdk:"id"`
34-
Name types.String `tfsdk:"name"`
35-
Url types.String `tfsdk:"url"`
36-
Method types.String `tfsdk:"method"`
37-
RequestBody types.String `tfsdk:"request_body"`
38-
RequestBodyFile types.String `tfsdk:"request_body_file"`
39-
RequestMultipart *MultipartConfigModel `tfsdk:"request_multipart"`
40-
Headers types.Map `tfsdk:"headers"`
41-
DigestAuth *DigestAuthModel `tfsdk:"digest_auth"`
42-
RequestParameters types.Map `tfsdk:"request_parameters"`
43-
RequestUrlString types.String `tfsdk:"request_url_string"`
44-
CertFile types.String `tfsdk:"cert_file"`
45-
KeyFile types.String `tfsdk:"key_file"`
46-
CaCertFile types.String `tfsdk:"ca_cert_file"`
47-
CaCertDirectory types.String `tfsdk:"ca_cert_directory"`
48-
SkipTlsVerify types.Bool `tfsdk:"skip_tls_verify"`
49-
RetryInterval types.Int64 `tfsdk:"retry_interval"`
50-
MaxRetry types.Int64 `tfsdk:"max_retry"`
51-
Timeout types.Int64 `tfsdk:"timeout"`
52-
Response types.String `tfsdk:"response"`
53-
SensitiveResponse types.String `tfsdk:"sensitive_response"`
54-
ResponseSensitive types.Bool `tfsdk:"response_sensitive"`
55-
ResponseCodes types.List `tfsdk:"response_codes"`
56-
StatusCode types.String `tfsdk:"status_code"`
33+
ID types.String `tfsdk:"id"`
34+
Name types.String `tfsdk:"name"`
35+
Url types.String `tfsdk:"url"`
36+
Method types.String `tfsdk:"method"`
37+
RequestBody types.String `tfsdk:"request_body"`
38+
RequestBodyFile types.String `tfsdk:"request_body_file"`
39+
RequestMultipart *MultipartConfigModel `tfsdk:"request_multipart"`
40+
Headers types.Map `tfsdk:"headers"`
41+
DigestAuth *DigestAuthModel `tfsdk:"digest_auth"`
42+
RequestParameters types.Map `tfsdk:"request_parameters"`
43+
RequestUrlString types.String `tfsdk:"request_url_string"`
44+
CertFile types.String `tfsdk:"cert_file"`
45+
KeyFile types.String `tfsdk:"key_file"`
46+
CaCertFile types.String `tfsdk:"ca_cert_file"`
47+
CaCertDirectory types.String `tfsdk:"ca_cert_directory"`
48+
SkipTlsVerify types.Bool `tfsdk:"skip_tls_verify"`
49+
RetryInterval types.Int64 `tfsdk:"retry_interval"`
50+
MaxRetry types.Int64 `tfsdk:"max_retry"`
51+
Timeout types.Int64 `tfsdk:"timeout"`
52+
Response types.String `tfsdk:"response"`
53+
SensitiveResponse types.String `tfsdk:"sensitive_response"`
54+
ResponseBase64 types.String `tfsdk:"response_base64"`
55+
SensitiveResponseBase64 types.String `tfsdk:"sensitive_response_base64"`
56+
ResponseSensitive types.Bool `tfsdk:"response_sensitive"`
57+
ResponseCodes types.List `tfsdk:"response_codes"`
58+
StatusCode types.String `tfsdk:"status_code"`
5759
}
5860

5961
func (d *CurlDataSource) Metadata(ctx context.Context, req datasource.MetadataRequest, resp *datasource.MetadataResponse) {
@@ -139,12 +141,21 @@ func (d *CurlDataSource) Schema(ctx context.Context, req datasource.SchemaReques
139141
},
140142
"response": schema.StringAttribute{
141143
Computed: true,
142-
MarkdownDescription: "JSON response received from request. Empty when `response_sensitive` is `true`; use `sensitive_response` instead.",
144+
MarkdownDescription: "JSON response received from request. Empty when `response_sensitive` is `true`; use `sensitive_response` instead. For binary responses, use `response_base64` instead.",
143145
},
144146
"sensitive_response": schema.StringAttribute{
145147
Computed: true,
146148
Sensitive: true,
147-
MarkdownDescription: "JSON response received from request, marked as sensitive so it is not displayed in plan output. Populated only when `response_sensitive` is `true`.",
149+
MarkdownDescription: "JSON response received from request, marked as sensitive so it is not displayed in plan output. Populated only when `response_sensitive` is `true`. For binary responses, use `sensitive_response_base64` instead.",
150+
},
151+
"response_base64": schema.StringAttribute{
152+
Computed: true,
153+
MarkdownDescription: "Response body encoded as base64 (standard) as defined in RFC 4648. Use this for binary content. Empty when `response_sensitive` is `true`; use `sensitive_response_base64` instead.",
154+
},
155+
"sensitive_response_base64": schema.StringAttribute{
156+
Computed: true,
157+
Sensitive: true,
158+
MarkdownDescription: "Response body encoded as base64 (standard) as defined in RFC 4648, marked as sensitive. Populated only when `response_sensitive` is `true`.",
148159
},
149160
"response_sensitive": schema.BoolAttribute{
150161
Optional: true,
@@ -302,7 +313,10 @@ func (d *CurlDataSource) Read(ctx context.Context, req datasource.ReadRequest, r
302313
}
303314

304315
data.RequestUrlString = types.StringValue(request.URL.String())
305-
setDataSourceResponseValues(&data, bodyString)
316+
if responseBodyContainsInvalidUTF8(body) {
317+
tflog.Warn(ctx, "Response body is not valid UTF-8; use response_base64 for binary content")
318+
}
319+
setDataSourceResponseValuesFromBytes(&data, body, bodyString)
306320
data.StatusCode = types.StringValue(strconv.Itoa(statusCode))
307321

308322
// Save data into Terraform state.

‎internal/provider/curl_data_source_test.go‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package provider
22

33
import (
4+
"encoding/base64"
45
"fmt"
56
"github.com/hashicorp/terraform-plugin-testing/helper/acctest"
67
"github.com/hashicorp/terraform-plugin-testing/helper/resource"
@@ -458,3 +459,90 @@ data "terracurl_request" "default_test" {
458459
}
459460
`, name)
460461
}
462+
463+
func TestAccDataSourceCurlResponseBase64(t *testing.T) {
464+
t.Setenv("TF_ACC", "true")
465+
t.Setenv("USE_DEFAULT_CLIENT_FOR_TESTS", "true")
466+
467+
binaryBody := []byte{0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x01, 0x02}
468+
expectedBase64 := base64.StdEncoding.EncodeToString(binaryBody)
469+
470+
httpmock.Activate()
471+
defer httpmock.DeactivateAndReset()
472+
httpmock.RegisterResponder(
473+
"GET",
474+
"https://example.com/binary",
475+
httpmock.NewBytesResponder(200, binaryBody),
476+
)
477+
478+
rName := acctest.RandStringFromCharSet(10, acctest.CharSetAlphaNum)
479+
480+
resource.Test(t, resource.TestCase{
481+
PreCheck: func() { testAccPreCheck(t) },
482+
ProtoV6ProviderFactories: testAccProtoV6ProviderFactories,
483+
Steps: []resource.TestStep{
484+
{
485+
Config: testAccDataSourceCurlResponseBase64(rName),
486+
Check: resource.ComposeTestCheckFunc(
487+
resource.TestCheckResourceAttr("data.terracurl_request.binary_test", "response_base64", expectedBase64),
488+
resource.TestCheckResourceAttr("data.terracurl_request.binary_test", "sensitive_response_base64", ""),
489+
),
490+
},
491+
},
492+
})
493+
}
494+
495+
func testAccDataSourceCurlResponseBase64(name string) string {
496+
return fmt.Sprintf(`
497+
data "terracurl_request" "binary_test" {
498+
name = "%s"
499+
url = "https://example.com/binary"
500+
method = "GET"
501+
response_codes = ["200"]
502+
}
503+
`, name)
504+
}
505+
506+
func TestAccDataSourceCurlSensitiveResponseBase64(t *testing.T) {
507+
t.Setenv("TF_ACC", "true")
508+
t.Setenv("USE_DEFAULT_CLIENT_FOR_TESTS", "true")
509+
510+
binaryBody := []byte{0x00, 0x01, 0x02, 0xff}
511+
expectedBase64 := base64.StdEncoding.EncodeToString(binaryBody)
512+
513+
httpmock.Activate()
514+
defer httpmock.DeactivateAndReset()
515+
httpmock.RegisterResponder(
516+
"GET",
517+
"https://example.com/binary-sensitive",
518+
httpmock.NewBytesResponder(200, binaryBody),
519+
)
520+
521+
rName := acctest.RandStringFromCharSet(10, acctest.CharSetAlphaNum)
522+
523+
resource.Test(t, resource.TestCase{
524+
PreCheck: func() { testAccPreCheck(t) },
525+
ProtoV6ProviderFactories: testAccProtoV6ProviderFactories,
526+
Steps: []resource.TestStep{
527+
{
528+
Config: testAccDataSourceCurlSensitiveResponseBase64(rName),
529+
Check: resource.ComposeTestCheckFunc(
530+
resource.TestCheckResourceAttr("data.terracurl_request.binary_sensitive_test", "response_base64", ""),
531+
resource.TestCheckResourceAttr("data.terracurl_request.binary_sensitive_test", "sensitive_response_base64", expectedBase64),
532+
),
533+
},
534+
},
535+
})
536+
}
537+
538+
func testAccDataSourceCurlSensitiveResponseBase64(name string) string {
539+
return fmt.Sprintf(`
540+
data "terracurl_request" "binary_sensitive_test" {
541+
name = "%s"
542+
url = "https://example.com/binary-sensitive"
543+
method = "GET"
544+
response_codes = ["200"]
545+
response_sensitive = true
546+
}
547+
`, name)
548+
}

‎internal/provider/response_base64_test.go‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
package provider
2+
3+
import (
4+
"encoding/base64"
5+
"testing"
6+
7+
"github.com/hashicorp/terraform-plugin-framework/types"
8+
)
9+
10+
func TestEncodeResponseBase64(t *testing.T) {
11+
want := []byte{0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a}
12+
got := encodeResponseBase64(want)
13+
if got != "iVBORw0KGgo=" {
14+
t.Fatalf("got %q", got)
15+
}
16+
17+
if encodeResponseBase64(nil) != "" {
18+
t.Fatal("expected empty string for nil body")
19+
}
20+
if encodeResponseBase64([]byte{}) != "" {
21+
t.Fatal("expected empty string for empty body")
22+
}
23+
}
24+
25+
func TestSetDataSourceResponseValuesFromBytes(t *testing.T) {
26+
body := []byte{0x00, 0x01, 0x02}
27+
encoded := base64.StdEncoding.EncodeToString(body)
28+
29+
t.Run("non-sensitive", func(t *testing.T) {
30+
data := &CurlDataSourceModel{
31+
ResponseSensitive: types.BoolValue(false),
32+
}
33+
setDataSourceResponseValuesFromBytes(data, body, string(body))
34+
35+
if data.Response.ValueString() != string(body) {
36+
t.Fatalf("response got %q", data.Response.ValueString())
37+
}
38+
if data.ResponseBase64.ValueString() != encoded {
39+
t.Fatalf("response_base64 got %q want %q", data.ResponseBase64.ValueString(), encoded)
40+
}
41+
if data.SensitiveResponse.ValueString() != "" || data.SensitiveResponseBase64.ValueString() != "" {
42+
t.Fatal("expected sensitive response attrs to be empty")
43+
}
44+
})
45+
46+
t.Run("sensitive", func(t *testing.T) {
47+
data := &CurlDataSourceModel{
48+
ResponseSensitive: types.BoolValue(true),
49+
}
50+
setDataSourceResponseValuesFromBytes(data, body, string(body))
51+
52+
if data.Response.ValueString() != "" || data.ResponseBase64.ValueString() != "" {
53+
t.Fatal("expected non-sensitive response attrs to be empty")
54+
}
55+
if data.SensitiveResponse.ValueString() != string(body) {
56+
t.Fatalf("sensitive_response got %q", data.SensitiveResponse.ValueString())
57+
}
58+
if data.SensitiveResponseBase64.ValueString() != encoded {
59+
t.Fatalf("sensitive_response_base64 got %q want %q", data.SensitiveResponseBase64.ValueString(), encoded)
60+
}
61+
})
62+
}
63+
64+
func TestResponseBodyContainsInvalidUTF8(t *testing.T) {
65+
if !responseBodyContainsInvalidUTF8([]byte{0xff, 0xfe, 0xfd}) {
66+
t.Fatal("expected invalid UTF-8 detection")
67+
}
68+
if responseBodyContainsInvalidUTF8([]byte("hello")) {
69+
t.Fatal("expected valid UTF-8")
70+
}
71+
if responseBodyContainsInvalidUTF8(nil) {
72+
t.Fatal("expected empty body to be valid")
73+
}
74+
}

‎internal/provider/utilities.go‎

Lines changed: 28 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package provider
33
import (
44
"crypto/tls"
55
"crypto/x509"
6+
"encoding/base64"
67
"encoding/json"
78
"fmt"
89
"net/http"
@@ -11,6 +12,7 @@ import (
1112
"strconv"
1213
"strings"
1314
"time"
15+
"unicode/utf8"
1416

1517
"github.com/hashicorp/terraform-plugin-framework/attr"
1618
"github.com/hashicorp/terraform-plugin-framework/types"
@@ -57,6 +59,28 @@ func setResponseValue(sensitive bool, response, sensitiveResponse *types.String,
5759
}
5860
}
5961

62+
func encodeResponseBase64(body []byte) string {
63+
if len(body) == 0 {
64+
return ""
65+
}
66+
return base64.StdEncoding.EncodeToString(body)
67+
}
68+
69+
func setResponseBase64Value(sensitive bool, response, sensitiveResponse *types.String, body []byte) {
70+
encoded := encodeResponseBase64(body)
71+
if sensitive {
72+
*response = types.StringValue("")
73+
*sensitiveResponse = types.StringValue(encoded)
74+
} else {
75+
*response = types.StringValue(encoded)
76+
*sensitiveResponse = types.StringValue("")
77+
}
78+
}
79+
80+
func responseBodyContainsInvalidUTF8(body []byte) bool {
81+
return len(body) > 0 && !utf8.Valid(body)
82+
}
83+
6084
func responseSensitiveEnabled(value types.Bool) bool {
6185
if value.IsNull() || value.IsUnknown() {
6286
return false
@@ -107,13 +131,10 @@ func setEphemeralCloseResponse(data *CurlEphemeralModel, body string) {
107131
)
108132
}
109133

110-
func setDataSourceResponseValues(data *CurlDataSourceModel, body string) {
111-
setResponseValue(
112-
responseSensitiveEnabled(data.ResponseSensitive),
113-
&data.Response,
114-
&data.SensitiveResponse,
115-
body,
116-
)
134+
func setDataSourceResponseValuesFromBytes(data *CurlDataSourceModel, body []byte, bodyString string) {
135+
sensitive := responseSensitiveEnabled(data.ResponseSensitive)
136+
setResponseValue(sensitive, &data.Response, &data.SensitiveResponse, bodyString)
137+
setResponseBase64Value(sensitive, &data.ResponseBase64, &data.SensitiveResponseBase64, body)
117138
}
118139

119140
func responseCodeChecker(expectedStatusCodes types.List, receivedStatusCode int) bool {

‎templates/guides/file_and_multipart_bodies.md.tmpl‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -170,6 +170,12 @@ Write-only string bodies (`*_request_body_wo`) remain available for inline secre
170170

171171
See the [Destroy Response Templating guide](destroy_templating) for placeholder syntax.
172172

173+
## Binary response bodies (data source)
174+
175+
To download binary files without UTF-8 corruption, use the data source computed attributes `response_base64` and `sensitive_response_base64`. These encode the raw response bytes with RFC 4648 standard base64. Decode in Terraform with `base64decode()` (for example when writing to disk with `local_file`).
176+
177+
See [`examples/data-sources/binary_response_example`](../../examples/data-sources/binary_response_example/data-source.tf).
178+
173179
## Limitations
174180

175181
- File contents are loaded fully into memory at request time (same as embedding bytes in configuration).

0 commit comments

Comments
 (0)