redactor add: Add optional filtering like env vars

Author: DrJosh9000

clicommand/redactor_add.go

@@ -11,6 +11,8 @@ import (
 	"slices"
 	"strings"
 
+	"github.com/buildkite/agent/v3/env"
+	"github.com/buildkite/agent/v3/internal/redact"
 	"github.com/buildkite/agent/v3/jobapi"
 	"github.com/buildkite/agent/v3/logger"
 	"github.com/urfave/cli"
@@ -38,8 +40,10 @@ type RedactorAddConfig struct {
 	GlobalConfig
 	APIConfig
 
-	File   string `cli:"arg:0"`
-	Format string `cli:"format"`
+	File            string   `cli:"arg:0"`
+	Format          string   `cli:"format"`
+	ApplyVarsFilter bool     `cli:"apply-vars-filter"`
+	RedactedVars    []string `cli:"redacted-vars"`
 }
 
 var RedactorAddCommand = cli.Command{
@@ -86,6 +90,12 @@ JSON does not allow duplicate keys. If you repeat the same key ("key"), the JSON
 			EnvVar: "BUILDKITE_AGENT_REDACT_ADD_FORMAT",
 			Value:  FormatStringNone,
 		},
+		cli.BoolFlag{
+			Name:   "apply-vars-filter",
+			Usage:  fmt.Sprintf("When the input is in 'json' format, filters the secrets to redact using the same rules used to detect secrets from environment variables: secrets must be at least %d characters long, and names must match the patterns defined by --redacted-vars or $BUILDKITE_REDACTED_VARS.", redact.LengthMin),
+			EnvVar: "BUILDKITE_AGENT_REDACT_VARS_FILTER",
+		},
+		RedactedVars,
 	}),
 	Action: func(c *cli.Context) error {
 		ctx := context.Background()
@@ -144,14 +154,29 @@ func ParseSecrets(
 ) ([]string, error) {
 	switch cfg.Format {
 	case FormatStringJSON:
-		secrets := &map[string]string{}
+		secrets := map[string]string{}
 		if err := json.NewDecoder(secretsReader).Decode(&secrets); err != nil {
 			return nil, fmt.Errorf("failed to parse as string valued JSON: %w", err)
 		}
 
-		parsedSecrets := make([]string, 0, len(*secrets))
-		for _, secret := range *secrets {
-			parsedSecrets = append(parsedSecrets, secret)
+		var parsedSecrets []string
+		if cfg.ApplyVarsFilter {
+			matched, short, err := redact.Vars(cfg.RedactedVars, env.FromMap(secrets).DumpPairs())
+			if err != nil {
+				return nil, fmt.Errorf("couldn't match object keys against redacted-vars: %w", err)
+			}
+			if len(short) > 0 {
+				l.Warn("Some object keys had values below minimum length (%d bytes) and will not be redacted: %s", redact.LengthMin, strings.Join(short, ", "))
+			}
+			parsedSecrets = make([]string, 0, len(matched))
+			for _, m := range matched {
+				parsedSecrets = append(parsedSecrets, m.Value)
+			}
+		} else {
+			parsedSecrets = make([]string, 0, len(secrets))
+			for _, secret := range secrets {
+				parsedSecrets = append(parsedSecrets, secret)
+			}
 		}
 
 		return parsedSecrets, nil
@@ -165,7 +190,7 @@ func ParseSecrets(
 		return []string{strings.TrimSpace(string(readSecret))}, nil
 
 	default:
-		return nil, fmt.Errorf("%s: %w", cfg.Format, errUnknownFormat)
+		return nil, fmt.Errorf("%w %q", errUnknownFormat, cfg.Format)
 	}
 }
 

clicommand/redactor_add_test.go

@@ -1,58 +1,108 @@
 package clicommand_test
 
 import (
+	"encoding/json"
+	"errors"
 	"slices"
 	"strings"
 	"testing"
 
 	"github.com/buildkite/agent/v3/clicommand"
 	"github.com/buildkite/agent/v3/logger"
-	"gotest.tools/v3/assert"
+	"github.com/google/go-cmp/cmp"
 )
 
 func TestParseSecrets(t *testing.T) {
 	t.Parallel()
 
 	for _, tc := range []struct {
-		name              string
-		inputData         string
-		formatString      string
-		expectedSecrets   []string
-		errorTextContains string
+		name            string
+		inputData       string
+		formatString    string
+		applyVarsFilter bool
+		wantSecrets     []string
 	}{
 		{
-			name:            "json",
-			inputData:       `{"hello": "world", "password": "hunter2"}`,
-			formatString:    clicommand.FormatStringJSON,
-			expectedSecrets: []string{"world", "hunter2"},
+			name:         "json",
+			inputData:    `{"hello": "world", "password": "hunter2"}`,
+			formatString: clicommand.FormatStringJSON,
+			wantSecrets:  []string{"world", "hunter2"},
 		},
 		{
-			name:            "plaintext",
-			inputData:       "hunter2\n",
-			formatString:    clicommand.FormatStringNone,
-			expectedSecrets: []string{"hunter2"},
+			name:         "plaintext",
+			inputData:    "hunter2\n",
+			formatString: clicommand.FormatStringNone,
+			wantSecrets:  []string{"hunter2"},
 		},
+
 		{
-			name:              "invalid_json",
-			inputData:         `{"hello": 1, "password": "hunter2"}`,
-			formatString:      clicommand.FormatStringJSON,
-			errorTextContains: "failed to parse as string valued JSON",
+			name:            "vars filter",
+			inputData:       `{"HELLO": "1", "MY_PASSWORD": "hunter2"}`,
+			applyVarsFilter: true,
+			formatString:    clicommand.FormatStringJSON,
+			wantSecrets:     []string{"hunter2"},
 		},
 	} {
 		t.Run(tc.name, func(t *testing.T) {
 			t.Parallel()
 
 			input := strings.NewReader(tc.inputData)
-			secrets, err := clicommand.ParseSecrets(logger.Discard, clicommand.RedactorAddConfig{Format: tc.formatString}, input)
-			if tc.errorTextContains != "" {
-				assert.ErrorContains(t, err, tc.errorTextContains)
-				return
+			secrets, err := clicommand.ParseSecrets(
+				logger.Discard,
+				clicommand.RedactorAddConfig{
+					Format:          tc.formatString,
+					ApplyVarsFilter: tc.applyVarsFilter,
+					RedactedVars:    *clicommand.RedactedVars.Value,
+				},
+				input,
+			)
+			if err != nil {
+				t.Errorf("clicommand.ParseSecrets(logger, cfg, %q) error = %v", input, err)
 			}
-			assert.NilError(t, err)
 
 			slices.Sort(secrets)
-			slices.Sort(tc.expectedSecrets)
-			assert.DeepEqual(t, secrets, tc.expectedSecrets)
+			slices.Sort(tc.wantSecrets)
+			if diff := cmp.Diff(secrets, tc.wantSecrets); diff != "" {
+				t.Errorf("clicommand.ParseSecrets(logger, cfg, %q) secrets diff (-got +want):\n%s", input, diff)
+			}
+		})
+	}
+}
+
+func TestParseSecrets_JSON(t *testing.T) {
+	t.Parallel()
+
+	for _, tc := range []struct {
+		name      string
+		inputData string
+		wantError any
+	}{
+		{
+			name:      "type mismatch",
+			inputData: `{"hello": 1, "password": "hunter2"}`,
+			wantError: new(*json.UnmarshalTypeError),
+		},
+		{
+			name:      "syntax error",
+			inputData: `}}{"hello": , "pas: "hun'}`,
+			wantError: new(*json.SyntaxError),
+		},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+
+			input := strings.NewReader(tc.inputData)
+
+			_, err := clicommand.ParseSecrets(
+				logger.Discard,
+				clicommand.RedactorAddConfig{
+					Format: clicommand.FormatStringJSON,
+				},
+				input,
+			)
+			if !errors.As(err, tc.wantError) {
+				t.Errorf("clicommand.ParseSecrets(logger, cfg, %q) error = %v, want error wrapping %T", input, err, tc.wantError)
+			}
 		})
 	}
 }