Merge pull request #737 from mozilla/develop

v3.6.1 - develop -> master

Author: ajvb

CHANGELOG.rst

@@ -1,6 +1,24 @@
 Changelog
 =========
 
+3.6.1
+-----
+Features:
+
+    * Add support for --unencrypted-regex (#715)
+
+Changes:
+
+    * Use keys.openpgp.org instead of gpg.mozilla.org (#732)
+    * Upgrade AWS SDK version (#714)
+    * Support --input-type for exec-file (#699)
+
+Bug fixes:
+
+    * Fixes broken Vault tests (#731)
+    * Revert "Add standard newline/quoting behavior to dotenv store" (#706)
+
+
 3.6.0
 -----
 Features:

README.rst

@@ -673,10 +673,9 @@ Example: place the following in your ``~/.bashrc``
 Specify a different GPG key server
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-By default, ``sops`` uses the key server ``gpg.mozilla.org`` to retrieve the GPG
+By default, ``sops`` uses the key server ``keys.openpgp.org`` to retrieve the GPG
 keys that are not present in the local keyring.
-To use a different GPG key server, set the ``SOPS_GPG_KEYSERVER`` environment
-variable.
+This is no longer configurable. You can learn more about why from this write-up: [SKS Keyserver Network Under Attack](https://gist.github.com/rjhansen/67ab921ffb4084c865b3618d6955275f).
 
 Example: place the following in your ``~/.bashrc``
 
@@ -1426,9 +1425,20 @@ will encrypt the values under the ``data`` and ``stringData`` keys in a YAML fil
 containing kubernetes secrets.  It will not encrypt other values that help you to
 navigate the file, like ``metadata`` which contains the secrets' names.
 
+Conversely, you can opt in to only left certain keys without encrypting by using the 
+``--unencrypted-regex`` option, which will leave the values unencrypted of those keys 
+that match the supplied regular expression. For example, this command:
+
+.. code:: bash
+
+  $ sops --encrypt --unencrypted-regex '^(description|metadata)$' k8s-secrets.yaml
+
+will not encrypt the values under the ``description`` and ``metadata`` keys in a YAML file
+containing kubernetes secrets, while encrypting everything else.
+
 You can also specify these options in the ``.sops.yaml`` config file.
 
-Note: these three options ``--unencrypted-suffix``, ``--encrypted-suffix``, and ``--encrypted-regex`` are
+Note: these four options ``--unencrypted-suffix``, ``--encrypted-suffix``, ``--encrypted-regex`` and ``--unencrypted-regex`` are
 mutually exclusive and cannot all be used in the same file.
 
 Encryption Protocol

cmd/sops/edit.go

@@ -37,6 +37,7 @@ type editExampleOpts struct {
 	editOpts
 	UnencryptedSuffix string
 	EncryptedSuffix   string
+	UnencryptedRegex  string
 	EncryptedRegex    string
 	KeyGroups         []sops.KeyGroup
 	GroupThreshold    int
@@ -66,6 +67,7 @@ func editExample(opts editExampleOpts) ([]byte, error) {
 			KeyGroups:         opts.KeyGroups,
 			UnencryptedSuffix: opts.UnencryptedSuffix,
 			EncryptedSuffix:   opts.EncryptedSuffix,
+			UnencryptedRegex:  opts.UnencryptedRegex,
 			EncryptedRegex:    opts.EncryptedRegex,
 			Version:           version.Version,
 			ShamirThreshold:   opts.GroupThreshold,
@@ -132,7 +134,7 @@ func editTree(opts editOpts, tree *sops.Tree, dataKey []byte) ([]byte, error) {
 	if err != nil {
 		return nil, common.NewExitError(fmt.Sprintf("Could not write output file: %s", err), codes.CouldNotWriteOutputFile)
 	}
-	
+
 	// Close temporary file, since Windows won't delete the file unless it's closed beforehand
 	defer tmpfile.Close()
 

cmd/sops/encrypt.go

@@ -22,6 +22,7 @@ type encryptOpts struct {
 	KeyServices       []keyservice.KeyServiceClient
 	UnencryptedSuffix string
 	EncryptedSuffix   string
+	UnencryptedRegex  string
 	EncryptedRegex    string
 	KeyGroups         []sops.KeyGroup
 	GroupThreshold    int
@@ -77,6 +78,7 @@ func encrypt(opts encryptOpts) (encryptedFile []byte, err error) {
 			KeyGroups:         opts.KeyGroups,
 			UnencryptedSuffix: opts.UnencryptedSuffix,
 			EncryptedSuffix:   opts.EncryptedSuffix,
+			UnencryptedRegex:  opts.UnencryptedRegex,
 			EncryptedRegex:    opts.EncryptedRegex,
 			Version:           version.Version,
 			ShamirThreshold:   opts.GroupThreshold,

cmd/sops/main.go

@@ -109,7 +109,6 @@ func main() {
    the "add-{kms,pgp,gcp-kms,azure-kv,hc-vault-transit}" and "rm-{kms,pgp,gcp-kms,azure-kv,hc-vault-transit}" flags.
 
    To use a different GPG binary than the one in your PATH, set SOPS_GPG_EXEC.
-   To use a GPG key server other than gpg.mozilla.org, set SOPS_GPG_KEYSERVER.
 
    To select a different editor than the default (vim), set EDITOR.
 
@@ -184,6 +183,14 @@ func main() {
 					Name:  "user",
 					Usage: "the user to run the command as",
 				},
+				cli.StringFlag{
+					Name:  "input-type",
+					Usage: "currently json, yaml, dotenv and binary are supported. If not set, sops will use the file's extension to determine the type",
+				},
+				cli.StringFlag{
+					Name:  "output-type",
+					Usage: "currently json, yaml, dotenv and binary are supported. If not set, sops will use the input file's extension to determine the output format",
+				},
 			}, keyserviceFlags...),
 			Action: func(c *cli.Context) error {
 				if len(c.Args()) != 2 {
@@ -617,6 +624,10 @@ func main() {
 			Name:  "encrypted-suffix",
 			Usage: "override the encrypted key suffix. When empty, all keys will be encrypted, unless otherwise marked with unencrypted-suffix.",
 		},
+		cli.StringFlag{
+			Name:  "unencrypted-regex",
+			Usage: "set the unencrypted key suffix. When specified, only keys matching the regex will be left unencrypted.",
+		},
 		cli.StringFlag{
 			Name:  "encrypted-regex",
 			Usage: "set the encrypted key suffix. When specified, only keys matching the regex will be encrypted.",
@@ -674,6 +685,7 @@ func main() {
 		unencryptedSuffix := c.String("unencrypted-suffix")
 		encryptedSuffix := c.String("encrypted-suffix")
 		encryptedRegex := c.String("encrypted-regex")
+		unencryptedRegex := c.String("unencrypted-regex")
 		conf, err := loadConfig(c, fileName, nil)
 		if err != nil {
 			return toExitError(err)
@@ -689,6 +701,9 @@ func main() {
 			if encryptedRegex == "" {
 				encryptedRegex = conf.EncryptedRegex
 			}
+			if unencryptedRegex == "" {
+				unencryptedRegex = conf.UnencryptedRegex
+			}
 		}
 
 		cryptRuleCount := 0
@@ -701,9 +716,12 @@ func main() {
 		if encryptedRegex != "" {
 			cryptRuleCount++
 		}
+		if unencryptedRegex != "" {
+			cryptRuleCount++
+		}
 
 		if cryptRuleCount > 1 {
-			return common.NewExitError("Error: cannot use more than one of encrypted_suffix, unencrypted_suffix, or encrypted_regex in the same file", codes.ErrorConflictingParameters)
+			return common.NewExitError("Error: cannot use more than one of encrypted_suffix, unencrypted_suffix, encrypted_regex or unencrypted_regex in the same file", codes.ErrorConflictingParameters)
 		}
 
 		// only supply the default UnencryptedSuffix when EncryptedSuffix and EncryptedRegex are not provided
@@ -734,6 +752,7 @@ func main() {
 				Cipher:            aes.NewCipher(),
 				UnencryptedSuffix: unencryptedSuffix,
 				EncryptedSuffix:   encryptedSuffix,
+				UnencryptedRegex:  unencryptedRegex,
 				EncryptedRegex:    encryptedRegex,
 				KeyServices:       svcs,
 				KeyGroups:         groups,
@@ -871,6 +890,7 @@ func main() {
 					editOpts:          opts,
 					UnencryptedSuffix: unencryptedSuffix,
 					EncryptedSuffix:   encryptedSuffix,
+					UnencryptedRegex:  unencryptedRegex,
 					EncryptedRegex:    encryptedRegex,
 					KeyGroups:         groups,
 					GroupThreshold:    threshold,

cmd/sops/subcommand/exec/exec.go

@@ -1,14 +1,13 @@
 package exec
 
 import (
-	"fmt"
+	"bytes"
 	"io/ioutil"
 	"os"
 	"runtime"
 	"strings"
 
 	"go.mozilla.org/sops/v3/logging"
-	"go.mozilla.org/sops/v3/stores/dotenv"
 
 	"github.com/sirupsen/logrus"
 )
@@ -85,18 +84,15 @@ func ExecWithEnv(opts ExecOpts) error {
 	}
 
 	env := os.Environ()
-	store := dotenv.Store{}
-
-	branches, err := store.LoadPlainFile(opts.Plaintext)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	for _, item := range branches[0] {
-		if item.Value == nil {
+	lines := bytes.Split(opts.Plaintext, []byte("\n"))
+	for _, line := range lines {
+		if len(line) == 0 {
+			continue
+		}
+		if line[0] == '#' {
 			continue
 		}
-		env = append(env, fmt.Sprintf("%s=%s", item.Key, item.Value))
+		env = append(env, string(line))
 	}
 
 	cmd := BuildCommand(opts.Command)

config/config.go

@@ -117,6 +117,7 @@ type creationRule struct {
 	ShamirThreshold   int        `yaml:"shamir_threshold"`
 	UnencryptedSuffix string     `yaml:"unencrypted_suffix"`
 	EncryptedSuffix   string     `yaml:"encrypted_suffix"`
+	UnencryptedRegex  string     `yaml:"unencrypted_regex"`
 	EncryptedRegex    string     `yaml:"encrypted_regex"`
 }
 
@@ -135,6 +136,7 @@ type Config struct {
 	ShamirThreshold   int
 	UnencryptedSuffix string
 	EncryptedSuffix   string
+	UnencryptedRegex  string
 	EncryptedRegex    string
 	Destination       publish.Destination
 	OmitExtensions    bool
@@ -235,6 +237,7 @@ func configFromRule(rule *creationRule, kmsEncryptionContext map[string]*string)
 		ShamirThreshold:   rule.ShamirThreshold,
 		UnencryptedSuffix: rule.UnencryptedSuffix,
 		EncryptedSuffix:   rule.EncryptedSuffix,
+		UnencryptedRegex:  rule.UnencryptedRegex,
 		EncryptedRegex:    rule.EncryptedRegex,
 	}, nil
 }

config/config_test.go

@@ -137,6 +137,7 @@ creation_rules:
     kms: "1"
     pgp: "2"
     encrypted_regex: "^enc:"
+    unencrypted_regex: "^dec:"
     `)
 
 var sampleConfigWithInvalidParameters = []byte(`
@@ -349,6 +350,12 @@ func TestLoadConfigFileWithEncryptedSuffix(t *testing.T) {
 	assert.Equal(t, "_enc", conf.EncryptedSuffix)
 }
 
+func TestLoadConfigFileWithUnencryptedRegex(t *testing.T) {
+	conf, err := parseCreationRuleForFile(parseConfigFile(sampleConfigWithRegexParameters, t), "barbar", nil)
+	assert.Equal(t, nil, err)
+	assert.Equal(t, "^dec:", conf.UnencryptedRegex)
+}
+
 func TestLoadConfigFileWithEncryptedRegex(t *testing.T) {
 	conf, err := parseCreationRuleForFile(parseConfigFile(sampleConfigWithRegexParameters, t), "barbar", nil)
 	assert.Equal(t, nil, err)
@@ -372,9 +379,9 @@ func TestLoadConfigFileWithVaultDestinationRules(t *testing.T) {
 	conf, err := parseDestinationRuleForFile(parseConfigFile(sampleConfigWithVaultDestinationRules, t), "vault-v2/barfoo", nil)
 	assert.Nil(t, err)
 	assert.NotNil(t, conf.Destination)
-	assert.Equal(t, "http://127.0.0.1:8200/v1/secret/data/foobar/barfoo", conf.Destination.Path("barfoo"))
+	assert.Contains(t, conf.Destination.Path("barfoo"), "/v1/secret/data/foobar/barfoo")
 	conf, err = parseDestinationRuleForFile(parseConfigFile(sampleConfigWithVaultDestinationRules, t), "vault-v1/barfoo", nil)
 	assert.Nil(t, err)
 	assert.NotNil(t, conf.Destination)
-	assert.Equal(t, "http://127.0.0.1:8200/v1/kv/barfoo/barfoo", conf.Destination.Path("barfoo"))
+	assert.Contains(t, conf.Destination.Path("barfoo"), "/v1/kv/barfoo/barfoo")
 }

go.mod

@@ -12,7 +12,7 @@ require (
 	github.com/Azure/go-autorest/autorest/validation v0.2.0 // indirect
 	github.com/Microsoft/go-winio v0.4.14 // indirect
 	github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect
-	github.com/aws/aws-sdk-go v1.23.13
+	github.com/aws/aws-sdk-go v1.33.18
 	github.com/blang/semver v3.5.1+incompatible
 	github.com/cenkalti/backoff v2.2.1+incompatible // indirect
 	github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc // indirect
@@ -34,13 +34,14 @@ require (
 	github.com/opencontainers/image-spec v1.0.1 // indirect
 	github.com/opencontainers/runc v0.1.1 // indirect
 	github.com/ory/dockertest v3.3.4+incompatible
-	github.com/pkg/errors v0.8.1
+	github.com/pkg/errors v0.9.1
 	github.com/sirupsen/logrus v1.4.2
 	github.com/smartystreets/goconvey v0.0.0-20190710185942-9d28bd7c0945 // indirect
-	github.com/stretchr/testify v1.3.0
+	github.com/stretchr/testify v1.5.1
+	github.com/vektra/mockery v1.1.2 // indirect
 	go.mozilla.org/gopgagent v0.0.0-20170926210634-4d7ea76ff71a
-	golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4
-	golang.org/x/net v0.0.0-20190724013045-ca1201d0de80
+	golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550
+	golang.org/x/net v0.0.0-20200226121028-0de0cce0169b
 	golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45
 	google.golang.org/api v0.7.0
 	google.golang.org/grpc v1.22.1