bug 1567320: add support for funnel_experiment and funnel_variation a… (#95)

* bug 1567320: add support for funnel_experiment and funnel_variation attribution keys

* add funnel params to test and fix length test

* add original test back

* validator: increase accepted code length to 600

* s/funnel_//

* set maxUnescapedCodeLen to match AttributionCode.jsm

* fix pingdom test

* fix tests and allow blank inputs

* add ua attribution key

* return error if code is empty

* tests to account for code is empty

Author: oremj

attributioncode/validator.go

@@ -5,18 +5,33 @@ import (
 	"crypto/sha256"
 	"encoding/base64"
 	"encoding/hex"
+	"fmt"
 	"net/url"
 	"time"
 
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 )
 
+// Set to match https://searchfox.org/mozilla-central/rev/a92ed79b0bc746159fc31af1586adbfa9e45e264/browser/components/attribution/AttributionCode.jsm#24
+const maxUnescapedCodeLen = 1010
+
 var validAttributionKeys = map[string]bool{
-	"source":   true,
-	"medium":   true,
-	"campaign": true,
-	"content":  true,
+	"source":     true,
+	"medium":     true,
+	"campaign":   true,
+	"content":    true,
+	"experiment": true,
+	"variation":  true,
+	"ua":         true,
+}
+
+// If any of these are not set in the incoming payload, they will be set to '(not set)'
+var requiredAttributionKeys = []string{
+	"source",
+	"medium",
+	"campaign",
+	"content",
 }
 
 var base64Decoder = base64.URLEncoding.WithPadding('.')
@@ -38,6 +53,12 @@ func NewValidator(hmacKey string, timeout time.Duration) *Validator {
 // Validate validates and sanitizes attribution code and signature
 func (v *Validator) Validate(code, sig string) (string, error) {
 	logEntry := logrus.WithField("b64code", code)
+
+	if code == "" {
+		logEntry.Error("code is empty")
+		return "", errors.New("code is empty")
+	}
+
 	if len(code) > 5000 {
 		logEntry.WithField("code_len", len(code)).Error("code longer than 5000 characters")
 		return "", errors.New("base64 code longer than 5000 characters")
@@ -50,9 +71,10 @@ func (v *Validator) Validate(code, sig string) (string, error) {
 	}
 
 	logEntry = logrus.WithField("code", unEscapedCode)
-	if len(unEscapedCode) > 200 {
-		logEntry.WithField("code_len", len(code)).Error("code longer than 200 characters")
-		return "", errors.New("code longer than 200 characters")
+	if len(unEscapedCode) > maxUnescapedCodeLen {
+		errMsg := fmt.Sprintf("code longer than %d characters", maxUnescapedCodeLen)
+		logEntry.WithField("code_len", len(code)).Error(errMsg)
+		return "", errors.New(errMsg)
 	}
 
 	vals, err := url.ParseQuery(string(unEscapedCode))
@@ -78,17 +100,17 @@ func (v *Validator) Validate(code, sig string) (string, error) {
 		}
 	}
 
-	// all keys are included
-	if len(vals) != len(validAttributionKeys) {
-		logrus.Error("code is missing keys")
-		return "", errors.New("code is missing keys")
-	}
-
 	if source := vals.Get("source"); !isWhitelisted(source) {
 		logrus.WithField("source", source).Error("source is not in whitelist")
 		vals.Set("source", "(other)")
 	}
 
+	for _, val := range requiredAttributionKeys {
+		if vals.Get(val) == "" {
+			vals.Set(val, "(not set)")
+		}
+	}
+
 	return url.QueryEscape(vals.Encode()), nil
 }
 

attributioncode/validator_test.go

@@ -57,6 +57,14 @@ func TestValidateAttributionCode(t *testing.T) {
 			"c291cmNlPXd3dy5nb29nbGUuY29tJm1lZGl1bT1vcmdhbmljJmNhbXBhaWduPShub3Qgc2V0KSZjb250ZW50PShub3Qgc2V0KQ..", // source=www.google.com&medium=organic&campaign=(not set)&content=(not set)
 			"campaign%3D%2528not%2Bset%2529%26content%3D%2528not%2Bset%2529%26medium%3Dorganic%26source%3Dwww.google.com",
 		},
+		{
+			"c291cmNlPXd3dy5nb29nbGUuY29tJm1lZGl1bT1vcmdhbmljJmNhbXBhaWduPShub3Qgc2V0KQ..", // source=www.google.com&medium=organic&campaign=(not set)
+			"campaign%3D%2528not%2Bset%2529%26content%3D%2528not%2Bset%2529%26medium%3Dorganic%26source%3Dwww.google.com",
+		},
+		{
+			"c291cmNlPXd3dy5nb29nbGUuY29tJm1lZGl1bT1vcmdhbmljJmNhbXBhaWduPShub3Qgc2V0KSZjb250ZW50PShub3Qgc2V0KSZ2YXJpYXRpb249ZjEmZXhwZXJpbWVudD1lMQ..", // source=www.google.com&medium=organic&campaign=(not set)&content=(not set)&variation=f1&experiment=e1
+			"campaign%3D%2528not%2Bset%2529%26content%3D%2528not%2Bset%2529%26experiment%3De1%26medium%3Dorganic%26source%3Dwww.google.com%26variation%3Df1",
+		},
 	}
 	for _, c := range validCodes {
 		res, err := v.Validate(c.In, "")
@@ -77,17 +85,17 @@ func TestValidateAttributionCode(t *testing.T) {
 			"base64 code longer than 5000 characters",
 		},
 		{
-			"c291cmNlPWdvb2dsZS5jb21tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbSZtZWRpdW09b3JnYW5pYyZjYW1wYWlnbj0obm90IHNldCkmY29udGVudD0obm90IHNldCk.", // source=google.commmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmm&medium=organic&campaign=(not set)&content=(not set)
-			"code longer than 200 characters",
-		},
-		{
-			"bWVkaXVtPW9yZ2FuaWMmY2FtcGFpZ249KG5vdCBzZXQpJmNvbnRlbnQ9KG5vdCBzZXQp", // "medium=organic&campaign=(not set)&content=(not set)",
-			"code is missing keys",
+			"c291cmNlPWdvb2dsZS5jb21tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tbW1tJm1lZGl1bT1vcmdhbmljJmNhbXBhaWduPShub3Qgc2V0KSZjb250ZW50PShub3Qgc2V0KQ..", // source=google.commmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmm&medium=organic&campaign=(not set)&content=(not set)
+			"code longer than 1010 characters",
 		},
 		{
 			"bm90YXJlYWxrZXk9b3JnYW5pYyZjYW1wYWlnbj0obm90IHNldCkmY29udGVudD0obm90IHNldCk.", // "notarealkey=organic&campaign=(not set)&content=(not set)",
 			"notarealkey is not a valid attribution key",
 		},
+		{
+			"", // blank
+			"code is empty",
+		},
 	}
 	for _, c := range invalidCodes {
 		_, err := v.Validate(c.In, "")

stubservice/main.go

@@ -148,6 +148,8 @@ func pingdomHandler(w http.ResponseWriter, req *http.Request) {
 	attrQuery.Set("medium", "pingdom")
 	attrQuery.Set("campaign", "pingdom")
 	attrQuery.Set("content", "pingdom")
+	attrQuery.Set("experiment", "pingdom")
+	attrQuery.Set("variation", "pingdom")
 	b64AttrQuery := base64.URLEncoding.WithPadding('.').EncodeToString([]byte(attrQuery.Encode()))
 
 	query := url.Values{}

testing/stubtestclient/main.go

@@ -15,10 +15,12 @@ import (
 var (
 	baseURL string
 
-	campaign string
-	content  string
-	medium   string
-	source   string
+	campaign   string
+	content    string
+	medium     string
+	source     string
+	experiment string
+	variation  string
 
 	lang    string
 	os      string
@@ -50,6 +52,8 @@ func init() {
 	flag.StringVar(&content, "content", "testcontent", "content")
 	flag.StringVar(&medium, "medium", "testmedium", "medium")
 	flag.StringVar(&source, "source", "mozilla.com", "source")
+	flag.StringVar(&experiment, "experiment", "exp1", "experiment")
+	flag.StringVar(&variation, "variation", "var1", "variation")
 
 	flag.StringVar(&lang, "lang", "en-US", "")
 	flag.StringVar(&os, "os", "win", "")
@@ -66,6 +70,8 @@ func genCode() string {
 	query.Set("content", content)
 	query.Set("medium", medium)
 	query.Set("source", source)
+	query.Set("experiment", experiment)
+	query.Set("variation", variation)
 	query.Set("timestamp", fmt.Sprintf("%d", time.Now().UTC().Unix()))
 
 	b64Query := base64.URLEncoding.WithPadding('.').EncodeToString([]byte(query.Encode()))