add optimization of ACM certificate creation. Closes #452

It will now attempt to utilize any existing cert that can satisfy the domain,
and will otherwise create one with a wildcard so that future domains will
not require an additional cert

Author: tj

internal/util/util.go

@@ -351,9 +351,7 @@ func Md5(s string) string {
 	return hex.EncodeToString(h.Sum(nil))
 }
 
-// Domain returns the effective domain. For example
-// the string "api.example.com" becomes "example.com",
-// while "api.example.co.uk" becomes "example.co.uk".
+// Domain returns the effective domain (TLD plus one).
 func Domain(s string) string {
 	d, err := publicsuffix.EffectiveTLDPlusOne(s)
 	if err != nil {
@@ -363,6 +361,41 @@ func Domain(s string) string {
 	return d
 }
 
+// CertDomainNames returns the certificate domain name
+// and alternative names for a requested domain.
+func CertDomainNames(s string) []string {
+	// effective domain
+	if Domain(s) == s {
+		return []string{s, "*." + s}
+	}
+
+	// subdomain
+	return []string{RemoveSubdomains(s, 1), "*." + RemoveSubdomains(s, 1)}
+}
+
+// IsWildcardDomain returns true if the domain is a wildcard.
+func IsWildcardDomain(s string) bool {
+	return strings.HasPrefix(s, "*.")
+}
+
+// WildcardMatches returns true if wildcard is a wildcard domain
+// and it satisfies the given domain.
+func WildcardMatches(wildcard, domain string) bool {
+	if !IsWildcardDomain(wildcard) {
+		return false
+	}
+
+	w := RemoveSubdomains(wildcard, 1)
+	d := RemoveSubdomains(domain, 1)
+	return w == d
+}
+
+// RemoveSubdomains returns the domain without the n left-most subdomain(s).
+func RemoveSubdomains(s string, n int) string {
+	domains := strings.Split(s, ".")
+	return strings.Join(domains[n:], ".")
+}
+
 // ParseSections returns INI style sections from r.
 func ParseSections(r io.Reader) (sections []string, err error) {
 	s := bufio.NewScanner(r)
@@ -378,3 +411,16 @@ func ParseSections(r io.Reader) (sections []string, err error) {
 
 	return
 }
+
+// UniqueStrings returns a string slice of unique values.
+func UniqueStrings(s []string) (v []string) {
+	m := make(map[string]struct{})
+	for _, val := range s {
+		_, ok := m[val]
+		if !ok {
+			v = append(v, val)
+			m[val] = struct{}{}
+		}
+	}
+	return
+}

internal/util/util_test.go

@@ -85,6 +85,19 @@ func TestDomain(t *testing.T) {
 	assert.Equal(t, "example.co.uk", Domain("v1.api.example.co.uk"))
 }
 
+func TestCertDomainNames(t *testing.T) {
+	assert.Equal(t, []string{"example.com", "*.example.com"}, CertDomainNames("example.com"))
+	assert.Equal(t, []string{"example.com", "*.example.com"}, CertDomainNames("api.example.com"))
+	assert.Equal(t, []string{"api.example.com", "*.api.example.com"}, CertDomainNames("v1.api.example.com"))
+}
+
+func TestWildcardMatches(t *testing.T) {
+	assert.True(t, WildcardMatches("*.api.example.com", "v1.api.example.com"))
+	assert.True(t, WildcardMatches("*.example.com", "api.example.com"))
+	assert.False(t, WildcardMatches("example.com", "api.example.com"))
+	assert.False(t, WildcardMatches("*.api.example.com", "api.example.com"))
+}
+
 func TestParseSections(t *testing.T) {
 	r := strings.NewReader(`[personal]
 aws_access_key_id = personal_key

platform/lambda/lambda.go

@@ -8,6 +8,7 @@ import (
 	"io/ioutil"
 	"os"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/apex/log"
@@ -318,35 +319,34 @@ func (p *Platform) getHostedZone() (zones []*route53.HostedZone, err error) {
 func (p *Platform) createCerts() error {
 	s := session.New(aws.NewConfig().WithRegion("us-east-1"))
 	a := acm.New(s)
+	var domains []string
 
 	// existing certs
-	res, err := a.ListCertificates(&acm.ListCertificatesInput{
-		MaxItems: aws.Int64(1000),
-	})
-
+	log.Debug("fetching existing certs")
+	certs, err := getCerts(a)
 	if err != nil {
-		return errors.Wrap(err, "listing")
+		return errors.Wrap(err, "fetching certs")
 	}
 
-	var domains []string
-
 	// request certs
 	for _, s := range p.config.Stages.List() {
 		if s == nil {
 			continue
 		}
 
+		certDomains := util.CertDomainNames(s.Domain)
+
 		// see if the cert exists
 		log.Debugf("looking up cert for %s", s.Domain)
-		arn := getCert(res.CertificateSummaryList, s.Domain)
+		arn := getCert(certs, s.Domain)
 		if arn != "" {
 			log.Debugf("found cert for %s: %s", s.Domain, arn)
 			s.Cert = arn
 			continue
 		}
 
 		option := acm.DomainValidationOption{
-			DomainName:       &s.Domain,
+			DomainName:       aws.String(certDomains[0]),
 			ValidationDomain: aws.String(util.Domain(s.Domain)),
 		}
 
@@ -356,15 +356,16 @@ func (p *Platform) createCerts() error {
 
 		// request the cert
 		res, err := a.RequestCertificate(&acm.RequestCertificateInput{
-			DomainName:              &s.Domain,
+			DomainName:              aws.String(certDomains[0]),
 			DomainValidationOptions: options,
+			SubjectAlternativeNames: aws.StringSlice(certDomains[1:]),
 		})
 
 		if err != nil {
-			return errors.Wrapf(err, "requesting cert for %s", s.Domain)
+			return errors.Wrapf(err, "requesting cert for %v", certDomains)
 		}
 
-		domains = append(domains, s.Domain)
+		domains = append(domains, certDomains[0])
 		s.Cert = *res.CertificateArn
 	}
 
@@ -379,7 +380,7 @@ func (p *Platform) createCerts() error {
 
 	// wait for approval
 	for range time.Tick(4 * time.Second) {
-		res, err = a.ListCertificates(&acm.ListCertificatesInput{
+		res, err := a.ListCertificates(&acm.ListCertificatesInput{
 			MaxItems:            aws.Int64(1000),
 			CertificateStatuses: aws.StringSlice([]string{acm.CertificateStatusPendingValidation}),
 		})
@@ -787,12 +788,72 @@ func toEnv(env config.Environment, stage string) *lambda.Environment {
 	}
 }
 
-// getCert returns the ARN if the cert is present.
-func getCert(certs []*acm.CertificateSummary, domain string) string {
+// getCerts returns the certificates available.
+func getCerts(a *acm.ACM) (certs []*acm.CertificateDetail, err error) {
+	var g errgroup.Group
+	var mu sync.Mutex
+
+	res, err := a.ListCertificates(&acm.ListCertificatesInput{
+		MaxItems: aws.Int64(1000),
+	})
+
+	if err != nil {
+		return nil, errors.Wrap(err, "listing")
+	}
+
+	for _, c := range res.CertificateSummaryList {
+		c := c
+		g.Go(func() error {
+			res, err := a.DescribeCertificate(&acm.DescribeCertificateInput{
+				CertificateArn: c.CertificateArn,
+			})
+
+			if err != nil {
+				return errors.Wrap(err, "describing")
+			}
+
+			mu.Lock()
+			certs = append(certs, res.Certificate)
+			mu.Unlock()
+			return nil
+		})
+	}
+
+	err = g.Wait()
+	return
+}
+
+// getCert returns the ARN of a certificate with can satisfy domain,
+// favoring more specific certificates, then falling back on wildcards.
+func getCert(certs []*acm.CertificateDetail, domain string) string {
+	// exact domain
 	for _, c := range certs {
 		if *c.DomainName == domain {
 			return *c.CertificateArn
 		}
 	}
+
+	// exact alt
+	for _, c := range certs {
+		for _, a := range c.SubjectAlternativeNames {
+			if *a == domain {
+				return *c.CertificateArn
+			}
+		}
+	}
+
+	// wildcards
+	for _, c := range certs {
+		if util.WildcardMatches(*c.DomainName, domain) {
+			return *c.CertificateArn
+		}
+
+		for _, a := range c.SubjectAlternativeNames {
+			if util.WildcardMatches(*a, domain) {
+				return *c.CertificateArn
+			}
+		}
+	}
+
 	return ""
 }

platform/lambda/lambda_test.go

@@ -0,0 +1,57 @@
+package lambda
+
+import (
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/acm"
+	"github.com/tj/assert"
+)
+
+func TestGetCert(t *testing.T) {
+	certs := []*acm.CertificateDetail{
+		{
+			DomainName:     aws.String("example.com"),
+			CertificateArn: aws.String("arn:example.com"),
+			SubjectAlternativeNames: aws.StringSlice([]string{
+				"*.example.com",
+			}),
+		},
+		{
+			DomainName:     aws.String("*.apex.sh"),
+			CertificateArn: aws.String("arn:*.apex.sh"),
+		},
+		{
+			DomainName:     aws.String("api.example.com"),
+			CertificateArn: aws.String("arn:api.example.com"),
+			SubjectAlternativeNames: aws.StringSlice([]string{
+				"*.api.example.com",
+				"something.example.com",
+			}),
+		},
+	}
+
+	arn := getCert(certs, "example.com")
+	assert.Equal(t, "arn:example.com", arn)
+
+	arn = getCert(certs, "www.example.com")
+	assert.Equal(t, "arn:example.com", arn)
+
+	arn = getCert(certs, "api.example.com")
+	assert.Equal(t, "arn:api.example.com", arn)
+
+	arn = getCert(certs, "apex.sh")
+	assert.Empty(t, arn)
+
+	arn = getCert(certs, "api.apex.sh")
+	assert.Equal(t, "arn:*.apex.sh", arn)
+
+	arn = getCert(certs, "v1.api.example.com")
+	assert.Equal(t, "arn:api.example.com", arn)
+
+	arn = getCert(certs, "something.example.com")
+	assert.Equal(t, "arn:api.example.com", arn)
+
+	arn = getCert(certs, "staging.v1.api.example.com")
+	assert.Empty(t, arn)
+}

reporter/text/text.go

@@ -203,11 +203,12 @@ func (r *reporter) Start() {
 				}
 				fmt.Printf("\n")
 			case "platform.certs.create":
-				domains := e.Fields["domains"].([]string)
+				domains := util.UniqueStrings(e.Fields["domains"].([]string))
 				r.log("domains", "Check your email to approve the certificate")
 				r.pending("confirm", strings.Join(domains, ", "))
 			case "platform.certs.create.complete":
 				r.complete("confirm", "complete", e.Duration("duration"))
+				fmt.Printf("\n")
 			case "metrics", "metrics.complete":
 				fmt.Printf("\n")
 			case "metrics.value":