feat: add email validation function to lower bounce rates (#1845)

The goal is to only return an error when we have a very high confidence
the email won't be deliverable.

This is currently going to be added as a draft for the team to review. I
haven't actually implemented any paths that call this or configuration
around when it is activated.

---------

Co-authored-by: Chris Stockton <[email protected]>

Author: cstockton

internal/api/errorcodes.go

@@ -91,4 +91,5 @@ const (
 	//#nosec G101 -- Not a secret value.
 	ErrorCodeInvalidCredentials        ErrorCode = "invalid_credentials"
 	ErrorCodeEmailAddressNotAuthorized ErrorCode = "email_address_not_authorized"
+	ErrorCodeEmailAddressInvalid       ErrorCode = "email_address_invalid"
 )

internal/api/mail.go

@@ -601,7 +601,6 @@ func (a *API) checkEmailAddressAuthorization(email string) bool {
 }
 
 func (a *API) sendEmail(r *http.Request, tx *storage.Connection, u *models.User, emailActionType, otp, otpNew, tokenHashWithPrefix string) error {
-	mailer := a.Mailer()
 	ctx := r.Context()
 	config := a.config
 	referrerURL := utilities.GetReferrer(r, config)
@@ -675,20 +674,34 @@ func (a *API) sendEmail(r *http.Request, tx *storage.Connection, u *models.User,
 		return a.invokeHook(tx, r, &input, &output)
 	}
 
+	mr := a.Mailer()
+	var err error
 	switch emailActionType {
 	case mail.SignupVerification:
-		return mailer.ConfirmationMail(r, u, otp, referrerURL, externalURL)
+		err = mr.ConfirmationMail(r, u, otp, referrerURL, externalURL)
 	case mail.MagicLinkVerification:
-		return mailer.MagicLinkMail(r, u, otp, referrerURL, externalURL)
+		err = mr.MagicLinkMail(r, u, otp, referrerURL, externalURL)
 	case mail.ReauthenticationVerification:
-		return mailer.ReauthenticateMail(r, u, otp)
+		err = mr.ReauthenticateMail(r, u, otp)
 	case mail.RecoveryVerification:
-		return mailer.RecoveryMail(r, u, otp, referrerURL, externalURL)
+		err = mr.RecoveryMail(r, u, otp, referrerURL, externalURL)
 	case mail.InviteVerification:
-		return mailer.InviteMail(r, u, otp, referrerURL, externalURL)
+		err = mr.InviteMail(r, u, otp, referrerURL, externalURL)
 	case mail.EmailChangeVerification:
-		return mailer.EmailChangeMail(r, u, otpNew, otp, referrerURL, externalURL)
+		err = mr.EmailChangeMail(r, u, otpNew, otp, referrerURL, externalURL)
+	default:
+		err = errors.New("invalid email action type")
+	}
+
+	switch {
+	case errors.Is(err, mail.ErrInvalidEmailAddress),
+		errors.Is(err, mail.ErrInvalidEmailFormat),
+		errors.Is(err, mail.ErrInvalidEmailDNS):
+		return badRequestError(
+			ErrorCodeEmailAddressInvalid,
+			"Email address %q is invalid",
+			u.GetEmail())
 	default:
-		return errors.New("invalid email action type")
+		return err
 	}
 }

internal/conf/configuration.go

@@ -399,6 +399,33 @@ type MailerConfiguration struct {
 	OtpLength int  `json:"otp_length" split_words:"true"`
 
 	ExternalHosts []string `json:"external_hosts" split_words:"true"`
+
+	// EXPERIMENTAL: May be removed in a future release.
+	EmailValidationExtended       bool   `json:"email_validation_extended" split_words:"true" default:"false"`
+	EmailValidationServiceURL     string `json:"email_validation_service_url" split_words:"true"`
+	EmailValidationServiceHeaders string `json:"email_validation_service_key" split_words:"true"`
+
+	serviceHeaders map[string][]string `json:"-"`
+}
+
+func (c *MailerConfiguration) Validate() error {
+	headers := make(map[string][]string)
+
+	if c.EmailValidationServiceHeaders != "" {
+		err := json.Unmarshal([]byte(c.EmailValidationServiceHeaders), &headers)
+		if err != nil {
+			return fmt.Errorf("conf: SMTP headers not a map[string][]string format: %w", err)
+		}
+	}
+
+	if len(headers) > 0 {
+		c.serviceHeaders = headers
+	}
+	return nil
+}
+
+func (c *MailerConfiguration) GetEmailValidationServiceHeaders() map[string][]string {
+	return c.serviceHeaders
 }
 
 type PhoneProviderConfiguration struct {
@@ -1020,6 +1047,7 @@ func (c *GlobalConfiguration) Validate() error {
 		&c.Tracing,
 		&c.Metrics,
 		&c.SMTP,
+		&c.Mailer,
 		&c.SAML,
 		&c.Security,
 		&c.Sessions,

internal/conf/configuration_test.go

@@ -25,6 +25,7 @@ func TestGlobal(t *testing.T) {
 	os.Setenv("GOTRUE_HOOK_MFA_VERIFICATION_ATTEMPT_URI", "pg-functions://postgres/auth/count_failed_attempts")
 	os.Setenv("GOTRUE_HOOK_SEND_SMS_SECRETS", "v1,whsec_aWxpa2VzdXBhYmFzZXZlcnltdWNoYW5kaWhvcGV5b3Vkb3Rvbw==")
 	os.Setenv("GOTRUE_SMTP_HEADERS", `{"X-PM-Metadata-project-ref":["project_ref"],"X-SES-Message-Tags":["ses:feedback-id-a=project_ref,ses:feedback-id-b=$messageType"]}`)
+	os.Setenv("GOTRUE_MAILER_EMAIL_VALIDATION_SERVICE_HEADERS", `{"apikey":["test"]}`)
 	os.Setenv("GOTRUE_SMTP_LOGGING_ENABLED", "true")
 	gc, err := LoadGlobal("")
 	require.NoError(t, err)
@@ -34,6 +35,12 @@ func TestGlobal(t *testing.T) {
 	assert.Equal(t, "X-Request-ID", gc.API.RequestIDHeader)
 	assert.Equal(t, "pg-functions://postgres/auth/count_failed_attempts", gc.Hook.MFAVerificationAttempt.URI)
 
+	{
+		hdrs := gc.Mailer.GetEmailValidationServiceHeaders()
+		assert.Equal(t, 1, len(hdrs["apikey"]))
+		assert.Equal(t, "test", hdrs["apikey"][0])
+	}
+
 }
 
 func TestRateLimits(t *testing.T) {

internal/mailer/mailer.go

@@ -18,7 +18,6 @@ type Mailer interface {
 	MagicLinkMail(r *http.Request, user *models.User, otp, referrerURL string, externalURL *url.URL) error
 	EmailChangeMail(r *http.Request, user *models.User, otpNew, otpCurrent, referrerURL string, externalURL *url.URL) error
 	ReauthenticateMail(r *http.Request, user *models.User, otp string) error
-	ValidateEmail(email string) error
 	GetEmailActionLink(user *models.User, actionType, referrerURL string, externalURL *url.URL) (string, error)
 }
 
@@ -46,18 +45,21 @@ func NewMailer(globalConfig *conf.GlobalConfiguration) Mailer {
 	var mailClient MailClient
 	if globalConfig.SMTP.Host == "" {
 		logrus.Infof("Noop mail client being used for %v", globalConfig.SiteURL)
-		mailClient = &noopMailClient{}
+		mailClient = &noopMailClient{
+			EmailValidator: newEmailValidator(globalConfig.Mailer),
+		}
 	} else {
 		mailClient = &MailmeMailer{
-			Host:        globalConfig.SMTP.Host,
-			Port:        globalConfig.SMTP.Port,
-			User:        globalConfig.SMTP.User,
-			Pass:        globalConfig.SMTP.Pass,
-			LocalName:   u.Hostname(),
-			From:        from,
-			BaseURL:     globalConfig.SiteURL,
-			Logger:      logrus.StandardLogger(),
-			MailLogging: globalConfig.SMTP.LoggingEnabled,
+			Host:           globalConfig.SMTP.Host,
+			Port:           globalConfig.SMTP.Port,
+			User:           globalConfig.SMTP.User,
+			Pass:           globalConfig.SMTP.Pass,
+			LocalName:      u.Hostname(),
+			From:           from,
+			BaseURL:        globalConfig.SiteURL,
+			Logger:         logrus.StandardLogger(),
+			MailLogging:    globalConfig.SMTP.LoggingEnabled,
+			EmailValidator: newEmailValidator(globalConfig.Mailer),
 		}
 	}
 

internal/mailer/mailme.go

@@ -2,6 +2,7 @@ package mailer
 
 import (
 	"bytes"
+	"context"
 	"errors"
 	"html/template"
 	"io"
@@ -24,22 +25,29 @@ const TemplateExpiration = 10 * time.Second
 
 // MailmeMailer lets MailMe send templated mails
 type MailmeMailer struct {
-	From        string
-	Host        string
-	Port        int
-	User        string
-	Pass        string
-	BaseURL     string
-	LocalName   string
-	FuncMap     template.FuncMap
-	cache       *TemplateCache
-	Logger      logrus.FieldLogger
-	MailLogging bool
+	From           string
+	Host           string
+	Port           int
+	User           string
+	Pass           string
+	BaseURL        string
+	LocalName      string
+	FuncMap        template.FuncMap
+	cache          *TemplateCache
+	Logger         logrus.FieldLogger
+	MailLogging    bool
+	EmailValidator *EmailValidator
 }
 
 // Mail sends a templated mail. It will try to load the template from a URL, and
 // otherwise fall back to the default
-func (m *MailmeMailer) Mail(to, subjectTemplate, templateURL, defaultTemplate string, templateData map[string]interface{}, headers map[string][]string, typ string) error {
+func (m *MailmeMailer) Mail(
+	ctx context.Context,
+	to, subjectTemplate, templateURL, defaultTemplate string,
+	templateData map[string]interface{},
+	headers map[string][]string,
+	typ string,
+) error {
 	if m.FuncMap == nil {
 		m.FuncMap = map[string]interface{}{}
 	}
@@ -51,6 +59,12 @@ func (m *MailmeMailer) Mail(to, subjectTemplate, templateURL, defaultTemplate st
 		}
 	}
 
+	if m.EmailValidator != nil {
+		if err := m.EmailValidator.Validate(ctx, to); err != nil {
+			return err
+		}
+	}
+
 	tmp, err := template.New("Subject").Funcs(template.FuncMap(m.FuncMap)).Parse(subjectTemplate)
 	if err != nil {
 		return err

internal/mailer/noop.go

@@ -1,14 +1,28 @@
 package mailer
 
 import (
+	"context"
 	"errors"
 )
 
-type noopMailClient struct{}
+type noopMailClient struct {
+	EmailValidator *EmailValidator
+}
 
-func (m *noopMailClient) Mail(to, subjectTemplate, templateURL, defaultTemplate string, templateData map[string]interface{}, headers map[string][]string, typ string) error {
+func (m *noopMailClient) Mail(
+	ctx context.Context,
+	to, subjectTemplate, templateURL, defaultTemplate string,
+	templateData map[string]interface{},
+	headers map[string][]string,
+	typ string,
+) error {
 	if to == "" {
 		return errors.New("to field cannot be empty")
 	}
+	if m.EmailValidator != nil {
+		if err := m.EmailValidator.Validate(ctx, to); err != nil {
+			return err
+		}
+	}
 	return nil
 }

internal/mailer/template.go

@@ -1,18 +1,37 @@
 package mailer
 
 import (
+	"context"
 	"fmt"
 	"net/http"
 	"net/url"
 	"strings"
 
-	"github.com/badoux/checkmail"
 	"github.com/supabase/auth/internal/conf"
 	"github.com/supabase/auth/internal/models"
 )
 
+type MailRequest struct {
+	To              string
+	SubjectTemplate string
+	TemplateURL     string
+	DefaultTemplate string
+	TemplateData    map[string]interface{}
+	Headers         map[string][]string
+	Type            string
+}
+
 type MailClient interface {
-	Mail(string, string, string, string, map[string]interface{}, map[string][]string, string) error
+	Mail(
+		ctx context.Context,
+		to string,
+		subjectTemplate string,
+		templateURL string,
+		defaultTemplate string,
+		templateData map[string]interface{},
+		headers map[string][]string,
+		typ string,
+	) error
 }
 
 // TemplateMailer will send mail and use templates from the site for easy mail styling
@@ -81,12 +100,6 @@ const defaultReauthenticateMail = `<h2>Confirm reauthentication</h2>
 
 <p>Enter the code: {{ .Token }}</p>`
 
-// ValidateEmail returns nil if the email is valid,
-// otherwise an error indicating the reason it is invalid
-func (m TemplateMailer) ValidateEmail(email string) error {
-	return checkmail.ValidateFormat(email)
-}
-
 func (m *TemplateMailer) Headers(messageType string) map[string][]string {
 	originalHeaders := m.Config.SMTP.NormalizedHeaders()
 
@@ -145,6 +158,7 @@ func (m *TemplateMailer) InviteMail(r *http.Request, user *models.User, otp, ref
 	}
 
 	return m.Mailer.Mail(
+		r.Context(),
 		user.GetEmail(),
 		withDefault(m.Config.Mailer.Subjects.Invite, "You have been invited"),
 		m.Config.Mailer.Templates.Invite,
@@ -177,6 +191,7 @@ func (m *TemplateMailer) ConfirmationMail(r *http.Request, user *models.User, ot
 	}
 
 	return m.Mailer.Mail(
+		r.Context(),
 		user.GetEmail(),
 		withDefault(m.Config.Mailer.Subjects.Confirmation, "Confirm Your Email"),
 		m.Config.Mailer.Templates.Confirmation,
@@ -197,6 +212,7 @@ func (m *TemplateMailer) ReauthenticateMail(r *http.Request, user *models.User,
 	}
 
 	return m.Mailer.Mail(
+		r.Context(),
 		user.GetEmail(),
 		withDefault(m.Config.Mailer.Subjects.Reauthentication, "Confirm reauthentication"),
 		m.Config.Mailer.Templates.Reauthentication,
@@ -237,7 +253,10 @@ func (m *TemplateMailer) EmailChangeMail(r *http.Request, user *models.User, otp
 		})
 	}
 
-	errors := make(chan error)
+	ctx, cancel := context.WithCancel(r.Context())
+	defer cancel()
+
+	errors := make(chan error, len(emails))
 	for _, email := range emails {
 		path, err := getPath(
 			m.Config.Mailer.URLPaths.EmailChange,
@@ -263,6 +282,7 @@ func (m *TemplateMailer) EmailChangeMail(r *http.Request, user *models.User, otp
 				"RedirectTo":      referrerURL,
 			}
 			errors <- m.Mailer.Mail(
+				ctx,
 				address,
 				withDefault(m.Config.Mailer.Subjects.EmailChange, "Confirm Email Change"),
 				template,
@@ -280,7 +300,6 @@ func (m *TemplateMailer) EmailChangeMail(r *http.Request, user *models.User, otp
 			return e
 		}
 	}
-
 	return nil
 }
 
@@ -305,6 +324,7 @@ func (m *TemplateMailer) RecoveryMail(r *http.Request, user *models.User, otp, r
 	}
 
 	return m.Mailer.Mail(
+		r.Context(),
 		user.GetEmail(),
 		withDefault(m.Config.Mailer.Subjects.Recovery, "Reset Your Password"),
 		m.Config.Mailer.Templates.Recovery,
@@ -337,6 +357,7 @@ func (m *TemplateMailer) MagicLinkMail(r *http.Request, user *models.User, otp,
 	}
 
 	return m.Mailer.Mail(
+		r.Context(),
 		user.GetEmail(),
 		withDefault(m.Config.Mailer.Subjects.MagicLink, "Your Magic Link"),
 		m.Config.Mailer.Templates.MagicLink,
@@ -347,19 +368,6 @@ func (m *TemplateMailer) MagicLinkMail(r *http.Request, user *models.User, otp,
 	)
 }
 
-// Send can be used to send one-off emails to users
-func (m TemplateMailer) Send(user *models.User, subject, body string, data map[string]interface{}) error {
-	return m.Mailer.Mail(
-		user.GetEmail(),
-		subject,
-		"",
-		body,
-		data,
-		m.Headers("other"),
-		"other",
-	)
-}
-
 // GetEmailActionLink returns a magiclink, recovery or invite link based on the actionType passed.
 func (m TemplateMailer) GetEmailActionLink(user *models.User, actionType, referrerURL string, externalURL *url.URL) (string, error) {
 	var err error