(self host 1/?): allow users to configure their own email service (#972)

* feat: allow different email providers to be used when self-hosting, or allow users to disable email

* fix: handle errors and fix types

* chore: capitalize Gmail properly

Co-authored-by: Kalil Smith-Nuevelle <[email protected]>

---------

Co-authored-by: Kalil Smith-Nuevelle <[email protected]>

Author: tefkah

core/actions/email/run.ts

@@ -16,6 +16,7 @@ import { getPubsWithRelatedValuesAndChildren } from "~/lib/server";
 import { getCommunitySlug } from "~/lib/server/cache/getCommunitySlug";
 import * as Email from "~/lib/server/email";
 import { renderMarkdownWithPub } from "~/lib/server/render/pub/renderMarkdownWithPub";
+import { isClientException } from "~/lib/serverActions";
 import { defineRun } from "../types";
 
 export const run = defineRun<typeof action>(async ({ pub, config, args, communityId }) => {
@@ -88,26 +89,39 @@ export const run = defineRun<typeof action>(async ({ pub, config, args, communit
 			true
 		);
 
-		await Email.generic({
+		const result = await Email.generic({
 			to: expect(recipient?.user.email ?? recipientEmail),
 			subject,
 			html,
 		}).send();
+
+		if (isClientException(result)) {
+			logger.error({
+				msg: "An error occurred while sending an email",
+				error: result.error,
+				pub,
+				config,
+				args,
+				renderMarkdownWithPubContext,
+			});
+		} else {
+			logger.info({
+				msg: "Successfully sent email",
+				pub,
+				config,
+				args,
+				renderMarkdownWithPubContext,
+			});
+		}
+
+		return result;
 	} catch (error) {
-		logger.error({ msg: "email", error });
+		logger.error({ msg: "Failed to send email", error });
 
 		return {
 			title: "Failed to Send Email",
 			error: error.message,
 			cause: error,
 		};
 	}
-
-	logger.info({ msg: "email", pub, config, args });
-
-	return {
-		success: true,
-		report: "Email sent",
-		data: {},
-	};
 });

core/actions/googleDriveImport/getGDriveFiles.ts

@@ -1,3 +1,5 @@
+import type { Auth } from "googleapis";
+
 import { google } from "googleapis";
 
 import { logger } from "logger";
@@ -8,7 +10,21 @@ import { env } from "~/lib/env/env.mjs";
 // const keyFilePath = path.join(process.cwd(), 'src/utils/google/keyFile.json');
 // const keyFile = JSON.parse(fs.readFileSync(keyFilePath, 'utf8'));
 // const keyFile = JSON.parse(env.GCLOUD_KEY_FILE);
-const keyFile = JSON.parse(Buffer.from(env.GCLOUD_KEY_FILE, "base64").toString());
+
+let keyFile: Auth.JWTInput;
+
+try {
+	if (!env.GCLOUD_KEY_FILE) {
+		throw new Error(
+			"GCLOUD_KEY_FILE is not set. You must set this to use the Google Drive import."
+		);
+	}
+
+	keyFile = JSON.parse(Buffer.from(env.GCLOUD_KEY_FILE, "base64").toString());
+} catch (e) {
+	logger.error("Error parsing Google Cloud key file");
+	throw e;
+}
 
 // Configure a JWT auth client
 const auth = new google.auth.JWT(keyFile.client_email, undefined, keyFile.private_key, [

core/lib/env/env.mjs

@@ -3,11 +3,23 @@
 import { createEnv } from "@t3-oss/env-nextjs";
 import { z } from "zod";
 
+/**
+ * Parameters which are optional if the app is self-hosted
+ * but we do want checked for our AWS deploys
+ *
+ * @template {import("zod").ZodTypeAny} Z
+ * @param {Z} schema
+ */
+const selfHostedOptional = (schema) => {
+	return process.env.SELF_HOSTED ? schema.optional() : schema;
+};
+
 export const env = createEnv({
 	shared: {
 		NODE_ENV: z.enum(["development", "production", "test"]).optional(),
 	},
 	server: {
+		SELF_HOSTED: z.string().optional(),
 		API_KEY: z.string(),
 		ASSETS_BUCKET_NAME: z.string(),
 		ASSETS_REGION: z.string(),
@@ -21,16 +33,20 @@ export const env = createEnv({
 		KYSELY_DEBUG: z.string().optional(),
 		KYSELY_ARTIFICIAL_LATENCY: z.coerce.number().optional(),
 		LOG_LEVEL: z.enum(["debug", "info", "warn", "error"]).optional(),
-		MAILGUN_SMTP_PASSWORD: z.string(),
-		MAILGUN_SMTP_USERNAME: z.string(),
-		MAILGUN_SMTP_HOST: z.string(),
-		MAILGUN_SMTP_PORT: z.string(),
+		MAILGUN_SMTP_PASSWORD: selfHostedOptional(z.string()),
+		MAILGUN_SMTP_USERNAME: selfHostedOptional(z.string()),
+		MAILGUN_SMTP_HOST: selfHostedOptional(z.string()),
+		MAILGUN_SMTP_PORT: selfHostedOptional(z.string()),
+		MAILGUN_SMTP_FROM: z.string().optional(),
+		MAILGUN_SMTP_FROM_NAME: z.string().optional(),
+		MAILGUN_INSECURE_SENDMAIL: z.string().optional(),
+		MAILGUN_SMTP_SECURITY: z.enum(["ssl", "tls", "none"]).optional(),
 		OTEL_SERVICE_NAME: z.string().optional(),
 		HONEYCOMB_API_KEY: z.string().optional(),
 		PUBPUB_URL: z.string().url(),
 		INBUCKET_URL: z.string().url().optional(),
 		CI: z.string().or(z.boolean()).optional(),
-		GCLOUD_KEY_FILE: z.string(),
+		GCLOUD_KEY_FILE: selfHostedOptional(z.string()),
 		DATACITE_API_URL: z.string().optional(),
 		DATACITE_REPOSITORY_ID: z.string().optional(),
 		DATACITE_PASSWORD: z.string().optional(),

core/lib/server/email.tsx

@@ -11,17 +11,18 @@ import type { XOR } from "../types";
 import type { FormInviteLinkProps } from "./form";
 import { db } from "~/kysely/database";
 import { createMagicLink } from "~/lib/authentication/createMagicLink";
+import { env } from "../env/env.mjs";
 import { createFormInviteLink } from "./form";
-import { smtpclient } from "./mailgun";
+import { getSmtpClient } from "./mailgun";
 
 const FIFTEEN_MINUTES = 1000 * 60 * 15;
 
 type RequiredOptions = Required<Pick<SendMailOptions, "to" | "subject">> &
 	XOR<{ html: string }, { text: string }>;
 
 export const DEFAULT_OPTIONS = {
-	from: `[email protected]`,
-	name: `PubPub Team`,
+	from: env.MAILGUN_SMTP_FROM ?? "[email protected]",
+	name: env.MAILGUN_SMTP_FROM_NAME ?? "PubPub Team",
 } as const;
 
 // export class Email {
@@ -33,7 +34,9 @@ function buildSend(emailPromise: () => Promise<RequiredOptions>) {
 			options?: Partial<Omit<SendMailOptions, "to" | "subject" | "html">> & {
 				name?: string;
 			}
-		) => Promise<{ success: true; report?: string } | { error: string }>,
+		) => Promise<
+			{ success: true; report?: string; data: Record<string, unknown> } | { error: string }
+		>,
 	};
 }
 
@@ -54,7 +57,7 @@ async function send(
 			},
 		});
 
-		const send = await smtpclient.sendMail({
+		await getSmtpClient().sendMail({
 			from: `${options?.name ?? DEFAULT_OPTIONS.name} <${options?.from ?? DEFAULT_OPTIONS.from}>`,
 			to: required.to,
 			subject: required.subject,
@@ -65,6 +68,8 @@ async function send(
 
 		return {
 			success: true,
+			report: "Email sent",
+			data: {},
 		};
 	} catch (error) {
 		logger.error({

core/lib/server/mailgun.ts

@@ -1,14 +1,82 @@
+import type SMTPPool from "nodemailer/lib/smtp-pool";
+
 import nodemailer from "nodemailer";
 
 import { env } from "~/lib/env/env.mjs";
 
-export const smtpclient = nodemailer.createTransport({
-	pool: true,
-	host: env.MAILGUN_SMTP_HOST,
-	port: parseInt(env.MAILGUN_SMTP_PORT),
-	secure: env.MAILGUN_SMTP_HOST !== "localhost" && !env.CI,
-	auth: {
-		user: env.MAILGUN_SMTP_USERNAME,
-		pass: env.MAILGUN_SMTP_PASSWORD,
+let smtpclient: nodemailer.Transporter;
+
+const TLS_CONFIG = {
+	secure: false,
+	opportunisticTLS: true,
+	tls: {
+		ciphers: "SSLv3",
+		rejectUnauthorized: false,
 	},
-});
+} as const satisfies Partial<SMTPPool.Options>;
+
+const SSL_CONFIG = {
+	secure: true,
+} as const satisfies Partial<SMTPPool.Options>;
+
+const NO_CONFIG = {
+	secure: false,
+	opportunisticTLS: true,
+} as const satisfies Partial<SMTPPool.Options>;
+
+const guessSecurityType = () => {
+	if (env.MAILGUN_SMTP_PORT === "465") {
+		return "ssl";
+	}
+
+	if (env.MAILGUN_SMTP_PORT === "587") {
+		return "tls";
+	}
+
+	return "none";
+};
+
+const getSecurityConfig = () => {
+	const securityType = env.MAILGUN_SMTP_SECURITY ?? guessSecurityType();
+
+	if (securityType === "ssl") {
+		return SSL_CONFIG;
+	}
+
+	if (securityType === "tls") {
+		return TLS_CONFIG;
+	}
+
+	return NO_CONFIG;
+};
+
+export const getSmtpClient = () => {
+	const securityConfig = getSecurityConfig();
+
+	if (
+		!env.MAILGUN_SMTP_HOST ||
+		!env.MAILGUN_SMTP_PORT ||
+		!env.MAILGUN_SMTP_USERNAME ||
+		!env.MAILGUN_SMTP_PASSWORD
+	) {
+		throw new Error(
+			"Missing required SMTP configuration. Please set MAILGUN_SMTP_HOST, MAILGUN_SMTP_PORT, MAILGUN_SMTP_USERNAME, and MAILGUN_SMTP_PASSWORD in order to send emails."
+		);
+	}
+
+	if (!smtpclient) {
+		smtpclient = nodemailer.createTransport({
+			...securityConfig,
+			pool: true,
+			host: env.MAILGUN_SMTP_HOST,
+			port: parseInt(env.MAILGUN_SMTP_PORT),
+			secure: securityConfig.secure && env.MAILGUN_SMTP_HOST !== "localhost" && !env.CI,
+			auth: {
+				user: env.MAILGUN_SMTP_USERNAME,
+				pass: env.MAILGUN_SMTP_PASSWORD,
+			},
+		});
+	}
+
+	return smtpclient;
+};

self-host/README.md

@@ -0,0 +1,76 @@
+# Self-host documentation
+
+For the most part, self-hosting PubPub is a matter of deploying the app and the database, which you can do with the accompanying docker-compose file in `self-host/docker-compose.yml`.
+
+However, there are a few key things you need to know about.
+
+## Email
+
+To be able to send emails, you need to set some kind of email provider.
+
+We recommend using [Mailgun](https://www.mailgun.com/).
+
+Other common options are [SendGrid](https://sendgrid.com/) and [Postmark](https://postmarkapp.com/).
+
+You can also use an existing GMail or Office365 account to relay emails through PubPub.
+
+### Setup
+
+#### Mailgun
+
+To use Mailgun, you will need to create an account on [Mailgun](https://www.mailgun.com/) and set the following environment variables:
+
+```sh
+MAILGUN_SMTP_HOST="smtp.mailgun.org"
+MAILGUN_SMTP_PORT=587
+MAILGUN_SMTP_USERNAME="[email protected]"
+MAILGUN_SMTP_PASSWORD="your-mailgun-password"
+MAILGUN_SMTP_FROM="[email protected]"
+MAILGUN_SMTP_FROM_NAME="Your Organization"
+```
+
+#### Gmail
+
+To use Gmail to relay emails through PubPub, you will need to create an [app password](https://support.google.com/accounts/answer/185833?hl=en).
+
+You will be limited to 2000 emails per day by default this way.
+
+```sh
+MAILGUN_SMTP_HOST="smtp.gmail.com"
+MAILGUN_SMTP_PORT=587 # or 465 for SSL
+MAILGUN_SMTP_USERNAME="[email protected]"
+MAILGUN_SMTP_PASSWORD="your app password" # this will be a 16 character string
+MAILGUN_SMTP_FROM="[email protected]" # technically optional, but you will almost definitely need to set this.
+MAILGUN_SMTP_FROM_NAME="Your Organization" # Optional, will default to "PubPub Team"
+```
+
+If you need a higher limit of 10,000 emails, you can use the SMTP relay service. This will require extra configuration however:
+https://support.google.com/a/answer/176600?hl=en
+
+```sh
+MAILGUN_SMTP_HOST="smtp-relay.gmail.com"
+MAILGUN_SMTP_PORT=587 # or 465 for SSL
+MAILGUN_SMTP_USERNAME="[email protected]"
+MAILGUN_SMTP_PASSWORD="your app password" # this will be a 16 character string
+MAILGUN_SMTP_FROM="[email protected]" # technically optional, but you will almost definitely need to set this.
+MAILGUN_SMTP_FROM_NAME="Your Organization" # Optional, will default to "PubPub Team"
+```
+
+#### Office 365
+
+You can (for now) send emails through Office 365 Outlook/Exchange through SMTP, although Microsoft has repeatedly stated they will likely deprecate this feature in the future.
+
+You cannot send emails through shared mailboxes, you will need to an existing Microsoft account with a valid Office 365 subscription.
+
+```sh
+MAILGUN_SMTP_HOST="smtp.office365.com"
+MAILGUN_SMTP_PORT=587
+MAILGUN_SMTP_USERNAME="[email protected]"
+MAILGUN_SMTP_PASSWORD="your-password"
+MAILGUN_SMTP_FROM="[email protected]" # technically optional, but you will almost definitely need to set this, as it will use `[email protected]` by default.
+MAILGUN_SMTP_FROM_NAME="Your Organization" # Optional, will default to "PubPub Team"
+```
+
+#### No email
+
+You can technically leave the email provider blank, but this will disable the email functionality. The email action will still be visible in the UI, but it will fail when you try to send an email.