feat: Add persistent volumes and encrypted secret management for deployments

.gitignore

@@ -30,3 +30,4 @@ deno.lock
 *.njsproj
 *.sln
 *.sw?
+testing/

src/backend/.env.sample

@@ -7,4 +7,7 @@ MONGO_APP_ID=...
 SENTRY_DSN=...
 FRONTEND=...
 ADMIN_LIST=admin1|admin2
-MEMORY_LIMIT=500m
+MEMORY_LIMIT=500m
+FULLCHAIN_LOCATION=...
+PRIVKEY_LOCATION=...
+SECRET_MASTER_KEY=32characterlongkey

src/backend/db.ts

@@ -1,5 +1,6 @@
 import getProviderUser from "./utils/get-user.ts";
 import DfContentMap from "./types/maps_interface.ts";
+import type { EncryptedData } from "./utils/encryption.ts";
 
 const DATA_API_KEY = Deno.env.get("MONGO_API_KEY")!;
 const APP_ID = Deno.env.get("MONGO_APP_ID");
@@ -26,6 +27,24 @@ const MONGO_URLs = {
   delete: new URL(`${BASE_URI}/action/deleteOne`),
 };
 
+interface StoredSecret {
+  subdomain: string;
+  encrypted_secrets: string;
+  iv: string;
+  tag: string;
+  created_at: string;
+  updated_at: string;
+  version: number;
+}
+
+interface VolumeMetadata {
+  subdomain: string;
+  volume_name: string;
+  mount_path: string;
+  created_at: string;
+  last_used_at: string;
+}
+
 // Function to update access token on db if user exists
 async function checkUser(accessToken: string, provider: string) {
   const userId = await getProviderUser(accessToken, provider);
@@ -116,4 +135,188 @@ async function deleteMaps(document: DfContentMap, ADMIN_LIST: string[]) {
   return data;
 }
 
-export { addMaps, checkUser, deleteMaps, getMaps };
+// Get encrypted secrets for a project
+async function getSecretsForProject(subdomain: string): Promise<StoredSecret | null> {
+  const query = {
+    collection: "project_secrets",
+    database: DATABASE,
+    dataSource: DATA_SOURCE,
+    filter: { "subdomain": subdomain },
+  };
+  options.body = JSON.stringify(query);
+
+  const resp = await fetch(MONGO_URLs.find.toString(), options);
+  const data = await resp.json();
+
+  if (data.documents && data.documents.length > 0) {
+    return data.documents[0] as StoredSecret;
+  }
+  return null;
+}
+
+async function upsertSecrets(
+  subdomain: string,
+  encryptedData: EncryptedData,
+): Promise<boolean> {
+  const now = new Date().toISOString();
+
+  const existing = await getSecretsForProject(subdomain);
+
+  const document: StoredSecret = {
+    subdomain: subdomain,
+    encrypted_secrets: encryptedData.encrypted,
+    iv: encryptedData.iv,
+    tag: encryptedData.tag,
+    created_at: existing?.created_at || now,
+    updated_at: now,
+    version: existing ? existing.version + 1 : 1,
+  };
+
+  if (existing) {
+    const query = {
+      collection: "project_secrets",
+      database: DATABASE,
+      dataSource: DATA_SOURCE,
+      filter: { "subdomain": subdomain },
+      update: {
+        $set: {
+          encrypted_secrets: document.encrypted_secrets,
+          iv: document.iv,
+          tag: document.tag,
+          updated_at: document.updated_at,
+          version: document.version,
+        },
+      },
+    };
+    options.body = JSON.stringify(query);
+
+    const resp = await fetch(MONGO_URLs.update.toString(), options);
+    const data = await resp.json();
+
+    return data.matchedCount === 1;
+  } else {
+    const query = {
+      collection: "project_secrets",
+      database: DATABASE,
+      dataSource: DATA_SOURCE,
+      document: document,
+    };
+    options.body = JSON.stringify(query);
+
+    const resp = await fetch(MONGO_URLs.insert.toString(), options);
+    const data = await resp.json();
+
+    return data.insertedId !== undefined;
+  }
+}
+
+async function deleteSecretsForProject(subdomain: string): Promise<boolean> {
+  const query = {
+    collection: "project_secrets",
+    database: DATABASE,
+    dataSource: DATA_SOURCE,
+    filter: { "subdomain": subdomain },
+  };
+  options.body = JSON.stringify(query);
+
+  const resp = await fetch(MONGO_URLs.delete.toString(), options);
+  const data = await resp.json();
+
+  return data.deletedCount > 0;
+}
+
+async function getVolumeMetadata(subdomain: string): Promise<VolumeMetadata | null> {
+  const query = {
+    collection: "volume_metadata",
+    database: DATABASE,
+    dataSource: DATA_SOURCE,
+    filter: { "subdomain": subdomain },
+  };
+  options.body = JSON.stringify(query);
+
+  const resp = await fetch(MONGO_URLs.find.toString(), options);
+  const data = await resp.json();
+
+  if (data.documents && data.documents.length > 0) {
+    return data.documents[0] as VolumeMetadata;
+  }
+  return null;
+}
+
+async function upsertVolumeMetadata(
+  subdomain: string,
+  volumeName: string,
+  mountPath: string,
+): Promise<boolean> {
+  const now = new Date().toISOString();
+  const existing = await getVolumeMetadata(subdomain);
+
+  const document: VolumeMetadata = {
+    subdomain: subdomain,
+    volume_name: volumeName,
+    mount_path: mountPath,
+    created_at: existing?.created_at || now,
+    last_used_at: now,
+  };
+
+  if (existing) {
+    const query = {
+      collection: "volume_metadata",
+      database: DATABASE,
+      dataSource: DATA_SOURCE,
+      filter: { "subdomain": subdomain },
+      update: {
+        $set: {
+          last_used_at: document.last_used_at,
+        },
+      },
+    };
+    options.body = JSON.stringify(query);
+
+    const resp = await fetch(MONGO_URLs.update.toString(), options);
+    const data = await resp.json();
+
+    return data.matchedCount === 1;
+  } else {
+    const query = {
+      collection: "volume_metadata",
+      database: DATABASE,
+      dataSource: DATA_SOURCE,
+      document: document,
+    };
+    options.body = JSON.stringify(query);
+
+    const resp = await fetch(MONGO_URLs.insert.toString(), options);
+    const data = await resp.json();
+
+    return data.insertedId !== undefined;
+  }
+}
+
+async function deleteVolumeMetadata(subdomain: string): Promise<boolean> {
+  const query = {
+    collection: "volume_metadata",
+    database: DATABASE,
+    dataSource: DATA_SOURCE,
+    filter: { "subdomain": subdomain },
+  };
+  options.body = JSON.stringify(query);
+
+  const resp = await fetch(MONGO_URLs.delete.toString(), options);
+  const data = await resp.json();
+
+  return data.deletedCount > 0;
+}
+
+export {
+  addMaps,
+  checkUser,
+  deleteMaps,
+  getMaps,
+  getSecretsForProject,
+  upsertSecrets,
+  deleteSecretsForProject,
+  getVolumeMetadata,
+  upsertVolumeMetadata,
+  deleteVolumeMetadata,
+};

src/backend/scripts.ts

@@ -1,14 +1,43 @@
 import { exec } from "./dependencies.ts";
 import dockerize from "./utils/container.ts";
 import DfContentMap from "./types/maps_interface.ts";
+import {
+  getSecretsForProject,
+  upsertVolumeMetadata,
+} from "./db.ts";
+import { getEncryptionService } from "./utils/encryption.ts";
 
 const MEMORY_LIMIT = Deno.env.get("MEMORY_LIMIT");
+const VOLUME_MOUNT_PATH = "/app/data";
+
+function mergeEnvVars(
+  envContent: string,
+  secrets: Record<string, string>,
+): string {
+  const lines: string[] = [];
+
+  if (envContent && envContent.trim()) {
+    const envLines = envContent.split("\n").filter((line) => line.trim());
+    lines.push(...envLines);
+  }
+
+  for (const [key, value] of Object.entries(secrets)) {
+    const escapedValue = value.replace(/(["$`\\])/g, "\\$1");
+    lines.push(`${key}=${escapedValue}`);
+  }
+
+  return lines.join("\n");
+}
+
+function sanitizeSubdomain(subdomain: string): string {
+  return subdomain.replace(/[.:\/]/g, "-");
+}
 
 async function addScript(
   document: DfContentMap,
   env_content: string,
   static_content: string,
-  dockerfile_present:string,
+  dockerfile_present: string,
   stack: string,
   port: string,
   build_cmds: string,
@@ -17,30 +46,84 @@ async function addScript(
     await exec(
       `bash -c "echo 'bash ../../src/backend/shell_scripts/automate.sh -u ${document.resource} ${document.subdomain}' > /hostpipe/pipe"`,
     );
+    return;
   } else if (document.resource_type === "PORT") {
     await exec(
       `bash -c "echo 'bash ../../src/backend/shell_scripts/automate.sh -p ${document.resource} ${document.subdomain}' > /hostpipe/pipe"`,
     );
-  } else if (document.resource_type === "GITHUB" && static_content == "Yes") {
-    Deno.writeTextFile(`/hostpipe/.env`, env_content);
+    return;
+  }
+
+  let envFilePath: string | null = null;
+  let hasSecrets = false;
+
+  try {
+    const volumeName = `df-vol-${sanitizeSubdomain(document.subdomain)}`;
     await exec(
-      `bash -c "echo 'bash ../../src/backend/shell_scripts/container.sh -s ${document.subdomain} ${document.resource} 80 ${MEMORY_LIMIT}' > /hostpipe/pipe"`,
+      `bash -c "echo 'bash ../../src/backend/shell_scripts/volume.sh create ${document.subdomain}' > /hostpipe/pipe"`,
     );
-  } else if (document.resource_type === "GITHUB" && static_content == "No") {
-    if(dockerfile_present === 'No'){
-      const dockerfile = dockerize(stack, port, build_cmds);
-    Deno.writeTextFile(`/hostpipe/Dockerfile`, dockerfile);
-    Deno.writeTextFile(`/hostpipe/.env`, env_content);
+
+    await new Promise((resolve) => setTimeout(resolve, 500));
+
+    await upsertVolumeMetadata(document.subdomain, volumeName, VOLUME_MOUNT_PATH);
+
+    const storedSecret = await getSecretsForProject(document.subdomain);
+    let decryptedSecrets: Record<string, string> = {};
+
+    if (storedSecret) {
+      const encryptionService = getEncryptionService();
+      if (encryptionService.isInitialized()) {
+        try {
+          decryptedSecrets = await encryptionService.decryptSecrets({
+            encrypted: storedSecret.encrypted_secrets,
+            iv: storedSecret.iv,
+            tag: storedSecret.tag,
+          });
+          hasSecrets = Object.keys(decryptedSecrets).length > 0;
+        } catch (error) {
+          console.error(`Failed to decrypt secrets for ${document.subdomain}:`, error);
+        }
+      }
+    }
+
+    const allEnvVars = mergeEnvVars(env_content || "", decryptedSecrets);
+
+    if (allEnvVars.trim() || hasSecrets) {
+      envFilePath = `/tmp/${sanitizeSubdomain(document.subdomain)}-${Date.now()}.env`;
+      await Deno.writeTextFile(envFilePath, allEnvVars);
+    }
+  } catch (error) {
+    console.error("Error preparing volumes/secrets for deployment:", error);
+  }
+
+  if (document.resource_type === "GITHUB" && static_content == "Yes") {
+    if (env_content && !hasSecrets) {
+      Deno.writeTextFile(`/hostpipe/.env`, env_content);
+    }
+
+    const envFileArg = envFilePath ? ` ${envFilePath}` : "";
     await exec(
-      `bash -c "echo 'bash ../../src/backend/shell_scripts/container.sh -g ${document.subdomain} ${document.resource} ${port} ${MEMORY_LIMIT}' > /hostpipe/pipe"`,
+      `bash -c "echo 'bash ../../src/backend/shell_scripts/container.sh -s ${document.subdomain} ${document.resource} 80 ${MEMORY_LIMIT}${envFileArg}' > /hostpipe/pipe"`,
     );
-    }else if(dockerfile_present === 'Yes'){
+  } else if (document.resource_type === "GITHUB" && static_content == "No") {
+    if (dockerfile_present === "No") {
+      const dockerfile = dockerize(stack, port, build_cmds);
+      Deno.writeTextFile(`/hostpipe/Dockerfile`, dockerfile);
 
+      if (env_content && !hasSecrets) {
+        Deno.writeTextFile(`/hostpipe/.env`, env_content);
+      }
+
+      const envFileArg = envFilePath ? ` ${envFilePath}` : "";
+      await exec(
+        `bash -c "echo 'bash ../../src/backend/shell_scripts/container.sh -g ${document.subdomain} ${document.resource} ${port} ${MEMORY_LIMIT}${envFileArg}' > /hostpipe/pipe"`,
+      );
+    } else if (dockerfile_present === "Yes") {
+      const envFileArg = envFilePath ? ` ${envFilePath}` : "";
       await exec(
-        `bash -c "echo 'bash ../../src/backend/shell_scripts/container.sh -d ${document.subdomain} ${document.resource} ${port} ${MEMORY_LIMIT}' > /hostpipe/pipe"`,
+        `bash -c "echo 'bash ../../src/backend/shell_scripts/container.sh -d ${document.subdomain} ${document.resource} ${port} ${MEMORY_LIMIT}${envFileArg}' > /hostpipe/pipe"`,
       );
     }
-
   }
 }