feat: sync to electric

Author: shortdiv

Diff for: server/index.js

@@ -2,7 +2,7 @@ import { electricSync } from "@electric-sql/pglite-sync";
 import Fastify from "fastify";
 import validate from "uuid-validate";
 
-import { populate } from "./populate-pglite.js";
+import { generateAndSyncToElectric } from "./populate-pglite.js";
 
 const fastify = Fastify({
   logger: true,
@@ -17,7 +17,9 @@ const db = await PGlite.create({
 
 console.log("Waiting for db to be ready");
 await db.waitReady;
-await populate(db);
+
+console.log("Syncing shapes");
+await generateAndSyncToElectric(db);
 
 fastify.get("/users", async (_req, reply) => {
   const res = await db.exec(
@@ -36,10 +38,14 @@ fastify.get("/users/:userId", async (req, reply) => {
   if (!validate(userId)) {
     reply.send(`The id provided ${userId} is not a valid UUID`);
   }
-  const res = await db.exec(
+  const res = await db.live.query(
     `
     SELECT * from users WHERE id = '${userId}'
   `,
+    null,
+    (res) => {
+      JSON.stringify(res.rows, null, 2);
+    },
   );
   const rows = res[0].rows;
 

Diff for: server/populate-pglite.js

@@ -1,22 +1,5 @@
-import { generateUsers } from "../db/generate-data.js";
-
-const USERS_TO_LOAD = 1000000;
-
-async function makeInsertQuery(db, data) {
-  const columns = Object.keys(data);
-  const columnNames = columns.join(`, `);
-  const values = columns.map((column) => data[column]);
-  const sql = `
-    INSERT INTO users (${columnNames})
-    VALUES (${values.map((value) => `'${value}'`)}) 
-  `;
-  console.log("Inserting data...");
-  return await db.exec(sql);
-}
-
-//generate users//
-export async function populate(db) {
-  //run pglite queries
+//generate table and sync to electric//
+export async function generateAndSyncToElectric(db) {
   await db.exec(`
   CREATE TABLE IF NOT EXISTS users (
     id uuid PRIMARY KEY,
@@ -35,23 +18,17 @@ export async function populate(db) {
   )
 `);
 
-  const users = generateUsers(USERS_TO_LOAD);
-  const userCount = users.length;
-  const batchSize = 100;
-  for (let i = 0; i < userCount; i += batchSize) {
-    db.exec(`SET CONSTRAINTS ALL DEFERRED;`); // disable FK checks
-
-    const batch = users.slice(i, i + batchSize);
-    const promises = batch.map(async (user, index) => {
-      if ((i + index + 1) % 100 === 0 || i + index + 1 === userCount) {
-        process.stdout.write(`Loading user ${i + index + 1} of ${userCount}\r`);
-      }
-      return await makeInsertQuery(db, user);
-    });
-    try {
-      await Promise.all(promises);
-    } catch (err) {
-      console.error("Batch failed", err);
-    }
-  }
+  const shape = await db.electric.syncShapeToTable({
+    shape: {
+      url: `${env.ELECTRIC_URL}/v1/shape`,
+      params: {
+        table: "users",
+        sourceSecret: env.ELECTRIC_SOURCE_SECRET,
+        sourceId: env.ELECTRIC_SOURCE_ID,
+      },
+    },
+    table: "users",
+    primaryKey: ["id"],
+  });
+  return shape;
 }

Diff for: sst.config.ts

@@ -18,6 +18,16 @@ export default $config({
     };
   },
   async run() {
+    // env var checks
+    if (
+      !process.env.ELECTRIC_SOURCE_ID &&
+      !process.env.ELECTRIC_SOURCE_SECRET
+    ) {
+      throw new Error(
+        `ELECTRIC_SOURCE_ID and ELECTRIC_SOURCE_SECRET is not set`,
+      );
+    }
+
     const { getNeonConnectionString, createNeonDb } = await import("./neon");
     // Create a db in Neon
     const project = neon.getProjectOutput({ id: `square-flower-52864146` });