add support for persistent storage

ansible/playbooks/swarm_apt.yml

@@ -47,6 +47,13 @@
         update_cache: yes
         cache_valid_time: 3600
 
+    - name: apt | Install autofs
+      apt:
+        name: autofs
+        state: present
+        update_cache: yes
+        cache_valid_time: 3600
+
     - name: Install nginx
       apt:
         name: nginx

swarm-syncer/beamup-sync-and-deploy

@@ -27,11 +27,11 @@ services:
 EOF
 docker stack rm --detach=false beamup_control
 sleep 10
-docker stack deploy --detach=false --compose-file registry.yaml beamup_control
+docker stack deploy --detach=false --compose-file registry.yaml beamup_control --detach=false
 sleep 50
 
-beamup-sync-swarm apps.yaml apps.conf
+sudo beamup-sync-swarm apps.yaml apps.conf docker_volumes
 APPS_COUNT=`cat apps.yaml | grep image | wc -l`
 if [ $APPS_COUNT -gt 0 ] ; then
-	docker stack deploy --prune --compose-file apps.yaml beamup
+	docker stack deploy --prune --compose-file apps.yaml beamup --detach=false
 fi

swarm-syncer/beamup-sync-swarm

@@ -3,11 +3,14 @@
 const fs = require('fs')
 const http = require('http')
 const https = require('https')
+const { execSync } = require('child_process');
 const request = (http, args) => new Promise((resolve, reject) => http.request(args, resolve).on('error', reject).end(args.body))
 
 const START_PORT = 8000
 const TOTAL_APP_LIMIT = 300
 const REGISTRY_URL = 'http://127.0.0.1:5000'
+const VOLUME_SIZE = '1GB'
+const VOLUME_MOUNT_DIR = `/mnt/beamup`;
 
 // https://docs.docker.com/compose/compose-file/04-version-and-name/#version-top-level-element-obsolete
 // Compose Specification is being used
@@ -35,6 +38,8 @@ const APP_TMPL = (appName, image, port) => `   ${appName}:
         command: /start web
         ports:
           - '${port}:${port}'
+        volumes:
+          - ${VOLUME_MOUNT_DIR}/${appName}:/persistant_data
 `
 
 const APP_DOCKER_TMPL = (appName, image, port) => `   ${appName}:
@@ -55,6 +60,8 @@ const APP_DOCKER_TMPL = (appName, image, port) => `   ${appName}:
           - PORT=${port}
         ports:
           - '${port}:${port}'
+        volumes:
+          - ${VOLUME_MOUNT_DIR}/${appName}:/persistant_data
 `
 
 // https://github.com/docker-archive/for-aws/issues/104
@@ -107,6 +114,37 @@ server {
 
 `
 
+function createVolume(appName) {
+  if (!volumesDir) return;
+  const volumePath = `${volumesDir}/${appName}_data`;
+  try {
+    // If the volume file does not exist, create and format it
+    if (!fs.existsSync(volumePath)) {
+      execSync(`dd if=/dev/zero of="${volumePath}" bs=1M count=0 seek=${VOLUME_SIZE}`);
+      // Set up a loop device, format as ext4, set permissions, then detach
+      let loopdev = execSync(`losetup --find --show "${volumePath}"`).toString().trim();
+      const tmpMount = `/tmp/beamup-mnt-${appName}-${process.pid}`;
+      try {
+        execSync(`mkfs.ext4 -F "${loopdev}"`);
+        fs.mkdirSync(tmpMount, { recursive: true });
+        execSync(`mount "${loopdev}" "${tmpMount}"`);
+        execSync(`chmod 0777 "${tmpMount}"`);
+        execSync(`umount "${tmpMount}"`);
+        fs.rmdirSync(tmpMount);
+      } catch (err) {
+        // Try to clean up if mount fails
+        try { execSync(`umount "${tmpMount}"`); } catch (e) {}
+        try { fs.rmdirSync(tmpMount); } catch (e) {}
+        throw err;
+      } finally {
+        execSync(`losetup -d "${loopdev}"`);
+      }
+    }
+  } catch (err) {
+    console.error(`Error in createVolume for ${appName}:`, err);
+  }
+}
+
 async function getJSON(http, opts) {
 	const res = await request(http, opts)
 	if (res.statusCode !== 200) {
@@ -153,7 +191,10 @@ async function getConfigs() {
 	if (apps.length > TOTAL_APP_LIMIT) throw new Error('app limit exceeded')
 	const swarmHerokuishCfgs = apps.filter(app => !app.name.includes('docker')).map(app => APP_TMPL(app.name, app.fullImageName, app.port))
 	const swarmDockerfileCfgs = apps.filter(app => app.name.includes('docker')).map(app => APP_DOCKER_TMPL(app.name, app.fullImageName, app.port))
-	const swarmCfgs = swarmHerokuishCfgs.concat(swarmDockerfileCfgs)
+  apps.forEach(app => {
+    createVolume(app.name);
+  });
+  const swarmCfgs = swarmHerokuishCfgs.concat(swarmDockerfileCfgs);
 //	const swarmCfgs = swarmHerokuishCfgs.concat(swarmDockerfileCfgs,networkCfg)
 	const nginxCfgs = apps.map(app => APP_NGINX_TMPL(app.name, app.port))
 	return {
@@ -165,10 +206,17 @@ async function getConfigs() {
 
 const swarmCfg = process.argv[2]
 const nginxCfg = process.argv[3]
+const volumesDir = process.argv[4]
+
 if (!(swarmCfg && swarmCfg.endsWith('.yaml') && nginxCfg && nginxCfg.endsWith('.conf'))) {
-	console.log('usage: beamup-sync-swarm <path to swarm yaml> <path to nginx config>')
+	console.log('usage: beamup-sync-swarm <path to swarm yaml> <path to nginx config> <volumes directory>')
 	process.exit(1)
 }
+if (volumesDir) {
+  if (!fs.existsSync(volumesDir)) {
+    fs.mkdirSync(volumesDir, { recursive: true });
+  }
+}
 getConfigs().then(cfgs => {
 	fs.writeFileSync(swarmCfg, cfgs.swarm)
 	fs.writeFileSync(nginxCfg, cfgs.nginx)
@@ -191,4 +239,4 @@ getConfigs().then(cfgs => {
 			})
 		})
 	}
-})
+})