Skip to content

Commit 99840a0

Browse files
pajama-coderpajama-coder
committed
Mesh filesystem now supports file deletion and watching
1 parent bd85b92 commit 99840a0

File tree

5 files changed

+132
-67
lines changed

5 files changed

+132
-67
lines changed

‎agent/fs.js‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,18 @@
11
//
22
// Mesh Filesystem
33
//
4-
// /home
4+
// /users
55
// /<username>
6-
// /apps
7-
// /<provider>
8-
// /<appname>
96
// /shared
107
// /<username>
118
// /pkg
12-
// /apps
13-
// /<provider>
14-
// /<appname>
9+
// /apps
10+
// /<provider>
11+
// /<appname>
12+
// /users
13+
// /<username>
14+
// /shared
15+
// /<username>
1516
//
1617

1718
export default function(storeDir) {

‎agent/mesh.js‎

Lines changed: 74 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,8 @@ export default function (rootDir, config) {
1111
var agentLog = []
1212
var meshErrors = []
1313
var fs = null
14+
var fsWatchers = []
15+
var fsLastChangeTime = Date.now()
1416
var apps = null
1517
var exited = false
1618

@@ -1066,6 +1068,49 @@ export default function (rootDir, config) {
10661068
})
10671069
}
10681070

1071+
function watchFile(prefix) {
1072+
var resolve
1073+
var promise = new Promise(cb => resolve = cb)
1074+
var isWatching = fsWatchers.length > 0
1075+
var entry = fsWatchers.find(([k]) => k === prefix)
1076+
if (entry) {
1077+
entry[1].push(resolve)
1078+
} else {
1079+
fsWatchers.push([prefix, [resolve]])
1080+
}
1081+
if (!isWatching) startWatchingFiles()
1082+
return promise
1083+
}
1084+
1085+
function startWatchingFiles() {
1086+
new Timeout(5).wait().then(
1087+
() => discoverFiles(fsLastChangeTime)
1088+
).then(files => {
1089+
var paths = Object.keys(files)
1090+
if (paths.length > 0) {
1091+
fsLastChangeTime = Object.values(files).map(f => f.since).reduce(
1092+
(max, t) => (t > max ? t : max), fsLastChangeTime
1093+
)
1094+
fsWatchers.forEach(
1095+
([prefix, watchers]) => {
1096+
var changes = []
1097+
paths.forEach(path => {
1098+
if (path.startsWith(prefix)) {
1099+
changes.push(path)
1100+
}
1101+
})
1102+
if (changes.length > 0) {
1103+
watchers.forEach(resolve => resolve([...changes]))
1104+
watchers.length = 0
1105+
}
1106+
}
1107+
)
1108+
fsWatchers = fsWatchers.filter(([_, watchers]) => watchers.length > 0)
1109+
}
1110+
if (fsWatchers.length > 0) startWatchingFiles()
1111+
})
1112+
}
1113+
10691114
//
10701115
// Mesh API exposed to apps
10711116
//
@@ -1086,32 +1131,22 @@ export default function (rootDir, config) {
10861131
}
10871132

10881133
function makeAppFilesystem(provider, app) {
1089-
var prefixHome = `/home/${username}/`
1090-
var prefixHomeApp = prefixHome + `apps/${provider}/${app}/`
1091-
var prefixApp = `/apps/${provider}/${app}/`
1092-
var matchShared = new http.Match(`/shared/{username}/apps/${provider}/${app}/*`)
1093-
var matchShared = new http.Match('/shared/{username}' + prefixApp + '*')
1134+
var pathApp = `/apps/${provider}/${app}`
1135+
var pathUser = `/users/${username}/`
1136+
var pathShared = `/shared/`
1137+
var pathLocal = `/local/`
1138+
var pathAppUser = pathApp + pathUser
1139+
var pathAppShared = pathApp + pathShared
10941140

10951141
function pathToLocal(path) {
1096-
if (path.startsWith(prefixHomeApp)) {
1097-
return prefixHome + path.substring(prefixHomeApp.length)
1098-
}
1099-
var params = matchShared(path)
1100-
if (params) {
1101-
return `/shared/${params.username}/${params['*']}`
1142+
if (path.startsWith(pathAppUser) || path.startsWith(pathAppShared)) {
1143+
return path.substring(pathApp.length)
11021144
}
11031145
}
11041146

11051147
function pathToGlobal(path) {
1106-
if (path.startsWith(prefixHome)) {
1107-
return prefixHomeApp + path.substring(prefixHome.length)
1108-
}
1109-
if (path.startsWith('/shared/')) {
1110-
path = path.substring(8)
1111-
var i = path.indexOf('/')
1112-
if (i <= 0 || i + 1 == path.length) return
1113-
var username = path.substring(0, i)
1114-
return `/shared/${username}` + prefixApp + path.substring(i + 1)
1148+
if (path.startsWith(pathUser) || path.startsWith(pathShared)) {
1149+
return pathApp + path
11151150
}
11161151
}
11171152

@@ -1124,11 +1159,13 @@ export default function (rootDir, config) {
11241159
var list = []
11251160
Object.keys(files).forEach(path => {
11261161
var localPath = pathToLocal(path)
1127-
if (localPath && localPath.startsWith(prefix)) list.push(localPath)
1162+
if (localPath && localPath.startsWith(prefix)) {
1163+
list.push(path)
1164+
}
11281165
})
11291166
db.allFiles(meshName, provider, app).forEach(
11301167
path => {
1131-
var fullPath = '/local' + path
1168+
var fullPath = os.path.join(pathLocal, path)
11321169
if (fullPath.startsWith(prefix)) list.push(fullPath)
11331170
}
11341171
)
@@ -1139,9 +1176,9 @@ export default function (rootDir, config) {
11391176

11401177
function read(pathname) {
11411178
var path = os.path.normalize(pathname)
1142-
if (path.startsWith('/local/')) {
1179+
if (path.startsWith(pathLocal)) {
11431180
return Promise.resolve(
1144-
db.getFile(meshName, provider, app, path.substring(6))
1181+
db.getFile(meshName, provider, app, path.substring(pathLocal.length))
11451182
)
11461183
} else {
11471184
var globalPath = pathToGlobal(path)
@@ -1156,8 +1193,8 @@ export default function (rootDir, config) {
11561193
function write(pathname, data) {
11571194
if (typeof data === 'string') data = new Data(data)
11581195
var path = os.path.normalize(pathname)
1159-
if (path.startsWith('/local/')) {
1160-
db.setFile(meshName, provider, app, path.substring(6), data)
1196+
if (path.startsWith(pathLocal)) {
1197+
db.setFile(meshName, provider, app, path.substring(pathLocal.length), data)
11611198
} else {
11621199
var globalPath = pathToGlobal(path)
11631200
if (globalPath) {
@@ -1167,7 +1204,17 @@ export default function (rootDir, config) {
11671204
}
11681205
}
11691206

1170-
return { dir, read, write }
1207+
function watch(prefix) {
1208+
if (!prefix.endsWith('/')) prefix += '/'
1209+
var globalPath = pathToGlobal(prefix)
1210+
if (globalPath) {
1211+
return watchFile(globalPath).then(
1212+
paths => paths.map(path => pathToLocal(path)).filter(p=>p)
1213+
)
1214+
}
1215+
}
1216+
1217+
return { dir, read, write, watch }
11711218
}
11721219

11731220
function remoteQueryLog(ep) {

‎cli/main.js‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -279,7 +279,7 @@ function doCommand(meshName, epName, argv, program) {
279279
var type = args['<object type>']
280280
var name = args['<object name>']
281281
switch (type) {
282-
case 'app': return selectMesh(meshName).then(mesh => downloadApp(name, mesh))
282+
case 'app': return selectMeshEndpoint(meshName, epName).then(({ mesh, ep }) => downloadApp(name, mesh, ep))
283283
case 'file': return selectMesh(meshName).then(mesh => downloadFile(name, args['--output'], mesh))
284284
default: return invalidObjectType(type, 'download')
285285
}
@@ -294,7 +294,7 @@ function doCommand(meshName, epName, argv, program) {
294294
var type = args['<object type>']
295295
var name = args['<object name>']
296296
switch (type) {
297-
case 'app': return selectMesh(meshName).then(mesh => eraseApp(name, mesh))
297+
case 'app': return selectMeshEndpoint(meshName, epName).then(({ mesh, ep }) => eraseApp(name, mesh, ep))
298298
case 'file': return selectMesh(meshName).then(mesh => eraseFile(name, mesh))
299299
default: return invalidObjectType(type, 'erase')
300300
}
@@ -314,7 +314,7 @@ function doCommand(meshName, epName, argv, program) {
314314
var type = args['<object type>']
315315
var name = args['<object name>']
316316
switch (type) {
317-
case 'app': return selectMesh(meshName).then(mesh => publishApp(name, mesh))
317+
case 'app': return selectMeshEndpoint(meshName, epName).then(({ mesh, ep }) => publishApp(name, mesh, ep))
318318
case 'file': return selectMesh(meshName).then(mesh => publishFile(name, args['--input'], mesh))
319319
default: return invalidObjectType(type, 'publish')
320320
}
@@ -329,7 +329,7 @@ function doCommand(meshName, epName, argv, program) {
329329
var type = args['<object type>']
330330
var name = args['<object name>']
331331
switch (type) {
332-
case 'app': return selectMesh(meshName).then(mesh => unpublishApp(name, mesh))
332+
case 'app': return selectMeshEndpoint(meshName, epName).then(({ mesh, ep }) => unpublishApp(name, mesh, ep))
333333
case 'file': return selectMesh(meshName).then(mesh => unpublishFile(name, mesh))
334334
default: return invalidObjectType(type, 'unpublish')
335335
}
@@ -1039,7 +1039,7 @@ function describeApp(name, mesh, ep) {
10391039
// Command: download
10401040
//
10411041

1042-
function downloadApp(name) {
1042+
function downloadApp(name, mesh, ep) {
10431043
var appName = normalizeAppName(name)
10441044
if (!appName) throw 'missing app name'
10451045
return selectApp(appName, mesh, ep).then(app => {
@@ -1069,7 +1069,7 @@ function downloadFile(name, output, mesh) {
10691069
// Command: erase
10701070
//
10711071

1072-
function eraseApp(name, mesh) {
1072+
function eraseApp(name, mesh, ep) {
10731073
var appName = normalizeAppName(name)
10741074
if (!appName) throw 'missing app name'
10751075
return selectApp(appName, mesh, ep).then(app => {
@@ -1091,7 +1091,7 @@ function eraseFile(name, mesh) {
10911091
// Command: publish
10921092
//
10931093

1094-
function publishApp(name) {
1094+
function publishApp(name, mesh, ep) {
10951095
var appName = normalizeAppName(name)
10961096
if (!appName) throw 'missing app name'
10971097
return selectApp(appName, mesh, ep).then(app => {
@@ -1118,7 +1118,7 @@ function publishFile(name, input, mesh) {
11181118
// Command: unpublish
11191119
//
11201120

1121-
function unpublishApp(name) {
1121+
function unpublishApp(name, mesh, ep) {
11221122
var appName = normalizeAppName(name)
11231123
if (!appName) throw 'missing app name'
11241124
return selectApp(appName, mesh, ep).then(app => {

‎docs/Agent-API.md‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
# Agent API
22

3-
The Agent API is organized into 4 types of resources that can be accessed by the standard HTTP semantics. The 4 types of resources are:
3+
The Agent API is organized into 4 types of resources that can be accessed by standard HTTP semantics. The 4 types of resources are:
44

55
- Meshes
66
- Endpoints
@@ -163,8 +163,9 @@ Or returns raw binary content for `/file-data` based paths.
163163
Paths and methods:
164164

165165
```
166-
GET /api/meshes/{meshName}/files
166+
GET /api/meshes/{meshName}/files[?since={time}]
167167
GET /api/meshes/{meshName}/files/{pathname}
168+
DELETE /api/meshes/{meshName}/files/{pathname}
168169
GET /api/meshes/{meshName}/file-data/{pathname}
169170
POST /api/meshes/{meshName}/file-data/{pathname}
170171
DELETE /api/meshes/{meshName}/file-data/{pathname}

‎hub/main.js‎

Lines changed: 39 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -374,9 +374,19 @@ var postFilesystem = pipeline($=>$
374374
.replaceMessage(
375375
function (req) {
376376
var body = JSON.decode(req.body)
377-
var prefix = `/home/${$endpoint.username}/`
377+
var username = $endpoint.username
378+
var prefixUser = `/users/${username}/`
379+
var prefixShared = `/shared/${username}`
380+
var matchAppUser = new http.Match(`/apps/{provider}/{appname}/users/${username}/*`)
381+
var matchAppShared = new http.Match(`/apps/{provider}/{appname}/shared/${username}/*`)
382+
var canUpdate = (path) => (
383+
path.startsWith(prefixUser) ||
384+
path.startsWith(prefixShared) ||
385+
matchAppUser(path) ||
386+
matchAppShared(path)
387+
)
378388
Object.entries(body).map(
379-
([k, v]) => updateFileInfo(k, v, $endpoint.id, k.startsWith(prefix))
389+
([k, v]) => updateFileInfo(k, v, $endpoint.id, canUpdate(k))
380390
)
381391
return new Message({ status: 201 })
382392
}
@@ -588,27 +598,33 @@ function makeFileInfo(hash, size, time, since) {
588598
}
589599

590600
function updateFileInfo(pathname, f, ep, update) {
591-
var e = (files[pathname] ??= makeFileInfo('', 0, 0, 0))
592-
var t1 = e['T']
593-
var h1 = e['#']
594-
var t2 = f['T']
595-
var h2 = f['#']
596-
if (h2 === h1) {
597-
var sources = e['@']
598-
if (!sources.includes(ep)) sources.push(ep)
599-
if (update) e['T'] = Math.max(t1, t2)
600-
} else if (t2 > t1 && update) {
601-
e['#'] = h2
602-
e['$'] = f['$']
603-
e['T'] = t2
604-
e['+'] = Date.now()
605-
e['@'] = [ep]
606-
db.setFile(pathname, {
607-
hash: h2,
608-
size: e['$'],
609-
time: t2,
610-
since: e['+'],
611-
})
601+
var e = files[pathname]
602+
if (e || update) {
603+
if (!e) e = files[pathname] = makeFileInfo('', 0, 0, 0)
604+
var t1 = e['T']
605+
var h1 = e['#']
606+
var t2 = f['T']
607+
var h2 = f['#']
608+
if (h2 === h1) {
609+
var sources = e['@']
610+
if (!sources.includes(ep)) sources.push(ep)
611+
if (update && t2 > t1) {
612+
e['T'] = t2
613+
e['+'] = Date.now()
614+
}
615+
} else if (t2 > t1 && update) {
616+
e['#'] = h2
617+
e['$'] = f['$']
618+
e['T'] = t2
619+
e['+'] = Date.now()
620+
e['@'] = [ep]
621+
db.setFile(pathname, {
622+
hash: h2,
623+
size: e['$'],
624+
time: t2,
625+
since: e['+'],
626+
})
627+
}
612628
}
613629
}
614630

0 commit comments

Comments
 (0)