Skip to content

Commit 0d453a0

Browse files
PercoccoPercocco
committed
fix obsolescenza session e output
1 parent 2b49373 commit 0d453a0

2 files changed

Lines changed: 155 additions & 12 deletions

File tree

‎backend/src/utils/common.js‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -143,7 +143,9 @@ module.exports = {
143143
return (configIn || config).writers.includes('minioWriter');
144144
},
145145
async finish(obj) {
146+
console.log("-------------------TRACE-------------------")
146147
console.trace("Waiting for value to be set in object ", obj)
148+
console.log("-------------------TRACE-------------------")
147149
let logCounterFlag
148150
while (!obj.value) {
149151
await sleep(1)

‎backend/src/utils/utils.js‎

Lines changed: 153 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -37,12 +37,21 @@ const mongoose = require("mongoose");
3737

3838
function readDirRecursive(dir) {
3939
let results = [];
40-
const entries = fs.readdirSync(dir, { withFileTypes: true });
40+
let entries
41+
try {
42+
entries = fs.readdirSync(dir, { withFileTypes: true });
43+
}
44+
catch (error) {
45+
logger.error(`Error reading directory ${dir}:`, error)
46+
fs.mkdirSync(dir, { recursive: true });
47+
return results
48+
}
4149

4250
for (const entry of entries) {
4351
const fullPath = path.join(dir, entry.name);
4452
logger.debug(entry.name)
4553
if (entry.isDirectory()) {
54+
results.push({ folder: fullPath });
4655
results = results.concat(readDirRecursive(fullPath));
4756
} else {
4857
results.push(fullPath);
@@ -52,6 +61,46 @@ function readDirRecursive(dir) {
5261
return results;
5362
}
5463

64+
function pathIsOutput(path) {
65+
logger.debug(`Checking if path ${path} is an output path...`)
66+
logger.debug(
67+
path.substring(path.lastIndexOf("/") + 1, path.lastIndexOf(".")),
68+
parseInt(path.substring(path.lastIndexOf("/") + 1, path.lastIndexOf("."))),
69+
isNaN(parseInt(path.substring(path.lastIndexOf("/") + 1, path.lastIndexOf("."))))
70+
)
71+
return !isNaN(parseInt(path.substring(path.lastIndexOf("/") + 1, path.lastIndexOf("."))))
72+
}
73+
function pathIsSession(path) {
74+
return isNaN(parseInt(path.substring(path.lastIndexOf("/"), path.lastIndexOf("."))))
75+
}
76+
function getFolderFromOutputPath(path) {
77+
if (pathIsOutput(path))
78+
return path.substring(0, path.lastIndexOf("/"))
79+
else
80+
throw new Error("Path is not an output path")
81+
}
82+
function getSessionFromOutputPath(path) {
83+
if (pathIsOutput(path)) {
84+
let folder = getFolderFromOutputPath(path)
85+
return getSessionFromOutputFolderPath(folder)
86+
}
87+
else
88+
throw new Error("Path is not an output path")
89+
}
90+
91+
function getFolderFromSession(path) {
92+
if (pathIsSession(path)) {
93+
let folder = path.replace("./output/output", "./output/").replace(".json", "")
94+
return folder
95+
}
96+
else
97+
throw new Error("Path is not a session path")
98+
}
99+
100+
function getSessionFromOutputFolderPath(path) {
101+
return "./output/" + path.substring(2).split("/").join("") + ".json"
102+
}
103+
55104
function dropOutput(id) {
56105
const { execSync } = require("child_process");
57106
const os = require('os')
@@ -107,12 +156,12 @@ async function checkMaximumSpaceOverflow() {
107156
await Session.deleteOne({ sessionId: session.sessionId });
108157
usedMB -= size;
109158
if (usedMB <= (config.mongoMaxStorageMB || 500)) {
110-
stats = await mongoose.connection.db.stats({
159+
/*stats = await mongoose.connection.db.stats({
111160
scale: 1024 * 1024
112161
});
113162
usedMB = stats.storageSize;
114-
if (usedMB <= (config.mongoMaxStorageMB || 500))
115-
break
163+
if (usedMB <= (config.mongoMaxStorageMB || 500))*/
164+
break
116165
}
117166
}
118167
for (const coll of collections.filter(coll => coll.name.includes("output"))) {
@@ -136,27 +185,119 @@ async function checkMaximumSpaceOverflow() {
136185
let files = readDirRecursive("./output/")//fs.readdirSync("./output/");
137186
logger.debug(files)
138187
files = files
139-
.map(file => ({ file, time: fs.statSync("./" + file).mtime.getTime() }))
188+
.map(file => ({ file: (file.folder || file), time: fs.statSync("./" + (file.folder || file)).mtime.getTime(), path: "./" + (file.folder || file).replaceAll("\\", "/"), type: (file.folder ? "folder" : "file") }))
140189
.sort((a, b) => {
141190
const aTime = a.time;
142191
const bTime = b.time;
143192
return bTime - aTime;
144193
});
194+
let sessionedOutputs = {}
145195
for (const file of files) {
146196
//logger.debug(`Checking file ${file.file} for cleanup...`)
147-
const filePath = path.join("./", file.file);
148-
const stats = fs.statSync(filePath);
149-
usedMB += stats.size / (1024 * 1024);
197+
const filePath = "./" + file.file; // ./output\\shared123\1.json
198+
logger.debug(`Checking file ${filePath} | ${file.path} for cleanup...`)
199+
//if (Object.keys(sessionedOutputs).filter(key => key.includes(filePath.)))//sessionedOutputs[filePath] === undefined)
200+
if (pathIsOutput(file.path)) {
201+
//let folder = filePath.substring(0, filePath.lastIndexOf("\\"))
202+
//let searchingSession = folder.split("\\").join("") + ".json"
203+
logger.debug("Is an output file, looking for session file...")
204+
logger.debug({ file })
205+
//logger.debug(file.path)
206+
//let creatingId = file.path.substring(2, file.path.lastIndexOf("/")).replace("output/", "output") //./output/shared123/1.json -> outputshared123
207+
//logger.debug(creatingId)
208+
//creatingId.pop()
209+
//creatingId = "./output" + creatingId + ".json" //.join("") + ".json" // outputshared123 -> ./output/outputshared123.json
210+
//logger.debug(creatingId)
211+
if (sessionedOutputs[getSessionFromOutputPath(file.path)] === undefined)
212+
sessionedOutputs[getSessionFromOutputPath(file.path)] = {
213+
searchingSessionFound: false,
214+
searchingOutputFound: true,
215+
folderPath: getFolderFromOutputPath(file.path)//file.path.substring(0, file.path.lastIndexOf("/"))//filePath.substring(0, filePath.lastIndexOf("\\"))
216+
}
217+
else
218+
sessionedOutputs[getSessionFromOutputPath(file.path)].searchingOutputFound = true
219+
}
220+
else if (file.type === "folder") {
221+
logger.debug("Is a folder, looking for session file...")
222+
logger.debug(file.path)
223+
if (sessionedOutputs[getSessionFromOutputFolderPath(file.path)] === undefined)
224+
sessionedOutputs[getSessionFromOutputFolderPath(file.path)] = {
225+
searchingSessionFound: false,
226+
searchingOutputFound: false,
227+
folderPath: file.path
228+
}
229+
}
230+
else {
231+
logger.debug("Is a session file, looking for session file...")
232+
logger.debug(filePath)
233+
logger.debug(file.path)
234+
if (sessionedOutputs[file.path] === undefined)
235+
sessionedOutputs[file.path] = {
236+
searchingSessionFound: true,
237+
searchingOutputFound: false,
238+
filePath: file.path
239+
}
240+
else
241+
sessionedOutputs[file.path].searchingSessionFound = true
242+
}
243+
let stats
244+
try {
245+
stats = fs.statSync(file.path);
246+
} catch (error) {
247+
logger.error(`Error getting stats for file ${file.path}:`, error)
248+
if (cancel)
249+
logger.debug(`Maybe file ${file.path} was already deleted, dummy!`)
250+
continue
251+
}
252+
usedMB += stats?.size / (1024 * 1024) || 0;
150253
if (usedMB > (config.fileMaxStorageMB || 500))
151254
cancel = true
152255
if (cancel)
153256
try {
154-
fs.unlinkSync(filePath);
257+
let folder //file.path.substring(0, file.path.lastIndexOf("/"))//filePath.substring(0, filePath.lastIndexOf("\\"))
258+
if (pathIsOutput(file.path))
259+
folder = getFolderFromOutputPath(file.path)
260+
else if (file.type === "folder")
261+
folder = file.path
262+
else
263+
folder = getFolderFromSession(file.path)
264+
logger.debug(`Deleting file ${file.path} and folder ${folder} for cleanup...`)
265+
fs.rmSync(folder, { recursive: true, force: true });
266+
logger.info(`File ${file.path} and folder ${folder} deleted.`);
267+
let orphanSession //"./output/" + folder.split("/").join("") + ".json"
268+
if (pathIsOutput(file.path))
269+
orphanSession = getSessionFromOutputPath(file.path)
270+
else if (file.type === "folder")
271+
orphanSession = getSessionFromOutputFolderPath(file.path)
272+
else
273+
orphanSession = file.path
274+
logger.debug(`Checking for orphan session with id ${orphanSession} linked to deleted file...`)
275+
if (fs.existsSync(orphanSession))
276+
try {
277+
fs.unlinkSync(orphanSession);
278+
} catch (error) {
279+
logger.error(`Error deleting orphan session file ${orphanSession}:`, error)
280+
}
281+
else
282+
logger.debug(`No orphan session file ${orphanSession} found for deleted file.`)
283+
//fs.unlinkSync(filePath);
155284
//logger.info(`File ${file.file} deleted.`);
156285
} catch (err) {
157286
logger.error(`Error deleting file ${filePath}:`, err);
158287
}
159288
}
289+
for (let key in sessionedOutputs)
290+
if (!sessionedOutputs[key].searchingSessionFound) {
291+
logger.warn(`Orphan output detected: ${key}`)
292+
logger.debug(`Deleting folder ${sessionedOutputs[key].folderPath} for orphan output...`)
293+
fs.rmSync(sessionedOutputs[key].folderPath, { recursive: true, force: true });
294+
}
295+
else if (!sessionedOutputs[key].searchingOutputFound) {
296+
logger.warn(`Orphan session detected: ${key}`)
297+
logger.debug(`Deleting file ${sessionedOutputs[key].filePath} for orphan session...`)
298+
fs.unlinkSync(sessionedOutputs[key].filePath)
299+
}
300+
logger.debug(sessionedOutputs)
160301
logger.debug("Current filesystem storage size for sessions and outputs: ", Number(usedMB.toFixed(3)), " MB")
161302
}
162303

@@ -330,10 +471,10 @@ const parseFilePath = (pathString) => {
330471

331472
/*function spaceCleaner(object) {
332473
stack = [object];
333-
474+
334475
while (stack.length > 0) {
335476
let current = stack.pop();
336-
477+
337478
for (let sub in current) {
338479
if (typeof current[sub] === "object" && current[sub] !== null) {
339480
stack.push(current[sub]);
@@ -342,7 +483,7 @@ const parseFilePath = (pathString) => {
342483
}
343484
}
344485
}
345-
486+
346487
return object;
347488
}*/
348489

0 commit comments

Comments
 (0)