From 85f5dd868bd296e91f2bc5beb5add067bba0b7b1 Mon Sep 17 00:00:00 2001 From: bhatiadheeraj Date: Mon, 21 Aug 2023 20:14:21 +0000 Subject: [PATCH 1/4] added param for json and filedirectory --- api/models.js | 3 +++ bin/importdatalad.js | 12 ++++++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/api/models.js b/api/models.js index 72afa308..3242945a 100644 --- a/api/models.js +++ b/api/models.js @@ -840,6 +840,9 @@ var dlDatasetSchema = mongoose.Schema({ ], participants_info: mongoose.Schema.Types.Mixed, //metadata for participants info + phenotypes_json: mongoose.Schema.Types.Mixed, + phenotype_files: [String], //list of files in phenotypes folder + stats: { subjects: Number, sessions: Number, diff --git a/bin/importdatalad.js b/bin/importdatalad.js index 16e0456d..ae3e042c 100755 --- a/bin/importdatalad.js +++ b/bin/importdatalad.js @@ -10,9 +10,13 @@ const child_process = require('child_process'); const cli = require('brainlife'); const axios = require('axios'); -process.chdir('/mnt/datalad'); - +if(config.dataladDirectory) process.chdir(config.dataladDirectory); +else { + console.error("config.dataladDirectory is not set"); + process.exit(1); +} console.log("connecting"); + db.init(async err=>{ if(err) throw err; await load_datatypes(); @@ -123,6 +127,10 @@ function handle_bids(key, bids, cb) { if(bids.dataset_description) dldataset.dataset_description = bids.dataset_description; if(bids.participants) dldataset.participants = bids.participants; if(bids.participants_json) dldataset.participants_info = bids.participants_json; + // storing combines json and folder files + if(bids.phenotypes_json) dldataset.phenotypes_json = bids.phenotypes_json; + + if(bids.phenotype_files && bids.phenotype_files.length) dldataset.phenotype_files = bids.phenotype_files; //count let unique_subjects = []; From 8ab9d7420173c13ecfc49505dc5b4531562e7df9 Mon Sep 17 00:00:00 2001 From: bhatiadheeraj Date: Mon, 21 Aug 2023 22:04:09 +0000 Subject: [PATCH 2/4] wip --- api/controllers/datalad.js | 4 ++++ api/models.js | 3 +++ bin/importdatalad.js | 2 +- ui/src/project.vue | 8 ++++++++ 4 files changed, 16 insertions(+), 1 deletion(-) diff --git a/api/controllers/datalad.js b/api/controllers/datalad.js index 03e43e2f..0e554086 100644 --- a/api/controllers/datalad.js +++ b/api/controllers/datalad.js @@ -74,6 +74,10 @@ router.post('/import/:dataset_id', common.jwt(), (req, res, next)=>{ if(project.members.includes(req.user.sub)) canedit = true; if(!canedit) return next("you can't import to this project"); + if(dataset.phenotype_files) { + console.log("importing phenotype_files-----------------------------------------------------"); + project.phenotype_files = dataset.phenotype_files; + } //update participants info let participants = new db.Participants({ project, diff --git a/api/models.js b/api/models.js index 3242945a..44bba5d0 100644 --- a/api/models.js +++ b/api/models.js @@ -122,6 +122,9 @@ var projectSchema = mongoose.Schema({ //counts of publications (updated by common.update_project_stats) publications: Number, + //phenotype files + phenotype_files: [String], + groupanalysis: { sessions: [{ task_id: String, //amaretti task id diff --git a/bin/importdatalad.js b/bin/importdatalad.js index ae3e042c..ce9c24f6 100755 --- a/bin/importdatalad.js +++ b/bin/importdatalad.js @@ -32,7 +32,7 @@ db.init(async err=>{ */ console.log("loading dataset_description.json"); - //let datasets = child_process.execSync("find ./ -name dataset_description.json", {encoding: "utf8"}).split("\n").filter(dataset=>{ + // let datasets = child_process.execSync("find ./ -name dataset_description.json", {encoding: "utf8"}).split("\n").filter(dataset=>{ const datasets = fs.readFileSync(process.argv[2], "utf8").split("\n").filter(dataset=>{ //ignore some datasets if(dataset.startsWith("datasets.datalad.org/openneuro")) return false; diff --git a/ui/src/project.vue b/ui/src/project.vue index 52eca641..bb0ccff3 100644 --- a/ui/src/project.vue +++ b/ui/src/project.vue @@ -274,6 +274,14 @@ {{project.stats.comments}} + + + + + From a3ff5292f92e639d18105b3051d661b3efa439ba Mon Sep 17 00:00:00 2001 From: bhatiadheeraj Date: Tue, 22 Aug 2023 22:21:01 +0000 Subject: [PATCH 3/4] updated api,models,and ui --- api/controllers/datalad.js | 7 ++++--- api/models.js | 29 +++++++++++++++++++++++------ bin/importdatalad.js | 35 +++++++++++++++++++++++++++-------- ui/src/project.vue | 26 ++++++++++++++++++++++++-- 4 files changed, 78 insertions(+), 19 deletions(-) diff --git a/api/controllers/datalad.js b/api/controllers/datalad.js index 0e554086..106cde28 100644 --- a/api/controllers/datalad.js +++ b/api/controllers/datalad.js @@ -74,10 +74,11 @@ router.post('/import/:dataset_id', common.jwt(), (req, res, next)=>{ if(project.members.includes(req.user.sub)) canedit = true; if(!canedit) return next("you can't import to this project"); - if(dataset.phenotype_files) { + if(dataset.phenotypes) { console.log("importing phenotype_files-----------------------------------------------------"); - project.phenotype_files = dataset.phenotype_files; + project.phenotypes = dataset.phenotypes; } + //update participants info let participants = new db.Participants({ project, @@ -86,7 +87,7 @@ router.post('/import/:dataset_id', common.jwt(), (req, res, next)=>{ subjects: dataset.participants, columns: dataset.participants_info, //might be missing }); - common.publish("participant.create."+req.user.sub+"."+project._id, participants); //too much data? + // common.publish("participant.create."+req.user.sub+"."+project._id, participants); //too much data? participants.save(); db.DLDatasets.updateOne({_id: dataset._id}, {$inc: {import_count: 1} }).then(err=>{ diff --git a/api/models.js b/api/models.js index 44bba5d0..3e9599c5 100644 --- a/api/models.js +++ b/api/models.js @@ -41,6 +41,11 @@ var projectSchema = mongoose.Schema({ avatar: String, //url for avatar + phenotypes: [{ + type: mongoose.Schema.Types.ObjectId, + ref: 'Phenotype' + }], + //access control //* private - only the project member can access //* public - accessible by anyone @@ -122,9 +127,6 @@ var projectSchema = mongoose.Schema({ //counts of publications (updated by common.update_project_stats) publications: Number, - //phenotype files - phenotype_files: [String], - groupanalysis: { sessions: [{ task_id: String, //amaretti task id @@ -808,6 +810,20 @@ ruleSchema.pre('save', function(next) { }); exports.Rules = mongoose.model('Rules', ruleSchema); +const phenotypeSchema = mongoose.Schema({ + name: String, + file: String, + sidecar: String, + columns: Object, + data: Array, + dldataset: { + type: mongoose.Schema.Types.ObjectId, + ref: 'DLDataset' + } +}); + +exports.Phenotype = mongoose.model('Phenotype', phenotypeSchema); + ////////////////////////////////////////////////////////////// // // datalad collections @@ -843,8 +859,10 @@ var dlDatasetSchema = mongoose.Schema({ ], participants_info: mongoose.Schema.Types.Mixed, //metadata for participants info - phenotypes_json: mongoose.Schema.Types.Mixed, - phenotype_files: [String], //list of files in phenotypes folder + phenotypes: [{ + type: mongoose.Schema.Types.ObjectId, + ref: 'Phenotype' + }], stats: { subjects: Number, @@ -887,4 +905,3 @@ var commentSchema = mongoose.Schema({ }); exports.Comments = mongoose.model('Comments', commentSchema); - diff --git a/bin/importdatalad.js b/bin/importdatalad.js index ce9c24f6..58fb8171 100755 --- a/bin/importdatalad.js +++ b/bin/importdatalad.js @@ -32,8 +32,8 @@ db.init(async err=>{ */ console.log("loading dataset_description.json"); - // let datasets = child_process.execSync("find ./ -name dataset_description.json", {encoding: "utf8"}).split("\n").filter(dataset=>{ - const datasets = fs.readFileSync(process.argv[2], "utf8").split("\n").filter(dataset=>{ + let datasets = child_process.execSync("find ./ -name dataset_description.json", {encoding: "utf8"}).split("\n").filter(dataset=>{ + // const datasets = fs.readFileSync(process.argv[2], "utf8").split("\n").filter(dataset=>{ //ignore some datasets if(dataset.startsWith("datasets.datalad.org/openneuro")) return false; if(dataset.startsWith("datasets.datalad.org/openfmri")) return false; @@ -112,7 +112,7 @@ async function load_datatypes() { function handle_bids(key, bids, cb) { //upsert dl-dataset record - db.DLDatasets.findOne(key, (err, dldataset)=>{ + db.DLDatasets.findOne(key, async (err, dldataset)=>{ if(err) return cb(err); if(!bids.dataset_description) return cb(); @@ -127,11 +127,31 @@ function handle_bids(key, bids, cb) { if(bids.dataset_description) dldataset.dataset_description = bids.dataset_description; if(bids.participants) dldataset.participants = bids.participants; if(bids.participants_json) dldataset.participants_info = bids.participants_json; - // storing combines json and folder files - if(bids.phenotypes_json) dldataset.phenotypes_json = bids.phenotypes_json; - - if(bids.phenotype_files && bids.phenotype_files.length) dldataset.phenotype_files = bids.phenotype_files; + if(bids.phenotypes) { + let phenotypeIds = []; + for (let phenotypeData of bids.phenotypes) { + // create a new Phenotype object with the current phenotype data + const phenotype = new db.Phenotype({ + name: phenotypeData.name, + file: phenotypeData.file, + sidecar: phenotypeData.sidecar, + columns: phenotypeData.columns, + data: phenotypeData.data, + dldataset: dldataset._id + }); + + // console.log(phenotype.data); + // process.exit(1); + + // save the Phenotype object + const savedPhenotype = await phenotype.save(); + phenotypeIds.push(savedPhenotype._id); + + } + dldataset.phenotypes = phenotypeIds; + } + console.log("phenotypes added"); //count let unique_subjects = []; let unique_sessions = []; @@ -162,7 +182,6 @@ function handle_bids(key, bids, cb) { dldataset.save(err=>{ if(err) throw err; - //handle each items async.eachSeries(bids.datasets, (item, next_dataset)=>{ item.dataset.datatype = datatype_ids[item.dataset.datatype]; diff --git a/ui/src/project.vue b/ui/src/project.vue index bb0ccff3..c0a6b002 100644 --- a/ui/src/project.vue +++ b/ui/src/project.vue @@ -278,7 +278,7 @@ @@ -397,6 +397,15 @@
+
+ + + {{ file }} + +
@@ -608,6 +617,19 @@ export default { sortedPapers : function() { return this.project.relatedPapers.sort((a,b)=> b.citationCount - a.citationCount ); }, + filteredPhenotypes: function() { + let filtered = []; + this.project.phenotypes.forEach(phenotype => { + filtered.push( + { + tsv : phenotype.file.replace("phenotype/", ""), + json : phenotype.sidecar.replace("phenotype/", ""), + // jsonContent: phenotype.columns + } + ); + }); + return filtered; + }, }, mounted() { @@ -745,7 +767,7 @@ export default { find: JSON.stringify({ _id: projectId, }), - populate: "stats.apps.app", + populate: "stats.apps.app phenotypes", }}).then(res=>{ if(res.data.projects.length == 0) { this.error = "You don't have access to the project, or the project ID is invalid. Please contact the project owner and ask to add you to the member/guest of the project."; From 237f746ea9b8c5febe8d032589b364f7a4555f95 Mon Sep 17 00:00:00 2001 From: bhatiadheeraj Date: Wed, 23 Aug 2023 19:49:53 +0000 Subject: [PATCH 4/4] added the tab to show phenotypes and also shows json for phenotypes.json --- ui/src/project.vue | 64 +++++++++++++++++++++++++++++++--------------- 1 file changed, 43 insertions(+), 21 deletions(-) diff --git a/ui/src/project.vue b/ui/src/project.vue index c0a6b002..3a9ed719 100644 --- a/ui/src/project.vue +++ b/ui/src/project.vue @@ -255,6 +255,15 @@ + + + + - - - - @@ -303,9 +305,19 @@ style="overflow: auto; max-height: 500px;"/>
+ +
+ + + {{ file.tsv }} + {{ file.json }} + + + +
-
+
App Usage @@ -356,7 +368,7 @@
-
+

We found the following journals/articles related to this project based on name/description @@ -365,7 +377,7 @@

-
+
Please login to Comments.
@@ -397,15 +409,7 @@

-
- - - {{ file }} - -
+
@@ -454,6 +458,11 @@ + + + +
@@ -461,6 +470,7 @@