Skip to content

[cli] Allow one or more positional path args #1967

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
jameshadfield wants to merge 1 commit into
base: master
Choose a base branch
from
+326 −199
Open

[cli] Allow one or more positional path args #1967

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 8 additions & 5 deletions cli/develop.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ const webpack = require("webpack");
const webpackDevMiddleware = require("webpack-dev-middleware");
const webpackHotMiddleware = require("webpack-hot-middleware");
const utils = require("./utils");
const { loadAndAddHandlers, serveRelativeFilepaths } = require("./view");
const view = require("./view");
const version = require('../src/version').version;
const chalk = require('chalk');
const generateWebpackConfig = require("../webpack.config.js").default;
Expand All @@ -21,8 +21,7 @@ const addParser = (parser) => {
subparser.addArgument('--verbose', {action: "storeTrue", help: "Print more verbose progress messages in the terminal."});
subparser.addArgument('--extend', {action: "store", metavar: "JSON", help: "Client customisations to be applied. See documentation for more details. Note that hot reloading does not currently work for these customisations."});
subparser.addArgument('--handlers', {action: "store", metavar: "JS", help: "Overwrite the provided server handlers for client requests. See documentation for more details."});
subparser.addArgument('--datasetDir', {metavar: "PATH", help: "Directory where datasets (JSONs) are sourced. This is ignored if you define custom handlers."});
subparser.addArgument('--narrativeDir', {metavar: "PATH", help: "Directory where narratives (Markdown files) are sourced. This is ignored if you define custom handlers."});
view.addDatasetNarrativePathArgs(subparser)
subparser.addArgument('--includeTiming', {action: "storeTrue", help: "Do not strip timing functions. With these included the browser console will print timing measurements for a number of functions."});

/* there are some options which we deliberately do not document via `--help`. See build.js for explanations. */
Expand All @@ -31,6 +30,8 @@ const addParser = (parser) => {


const run = (args) => {
const dataPaths = view.processPathArguments(args)

/* Basic server set up */
const app = express();
app.set('port', process.env.PORT || 4000);
Expand Down Expand Up @@ -69,9 +70,11 @@ const run = (args) => {

let handlerMsg = "";
if (args.gh_pages) {
handlerMsg = serveRelativeFilepaths({app, dir: path.resolve(args.gh_pages)});
handlerMsg = view.serveRelativeFilepaths({app, dir: path.resolve(args.gh_pages)});
} else if (args.handlers) {
handlerMsg = view.customRouteHandlers(app, path.resolve(args.handlers));
} else {
handlerMsg = loadAndAddHandlers({app, handlersArg: args.handlers, datasetDir: args.datasetDir, narrativeDir: args.narrativeDir});
handlerMsg = view.defaultRouteHandlers({app, dataPaths});
}

app.get("*", (req, res) => res.redirect("/"));
Expand Down
159 changes: 96 additions & 63 deletions cli/server/getAvailable.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,94 +1,104 @@
const utils = require("../utils");
const fs = require('fs');
const path = require("path");
const { promisify } = require('util');
const { findAvailableSecondTreeOptions } = require('./getDatasetHelpers');

const readdir = promisify(fs.readdir);

const getAvailableDatasets = async (localDataPath) => {
const getAvailableDatasets = (dir, files) => {
const datasets = [];
/* NOTE: if there are v1 & v2 files with the same name the v2 JSON is
* preferentially fetched. E.g. if `zika.json`, `zika_meta.json` and
* `zika_tree.json` exist then only `zika.json` is viewable in auspice.
*/
try {
const files = await readdir(localDataPath);
/* v2 files -- match JSONs not ending with `_tree.json`, `_meta.json`,
`_tip-frequencies.json`, `_seq.json` */
const v2Files = files.filter((file) => (
file.endsWith(".json") &&
!file.includes("manifest") &&
!file.endsWith("_tree.json") &&
!file.endsWith("_meta.json") &&
!file.endsWith("_tip-frequencies.json") &&
!file.endsWith("_root-sequence.json") &&
!file.endsWith("_seq.json") &&
!file.endsWith("_measurements.json")
))

/* v2 files -- match JSONs not ending with `_tree.json`, `_meta.json`,
`_tip-frequencies.json`, `_seq.json` */
const v2Files = files.filter((file) => (
file.endsWith(".json") &&
!file.includes("manifest") &&
!file.endsWith("_tree.json") &&
!file.endsWith("_meta.json") &&
!file.endsWith("_tip-frequencies.json") &&
!file.endsWith("_root-sequence.json") &&
!file.endsWith("_seq.json") &&
!file.endsWith("_measurements.json")
))
.map((file) => file
.replace(".json", "")
.split("_")
.join("/")
);

v2Files.forEach((filepath) => {
datasets.push({
request: filepath,
v2: true,
secondTreeOptions: findAvailableSecondTreeOptions(filepath, v2Files),
fileType: 'dataset',
dir,
});
});

/* v1 files -- match files ending with `_tree.json` */
const v1Files = files
.filter((file) => file.endsWith("_tree.json"))
.map((file) => file
.replace(".json", "")
.replace("_tree.json", "")
.split("_")
.join("/")
);

v2Files.forEach((filepath) => {
datasets.push({
request: filepath,
v2: true,
secondTreeOptions: findAvailableSecondTreeOptions(filepath, v2Files)
});
});

/* v1 files -- match files ending with `_tree.json` */
const v1Files = files
.filter((file) => file.endsWith("_tree.json"))
.map((file) => file
.replace("_tree.json", "")
.split("_")
.join("/")
);

v1Files.forEach((filepath) => {
datasets.push({
request: filepath,
v2: false,
secondTreeOptions: findAvailableSecondTreeOptions(filepath, v1Files)
});
v1Files.forEach((filepath) => {
datasets.push({
request: filepath,
v2: false,
secondTreeOptions: findAvailableSecondTreeOptions(filepath, v1Files),
fileType: 'dataset',
dir,
});
} catch (err) {
utils.warn(`Couldn't collect available dataset files (path searched: ${localDataPath})`);
utils.verbose(err);
}
});
return datasets;
};

const getAvailableNarratives = async (localNarrativesPath) => {
let narratives = [];
try {
narratives = await readdir(localNarrativesPath);
narratives = narratives
.filter((file) => file.endsWith(".md") && file!=="README.md")
.map((file) => file.replace(".md", ""))
.map((file) => file.split("_").join("/"))
.map((filepath) => `narratives/${filepath}`)
.map((filepath) => ({request: filepath}));
} catch (err) {
utils.warn(`Couldn't collect available narratives (path searched: ${localNarrativesPath})`);
utils.verbose(err);
}
return narratives;
const getAvailableNarratives = (dir, files) => {
return files
.filter((file) => file.endsWith(".md") && file!=="README.md")
.map((file) => ({
fullPath: path.join(dir, file),
request: 'narratives/' + file.replace(".md", "").split("_").join("/"),
fileType: 'narrative',
}));
};

const setUpGetAvailableHandler = ({datasetsPath, narrativesPath}) => {
const setUpGetAvailableHandler = (dataPaths) => {
/* return Auspice's default handler for "/charon/getAvailable" requests.
* Remember that auspice itself only serves local files.
* Servers often use their own handler instead of this.
*/
return async (req, res) => {
utils.log("GET AVAILABLE returning locally available datasets & narratives");
const datasets = await getAvailableDatasets(datasetsPath);
const narratives = await getAvailableNarratives(narrativesPath);
res.json({datasets, narratives});
let resources = []
for (const [p, dataTypes] of Object.entries(dataPaths)) {
try {
// FUTURE TODO - recurse into subfolders (readdir can do this natively)
const files = await readdir(p);
if (dataTypes.has('datasets')) {
resources.push(...getAvailableDatasets(p, files))
}
if (dataTypes.has('narratives')) {
resources.push(...getAvailableNarratives(p, files))
}
} catch (err) {
utils.warn(`Couldn't collect files (path searched: ${p})`);
utils.verbose(err);
}
}
resources = unique(resources)

/* convert to the expected API response structure */
res.json(availableResponseStructure(resources));
};

};
Expand All @@ -97,5 +107,28 @@ const setUpGetAvailableHandler = ({datasetsPath, narrativesPath}) => {
module.exports = {
setUpGetAvailableHandler,
getAvailableDatasets,
getAvailableNarratives
getAvailableNarratives,
};

function availableResponseStructure(resources) {
const datasets = resources.filter((r) => r.fileType==='dataset')
.map((r) => ({request: r.request, buildUrl: r.buildUrl, secondTreeOptions: r.secondTreeOptions}));
const narratives = resources.filter((r) => r.fileType==='narrative')
.map((r) => ({request: r.request}));
return {datasets, narratives};
}

/**
* First match wins
*/
function unique(resources) {
const seen = {dataset: new Set(), narrative: new Set()};
return resources.filter((r) => {
if (seen[r.fileType].has(r.request)) {
// utils.verbose(`Dropping duplicate ${r.request} from available`); // Too verbose!
return false;
}
seen[r.fileType].add(r.request);
return true;
})
}
67 changes: 56 additions & 11 deletions cli/server/getDataset.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,24 +1,69 @@
const { promisify } = require('util');
const path = require("path");
const fs = require('fs');
const getAvailable = require("./getAvailable");
const helpers = require("./getDatasetHelpers");
const utils = require("../utils");

const setUpGetDatasetHandler = ({datasetsPath}) => {
const readdir = promisify(fs.readdir);

const setUpGetDatasetHandler = (dataPaths) => {
return async (req, res) => {
const allAvailable = []

let requestInfo;
try {
const availableDatasets = await getAvailable.getAvailableDatasets(datasetsPath);
const info = helpers.interpretRequest(req, datasetsPath);
const redirected = helpers.redirectIfDatapathMatchFound(res, info, availableDatasets);
if (redirected) return;
helpers.makeFetchAddresses(info, datasetsPath, availableDatasets);
await helpers.sendJson(res, info);
requestInfo = helpers.interpretRequest(req);
} catch (err) {
console.trace(err);
// Throw 404 when not available
const errorCode = err.message.endsWith("not in available datasets") ? 404 : 500;
return helpers.handleError(res, `couldn't fetch JSONs`, err.message, errorCode);
return helpers.handleError(res, err.message, err.message, 400);
}

/**
* Iterate through the dataPaths and return the first match we find
* (this "first match wins" approach mirrors that of `getAvailable`)
*/
for (const [p, dataTypes] of Object.entries(dataPaths)) {
if (!dataTypes.has('datasets')) continue;
try {
const files = await readdir(p);
const available = getAvailable.getAvailableDatasets(p, files);
const dataset = match(available, requestInfo);
if (dataset) {
requestInfo.address = dataset.v2 ?
path.join(dataset.dir, `${requestInfo.parts.join("_")}.json`) :
({
meta: path.join(dataset.dir, `${requestInfo.parts.join("_")}_meta.json`),
tree: path.join(dataset.dir, `${requestInfo.parts.join("_")}_tree.json`)
});
return await helpers.sendJson(res, requestInfo);
}
allAvailable.push(...available);
} catch (err) {
utils.warn(`Error reading datasets from ${p}: ${err.message}`)
}
}

/* No perfect match - attempt to find a close one... */
const closeMatchRequest = helpers.closestMatch(requestInfo.parts, allAvailable);
if (closeMatchRequest) {
utils.verbose(`Redirecting request to ${closeMatchRequest}`);
return res.redirect(`./getDataset?prefix=/${closeMatchRequest}`);
}

return helpers.handleError(res, `no matching dataset`, `no matching dataset`, 404);
};
};

function match(resources, requestInfo) {
for (const resource of resources) {
if (
(resource.fileType === requestInfo.dataType) &&
(resource.request === requestInfo.parts.join("/"))
) return resource;
}
return false;
}


module.exports = {
setUpGetDatasetHandler
Expand Down
Loading
Loading