diff --git a/package.json b/package.json index 430c55f5..cd10a797 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@macrostrat/api-v2", - "version": "2.1.8", + "version": "2.2.0", "description": "An API for stratigraphic and geological information (Version 2).", "main": "server.ts", "repository": { diff --git a/server.ts b/server.ts index 916d18f3..9115f914 100755 --- a/server.ts +++ b/server.ts @@ -2,48 +2,48 @@ const dotenv = require("dotenv"); // Load environment variables from .env file dotenv.config(); +import { buildAPI } from "./v2"; + var express = require("express"), - bodyParser = require("body-parser"), - //v1 = require("./v1"), - v2 = require("./v2"), - defs = require("./v2/defs"), - app = express(); + bodyParser = require("body-parser"); +//defs = require("./v2/defs"), -// parse application/x-www-form-urlencoded -app.use(bodyParser.urlencoded({ extended: false })); +//TODO: update port to designated env. +const listenPort = process.argv[2] ?? process.env.PORT ?? 5000; -// parse application/json -app.use(bodyParser.json()); +async function runServer() { + const app = express(); + // parse application/x-www-form-urlencoded + app.use(bodyParser.urlencoded({ extended: false })); -// parse application/vnd.api+json as json -app.use(bodyParser.json({ type: "application/vnd.api+json" })); + // parse application/json + app.use(bodyParser.json()); -// Load and prefix all routes with /api and appropriate version -app.use("/v2", v2); + // parse application/vnd.api+json as json + app.use(bodyParser.json({ type: "application/vnd.api+json" })); -app.route("/v1*").get(function (req, res, next) { - res.status(410).send({ - error: - "Macrostrat's v1 API has been retired. Please update your usage to newer endpoints.", - }); -}); + const v2 = await buildAPI(); -// If no version specified, fall back to more current -app.use("/", v2); + // Load and prefix all routes with /api and appropriate version + app.use("/v2", v2); -app.set("json spaces", 2); + app.route("/v1*").get(function (req, res, next) { + res.status(410).send({ + error: + "Macrostrat's v1 API has been retired. Please update your usage to newer endpoints.", + }); + }); -//TODO: update port to designated env. -app.port = process.argv[2] ?? process.env.PORT ?? 5000; + // If no version specified, fall back to more current + app.use("/", v2); -app.start = function () { - app.listen(app.port, function () { - console.log("Listening on port " + app.port); - }); -}; + app.set("json spaces", 2); -if (!module.parent) { - app.start(); + app.listen(listenPort, function () { + console.log("Listening on port " + listenPort); + }); } -module.exports = app; +runServer().catch((error) => { + console.error("Failed to start server:", error); +}); diff --git a/v2/column-cache-refresh.ts b/v2/column-cache-refresh.ts index 533815b7..91aeed2e 100644 --- a/v2/column-cache-refresh.ts +++ b/v2/column-cache-refresh.ts @@ -1,13 +1,11 @@ -const larkin = require("./larkin"); - module.exports = (req, res, next) => { if ( req.query && req.query.cacheRefreshKey && req.query.cacheRefreshKey === process.env.CACHE_REFRESH_KEY ) { - larkin.setupCache(); - res.json({ success: "cache refreshed" }); + res.status(404); + res.json({ fail: "internal column cache refresh is no longer supported" }); } else { res.status(401); res.json({ fail: "you do not have permissions to execute this action" }); diff --git a/v2/columns.ts b/v2/columns.ts index 8a10b3ef..1591865c 100644 --- a/v2/columns.ts +++ b/v2/columns.ts @@ -1,3 +1,5 @@ +import { handleUnitsRoute } from "./units"; + var api = require("./api"), async = require("async"), dbgeo = require("dbgeo"), @@ -12,24 +14,9 @@ module.exports = function (req, res, next, callback) { async.waterfall( [ - // First pass the request to the /units route and get the long response function (callback) { - if ("all" in req.query) { - larkin.cache.fetch("unitSummary", function (data) { - callback(null, data); - }); - } else { - callback(null, null); - } - }, - - function (data, callback) { - if (data) { - return callback(null, data); - } - //call units to group units by col_id - require("./units")(req, null, null, function (error, result) { + handleUnitsRoute(req, null, null, function (error, result) { if (error) { callback(error); } @@ -122,19 +109,22 @@ module.exports = function (req, res, next, callback) { return callback(null, null, []); } - if ("all" in req.query) { - if (req.query.format && api.acceptedFormats.geo[req.query.format]) { - larkin.cache.fetch("columnsGeom", function (data) { - callback(null, new_cols, data); - }); - } else { - larkin.cache.fetch("columnsNoGeom", function (data) { - callback(null, new_cols, data); - }); - } - } else { - callback(null, new_cols, null); - } + callback(null, new_cols, null); + + // TODO: this breaks "all" filtering and brings down the API + // if ("all" in req.query) { + // if (req.query.format && api.acceptedFormats.geo[req.query.format]) { + // larkin.cache.fetch("columnsGeom", function (data) { + // callback(null, new_cols, data); + // }); + // } else { + // larkin.cache.fetch("columnsNoGeom", function (data) { + // callback(null, new_cols, data); + // }); + // } + // } else { + // callback(null, new_cols, null); + // } }, // Using the unique column IDs returned from units, query columns diff --git a/v2/definitions/columns.ts b/v2/definitions/columns.ts index ad3f7928..dd0beb1a 100644 --- a/v2/definitions/columns.ts +++ b/v2/definitions/columns.ts @@ -2,6 +2,8 @@ var api = require("../api"); var larkin = require("../larkin"); var dbgeo = require("dbgeo"); +import { buildProjectsFilter } from "../utils"; + module.exports = function (req, res, next, cb) { if (Object.keys(req.query).length < 1) { return larkin.info(req, res, next); @@ -44,17 +46,23 @@ module.exports = function (req, res, next, cb) { where.push("cols.col_name = ANY(:col_name)"); params["col_name"] = larkin.parseMultipleStrings(req.query.col_name); } - if (req.query.project_id) { - where.push("cols.project_id = ANY(:project_id)"); - params["project_id"] = larkin.parseMultipleIds(req.query.project_id); - } + + const [whereClauses, projectParams] = buildProjectsFilter( + req, + "cols.project_id", + ); + where = where.concat(whereClauses); + Object.assign(params, projectParams); + + where.push("status_code = ANY(:status_code)"); if (req.query.status_code || req.query.status) { // `status` parameter still works but has been superseded by `status_code` // multiple status codes can be provided - where.push("status_code = ANY(:status_code)"); params["status_code"] = larkin.parseMultipleIds( req.query.status_code ?? req.query.status, ); + } else { + params["status_code"] = ["active"]; } if (where.length) { diff --git a/v2/definitions/groups.ts b/v2/definitions/groups.ts index 3ef5755f..80df5cd0 100644 --- a/v2/definitions/groups.ts +++ b/v2/definitions/groups.ts @@ -1,6 +1,8 @@ var api = require("../api"), larkin = require("../larkin"); +import { buildProjectsFilter } from "../utils"; + module.exports = function (req, res, next, cb) { if (Object.keys(req.query).length < 1) { return larkin.info(req, res, next); @@ -18,15 +20,16 @@ module.exports = function (req, res, next, cb) { if (req.query.col_group_id) { where.push("col_groups.id = ANY(:col_group_id)"); params["col_group_id"] = larkin.parseMultipleIds(req.query.col_group_id); - } else if (req.query.project_id) { - where.push("cols.project_id = ANY(:project_id)"); - params["project_id"] = larkin.parseMultipleIds(req.query.project_id); - } else if (req.query.col_id) { - where.push("cols.id = ANY(:col_id)"); - params["col_id"] = larkin.parseMultipleIds(req.query.col_id); } - where = where.length ? "WHERE " + where.join(" AND ") : ""; + const [projectWhereClauses, projectParams] = buildProjectsFilter( + req, + "cols.project_id", + ); + where = where.concat(projectWhereClauses); + Object.assign(params, projectParams); + + const whereClause = where.length ? "WHERE " + where.join(" AND ") : ""; let sql = `SELECT col_groups.id AS col_group_id, col_group, @@ -37,7 +40,7 @@ module.exports = function (req, res, next, cb) { FROM macrostrat.col_groups LEFT JOIN macrostrat.cols ON cols.col_group_id = col_groups.id LEFT JOIN macrostrat.units_sections ON units_sections.col_id = cols.id - ${where} + ${whereClause} GROUP BY col_groups.id, cols.project_id `; if ("sample" in req.query) { diff --git a/v2/definitions/projects.ts b/v2/definitions/projects.ts index f22c2ae2..2d9e437b 100644 --- a/v2/definitions/projects.ts +++ b/v2/definitions/projects.ts @@ -7,62 +7,83 @@ module.exports = function (req, res, next, cb) { } //There will be a discrepancy with a key in production. Updated in_proccess_cols to in_process_cols key. Values are //still the same. - var sql = `WITH in_proc AS ( - SELECT COUNT(DISTINCT id) AS c, project_id - FROM macrostrat.cols - WHERE status_code = 'in process' - GROUP BY project_id - ), - obs AS ( - SELECT COUNT(DISTINCT id) AS co, project_id - FROM macrostrat.cols - WHERE status_code = 'obsolete' - GROUP BY project_id - ), - col_area_sum AS ( - SELECT project_id, SUM(col_area) AS total_area - FROM macrostrat.cols - WHERE status_code = 'active' - GROUP BY project_id - ) - - SELECT - projects.id AS project_id, - projects.project, - projects.descrip, - projects.timescale_id, - COUNT(DISTINCT units_sections.col_id)::integer AS t_cols, - COALESCE(c, 0)::integer AS in_process_cols, - COALESCE(co, 0)::integer AS obsolete_cols, - COUNT(DISTINCT units_sections.unit_id)::integer AS t_units, - COALESCE(ROUND(total_area), 0)::integer AS area - - FROM macrostrat.projects - LEFT JOIN macrostrat.cols ON projects.id = cols.project_id - LEFT JOIN macrostrat.units_sections ON units_sections.col_id = cols.id - LEFT JOIN in_proc USING (project_id) - LEFT JOIN obs USING (project_id) - LEFT JOIN col_area_sum ON projects.id = col_area_sum.project_id - `; - var where = []; - var params = {}; + const where = []; + let params = {}; if (req.query.project_id) { where.push("projects.id = ANY(:project_id)"); params["project_id"] = larkin.parseMultipleIds(req.query.project_id); } - if (where.length) { - sql += ` WHERE ${where.join(" AND ")}`; + + const whereStatement = where.length > 0 ? where.join(" AND ") : "true"; + + let sql = ` + SELECT + p.id AS project_id, + p.project, + p.descrip, + p.timescale_id, + count(DISTINCT units_sections.col_id)::integer AS t_cols, + count(DISTINCT cols.id) FILTER ( WHERE cols.status_code = 'active' )::integer AS active_cols, + count(DISTINCT cols.id) FILTER ( WHERE cols.status_code = 'in process' )::integer AS in_process_cols, + count(DISTINCT cols.id) FILTER ( WHERE cols.status_code = 'obsolete' )::integer AS obsolete_cols, + count(DISTINCT units_sections.unit_id)::integer AS t_units, + coalesce(round(sum(DISTINCT cols.col_area) FILTER ( WHERE cols.status_code = 'active')), 0) AS area + FROM macrostrat.projects p + LEFT JOIN macrostrat.cols ON p.id = cols.project_id + LEFT JOIN macrostrat.units_sections ON units_sections.col_id = cols.id + WHERE ${whereStatement} + GROUP BY + p.id, + p.project, + p.descrip, + p.timescale_id + `; + + if (larkin.hasCapability("composite-projects")) { + /** Progressive enhancement for composite projects **/ + sql = ` + WITH composite_tree AS ( + SELECT pt.parent_id, array_agg(pt.child_id) children, jsonb_agg(to_jsonb(p)) AS members + FROM macrostrat.projects_tree pt + JOIN LATERAL ( + SELECT p.id, p.slug, p.project name + FROM macrostrat.projects p + WHERE p.id = pt.child_id + ) AS p ON true + GROUP BY pt.parent_id + ) + SELECT + p.id AS project_id, + p.slug, + p.project, + p.descrip, + p.timescale_id, + ct.members, + count(DISTINCT units_sections.col_id)::integer AS t_cols, + count(DISTINCT cols.id) FILTER ( WHERE cols.status_code = 'active' )::integer AS active_cols, + count(DISTINCT cols.id) FILTER ( WHERE cols.status_code = 'in process' )::integer AS in_process_cols, + count(DISTINCT cols.id) FILTER ( WHERE cols.status_code = 'obsolete' )::integer AS obsolete_cols, + count(DISTINCT units_sections.unit_id)::integer AS t_units, + coalesce(round(sum(DISTINCT cols.col_area) FILTER ( WHERE cols.status_code = 'active')), 0) AS area + FROM macrostrat.projects p + LEFT JOIN composite_tree ct + ON ct.parent_id = p.id + LEFT JOIN macrostrat.cols ON p.id = cols.project_id + OR (p.is_composite AND cols.project_id = ANY(ct.children)) + LEFT JOIN macrostrat.units_sections ON units_sections.col_id = cols.id + WHERE ${whereStatement} + GROUP BY + p.id, + p.project, + p.descrip, + p.timescale_id, + p.slug, + ct.children, + ct.members + `; } - sql += `\nGROUP BY - projects.id, - projects.project, - projects.descrip, - projects.timescale_id, - c, - co, - total_area;`; larkin.queryPg("burwell", sql, params, function (error, data) { if (error) { diff --git a/v2/fossils.ts b/v2/fossils.ts index b37a5110..4bf32473 100644 --- a/v2/fossils.ts +++ b/v2/fossils.ts @@ -1,8 +1,12 @@ +import { handleUnitsRoute } from "./units"; + var api = require("./api"), dbgeo = require("dbgeo"), async = require("async"), larkin = require("./larkin"); +import { buildProjectsFilter } from "./utils"; + module.exports = function (req, res, next) { if (Object.keys(req.query).length < 1) { return larkin.info(req, res, next); @@ -57,7 +61,7 @@ module.exports = function (req, res, next) { req.query.econ_type || req.query.econ_class ) { - require("./units")(req, null, null, function (error, result) { + handleUnitsRoute(req, null, null, function (error, result) { if (error) { callback(error); } @@ -122,12 +126,15 @@ module.exports = function (req, res, next) { ); } - if (req.query.project_id) { - where += " AND cols.project_id = ANY(:project_ids)"; - params["project_ids"] = larkin.parseMultipleIds( - req.query.project_id, - ); - } + const [projectWhereClauses, projectParams] = buildProjectsFilter( + req, + "cols.project_id", + ); + where += projectWhereClauses.length + ? " AND " + projectWhereClauses.join(" AND ") + : ""; + Object.assign(params, projectParams); + //TODO there is no pbdb table, so I removed LEFT JOIN pbdb.occ_matrix ON pbdb.coll_matrix.collection_no = pbdb.occ_matrix.collection_no //I also removed LEFT JOIN pbdb.taxon_lower ON pbdb.occ_matrix.orig_no = pbdb.taxon_lower.orig_no //removed JOIN pbdb.coll_matrix ON pbdb_matches.collection_no = pbdb.coll_matrix.collection_no diff --git a/v2/index.ts b/v2/index.ts index a5f0d67d..63cb628b 100644 --- a/v2/index.ts +++ b/v2/index.ts @@ -1,108 +1,108 @@ -var api = require("./api"), - larkin = require("./larkin"); +const api = require("./api"); +const larkin = require("./larkin"); -// Establish a connection to the database -//larkin.connectMySQL(); +import { handleUnitsRoute } from "./units"; -// Set up the column and unit cache -larkin.setupCache(); +export async function buildAPI() { + await larkin.checkCapabilities(api); -// Load route categories -api.use("/carto", require("./carto")); -api.use("/defs", require("./definitions")); -api.use("/grids", require("./grids")); -api.use("/mobile", require("./mobile")); + // Load route categories + api.use("/carto", require("./carto")); + api.use("/defs", require("./definitions")); + api.use("/grids", require("./grids")); + api.use("/mobile", require("./mobile")); -api.route("/").get(require("./root")); + api.route("/").get(require("./root")); -api.route("/meta").get(require("./meta")); + api.route("/meta").get(require("./meta")); -api.route("/changes").get(function (req, res, next) { - res.sendFile(__dirname + "/changes.html"); -}); + api.route("/changes").get(function (req, res, next) { + res.sendFile(__dirname + "/changes.html"); + }); -api.route("/columns/refresh-cache").get(require("./column-cache-refresh")); + api.route("/columns/refresh-cache").get(require("./column-cache-refresh")); -api.route("/columns").get(function (req, res, next) { - require("./columns")(req, res, next); -}); + api.route("/columns").get(function (req, res, next) { + require("./columns")(req, res, next); + }); -api.route("/sections").get(require("./sections")); + api.route("/sections").get(require("./sections")); -api.route("/units").get(function (req, res, next) { - require("./units")(req, res, next); -}); + api.route("/units").get(function (req, res, next) { + handleUnitsRoute(req, res, next); + }); -api.route("/fossils").get(require("./fossils")); + api.route("/fossils").get(require("./fossils")); -api.route("/stats").get(require("./stats")); + api.route("/stats").get(require("./stats")); -api.route("/paleogeography").get(require("./paleogeography")); + api.route("/paleogeography").get(require("./paleogeography")); -api.route("/geologic_units/gmna").get(require("./geologic_units_gmna")); + api.route("/geologic_units/gmna").get(require("./geologic_units_gmna")); -api.route("/geologic_units/gmus").get(require("./geologic_units_gmus")); + api.route("/geologic_units/gmus").get(require("./geologic_units_gmus")); -api.route("/geologic_units/burwell").get(function (req, res, next) { - require("./geologic_units_burwell")(req, res, next); -}); + api.route("/geologic_units/burwell").get(function (req, res, next) { + require("./geologic_units_burwell")(req, res, next); + }); -api.route("/geologic_units/map").get(function (req, res, next) { - require("./geologic_units_burwell")(req, res, next); -}); + api.route("/geologic_units/map").get(function (req, res, next) { + require("./geologic_units_burwell")(req, res, next); + }); -api - .route("/geologic_units/burwell/nearby") - .get(require("./geologic_units_burwell_nearby")); + api + .route("/geologic_units/burwell/nearby") + .get(require("./geologic_units_burwell_nearby")); -api - .route("/geologic_units/map/nearby") - .get(require("./geologic_units_burwell_nearby")); + api + .route("/geologic_units/map/nearby") + .get(require("./geologic_units_burwell_nearby")); -api - .route("/geologic_units/burwell/points") - .get(require("./geologic_units_burwell_points")); + api + .route("/geologic_units/burwell/points") + .get(require("./geologic_units_burwell_points")); -api - .route("/geologic_units/map/points") - .get(require("./geologic_units_burwell_points")); + api + .route("/geologic_units/map/points") + .get(require("./geologic_units_burwell_points")); -api.route("/geologic_units/map/legend").get(function (req, res, next) { - require("./geologic_units_burwell_legend")(req, res, next); -}); + api.route("/geologic_units/map/legend").get(function (req, res, next) { + require("./geologic_units_burwell_legend")(req, res, next); + }); -api.route("/elevation").get(function (req, res, next) { - require("./elevation")(req, res, next); -}); + api.route("/elevation").get(function (req, res, next) { + require("./elevation")(req, res, next); + }); -api.route("/places").get(function (req, res, next) { - require("./places")(req, res, next); -}); + api.route("/places").get(function (req, res, next) { + require("./places")(req, res, next); + }); -api.route("/measurements").get(require("./measurements")); + api.route("/measurements").get(require("./measurements")); -api.route("/age_model").get(require("./age_model")); + api.route("/age_model").get(require("./age_model")); -api.route("/eodp").get(require("./eodp")); -//api.route("/hillshade") -// .get(require("./hillshade")); + api.route("/eodp").get(require("./eodp")); + //api.route("/hillshade") + // .get(require("./hillshade")); -api.route("/boundaries").get(require("./boundaries")); + api.route("/boundaries").get(require("./boundaries")); -api.route("/hex-summary").get(require("./hex_summary")); + api.route("/hex-summary").get(require("./hex_summary")); -api.route("/hex-summary/max/:zoom").get(require("./hex_summary_max")); + api.route("/hex-summary/max/:zoom").get(require("./hex_summary_max")); -api.route("*").get(require("./catchall")); + api.route("*").get(require("./catchall")); -api.use(function (err, req, res, next) { - if (err.status !== 404) { - return next(); - } else if (err.status === 404) { - larkin.error(req, res, next, "404: Page not found", 404); - } else { - larkin.error(req, res, next, "500: Internal Server Error", 500); - } -}); + api.use(function (err, req, res, next) { + if (err.status !== 404) { + return next(); + } else if (err.status === 404) { + larkin.error(req, res, next, "404: Page not found", 404); + } else { + larkin.error(req, res, next, "500: Internal Server Error", 500); + } + }); -module.exports = api; + return api; +} diff --git a/v2/larkin.ts b/v2/larkin.ts index 99198450..f1f43b76 100644 --- a/v2/larkin.ts +++ b/v2/larkin.ts @@ -10,19 +10,39 @@ var async = require("async"), const named = require("yesql").pg; const { Client, Pool } = require("pg"); +enum APICapability { + COMPOSITE_PROJECTS = "composite-projects", +} + (function () { var larkin: any = {}; // Store a global mapping of connection pools, so we don't overload PG with connections const connectionPoolStore: { [key: string]: typeof Pool } = {}; - larkin.trace = function (...args: any[]) { + larkin.trace = function (type: string, ...args: any[]) { if (credentials.debug === false) { return; } console.log(...args); }; + larkin.queryPgAsync = function ( + db: string, + sql: string, + params: any = {}, + ): Promise { + return new Promise((resolve, reject) => { + larkin.queryPg(db, sql, params, (err: any, result: any) => { + if (err) { + reject(err); + } else { + resolve(result); + } + }); + }); + }; + //added new method to query from Maria data in the new PG database after migration larkin.queryPg = function (db, sql, params, callback) { //add console.logs for debug mode in the future @@ -128,6 +148,14 @@ const { Client, Pool } = require("pg"); }); }; + larkin.capabilities = new Set([]); + + larkin.hasCapability = function (capability: APICapability) { + /** Check to see if the API framework/database supports a given capability. + * This is not exposed to clients and is used for eventual consistency. */ + return larkin.capabilities.has(capability); + }; + larkin.toUnnamed = function (sql, params) { var placeholders = sql.match( /(?:\?)|(?::(\d+|(?::?[a-zA-Z][a-zA-Z0-9_]*)))/g, @@ -267,8 +295,19 @@ const { Client, Pool } = require("pg"); } }; - larkin.log = function (type, message) { - larkin.trace(type, message); + larkin.log = function (type, obj) { + let message = obj; + if (typeof obj === "object" && "message" in obj) { + message = obj.message; + } + if (type === "error") { + console.error("ERROR:", message); + } else if (type === "warning") { + console.warn("WARNING:", message); + } else if (type === "info") { + console.info("INFO", message); + } + larkin.trace(obj); }; // Will return all field definitions @@ -626,215 +665,49 @@ const { Client, Pool } = require("pg"); On API startup cache the following: + All columns (with geometries) + All columns (without geometries) - + A summary of all units To refresh these caches without restarting the API an HTTP request can be made to /columns/refresh-cache?cacheRefreshKey= with the value of exports.cacheRefreshKey found in ./credentials.js */ - larkin.setupCache = function () { - async.parallel( - { - unitSummary: function (callback) { - //get all units and summarize for columns - http.get( - //TODO: change url to match env. - "http://0.0.0.0:5000/v2/units?all&response=long", - function (res) { - var body = ""; - res.on("data", function (chunk) { - body += chunk; - }); - res.on("end", function () { - try { - var parsedBody = JSON.parse(body); - // Process the JSON as needed - } catch (e) { - console.error("Failed to parse JSON:", e); - console.error("Response body:", body); // Log the actual body for debugging - } - - if ( - parsedBody && - parsedBody.success && - parsedBody.success.data - ) { - var result = parsedBody.success.data; - } else { - console.error("Invalid response body:", body); - } - var cols = _.groupBy(result, function (d) { - return d.col_id; - }); - - var new_cols = {}; - - Object.keys(cols).forEach(function (col_id) { - new_cols[col_id] = { - max_thick: _.reduce( - cols[col_id].map(function (d) { - return d.max_thick; - }), - function (a, b) { - return a + b; - }, - 0, - ), - max_min_thick: _.reduce( - cols[col_id].map(function (d) { - if (d.min_thick === 0) { - return d.max_thick; - } else { - return d.min_thick; - } - }), - function (a, b) { - return a + b; - }, - 0, - ), - min_min_thick: _.reduce( - cols[col_id].map(function (d) { - return d.min_thick; - }), - function (a, b) { - return a + b; - }, - 0, - ), - - b_age: _.max(cols[col_id], function (d) { - return d.b_age; - }).b_age, - t_age: _.min(cols[col_id], function (d) { - return d.t_age; - }).t_age, - b_int_name: _.max(cols[col_id], function (d) { - return d.b_age; - }).b_int_name, - t_int_name: _.min(cols[col_id], function (d) { - return d.t_age; - }).t_int_name, - - pbdb_collections: _.reduce( - cols[col_id].map(function (d) { - return d.pbdb_collections; - }), - function (a, b) { - return a + b; - }, - 0, - ), - - lith: larkin.summarizeAttribute(cols[col_id], "lith"), - environ: larkin.summarizeAttribute(cols[col_id], "environ"), - econ: larkin.summarizeAttribute(cols[col_id], "econ"), - - t_units: cols[col_id].length, - t_sections: _.uniq( - cols[col_id].map(function (d) { - return d.section_id; - }), - ).length, - }; - }); - callback(null, new_cols); - }); - }, - ); - }, - columnsGeom: function (callback) { - // get all columns, with geometry - larkin.queryPg( - "burwell", - ` - SELECT - cols.id AS col_id, - col_name, - col_group, - col_groups.id AS col_group_id, - col AS group_col_id, - round(cols.col_area::numeric, 1) AS col_area, - cols.project_id, - string_agg(col_refs.ref_id::varchar, '|') AS refs, - ST_AsGeoJSON(col_areas.col_area) geojson - FROM macrostrat.cols - LEFT JOIN macrostrat.col_areas on col_areas.col_id = cols.id - LEFT JOIN macrostrat.col_groups ON col_groups.id = cols.col_group_id - LEFT JOIN macrostrat.col_refs ON cols.id = col_refs.col_id - WHERE status_code = 'active' - AND col_areas.col_area IS NOT NULL - GROUP BY col_areas.col_id, cols.id, col_groups.col_group, col_groups.id, col_areas.col_area - `, - [], - function (error, result) { - if (!error && result && result.rows) { - callback(null, result.rows); - } else { - larkin.trace( - "Could not set up column cache. Check postgres connection", - ); - callback(null); - } - }, - ); - }, + larkin.checkCapabilities = async function (): Promise> { + console.log("Checking API capabilities..."); + // COMPOSITE PROJECTS + // Enable enhanced support for projects + // Check if composite projects are supported + try { + await Promise.all([ + larkin.queryPgAsync( + "macrostrat", + `SELECT COUNT(*) FROM macrostrat.projects_tree`, + ), + larkin.queryPgAsync( + "macrostrat", + `SELECT slug FROM macrostrat.projects WHERE is_composite = true LIMIT 1`, + ), + larkin.queryPgAsync( + "macrostrat", + `SELECT macrostrat.core_project_ids()`, + ), + ]); + + larkin.capabilities.add(APICapability.COMPOSITE_PROJECTS); + } catch (e) { + console.log("Composite projects not supported"); + } - columnsNoGeom: function (callback) { - larkin.queryPg( - "burwell", - ` - SELECT - cols.id AS col_id, - col_name, - col_group, - col_groups.id AS col_group_id, - col AS group_col_id, - round(cols.col_area::numeric, 1) AS col_area, - cols.project_id, - string_agg(col_refs.ref_id::varchar, '|') AS refs - FROM macrostrat.cols - LEFT JOIN macrostrat.col_areas on col_areas.col_id = cols.id - LEFT JOIN macrostrat.col_groups ON col_groups.id = cols.col_group_id - LEFT JOIN macrostrat.col_refs ON cols.id = col_refs.col_id - WHERE status_code = 'active' - AND col_areas.col_area IS NOT NULL - GROUP BY col_areas.col_id, cols.id, col_groups.col_group, col_groups.id - `, - [], - function (error, result) { - if (!error && result && result.rows) { - callback(null, result.rows); - } else { - larkin.trace( - "Could not set up column cache. Check postgres connection", - ); - callback(null); - } - }, - ); - }, - }, - function (error, results) { - // Check if using Redis or not - if (larkin.cache.address) { - larkin.cache.set("unitSummary", JSON.stringify(results.unitSummary)); - larkin.cache.set("columnsGeom", JSON.stringify(results.columnsGeom)); - larkin.cache.set( - "columnsNoGeom", - JSON.stringify(results.columnsNoGeom), - ); - } else { - larkin.trace("Setting up column cache"); - larkin.cache.put("unitSummary", results.unitSummary); - larkin.cache.put("columnsGeom", results.columnsGeom); - larkin.cache.put("columnsNoGeom", results.columnsNoGeom); - } + if (larkin.capabilities.size > 0) { + let msg = "Progressive enhancements enabled:\n"; + larkin.capabilities.forEach((cap) => { + msg += `- ${cap}\n`; + }); + larkin.log("info", msg); + } else { + larkin.log("info", "No enhancements available."); + } - larkin.trace("Done prepping column cache"); - }, - ); + return larkin.capabilities; }; module.exports = larkin; diff --git a/v2/mobile/macro_summary.ts b/v2/mobile/macro_summary.ts index dd569b17..98346a2f 100644 --- a/v2/mobile/macro_summary.ts +++ b/v2/mobile/macro_summary.ts @@ -1,3 +1,5 @@ +import { handleUnitsRoute } from "../units"; + var api = require("../api"), async = require("async"), larkin = require("../larkin"), @@ -222,7 +224,7 @@ module.exports = function (req, res, next) { }, ); } else { - require("../units")( + handleUnitsRoute( { query: { unit_id: units[0].units } }, null, null, diff --git a/v2/mobile/map_query.ts b/v2/mobile/map_query.ts index f4537e49..95ae438c 100644 --- a/v2/mobile/map_query.ts +++ b/v2/mobile/map_query.ts @@ -1,3 +1,5 @@ +import { handleUnitsRoute } from "../units"; + var api = require("../api"); var async = require("async"); var larkin = require("../larkin"); @@ -386,7 +388,7 @@ module.exports = function (req, res, next) { bestFit && bestFit.strat_names ? bestFit.strat_names : []; if (macroUnits.length) { - require("../units")( + handleUnitsRoute( { query: { unit_id: macroUnits.join(",") } }, null, null, diff --git a/v2/sections.ts b/v2/sections.ts index e8acc0cf..8b16f346 100644 --- a/v2/sections.ts +++ b/v2/sections.ts @@ -1,3 +1,5 @@ +import { handleUnitsRoute } from "./units"; + var api = require("./api"), async = require("async"), _ = require("underscore"), @@ -20,7 +22,7 @@ module.exports = function (req, res, next) { [ // First pass the request to units and get the units back function (callback) { - require("./units")(req, null, null, function (error, result) { + handleUnitsRoute(req, null, null, function (error, result) { if (error) { return callback(error); } diff --git a/v2/stats.ts b/v2/stats.ts index 138a2443..554ec339 100644 --- a/v2/stats.ts +++ b/v2/stats.ts @@ -1,15 +1,12 @@ var api = require("./api"), - larkin = require("./larkin"), - multiline = require("multiline"); + larkin = require("./larkin"); -module.exports = function (req, res, next) { +module.exports = async function (req, res, next) { if (Object.keys(req.query).length < 1) { return larkin.info(req, res, next); } - var sql = multiline(function () { - /* - SELECT + const sql = `SELECT project_id, project, columns::integer, @@ -18,32 +15,27 @@ module.exports = function (req, res, next) { pbdb_collections::integer, measurements::integer, burwell_polygons::integer AS t_polys - FROM macrostrat.stats - */ - }); + FROM macrostrat.stats`; - var format = api.acceptedFormats.standard[req.query.format] + const format = api.acceptedFormats.standard[req.query.format] ? req.query.format : "json"; - larkin.queryPg("burwell", sql, [], function (error, data) { - if (error) { - larkin.error(req, res, next, error); - } else { - larkin.sendData( - req, - res, - next, - { - format: api.acceptedFormats.standard[req.query.format] - ? req.query.format - : "json", - bare: api.acceptedFormats.bare[req.query.format] ? true : false, - }, - { - data: data.rows, - }, - ); - } - }); + try { + const data = await larkin.queryPgAsync("burwell", sql, {}); + larkin.sendData( + req, + res, + next, + { + format, + bare: api.acceptedFormats.bare[req.query.format] ? true : false, + }, + { + data: data.rows, + }, + ); + } catch (error) { + larkin.error(req, res, next, error); + } }; diff --git a/v2/units.ts b/v2/units.ts index cf2ee638..2510235d 100644 --- a/v2/units.ts +++ b/v2/units.ts @@ -3,9 +3,10 @@ var api = require("./api"), dbgeo = require("dbgeo"), gp = require("geojson-precision"), larkin = require("./larkin"); -//need to repoint setupCache function in larkin.ts to accommodate for units.ts changes -module.exports = function (req, res, next, cb) { +import { buildProjectsFilter } from "./utils"; + +export function handleUnitsRoute(req, res, next, cb) { // If no parameters, send the route definition if (Object.keys(req.query).length < 1) { return larkin.info(req, res, next); @@ -332,7 +333,11 @@ module.exports = function (req, res, next, cb) { where += lithWhere.join(" OR ") + ")"; } - if (req.query.lith_att_id || req.query.lith_att || req.query.lith_att_type) { + if ( + req.query.lith_att_id || + req.query.lith_att || + req.query.lith_att_type + ) { let lithAttField; if (req.query.lith_att_id) { @@ -340,10 +345,14 @@ module.exports = function (req, res, next, cb) { params["lith_att"] = larkin.parseMultipleIds(req.query.lith_att_id); } else if (req.query.lith_att) { lithAttField = "lith_atts.lith_att"; - params["lith_att"] = larkin.parseMultipleStrings(req.query.lith_att); + params["lith_att"] = larkin.parseMultipleStrings( + req.query.lith_att, + ); } else if (req.query.lith_att_type) { lithAttField = "lith_atts.att_type"; - params["lith_att"] = larkin.parseMultipleStrings(req.query.lith_att_type); + params["lith_att"] = larkin.parseMultipleStrings( + req.query.lith_att_type, + ); } where += ` @@ -357,7 +366,6 @@ module.exports = function (req, res, next, cb) { )`; } - if (data.age_bottom !== 99999) { where += " AND b_age > :age_top AND t_age < :age_bottom"; params["age_top"] = data.age_top; @@ -391,9 +399,13 @@ module.exports = function (req, res, next, cb) { params["strat_ids"] = data.strat_ids; } - if (req.query.project_id) { - where += " AND lookup_units.project_id = ANY(:project_id)"; - params["project_id"] = larkin.parseMultipleIds(req.query.project_id); + const [whereClauses, projectParams] = buildProjectsFilter( + req, + "lookup_units.project_id", + ); + if (whereClauses.length > 0) { + where += " AND " + whereClauses.join(" AND "); + Object.assign(params, projectParams); } if ( @@ -690,4 +702,4 @@ module.exports = function (req, res, next, cb) { } }, ); -}; +} diff --git a/v2/utils/index.ts b/v2/utils/index.ts new file mode 100644 index 00000000..0104f7b3 --- /dev/null +++ b/v2/utils/index.ts @@ -0,0 +1,38 @@ +const larkin = require("../larkin"); + +type Params = Record< + string, + string | number | boolean | Array +>; + +export function buildProjectsFilter( + req, + field = "project_id", +): [string[], Params] { + const params: Params = {}; + const whereClauses: string[] = []; + + if (!larkin.hasCapability("composite-projects")) { + // If composite projects are not supported, default to active projects only + if (req.query.project_id) { + whereClauses.push(field + " = ANY(:project_id)"); + params["project_id"] = larkin.parseMultipleIds(req.query.project_id); + } + return [whereClauses, params]; + } + + // Newer handling of composite/core projects + if (req.query.project_id) { + if (req.query.project_id !== "all") { + whereClauses.push( + field + " = ANY(macrostrat.flattened_project_ids(:project_id))", + ); + params["project_id"] = larkin.parseMultipleIds(req.query.project_id); + } + } else { + // Default to active projects only + whereClauses.push(field + " = ANY(macrostrat.core_project_ids())"); + } + + return [whereClauses, params]; +}