Composite projects

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@macrostrat/api-v2",
-  "version": "2.1.8",
+  "version": "2.2.0",
   "description": "An API for stratigraphic and geological information (Version 2).",
   "main": "server.ts",
   "repository": {

server.ts

@@ -2,48 +2,48 @@ const dotenv = require("dotenv");
 // Load environment variables from .env file
 dotenv.config();
 
+import { buildAPI } from "./v2";
+
 var express = require("express"),
-  bodyParser = require("body-parser"),
-  //v1 = require("./v1"),
-  v2 = require("./v2"),
-  defs = require("./v2/defs"),
-  app = express();
+  bodyParser = require("body-parser");
+//defs = require("./v2/defs"),
 
-// parse application/x-www-form-urlencoded
-app.use(bodyParser.urlencoded({ extended: false }));
+//TODO: update port to designated env.
+const listenPort = process.argv[2] ?? process.env.PORT ?? 5000;
 
-// parse application/json
-app.use(bodyParser.json());
+async function runServer() {
+  const app = express();
+  // parse application/x-www-form-urlencoded
+  app.use(bodyParser.urlencoded({ extended: false }));
 
-// parse application/vnd.api+json as json
-app.use(bodyParser.json({ type: "application/vnd.api+json" }));
+  // parse application/json
+  app.use(bodyParser.json());
 
-// Load and prefix all routes with /api and appropriate version
-app.use("/v2", v2);
+  // parse application/vnd.api+json as json
+  app.use(bodyParser.json({ type: "application/vnd.api+json" }));
 
-app.route("/v1*").get(function (req, res, next) {
-  res.status(410).send({
-    error:
-      "Macrostrat's v1 API has been retired. Please update your usage to newer endpoints.",
-  });
-});
+  const v2 = await buildAPI();
 
-// If no version specified, fall back to more current
-app.use("/", v2);
+  // Load and prefix all routes with /api and appropriate version
+  app.use("/v2", v2);
 
-app.set("json spaces", 2);
+  app.route("/v1*").get(function (req, res, next) {
+    res.status(410).send({
+      error:
+        "Macrostrat's v1 API has been retired. Please update your usage to newer endpoints.",
+    });
+  });
 
-//TODO: update port to designated env.
-app.port = process.argv[2] ?? process.env.PORT ?? 5000;
+  // If no version specified, fall back to more current
+  app.use("/", v2);
 
-app.start = function () {
-  app.listen(app.port, function () {
-    console.log("Listening on port " + app.port);
-  });
-};
+  app.set("json spaces", 2);
 
-if (!module.parent) {
-  app.start();
+  app.listen(listenPort, function () {
+    console.log("Listening on port " + listenPort);
+  });
 }
 
-module.exports = app;
+runServer().catch((error) => {
+  console.error("Failed to start server:", error);
+});

v2/column-cache-refresh.ts

@@ -1,13 +1,11 @@
-const larkin = require("./larkin");
-
 module.exports = (req, res, next) => {
   if (
     req.query &&
     req.query.cacheRefreshKey &&
     req.query.cacheRefreshKey === process.env.CACHE_REFRESH_KEY
   ) {
-    larkin.setupCache();
-    res.json({ success: "cache refreshed" });
+    res.status(404);
+    res.json({ fail: "internal column cache refresh is no longer supported" });
   } else {
     res.status(401);
     res.json({ fail: "you do not have permissions to execute this action" });

v2/columns.ts

@@ -1,3 +1,5 @@
+import { handleUnitsRoute } from "./units";
+
 var api = require("./api"),
   async = require("async"),
   dbgeo = require("dbgeo"),
@@ -12,24 +14,9 @@ module.exports = function (req, res, next, callback) {
 
   async.waterfall(
     [
-      // First pass the request to the /units route and get the long response
       function (callback) {
-        if ("all" in req.query) {
-          larkin.cache.fetch("unitSummary", function (data) {
-            callback(null, data);
-          });
-        } else {
-          callback(null, null);
-        }
-      },
-
-      function (data, callback) {
-        if (data) {
-          return callback(null, data);
-        }
-
         //call units to group units by col_id
-        require("./units")(req, null, null, function (error, result) {
+        handleUnitsRoute(req, null, null, function (error, result) {
           if (error) {
             callback(error);
           }
@@ -122,19 +109,22 @@ module.exports = function (req, res, next, callback) {
           return callback(null, null, []);
         }
 
-        if ("all" in req.query) {
-          if (req.query.format && api.acceptedFormats.geo[req.query.format]) {
-            larkin.cache.fetch("columnsGeom", function (data) {
-              callback(null, new_cols, data);
-            });
-          } else {
-            larkin.cache.fetch("columnsNoGeom", function (data) {
-              callback(null, new_cols, data);
-            });
-          }
-        } else {
-          callback(null, new_cols, null);
-        }
+        callback(null, new_cols, null);
+
+        // TODO: this breaks "all" filtering and brings down the API
+        // if ("all" in req.query) {
+        //   if (req.query.format && api.acceptedFormats.geo[req.query.format]) {
+        //     larkin.cache.fetch("columnsGeom", function (data) {
+        //       callback(null, new_cols, data);
+        //     });
+        //   } else {
+        //     larkin.cache.fetch("columnsNoGeom", function (data) {
+        //       callback(null, new_cols, data);
+        //     });
+        //   }
+        // } else {
+        //   callback(null, new_cols, null);
+        // }
       },
 
       // Using the unique column IDs returned from units, query columns

v2/definitions/columns.ts

@@ -2,6 +2,8 @@ var api = require("../api");
 var larkin = require("../larkin");
 var dbgeo = require("dbgeo");
 
+import { buildProjectsFilter } from "../utils";
+
 module.exports = function (req, res, next, cb) {
   if (Object.keys(req.query).length < 1) {
     return larkin.info(req, res, next);
@@ -44,17 +46,23 @@ module.exports = function (req, res, next, cb) {
     where.push("cols.col_name = ANY(:col_name)");
     params["col_name"] = larkin.parseMultipleStrings(req.query.col_name);
   }
-  if (req.query.project_id) {
-    where.push("cols.project_id = ANY(:project_id)");
-    params["project_id"] = larkin.parseMultipleIds(req.query.project_id);
-  }
+
+  const [whereClauses, projectParams] = buildProjectsFilter(
+    req,
+    "cols.project_id",
+  );
+  where = where.concat(whereClauses);
+  Object.assign(params, projectParams);
+
+  where.push("status_code = ANY(:status_code)");
   if (req.query.status_code || req.query.status) {
     // `status` parameter still works but has been superseded by `status_code`
     // multiple status codes can be provided
-    where.push("status_code = ANY(:status_code)");
     params["status_code"] = larkin.parseMultipleIds(
       req.query.status_code ?? req.query.status,
     );
+  } else {
+    params["status_code"] = ["active"];
   }
 
   if (where.length) {

v2/definitions/groups.ts

@@ -1,6 +1,8 @@
 var api = require("../api"),
   larkin = require("../larkin");
 
+import { buildProjectsFilter } from "../utils";
+
 module.exports = function (req, res, next, cb) {
   if (Object.keys(req.query).length < 1) {
     return larkin.info(req, res, next);
@@ -18,15 +20,16 @@ module.exports = function (req, res, next, cb) {
   if (req.query.col_group_id) {
     where.push("col_groups.id = ANY(:col_group_id)");
     params["col_group_id"] = larkin.parseMultipleIds(req.query.col_group_id);
-  } else if (req.query.project_id) {
-    where.push("cols.project_id = ANY(:project_id)");
-    params["project_id"] = larkin.parseMultipleIds(req.query.project_id);
-  } else if (req.query.col_id) {
-    where.push("cols.id = ANY(:col_id)");
-    params["col_id"] = larkin.parseMultipleIds(req.query.col_id);
   }
 
-  where = where.length ? "WHERE " + where.join(" AND ") : "";
+  const [projectWhereClauses, projectParams] = buildProjectsFilter(
+    req,
+    "cols.project_id",
+  );
+  where = where.concat(projectWhereClauses);
+  Object.assign(params, projectParams);
+
+  const whereClause = where.length ? "WHERE " + where.join(" AND ") : "";
 
   let sql = `SELECT col_groups.id AS col_group_id,
        col_group,
@@ -37,7 +40,7 @@ module.exports = function (req, res, next, cb) {
     FROM macrostrat.col_groups
     LEFT JOIN macrostrat.cols ON cols.col_group_id = col_groups.id
     LEFT JOIN macrostrat.units_sections ON units_sections.col_id = cols.id
-    ${where}
+    ${whereClause}
     GROUP BY col_groups.id, cols.project_id `;
 
   if ("sample" in req.query) {

v2/definitions/projects.ts

@@ -7,62 +7,83 @@ module.exports = function (req, res, next, cb) {
   }
   //There will be a discrepancy with a key in production. Updated in_proccess_cols to in_process_cols key. Values are
   //still the same.
-  var sql = `WITH in_proc AS (
-        SELECT COUNT(DISTINCT id) AS c, project_id
-        FROM macrostrat.cols
-        WHERE status_code = 'in process'
-        GROUP BY project_id
-    ),
-    obs AS (
-        SELECT COUNT(DISTINCT id) AS co, project_id
-        FROM macrostrat.cols
-        WHERE status_code = 'obsolete'
-        GROUP BY project_id
-    ),
-    col_area_sum AS (
-        SELECT project_id, SUM(col_area) AS total_area
-        FROM macrostrat.cols
-        WHERE status_code = 'active'
-        GROUP BY project_id
-    )
-
-    SELECT
-        projects.id AS project_id,
-        projects.project,
-        projects.descrip,
-        projects.timescale_id,
-        COUNT(DISTINCT units_sections.col_id)::integer AS t_cols,
-        COALESCE(c, 0)::integer AS in_process_cols,
-        COALESCE(co, 0)::integer AS obsolete_cols,
-        COUNT(DISTINCT units_sections.unit_id)::integer AS t_units,
-        COALESCE(ROUND(total_area), 0)::integer AS area
-
-    FROM macrostrat.projects
-    LEFT JOIN macrostrat.cols ON projects.id = cols.project_id
-    LEFT JOIN macrostrat.units_sections ON units_sections.col_id = cols.id
-    LEFT JOIN in_proc USING (project_id)
-    LEFT JOIN obs USING (project_id)
-    LEFT JOIN col_area_sum ON projects.id = col_area_sum.project_id
-    `;
 
-  var where = [];
-  var params = {};
+  const where = [];
+  let params = {};
 
   if (req.query.project_id) {
     where.push("projects.id = ANY(:project_id)");
     params["project_id"] = larkin.parseMultipleIds(req.query.project_id);
   }
-  if (where.length) {
-    sql += ` WHERE ${where.join(" AND ")}`;
+
+  const whereStatement = where.length > 0 ? where.join(" AND ") : "true";
+
+  let sql = `
+    SELECT
+        p.id AS project_id,
+        p.project,
+        p.descrip,
+        p.timescale_id,
+        count(DISTINCT units_sections.col_id)::integer AS t_cols,
+        count(DISTINCT cols.id) FILTER ( WHERE cols.status_code = 'active' )::integer AS active_cols,
+        count(DISTINCT cols.id) FILTER ( WHERE cols.status_code = 'in process' )::integer AS in_process_cols,
+        count(DISTINCT cols.id) FILTER ( WHERE cols.status_code = 'obsolete' )::integer AS obsolete_cols,
+        count(DISTINCT units_sections.unit_id)::integer AS t_units,
+        coalesce(round(sum(DISTINCT cols.col_area) FILTER ( WHERE cols.status_code = 'active')), 0) AS area
+    FROM macrostrat.projects p
+    LEFT JOIN macrostrat.cols ON p.id = cols.project_id
+    LEFT JOIN macrostrat.units_sections ON units_sections.col_id = cols.id
+    WHERE ${whereStatement}
+    GROUP BY
+      p.id,
+      p.project,
+      p.descrip,
+      p.timescale_id
+  `;
+
+  if (larkin.hasCapability("composite-projects")) {
+    /** Progressive enhancement for composite projects **/
+    sql = `
+    WITH composite_tree AS (
+      SELECT pt.parent_id,  array_agg(pt.child_id) children, jsonb_agg(to_jsonb(p)) AS members
+      FROM macrostrat.projects_tree pt
+      JOIN LATERAL (
+        SELECT p.id, p.slug, p.project name
+        FROM macrostrat.projects p
+        WHERE p.id = pt.child_id
+      ) AS p ON true
+      GROUP BY pt.parent_id
+    )
+    SELECT
+        p.id AS project_id,
+        p.slug,
+        p.project,
+        p.descrip,
+        p.timescale_id,
+        ct.members,
+        count(DISTINCT units_sections.col_id)::integer AS t_cols,
+        count(DISTINCT cols.id) FILTER ( WHERE cols.status_code = 'active' )::integer AS active_cols,
+        count(DISTINCT cols.id) FILTER ( WHERE cols.status_code = 'in process' )::integer AS in_process_cols,
+        count(DISTINCT cols.id) FILTER ( WHERE cols.status_code = 'obsolete' )::integer AS obsolete_cols,
+        count(DISTINCT units_sections.unit_id)::integer AS t_units,
+        coalesce(round(sum(DISTINCT cols.col_area) FILTER ( WHERE cols.status_code = 'active')), 0) AS area
+    FROM macrostrat.projects p
+    LEFT JOIN composite_tree ct
+      ON ct.parent_id = p.id
+    LEFT JOIN macrostrat.cols ON p.id = cols.project_id
+          OR (p.is_composite AND cols.project_id = ANY(ct.children))
+    LEFT JOIN macrostrat.units_sections ON units_sections.col_id = cols.id
+    WHERE ${whereStatement}
+    GROUP BY
+      p.id,
+      p.project,
+      p.descrip,
+      p.timescale_id,
+      p.slug,
+      ct.children,
+      ct.members
+    `;
   }
-  sql += `\nGROUP BY
-    projects.id,
-    projects.project,
-    projects.descrip,
-    projects.timescale_id,
-    c,
-    co,
-    total_area;`;
 
   larkin.queryPg("burwell", sql, params, function (error, data) {
     if (error) {