Skip to content

Composite projects #266

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
davenquinn merged 18 commits into from
Dec 27, 2025
Merged

Composite projects #266

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
66456fd
Improve caching
davenquinn Dec 26, 2025
e6653a4
Starting point for updating projects route
davenquinn Dec 26, 2025
860408e
Streamline project SQL generation
davenquinn Dec 26, 2025
cc774ec
Improve logging
davenquinn Dec 26, 2025
3f17eff
Dramatically simplify projects query
davenquinn Dec 26, 2025
f359675
Updated project definitions to include composite projects
davenquinn Dec 26, 2025
f240d82
More info in projects route
davenquinn Dec 26, 2025
9c0a33f
Remove unitSummary cache and add function for project loading
davenquinn Dec 26, 2025
7348cb0
Filter based on composite projects
davenquinn Dec 26, 2025
e7a8e5a
Add basic async handler
davenquinn Dec 26, 2025
24817b8
Starting point for progressive enhancement of projects
davenquinn Dec 26, 2025
b23df7f
Enable progressive enhancements
davenquinn Dec 26, 2025
f5f9c27
Progressive enhancement - composite projects are enabled if schema mi…
davenquinn Dec 27, 2025
7d6462c
Completely remove internal caching
davenquinn Dec 27, 2025
34d68cf
Start process of converting to esm
davenquinn Dec 27, 2025
e6ae10b
Start integrating async lifecycle managers
davenquinn Dec 27, 2025
79425c2
Start moving towards esm
davenquinn Dec 27, 2025
25ed403
Update version
davenquinn Dec 27, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@macrostrat/api-v2",
"version": "2.1.8",
"version": "2.2.0",
"description": "An API for stratigraphic and geological information (Version 2).",
"main": "server.ts",
"repository": {
Expand Down
64 changes: 32 additions & 32 deletions server.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,48 +2,48 @@ const dotenv = require("dotenv");
// Load environment variables from .env file
dotenv.config();

import { buildAPI } from "./v2";

var express = require("express"),
bodyParser = require("body-parser"),
//v1 = require("./v1"),
v2 = require("./v2"),
defs = require("./v2/defs"),
app = express();
bodyParser = require("body-parser");
//defs = require("./v2/defs"),

// parse application/x-www-form-urlencoded
app.use(bodyParser.urlencoded({ extended: false }));
//TODO: update port to designated env.
const listenPort = process.argv[2] ?? process.env.PORT ?? 5000;

// parse application/json
app.use(bodyParser.json());
async function runServer() {
const app = express();
// parse application/x-www-form-urlencoded
app.use(bodyParser.urlencoded({ extended: false }));

// parse application/vnd.api+json as json
app.use(bodyParser.json({ type: "application/vnd.api+json" }));
// parse application/json
app.use(bodyParser.json());

// Load and prefix all routes with /api and appropriate version
app.use("/v2", v2);
// parse application/vnd.api+json as json
app.use(bodyParser.json({ type: "application/vnd.api+json" }));

app.route("/v1*").get(function (req, res, next) {
res.status(410).send({
error:
"Macrostrat's v1 API has been retired. Please update your usage to newer endpoints.",
});
});
const v2 = await buildAPI();

// If no version specified, fall back to more current
app.use("/", v2);
// Load and prefix all routes with /api and appropriate version
app.use("/v2", v2);

app.set("json spaces", 2);
app.route("/v1*").get(function (req, res, next) {
res.status(410).send({
error:
"Macrostrat's v1 API has been retired. Please update your usage to newer endpoints.",
});
});

//TODO: update port to designated env.
app.port = process.argv[2] ?? process.env.PORT ?? 5000;
// If no version specified, fall back to more current
app.use("/", v2);

app.start = function () {
app.listen(app.port, function () {
console.log("Listening on port " + app.port);
});
};
app.set("json spaces", 2);

if (!module.parent) {
app.start();
app.listen(listenPort, function () {
console.log("Listening on port " + listenPort);
});
}

module.exports = app;
runServer().catch((error) => {
console.error("Failed to start server:", error);
});
6 changes: 2 additions & 4 deletions v2/column-cache-refresh.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,13 +1,11 @@
const larkin = require("./larkin");

module.exports = (req, res, next) => {
if (
req.query &&
req.query.cacheRefreshKey &&
req.query.cacheRefreshKey === process.env.CACHE_REFRESH_KEY
) {
larkin.setupCache();
res.json({ success: "cache refreshed" });
res.status(404);
res.json({ fail: "internal column cache refresh is no longer supported" });
} else {
res.status(401);
res.json({ fail: "you do not have permissions to execute this action" });
Expand Down
48 changes: 19 additions & 29 deletions v2/columns.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
import { handleUnitsRoute } from "./units";

var api = require("./api"),
async = require("async"),
dbgeo = require("dbgeo"),
Expand All @@ -12,24 +14,9 @@ module.exports = function (req, res, next, callback) {

async.waterfall(
[
// First pass the request to the /units route and get the long response
function (callback) {
if ("all" in req.query) {
larkin.cache.fetch("unitSummary", function (data) {
callback(null, data);
});
} else {
callback(null, null);
}
},

function (data, callback) {
if (data) {
return callback(null, data);
}

//call units to group units by col_id
require("./units")(req, null, null, function (error, result) {
handleUnitsRoute(req, null, null, function (error, result) {
if (error) {
callback(error);
}
Expand Down Expand Up @@ -122,19 +109,22 @@ module.exports = function (req, res, next, callback) {
return callback(null, null, []);
}

if ("all" in req.query) {
if (req.query.format && api.acceptedFormats.geo[req.query.format]) {
larkin.cache.fetch("columnsGeom", function (data) {
callback(null, new_cols, data);
});
} else {
larkin.cache.fetch("columnsNoGeom", function (data) {
callback(null, new_cols, data);
});
}
} else {
callback(null, new_cols, null);
}
callback(null, new_cols, null);

// TODO: this breaks "all" filtering and brings down the API
// if ("all" in req.query) {
// if (req.query.format && api.acceptedFormats.geo[req.query.format]) {
// larkin.cache.fetch("columnsGeom", function (data) {
// callback(null, new_cols, data);
// });
// } else {
// larkin.cache.fetch("columnsNoGeom", function (data) {
// callback(null, new_cols, data);
// });
// }
// } else {
// callback(null, new_cols, null);
// }
},

// Using the unique column IDs returned from units, query columns
Expand Down
18 changes: 13 additions & 5 deletions v2/definitions/columns.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@ var api = require("../api");
var larkin = require("../larkin");
var dbgeo = require("dbgeo");

import { buildProjectsFilter } from "../utils";

module.exports = function (req, res, next, cb) {
if (Object.keys(req.query).length < 1) {
return larkin.info(req, res, next);
Expand Down Expand Up @@ -44,17 +46,23 @@ module.exports = function (req, res, next, cb) {
where.push("cols.col_name = ANY(:col_name)");
params["col_name"] = larkin.parseMultipleStrings(req.query.col_name);
}
if (req.query.project_id) {
where.push("cols.project_id = ANY(:project_id)");
params["project_id"] = larkin.parseMultipleIds(req.query.project_id);
}

const [whereClauses, projectParams] = buildProjectsFilter(
req,
"cols.project_id",
);
where = where.concat(whereClauses);
Object.assign(params, projectParams);

where.push("status_code = ANY(:status_code)");
if (req.query.status_code || req.query.status) {
// `status` parameter still works but has been superseded by `status_code`
// multiple status codes can be provided
where.push("status_code = ANY(:status_code)");
params["status_code"] = larkin.parseMultipleIds(
req.query.status_code ?? req.query.status,
);
} else {
params["status_code"] = ["active"];
}

if (where.length) {
Expand Down
19 changes: 11 additions & 8 deletions v2/definitions/groups.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
var api = require("../api"),
larkin = require("../larkin");

import { buildProjectsFilter } from "../utils";

module.exports = function (req, res, next, cb) {
if (Object.keys(req.query).length < 1) {
return larkin.info(req, res, next);
Expand All @@ -18,15 +20,16 @@ module.exports = function (req, res, next, cb) {
if (req.query.col_group_id) {
where.push("col_groups.id = ANY(:col_group_id)");
params["col_group_id"] = larkin.parseMultipleIds(req.query.col_group_id);
} else if (req.query.project_id) {
where.push("cols.project_id = ANY(:project_id)");
params["project_id"] = larkin.parseMultipleIds(req.query.project_id);
} else if (req.query.col_id) {
where.push("cols.id = ANY(:col_id)");
params["col_id"] = larkin.parseMultipleIds(req.query.col_id);
}

where = where.length ? "WHERE " + where.join(" AND ") : "";
const [projectWhereClauses, projectParams] = buildProjectsFilter(
req,
"cols.project_id",
);
where = where.concat(projectWhereClauses);
Object.assign(params, projectParams);

const whereClause = where.length ? "WHERE " + where.join(" AND ") : "";

let sql = `SELECT col_groups.id AS col_group_id,
col_group,
Expand All @@ -37,7 +40,7 @@ module.exports = function (req, res, next, cb) {
FROM macrostrat.col_groups
LEFT JOIN macrostrat.cols ON cols.col_group_id = col_groups.id
LEFT JOIN macrostrat.units_sections ON units_sections.col_id = cols.id
${where}
${whereClause}
GROUP BY col_groups.id, cols.project_id `;

if ("sample" in req.query) {
Expand Down
119 changes: 70 additions & 49 deletions v2/definitions/projects.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,62 +7,83 @@ module.exports = function (req, res, next, cb) {
}
//There will be a discrepancy with a key in production. Updated in_proccess_cols to in_process_cols key. Values are
//still the same.
var sql = `WITH in_proc AS (
SELECT COUNT(DISTINCT id) AS c, project_id
FROM macrostrat.cols
WHERE status_code = 'in process'
GROUP BY project_id
),
obs AS (
SELECT COUNT(DISTINCT id) AS co, project_id
FROM macrostrat.cols
WHERE status_code = 'obsolete'
GROUP BY project_id
),
col_area_sum AS (
SELECT project_id, SUM(col_area) AS total_area
FROM macrostrat.cols
WHERE status_code = 'active'
GROUP BY project_id
)

SELECT
projects.id AS project_id,
projects.project,
projects.descrip,
projects.timescale_id,
COUNT(DISTINCT units_sections.col_id)::integer AS t_cols,
COALESCE(c, 0)::integer AS in_process_cols,
COALESCE(co, 0)::integer AS obsolete_cols,
COUNT(DISTINCT units_sections.unit_id)::integer AS t_units,
COALESCE(ROUND(total_area), 0)::integer AS area

FROM macrostrat.projects
LEFT JOIN macrostrat.cols ON projects.id = cols.project_id
LEFT JOIN macrostrat.units_sections ON units_sections.col_id = cols.id
LEFT JOIN in_proc USING (project_id)
LEFT JOIN obs USING (project_id)
LEFT JOIN col_area_sum ON projects.id = col_area_sum.project_id
`;

var where = [];
var params = {};
const where = [];
let params = {};

if (req.query.project_id) {
where.push("projects.id = ANY(:project_id)");
params["project_id"] = larkin.parseMultipleIds(req.query.project_id);
}
if (where.length) {
sql += ` WHERE ${where.join(" AND ")}`;

const whereStatement = where.length > 0 ? where.join(" AND ") : "true";

let sql = `
SELECT
p.id AS project_id,
p.project,
p.descrip,
p.timescale_id,
count(DISTINCT units_sections.col_id)::integer AS t_cols,
count(DISTINCT cols.id) FILTER ( WHERE cols.status_code = 'active' )::integer AS active_cols,
count(DISTINCT cols.id) FILTER ( WHERE cols.status_code = 'in process' )::integer AS in_process_cols,
count(DISTINCT cols.id) FILTER ( WHERE cols.status_code = 'obsolete' )::integer AS obsolete_cols,
count(DISTINCT units_sections.unit_id)::integer AS t_units,
coalesce(round(sum(DISTINCT cols.col_area) FILTER ( WHERE cols.status_code = 'active')), 0) AS area
FROM macrostrat.projects p
LEFT JOIN macrostrat.cols ON p.id = cols.project_id
LEFT JOIN macrostrat.units_sections ON units_sections.col_id = cols.id
WHERE ${whereStatement}
GROUP BY
p.id,
p.project,
p.descrip,
p.timescale_id
`;

if (larkin.hasCapability("composite-projects")) {
/** Progressive enhancement for composite projects **/
sql = `
WITH composite_tree AS (
SELECT pt.parent_id, array_agg(pt.child_id) children, jsonb_agg(to_jsonb(p)) AS members
FROM macrostrat.projects_tree pt
JOIN LATERAL (
SELECT p.id, p.slug, p.project name
FROM macrostrat.projects p
WHERE p.id = pt.child_id
) AS p ON true
GROUP BY pt.parent_id
)
SELECT
p.id AS project_id,
p.slug,
p.project,
p.descrip,
p.timescale_id,
ct.members,
count(DISTINCT units_sections.col_id)::integer AS t_cols,
count(DISTINCT cols.id) FILTER ( WHERE cols.status_code = 'active' )::integer AS active_cols,
count(DISTINCT cols.id) FILTER ( WHERE cols.status_code = 'in process' )::integer AS in_process_cols,
count(DISTINCT cols.id) FILTER ( WHERE cols.status_code = 'obsolete' )::integer AS obsolete_cols,
count(DISTINCT units_sections.unit_id)::integer AS t_units,
coalesce(round(sum(DISTINCT cols.col_area) FILTER ( WHERE cols.status_code = 'active')), 0) AS area
FROM macrostrat.projects p
LEFT JOIN composite_tree ct
ON ct.parent_id = p.id
LEFT JOIN macrostrat.cols ON p.id = cols.project_id
OR (p.is_composite AND cols.project_id = ANY(ct.children))
LEFT JOIN macrostrat.units_sections ON units_sections.col_id = cols.id
WHERE ${whereStatement}
GROUP BY
p.id,
p.project,
p.descrip,
p.timescale_id,
p.slug,
ct.children,
ct.members
`;
}
sql += `\nGROUP BY
projects.id,
projects.project,
projects.descrip,
projects.timescale_id,
c,
co,
total_area;`;

larkin.queryPg("burwell", sql, params, function (error, data) {
if (error) {
Expand Down
Loading