Skip to content

Commit 6149fab

Browse files
davenquinndavenquinn
committed
* 'main' of https://github.com/UW-Macrostrat/macrostrat-api: Update version Start moving towards esm Start integrating async lifecycle managers Start process of converting to esm Completely remove internal caching Progressive enhancement - composite projects are enabled if schema migrations are in place Enable progressive enhancements Starting point for progressive enhancement of projects Add basic async handler Filter based on composite projects Remove unitSummary cache and add function for project loading More info in projects route Updated project definitions to include composite projects Dramatically simplify projects query Improve logging Streamline project SQL generation Starting point for updating projects route Improve caching
2 parents e223477 + 115b9ae commit 6149fab

File tree

16 files changed

+405
-457
lines changed

16 files changed

+405
-457
lines changed

‎package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "@macrostrat/api-v2",
3-
"version": "2.1.8",
3+
"version": "2.2.0",
44
"description": "An API for stratigraphic and geological information (Version 2).",
55
"main": "server.ts",
66
"repository": {

‎server.ts‎

Lines changed: 32 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -2,48 +2,48 @@ const dotenv = require("dotenv");
22
// Load environment variables from .env file
33
dotenv.config();
44

5+
import { buildAPI } from "./v2";
6+
57
var express = require("express"),
6-
bodyParser = require("body-parser"),
7-
//v1 = require("./v1"),
8-
v2 = require("./v2"),
9-
defs = require("./v2/defs"),
10-
app = express();
8+
bodyParser = require("body-parser");
9+
//defs = require("./v2/defs"),
1110

12-
// parse application/x-www-form-urlencoded
13-
app.use(bodyParser.urlencoded({ extended: false }));
11+
//TODO: update port to designated env.
12+
const listenPort = process.argv[2] ?? process.env.PORT ?? 5000;
1413

15-
// parse application/json
16-
app.use(bodyParser.json());
14+
async function runServer() {
15+
const app = express();
16+
// parse application/x-www-form-urlencoded
17+
app.use(bodyParser.urlencoded({ extended: false }));
1718

18-
// parse application/vnd.api+json as json
19-
app.use(bodyParser.json({ type: "application/vnd.api+json" }));
19+
// parse application/json
20+
app.use(bodyParser.json());
2021

21-
// Load and prefix all routes with /api and appropriate version
22-
app.use("/v2", v2);
22+
// parse application/vnd.api+json as json
23+
app.use(bodyParser.json({ type: "application/vnd.api+json" }));
2324

24-
app.route("/v1*").get(function (req, res, next) {
25-
res.status(410).send({
26-
error:
27-
"Macrostrat's v1 API has been retired. Please update your usage to newer endpoints.",
28-
});
29-
});
25+
const v2 = await buildAPI();
3026

31-
// If no version specified, fall back to more current
32-
app.use("/", v2);
27+
// Load and prefix all routes with /api and appropriate version
28+
app.use("/v2", v2);
3329

34-
app.set("json spaces", 2);
30+
app.route("/v1*").get(function (req, res, next) {
31+
res.status(410).send({
32+
error:
33+
"Macrostrat's v1 API has been retired. Please update your usage to newer endpoints.",
34+
});
35+
});
3536

36-
//TODO: update port to designated env.
37-
app.port = process.argv[2] ?? process.env.PORT ?? 5000;
37+
// If no version specified, fall back to more current
38+
app.use("/", v2);
3839

39-
app.start = function () {
40-
app.listen(app.port, function () {
41-
console.log("Listening on port " + app.port);
42-
});
43-
};
40+
app.set("json spaces", 2);
4441

45-
if (!module.parent) {
46-
app.start();
42+
app.listen(listenPort, function () {
43+
console.log("Listening on port " + listenPort);
44+
});
4745
}
4846

49-
module.exports = app;
47+
runServer().catch((error) => {
48+
console.error("Failed to start server:", error);
49+
});

‎v2/column-cache-refresh.ts‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,11 @@
1-
const larkin = require("./larkin");
2-
31
module.exports = (req, res, next) => {
42
if (
53
req.query &&
64
req.query.cacheRefreshKey &&
75
req.query.cacheRefreshKey === process.env.CACHE_REFRESH_KEY
86
) {
9-
larkin.setupCache();
10-
res.json({ success: "cache refreshed" });
7+
res.status(404);
8+
res.json({ fail: "internal column cache refresh is no longer supported" });
119
} else {
1210
res.status(401);
1311
res.json({ fail: "you do not have permissions to execute this action" });

‎v2/columns.ts‎

Lines changed: 19 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
import { handleUnitsRoute } from "./units";
2+
13
var api = require("./api"),
24
async = require("async"),
35
dbgeo = require("dbgeo"),
@@ -12,24 +14,9 @@ module.exports = function (req, res, next, callback) {
1214

1315
async.waterfall(
1416
[
15-
// First pass the request to the /units route and get the long response
1617
function (callback) {
17-
if ("all" in req.query) {
18-
larkin.cache.fetch("unitSummary", function (data) {
19-
callback(null, data);
20-
});
21-
} else {
22-
callback(null, null);
23-
}
24-
},
25-
26-
function (data, callback) {
27-
if (data) {
28-
return callback(null, data);
29-
}
30-
3118
//call units to group units by col_id
32-
require("./units")(req, null, null, function (error, result) {
19+
handleUnitsRoute(req, null, null, function (error, result) {
3320
if (error) {
3421
callback(error);
3522
}
@@ -122,19 +109,22 @@ module.exports = function (req, res, next, callback) {
122109
return callback(null, null, []);
123110
}
124111

125-
if ("all" in req.query) {
126-
if (req.query.format && api.acceptedFormats.geo[req.query.format]) {
127-
larkin.cache.fetch("columnsGeom", function (data) {
128-
callback(null, new_cols, data);
129-
});
130-
} else {
131-
larkin.cache.fetch("columnsNoGeom", function (data) {
132-
callback(null, new_cols, data);
133-
});
134-
}
135-
} else {
136-
callback(null, new_cols, null);
137-
}
112+
callback(null, new_cols, null);
113+
114+
// TODO: this breaks "all" filtering and brings down the API
115+
// if ("all" in req.query) {
116+
// if (req.query.format && api.acceptedFormats.geo[req.query.format]) {
117+
// larkin.cache.fetch("columnsGeom", function (data) {
118+
// callback(null, new_cols, data);
119+
// });
120+
// } else {
121+
// larkin.cache.fetch("columnsNoGeom", function (data) {
122+
// callback(null, new_cols, data);
123+
// });
124+
// }
125+
// } else {
126+
// callback(null, new_cols, null);
127+
// }
138128
},
139129

140130
// Using the unique column IDs returned from units, query columns

‎v2/definitions/columns.ts‎

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@ var api = require("../api");
22
var larkin = require("../larkin");
33
var dbgeo = require("dbgeo");
44

5+
import { buildProjectsFilter } from "../utils";
6+
57
module.exports = function (req, res, next, cb) {
68
if (Object.keys(req.query).length < 1) {
79
return larkin.info(req, res, next);
@@ -44,17 +46,23 @@ module.exports = function (req, res, next, cb) {
4446
where.push("cols.col_name = ANY(:col_name)");
4547
params["col_name"] = larkin.parseMultipleStrings(req.query.col_name);
4648
}
47-
if (req.query.project_id) {
48-
where.push("cols.project_id = ANY(:project_id)");
49-
params["project_id"] = larkin.parseMultipleIds(req.query.project_id);
50-
}
49+
50+
const [whereClauses, projectParams] = buildProjectsFilter(
51+
req,
52+
"cols.project_id",
53+
);
54+
where = where.concat(whereClauses);
55+
Object.assign(params, projectParams);
56+
57+
where.push("status_code = ANY(:status_code)");
5158
if (req.query.status_code || req.query.status) {
5259
// `status` parameter still works but has been superseded by `status_code`
5360
// multiple status codes can be provided
54-
where.push("status_code = ANY(:status_code)");
5561
params["status_code"] = larkin.parseMultipleIds(
5662
req.query.status_code ?? req.query.status,
5763
);
64+
} else {
65+
params["status_code"] = ["active"];
5866
}
5967

6068
if (where.length) {

‎v2/definitions/groups.ts‎

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
var api = require("../api"),
22
larkin = require("../larkin");
33

4+
import { buildProjectsFilter } from "../utils";
5+
46
module.exports = function (req, res, next, cb) {
57
if (Object.keys(req.query).length < 1) {
68
return larkin.info(req, res, next);
@@ -18,15 +20,16 @@ module.exports = function (req, res, next, cb) {
1820
if (req.query.col_group_id) {
1921
where.push("col_groups.id = ANY(:col_group_id)");
2022
params["col_group_id"] = larkin.parseMultipleIds(req.query.col_group_id);
21-
} else if (req.query.project_id) {
22-
where.push("cols.project_id = ANY(:project_id)");
23-
params["project_id"] = larkin.parseMultipleIds(req.query.project_id);
24-
} else if (req.query.col_id) {
25-
where.push("cols.id = ANY(:col_id)");
26-
params["col_id"] = larkin.parseMultipleIds(req.query.col_id);
2723
}
2824

29-
where = where.length ? "WHERE " + where.join(" AND ") : "";
25+
const [projectWhereClauses, projectParams] = buildProjectsFilter(
26+
req,
27+
"cols.project_id",
28+
);
29+
where = where.concat(projectWhereClauses);
30+
Object.assign(params, projectParams);
31+
32+
const whereClause = where.length ? "WHERE " + where.join(" AND ") : "";
3033

3134
let sql = `SELECT col_groups.id AS col_group_id,
3235
col_group,
@@ -37,7 +40,7 @@ module.exports = function (req, res, next, cb) {
3740
FROM macrostrat.col_groups
3841
LEFT JOIN macrostrat.cols ON cols.col_group_id = col_groups.id
3942
LEFT JOIN macrostrat.units_sections ON units_sections.col_id = cols.id
40-
${where}
43+
${whereClause}
4144
GROUP BY col_groups.id, cols.project_id `;
4245

4346
if ("sample" in req.query) {

‎v2/definitions/projects.ts‎

Lines changed: 70 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -7,62 +7,83 @@ module.exports = function (req, res, next, cb) {
77
}
88
//There will be a discrepancy with a key in production. Updated in_proccess_cols to in_process_cols key. Values are
99
//still the same.
10-
var sql = `WITH in_proc AS (
11-
SELECT COUNT(DISTINCT id) AS c, project_id
12-
FROM macrostrat.cols
13-
WHERE status_code = 'in process'
14-
GROUP BY project_id
15-
),
16-
obs AS (
17-
SELECT COUNT(DISTINCT id) AS co, project_id
18-
FROM macrostrat.cols
19-
WHERE status_code = 'obsolete'
20-
GROUP BY project_id
21-
),
22-
col_area_sum AS (
23-
SELECT project_id, SUM(col_area) AS total_area
24-
FROM macrostrat.cols
25-
WHERE status_code = 'active'
26-
GROUP BY project_id
27-
)
28-
29-
SELECT
30-
projects.id AS project_id,
31-
projects.project,
32-
projects.descrip,
33-
projects.timescale_id,
34-
COUNT(DISTINCT units_sections.col_id)::integer AS t_cols,
35-
COALESCE(c, 0)::integer AS in_process_cols,
36-
COALESCE(co, 0)::integer AS obsolete_cols,
37-
COUNT(DISTINCT units_sections.unit_id)::integer AS t_units,
38-
COALESCE(ROUND(total_area), 0)::integer AS area
39-
40-
FROM macrostrat.projects
41-
LEFT JOIN macrostrat.cols ON projects.id = cols.project_id
42-
LEFT JOIN macrostrat.units_sections ON units_sections.col_id = cols.id
43-
LEFT JOIN in_proc USING (project_id)
44-
LEFT JOIN obs USING (project_id)
45-
LEFT JOIN col_area_sum ON projects.id = col_area_sum.project_id
46-
`;
4710

48-
var where = [];
49-
var params = {};
11+
const where = [];
12+
let params = {};
5013

5114
if (req.query.project_id) {
5215
where.push("projects.id = ANY(:project_id)");
5316
params["project_id"] = larkin.parseMultipleIds(req.query.project_id);
5417
}
55-
if (where.length) {
56-
sql += ` WHERE ${where.join(" AND ")}`;
18+
19+
const whereStatement = where.length > 0 ? where.join(" AND ") : "true";
20+
21+
let sql = `
22+
SELECT
23+
p.id AS project_id,
24+
p.project,
25+
p.descrip,
26+
p.timescale_id,
27+
count(DISTINCT units_sections.col_id)::integer AS t_cols,
28+
count(DISTINCT cols.id) FILTER ( WHERE cols.status_code = 'active' )::integer AS active_cols,
29+
count(DISTINCT cols.id) FILTER ( WHERE cols.status_code = 'in process' )::integer AS in_process_cols,
30+
count(DISTINCT cols.id) FILTER ( WHERE cols.status_code = 'obsolete' )::integer AS obsolete_cols,
31+
count(DISTINCT units_sections.unit_id)::integer AS t_units,
32+
coalesce(round(sum(DISTINCT cols.col_area) FILTER ( WHERE cols.status_code = 'active')), 0) AS area
33+
FROM macrostrat.projects p
34+
LEFT JOIN macrostrat.cols ON p.id = cols.project_id
35+
LEFT JOIN macrostrat.units_sections ON units_sections.col_id = cols.id
36+
WHERE ${whereStatement}
37+
GROUP BY
38+
p.id,
39+
p.project,
40+
p.descrip,
41+
p.timescale_id
42+
`;
43+
44+
if (larkin.hasCapability("composite-projects")) {
45+
/** Progressive enhancement for composite projects **/
46+
sql = `
47+
WITH composite_tree AS (
48+
SELECT pt.parent_id, array_agg(pt.child_id) children, jsonb_agg(to_jsonb(p)) AS members
49+
FROM macrostrat.projects_tree pt
50+
JOIN LATERAL (
51+
SELECT p.id, p.slug, p.project name
52+
FROM macrostrat.projects p
53+
WHERE p.id = pt.child_id
54+
) AS p ON true
55+
GROUP BY pt.parent_id
56+
)
57+
SELECT
58+
p.id AS project_id,
59+
p.slug,
60+
p.project,
61+
p.descrip,
62+
p.timescale_id,
63+
ct.members,
64+
count(DISTINCT units_sections.col_id)::integer AS t_cols,
65+
count(DISTINCT cols.id) FILTER ( WHERE cols.status_code = 'active' )::integer AS active_cols,
66+
count(DISTINCT cols.id) FILTER ( WHERE cols.status_code = 'in process' )::integer AS in_process_cols,
67+
count(DISTINCT cols.id) FILTER ( WHERE cols.status_code = 'obsolete' )::integer AS obsolete_cols,
68+
count(DISTINCT units_sections.unit_id)::integer AS t_units,
69+
coalesce(round(sum(DISTINCT cols.col_area) FILTER ( WHERE cols.status_code = 'active')), 0) AS area
70+
FROM macrostrat.projects p
71+
LEFT JOIN composite_tree ct
72+
ON ct.parent_id = p.id
73+
LEFT JOIN macrostrat.cols ON p.id = cols.project_id
74+
OR (p.is_composite AND cols.project_id = ANY(ct.children))
75+
LEFT JOIN macrostrat.units_sections ON units_sections.col_id = cols.id
76+
WHERE ${whereStatement}
77+
GROUP BY
78+
p.id,
79+
p.project,
80+
p.descrip,
81+
p.timescale_id,
82+
p.slug,
83+
ct.children,
84+
ct.members
85+
`;
5786
}
58-
sql += `\nGROUP BY
59-
projects.id,
60-
projects.project,
61-
projects.descrip,
62-
projects.timescale_id,
63-
c,
64-
co,
65-
total_area;`;
6687

6788
larkin.queryPg("burwell", sql, params, function (error, data) {
6889
if (error) {

0 commit comments

Comments
 (0)