From 077a38ef8c9efcfabdd209b890b53643d2cc9617 Mon Sep 17 00:00:00 2001 From: Thomson Comer Date: Wed, 31 Aug 2022 18:11:40 -0500 Subject: [PATCH 01/43] Update for branch-22.08 --- dev/dockerfiles/devel/main.Dockerfile | 2 +- modules/cudf/src/column/json.cpp | 13 +++- modules/demo/api-server/README.md | 7 +- modules/demo/api-server/package.json | 2 +- .../api-server/routes/graphology/index.js | 64 +++++++++--------- modules/demo/api-server/test/fixtures.js | 8 +-- .../api-server/test/routes/graphology.test.js | 67 ++++++++++++++----- modules/demo/api-server/util/gpu_cache.js | 12 ++-- 8 files changed, 107 insertions(+), 68 deletions(-) diff --git a/dev/dockerfiles/devel/main.Dockerfile b/dev/dockerfiles/devel/main.Dockerfile index 1bd5c5944..47509060b 100644 --- a/dev/dockerfiles/devel/main.Dockerfile +++ b/dev/dockerfiles/devel/main.Dockerfile @@ -219,7 +219,7 @@ ARG NODE_WEBRTC_VERSION=0.4.7 RUN export DEBIAN_FRONTEND=noninteractive \ && apt update \ && apt install --no-install-recommends -y \ - jq entr ssh vim nano sudo less bash-completion \ + jq entr ssh vim nano sudo less bash-completion ripgrep fzf \ # X11 dependencies libxi-dev libxrandr-dev libxinerama-dev libxcursor-dev \ # node-canvas dependencies diff --git a/modules/cudf/src/column/json.cpp b/modules/cudf/src/column/json.cpp index 0cfd6ce32..86e5c642f 100644 --- a/modules/cudf/src/column/json.cpp +++ b/modules/cudf/src/column/json.cpp @@ -21,10 +21,17 @@ namespace nv { Column::wrapper_t Column::get_json_object(std::string const& json_path, rmm::mr::device_memory_resource* mr) { + auto options = cudf::strings::get_json_object_options{}; + options.set_missing_fields_as_nulls(true); try { - return Column::New(Env(), - cudf::strings::get_json_object( - this->view(), json_path, cudf::strings::get_json_object_options{}, mr)); + auto col = + Column::New(Env(), + cudf::strings::get_json_object( + this->view(), json_path, cudf::strings::get_json_object_options{}, mr)); + cudf::scalar& valid_count = *col->is_valid(mr)->sum(mr); + auto& count_scalar = static_cast&>(valid_count); + col->set_null_count(count_scalar.value()); + return col; } catch (std::exception const& e) { NAPI_THROW(Napi::Error::New(Env(), e.what())); } } diff --git a/modules/demo/api-server/README.md b/modules/demo/api-server/README.md index 332f85d2a..0afbcc40d 100644 --- a/modules/demo/api-server/README.md +++ b/modules/demo/api-server/README.md @@ -17,6 +17,7 @@ that utilizes this GPU-accelerated data for rendering larger datasets than available via only CPU. ## Main Dependencies + - @rapidsai/cudf - fastify - fastify-arrow @@ -34,6 +35,7 @@ yarn ``` To run the demo + ```bash # Select the api-server demo from the list of demos yarn demo @@ -44,7 +46,7 @@ yarn start ## Dataset -Run the graph generator at https://github.com/thomcom/sigma.js/blob/add-gpu-graph-to-example/examples/extra-large-graphs/generate-graph.js +Run the graph generator at to create a very large graph using the object ```js @@ -66,7 +68,7 @@ Which will create a file `./large-graph.json`. Copy `./large-graph.json` into `a API request to the location of the file relative to `routes/graphology/index.js`: ``` -curl http://localhost:3000/graphology/read_large_demo?filename=../../large-graph.json +curl http://localhost:3010/graphology/read_large_demo?filename=../../large-graph.json ``` Which will use parallel JSON parsing to load the graph onto the GPU. @@ -86,4 +88,3 @@ Which will use parallel JSON parsing to load the graph onto the GPU. /graphology/edges /graphology/release ``` - diff --git a/modules/demo/api-server/package.json b/modules/demo/api-server/package.json index 7b8f1743f..94584f1c5 100644 --- a/modules/demo/api-server/package.json +++ b/modules/demo/api-server/package.json @@ -28,7 +28,7 @@ "@fastify/cors": "latest", "@fastify/sensible": "^4.0.0", "@types/node": "17.0.33", - "fastify": "^3.0.0", + "fastify": "^4.0.0", "fastify-arrow": "1.0.0", "fastify-cli": "^3.0.1", "fastify-plugin": "^3.0.0" diff --git a/modules/demo/api-server/routes/graphology/index.js b/modules/demo/api-server/routes/graphology/index.js index bb0a31ead..8635c0db2 100644 --- a/modules/demo/api-server/routes/graphology/index.js +++ b/modules/demo/api-server/routes/graphology/index.js @@ -12,15 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -const Fs = require('fs'); -const {Utf8String, Int32, Uint32, Float32, DataFrame, Series, Float64} = - require('@rapidsai/cudf'); -const {RecordBatchStreamWriter, Field, Vector, List, Table} = require('apache-arrow'); -const Path = require('path'); -const {promisify} = require('util'); -const Stat = promisify(Fs.stat); -const fastifyCors = require('@fastify/cors'); -const fastify = require('fastify'); +const Fs = require('fs'); +const {Utf8String, Int32, Uint32, Float32, DataFrame, Series, Float64} = require('@rapidsai/cudf'); +const {RecordBatchStreamWriter, Field, Vector, List, Table} = require('apache-arrow'); +const Path = require('path'); +const {promisify} = require('util'); +const Stat = promisify(Fs.stat); +const fastifyCors = require('@fastify/cors'); +const fastify = require('fastify'); const arrowPlugin = require('fastify-arrow'); const gpu_cache = require('../../util/gpu_cache.js'); @@ -28,7 +27,7 @@ const root_schema = require('../../util/schema.js'); module.exports = async function(fastify, opts) { fastify.register(arrowPlugin); - fastify.register(fastifyCors, {origin: 'http://localhost:3000'}); + fastify.register(fastifyCors, {origin: 'http://localhost:3001'}); fastify.decorate('setDataframe', gpu_cache.setDataframe); fastify.decorate('getDataframe', gpu_cache.getDataframe); fastify.decorate('listDataframes', gpu_cache.listDataframes); @@ -177,7 +176,7 @@ module.exports = async function(fastify, opts) { const table = await fastify.getDataframe(request.params.table); if (table == undefined) { result.message = 'Table not found'; - reply.code(404).send(result); + await reply.code(404).send(result); } else { try { const name = request.params.column; @@ -186,16 +185,16 @@ module.exports = async function(fastify, opts) { newDfObject[name] = column; const result = new DataFrame(newDfObject); const writer = RecordBatchStreamWriter.writeAll(result.toArrow()); - reply.code(200).send(writer.toNodeStream()); + await reply.code(200).send(writer.toNodeStream()); } catch (e) { if (e.substring('Unknown column name') != -1) { result.message = e; console.log(result); - reply.code(404).send(result); + await reply.code(404).send(result); } else { result.message = e; console.log(result); - reply.code(500).send(result); + await reply.code(500).send(result); } } } @@ -212,10 +211,10 @@ module.exports = async function(fastify, opts) { const table = await fastify.getDataframe(request.params.table); if (table == undefined) { result.message = 'Table not found'; - reply.code(404).send(result); + await reply.code(404).send(result); } else { const writer = RecordBatchStreamWriter.writeAll(table.toArrow()); - reply.code(200).send(writer.toNodeStream()); + await reply.code(200).send(writer.toNodeStream()); } } }); @@ -229,7 +228,7 @@ module.exports = async function(fastify, opts) { const df = await fastify.getDataframe('nodes'); if (df == undefined) { result.message = 'Table not found'; - reply.code(404).send(result); + await reply.code(404).send(result); } else { // compute xmin, xmax, ymin, ymax const x = df.get('x'); @@ -240,7 +239,7 @@ module.exports = async function(fastify, opts) { result.bounds = {xmin: xmin, xmax: xmax, ymin: ymin, ymax: ymax}; result.message = 'Success'; result.success = true; - reply.code(200).send(result); + await reply.code(200).send(result); } } }); @@ -254,7 +253,7 @@ module.exports = async function(fastify, opts) { const df = await fastify.getDataframe('nodes'); if (df == undefined) { result.message = 'Table not found'; - reply.code(404).send(result); + await reply.code(404).send(result); } else { // tile x, y, size, color let tiled = Series.sequence({type: new Float32, init: 0.0, size: (4 * df.numRows)}); @@ -263,25 +262,23 @@ module.exports = async function(fastify, opts) { // Duplicatin the sigma.j createNormalizationFunction here because there's no other way // to let the Graph object compute it. // - let x = df.get('x'); - let y = df.get('y'); + const x = df.get('x'); + const y = df.get('y'); let color = df.get('color'); + const color_ints = color.hexToIntegers(new Uint32).bitwiseOr(0xef000000); const [xMin, xMax] = x.minmax(); const [yMin, yMax] = y.minmax(); const ratio = Math.max(xMax - xMin, yMax - yMin); const dX = (xMax + xMin) / 2.0; const dY = (yMax + yMin) / 2.0; - x = x.add(-1.0 * dX).mul(1.0 / ratio).add(0.5); - y = y.add(-1.0 * dY).mul(1.0 / ratio).add(0.5); - tiled = tiled.scatter(x, base_offset.cast(new Int32)); - tiled = tiled.scatter(y, base_offset.add(1).cast(new Int32)); + const x_scaled = x.add(-1.0 * dX).mul(1.0 / ratio).add(0.5); + const y_scaled = y.add(-1.0 * dY).mul(1.0 / ratio).add(0.5); + tiled = tiled.scatter(x_scaled, base_offset.cast(new Int32)); + tiled = tiled.scatter(y_scaled, base_offset.add(1).cast(new Int32)); tiled = tiled.scatter(df.get('size').mul(2), base_offset.add(2).cast(new Int32)); - color = color.hexToIntegers(new Uint32).bitwiseOr(0xef000000); - // color = Series.sequence({size: color.length, type: new Int32, init: 0xff0000ff, step: - // 0}); - tiled = tiled.scatter(color.view(new Float32), base_offset.add(3).cast(new Int32)); + tiled = tiled.scatter(color_ints.view(new Float32), base_offset.add(3).cast(new Int32)); const writer = RecordBatchStreamWriter.writeAll(new DataFrame({nodes: tiled}).toArrow()); - reply.code(200).send(writer.toNodeStream()); + await reply.code(200).send(writer.toNodeStream()); } } }); @@ -298,7 +295,7 @@ module.exports = async function(fastify, opts) { const edges = await fastify.getDataframe('edges'); if (df == undefined) { result.message = 'Table not found'; - reply.code(404).send(result); + await reply.code(404).send(result); } else { // tile x, y, size, color let tiled = Series.sequence({type: new Float32, init: 0.0, size: (6 * edges.numRows)}); @@ -309,6 +306,7 @@ module.exports = async function(fastify, opts) { // // Remap the indices in the key table to their real targets. See // https://github.com/rapidsai/node/issue/397 + /** Series */ let keys = df.get('key'); let source_map = edges.get('source'); let source = keys.gather(source_map, false); @@ -343,7 +341,7 @@ module.exports = async function(fastify, opts) { Series.new(base_offset.mul(2).add(4)).cast(new Int32)); tiled = tiled.scatter(color, Series.new(base_offset.mul(2).add(5).cast(new Int32))); const writer = RecordBatchStreamWriter.writeAll(new DataFrame({edges: tiled}).toArrow()); - reply.code(200).send(writer.toNodeStream()); + await reply.code(200).send(writer.toNodeStream()); } } }); @@ -353,7 +351,7 @@ module.exports = async function(fastify, opts) { url: '/release', handler: async (request, reply) => { await fastify.clearDataFrames(); - reply.code(200).send({message: 'OK'}) + await reply.code(200).send({message: 'OK'}) } }); } diff --git a/modules/demo/api-server/test/fixtures.js b/modules/demo/api-server/test/fixtures.js index d3de113d5..330bfd82e 100644 --- a/modules/demo/api-server/test/fixtures.js +++ b/modules/demo/api-server/test/fixtures.js @@ -84,8 +84,8 @@ const json_large = { } ], "edges": [ - {"key": "geid_115_98", "source": "291", "target": "290"}, - {"key": "geid_115_99", "source": "290", "target": "291"} + {"key": "geid_115_98", "source": "1", "target": "0"}, + {"key": "geid_115_99", "source": "0", "target": "1"} ], "options": {"type": "mixed", "multi": false, "allowSelfLoops": true} }` @@ -120,8 +120,8 @@ const json_out_of_order = { } ], "edges": [ - {"key": "geid_115_98", "source": "290", "target": "291"}, - {"key": "geid_115_99", "source": "291", "target": "290"} + {"key": "geid_115_98", "source": "0", "target": "1"}, + {"key": "geid_115_99", "source": "1", "target": "0"} ], "options": {"type": "mixed", "multi": false, "allowSelfLoops": true} }` diff --git a/modules/demo/api-server/test/routes/graphology.test.js b/modules/demo/api-server/test/routes/graphology.test.js index b976e46f2..eb53af646 100644 --- a/modules/demo/api-server/test/routes/graphology.test.js +++ b/modules/demo/api-server/test/routes/graphology.test.js @@ -193,7 +193,7 @@ test('nodes', async (t) => { const res = await app.inject( {method: 'GET', url: '/graphology/nodes', headers: {'accepts': 'application/octet-stream'}}); t.same(res.statusCode, 200); - const table = tableFromIPC(res.rawPayload); + const table = await tableFromIPC(res.rawPayload); t.ok(table.getChild('nodes')); t.same(table.getChild('nodes').toArray(), new Float32Array([ 0.02944733388721943, @@ -205,25 +205,62 @@ test('nodes', async (t) => { 2, -5.515159729197043e+28 ])) + const release = await app.inject({method: 'POST', url: '/graphology/release'}); }); test('nodes/bounds', async (t) => { - const dir = t.testdir(json_good); - const rpath = '../../test/routes/' + dir.substring(dir.lastIndexOf('/')) + '/json_good.txt'; + const dir = t.testdir(json_large); + const rpath = '../../test/routes/' + dir.substring(dir.lastIndexOf('/')) + '/json_large.txt'; const app = await build(t); - const load = await app.inject({method: 'POST', url: '/graphology/read_json?filename=' + rpath}); - const res = await app.inject({method: 'GET', url: '/graphology/nodes/bounds'}); - const release = await app.inject({method: 'POST', url: '/graphology/release'}); + const load = + await app.inject({method: 'POST', url: '/graphology/read_large_demo?filename=' + rpath}); + const res = await app.inject({method: 'GET', url: '/graphology/nodes/bounds'}); t.same(JSON.parse(res.payload), { 'success': true, 'message': 'Success', 'bounds': { - 'xmin': -278.2200012207031, - 'xmax': -1.9823756217956543, - 'ymin': 250.4990692138672, - 'ymax': 436.01004028320307 + 'xmin': -13.364311218261719, + 'xmax': 1.3836898803710938, + 'ymin': -11.536596298217773, + 'ymax': 4.134339332580566, } }); + const release = await app.inject({method: 'POST', url: '/graphology/release'}); +}); + +test('nodes then nodes/bounds', async (t) => { + const dir = t.testdir(json_large); + const rpath = '../../test/routes/' + dir.substring(dir.lastIndexOf('/')) + '/json_large.txt'; + const app = await build(t); + const load = + await app.inject({method: 'POST', url: '/graphology/read_large_demo?filename=' + rpath}); + const nodes = await app.inject( + {method: 'GET', url: '/graphology/nodes', headers: {'accepts': 'application/octet-stream'}}); + t.same(nodes.statusCode, 200); + const table = await tableFromIPC(nodes.rawPayload); + t.ok(table.getChild('nodes')); + t.same(table.getChild('nodes').toArray(), new Float32Array([ + 0.02944733388721943, + 1, + 0, + -1.4006860109112203e+29, + 0.9705526828765869, + 0, + 2, + -5.515159729197043e+28 + ])) + const bounds = await app.inject({method: 'GET', url: '/graphology/nodes/bounds'}); + t.same(JSON.parse(bounds.payload), { + 'success': true, + 'message': 'Success', + 'bounds': { + 'xmin': -13.364311218261719, + 'xmax': 1.3836898803710938, + 'ymin': -11.536596298217773, + 'ymax': 4.134339332580566, + } + }); + const release = await app.inject({method: 'POST', url: '/graphology/release'}); }); test('edges', async (t) => { @@ -239,8 +276,8 @@ test('edges', async (t) => { const release = await app.inject({method: 'POST', url: '/graphology/release'}); t.ok(table.getChild('edges')); t.same(table.getChild('edges').toArray(), new Float32Array([ - 0.02944733388721943, - 1, + 0.9705526828765869, + 0, -1.701910173408654e+38, 0.02944733388721943, 1, @@ -248,13 +285,13 @@ test('edges', async (t) => { 0.02944733388721943, 1, -1.701910173408654e+38, - 0.02944733388721943, - 1, + 0.9705526828765869, + 0, -1.701910173408654e+38 ])) }); -test('edges out of order', async (t) => { +test('edges out of order', {only: true}, async (t) => { const dir = t.testdir(json_out_of_order); const rpath = '../../test/routes/' + dir.substring(dir.lastIndexOf('/')) + '/json_out_of_order.txt'; diff --git a/modules/demo/api-server/util/gpu_cache.js b/modules/demo/api-server/util/gpu_cache.js index edcaeb032..1188346a6 100644 --- a/modules/demo/api-server/util/gpu_cache.js +++ b/modules/demo/api-server/util/gpu_cache.js @@ -29,12 +29,10 @@ function json_key_attributes_to_dataframe(str) { const no_open_list = str.split('[\n').gather([1], false); const tokenized = no_open_list.split('},'); const keys = tokenized.getJSONObject('.key'); - keys.setNullMask(1, 0); - arr['key'] = keys.cast(new Int32); + arr['key'] = keys.cast(new Int32); columns.forEach((col, ix) => { const parse_result = tokenized.getJSONObject('.attributes.' + columns[ix]); - parse_result.setNullMask([], 0); - arr[col] = parse_result.cast(dtypes[ix]); + arr[col] = parse_result.cast(dtypes[ix]); }); const result = new DataFrame(arr); return result; @@ -46,8 +44,7 @@ function json_aos_to_dataframe(str, columns, dtypes) { const no_open_list = str.split('[\n').gather([1], false); const tokenized = no_open_list.split('},'); const parse_result = tokenized.getJSONObject('.' + columns[ix]); - parse_result.setNullMask(1, 0); - arr[col] = parse_result.cast(dtypes[ix]); + arr[col] = parse_result.cast(dtypes[ix]); }); const result = new DataFrame(arr); return result; @@ -60,8 +57,7 @@ function json_aoa_to_dataframe(str, dtypes) { dtypes.forEach((_, ix) => { const get_ix = `[${ix}]`; const parse_result = tokenized.getJSONObject(get_ix); - parse_result.setNullMask([], 0); - arr[ix] = parse_result.cast(dtypes[ix]); + arr[ix] = parse_result.cast(dtypes[ix]); }); const result = new DataFrame(arr); return result; From 99d76adffa842d1f51f4b56f9f3283a91ba1bcbf Mon Sep 17 00:00:00 2001 From: "H. Thomson Comer" Date: Thu, 1 Sep 2022 16:28:05 -0500 Subject: [PATCH 02/43] Modify edge mapper to use a table .join, with error handling, instead of blindly gathering. --- .../api-server/routes/graphology/index.js | 117 +++++++++++------- modules/demo/api-server/test/fixtures.js | 39 +++++- .../api-server/test/routes/graphology.test.js | 42 +++++-- 3 files changed, 139 insertions(+), 59 deletions(-) diff --git a/modules/demo/api-server/routes/graphology/index.js b/modules/demo/api-server/routes/graphology/index.js index 8635c0db2..7ae0ce1dd 100644 --- a/modules/demo/api-server/routes/graphology/index.js +++ b/modules/demo/api-server/routes/graphology/index.js @@ -297,51 +297,78 @@ module.exports = async function(fastify, opts) { result.message = 'Table not found'; await reply.code(404).send(result); } else { - // tile x, y, size, color - let tiled = Series.sequence({type: new Float32, init: 0.0, size: (6 * edges.numRows)}); - let base_offset = Series.sequence({type: new Int32, init: 0.0, size: edges.numRows}).mul(3); - // - // Duplicatin the sigma.js createNormalizationFunction here because this is the best way - // to let the Graph object compute it on GPU. - // - // Remap the indices in the key table to their real targets. See - // https://github.com/rapidsai/node/issue/397 - /** Series */ - let keys = df.get('key'); - let source_map = edges.get('source'); - let source = keys.gather(source_map, false); - let target_map = edges.get('target'); - let target = keys.gather(target_map, false); - let x = df.get('x'); - let y = df.get('y'); - const [xMin, xMax] = x.minmax(); - const [yMin, yMax] = y.minmax(); - const ratio = Math.max(xMax - xMin, yMax - yMin); - const dX = (xMax + xMin) / 2.0; - const dY = (yMax + yMin) / 2.0; - x = x.add(-1.0 * dX).mul(1.0 / ratio).add(0.5); - y = y.add(-1.0 * dY).mul(1.0 / ratio).add(0.5); - const source_xmap = x.gather(source, false); - const source_ymap = y.gather(source, false); - const target_xmap = x.gather(target, false); - const target_ymap = y.gather(target, false); - const color = Series.new(['#999']) - .hexToIntegers(new Int32) - .bitwiseOr(0xff000000) - .view(new Float32) - .toArray()[0]; - tiled = - tiled.scatter(Series.new(source_xmap), Series.new(base_offset.mul(2)).cast(new Int32)); - tiled = tiled.scatter(Series.new(source_ymap), - Series.new(base_offset.mul(2).add(1)).cast(new Int32)); - tiled = tiled.scatter(color, Series.new(base_offset.mul(2).add(2).cast(new Int32))); - tiled = tiled.scatter(Series.new(target_xmap), - Series.new(base_offset.mul(2).add(3)).cast(new Int32)); - tiled = tiled.scatter(Series.new(target_ymap), - Series.new(base_offset.mul(2).add(4)).cast(new Int32)); - tiled = tiled.scatter(color, Series.new(base_offset.mul(2).add(5).cast(new Int32))); - const writer = RecordBatchStreamWriter.writeAll(new DataFrame({edges: tiled}).toArrow()); - await reply.code(200).send(writer.toNodeStream()); + try { + // tile x, y, size, color + let tiled = Series.sequence({type: new Float32, init: 0.0, size: (6 * edges.numRows)}); + let base_offset = + Series.sequence({type: new Int32, init: 0.0, size: edges.numRows}).mul(3); + // + // Duplicatin the sigma.js createNormalizationFunction here because this is the best way + // to let the Graph object compute it on GPU. + // + // Remap the indices in the key table to their real targets. See + // https://github.com/rapidsai/node/issue/397 + /** Series */ + const keys = df.get('key'); + const keys_df = new DataFrame({'keys': keys}); + const source_unordered = edges.get('source'); + const target_unordered = edges.get('target'); + source_df = new DataFrame({ + 'keys': source_unordered, + 'idx': Series.sequence({size: source_unordered.length, init: 0}) + }); + target_df = new DataFrame({ + 'keys': target_unordered, + 'idx': Series.sequence({size: target_unordered.length, init: 0}) + }); + const source_idx_df = keys_df.join({other: source_df, on: ['keys'], how: 'left'}); + const target_idx_df = keys_df.join({other: target_df, on: ['keys'], how: 'left'}); + /** Series */ + let source_map = source_idx_df.get('idx') + let target_map = target_idx_df.get('idx') + if (source_map.nullCount > 0) { throw 'Edge sources do not match node keys'; } + if (target_map.nullCount > 0) { throw 'Edge targets do not match node keys'; } + let x = df.get('x'); + let y = df.get('y'); + const [xMin, xMax] = x.minmax(); + const [yMin, yMax] = y.minmax(); + const ratio = Math.max(xMax - xMin, yMax - yMin); + const dX = (xMax + xMin) / 2.0; + const dY = (yMax + yMin) / 2.0; + x = x.add(-1.0 * dX).mul(1.0 / ratio).add(0.5); + y = y.add(-1.0 * dY).mul(1.0 / ratio).add(0.5); + + const source_xmap = x.gather(source_map, false); + const source_ymap = y.gather(source_map, false); + const target_xmap = x.gather(target_map, false); + const target_ymap = y.gather(target_map, false); + const color = Series.new(['#999']) + .hexToIntegers(new Int32) + .bitwiseOr(0xff000000) + .view(new Float32) + .toArray()[0]; + tiled = + tiled.scatter(Series.new(source_xmap), Series.new(base_offset.mul(2)).cast(new Int32)); + tiled = tiled.scatter(Series.new(source_ymap), + Series.new(base_offset.mul(2).add(1)).cast(new Int32)); + tiled = tiled.scatter(color, Series.new(base_offset.mul(2).add(2).cast(new Int32))); + tiled = tiled.scatter(Series.new(target_xmap), + Series.new(base_offset.mul(2).add(3)).cast(new Int32)); + tiled = tiled.scatter(Series.new(target_ymap), + Series.new(base_offset.mul(2).add(4)).cast(new Int32)); + tiled = tiled.scatter(color, Series.new(base_offset.mul(2).add(5).cast(new Int32))); + const writer = RecordBatchStreamWriter.writeAll(new DataFrame({edges: tiled}).toArrow()); + await reply.code(200).send(writer.toNodeStream()); + } catch (e) { + if (e.includes('do not match') >= 0) { + result.statusCode = 422; + } else { + result.statusCode = 500; + } + result.success = false; + result.message = e; + await reply.code(result.statusCode).send(result); + } } } }); diff --git a/modules/demo/api-server/test/fixtures.js b/modules/demo/api-server/test/fixtures.js index 330bfd82e..35df27442 100644 --- a/modules/demo/api-server/test/fixtures.js +++ b/modules/demo/api-server/test/fixtures.js @@ -93,6 +93,42 @@ const json_large = { const json_out_of_order = { 'json_out_of_order.txt': + ` { + "attributes": {}, + "nodes": [ + { + "key": "290", + "attributes": { + "cluster": 0, + "x": -13.364310772761677, + "y": 4.134339113107921, + "size": 0, + "label": "Node n°291, in cluster n°0", + "color": "#e24b04" + } + }, + { + "key": "291", + "attributes": { + "cluster": 1, + "x": 1.3836898237261988, + "y": -11.536596764896206, + "size": 1, + "label": "Node n°292, in cluster n°1", + "color": "#323455" + } + } + ], + "edges": [ + {"key": "geid_115_98", "source": "290", "target": "291"}, + {"key": "geid_115_99", "source": "291", "target": "290"} + ], + "options": {"type": "mixed", "multi": false, "allowSelfLoops": true} + }` +}; + +const json_bad_map = { + 'json_bad_map.txt': ` { "attributes": {}, "nodes": [ @@ -130,5 +166,6 @@ const json_out_of_order = { module.exports = { json_good: json_good, json_large: json_large, - json_out_of_order: json_out_of_order + json_out_of_order: json_out_of_order, + json_bad_map: json_bad_map, }; diff --git a/modules/demo/api-server/test/routes/graphology.test.js b/modules/demo/api-server/test/routes/graphology.test.js index eb53af646..f0242fdc8 100644 --- a/modules/demo/api-server/test/routes/graphology.test.js +++ b/modules/demo/api-server/test/routes/graphology.test.js @@ -14,11 +14,11 @@ 'use strict' -const {dir} = require('console'); -const {test} = require('tap'); -const {build} = require('../helper'); -const {tableFromIPC, RecordBatchStreamWriter} = require('apache-arrow'); -const {json_large, json_good, json_out_of_order} = require('../fixtures.js'); +const {dir} = require('console'); +const {test} = require('tap'); +const {build} = require('../helper'); +const {tableFromIPC, RecordBatchStreamWriter} = require('apache-arrow'); +const {json_large, json_good, json_out_of_order, json_bad_map} = require('../fixtures.js'); test('graphology root returns api description', async t => { const app = await build(t); @@ -291,7 +291,7 @@ test('edges', async (t) => { ])) }); -test('edges out of order', {only: true}, async (t) => { +test('edges and nodes do not begin with 0', async (t) => { const dir = t.testdir(json_out_of_order); const rpath = '../../test/routes/' + dir.substring(dir.lastIndexOf('/')) + '/json_out_of_order.txt'; @@ -300,22 +300,38 @@ test('edges out of order', {only: true}, async (t) => { await app.inject({method: 'POST', url: '/graphology/read_large_demo?filename=' + rpath}); const res = await app.inject( {method: 'GET', url: '/graphology/edges', header: {'accepts': 'application/octet-stream'}}); - t.equal(res.statusCode, 200); - const table = tableFromIPC(res.rawPayload); const release = await app.inject({method: 'POST', url: '/graphology/release'}); + debugger; + t.equal(res.statusCode, 200); + const table = tableFromIPC(res.rawPayload); t.ok(table.getChild('edges')); t.same(table.getChild('edges').toArray(), new Float32Array([ 0.02944733388721943, 1, -1.701910173408654e+38, - 0.02944733388721943, - 1, + 0.9705526828765869, + 0, -1.701910173408654e+38, - 0.02944733388721943, - 1, + 0.9705526828765869, + 0, -1.701910173408654e+38, 0.02944733388721943, 1, - -1.701910173408654e+38 + -1.701910173408654e+38, ])) }); + +test('edge keys do not match node keys', {only: true}, async (t) => { + const dir = t.testdir(json_bad_map); + const rpath = '../../test/routes/' + dir.substring(dir.lastIndexOf('/')) + '/json_bad_map.txt'; + const app = await build(t); + const load = + await app.inject({method: 'POST', url: '/graphology/read_large_demo?filename=' + rpath}); + const res = await app.inject( + {method: 'GET', url: '/graphology/edges', header: {'accepts': 'application/octet-stream'}}); + const release = await app.inject({method: 'POST', url: '/graphology/release'}); + debugger; + t.equal(res.statusCode, 422); + t.same(JSON.parse(res.payload), + {success: false, message: 'Edge sources do not match node keys', statusCode: 422}); +}); From 7f699398ea53b5483ec677370f1a42d47bb1d4d0 Mon Sep 17 00:00:00 2001 From: Thomson Comer Date: Fri, 2 Sep 2022 10:08:31 -0500 Subject: [PATCH 03/43] Remove set_missing_fields_as_nulls workaround. --- modules/cudf/src/column/json.cpp | 2 -- modules/demo/api-server/routes/graphology/index.js | 9 +++------ 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/modules/cudf/src/column/json.cpp b/modules/cudf/src/column/json.cpp index 86e5c642f..1b79cc3ac 100644 --- a/modules/cudf/src/column/json.cpp +++ b/modules/cudf/src/column/json.cpp @@ -21,8 +21,6 @@ namespace nv { Column::wrapper_t Column::get_json_object(std::string const& json_path, rmm::mr::device_memory_resource* mr) { - auto options = cudf::strings::get_json_object_options{}; - options.set_missing_fields_as_nulls(true); try { auto col = Column::New(Env(), diff --git a/modules/demo/api-server/routes/graphology/index.js b/modules/demo/api-server/routes/graphology/index.js index 7ae0ce1dd..42286fcff 100644 --- a/modules/demo/api-server/routes/graphology/index.js +++ b/modules/demo/api-server/routes/graphology/index.js @@ -306,9 +306,7 @@ module.exports = async function(fastify, opts) { // Duplicatin the sigma.js createNormalizationFunction here because this is the best way // to let the Graph object compute it on GPU. // - // Remap the indices in the key table to their real targets. See - // https://github.com/rapidsai/node/issue/397 - /** Series */ + // Remap the indices in the key table to their real targets. const keys = df.get('key'); const keys_df = new DataFrame({'keys': keys}); const source_unordered = edges.get('source'); @@ -323,9 +321,8 @@ module.exports = async function(fastify, opts) { }); const source_idx_df = keys_df.join({other: source_df, on: ['keys'], how: 'left'}); const target_idx_df = keys_df.join({other: target_df, on: ['keys'], how: 'left'}); - /** Series */ - let source_map = source_idx_df.get('idx') - let target_map = target_idx_df.get('idx') + let source_map = source_idx_df.get('idx') + let target_map = target_idx_df.get('idx') if (source_map.nullCount > 0) { throw 'Edge sources do not match node keys'; } if (target_map.nullCount > 0) { throw 'Edge targets do not match node keys'; } let x = df.get('x'); From adda8109b03b249f2368f4fc75817bd11cb90bca Mon Sep 17 00:00:00 2001 From: "H. Thomson Comer" Date: Fri, 2 Sep 2022 11:10:26 -0500 Subject: [PATCH 04/43] First bits of template for abstract request handling. --- modules/demo/api-server/index.js | 2 +- modules/demo/api-server/util/schema.js | 11 ++ yarn.lock | 189 ++++++++++++++++++++++++- 3 files changed, 196 insertions(+), 6 deletions(-) diff --git a/modules/demo/api-server/index.js b/modules/demo/api-server/index.js index 0d97cdbc5..c7ef575d2 100755 --- a/modules/demo/api-server/index.js +++ b/modules/demo/api-server/index.js @@ -19,7 +19,7 @@ const Path = require('path'); // Change cwd to the example dir so relative file paths are resolved process.chdir(__dirname); -const fastify = require.resolve('fastify-cli/cli.js'); +const fastify = require.resolve('fastify-cli/cli.js')({logger: true}); const {spawnSync} = require('child_process'); diff --git a/modules/demo/api-server/util/schema.js b/modules/demo/api-server/util/schema.js index 1756ded74..b9589b888 100644 --- a/modules/demo/api-server/util/schema.js +++ b/modules/demo/api-server/util/schema.js @@ -15,6 +15,17 @@ 'use strict'; const schema = { + gpu: { + description: 'An abstract interface to the node-rapids api, supported by a server.', + schema: { + '/': { + method: 'The name of the method to apply to gpu_cache data.', + caller: 'Either an object that has been stored in the gpu_cache or a static module name.', + arguments: 'Correctly specified arguments to the gpu_cache method.', + result: 'Either a result code specifying success or failure or an arrow data buffer.', + } + } + }, graphology: { description: 'The graphology api provides GPU acceleration of graphology datasets.', schema: { diff --git a/yarn.lock b/yarn.lock index 7c406df63..64e81a6ee 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1298,6 +1298,15 @@ dependencies: ajv "^6.12.6" +"@fastify/ajv-compiler@^3.1.1": + version "3.2.0" + resolved "https://registry.npmjs.org/@fastify/ajv-compiler/-/ajv-compiler-3.2.0.tgz#a165ffb877fe239571a68f7b22143034176dcb8a" + integrity sha512-JrqgKmZoh1AJojDZk699DupQ9+tz5gSy7/w+5DrkXy5whM5IcqdV3SjG5qnOqgVJT1nPtUMDY0xYus2j6vwJiw== + dependencies: + ajv "^8.10.0" + ajv-formats "^2.1.1" + fast-uri "^2.0.0" + "@fastify/autoload@^4.0.0": version "4.0.1" resolved "https://registry.npmjs.org/@fastify/autoload/-/autoload-4.0.1.tgz#7b3008af96ef0cd20926e01b04aaae0a2c4e225b" @@ -1321,11 +1330,28 @@ fastify-plugin "^4.0.0" mnemonist "0.39.2" +"@fastify/deepmerge@^1.0.0": + version "1.1.0" + resolved "https://registry.npmjs.org/@fastify/deepmerge/-/deepmerge-1.1.0.tgz#91f0a5a27034ff76b7bece63a5906894940ace82" + integrity sha512-E8Hfdvs1bG6u0N4vN5Nty6JONUfTdOciyD5rn8KnEsLKIenvOVcr210BQR9t34PRkNyjqnMLGk3e0BsaxRdL+g== + "@fastify/error@^2.0.0": version "2.0.0" resolved "https://registry.npmjs.org/@fastify/error/-/error-2.0.0.tgz#a9f94af56eb934f0ab1ce4ef9f0ced6ebf2319dc" integrity sha512-wI3fpfDT0t7p8E6dA2eTECzzOd+bZsZCJ2Hcv+Onn2b7ZwK3RwD27uW2QDaMtQhAfWQQP+WNK7nKf0twLsBf9w== +"@fastify/error@^3.0.0": + version "3.0.0" + resolved "https://registry.npmjs.org/@fastify/error/-/error-3.0.0.tgz#bfcb7b33cec0196413083a91ef2edc7b2c88455b" + integrity sha512-dPRyT40GiHRzSCll3/Jn2nPe25+E1VXc9tDwRAIKwFCxd5Np5wzgz1tmooWG3sV0qKgrBibihVoCna2ru4SEFg== + +"@fastify/fast-json-stringify-compiler@^4.0.0": + version "4.1.0" + resolved "https://registry.npmjs.org/@fastify/fast-json-stringify-compiler/-/fast-json-stringify-compiler-4.1.0.tgz#ebf657ce4ec88e27ba311f7560eaa0b37de8719d" + integrity sha512-cTKBV2J9+u6VaKDhX7HepSfPSzw+F+TSd+k0wzifj4rG+4E5PjSFJCk19P8R6tr/72cuzgGd+mbB3jFT6lvAgw== + dependencies: + fast-json-stringify "^5.0.0" + "@fastify/multipart@^6.0.0": version "6.0.0" resolved "https://registry.npmjs.org/@fastify/multipart/-/multipart-6.0.0.tgz#da7e80b589b3874b3964145ec6e13fd69bd7e6cf" @@ -3645,7 +3671,14 @@ abbrev@1: resolved "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz#f8f2c887ad10bf67f634f005b6987fed3179aac8" integrity sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q== -abstract-logging@^2.0.0: +abort-controller@^3.0.0: + version "3.0.0" + resolved "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz#eaf54d53b62bae4138e809ca225c8439a6efb392" + integrity sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg== + dependencies: + event-target-shim "^5.0.0" + +abstract-logging@^2.0.0, abstract-logging@^2.0.1: version "2.0.1" resolved "https://registry.npmjs.org/abstract-logging/-/abstract-logging-2.0.1.tgz#6b0c371df212db7129b57d2e7fcf282b8bf1c839" integrity sha512-2BjRTZxTPvheOvGbBslFSYOUkr+SjPtOnrLP33f+VIWLzezQpZcqVg7ja3L4dBXmzzgwT+a029jRx5PCi3JuiA== @@ -3722,6 +3755,13 @@ aggregate-error@^3.0.0: clean-stack "^2.0.0" indent-string "^4.0.0" +ajv-formats@^2.1.1: + version "2.1.1" + resolved "https://registry.npmjs.org/ajv-formats/-/ajv-formats-2.1.1.tgz#6e669400659eb74973bbf2e33327180a0996b520" + integrity sha512-Wx0Kx52hxE7C18hkMEggYlEifqWZtYaRgouJor+WMdPnQyEK13vgEWyVNup7SoeeoLMsr4kf5h6dOW11I15MUA== + dependencies: + ajv "^8.0.0" + ajv@^6.10.0, ajv@^6.11.0, ajv@^6.12.3, ajv@^6.12.4, ajv@^6.12.6: version "6.12.6" resolved "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz#baf5a62e802b07d977034586f8c3baf5adf26df4" @@ -3732,7 +3772,7 @@ ajv@^6.10.0, ajv@^6.11.0, ajv@^6.12.3, ajv@^6.12.4, ajv@^6.12.6: json-schema-traverse "^0.4.1" uri-js "^4.2.2" -ajv@^8.1.0: +ajv@^8.0.0, ajv@^8.1.0, ajv@^8.10.0: version "8.11.0" resolved "https://registry.npmjs.org/ajv/-/ajv-8.11.0.tgz#977e91dd96ca669f54a11e23e378e33b884a565f" integrity sha512-wGgprdCvMalC0BztXvitD2hC04YffAvtsUn93JbGXYLAtCUO4xd17mCCZQxUOItiBwZvJScWo8NIvQMQ71rdpg== @@ -4148,6 +4188,15 @@ avvio@^7.1.2: fastq "^1.6.1" queue-microtask "^1.1.2" +avvio@^8.1.3: + version "8.2.0" + resolved "https://registry.npmjs.org/avvio/-/avvio-8.2.0.tgz#aff28b0266617bf07ffc1c2d5f4220c3663ce1c2" + integrity sha512-bbCQdg7bpEv6kGH41RO/3B2/GMMmJSo2iBK+X8AWN9mujtfUipMDfIjsgHCfpnKqoGEQrrmCDKSa5OQ19+fDmg== + dependencies: + archy "^1.0.0" + debug "^4.0.0" + fastq "^1.6.1" + aws-sign2@~0.7.0: version "0.7.0" resolved "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz#b46e890934a9591f2d2f6f86d7e6a9f1b3fe76a8" @@ -6943,6 +6992,11 @@ eve@~0.5.1: resolved "https://registry.npmjs.org/eve/-/eve-0.5.4.tgz#67d080b9725291d7e389e34c26860dd97f1debaa" integrity sha512-aqprQ9MAOh1t66PrHxDFmMXPlgNO6Uv1uqvxmwjprQV50jaQ2RqO7O1neY4PJwC+hMnkyMDphu2AQPOPZdjQog== +event-target-shim@^5.0.0: + version "5.0.1" + resolved "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz#5d4d3ebdf9583d63a5333ce2deb7480ab2b05789" + integrity sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ== + eventemitter3@^3.1.0: version "3.1.2" resolved "https://registry.npmjs.org/eventemitter3/-/eventemitter3-3.1.2.tgz#2d3d48f9c346698fce83a85d7d664e98535df6e7" @@ -7156,12 +7210,24 @@ fast-json-stringify@^2.5.2: rfdc "^1.2.0" string-similarity "^4.0.1" +fast-json-stringify@^5.0.0: + version "5.2.0" + resolved "https://registry.npmjs.org/fast-json-stringify/-/fast-json-stringify-5.2.0.tgz#0a637b698db602cd8f0c931b0025cf4858e9f5ce" + integrity sha512-u5jtrcAK9RINW15iuDKnsuuhqmqre4AmDMp3crRTjUMdAuHMpQUt3IfoMm5wlJm59b74PcajqOl3SjgnC5FPmw== + dependencies: + "@fastify/deepmerge" "^1.0.0" + ajv "^8.10.0" + ajv-formats "^2.1.1" + fast-deep-equal "^3.1.3" + fast-uri "^2.1.0" + rfdc "^1.2.0" + fast-levenshtein@^2.0.6, fast-levenshtein@~2.0.6: version "2.0.6" resolved "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz#3d8a5c66883a16a30ca8643e851f19baa7797917" integrity sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw== -fast-redact@^3.0.0: +fast-redact@^3.0.0, fast-redact@^3.1.1: version "3.1.2" resolved "https://registry.npmjs.org/fast-redact/-/fast-redact-3.1.2.tgz#d58e69e9084ce9fa4c1a6fa98a3e1ecf5d7839aa" integrity sha512-+0em+Iya9fKGfEQGcd62Yv6onjBmmhV1uh86XVfOU8VwAe6kaFdQCWI9s0/Nnugx5Vd9tdbZ7e6gE2tR9dzXdw== @@ -7171,6 +7237,11 @@ fast-safe-stringify@^2.0.8: resolved "https://registry.npmjs.org/fast-safe-stringify/-/fast-safe-stringify-2.1.1.tgz#c406a83b6e70d9e35ce3b30a81141df30aeba884" integrity sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA== +fast-uri@^2.0.0, fast-uri@^2.1.0: + version "2.1.0" + resolved "https://registry.npmjs.org/fast-uri/-/fast-uri-2.1.0.tgz#9279432d6b53675c90116b947ed2bbba582d6fb5" + integrity sha512-qKRta6N7BWEFVlyonVY/V+BMLgFqktCUV0QjT259ekAIlbVrMaFnFLxJ4s/JPl4tou56S1BzPufI60bLe29fHA== + fastify-arrow@1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/fastify-arrow/-/fastify-arrow-1.0.0.tgz#49435f441bae61ace524dc0395b620f1cdffe167" @@ -7313,6 +7384,26 @@ fastify@^3.0.0: semver "^7.3.2" tiny-lru "^8.0.1" +fastify@^4.0.0: + version "4.5.3" + resolved "https://registry.npmjs.org/fastify/-/fastify-4.5.3.tgz#df4f00347f06e8f3d6ab35788d2d140668c2533c" + integrity sha512-Q8Zvkmg7GnioMCDX1jT2Q7iRqjywlnDZ1735D2Ipf7ashCM/3/bqPKv2Jo1ZF2iDExct2eP1C/tdhcj0GG/OuQ== + dependencies: + "@fastify/ajv-compiler" "^3.1.1" + "@fastify/error" "^3.0.0" + "@fastify/fast-json-stringify-compiler" "^4.0.0" + abstract-logging "^2.0.1" + avvio "^8.1.3" + find-my-way "^7.0.0" + light-my-request "^5.5.1" + pino "^8.0.0" + process-warning "^2.0.0" + proxy-addr "^2.0.7" + rfdc "^1.3.0" + secure-json-parse "^2.4.0" + semver "^7.3.7" + tiny-lru "^8.0.2" + fastq@^1.6.0, fastq@^1.6.1: version "1.13.0" resolved "https://registry.npmjs.org/fastq/-/fastq-1.13.0.tgz#616760f88a7526bdfc596b7cab8c18938c36b98c" @@ -7431,6 +7522,14 @@ find-my-way@^4.1.0, find-my-way@^4.5.0: safe-regex2 "^2.0.0" semver-store "^0.3.0" +find-my-way@^7.0.0: + version "7.0.1" + resolved "https://registry.npmjs.org/find-my-way/-/find-my-way-7.0.1.tgz#079d6a8b474754e073c75778da678f59dedd620f" + integrity sha512-w05SaOPg54KqBof/RDA+75n1R48V7ZZNPL3nR17jJJs5dgZpR3ivfrMWOyx7BVFQgCLhYRG05hfgFCohYvSUXA== + dependencies: + fast-deep-equal "^3.1.3" + safe-regex2 "^2.0.0" + find-parent-dir@^0.3.0: version "0.3.1" resolved "https://registry.npmjs.org/find-parent-dir/-/find-parent-dir-0.3.1.tgz#c5c385b96858c3351f95d446cab866cbf9f11125" @@ -9973,6 +10072,15 @@ light-my-request@^4.2.0: process-warning "^1.0.0" set-cookie-parser "^2.4.1" +light-my-request@^5.5.1: + version "5.5.1" + resolved "https://registry.npmjs.org/light-my-request/-/light-my-request-5.5.1.tgz#566d90928b9b960d44b6b2b74e072eec1f7015e4" + integrity sha512-Zd4oZjF7axSyc5rYQsbB0qsgY4LFFviZSbEywxf7Vi5UE3y3c7tYF/GeheQjBNYY+pQ55BF8UGGJTjneoxOS1w== + dependencies: + cookie "^0.5.0" + process-warning "^2.0.0" + set-cookie-parser "^2.4.1" + lines-and-columns@^1.1.6: version "1.2.4" resolved "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz#eca284f75d2965079309dc0ad9255abb2ebc1632" @@ -11280,6 +11388,11 @@ octokit-pagination-methods@^1.1.0: resolved "https://registry.npmjs.org/octokit-pagination-methods/-/octokit-pagination-methods-1.1.0.tgz#cf472edc9d551055f9ef73f6e42b4dbb4c80bea4" integrity sha512-fZ4qZdQ2nxJvtcasX7Ghl+WlWS/d9IgnBIwFZXVNNZUmzpno91SX5bc5vuxiuKoCtK78XxGGNuSCrDC7xYB3OQ== +on-exit-leak-free@^2.1.0: + version "2.1.0" + resolved "https://registry.npmjs.org/on-exit-leak-free/-/on-exit-leak-free-2.1.0.tgz#5c703c968f7e7f851885f6459bf8a8a57edc9cc4" + integrity sha512-VuCaZZAjReZ3vUwgOB8LxAosIurDiAW0s13rI1YwmaP++jvcxP77AWoQvenZebpCA2m8WC1/EosPYPMjnRAp/w== + on-finished@~2.3.0: version "2.3.0" resolved "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz#20f1336481b083cd75337992a16971aa2d906947" @@ -11790,6 +11903,14 @@ pinkie@^2.0.0: resolved "https://registry.npmjs.org/pinkie/-/pinkie-2.0.4.tgz#72556b80cfa0d48a974e80e77248e80ed4f7f870" integrity sha512-MnUuEycAemtSaeFSjXKW/aroV7akBbY+Sv+RkyqFjgAe73F+MR0TBWKBRDkmfWq/HiFmdavfZ1G7h4SPZXaCSg== +pino-abstract-transport@v1.0.0: + version "1.0.0" + resolved "https://registry.npmjs.org/pino-abstract-transport/-/pino-abstract-transport-1.0.0.tgz#cc0d6955fffcadb91b7b49ef220a6cc111d48bb3" + integrity sha512-c7vo5OpW4wIS42hUVcT5REsL8ZljsUfBjqV/e2sFxmFEFZiq1XLUp5EYLtuDH6PEHq9W1egWqRbnLUP5FuZmOA== + dependencies: + readable-stream "^4.0.0" + split2 "^4.0.0" + pino-colada@^2.2.2: version "2.2.2" resolved "https://registry.npmjs.org/pino-colada/-/pino-colada-2.2.2.tgz#d52a9a20ea4a2916d54a6ec97100ae7898885e4c" @@ -11806,6 +11927,11 @@ pino-std-serializers@^3.1.0: resolved "https://registry.npmjs.org/pino-std-serializers/-/pino-std-serializers-3.2.0.tgz#b56487c402d882eb96cd67c257868016b61ad671" integrity sha512-EqX4pwDPrt3MuOAAUBMU0Tk5kR/YcCM5fNPEzgCO2zJ5HfX0vbiH9HbJglnyeQsN96Kznae6MWD47pZB5avTrg== +pino-std-serializers@^6.0.0: + version "6.0.0" + resolved "https://registry.npmjs.org/pino-std-serializers/-/pino-std-serializers-6.0.0.tgz#4c20928a1bafca122fdc2a7a4a171ca1c5f9c526" + integrity sha512-mMMOwSKrmyl+Y12Ri2xhH1lbzQxwwpuru9VjyJpgFIH4asSj88F2csdMwN6+M5g1Ll4rmsYghHLQJw81tgZ7LQ== + pino@^6.13.0: version "6.14.0" resolved "https://registry.npmjs.org/pino/-/pino-6.14.0.tgz#b745ea87a99a6c4c9b374e4f29ca7910d4c69f78" @@ -11819,6 +11945,23 @@ pino@^6.13.0: quick-format-unescaped "^4.0.3" sonic-boom "^1.0.2" +pino@^8.0.0: + version "8.4.2" + resolved "https://registry.npmjs.org/pino/-/pino-8.4.2.tgz#5de76e81b36e173d74244e0af4543e7ae241dbfd" + integrity sha512-PlXDeGhJZfAuVay+wtlS02s5j8uisQveZExYdAm9MwwxUQSz9R7Q78XtjM2tTa4sa5KJmygimZjZxXXuHgV6ew== + dependencies: + atomic-sleep "^1.0.0" + fast-redact "^3.1.1" + on-exit-leak-free "^2.1.0" + pino-abstract-transport v1.0.0 + pino-std-serializers "^6.0.0" + process-warning "^2.0.0" + quick-format-unescaped "^4.0.3" + real-require "^0.2.0" + safe-stable-stringify "^2.3.1" + sonic-boom "^3.1.0" + thread-stream "^2.0.0" + pirates@^4.0.0, pirates@^4.0.1: version "4.0.5" resolved "https://registry.npmjs.org/pirates/-/pirates-4.0.5.tgz#feec352ea5c3268fb23a37c702ab1699f35a5f3b" @@ -11982,6 +12125,11 @@ process-warning@^1.0.0: resolved "https://registry.npmjs.org/process-warning/-/process-warning-1.0.0.tgz#980a0b25dc38cd6034181be4b7726d89066b4616" integrity sha512-du4wfLyj4yCZq1VupnVSZmRsPJsNuxoDQFdCFHLaYiEbFBD7QE0a+I4D7hOxrVnh78QE/YipFAj9lXHiXocV+Q== +process-warning@^2.0.0: + version "2.0.0" + resolved "https://registry.npmjs.org/process-warning/-/process-warning-2.0.0.tgz#341dbeaac985b90a04ebcd844d50097c7737b2ee" + integrity sha512-+MmoAXoUX+VTHAlwns0h+kFUWFs/3FZy+ZuchkgjyOu3oioLAo2LB5aCfKPh2+P9O18i3m43tUEv3YqttSy0Ww== + process@0.11.10, process@^0.11.1, process@^0.11.10: version "0.11.10" resolved "https://registry.npmjs.org/process/-/process-0.11.10.tgz#7332300e840161bda3e69a1d1d91a7d4bc16f182" @@ -12570,6 +12718,13 @@ read@1, read@~1.0.1: string_decoder "^1.1.1" util-deprecate "^1.0.1" +readable-stream@^4.0.0: + version "4.1.0" + resolved "https://registry.npmjs.org/readable-stream/-/readable-stream-4.1.0.tgz#280d0a29f559d3fb684a277254e02b6f61ae0631" + integrity sha512-sVisi3+P2lJ2t0BPbpK629j8wRW06yKGJUcaLAGXPAUhyUxVJm7VsCTit1PFgT4JHUDMrGNR+ZjSKpzGaRF3zw== + dependencies: + abort-controller "^3.0.0" + readable-stream@~1.0.26-2: version "1.0.34" resolved "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz#125820e34bc842d2f2aaafafe4c2916ee32c157c" @@ -12634,6 +12789,11 @@ readline2@^1.0.1: is-fullwidth-code-point "^1.0.0" mute-stream "0.0.5" +real-require@^0.2.0: + version "0.2.0" + resolved "https://registry.npmjs.org/real-require/-/real-require-0.2.0.tgz#209632dea1810be2ae063a6ac084fee7e33fba78" + integrity sha512-57frrGM/OCTLqLOAh0mhVA9VBMHd+9U7Zb2THMGdBUoZVOtGbJzjxsYGDJ3A9AYYCP4hn6y1TVbaOfzWtm5GFg== + rechoir@^0.6.2: version "0.6.2" resolved "https://registry.npmjs.org/rechoir/-/rechoir-0.6.2.tgz#85204b54dba82d5742e28c96756ef43af50e3384" @@ -13085,6 +13245,11 @@ safe-regex@^1.1.0: dependencies: ret "~0.1.10" +safe-stable-stringify@^2.3.1: + version "2.3.1" + resolved "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-2.3.1.tgz#ab67cbe1fe7d40603ca641c5e765cb942d04fc73" + integrity sha512-kYBSfT+troD9cDA85VDnHZ1rpHC50O0g1e6WlGHVCz/g+JS+9WKLj+XwFYyR8UbrZN8ll9HUpDAAddY58MGisg== + "safer-buffer@>= 2.1.2 < 3", "safer-buffer@>= 2.1.2 < 3.0.0", safer-buffer@^2.0.2, safer-buffer@^2.1.0, safer-buffer@~2.1.0: version "2.1.2" resolved "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a" @@ -13560,6 +13725,13 @@ sonic-boom@^1.0.2: atomic-sleep "^1.0.0" flatstr "^1.0.12" +sonic-boom@^3.1.0: + version "3.2.0" + resolved "https://registry.npmjs.org/sonic-boom/-/sonic-boom-3.2.0.tgz#ce9f2de7557e68be2e52c8df6d9b052e7d348143" + integrity sha512-SbbZ+Kqj/XIunvIAgUZRlqd6CGQYq71tRRbXR92Za8J/R3Yh4Av+TWENiSiEgnlwckYLyP0YZQWVfyNC0dzLaA== + dependencies: + atomic-sleep "^1.0.0" + sort-keys@^2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/sort-keys/-/sort-keys-2.0.0.tgz#658535584861ec97d730d6cf41822e1f56684128" @@ -13689,7 +13861,7 @@ split2@^3.0.0: dependencies: readable-stream "^3.0.0" -split2@^4.1.0: +split2@^4.0.0, split2@^4.1.0: version "4.1.0" resolved "https://registry.npmjs.org/split2/-/split2-4.1.0.tgz#101907a24370f85bb782f08adaabe4e281ecf809" integrity sha512-VBiJxFkxiXRlUIeyMQi8s4hgvKCSjtknJv/LVYbrgALPwf5zSKmEwV9Lst25AkvMDnvxODugjdl6KZgwKM1WYQ== @@ -14335,6 +14507,13 @@ thenify-all@^1.0.0: dependencies: any-promise "^1.0.0" +thread-stream@^2.0.0: + version "2.1.0" + resolved "https://registry.npmjs.org/thread-stream/-/thread-stream-2.1.0.tgz#d560dd8b9d09482b0e2e876a96c229c374870836" + integrity sha512-5+Pf2Ya31CsZyIPYYkhINzdTZ3guL+jHq7D8lkBybgGcSQIKDbid3NJku3SpCKeE/gACWAccDA/rH2B6doC5aA== + dependencies: + real-require "^0.2.0" + throat@^5.0.0: version "5.0.0" resolved "https://registry.npmjs.org/throat/-/throat-5.0.0.tgz#c5199235803aad18754a667d659b5e72ce16764b" @@ -14380,7 +14559,7 @@ tiny-lru@^7.0.0: resolved "https://registry.npmjs.org/tiny-lru/-/tiny-lru-7.0.6.tgz#b0c3cdede1e5882aa2d1ae21cb2ceccf2a331f24" integrity sha512-zNYO0Kvgn5rXzWpL0y3RS09sMK67eGaQj9805jlK9G6pSadfriTczzLHFXa/xcW4mIRfmlB9HyQ/+SgL0V1uow== -tiny-lru@^8.0.1: +tiny-lru@^8.0.1, tiny-lru@^8.0.2: version "8.0.2" resolved "https://registry.npmjs.org/tiny-lru/-/tiny-lru-8.0.2.tgz#812fccbe6e622ded552e3ff8a4c3b5ff34a85e4c" integrity sha512-ApGvZ6vVvTNdsmt676grvCkUCGwzG9IqXma5Z07xJgiC5L7akUMof5U8G2JTI9Rz/ovtVhJBlY6mNhEvtjzOIg== From e685652f07b10499ac0a3d18fc71efaa8277e0ca Mon Sep 17 00:00:00 2001 From: "H. Thomson Comer" Date: Fri, 2 Sep 2022 11:43:35 -0500 Subject: [PATCH 05/43] Add routes/gpu --- modules/demo/api-server/routes/gpu/index.js | 84 +++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 modules/demo/api-server/routes/gpu/index.js diff --git a/modules/demo/api-server/routes/gpu/index.js b/modules/demo/api-server/routes/gpu/index.js new file mode 100644 index 000000000..50d5a4f28 --- /dev/null +++ b/modules/demo/api-server/routes/gpu/index.js @@ -0,0 +1,84 @@ +// Copyright (c) 2022, NVIDIA CORPORATION. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +const Fs = require('fs'); +const {Utf8String, Int32, Uint32, Float32, DataFrame, Series, Float64} = require('@rapidsai/cudf'); +const {RecordBatchStreamWriter, Field, Vector, List, Table} = require('apache-arrow'); +const Path = require('path'); +const {promisify} = require('util'); +const Stat = promisify(Fs.stat); +const fastifyCors = require('@fastify/cors'); +const fastify = require('fastify'); + +const arrowPlugin = require('fastify-arrow'); +const gpu_cache = require('../../util/gpu_cache.js'); +const root_schema = require('../../util/schema.js'); + +module.exports = async function(fastify, opts) { + fastify.register(arrowPlugin); + fastify.register(fastifyCors, {origin: '*'}); + fastify.decorate('setDataframe', gpu_cache.setDataframe); + fastify.decorate('getDataframe', gpu_cache.getDataframe); + fastify.decorate('gpu', gpu_cache); + + const get_handler = + async (request, reply) => { + const query = request.query; + request.log.info('Parsing Query:'); + request.log.info(query); + request.log.info('Sending query to gpu_cache'); + request.log.info('Updating result'); + request.log.info('Sending cache.tick'); + let result = { + 'params': JSON.stringify(query), + success: true, + message: `gpu method:${request.method} placeholder`, + statusCode: 200 + }; + return result + } + + const post_handler = + async (request, reply) => { + const query = request.query; + request.log.info('Parsing Query:'); + request.log.info(query); + request.log.info('Sending query to gpu_cache'); + request.log.info('Updating result'); + request.log.info('Sending cache.tick'); + let result = { + 'params': JSON.stringify(query), + success: true, + message: `gpu method:${request.method} placeholder`, + statusCode: 200 + }; + return result + } + + const get_schema = { + logLevel: 'debug', + schema: { + response: { + 200: { + type: 'object', + properties: + {success: {type: 'boolean'}, message: {type: 'string'}, params: {type: 'string'}} + } + } + } + }; + + fastify.get('/', {...get_schema, handler: get_handler}); + fastify.post('/', {...get_schema, handler: post_handler}); +} From 5093b8628d3f2b3bdb7d45c9120dab8be84952ee Mon Sep 17 00:00:00 2001 From: Thomson Comer Date: Tue, 6 Sep 2022 12:40:14 -0500 Subject: [PATCH 06/43] Add Series.repeat --- modules/cudf/src/column.cpp | 2 ++ modules/cudf/src/column.ts | 10 ++++++++++ modules/cudf/src/node_cudf/column.hpp | 5 +++++ modules/cudf/src/series.ts | 21 +++++++++++++++++++++ package.json | 3 ++- 5 files changed, 40 insertions(+), 1 deletion(-) diff --git a/modules/cudf/src/column.cpp b/modules/cudf/src/column.cpp index a3fb08533..ad4c73b52 100644 --- a/modules/cudf/src/column.cpp +++ b/modules/cudf/src/column.cpp @@ -49,6 +49,8 @@ Napi::Function Column::Init(Napi::Env const& env, Napi::Object exports) { // column/filling.cpp InstanceMethod<&Column::fill>("fill"), InstanceMethod<&Column::fill_in_place>("fillInPlace"), + // column/repeat.cpp + InstanceMethod<&Column::repeat>("repeat"), // column/binaryop.cpp InstanceMethod<&Column::add>("add"), InstanceMethod<&Column::sub>("sub"), diff --git a/modules/cudf/src/column.ts b/modules/cudf/src/column.ts index 12baeac9c..8f2568403 100644 --- a/modules/cudf/src/column.ts +++ b/modules/cudf/src/column.ts @@ -244,6 +244,16 @@ export interface Column { */ fill(value: Scalar, begin?: number, end?: number, memoryResource?: MemoryResource): Column; + /** + * Repeats the values of this n times. + * + * @param repeats The number of times to repeat the column. + * @param memoryResource The optional MemoryResource used to allocate the result Column's device + * memory. + */ + repeat(value: Scalar, begin?: number, end?: number, memoryResource?: MemoryResource): + Column; + /** * Fills a range of elements in-place in a column with a scalar value. * diff --git a/modules/cudf/src/node_cudf/column.hpp b/modules/cudf/src/node_cudf/column.hpp index 37365a98b..fa329d9c8 100644 --- a/modules/cudf/src/node_cudf/column.hpp +++ b/modules/cudf/src/node_cudf/column.hpp @@ -715,6 +715,11 @@ struct Column : public EnvLocalObjectWrap { cudf::scalar const& value, rmm::mr::device_memory_resource* mr = rmm::mr::get_current_device_resource()); + // column/filling/repeat.cpp + Column::wrapper_t repeat( + cudf::size_type repeats, + rmm::mr::device_memory_resource* mr = rmm::mr::get_current_device_resource()); + // column/replace.cpp Column::wrapper_t replace_nulls( cudf::column_view const& replacement, diff --git a/modules/cudf/src/series.ts b/modules/cudf/src/series.ts index 148931df9..710487125 100644 --- a/modules/cudf/src/series.ts +++ b/modules/cudf/src/series.ts @@ -807,6 +807,27 @@ export class AbstractSeries { this._col.fill(new Scalar({type: this.type, value}), begin, end, memoryResource)); } + /** + * Repeats a Series n times, returning a new Series. + * + * @param repeats The number of times to repeat this. + * + * @example + * ```typescript + * import {Series} from '@rapidsai/cudf'; + * + * // Float64Series + * Series.new([1, 2, 3]).repeat(2) // [1, 2, 3, 1, 2, 3] + * // StringSeries + * Series.new(["foo", "bar", "test"]).repeat(2) // ["foo", "bar", "test", "foo", "bar", "test"] + * // Bool8Series + * Series.new([true, true, true]).repeat(2) // [true, false, false, true, false, false] + * ``` + */ + repeat(repeats: T['scalarType'], memoryResource?: MemoryResource): Series { + return this.__construct(this._col.repeat(repeats, memoryResource)); + } + /** * Fills a range of elements in-place in a column with a scalar value. * diff --git a/package.json b/package.json index 3b85be9b1..aa7d8bd3c 100644 --- a/package.json +++ b/package.json @@ -94,7 +94,8 @@ "modules/demo/client-server", "modules/demo/api-server", "modules/demo/viz-app", - "modules/demo/sql/*" + "modules/demo/sql/*", + "modules/external/attractor-viz-node-rapids/*" ], "dependencies": { "@typescript-eslint/eslint-plugin": "5.30.0", From c302c018189d740a400ad5010cf24f0a7b0bf853 Mon Sep 17 00:00:00 2001 From: "H. Thomson Comer" Date: Wed, 5 Oct 2022 16:36:16 -0500 Subject: [PATCH 07/43] Fix yarn.lock --- yarn.lock | 41 ----------------------------------------- 1 file changed, 41 deletions(-) diff --git a/yarn.lock b/yarn.lock index a352cf71e..b30273de8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7241,15 +7241,9 @@ fast-json-stringify@^2.5.2: string-similarity "^4.0.1" fast-json-stringify@^5.0.0: -<<<<<<< HEAD - version "5.2.0" - resolved "https://registry.npmjs.org/fast-json-stringify/-/fast-json-stringify-5.2.0.tgz#0a637b698db602cd8f0c931b0025cf4858e9f5ce" - integrity sha512-u5jtrcAK9RINW15iuDKnsuuhqmqre4AmDMp3crRTjUMdAuHMpQUt3IfoMm5wlJm59b74PcajqOl3SjgnC5FPmw== -======= version "5.3.0" resolved "https://registry.npmjs.org/fast-json-stringify/-/fast-json-stringify-5.3.0.tgz#e3f6a86b68fbf9b8ff0e7c892141a719d128b6bb" integrity sha512-jTlJV/VAaYMtYl5G41uEL8UQT7/fT5W6LuxKxIS/Lpm6bXxmR+reF3m3WgP/WwxXybH61O+xhWK7n9uAsY6zGA== ->>>>>>> main dependencies: "@fastify/deepmerge" "^1.0.0" ajv "^8.10.0" @@ -7263,8 +7257,6 @@ fast-levenshtein@^2.0.6, fast-levenshtein@~2.0.6: resolved "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz#3d8a5c66883a16a30ca8643e851f19baa7797917" integrity sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw== -<<<<<<< HEAD -======= fast-querystring@^1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/fast-querystring/-/fast-querystring-1.0.0.tgz#d6151cd025d4b100e09e24045f6c35ae9ff191ef" @@ -7272,7 +7264,6 @@ fast-querystring@^1.0.0: dependencies: fast-decode-uri-component "^1.0.1" ->>>>>>> main fast-redact@^3.0.0, fast-redact@^3.1.1: version "3.1.2" resolved "https://registry.npmjs.org/fast-redact/-/fast-redact-3.1.2.tgz#d58e69e9084ce9fa4c1a6fa98a3e1ecf5d7839aa" @@ -7431,15 +7422,9 @@ fastify@^3.0.0: tiny-lru "^8.0.1" fastify@^4.0.0: -<<<<<<< HEAD - version "4.5.3" - resolved "https://registry.npmjs.org/fastify/-/fastify-4.5.3.tgz#df4f00347f06e8f3d6ab35788d2d140668c2533c" - integrity sha512-Q8Zvkmg7GnioMCDX1jT2Q7iRqjywlnDZ1735D2Ipf7ashCM/3/bqPKv2Jo1ZF2iDExct2eP1C/tdhcj0GG/OuQ== -======= version "4.6.0" resolved "https://registry.npmjs.org/fastify/-/fastify-4.6.0.tgz#47524173eb4013a1f98fb718ba1263f67a02a39a" integrity sha512-EgWUvcJNvsql1R4g5/ce866BYk8SgJKjGh6AI0e9BR+NidP7hqX1ObiwHEVbkR15A9XwMtkKd3TE/tFZCjsqnA== ->>>>>>> main dependencies: "@fastify/ajv-compiler" "^3.1.1" "@fastify/error" "^3.0.0" @@ -7575,20 +7560,12 @@ find-my-way@^4.1.0, find-my-way@^4.5.0: semver-store "^0.3.0" find-my-way@^7.0.0: -<<<<<<< HEAD - version "7.0.1" - resolved "https://registry.npmjs.org/find-my-way/-/find-my-way-7.0.1.tgz#079d6a8b474754e073c75778da678f59dedd620f" - integrity sha512-w05SaOPg54KqBof/RDA+75n1R48V7ZZNPL3nR17jJJs5dgZpR3ivfrMWOyx7BVFQgCLhYRG05hfgFCohYvSUXA== - dependencies: - fast-deep-equal "^3.1.3" -======= version "7.2.0" resolved "https://registry.npmjs.org/find-my-way/-/find-my-way-7.2.0.tgz#35090bb1f738e58e005b6831873c01648cb21851" integrity sha512-27SFA5sSYDYFZCQ/7SSJB0yhStTP/qxKP1OEC8feZvkHFRuD3fGcQ97Y+0w8HpKTDfMYWXGU3h2ETRGt5zPWyA== dependencies: fast-deep-equal "^3.1.3" fast-querystring "^1.0.0" ->>>>>>> main safe-regex2 "^2.0.0" find-parent-dir@^0.3.0: @@ -10138,15 +10115,9 @@ light-my-request@^4.2.0: set-cookie-parser "^2.4.1" light-my-request@^5.5.1: -<<<<<<< HEAD - version "5.5.1" - resolved "https://registry.npmjs.org/light-my-request/-/light-my-request-5.5.1.tgz#566d90928b9b960d44b6b2b74e072eec1f7015e4" - integrity sha512-Zd4oZjF7axSyc5rYQsbB0qsgY4LFFviZSbEywxf7Vi5UE3y3c7tYF/GeheQjBNYY+pQ55BF8UGGJTjneoxOS1w== -======= version "5.6.1" resolved "https://registry.npmjs.org/light-my-request/-/light-my-request-5.6.1.tgz#cff5c75d8cb35a354433d75406fea74a2f8bcdb1" integrity sha512-sbJnC1UBRivi9L1kICr3CESb82pNiPNB3TvtdIrZZqW0Qh8uDXvoywMmWKZlihDcmw952CMICCzM+54LDf+E+g== ->>>>>>> main dependencies: cookie "^0.5.0" process-warning "^2.0.0" @@ -12025,15 +11996,9 @@ pino@^6.13.0: sonic-boom "^1.0.2" pino@^8.0.0: -<<<<<<< HEAD - version "8.4.2" - resolved "https://registry.npmjs.org/pino/-/pino-8.4.2.tgz#5de76e81b36e173d74244e0af4543e7ae241dbfd" - integrity sha512-PlXDeGhJZfAuVay+wtlS02s5j8uisQveZExYdAm9MwwxUQSz9R7Q78XtjM2tTa4sa5KJmygimZjZxXXuHgV6ew== -======= version "8.5.0" resolved "https://registry.npmjs.org/pino/-/pino-8.5.0.tgz#60943fa2ec0ac4f22b1f8fde199cc2488547261e" integrity sha512-PuD6sOti8Y+p9zRoNB5dibmfjfM/OU2tEtJFICxw5ulXi1d0qnq/Rt3CsR6aBEAOeyCXP+ZUfiNWW+tt55pNzg== ->>>>>>> main dependencies: atomic-sleep "^1.0.0" fast-redact "^3.1.1" @@ -14584,15 +14549,9 @@ thenify-all@^1.0.0: any-promise "^1.0.0" thread-stream@^2.0.0: -<<<<<<< HEAD - version "2.1.0" - resolved "https://registry.npmjs.org/thread-stream/-/thread-stream-2.1.0.tgz#d560dd8b9d09482b0e2e876a96c229c374870836" - integrity sha512-5+Pf2Ya31CsZyIPYYkhINzdTZ3guL+jHq7D8lkBybgGcSQIKDbid3NJku3SpCKeE/gACWAccDA/rH2B6doC5aA== -======= version "2.2.0" resolved "https://registry.npmjs.org/thread-stream/-/thread-stream-2.2.0.tgz#310c03a253f729094ce5d4638ef5186dfa80a9e8" integrity sha512-rUkv4/fnb4rqy/gGy7VuqK6wE1+1DOCOWy4RMeaV69ZHMP11tQKZvZSip1yTgrKCMZzEMcCL/bKfHvSfDHx+iQ== ->>>>>>> main dependencies: real-require "^0.2.0" From d4e721f2b3544562923bb7a4840b0f5211ea803c Mon Sep 17 00:00:00 2001 From: "H. Thomson Comer" Date: Tue, 11 Oct 2022 10:46:13 -0500 Subject: [PATCH 08/43] Clean up and pass a couple of tests. --- modules/demo/api-server/routes/graphology/index.js | 4 +++- .../demo/api-server/test/routes/graphology.test.js | 14 ++++++-------- modules/demo/api-server/test/routes/root.test.js | 11 +++++++++++ 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/modules/demo/api-server/routes/graphology/index.js b/modules/demo/api-server/routes/graphology/index.js index 42286fcff..872bda0e3 100644 --- a/modules/demo/api-server/routes/graphology/index.js +++ b/modules/demo/api-server/routes/graphology/index.js @@ -34,7 +34,9 @@ module.exports = async function(fastify, opts) { fastify.decorate('readGraphology', gpu_cache.readGraphology); fastify.decorate('readLargeGraphDemo', gpu_cache.readLargeGraphDemo); fastify.decorate('clearDataFrames', gpu_cache.clearDataframes); - fastify.get('/', async function(request, reply) { return root_schema; }); + fastify.get('/', async function(request, reply) { + return root_schema['graphology']; + }); fastify.route({ method: 'POST', diff --git a/modules/demo/api-server/test/routes/graphology.test.js b/modules/demo/api-server/test/routes/graphology.test.js index f0242fdc8..e051458bb 100644 --- a/modules/demo/api-server/test/routes/graphology.test.js +++ b/modules/demo/api-server/test/routes/graphology.test.js @@ -24,7 +24,6 @@ test('graphology root returns api description', async t => { const app = await build(t); const res = await app.inject({url: '/graphology'}) t.same(JSON.parse(res.payload), { - graphology: { description: 'The graphology api provides GPU acceleration of graphology datasets.', schema: { read_json: { @@ -58,17 +57,16 @@ test('graphology root returns api description', async t => { nodes: {bounds: {returns: 'Returns the x and y bounds to be used in rendering.'}}, edges: {return: 'Returns the existing edges table after applying normalization for sigma.js'} - } } - }) +}) }); test('read_json no filename', async t => { - const app = await build(t); - const res = await app.inject({method: 'POST', url: '/graphology/read_json'}); - t.same( - JSON.parse(res.payload), - {success: false, message: 'Parameter filename is required', params: '{}', statusCode: 400}); +const app = await build(t); +const res = await app.inject({method: 'POST', url: '/graphology/read_json'}); +t.same( + JSON.parse(res.payload), + {success: false, message: 'Parameter filename is required', params: '{}', statusCode: 400}); }); test('read_json no file', async (t) => { diff --git a/modules/demo/api-server/test/routes/root.test.js b/modules/demo/api-server/test/routes/root.test.js index 564498d71..290b4e375 100644 --- a/modules/demo/api-server/test/routes/root.test.js +++ b/modules/demo/api-server/test/routes/root.test.js @@ -21,6 +21,17 @@ test('root returns API description', async (t) => { const app = await build(t); const res = await app.inject({url: '/'}); t.same(JSON.parse(res.payload), { + gpu: { + description: 'An abstract interface to the node-rapids api, supported by a server.', + schema: { + '/': { + method: 'The name of the method to apply to gpu_cache data.', + caller: 'Either an object that has been stored in the gpu_cache or a static module name.', + arguments: 'Correctly specified arguments to the gpu_cache method.', + result: 'Either a result code specifying success or failure or an arrow data buffer.', + } + } + }, graphology: { description: 'The graphology api provides GPU acceleration of graphology datasets.', schema: { From cc3cbaba09a552806066b788deccf80309f14440 Mon Sep 17 00:00:00 2001 From: "H. Thomson Comer" Date: Tue, 11 Oct 2022 12:29:49 -0500 Subject: [PATCH 09/43] Adding flexibility to gpu handler. --- modules/demo/api-server/routes/gpu/index.js | 38 +++++++++++++++------ 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/modules/demo/api-server/routes/gpu/index.js b/modules/demo/api-server/routes/gpu/index.js index 50d5a4f28..66f15a2ae 100644 --- a/modules/demo/api-server/routes/gpu/index.js +++ b/modules/demo/api-server/routes/gpu/index.js @@ -49,19 +49,33 @@ module.exports = async function(fastify, opts) { return result } + const cudf = async() => { + console.log(route); + console.log(args); + } + + const cudf_dispatcher = async (route, args) => { + Function(route[0])(route, args); + console.log(route); + console.log(args); + } + const post_handler = async (request, reply) => { - const query = request.query; - request.log.info('Parsing Query:'); - request.log.info(query); - request.log.info('Sending query to gpu_cache'); - request.log.info('Updating result'); - request.log.info('Sending cache.tick'); - let result = { - 'params': JSON.stringify(query), - success: true, - message: `gpu method:${request.method} placeholder`, - statusCode: 200 + request.log.info('Parsing Url:'); + const url = request.url.split('/'); + url.shift(); + url.shift(); + const query = request.query; + request.log.info('Sending query to gpu_cache'); + cudf_dispatcher(url, query); + request.log.info('Updating result'); + request.log.info('Sending cache.tick'); + let result = { + 'params': JSON.stringify(query), + success: true, + message: `gpu method:${request.method} placeholder`, + statusCode: 200 }; return result } @@ -81,4 +95,6 @@ module.exports = async function(fastify, opts) { fastify.get('/', {...get_schema, handler: get_handler}); fastify.post('/', {...get_schema, handler: post_handler}); + fastify.get('/*', {...get_schema, handler: get_handler}); + fastify.post('/*', {...get_schema, handler: post_handler}); } From 0eca0f4653ef8eab3741fec5a35de0a90feff5a5 Mon Sep 17 00:00:00 2001 From: "H. Thomson Comer" Date: Tue, 11 Oct 2022 12:59:26 -0500 Subject: [PATCH 10/43] Fiddling with dynamic dispatch --- modules/demo/api-server/index.js | 2 +- modules/demo/api-server/routes/gpu/index.js | 46 +++++++++++---------- 2 files changed, 26 insertions(+), 22 deletions(-) diff --git a/modules/demo/api-server/index.js b/modules/demo/api-server/index.js index c7ef575d2..0d97cdbc5 100755 --- a/modules/demo/api-server/index.js +++ b/modules/demo/api-server/index.js @@ -19,7 +19,7 @@ const Path = require('path'); // Change cwd to the example dir so relative file paths are resolved process.chdir(__dirname); -const fastify = require.resolve('fastify-cli/cli.js')({logger: true}); +const fastify = require.resolve('fastify-cli/cli.js'); const {spawnSync} = require('child_process'); diff --git a/modules/demo/api-server/routes/gpu/index.js b/modules/demo/api-server/routes/gpu/index.js index 66f15a2ae..ceb7ddc89 100644 --- a/modules/demo/api-server/routes/gpu/index.js +++ b/modules/demo/api-server/routes/gpu/index.js @@ -20,6 +20,7 @@ const {promisify} = require const Stat = promisify(Fs.stat); const fastifyCors = require('@fastify/cors'); const fastify = require('fastify'); +const cudf_api = require('@rapidsai/cudf'); const arrowPlugin = require('fastify-arrow'); const gpu_cache = require('../../util/gpu_cache.js'); @@ -49,33 +50,36 @@ module.exports = async function(fastify, opts) { return result } - const cudf = async() => { - console.log(route); - console.log(args); + const cudf = + async (route, args) => { + debugger; + const evalString = route.join('.'); + const paramsString = '(' + args + ')'; + eval(evalString + paramsString); } - const cudf_dispatcher = async (route, args) => { - Function(route[0])(route, args); - console.log(route); - console.log(args); + const cudf_dispatcher = + async (route, args) => { + const fn = Function(route[0]); + eval(route[0])(route, args); } const post_handler = async (request, reply) => { - request.log.info('Parsing Url:'); - const url = request.url.split('/'); - url.shift(); - url.shift(); - const query = request.query; - request.log.info('Sending query to gpu_cache'); - cudf_dispatcher(url, query); - request.log.info('Updating result'); - request.log.info('Sending cache.tick'); - let result = { - 'params': JSON.stringify(query), - success: true, - message: `gpu method:${request.method} placeholder`, - statusCode: 200 + request.log.info('Parsing Url:'); + const url = request.url.split('/'); + url.shift(); + url.shift(); + const query = request.query; + request.log.info('Sending query to gpu_cache'); + cudf_dispatcher(url, query); + request.log.info('Updating result'); + request.log.info('Sending cache.tick'); + let result = { + 'params': JSON.stringify(query), + success: true, + message: `gpu method:${request.method} placeholder`, + statusCode: 200 }; return result } From ba4c8670b99a1c86faa7cc7dbf9f17a3c8712967 Mon Sep 17 00:00:00 2001 From: "H. Thomson Comer" Date: Wed, 12 Oct 2022 12:20:43 -0500 Subject: [PATCH 11/43] Drop the abstract server. --- modules/demo/api-server/routes/gpu/index.js | 116 ++++++++++---------- modules/demo/api-server/util/gpu_cache.js | 5 + 2 files changed, 61 insertions(+), 60 deletions(-) diff --git a/modules/demo/api-server/routes/gpu/index.js b/modules/demo/api-server/routes/gpu/index.js index ceb7ddc89..77ad1d55e 100644 --- a/modules/demo/api-server/routes/gpu/index.js +++ b/modules/demo/api-server/routes/gpu/index.js @@ -12,15 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -const Fs = require('fs'); const {Utf8String, Int32, Uint32, Float32, DataFrame, Series, Float64} = require('@rapidsai/cudf'); -const {RecordBatchStreamWriter, Field, Vector, List, Table} = require('apache-arrow'); +const {RecordBatchStreamWriter, Field, Vector, List} = require('apache-arrow'); const Path = require('path'); const {promisify} = require('util'); +const Fs = require('fs'); const Stat = promisify(Fs.stat); const fastifyCors = require('@fastify/cors'); -const fastify = require('fastify'); -const cudf_api = require('@rapidsai/cudf'); +const fastify = require('fastify')({logger: true}); const arrowPlugin = require('fastify-arrow'); const gpu_cache = require('../../util/gpu_cache.js'); @@ -31,58 +30,7 @@ module.exports = async function(fastify, opts) { fastify.register(fastifyCors, {origin: '*'}); fastify.decorate('setDataframe', gpu_cache.setDataframe); fastify.decorate('getDataframe', gpu_cache.getDataframe); - fastify.decorate('gpu', gpu_cache); - - const get_handler = - async (request, reply) => { - const query = request.query; - request.log.info('Parsing Query:'); - request.log.info(query); - request.log.info('Sending query to gpu_cache'); - request.log.info('Updating result'); - request.log.info('Sending cache.tick'); - let result = { - 'params': JSON.stringify(query), - success: true, - message: `gpu method:${request.method} placeholder`, - statusCode: 200 - }; - return result - } - - const cudf = - async (route, args) => { - debugger; - const evalString = route.join('.'); - const paramsString = '(' + args + ')'; - eval(evalString + paramsString); - } - - const cudf_dispatcher = - async (route, args) => { - const fn = Function(route[0]); - eval(route[0])(route, args); - } - - const post_handler = - async (request, reply) => { - request.log.info('Parsing Url:'); - const url = request.url.split('/'); - url.shift(); - url.shift(); - const query = request.query; - request.log.info('Sending query to gpu_cache'); - cudf_dispatcher(url, query); - request.log.info('Updating result'); - request.log.info('Sending cache.tick'); - let result = { - 'params': JSON.stringify(query), - success: true, - message: `gpu method:${request.method} placeholder`, - statusCode: 200 - }; - return result - } + fastify.decorate('readCSV', gpu_cache.readCSV); const get_schema = { logLevel: 'debug', @@ -97,8 +45,56 @@ module.exports = async function(fastify, opts) { } }; - fastify.get('/', {...get_schema, handler: get_handler}); - fastify.post('/', {...get_schema, handler: post_handler}); - fastify.get('/*', {...get_schema, handler: get_handler}); - fastify.post('/*', {...get_schema, handler: post_handler}); + fastify.get('/', {...get_schema, handler: () => root_schema['gpu']}); + fastify.post('/', {...get_schema, handler: () => root_schema['gpu']}); + fastify.route({ + method: 'POST', + url: '/DataFrame/readCSV', + schema: {}, + handler: async (request, reply) => { + const path = Path.join(__dirname, request.body); + const stats = await Stat(path); + const message = 'File is available'; + const cacheObject = await fastify.readCSV({ + header: 0, + sourceType: 'files', + sources: [path], + }); + const name = request.body.replace('/\//g', '_'); + await fastify.setDataframe(name, cacheObject); + await reply.code(200).send({success: true, message: 'CSV file in GPU memory', params: name}); + } + }); + + fastify.route({ + method: 'GET', + url: '/get_column/:table/:column', + schema: {querystring: {table: {type: 'string'}, 'column': {type: 'string'}}}, + handler: async (request, reply) => { + let message = 'Error'; + let result = {'params': JSON.stringify(request.params), success: false, message: message}; + const table = await fastify.getDataframe(request.params.table); + if (table == undefined) { + result.message = 'Table not found'; + await reply.code(404).send(result); + } else { + try { + const name = request.params.column; + const column = table.get(name); + const newDfObject = {}; + newDfObject[name] = column; + const result = new DataFrame(newDfObject); + const writer = RecordBatchStreamWriter.writeAll(result.toArrow()); + await reply.code(200).send(writer.toNodeStream()); + } catch (e) { + if (e.substring('Unknown column name') != -1) { + result.message = e; + await reply.code(404).send(result); + } else { + await reply.code(500).send(result); + } + } + } + } + }); } diff --git a/modules/demo/api-server/util/gpu_cache.js b/modules/demo/api-server/util/gpu_cache.js index 1188346a6..f7c29040f 100644 --- a/modules/demo/api-server/util/gpu_cache.js +++ b/modules/demo/api-server/util/gpu_cache.js @@ -143,5 +143,10 @@ module.exports = { ]); const edges = json_aoa_to_dataframe(tedges, [new Utf8String, new Utf8String]); return {nodes: nodes, edges: edges, tags: tags, clusters: clusters}; + }, + + async readCSV(options) { + const result = await DataFrame.readCSV(options); + return result; } } From 1e1758fd1bce084274fb6fd00dad8bac1f2bf615 Mon Sep 17 00:00:00 2001 From: "H. Thomson Comer" Date: Thu, 13 Oct 2022 13:51:18 -0500 Subject: [PATCH 12/43] Particles endpoints to send data to client. --- modules/demo/api-server/index.js | 2 +- modules/demo/api-server/package.json | 4 +- modules/demo/api-server/routes/gpu/index.js | 41 ++++++++++++++------- 3 files changed, 31 insertions(+), 16 deletions(-) diff --git a/modules/demo/api-server/index.js b/modules/demo/api-server/index.js index 0d97cdbc5..a957bca54 100755 --- a/modules/demo/api-server/index.js +++ b/modules/demo/api-server/index.js @@ -29,5 +29,5 @@ const env = { }; spawnSync(process.execPath, - [fastify, 'start', '-l', 'info', '-P', '-p', '3010', '-w', 'app.js'], + [fastify, 'start', '-l', 'info', '--ignore-watch', '-P', '-p', '3010', 'app.js'], {env, cwd: __dirname, stdio: 'inherit'}); diff --git a/modules/demo/api-server/package.json b/modules/demo/api-server/package.json index fc6266bea..2751cef1d 100644 --- a/modules/demo/api-server/package.json +++ b/modules/demo/api-server/package.json @@ -12,8 +12,8 @@ "description": "A fastify-based web server that provides browser-access to GPU resources.", "scripts": { "test": "tap \"test/**/*.test.js\"", - "start": "fastify start -l info -P -p 3010 -w app.js", - "dev": "fastify start -w -l info -P -p 3010 app.js" + "start": "fastify start --ignore-watch -l info -P -p 3010 -w app.js", + "dev": "fastify start --ignore-watch -w -l info -P -p 3010 app.js" }, "keywords": [ "rapids.ai", diff --git a/modules/demo/api-server/routes/gpu/index.js b/modules/demo/api-server/routes/gpu/index.js index 77ad1d55e..79afccbc9 100644 --- a/modules/demo/api-server/routes/gpu/index.js +++ b/modules/demo/api-server/routes/gpu/index.js @@ -45,24 +45,39 @@ module.exports = async function(fastify, opts) { } }; + const filePath = () => Path.join(__dirname, '../../public'); + fastify.get('/', {...get_schema, handler: () => root_schema['gpu']}); - fastify.post('/', {...get_schema, handler: () => root_schema['gpu']}); + fastify.route({ method: 'POST', url: '/DataFrame/readCSV', schema: {}, handler: async (request, reply) => { - const path = Path.join(__dirname, request.body); - const stats = await Stat(path); - const message = 'File is available'; - const cacheObject = await fastify.readCSV({ - header: 0, - sourceType: 'files', - sources: [path], - }); - const name = request.body.replace('/\//g', '_'); - await fastify.setDataframe(name, cacheObject); - await reply.code(200).send({success: true, message: 'CSV file in GPU memory', params: name}); + let message = 'Error'; + let result = {'params': request.body, success: false, message: message}; + try { + const path = Path.join(filePath(), request.body); + const stats = await Stat(path); + const message = 'File is available'; + const cacheObject = await fastify.readCSV({ + header: 0, + sourceType: 'files', + sources: [path], + }); + const name = request.body.replace('/\//g', '_'); + await fastify.setDataframe(name, cacheObject); + result.success = true; + result.message = 'CSV file in GPU memory'; + await reply.code(200).send(result); + } catch (e) { + result.message = e.message; + if (e.message.search('no such file or directory') != -1) { + await reply.code(404).send(result); + } else { + await reply.code(500).send(result); + } + } } }); @@ -87,7 +102,7 @@ module.exports = async function(fastify, opts) { const writer = RecordBatchStreamWriter.writeAll(result.toArrow()); await reply.code(200).send(writer.toNodeStream()); } catch (e) { - if (e.substring('Unknown column name') != -1) { + if (e.message.search('Unknown column name') != -1) { result.message = e; await reply.code(404).send(result); } else { From 316a6a63566dcf485420b2f55830b324b0f4abd6 Mon Sep 17 00:00:00 2001 From: "H. Thomson Comer" Date: Fri, 14 Oct 2022 14:48:42 -0500 Subject: [PATCH 13/43] Fix a couple errors. --- modules/demo/api-server/routes/gpu/index.js | 2 +- modules/demo/api-server/util/gpu_cache.js | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/modules/demo/api-server/routes/gpu/index.js b/modules/demo/api-server/routes/gpu/index.js index 79afccbc9..f76f7cb8c 100644 --- a/modules/demo/api-server/routes/gpu/index.js +++ b/modules/demo/api-server/routes/gpu/index.js @@ -72,7 +72,7 @@ module.exports = async function(fastify, opts) { await reply.code(200).send(result); } catch (e) { result.message = e.message; - if (e.message.search('no such file or directory') != -1) { + if (e.message.search('no such file or directory') !== -1) { await reply.code(404).send(result); } else { await reply.code(500).send(result); diff --git a/modules/demo/api-server/util/gpu_cache.js b/modules/demo/api-server/util/gpu_cache.js index f7c29040f..6483efb4b 100644 --- a/modules/demo/api-server/util/gpu_cache.js +++ b/modules/demo/api-server/util/gpu_cache.js @@ -19,7 +19,10 @@ let timeout = -1; let datasets = {}; function clearCachedGPUData() { - for (const key in datasets) { datasets[key] = null; } + for (const key in datasets) { + datasets[key].dispose(); + datasets[key] = null; + } }; function json_key_attributes_to_dataframe(str) { @@ -68,6 +71,7 @@ module.exports = { if (timeout) { clearTimeout(timeout); } timeout = setTimeout(clearCachedGPUData, 10 * 60 * 1000); if (datasets === null) { + for (key in Object.datasets.keys()) { datasets.key.destroy(); } datasets = {}; } datasets[name] = dataframe; From 5288d6c64825439dd0d12249115f9be199b5627c Mon Sep 17 00:00:00 2001 From: "H. Thomson Comer" Date: Thu, 10 Nov 2022 13:57:24 -0600 Subject: [PATCH 14/43] Refactor particles handler to take two routes. --- .../demo/api-server/routes/particles/index.js | 135 ++++++++++++++++++ 1 file changed, 135 insertions(+) create mode 100644 modules/demo/api-server/routes/particles/index.js diff --git a/modules/demo/api-server/routes/particles/index.js b/modules/demo/api-server/routes/particles/index.js new file mode 100644 index 000000000..99bc06456 --- /dev/null +++ b/modules/demo/api-server/routes/particles/index.js @@ -0,0 +1,135 @@ +// Copyright (c) 2022, NVIDIA CORPORATION. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +const {Utf8String, Int32, Uint32, Float32, DataFrame, Series, Float64} = require('@rapidsai/cudf'); +const {RecordBatchStreamWriter, Field, Vector, List} = require('apache-arrow'); +const Path = require('path'); +const {promisify} = require('util'); +const Fs = require('fs'); +const Stat = promisify(Fs.stat); +const fastify = require('fastify')({logger: {level: 'debug'}}); +const fastifyCors = require('@fastify/cors'); + +const arrowPlugin = require('fastify-arrow'); +const gpu_cache = require('../../util/gpu_cache.js'); +const root_schema = require('../../util/schema.js'); + +module.exports = async function(fastify, opts) { + fastify.register(fastifyCors, {origin: '*'}); + fastify.register(arrowPlugin); + fastify.decorate('setDataframe', gpu_cache.setDataframe); + fastify.decorate('getDataframe', gpu_cache.getDataframe); + fastify.decorate('readCSV', gpu_cache.readCSV); + + const get_schema = { + logLevel: 'debug', + schema: { + response: { + 200: { + type: 'object', + properties: + {success: {type: 'boolean'}, message: {type: 'string'}, params: {type: 'string'}} + } + } + } + }; + + fastify.get('/', {...get_schema, handler: () => root_schema['particles']}); + fastify.post('/', {...get_schema, handler: () => root_schema['particles']}); + + const handler = async (request, reply) => { + let message = 'Error'; + let result = {'params': JSON.stringify(request.params), success: false, message: message}; + console.log(result); + const table = await fastify.getDataframe(request.params.table); + if (table == undefined) { + result.message = 'Table not found'; + await reply.code(404).send(result); + } else { + try { + const x = table.get('Longitude'); + const y = table.get('Latitude'); + // const state = table.get('State'); + // const zip = table.get('Zip_Code') + // Produce r,g,b from state + const color_map = [ + {'r': 0, 'g': 0, 'b': 0}, + {'r': 255, 'g': 0, 'b': 0}, + {'r': 0, 'g': 255, 'b': 0}, + {'r': 255, 'g': 255, 'b': 0}, + {'r': 0, 'g': 0, 'b': 255}, + {'r': 255, 'g': 0, 'b': 255}, + {'r': 0, 'g': 255, 'b': 255}, + {'r': 255, 'g': 255, 'b': 255} + ]; + // TODO: convert state to color by state index + const r = Series.sequence({type: new Int32, init: 255.0, size: x.length}).fill(0); + const g = Series.sequence({type: new Int32, init: 255.0, size: x.length}).fill(0); + const b = Series.sequence({type: new Int32, init: 255.0, size: x.length}).fill(0); + + // Map x, y, r, g, b to offsets for client display + let tiled = Series.sequence({type: new Float32, init: 0.0, size: (7 * x.length)}); + let base_offset = Series.sequence({type: new Int32, init: 0.0, size: x.length}).mul(7); + tiled = tiled.scatter(x, base_offset.cast(new Int32)); + x.dispose(); + tiled = tiled.scatter(y, base_offset.add(1).cast(new Int32)); + y.dispose(); + tiled = tiled.scatter(1.0, base_offset.add(2).cast(new Int32)); + tiled = tiled.scatter(1.0, base_offset.add(3).cast(new Int32)); + tiled = tiled.scatter(r, base_offset.add(4).cast(new Int32)); + r.dispose(); + tiled = tiled.scatter(g, base_offset.add(5).cast(new Int32)); + g.dispose(); + tiled = tiled.scatter(b, base_offset.add(6).cast(new Int32)); + b.dispose(); + const result = new DataFrame({'gpu_buffer': tiled}); + const writer = RecordBatchStreamWriter.writeAll(result.toArrow()); + await reply.code(200).send(writer.toNodeStream()); + } catch (e) { + result.message = e.message; + if (e.message.search('Unknown column name') != -1) { + result.message = { + error: result.message, + message: + 'Imported CSV file must contain four columns: State, Zip_Code, Longitude, and Latitude' + }; + await reply.code(500).send(result); + } else { + await reply.code(500).send(result); + } + } + } + }; + + fastify.route({ + method: 'GET', + url: '/get_shader_column/:table/:xmin/:xmax/:ymin/:ymax', + schema: { + querystring: { + table: {type: 'string'}, + xmin: {type: 'string'}, + xmax: {type: 'string'}, + ymin: {type: 'string'}, + ymax: {type: 'string'} + } + }, + handler: handler + }); + fastify.route({ + method: 'GET', + url: '/get_shader_column/:table', + schema: {querystring: {table: {type: 'string'}}}, + handler: handler + }); +}; From dee37c807032d0383809a17283ad2817167544d1 Mon Sep 17 00:00:00 2001 From: "H. Thomson Comer" Date: Thu, 10 Nov 2022 15:49:35 -0600 Subject: [PATCH 15/43] Add lon/lat filtering to request. --- .../demo/api-server/routes/particles/index.js | 39 ++++++++++++------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/modules/demo/api-server/routes/particles/index.js b/modules/demo/api-server/routes/particles/index.js index 99bc06456..36e9046e8 100644 --- a/modules/demo/api-server/routes/particles/index.js +++ b/modules/demo/api-server/routes/particles/index.js @@ -12,14 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -const {Utf8String, Int32, Uint32, Float32, DataFrame, Series, Float64} = require('@rapidsai/cudf'); -const {RecordBatchStreamWriter, Field, Vector, List} = require('apache-arrow'); -const Path = require('path'); -const {promisify} = require('util'); -const Fs = require('fs'); -const Stat = promisify(Fs.stat); -const fastify = require('fastify')({logger: {level: 'debug'}}); -const fastifyCors = require('@fastify/cors'); +const {Int32, Float32, DataFrame, Series} = require('@rapidsai/cudf'); +const {RecordBatchStreamWriter} = require('apache-arrow'); +const fastify = require('fastify')({logger: {level: 'debug'}}); +const fastifyCors = require('@fastify/cors'); const arrowPlugin = require('fastify-arrow'); const gpu_cache = require('../../util/gpu_cache.js'); @@ -58,8 +54,23 @@ module.exports = async function(fastify, opts) { await reply.code(404).send(result); } else { try { - const x = table.get('Longitude'); - const y = table.get('Latitude'); + let x = undefined; + let y = undefined; + if (request.params.xmin != undefined && request.params.xmax != undefined && + request.params.ymin != undefined && request.params.ymax != undefined) { + const x_unfiltered = table.get('Longitude'); + const x_gt = x_unfiltered._col.gt(parseFloat(request.params.xmin)); + const x_lt = x_unfiltered._col.lt(parseFloat(request.params.xmax)); + const y_unfiltered = table.get('Latitude'); + const y_gt = y_unfiltered._col.gt(parseFloat(request.params.ymin)); + const y_lt = y_unfiltered._col.lt(parseFloat(request.params.ymax)); + const x_y = x_lt.bitwiseAnd(x_gt).bitwiseAnd(y_lt).bitwiseAnd(y_gt); + x = x_unfiltered.filter(Series.new(x_y)); + y = y_unfiltered.filter(Series.new(x_y)); + } else { + x = table.get('Longitude'); + y = table.get('Latitude'); + } // const state = table.get('State'); // const zip = table.get('Zip_Code') // Produce r,g,b from state @@ -118,10 +129,10 @@ module.exports = async function(fastify, opts) { schema: { querystring: { table: {type: 'string'}, - xmin: {type: 'string'}, - xmax: {type: 'string'}, - ymin: {type: 'string'}, - ymax: {type: 'string'} + xmin: {type: 'number'}, + xmax: {type: 'number'}, + ymin: {type: 'number'}, + ymax: {type: 'number'} } }, handler: handler From 8e1905382bd63df7e8a684cf95dbd56f658bf356 Mon Sep 17 00:00:00 2001 From: "H. Thomson Comer" Date: Wed, 16 Nov 2022 11:34:25 -0600 Subject: [PATCH 16/43] Tweaking for better filtering, memory safety. --- modules/demo/api-server/package.json | 2 +- modules/demo/api-server/routes/gpu/index.js | 8 +++++--- modules/demo/api-server/routes/particles/index.js | 12 +++++++----- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/modules/demo/api-server/package.json b/modules/demo/api-server/package.json index 2751cef1d..e52600c1d 100644 --- a/modules/demo/api-server/package.json +++ b/modules/demo/api-server/package.json @@ -12,7 +12,7 @@ "description": "A fastify-based web server that provides browser-access to GPU resources.", "scripts": { "test": "tap \"test/**/*.test.js\"", - "start": "fastify start --ignore-watch -l info -P -p 3010 -w app.js", + "start": "fastify start --ignore-watch -l info -P -p 3010 app.js", "dev": "fastify start --ignore-watch -w -l info -P -p 3010 app.js" }, "keywords": [ diff --git a/modules/demo/api-server/routes/gpu/index.js b/modules/demo/api-server/routes/gpu/index.js index f76f7cb8c..4be6276b2 100644 --- a/modules/demo/api-server/routes/gpu/index.js +++ b/modules/demo/api-server/routes/gpu/index.js @@ -57,9 +57,11 @@ module.exports = async function(fastify, opts) { let message = 'Error'; let result = {'params': request.body, success: false, message: message}; try { - const path = Path.join(filePath(), request.body); - const stats = await Stat(path); - const message = 'File is available'; + const path = Path.join(filePath(), request.body); + const stats = await Stat(path); + const message = 'File is available'; + const currentDataFrame = await fastify.getDataframe(request.body); + if (currentDataFrame !== undefined) { currentDataFrame.dispose(); } const cacheObject = await fastify.readCSV({ header: 0, sourceType: 'files', diff --git a/modules/demo/api-server/routes/particles/index.js b/modules/demo/api-server/routes/particles/index.js index 36e9046e8..caa4a3982 100644 --- a/modules/demo/api-server/routes/particles/index.js +++ b/modules/demo/api-server/routes/particles/index.js @@ -47,7 +47,6 @@ module.exports = async function(fastify, opts) { const handler = async (request, reply) => { let message = 'Error'; let result = {'params': JSON.stringify(request.params), success: false, message: message}; - console.log(result); const table = await fastify.getDataframe(request.params.table); if (table == undefined) { result.message = 'Table not found'; @@ -59,11 +58,11 @@ module.exports = async function(fastify, opts) { if (request.params.xmin != undefined && request.params.xmax != undefined && request.params.ymin != undefined && request.params.ymax != undefined) { const x_unfiltered = table.get('Longitude'); - const x_gt = x_unfiltered._col.gt(parseFloat(request.params.xmin)); - const x_lt = x_unfiltered._col.lt(parseFloat(request.params.xmax)); + const x_gt = x_unfiltered._col.gt(parseInt(request.params.xmin)); + const x_lt = x_unfiltered._col.lt(parseInt(request.params.xmax)); const y_unfiltered = table.get('Latitude'); - const y_gt = y_unfiltered._col.gt(parseFloat(request.params.ymin)); - const y_lt = y_unfiltered._col.lt(parseFloat(request.params.ymax)); + const y_gt = y_unfiltered._col.gt(parseInt(request.params.ymin)); + const y_lt = y_unfiltered._col.lt(parseInt(request.params.ymax)); const x_y = x_lt.bitwiseAnd(x_gt).bitwiseAnd(y_lt).bitwiseAnd(y_gt); x = x_unfiltered.filter(Series.new(x_y)); y = y_unfiltered.filter(Series.new(x_y)); @@ -107,6 +106,9 @@ module.exports = async function(fastify, opts) { const result = new DataFrame({'gpu_buffer': tiled}); const writer = RecordBatchStreamWriter.writeAll(result.toArrow()); await reply.code(200).send(writer.toNodeStream()); + tiled.dispose(); + result.dispose(); + writer.close(); } catch (e) { result.message = e.message; if (e.message.search('Unknown column name') != -1) { From 60c48de1adc672965e831cd3f68936b45302240e Mon Sep 17 00:00:00 2001 From: "H. Thomson Comer" Date: Thu, 17 Nov 2022 10:01:46 -0600 Subject: [PATCH 17/43] Drop z, w, r, g, b from particles request. --- modules/demo/api-server/routes/particles/index.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/modules/demo/api-server/routes/particles/index.js b/modules/demo/api-server/routes/particles/index.js index caa4a3982..eff267328 100644 --- a/modules/demo/api-server/routes/particles/index.js +++ b/modules/demo/api-server/routes/particles/index.js @@ -89,12 +89,13 @@ module.exports = async function(fastify, opts) { const b = Series.sequence({type: new Int32, init: 255.0, size: x.length}).fill(0); // Map x, y, r, g, b to offsets for client display - let tiled = Series.sequence({type: new Float32, init: 0.0, size: (7 * x.length)}); - let base_offset = Series.sequence({type: new Int32, init: 0.0, size: x.length}).mul(7); + let tiled = Series.sequence({type: new Float32, init: 0.0, size: (2 * x.length)}); + let base_offset = Series.sequence({type: new Int32, init: 0.0, size: x.length}).mul(2); tiled = tiled.scatter(x, base_offset.cast(new Int32)); x.dispose(); tiled = tiled.scatter(y, base_offset.add(1).cast(new Int32)); y.dispose(); + /* tiled = tiled.scatter(1.0, base_offset.add(2).cast(new Int32)); tiled = tiled.scatter(1.0, base_offset.add(3).cast(new Int32)); tiled = tiled.scatter(r, base_offset.add(4).cast(new Int32)); @@ -103,6 +104,7 @@ module.exports = async function(fastify, opts) { g.dispose(); tiled = tiled.scatter(b, base_offset.add(6).cast(new Int32)); b.dispose(); + */ const result = new DataFrame({'gpu_buffer': tiled}); const writer = RecordBatchStreamWriter.writeAll(result.toArrow()); await reply.code(200).send(writer.toNodeStream()); From e8f6b59519affa697534b52dead0208d297e0be0 Mon Sep 17 00:00:00 2001 From: "H. Thomson Comer" Date: Thu, 17 Nov 2022 14:58:29 -0600 Subject: [PATCH 18/43] Add logging for timed out currentDataframe.dispose() that causes a 500 --- modules/demo/api-server/routes/gpu/index.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/modules/demo/api-server/routes/gpu/index.js b/modules/demo/api-server/routes/gpu/index.js index 4be6276b2..a70111a92 100644 --- a/modules/demo/api-server/routes/gpu/index.js +++ b/modules/demo/api-server/routes/gpu/index.js @@ -61,7 +61,12 @@ module.exports = async function(fastify, opts) { const stats = await Stat(path); const message = 'File is available'; const currentDataFrame = await fastify.getDataframe(request.body); - if (currentDataFrame !== undefined) { currentDataFrame.dispose(); } + if (currentDataFrame !== undefined) { + console.log('Found existing dataframe.'); + console.log(request.body); + console.log(currentDataFrame); + currentDataFrame.dispose(); + } const cacheObject = await fastify.readCSV({ header: 0, sourceType: 'files', From 7e51b8842fe9420d98c62ee2760dbbf84285eca5 Mon Sep 17 00:00:00 2001 From: "H. Thomson Comer" Date: Mon, 21 Nov 2022 14:32:23 -0600 Subject: [PATCH 19/43] Pass all tests and add some coverage, particles tests. --- modules/demo/api-server/routes/gpu/index.js | 14 +-- .../api-server/routes/graphology/index.js | 4 +- modules/demo/api-server/test/fixtures.js | 10 ++ .../api-server/test/plugins/gpu_cache.test.js | 38 +++++- .../api-server/test/routes/graphology.test.js | 90 +++++++------- .../demo/api-server/test/routes/root.test.js | 110 ++++++++++++------ modules/demo/api-server/util/gpu_cache.js | 9 +- modules/demo/api-server/util/schema.js | 39 +++++++ 8 files changed, 221 insertions(+), 93 deletions(-) diff --git a/modules/demo/api-server/routes/gpu/index.js b/modules/demo/api-server/routes/gpu/index.js index a70111a92..43c414992 100644 --- a/modules/demo/api-server/routes/gpu/index.js +++ b/modules/demo/api-server/routes/gpu/index.js @@ -31,6 +31,7 @@ module.exports = async function(fastify, opts) { fastify.decorate('setDataframe', gpu_cache.setDataframe); fastify.decorate('getDataframe', gpu_cache.getDataframe); fastify.decorate('readCSV', gpu_cache.readCSV); + fastify.decorate('publicPath', gpu_cache.publicPath); const get_schema = { logLevel: 'debug', @@ -45,8 +46,6 @@ module.exports = async function(fastify, opts) { } }; - const filePath = () => Path.join(__dirname, '../../public'); - fastify.get('/', {...get_schema, handler: () => root_schema['gpu']}); fastify.route({ @@ -57,10 +56,10 @@ module.exports = async function(fastify, opts) { let message = 'Error'; let result = {'params': request.body, success: false, message: message}; try { - const path = Path.join(filePath(), request.body); + const path = Path.join(fastify.publicPath(), request.body.filename); const stats = await Stat(path); const message = 'File is available'; - const currentDataFrame = await fastify.getDataframe(request.body); + const currentDataFrame = await fastify.getDataframe(request.body.filename); if (currentDataFrame !== undefined) { console.log('Found existing dataframe.'); console.log(request.body); @@ -72,10 +71,11 @@ module.exports = async function(fastify, opts) { sourceType: 'files', sources: [path], }); - const name = request.body.replace('/\//g', '_'); + const name = request.body.filename; // request.body.replace('/\//g', '_'); await fastify.setDataframe(name, cacheObject); - result.success = true; - result.message = 'CSV file in GPU memory'; + result.success = true; + result.message = 'CSV file in GPU memory.'; + result.statusCode = 200; await reply.code(200).send(result); } catch (e) { result.message = e.message; diff --git a/modules/demo/api-server/routes/graphology/index.js b/modules/demo/api-server/routes/graphology/index.js index 872bda0e3..032a39bdc 100644 --- a/modules/demo/api-server/routes/graphology/index.js +++ b/modules/demo/api-server/routes/graphology/index.js @@ -34,9 +34,7 @@ module.exports = async function(fastify, opts) { fastify.decorate('readGraphology', gpu_cache.readGraphology); fastify.decorate('readLargeGraphDemo', gpu_cache.readLargeGraphDemo); fastify.decorate('clearDataFrames', gpu_cache.clearDataframes); - fastify.get('/', async function(request, reply) { - return root_schema['graphology']; - }); + fastify.get('/', async function(request, reply) { return root_schema['graphology']; }); fastify.route({ method: 'POST', diff --git a/modules/demo/api-server/test/fixtures.js b/modules/demo/api-server/test/fixtures.js index 35df27442..0f932f9e6 100644 --- a/modules/demo/api-server/test/fixtures.js +++ b/modules/demo/api-server/test/fixtures.js @@ -163,9 +163,19 @@ const json_bad_map = { }` }; +const csv_base = { + 'csv_base.csv': + `Index,Name,Int,Float + 0,"bob",1,1.0 + 1,"george",2,2.0 + 2,"sam",3,3.0 + ` +}; + module.exports = { json_good: json_good, json_large: json_large, json_out_of_order: json_out_of_order, json_bad_map: json_bad_map, + csv_base: csv_base }; diff --git a/modules/demo/api-server/test/plugins/gpu_cache.test.js b/modules/demo/api-server/test/plugins/gpu_cache.test.js index b8453a337..99ee0ecad 100644 --- a/modules/demo/api-server/test/plugins/gpu_cache.test.js +++ b/modules/demo/api-server/test/plugins/gpu_cache.test.js @@ -20,13 +20,14 @@ const Support = require('../../plugins/support') const fixtures = require('../fixtures.js'); const gpuCache = require('../../util/gpu_cache.js'); -test('clearCachedGPUData()', async t => { +test('set/getDataframe', async t => { await gpuCache.setDataframe('bob', 5); const result = await gpuCache.getDataframe('bob'); + await gpuCache.clearDataframes(); t.equal(result, 5); }); -test('read_large_graph_demo', async t => { +test('readLargeGraphDemo', async t => { const dir = t.testdir(fixtures); const result = await gpuCache.readLargeGraphDemo(dir + '/json_large/json_large.txt'); await gpuCache.clearDataframes(); @@ -39,3 +40,36 @@ test('readGraphology', async t => { await gpuCache.clearDataframes(); t.same(Object.keys(result), ['nodes', 'edges', 'tags', 'clusters']); }); + +test('readCSV', {only: true}, async t => { + const dir = t.testdir(fixtures); + const path = dir + '/csv_base/csv_base.csv'; + const result = await gpuCache.readCSV({ + header: 0, + sourceType: 'files', + sources: [path], + }); + t.same(result.names, ['Index', 'Name', 'Int', 'Float']); +}); + +test('listDataframes', async t => { + await gpuCache.setDataframe('bob', 5); + await gpuCache.setDataframe('george', 6); + const result = await gpuCache.listDataframes(); + await gpuCache.clearDataframes(); + t.same(result, ['bob', 'george']); +}); + +test('clearDataframes', async t => { + await gpuCache.setDataframe('bob', 5); + await gpuCache.setDataframe('george', 6); + await gpuCache.clearDataframes(); + const result = await gpuCache.listDataframes(); + t.same(result, []); +}); + +test('_setPathForTesting', {only: true}, async t => { + await gpuCache._setPathForTesting('abcdef'); + const result = await gpuCache.publicPath(); + t.same(result, 'abcdef'); +}); diff --git a/modules/demo/api-server/test/routes/graphology.test.js b/modules/demo/api-server/test/routes/graphology.test.js index e051458bb..2e449725f 100644 --- a/modules/demo/api-server/test/routes/graphology.test.js +++ b/modules/demo/api-server/test/routes/graphology.test.js @@ -14,59 +14,60 @@ 'use strict' -const {dir} = require('console'); -const {test} = require('tap'); -const {build} = require('../helper'); -const {tableFromIPC, RecordBatchStreamWriter} = require('apache-arrow'); -const {json_large, json_good, json_out_of_order, json_bad_map} = require('../fixtures.js'); +const {dir} = require('console'); +const {test} = require('tap'); +const {build} = require('../helper'); +const {tableFromIPC, RecordBatchStreamWriter} = require('apache-arrow'); +const {json_large, json_good, json_out_of_order, json_bad_map, csv_base} = + require('../fixtures.js'); +const gpu_cache = require('../../util/gpu_cache'); -test('graphology root returns api description', async t => { +test('graphology root returns api description', {only: true}, async t => { const app = await build(t); const res = await app.inject({url: '/graphology'}) t.same(JSON.parse(res.payload), { - description: 'The graphology api provides GPU acceleration of graphology datasets.', - schema: { - read_json: { - filename: 'A URI to a graphology json dataset file.', - result: `Causes the node-rapids backend to attempt to load the json object specified + description: 'The graphology api provides GPU acceleration of graphology datasets.', + schema: { + read_json: { + filename: 'A URI to a graphology json dataset file.', + result: `Causes the node-rapids backend to attempt to load the json object specified by :filename. The GPU will attempt to parse the json file asynchronously and will return OK/ Not Found/ or Fail based on the file status. If the load is successful, four tables will be created in the node-rapids backend: nodes, edges, clusters, and tags. The root objects in the json target must match these names and order.`, - returns: 'Result OK/Not Found/Fail' - }, - read_large_demo: { - filename: - 'A URI to a graphology json dataset file matching the sigma.js/examples/large-demos spec.', - result: `Produces the same result as 'read_json'. + returns: 'Result OK/Not Found/Fail' + }, + read_large_demo: { + filename: + 'A URI to a graphology json dataset file matching the sigma.js/examples/large-demos spec.', + result: `Produces the same result as 'read_json'. If the load is successful, three tables will be created in the node-rapids backend: nodes, edges, and options.`, - returns: 'Result OK/Not Found/Fail' - }, - list_tables: {returns: 'Tables that are available presently in GPU memory.'}, - get_table: { - ':table': - {table: 'The name of the table that has been allocated previously into GPU memory.'} - }, - get_column: {':table': {':column': {table: 'The table name', column: 'The column name'}}}, - nodes: { - returns: - 'Returns the existing nodes table after applying normalization functions for sigma.js' - }, - nodes: {bounds: {returns: 'Returns the x and y bounds to be used in rendering.'}}, - edges: - {return: 'Returns the existing edges table after applying normalization for sigma.js'} + returns: 'Result OK/Not Found/Fail' + }, + list_tables: {returns: 'Tables that are available presently in GPU memory.'}, + get_table: { + ':table': + {table: 'The name of the table that has been allocated previously into GPU memory.'} + }, + get_column: {':table': {':column': {table: 'The table name', column: 'The column name'}}}, + nodes: { + returns: + 'Returns the existing nodes table after applying normalization functions for sigma.js' + }, + nodes: {bounds: {returns: 'Returns the x and y bounds to be used in rendering.'}}, + edges: {return: 'Returns the existing edges table after applying normalization for sigma.js'} } -}) + }) }); test('read_json no filename', async t => { -const app = await build(t); -const res = await app.inject({method: 'POST', url: '/graphology/read_json'}); -t.same( - JSON.parse(res.payload), - {success: false, message: 'Parameter filename is required', params: '{}', statusCode: 400}); + const app = await build(t); + const res = await app.inject({method: 'POST', url: '/graphology/read_json'}); + t.same( + JSON.parse(res.payload), + {success: false, message: 'Parameter filename is required', params: '{}', statusCode: 400}); }); test('read_json no file', async (t) => { @@ -127,10 +128,15 @@ test('read_json incorrect format', async (t) => { test('read_json file good', async (t) => { const dir = t.testdir(json_good); const rpath = '../../test/routes/' + dir.substring(dir.lastIndexOf('/')) + '/json_good.txt'; - const app = await build(t); - const res = await app.inject({method: 'POST', url: '/graphology/read_json?filename=' + rpath}); + /* + const build = t.mock('../../routes/graphology/index.js', + {'../../util/gpu_cache.js': {publicPath: () => rpath}}); + */ + const app = await build(t); + const res = await app.inject({method: 'POST', url: '/graphology/read_json?filename=' + rpath}); const release = await app.inject({method: 'POST', url: '/graphology/release'}); const payload = JSON.parse(res.payload); + console.log(payload); t.equal(payload.message, 'File read onto GPU.'); t.equal(payload.success, true); }); @@ -299,7 +305,6 @@ test('edges and nodes do not begin with 0', async (t) => { const res = await app.inject( {method: 'GET', url: '/graphology/edges', header: {'accepts': 'application/octet-stream'}}); const release = await app.inject({method: 'POST', url: '/graphology/release'}); - debugger; t.equal(res.statusCode, 200); const table = tableFromIPC(res.rawPayload); t.ok(table.getChild('edges')); @@ -319,7 +324,7 @@ test('edges and nodes do not begin with 0', async (t) => { ])) }); -test('edge keys do not match node keys', {only: true}, async (t) => { +test('edge keys do not match node keys', async (t) => { const dir = t.testdir(json_bad_map); const rpath = '../../test/routes/' + dir.substring(dir.lastIndexOf('/')) + '/json_bad_map.txt'; const app = await build(t); @@ -328,7 +333,6 @@ test('edge keys do not match node keys', {only: true}, async (t) => { const res = await app.inject( {method: 'GET', url: '/graphology/edges', header: {'accepts': 'application/octet-stream'}}); const release = await app.inject({method: 'POST', url: '/graphology/release'}); - debugger; t.equal(res.statusCode, 422); t.same(JSON.parse(res.payload), {success: false, message: 'Edge sources do not match node keys', statusCode: 422}); diff --git a/modules/demo/api-server/test/routes/root.test.js b/modules/demo/api-server/test/routes/root.test.js index 290b4e375..04624a64f 100644 --- a/modules/demo/api-server/test/routes/root.test.js +++ b/modules/demo/api-server/test/routes/root.test.js @@ -21,52 +21,90 @@ test('root returns API description', async (t) => { const app = await build(t); const res = await app.inject({url: '/'}); t.same(JSON.parse(res.payload), { - gpu: { - description: 'An abstract interface to the node-rapids api, supported by a server.', - schema: { - '/': { - method: 'The name of the method to apply to gpu_cache data.', - caller: 'Either an object that has been stored in the gpu_cache or a static module name.', - arguments: 'Correctly specified arguments to the gpu_cache method.', - result: 'Either a result code specifying success or failure or an arrow data buffer.', - } + gpu: { + description: 'An abstract interface to the node-rapids api, supported by a server.', + schema: { + '/': { + method: 'The name of the method to apply to gpu_cache data.', + caller: 'Either an object that has been stored in the gpu_cache or a static module name.', + arguments: 'Correctly specified arguments to the gpu_cache method.', + result: 'Either a result code specifying success or failure or an arrow data buffer.', + }, + 'DataFrame/readCSV': { + method: 'POST', + params: { + filename: 'The name of the file, stored in the server\'s public/ folder, of the csv file.' + }, + result: `Causes the node-rapids backend to attempt to load the csv file specified + by :filename. The GPU will attempt to parse the CSV file asynchronously and will + return OK/ Not Found/ or Fail based on the file status.`, + returns: '500/404/200' + }, + 'get_column/:table/:column': { + method: 'GET', + params: { + ':table': + 'The filename of a previously loaded dataset, for example with `DataFrame/readCSV`', + ':column': 'A valid column name in a DataFrame that has been previously loaded.' + }, + returns: 'An Arrow `RecordBatchStreamWriter` stream of the columnar data.' } - }, - graphology: { - description: 'The graphology api provides GPU acceleration of graphology datasets.', - schema: { - read_json: { - filename: 'A URI to a graphology json dataset file.', - result: `Causes the node-rapids backend to attempt to load the json object specified + } + }, + graphology: { + description: 'The graphology api provides GPU acceleration of graphology datasets.', + schema: { + read_json: { + filename: 'A URI to a graphology json dataset file.', + result: `Causes the node-rapids backend to attempt to load the json object specified by :filename. The GPU will attempt to parse the json file asynchronously and will return OK/ Not Found/ or Fail based on the file status. If the load is successful, four tables will be created in the node-rapids backend: nodes, edges, clusters, and tags. The root objects in the json target must match these names and order.`, - returns: 'Result OK/Not Found/Fail' - }, - read_large_demo: { - filename: - 'A URI to a graphology json dataset file matching the sigma.js/examples/large-demos spec.', - result: `Produces the same result as 'read_json'. + returns: 'Result OK/Not Found/Fail' + }, + read_large_demo: { + filename: + 'A URI to a graphology json dataset file matching the sigma.js/examples/large-demos spec.', + result: `Produces the same result as 'read_json'. If the load is successful, three tables will be created in the node-rapids backend: nodes, edges, and options.`, - returns: 'Result OK/Not Found/Fail' - }, - list_tables: {returns: 'Tables that are available presently in GPU memory.'}, - get_table: { - ':table': - {table: 'The name of the table that has been allocated previously into GPU memory.'} - }, - get_column: {':table': {':column': {table: 'The table name', column: 'The column name'}}}, - nodes: { - returns: - 'Returns the existing nodes table after applying normalization functions for sigma.js' + returns: 'Result OK/Not Found/Fail' + }, + list_tables: {returns: 'Tables that are available presently in GPU memory.'}, + get_table: { + ':table': + {table: 'The name of the table that has been allocated previously into GPU memory.'} + }, + get_column: {':table': {':column': {table: 'The table name', column: 'The column name'}}}, + nodes: { + returns: + 'Returns the existing nodes table after applying normalization functions for sigma.js' + }, + nodes: {bounds: {returns: 'Returns the x and y bounds to be used in rendering.'}}, + edges: {return: 'Returns the existing edges table after applying normalization for sigma.js'} + } + }, + particles: { + description: + 'The API responsible for parsing particles CSV files for the point-budget API demo.', + schema: { + 'get_shader_column/:table/:xmin/:xmax/:ymin/:ymax': { + method: 'POST', + params: { + ':table': 'The name of the CSV file previously loaded with `DataFrame/readCSV`', + 'xmin (optional)': 'Don\'t return results outside of xmin', + 'xmax (optional)': 'Don\'t return results outside of xmax', + 'ymin (optional)': 'Don\'t return results outside of ymin', + 'ymax (optional)': 'Don\'t return results outside of ymax' }, - nodes: {bounds: {returns: 'Returns the x and y bounds to be used in rendering.'}}, - edges: - {return: 'Returns the existing edges table after applying normalization for sigma.js'} + result: `Returns the Longitude and Latitude columns of a table that has been read previously + with DataFrame/readCSV. The Longitude and Latitude will be packed into a a single column and + interleaved.`, + return: 'Returns an Arrow stream of lon/lat values as a Table containing a single column.' } } + } }); }); diff --git a/modules/demo/api-server/util/gpu_cache.js b/modules/demo/api-server/util/gpu_cache.js index 6483efb4b..01f970205 100644 --- a/modules/demo/api-server/util/gpu_cache.js +++ b/modules/demo/api-server/util/gpu_cache.js @@ -14,13 +14,14 @@ const {Bool8, Utf8String, Int32, Int64, DataFrame, Series, Float32, Float64} = require('@rapidsai/cudf'); +const Path = require('path'); let timeout = -1; let datasets = {}; function clearCachedGPUData() { for (const key in datasets) { - datasets[key].dispose(); + const dataset = datasets[key]; datasets[key] = null; } }; @@ -66,12 +67,13 @@ function json_aoa_to_dataframe(str, dtypes) { return result; } +let _publicPath = Path.join(__dirname, '../../public'); + module.exports = { async setDataframe(name, dataframe) { if (timeout) { clearTimeout(timeout); } timeout = setTimeout(clearCachedGPUData, 10 * 60 * 1000); if (datasets === null) { - for (key in Object.datasets.keys()) { datasets.key.destroy(); } datasets = {}; } datasets[name] = dataframe; @@ -87,6 +89,9 @@ module.exports = { datasets = null; }, + _setPathForTesting(path) { _publicPath = path; }, + publicPath() { return _publicPath; }, + async readLargeGraphDemo(path) { console.log('readLargeGraphDemo'); const dataset = Series.readText(path, ''); diff --git a/modules/demo/api-server/util/schema.js b/modules/demo/api-server/util/schema.js index b9589b888..0ff7555ab 100644 --- a/modules/demo/api-server/util/schema.js +++ b/modules/demo/api-server/util/schema.js @@ -23,6 +23,25 @@ const schema = { caller: 'Either an object that has been stored in the gpu_cache or a static module name.', arguments: 'Correctly specified arguments to the gpu_cache method.', result: 'Either a result code specifying success or failure or an arrow data buffer.', + }, + 'DataFrame/readCSV': { + method: 'POST', + params: { + filename: 'The name of the file, stored in the server\'s public/ folder, of the csv file.' + }, + result: `Causes the node-rapids backend to attempt to load the csv file specified + by :filename. The GPU will attempt to parse the CSV file asynchronously and will + return OK/ Not Found/ or Fail based on the file status.`, + returns: '500/404/200' + }, + 'get_column/:table/:column': { + method: 'GET', + params: { + ':table': + 'The filename of a previously loaded dataset, for example with `DataFrame/readCSV`', + ':column': 'A valid column name in a DataFrame that has been previously loaded.' + }, + returns: 'An Arrow `RecordBatchStreamWriter` stream of the columnar data.' } } }, @@ -60,6 +79,26 @@ const schema = { nodes: {bounds: {returns: 'Returns the x and y bounds to be used in rendering.'}}, edges: {return: 'Returns the existing edges table after applying normalization for sigma.js'} } + }, + particles: { + description: + 'The API responsible for parsing particles CSV files for the point-budget API demo.', + schema: { + 'get_shader_column/:table/:xmin/:xmax/:ymin/:ymax': { + method: 'POST', + params: { + ':table': 'The name of the CSV file previously loaded with `DataFrame/readCSV`', + 'xmin (optional)': 'Don\'t return results outside of xmin', + 'xmax (optional)': 'Don\'t return results outside of xmax', + 'ymin (optional)': 'Don\'t return results outside of ymin', + 'ymax (optional)': 'Don\'t return results outside of ymax' + }, + result: `Returns the Longitude and Latitude columns of a table that has been read previously + with DataFrame/readCSV. The Longitude and Latitude will be packed into a a single column and + interleaved.`, + return: 'Returns an Arrow stream of lon/lat values as a Table containing a single column.' + } + } } }; From 7401333851ad03ab23f941b54682ba14fe672ba8 Mon Sep 17 00:00:00 2001 From: "H. Thomson Comer" Date: Mon, 21 Nov 2022 14:32:45 -0600 Subject: [PATCH 20/43] Add new test files. --- .../demo/api-server/test/routes/gpu.test.js | 42 +++++++++++++++++++ .../api-server/test/routes/particles.test.js | 40 ++++++++++++++++++ 2 files changed, 82 insertions(+) create mode 100644 modules/demo/api-server/test/routes/gpu.test.js create mode 100644 modules/demo/api-server/test/routes/particles.test.js diff --git a/modules/demo/api-server/test/routes/gpu.test.js b/modules/demo/api-server/test/routes/gpu.test.js new file mode 100644 index 000000000..203b64494 --- /dev/null +++ b/modules/demo/api-server/test/routes/gpu.test.js @@ -0,0 +1,42 @@ +// Copyright (c) 2022, NVIDIA CORPORATION. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +'use strict' + +const {dir} = require('console'); +const {test} = require('tap'); +const {build} = require('../helper'); +const {tableFromIPC, RecordBatchStreamWriter} = require('apache-arrow'); +const {json_large, json_good, json_out_of_order, json_bad_map, csv_base} = + require('../fixtures.js'); +const gpu_cache = require('../../util/gpu_cache'); + +test('read_csv', async (t) => { + const dir = t.testdir(csv_base); + const rpath = 'test/routes/' + dir.substring(dir.lastIndexOf('/')); + const app = await build(t); + gpu_cache._setPathForTesting(rpath); + const res = await app.inject( + {method: 'POST', url: '/gpu/DataFrame/readCSV', body: {filename: 'csv_base.csv'}}); + const release = await app.inject({method: 'POST', url: '/graphology/release'}); + t.equal(res.statusCode, 200); + console.log(res.statusCode); + console.log(res.payload); + t.same(JSON.parse(res.payload), { + success: true, + message: 'CSV file in GPU memory.', + statusCode: 200, + params: {filename: 'csv_base.csv'} + }); +}); diff --git a/modules/demo/api-server/test/routes/particles.test.js b/modules/demo/api-server/test/routes/particles.test.js new file mode 100644 index 000000000..e5a2f1ea1 --- /dev/null +++ b/modules/demo/api-server/test/routes/particles.test.js @@ -0,0 +1,40 @@ +// Copyright (c) 2022, NVIDIA CORPORATION. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +'use strict' + +const {dir} = require('console'); +const {test} = require('tap'); +const {build} = require('../helper'); +const {tableFromIPC, RecordBatchStreamWriter} = require('apache-arrow'); +const {json_large, json_good, json_out_of_order, json_bad_map, csv_base} = + require('../fixtures.js'); +const gpu_cache = require('../../util/gpu_cache'); + +test('read_csv', async (t) => { + const dir = t.testdir(csv_base); + const rpath = 'test/routes/' + dir.substring(dir.lastIndexOf('/')); + const app = await build(t); + gpu_cache._setPathForTesting(rpath); + const res = await app.inject( + {method: 'POST', url: '/gpu/DataFrame/readCSV', body: {filename: 'csv_base.csv'}}); + const release = await app.inject({method: 'POST', url: '/graphology/release'}); + t.equal(res.statusCode, 200); + t.same(JSON.parse(res.payload), { + success: true, + message: 'CSV file in GPU memory.', + statusCode: 200, + params: {filename: 'csv_base.csv'} + }); +}); From 4fdc5d7ae7bec73ba328205be51d6145188530fc Mon Sep 17 00:00:00 2001 From: "H. Thomson Comer" Date: Mon, 21 Nov 2022 16:53:59 -0600 Subject: [PATCH 21/43] Write unit tests for particles/get_shader_column in preparation of writing :npoints argument --- .../demo/api-server/routes/particles/index.js | 2 +- modules/demo/api-server/test/fixtures.js | 17 +++++-- .../api-server/test/routes/particles.test.js | 49 ++++++++++++++----- modules/demo/api-server/util/gpu_cache.js | 2 +- 4 files changed, 53 insertions(+), 17 deletions(-) diff --git a/modules/demo/api-server/routes/particles/index.js b/modules/demo/api-server/routes/particles/index.js index eff267328..f459f92da 100644 --- a/modules/demo/api-server/routes/particles/index.js +++ b/modules/demo/api-server/routes/particles/index.js @@ -46,7 +46,7 @@ module.exports = async function(fastify, opts) { const handler = async (request, reply) => { let message = 'Error'; - let result = {'params': JSON.stringify(request.params), success: false, message: message}; + let result = {'params': request.params, success: false, message: message}; const table = await fastify.getDataframe(request.params.table); if (table == undefined) { result.message = 'Table not found'; diff --git a/modules/demo/api-server/test/fixtures.js b/modules/demo/api-server/test/fixtures.js index 0f932f9e6..e7d70c2e7 100644 --- a/modules/demo/api-server/test/fixtures.js +++ b/modules/demo/api-server/test/fixtures.js @@ -168,8 +168,18 @@ const csv_base = { `Index,Name,Int,Float 0,"bob",1,1.0 1,"george",2,2.0 - 2,"sam",3,3.0 - ` + 2,"sam",3,3.0` +}; + +const csv_particles = { + 'csv_particles.csv': + `Index,Longitude,Latitude + 0, -105, 40 + 1, -106, 41 + 2, -107, 42 + 3, -108, 43 + 4, -109, 44 + 5, -110, 45` }; module.exports = { @@ -177,5 +187,6 @@ module.exports = { json_large: json_large, json_out_of_order: json_out_of_order, json_bad_map: json_bad_map, - csv_base: csv_base + csv_base: csv_base, + csv_particles: csv_particles }; diff --git a/modules/demo/api-server/test/routes/particles.test.js b/modules/demo/api-server/test/routes/particles.test.js index e5a2f1ea1..54135d28e 100644 --- a/modules/demo/api-server/test/routes/particles.test.js +++ b/modules/demo/api-server/test/routes/particles.test.js @@ -18,23 +18,48 @@ const {dir} = require('console'); const {test} = require('tap'); const {build} = require('../helper'); const {tableFromIPC, RecordBatchStreamWriter} = require('apache-arrow'); -const {json_large, json_good, json_out_of_order, json_bad_map, csv_base} = +const {json_large, json_good, json_out_of_order, json_bad_map, csv_base, csv_particles} = require('../fixtures.js'); const gpu_cache = require('../../util/gpu_cache'); -test('read_csv', async (t) => { - const dir = t.testdir(csv_base); +test('get_shader_column/:table wrong table', async (t) => { + const app = await build(t); + const res = await app.inject({method: 'GET', url: '/particles/get_shader_column/no_table'}); + t.equal(res.statusCode, 404); + const expected = {params: {table: 'no_table'}, success: false, message: 'Table not found'}; + console.log(res.payload); + const got = JSON.parse(res.payload); + t.same(got, expected); +}); + +test('get_shader_column/:table', async (t) => { + const dir = t.testdir(csv_particles); + const rpath = 'test/routes/' + dir.substring(dir.lastIndexOf('/')); + const app = await build(t); + gpu_cache._setPathForTesting(rpath); + const load = await app.inject( + {method: 'POST', url: '/gpu/DataFrame/readCSV', body: {filename: 'csv_particles.csv'}}); + const res = + await app.inject({method: 'GET', url: '/particles/get_shader_column/csv_particles.csv'}); + const expected = [-105, 40, -106, 41, -107, 42, -108, 43, -109, 44, -110, 45]; + const got = tableFromIPC(res.rawPayload).getChild('gpu_buffer').toArray(); + const release = await app.inject({method: 'POST', url: '/graphology/release'}); + t.same(got, expected); +}); + +test('get_shader_column/:table/:xmin/:xmax/:ymin/:ymax', async (t) => { + const dir = t.testdir(csv_particles); const rpath = 'test/routes/' + dir.substring(dir.lastIndexOf('/')); const app = await build(t); gpu_cache._setPathForTesting(rpath); + const load = await app.inject( + {method: 'POST', url: '/gpu/DataFrame/readCSV', body: {filename: 'csv_particles.csv'}}); const res = await app.inject( - {method: 'POST', url: '/gpu/DataFrame/readCSV', body: {filename: 'csv_base.csv'}}); - const release = await app.inject({method: 'POST', url: '/graphology/release'}); - t.equal(res.statusCode, 200); - t.same(JSON.parse(res.payload), { - success: true, - message: 'CSV file in GPU memory.', - statusCode: 200, - params: {filename: 'csv_base.csv'} - }); + {method: 'GET', url: '/particles/get_shader_column/csv_particles.csv/-109/-106/41/44'}); + const expected = [-107, 42, -108, 43]; + const got = tableFromIPC(res.rawPayload).getChild('gpu_buffer').toArray(); + const release = await app.inject({method: 'POST', url: '/graphology/release'}); + t.same(got, expected); }); + +test('get_shader_column/:table/:npoints', async (t) => {}); diff --git a/modules/demo/api-server/util/gpu_cache.js b/modules/demo/api-server/util/gpu_cache.js index 01f970205..ddef25b00 100644 --- a/modules/demo/api-server/util/gpu_cache.js +++ b/modules/demo/api-server/util/gpu_cache.js @@ -79,7 +79,7 @@ module.exports = { datasets[name] = dataframe; }, - async getDataframe(name) { return datasets[name]; }, + async getDataframe(name) { return datasets != null ? datasets[name] : undefined; }, async listDataframes() { return datasets != null ? Object.keys(datasets) : []; }, From f28d2777b7eecac9040a97c186bb040cefcfc1ec Mon Sep 17 00:00:00 2001 From: "H. Thomson Comer" Date: Tue, 22 Nov 2022 09:54:36 -0600 Subject: [PATCH 22/43] Write npoints test and refactor endpoint. --- .../demo/api-server/routes/particles/index.js | 52 ++++++------------- .../api-server/test/routes/particles.test.js | 15 +++++- 2 files changed, 30 insertions(+), 37 deletions(-) diff --git a/modules/demo/api-server/routes/particles/index.js b/modules/demo/api-server/routes/particles/index.js index f459f92da..cca205000 100644 --- a/modules/demo/api-server/routes/particles/index.js +++ b/modules/demo/api-server/routes/particles/index.js @@ -44,6 +44,14 @@ module.exports = async function(fastify, opts) { fastify.get('/', {...get_schema, handler: () => root_schema['particles']}); fastify.post('/', {...get_schema, handler: () => root_schema['particles']}); + const filterPoints = + (column, min, max) => { + const gt = column._col.gt(parseInt(min)); + const lt = column._col.lt(parseInt(max)); + const mask = gt.bitwiseAnd(lt); + return column.filter(Series.new(mask)); + } + const handler = async (request, reply) => { let message = 'Error'; let result = {'params': request.params, success: false, message: message}; @@ -57,36 +65,12 @@ module.exports = async function(fastify, opts) { let y = undefined; if (request.params.xmin != undefined && request.params.xmax != undefined && request.params.ymin != undefined && request.params.ymax != undefined) { - const x_unfiltered = table.get('Longitude'); - const x_gt = x_unfiltered._col.gt(parseInt(request.params.xmin)); - const x_lt = x_unfiltered._col.lt(parseInt(request.params.xmax)); - const y_unfiltered = table.get('Latitude'); - const y_gt = y_unfiltered._col.gt(parseInt(request.params.ymin)); - const y_lt = y_unfiltered._col.lt(parseInt(request.params.ymax)); - const x_y = x_lt.bitwiseAnd(x_gt).bitwiseAnd(y_lt).bitwiseAnd(y_gt); - x = x_unfiltered.filter(Series.new(x_y)); - y = y_unfiltered.filter(Series.new(x_y)); + x = filterPoints(table.get('Longitude'), request.params.xmin, request.params.xmax); + y = filterPoints(table.get('Latitude'), request.params.ymin, request.params.ymax); } else { x = table.get('Longitude'); y = table.get('Latitude'); } - // const state = table.get('State'); - // const zip = table.get('Zip_Code') - // Produce r,g,b from state - const color_map = [ - {'r': 0, 'g': 0, 'b': 0}, - {'r': 255, 'g': 0, 'b': 0}, - {'r': 0, 'g': 255, 'b': 0}, - {'r': 255, 'g': 255, 'b': 0}, - {'r': 0, 'g': 0, 'b': 255}, - {'r': 255, 'g': 0, 'b': 255}, - {'r': 0, 'g': 255, 'b': 255}, - {'r': 255, 'g': 255, 'b': 255} - ]; - // TODO: convert state to color by state index - const r = Series.sequence({type: new Int32, init: 255.0, size: x.length}).fill(0); - const g = Series.sequence({type: new Int32, init: 255.0, size: x.length}).fill(0); - const b = Series.sequence({type: new Int32, init: 255.0, size: x.length}).fill(0); // Map x, y, r, g, b to offsets for client display let tiled = Series.sequence({type: new Float32, init: 0.0, size: (2 * x.length)}); @@ -95,16 +79,6 @@ module.exports = async function(fastify, opts) { x.dispose(); tiled = tiled.scatter(y, base_offset.add(1).cast(new Int32)); y.dispose(); - /* - tiled = tiled.scatter(1.0, base_offset.add(2).cast(new Int32)); - tiled = tiled.scatter(1.0, base_offset.add(3).cast(new Int32)); - tiled = tiled.scatter(r, base_offset.add(4).cast(new Int32)); - r.dispose(); - tiled = tiled.scatter(g, base_offset.add(5).cast(new Int32)); - g.dispose(); - tiled = tiled.scatter(b, base_offset.add(6).cast(new Int32)); - b.dispose(); - */ const result = new DataFrame({'gpu_buffer': tiled}); const writer = RecordBatchStreamWriter.writeAll(result.toArrow()); await reply.code(200).send(writer.toNodeStream()); @@ -141,6 +115,12 @@ module.exports = async function(fastify, opts) { }, handler: handler }); + fastify.route({ + method: 'GET', + url: '/get_shader_column/:table/:npoints', + schema: {querystring: {table: {type: 'string'}, npoints: {type: 'number'}}}, + handler: handler + }); fastify.route({ method: 'GET', url: '/get_shader_column/:table', diff --git a/modules/demo/api-server/test/routes/particles.test.js b/modules/demo/api-server/test/routes/particles.test.js index 54135d28e..fda5c8bcd 100644 --- a/modules/demo/api-server/test/routes/particles.test.js +++ b/modules/demo/api-server/test/routes/particles.test.js @@ -62,4 +62,17 @@ test('get_shader_column/:table/:xmin/:xmax/:ymin/:ymax', async (t) => { t.same(got, expected); }); -test('get_shader_column/:table/:npoints', async (t) => {}); +test('get_shader_column/:table/:npoints', async (t) => { + const dir = t.testdir(csv_particles); + const rpath = 'test/routes/' + dir.substring(dir.lastIndexOf('/')); + const app = await build(t); + gpu_cache._setPathForTesting(rpath); + const load = await app.inject( + {method: 'POST', url: '/gpu/DataFrame/readCSV', body: {filename: 'csv_particles.csv'}}); + const res = + await app.inject({method: 'GET', url: '/particles/get_shader_column/csv_particles.csv/1'}); + const release = await app.inject({method: 'POST', url: '/graphology/release'}); + const expected = 2; + const got = tableFromIPC(res.rawPayload).getChild('gpu_buffer').toArray(); + t.equal(expected, got.length); +}); From f371cc7248d3e7c0133f1b824f3fa87e1fc81fe9 Mon Sep 17 00:00:00 2001 From: "H. Thomson Comer" Date: Tue, 22 Nov 2022 16:39:03 -0600 Subject: [PATCH 23/43] Add quadtree loading route. --- .../demo/api-server/routes/particles/index.js | 20 ++-- .../demo/api-server/routes/quadtree/index.js | 92 +++++++++++++++++++ modules/demo/api-server/test/fixtures.js | 17 +++- .../api-server/test/routes/graphology.test.js | 2 +- .../api-server/test/routes/particles.test.js | 15 --- .../api-server/test/routes/quadtree.test.js | 41 +++++++++ 6 files changed, 164 insertions(+), 23 deletions(-) create mode 100644 modules/demo/api-server/routes/quadtree/index.js create mode 100644 modules/demo/api-server/test/routes/quadtree.test.js diff --git a/modules/demo/api-server/routes/particles/index.js b/modules/demo/api-server/routes/particles/index.js index cca205000..fbc663d3d 100644 --- a/modules/demo/api-server/routes/particles/index.js +++ b/modules/demo/api-server/routes/particles/index.js @@ -117,14 +117,22 @@ module.exports = async function(fastify, opts) { }); fastify.route({ method: 'GET', - url: '/get_shader_column/:table/:npoints', - schema: {querystring: {table: {type: 'string'}, npoints: {type: 'number'}}}, + url: '/get_shader_column/:table', + schema: {querystring: {table: {type: 'string'}}}, handler: handler }); fastify.route({ - method: 'GET', - url: '/get_shader_column/:table', + method: 'POST', + url: '/quadtree/create/:table', schema: {querystring: {table: {type: 'string'}}}, - handler: handler + handler: async (request, reply) => { + let message = 'Error'; + let result = {'params': request.params, success: false, message: message}; + const table = await fastify.getDataframe(request.params.table); + if (table == undefined) { + result.message = 'Table not found'; + await reply.code(404).send(result); + } + } }); -}; +} diff --git a/modules/demo/api-server/routes/quadtree/index.js b/modules/demo/api-server/routes/quadtree/index.js new file mode 100644 index 000000000..02298f7fa --- /dev/null +++ b/modules/demo/api-server/routes/quadtree/index.js @@ -0,0 +1,92 @@ +// Copyright (c) 2022, NVIDIA CORPORATION. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +const {Int32, Float32, DataFrame, Series} = require('@rapidsai/cudf'); +const {RecordBatchStreamWriter} = require('apache-arrow'); +const fastify = require('fastify')({logger: {level: 'debug'}}); +const fastifyCors = require('@fastify/cors'); +const cuspatial = require('@rapidsai/cuspatial'); + +const arrowPlugin = require('fastify-arrow'); +const gpu_cache = require('../../util/gpu_cache.js'); +const root_schema = require('../../util/schema.js'); + +module.exports = async function(fastify, opts) { + fastify.register(fastifyCors, {origin: '*'}); + fastify.register(arrowPlugin); + fastify.decorate('setDataframe', gpu_cache.setDataframe); + fastify.decorate('getDataframe', gpu_cache.getDataframe); + fastify.decorate('readCSV', gpu_cache.readCSV); + + const get_schema = { + logLevel: 'debug', + schema: { + response: { + 200: { + type: 'object', + properties: + {success: {type: 'boolean'}, message: {type: 'string'}, params: {type: 'string'}} + } + } + } + }; + + fastify.get('/', {...get_schema, handler: () => root_schema['quadtree']}); + fastify.post('/', {...get_schema, handler: () => root_schema['quadtree']}); + + fastify.route({ + method: 'POST', + url: '/create/:table', + schema: {querystring: {table: {type: 'string'}}}, + handler: async (request, reply) => { + let message = 'Error'; + let result = {'params': request.params, success: false, message: message}; + const table = await fastify.getDataframe(request.params.table); + if (table == undefined) { + result.message = 'Table not found'; + await reply.code(404).send(result); + } else { + const [xMin, xMax, yMin, yMax] = [ + parseFloat(table.get('x').min()), + parseFloat(table.get('x').max()), + parseFloat(table.get('y').min()), + parseFloat(table.get('y').max()), + ]; + try { + const quadtree = cuspatial.Quadtree.new({ + x: table.get('x'), + y: table.get('y'), + xMin, + xMax, + yMin, + yMax, + scale: 0, + maxDepth: 15, + minSize: 1e5 + }); + const saved = await fastify.setDataframe(request.params.table); + result.message = 'Quadtree created'; + result.success = true; + result.statusCode = 200; + await reply.code(result.statusCode).send(result); + } catch (e) { + result.message = e; + result.success = false; + result.statusCode = 500; + await reply.code(result.statusCode).send(result); + } + } + } + }); +} diff --git a/modules/demo/api-server/test/fixtures.js b/modules/demo/api-server/test/fixtures.js index e7d70c2e7..8ad838150 100644 --- a/modules/demo/api-server/test/fixtures.js +++ b/modules/demo/api-server/test/fixtures.js @@ -182,11 +182,26 @@ const csv_particles = { 5, -110, 45` }; +const csv_quadtree = { + 'csv_quadtree.csv': + `Index,x,y + 0,-4.0,4.0 + 1,-3.0,3.0 + 2,-2.0,2.0 + 3,-1.0,1.0 + 4,0.0,0.0 + 5,1.0,-1.0 + 6,2.0,-2.0 + 7,3.0,-3.0 + 8,4.0,-4.0` +}; + module.exports = { json_good: json_good, json_large: json_large, json_out_of_order: json_out_of_order, json_bad_map: json_bad_map, csv_base: csv_base, - csv_particles: csv_particles + csv_particles: csv_particles, + csv_quadtree: csv_quadtree }; diff --git a/modules/demo/api-server/test/routes/graphology.test.js b/modules/demo/api-server/test/routes/graphology.test.js index 2e449725f..648c2252d 100644 --- a/modules/demo/api-server/test/routes/graphology.test.js +++ b/modules/demo/api-server/test/routes/graphology.test.js @@ -22,7 +22,7 @@ const {json_large, json_good, json_out_of_order, json_bad_map, csv_base} = require('../fixtures.js'); const gpu_cache = require('../../util/gpu_cache'); -test('graphology root returns api description', {only: true}, async t => { +test('graphology root returns api description', async t => { const app = await build(t); const res = await app.inject({url: '/graphology'}) t.same(JSON.parse(res.payload), { diff --git a/modules/demo/api-server/test/routes/particles.test.js b/modules/demo/api-server/test/routes/particles.test.js index fda5c8bcd..673b34627 100644 --- a/modules/demo/api-server/test/routes/particles.test.js +++ b/modules/demo/api-server/test/routes/particles.test.js @@ -61,18 +61,3 @@ test('get_shader_column/:table/:xmin/:xmax/:ymin/:ymax', async (t) => { const release = await app.inject({method: 'POST', url: '/graphology/release'}); t.same(got, expected); }); - -test('get_shader_column/:table/:npoints', async (t) => { - const dir = t.testdir(csv_particles); - const rpath = 'test/routes/' + dir.substring(dir.lastIndexOf('/')); - const app = await build(t); - gpu_cache._setPathForTesting(rpath); - const load = await app.inject( - {method: 'POST', url: '/gpu/DataFrame/readCSV', body: {filename: 'csv_particles.csv'}}); - const res = - await app.inject({method: 'GET', url: '/particles/get_shader_column/csv_particles.csv/1'}); - const release = await app.inject({method: 'POST', url: '/graphology/release'}); - const expected = 2; - const got = tableFromIPC(res.rawPayload).getChild('gpu_buffer').toArray(); - t.equal(expected, got.length); -}); diff --git a/modules/demo/api-server/test/routes/quadtree.test.js b/modules/demo/api-server/test/routes/quadtree.test.js new file mode 100644 index 000000000..45bf4aa20 --- /dev/null +++ b/modules/demo/api-server/test/routes/quadtree.test.js @@ -0,0 +1,41 @@ +// Copyright (c) 2022, NVIDIA CORPORATION. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +'use strict' + +const {dir} = require('console'); +const {test} = require('tap'); +const {build} = require('../helper'); +const {tableFromIPC, RecordBatchStreamWriter} = require('apache-arrow'); +const {csv_quadtree} = require('../fixtures.js'); +const gpu_cache = require('../../util/gpu_cache'); + +test('quadtree/create/:table', {only: true}, async (t) => { + const dir = t.testdir(csv_quadtree); + const rpath = 'test/routes/' + dir.substring(dir.lastIndexOf('/')); + const app = await build(t); + gpu_cache._setPathForTesting(rpath); + const load = await app.inject( + {method: 'POST', url: '/gpu/DataFrame/readCSV', body: {filename: 'csv_quadtree.csv'}}); + const res = await app.inject({method: 'POST', url: '/quadtree/create/csv_quadtree.csv'}); + const release = await app.inject({method: 'POST', url: '/graphology/release'}); + const result = JSON.parse(res.payload); + t.same(result.message, 'Quadtree created'); + t.same(result, { + statusCode: 200, + message: 'Quadtree created', + params: {table: 'csv_quadtree.csv'}, + success: true + }) +}); From 54d3d31219ac80b4f7ac07080e57581b04b27be4 Mon Sep 17 00:00:00 2001 From: "H. Thomson Comer" Date: Wed, 23 Nov 2022 09:14:17 -0600 Subject: [PATCH 24/43] Add set_polygons_quadtree and refactor gpu_cache. --- modules/demo/api-server/routes/gpu/index.js | 10 +- .../api-server/routes/graphology/index.js | 30 +++--- .../demo/api-server/routes/particles/index.js | 8 +- .../demo/api-server/routes/quadtree/index.js | 55 +++++++++-- .../api-server/test/plugins/gpu_cache.test.js | 14 +-- .../demo/api-server/test/routes/gpu.test.js | 3 - .../api-server/test/routes/quadtree.test.js | 33 ++++++- modules/demo/api-server/util/gpu_cache.js | 94 ++++++++++--------- 8 files changed, 154 insertions(+), 93 deletions(-) diff --git a/modules/demo/api-server/routes/gpu/index.js b/modules/demo/api-server/routes/gpu/index.js index 43c414992..a0e6f3df0 100644 --- a/modules/demo/api-server/routes/gpu/index.js +++ b/modules/demo/api-server/routes/gpu/index.js @@ -28,8 +28,8 @@ const root_schema = require('../../util/schema.js'); module.exports = async function(fastify, opts) { fastify.register(arrowPlugin); fastify.register(fastifyCors, {origin: '*'}); - fastify.decorate('setDataframe', gpu_cache.setDataframe); - fastify.decorate('getDataframe', gpu_cache.getDataframe); + fastify.decorate('cacheObject', gpu_cache.cacheObject); + fastify.decorate('getData', gpu_cache.getData); fastify.decorate('readCSV', gpu_cache.readCSV); fastify.decorate('publicPath', gpu_cache.publicPath); @@ -59,7 +59,7 @@ module.exports = async function(fastify, opts) { const path = Path.join(fastify.publicPath(), request.body.filename); const stats = await Stat(path); const message = 'File is available'; - const currentDataFrame = await fastify.getDataframe(request.body.filename); + const currentDataFrame = await fastify.getData(request.body.filename); if (currentDataFrame !== undefined) { console.log('Found existing dataframe.'); console.log(request.body); @@ -72,7 +72,7 @@ module.exports = async function(fastify, opts) { sources: [path], }); const name = request.body.filename; // request.body.replace('/\//g', '_'); - await fastify.setDataframe(name, cacheObject); + await fastify.cacheObject(name, cacheObject); result.success = true; result.message = 'CSV file in GPU memory.'; result.statusCode = 200; @@ -95,7 +95,7 @@ module.exports = async function(fastify, opts) { handler: async (request, reply) => { let message = 'Error'; let result = {'params': JSON.stringify(request.params), success: false, message: message}; - const table = await fastify.getDataframe(request.params.table); + const table = await fastify.getData(request.params.table); if (table == undefined) { result.message = 'Table not found'; await reply.code(404).send(result); diff --git a/modules/demo/api-server/routes/graphology/index.js b/modules/demo/api-server/routes/graphology/index.js index 032a39bdc..bffcfcea9 100644 --- a/modules/demo/api-server/routes/graphology/index.js +++ b/modules/demo/api-server/routes/graphology/index.js @@ -28,8 +28,8 @@ const root_schema = require('../../util/schema.js'); module.exports = async function(fastify, opts) { fastify.register(arrowPlugin); fastify.register(fastifyCors, {origin: 'http://localhost:3001'}); - fastify.decorate('setDataframe', gpu_cache.setDataframe); - fastify.decorate('getDataframe', gpu_cache.getDataframe); + fastify.decorate('cacheObject', gpu_cache.cacheObject); + fastify.decorate('getData', gpu_cache.getData); fastify.decorate('listDataframes', gpu_cache.listDataframes); fastify.decorate('readGraphology', gpu_cache.readGraphology); fastify.decorate('readLargeGraphDemo', gpu_cache.readLargeGraphDemo); @@ -76,9 +76,9 @@ module.exports = async function(fastify, opts) { result.success = true; try { const graphology = await fastify.readLargeGraphDemo(path); - await fastify.setDataframe('nodes', graphology['nodes']); - await fastify.setDataframe('edges', graphology['edges']); - await fastify.setDataframe('options', graphology['options']); + await fastify.cacheObject('nodes', graphology['nodes']); + await fastify.cacheObject('edges', graphology['edges']); + await fastify.cacheObject('options', graphology['options']); result.message = 'File read onto GPU.'; } catch (e) { result.success = false; @@ -135,10 +135,10 @@ module.exports = async function(fastify, opts) { result.success = true; try { const graphology = await fastify.readGraphology(path); - await fastify.setDataframe('nodes', graphology['nodes']); - await fastify.setDataframe('edges', graphology['edges']); - await fastify.setDataframe('clusters', graphology['clusters']); - await fastify.setDataframe('tags', graphology['tags']); + await fastify.cacheObject('nodes', graphology['nodes']); + await fastify.cacheObject('edges', graphology['edges']); + await fastify.cacheObject('clusters', graphology['clusters']); + await fastify.cacheObject('tags', graphology['tags']); result.message = 'File read onto GPU.'; } catch (e) { result.success = false; @@ -173,7 +173,7 @@ module.exports = async function(fastify, opts) { handler: async (request, reply) => { let message = 'Error'; let result = {'params': JSON.stringify(request.params), success: false, message: message}; - const table = await fastify.getDataframe(request.params.table); + const table = await fastify.getData(request.params.table); if (table == undefined) { result.message = 'Table not found'; await reply.code(404).send(result); @@ -208,7 +208,7 @@ module.exports = async function(fastify, opts) { handler: async (request, reply) => { let message = 'Error'; let result = {'params': JSON.stringify(request.params), success: false, message: message}; - const table = await fastify.getDataframe(request.params.table); + const table = await fastify.getData(request.params.table); if (table == undefined) { result.message = 'Table not found'; await reply.code(404).send(result); @@ -225,7 +225,7 @@ module.exports = async function(fastify, opts) { handler: async (request, reply) => { let message = 'Error'; let result = {success: false, message: message}; - const df = await fastify.getDataframe('nodes'); + const df = await fastify.getData('nodes'); if (df == undefined) { result.message = 'Table not found'; await reply.code(404).send(result); @@ -250,7 +250,7 @@ module.exports = async function(fastify, opts) { handler: async (request, reply) => { let message = 'Error'; let result = {success: false, message: message}; - const df = await fastify.getDataframe('nodes'); + const df = await fastify.getData('nodes'); if (df == undefined) { result.message = 'Table not found'; await reply.code(404).send(result); @@ -290,9 +290,9 @@ module.exports = async function(fastify, opts) { let message = 'Error'; let result = {success: false, message: message}; /** @type DataFrame<{x: Float32, y: Float32}> */ - const df = await fastify.getDataframe('nodes'); + const df = await fastify.getData('nodes'); /** @type DataFrame<{x: Int32, y: Int32}> */ - const edges = await fastify.getDataframe('edges'); + const edges = await fastify.getData('edges'); if (df == undefined) { result.message = 'Table not found'; await reply.code(404).send(result); diff --git a/modules/demo/api-server/routes/particles/index.js b/modules/demo/api-server/routes/particles/index.js index fbc663d3d..4ec761cee 100644 --- a/modules/demo/api-server/routes/particles/index.js +++ b/modules/demo/api-server/routes/particles/index.js @@ -24,8 +24,8 @@ const root_schema = require('../../util/schema.js'); module.exports = async function(fastify, opts) { fastify.register(fastifyCors, {origin: '*'}); fastify.register(arrowPlugin); - fastify.decorate('setDataframe', gpu_cache.setDataframe); - fastify.decorate('getDataframe', gpu_cache.getDataframe); + fastify.decorate('cacheObject', gpu_cache.cacheObject); + fastify.decorate('getData', gpu_cache.getData); fastify.decorate('readCSV', gpu_cache.readCSV); const get_schema = { @@ -55,7 +55,7 @@ module.exports = async function(fastify, opts) { const handler = async (request, reply) => { let message = 'Error'; let result = {'params': request.params, success: false, message: message}; - const table = await fastify.getDataframe(request.params.table); + const table = await fastify.getData(request.params.table); if (table == undefined) { result.message = 'Table not found'; await reply.code(404).send(result); @@ -128,7 +128,7 @@ module.exports = async function(fastify, opts) { handler: async (request, reply) => { let message = 'Error'; let result = {'params': request.params, success: false, message: message}; - const table = await fastify.getDataframe(request.params.table); + const table = await fastify.getData(request.params.table); if (table == undefined) { result.message = 'Table not found'; await reply.code(404).send(result); diff --git a/modules/demo/api-server/routes/quadtree/index.js b/modules/demo/api-server/routes/quadtree/index.js index 02298f7fa..1efebe7cd 100644 --- a/modules/demo/api-server/routes/quadtree/index.js +++ b/modules/demo/api-server/routes/quadtree/index.js @@ -25,8 +25,8 @@ const root_schema = require('../../util/schema.js'); module.exports = async function(fastify, opts) { fastify.register(fastifyCors, {origin: '*'}); fastify.register(arrowPlugin); - fastify.decorate('setDataframe', gpu_cache.setDataframe); - fastify.decorate('getDataframe', gpu_cache.getDataframe); + fastify.decorate('cacheObject', gpu_cache.cacheObject); + fastify.decorate('getData', gpu_cache.getData); fastify.decorate('readCSV', gpu_cache.readCSV); const get_schema = { @@ -52,21 +52,27 @@ module.exports = async function(fastify, opts) { handler: async (request, reply) => { let message = 'Error'; let result = {'params': request.params, success: false, message: message}; - const table = await fastify.getDataframe(request.params.table); + const table = await fastify.getData(request.params.table); + if (request.body.xAxisName === undefined || request.body.yAxisName === undefined) { + result.message = 'xAxisName or yAxisName undefined, specify them in POST body.'; + result.code = 400; + await reply.code(result.code).send(result); + return; + } if (table == undefined) { result.message = 'Table not found'; await reply.code(404).send(result); } else { const [xMin, xMax, yMin, yMax] = [ - parseFloat(table.get('x').min()), - parseFloat(table.get('x').max()), - parseFloat(table.get('y').min()), - parseFloat(table.get('y').max()), + parseFloat(table.get(request.body.xAxisName).min()), + parseFloat(table.get(request.body.xAxisName).max()), + parseFloat(table.get(request.body.yAxisName).min()), + parseFloat(table.get(request.body.yAxisName).max()), ]; try { const quadtree = cuspatial.Quadtree.new({ - x: table.get('x'), - y: table.get('y'), + x: table.get(request.body.xAxisName), + y: table.get(request.body.yAxisName), xMin, xMax, yMin, @@ -75,7 +81,7 @@ module.exports = async function(fastify, opts) { maxDepth: 15, minSize: 1e5 }); - const saved = await fastify.setDataframe(request.params.table); + const saved = await fastify.cacheObject(request.params.table); result.message = 'Quadtree created'; result.success = true; result.statusCode = 200; @@ -89,4 +95,33 @@ module.exports = async function(fastify, opts) { } } }); + + fastify.route({ + method: 'POST', + url: '/set_polygons_quadtree', + schema: { + querystring: + {polygon_offset: {type: 'array'}, ring_offset: {type: 'array'}, points: {type: 'array'}} + }, + handler: async (request, reply) => { + let message = 'Error'; + let result = {'params': request.params, success: false, message: message}; + try { + const polygon_offset = Series.new(request.body.polygon_offset); + const ring_offset = Series.new(request.body.ring_offset); + const points = Series.new(request.body.points); + fastify.cacheObject(request.body.name, {polygon_offset, ring_offset, points}); + result.message = 'Set polygon ' + request.body.name; + result.success = true; + result.statusCode = 200; + result.params = request.body; + await reply.code(result.statusCode).send(result); + } catch (e) { + result.message = e; + result.success = false; + result.statusCode = 500; + await reply.code(result.statusCode).send(result); + } + } + }); } diff --git a/modules/demo/api-server/test/plugins/gpu_cache.test.js b/modules/demo/api-server/test/plugins/gpu_cache.test.js index 99ee0ecad..b29acaec9 100644 --- a/modules/demo/api-server/test/plugins/gpu_cache.test.js +++ b/modules/demo/api-server/test/plugins/gpu_cache.test.js @@ -20,9 +20,9 @@ const Support = require('../../plugins/support') const fixtures = require('../fixtures.js'); const gpuCache = require('../../util/gpu_cache.js'); -test('set/getDataframe', async t => { - await gpuCache.setDataframe('bob', 5); - const result = await gpuCache.getDataframe('bob'); +test('set/getData', async t => { + await gpuCache.cacheObject('bob', 5); + const result = await gpuCache.getData('bob'); await gpuCache.clearDataframes(); t.equal(result, 5); }); @@ -53,16 +53,16 @@ test('readCSV', {only: true}, async t => { }); test('listDataframes', async t => { - await gpuCache.setDataframe('bob', 5); - await gpuCache.setDataframe('george', 6); + await gpuCache.cacheObject('bob', 5); + await gpuCache.cacheObject('george', 6); const result = await gpuCache.listDataframes(); await gpuCache.clearDataframes(); t.same(result, ['bob', 'george']); }); test('clearDataframes', async t => { - await gpuCache.setDataframe('bob', 5); - await gpuCache.setDataframe('george', 6); + await gpuCache.cacheObject('bob', 5); + await gpuCache.cacheObject('george', 6); await gpuCache.clearDataframes(); const result = await gpuCache.listDataframes(); t.same(result, []); diff --git a/modules/demo/api-server/test/routes/gpu.test.js b/modules/demo/api-server/test/routes/gpu.test.js index 203b64494..1f3148d4c 100644 --- a/modules/demo/api-server/test/routes/gpu.test.js +++ b/modules/demo/api-server/test/routes/gpu.test.js @@ -30,9 +30,6 @@ test('read_csv', async (t) => { const res = await app.inject( {method: 'POST', url: '/gpu/DataFrame/readCSV', body: {filename: 'csv_base.csv'}}); const release = await app.inject({method: 'POST', url: '/graphology/release'}); - t.equal(res.statusCode, 200); - console.log(res.statusCode); - console.log(res.payload); t.same(JSON.parse(res.payload), { success: true, message: 'CSV file in GPU memory.', diff --git a/modules/demo/api-server/test/routes/quadtree.test.js b/modules/demo/api-server/test/routes/quadtree.test.js index 45bf4aa20..71b95b63e 100644 --- a/modules/demo/api-server/test/routes/quadtree.test.js +++ b/modules/demo/api-server/test/routes/quadtree.test.js @@ -21,17 +21,20 @@ const {tableFromIPC, RecordBatchStreamWriter} = require('apache-arrow'); const {csv_quadtree} = require('../fixtures.js'); const gpu_cache = require('../../util/gpu_cache'); -test('quadtree/create/:table', {only: true}, async (t) => { +test('quadtree/create/:table', async (t) => { const dir = t.testdir(csv_quadtree); const rpath = 'test/routes/' + dir.substring(dir.lastIndexOf('/')); const app = await build(t); gpu_cache._setPathForTesting(rpath); const load = await app.inject( {method: 'POST', url: '/gpu/DataFrame/readCSV', body: {filename: 'csv_quadtree.csv'}}); - const res = await app.inject({method: 'POST', url: '/quadtree/create/csv_quadtree.csv'}); + const res = await app.inject({ + method: 'POST', + url: '/quadtree/create/csv_quadtree.csv', + body: {xAxisName: 'x', yAxisName: 'y'} + }); const release = await app.inject({method: 'POST', url: '/graphology/release'}); const result = JSON.parse(res.payload); - t.same(result.message, 'Quadtree created'); t.same(result, { statusCode: 200, message: 'Quadtree created', @@ -39,3 +42,27 @@ test('quadtree/create/:table', {only: true}, async (t) => { success: true }) }); + +test('quadtree/set_polygons', {only: true}, async (t) => { + const dir = t.testdir(csv_quadtree); + const rpath = 'test/routes/' + dir.substring(dir.lastIndexOf('/')); + const app = await build(t); + gpu_cache._setPathForTesting(rpath); + const load = await app.inject( + {method: 'POST', url: '/gpu/DataFrame/readCSV', body: {filename: 'csv_quadtree.csv'}}); + const res = await app.inject({ + method: 'POST', + url: '/quadtree/set_polygons_quadtree', + body: + {name: 'test', polygon_offset: [0, 1], ring_offset: [0, 4], points: [0, 0, 1, 1, 2, 2, 3, 3]} + }); + const release = await app.inject({method: 'POST', url: '/graphology/release'}); + const result = JSON.parse(res.payload); + t.same(result, { + statusCode: 200, + message: 'Set polygon test', + params: + {name: 'test', polygon_offset: [0, 1], ring_offset: [0, 4], points: [0, 0, 1, 1, 2, 2, 3, 3]}, + success: true + }) +}); diff --git a/modules/demo/api-server/util/gpu_cache.js b/modules/demo/api-server/util/gpu_cache.js index ddef25b00..5678fc1c1 100644 --- a/modules/demo/api-server/util/gpu_cache.js +++ b/modules/demo/api-server/util/gpu_cache.js @@ -40,7 +40,7 @@ function json_key_attributes_to_dataframe(str) { }); const result = new DataFrame(arr); return result; -} +}; function json_aos_to_dataframe(str, columns, dtypes) { let arr = {}; @@ -52,7 +52,7 @@ function json_aos_to_dataframe(str, columns, dtypes) { }); const result = new DataFrame(arr); return result; -} +}; function json_aoa_to_dataframe(str, dtypes) { let arr = {}; @@ -65,79 +65,81 @@ function json_aoa_to_dataframe(str, dtypes) { }); const result = new DataFrame(arr); return result; -} +}; let _publicPath = Path.join(__dirname, '../../public'); +const cacheObject = async (name, data) => { + if (timeout) { clearTimeout(timeout); } + timeout = setTimeout(clearCachedGPUData, 10 * 60 * 1000); + if (datasets === null) { + datasets = {}; + } + datasets[name] = data; +}; + module.exports = { - async setDataframe(name, dataframe) { - if (timeout) { clearTimeout(timeout); } - timeout = setTimeout(clearCachedGPUData, 10 * 60 * 1000); - if (datasets === null) { - datasets = {}; - } - datasets[name] = dataframe; - }, + async cacheObject(name, data) { cacheObject(name, data); }, - async getDataframe(name) { return datasets != null ? datasets[name] : undefined; }, + async getData(name) { return datasets != null ? datasets[name] : undefined; }, async listDataframes() { return datasets != null ? Object.keys(datasets) : []; }, async clearDataframes() { clearCachedGPUData(); clearTimeout(timeout); - datasets = null; + datasets = null; }, - _setPathForTesting(path) { _publicPath = path; }, + _setPathForTesting(path) { _publicPath= path; }, publicPath() { return _publicPath; }, async readLargeGraphDemo(path) { console.log('readLargeGraphDemo'); - const dataset = Series.readText(path, ''); - let split = dataset.split('"options":'); + const dataset= Series.readText(path, ''); + let split = dataset.split('"options":'); if (split.length <= 1) { throw 'Bad readLargeGraphDemo format: options not found.'; }; - const toptions = split.gather([1], false); - let rest = split.gather([0], false); - split = rest.split('"edges":'); + const toptions= split.gather([1], false); + let rest = split.gather([0], false); + split = rest.split('"edges":'); if (split.length <= 1) { throw 'Bad readLargeGraphDemo format: edges not found.'; }; - const tedges = split.gather([1], false); - rest = split.gather([0], false); - split = rest.split('"nodes":'); + const tedges= split.gather([1], false); + rest = split.gather([0], false); + split = rest.split('"nodes":'); if (split.length <= 1) { throw 'Bad readLargeGraphDemo format: nodes not found.'; }; - const tnodes = split.gather([1], false); - const nodes = json_key_attributes_to_dataframe(tnodes); - const edges = json_aos_to_dataframe( + const tnodes= split.gather([1], false); + const nodes= json_key_attributes_to_dataframe(tnodes); + const edges= json_aos_to_dataframe( tedges, ['key', 'source', 'target'], [new Utf8String, new Int64, new Int64]); - let optionsArr = {}; - optionsArr['type'] = Series.new(toptions.getJSONObject('.type')); - optionsArr['multi'] = Series.new(toptions.getJSONObject('.multi')); - optionsArr['allowSelfLoops'] = Series.new(toptions.getJSONObject('.allowSelfLoops')); - const options = new DataFrame(optionsArr); + let optionsArr= {}; + optionsArr['type']= Series.new(toptions.getJSONObject('.type')); + optionsArr['multi']= Series.new(toptions.getJSONObject('.multi')); + optionsArr['allowSelfLoops']= Series.new(toptions.getJSONObject('.allowSelfLoops')); + const options= new DataFrame(optionsArr); return {nodes: nodes, edges: edges, options: options}; }, async readGraphology(path) { console.log('readGraphology'); - const dataset = Series.readText(path, ''); + const dataset= Series.readText(path, ''); if (dataset.length == 0) { throw 'File does not exist or is empty.' } - let split = dataset.split('"tags":'); + let split = dataset.split('"tags":'); if (split.length <= 1) { throw 'Bad graphology format: tags not found.'; } - const ttags = split.gather([1], false); - let rest = split.gather([0], false); - split = rest.split('"clusters":'); + const ttags= split.gather([1], false); + let rest = split.gather([0], false); + split = rest.split('"clusters":'); if (split.length <= 1) { throw 'Bad graphology format: clusters not found.'; } - const tclusters = split.gather([1], false); - rest = split.gather([0], false); - split = rest.split('"edges":'); + const tclusters= split.gather([1], false); + rest = split.gather([0], false); + split = rest.split('"edges":'); if (split.length <= 1) { throw 'Bad graphology format: edges not found.'; } - const tedges = split.gather([1], false); - rest = split.gather([0], false); - split = rest.split('"nodes":'); + const tedges= split.gather([1], false); + rest = split.gather([0], false); + split = rest.split('"nodes":'); if (split.length <= 1) { throw 'Bad graphology format: nodes not found.'; } - const tnodes = split.gather([1], false); - const tags = json_aos_to_dataframe(ttags, ['key', 'image'], [new Utf8String, new Utf8String]); - const clusters = json_aos_to_dataframe( + const tnodes= split.gather([1], false); + const tags = json_aos_to_dataframe(ttags, ['key', 'image'], [new Utf8String, new Utf8String]); + const clusters= json_aos_to_dataframe( tclusters, ['key', 'color', 'clusterLabel'], [new Int64, new Utf8String, new Utf8String]); const nodes = json_aos_to_dataframe(tnodes, ['key', 'label', 'tag', 'URL', 'cluster', 'x', 'y', 'score'], [ @@ -150,12 +152,12 @@ module.exports = { new Float64, new Int32 ]); - const edges = json_aoa_to_dataframe(tedges, [new Utf8String, new Utf8String]); + const edges= json_aoa_to_dataframe(tedges, [new Utf8String, new Utf8String]); return {nodes: nodes, edges: edges, tags: tags, clusters: clusters}; }, async readCSV(options) { - const result = await DataFrame.readCSV(options); + const result= await DataFrame.readCSV(options); return result; } } From 6b2a0827a004c60e5f8c6d5c37b3d9f525164b5c Mon Sep 17 00:00:00 2001 From: "H. Thomson Comer" Date: Wed, 23 Nov 2022 09:40:41 -0600 Subject: [PATCH 25/43] Set poly types. --- modules/demo/api-server/routes/quadtree/index.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/demo/api-server/routes/quadtree/index.js b/modules/demo/api-server/routes/quadtree/index.js index 1efebe7cd..bc7e26a21 100644 --- a/modules/demo/api-server/routes/quadtree/index.js +++ b/modules/demo/api-server/routes/quadtree/index.js @@ -107,9 +107,9 @@ module.exports = async function(fastify, opts) { let message = 'Error'; let result = {'params': request.params, success: false, message: message}; try { - const polygon_offset = Series.new(request.body.polygon_offset); - const ring_offset = Series.new(request.body.ring_offset); - const points = Series.new(request.body.points); + const polygon_offset = Series.new(new Int32Array(request.body.polygon_offset)); + const ring_offset = Series.new(new Int32Array(request.body.ring_offset)); + const points = Series.new(new Float32Array(request.body.points)); fastify.cacheObject(request.body.name, {polygon_offset, ring_offset, points}); result.message = 'Set polygon ' + request.body.name; result.success = true; From 43aef81408084daab4eedc7feb399f8e5d017206 Mon Sep 17 00:00:00 2001 From: "H. Thomson Comer" Date: Wed, 23 Nov 2022 14:09:58 -0600 Subject: [PATCH 26/43] Write quadtree/get_points --- .../demo/api-server/routes/quadtree/index.js | 63 ++++++++++++++++--- .../api-server/test/routes/quadtree.test.js | 39 +++++++++++- modules/demo/api-server/util/gpu_cache.js | 1 + 3 files changed, 92 insertions(+), 11 deletions(-) diff --git a/modules/demo/api-server/routes/quadtree/index.js b/modules/demo/api-server/routes/quadtree/index.js index bc7e26a21..942dc2530 100644 --- a/modules/demo/api-server/routes/quadtree/index.js +++ b/modules/demo/api-server/routes/quadtree/index.js @@ -27,6 +27,7 @@ module.exports = async function(fastify, opts) { fastify.register(arrowPlugin); fastify.decorate('cacheObject', gpu_cache.cacheObject); fastify.decorate('getData', gpu_cache.getData); + fastify.decorate('listDataframes', gpu_cache.listDataframes); fastify.decorate('readCSV', gpu_cache.readCSV); const get_schema = { @@ -70,7 +71,7 @@ module.exports = async function(fastify, opts) { parseFloat(table.get(request.body.yAxisName).max()), ]; try { - const quadtree = cuspatial.Quadtree.new({ + const quadtree = cuspatial.Quadtree.new({ x: table.get(request.body.xAxisName), y: table.get(request.body.yAxisName), xMin, @@ -81,10 +82,12 @@ module.exports = async function(fastify, opts) { maxDepth: 15, minSize: 1e5 }); - const saved = await fastify.cacheObject(request.params.table); - result.message = 'Quadtree created'; - result.success = true; - result.statusCode = 200; + const quadtree_name = request.params.table + '_quadtree'; + request.params.quadtree = quadtree_name + const saved = await fastify.cacheObject(quadtree_name, quadtree); + result.message = 'Quadtree created'; + result.success = true; + result.statusCode = 200; await reply.code(result.statusCode).send(result); } catch (e) { result.message = e; @@ -100,8 +103,12 @@ module.exports = async function(fastify, opts) { method: 'POST', url: '/set_polygons_quadtree', schema: { - querystring: - {polygon_offset: {type: 'array'}, ring_offset: {type: 'array'}, points: {type: 'array'}} + querystring: { + name: {type: 'string'}, + polygon_offset: {type: 'array'}, + ring_offset: {type: 'array'}, + points: {type: 'array'} + } }, handler: async (request, reply) => { let message = 'Error'; @@ -109,8 +116,9 @@ module.exports = async function(fastify, opts) { try { const polygon_offset = Series.new(new Int32Array(request.body.polygon_offset)); const ring_offset = Series.new(new Int32Array(request.body.ring_offset)); - const points = Series.new(new Float32Array(request.body.points)); - fastify.cacheObject(request.body.name, {polygon_offset, ring_offset, points}); + const points = Series.new(new Float64Array(request.body.points)); + const cached = + await fastify.cacheObject(request.body.name, {polygon_offset, ring_offset, points}); result.message = 'Set polygon ' + request.body.name; result.success = true; result.statusCode = 200; @@ -124,4 +132,41 @@ module.exports = async function(fastify, opts) { } } }); + + fastify.route({ + method: 'GET', + url: '/get_points/:quadtree/:polygon', + schema: {querystring: {quadtree: {type: 'string'}, polygon: {type: 'string'}}}, + handler: async (request, reply) => { + let message = 'Error'; + let result = {'params': request.params, success: false, message: message}; + try { + const quadtree = await fastify.getData(request.params.quadtree); + const {polygon_offset, ring_offset, points} = await fastify.getData(request.params.polygon); + const data = await fastify.listDataframes(); + const pts = cuspatial.makePoints( + points.gather(Series.sequence({size: points.length, step: 2, init: 0})), + points.gather(Series.sequence({size: points.length, step: 2, init: 1}))); + const polylines = cuspatial.makePolylines(pts, ring_offset); + const polygons = cuspatial.makePolygons(polylines, polygon_offset); + const polyPointPairs = quadtree.pointInPolygon(polygons); + const resultPoints = quadtree.points.gather(polyPointPairs.get('point_index')); + const numPoints = resultPoints.get('x').length + let result_col = Series.sequence({size: numPoints * 2, step: 0, init: 0}); + result_col = result_col.scatter(resultPoints.get('x'), + Series.sequence({size: numPoints, step: 2, init: 0})); + result_col = result_col.scatter(resultPoints.get('y'), + Series.sequence({size: numPoints, step: 2, init: 1})); + result = new DataFrame({'points_in_polygon': result_col}) + const writer = RecordBatchStreamWriter.writeAll(result.toArrow()); + writer.close(); + await reply.code(200).send(writer.toNodeStream()); + } catch (e) { + result.message = e; + result.success = false; + result.statusCode = 500; + await reply.code(result.statusCode).send(result); + } + } + }); } diff --git a/modules/demo/api-server/test/routes/quadtree.test.js b/modules/demo/api-server/test/routes/quadtree.test.js index 71b95b63e..7c9b8444a 100644 --- a/modules/demo/api-server/test/routes/quadtree.test.js +++ b/modules/demo/api-server/test/routes/quadtree.test.js @@ -38,12 +38,12 @@ test('quadtree/create/:table', async (t) => { t.same(result, { statusCode: 200, message: 'Quadtree created', - params: {table: 'csv_quadtree.csv'}, + params: {table: 'csv_quadtree.csv', quadtree: 'csv_quadtree.csv_quadtree'}, success: true }) }); -test('quadtree/set_polygons', {only: true}, async (t) => { +test('quadtree/set_polygons', async (t) => { const dir = t.testdir(csv_quadtree); const rpath = 'test/routes/' + dir.substring(dir.lastIndexOf('/')); const app = await build(t); @@ -66,3 +66,38 @@ test('quadtree/set_polygons', {only: true}, async (t) => { success: true }) }); + +test('quadtree/get_points', {only: true}, async (t) => { + const dir = t.testdir(csv_quadtree); + const rpath = 'test/routes/' + dir.substring(dir.lastIndexOf('/')); + const app = await build(t); + gpu_cache._setPathForTesting(rpath); + const load = await app.inject( + {method: 'POST', url: '/gpu/DataFrame/readCSV', body: {filename: 'csv_quadtree.csv'}}); + const create = await app.inject({ + method: 'POST', + url: '/quadtree/create/csv_quadtree.csv', + body: {xAxisName: 'x', yAxisName: 'y'} + }); + const quadtree_name = JSON.parse(create.payload).params.quadtree; + const set_poly = await app.inject({ + method: 'POST', + url: '/quadtree/set_polygons_quadtree', + body: { + name: 'test', + polygon_offset: [0, 1], + ring_offset: [0, 4], + points: [-2, -2, -2, 2, 2, 2, 2, -2] + } + }); + const polygons_name = JSON.parse(set_poly.payload).params.name; + const res = await app.inject({ + method: 'GET', + url: 'quadtree/get_points/' + quadtree_name + '/' + polygons_name, + }) + const release = await app.inject({method: 'POST', url: '/graphology/release'}); + const table = tableFromIPC(res.rawPayload); + const got = table.getChild('points_in_polygon').toArray(); + const expected = [1, -1, -1, 1, 0, 0]; + t.same(expected, got); +}); diff --git a/modules/demo/api-server/util/gpu_cache.js b/modules/demo/api-server/util/gpu_cache.js index 5678fc1c1..c376b5f0e 100644 --- a/modules/demo/api-server/util/gpu_cache.js +++ b/modules/demo/api-server/util/gpu_cache.js @@ -82,6 +82,7 @@ module.exports = { async cacheObject(name, data) { cacheObject(name, data); }, async getData(name) { return datasets != null ? datasets[name] : undefined; }, + getDataSync(name) { return datasets != null ? datasets[name] : undefined; }, async listDataframes() { return datasets != null ? Object.keys(datasets) : []; }, From 63aee4dcd68a528d29f1506d531d7eab4d833691 Mon Sep 17 00:00:00 2001 From: "H. Thomson Comer" Date: Wed, 30 Nov 2022 16:56:16 -0600 Subject: [PATCH 27/43] Update modules/demo/api-server/package.json --- modules/demo/api-server/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/demo/api-server/package.json b/modules/demo/api-server/package.json index e52600c1d..830cef317 100644 --- a/modules/demo/api-server/package.json +++ b/modules/demo/api-server/package.json @@ -13,7 +13,7 @@ "scripts": { "test": "tap \"test/**/*.test.js\"", "start": "fastify start --ignore-watch -l info -P -p 3010 app.js", - "dev": "fastify start --ignore-watch -w -l info -P -p 3010 app.js" + "dev": "fastify start --ignore-watch -l info -P -p 3010 app.js" }, "keywords": [ "rapids.ai", From 6228a6152d17b674c4c0ba2fae02a1c14078e76a Mon Sep 17 00:00:00 2001 From: "H. Thomson Comer" Date: Wed, 30 Nov 2022 16:59:10 -0600 Subject: [PATCH 28/43] Update package.json --- package.json | 1 - 1 file changed, 1 deletion(-) diff --git a/package.json b/package.json index 591fac172..7f28d1ad1 100644 --- a/package.json +++ b/package.json @@ -95,7 +95,6 @@ "modules/demo/api-server", "modules/demo/viz-app", "modules/demo/sql/*", - "modules/external/attractor-viz-node-rapids/*" ], "dependencies": { "@typescript-eslint/eslint-plugin": "5.30.0", From bc7f5bd52c70ae7a4265fe29dbe0d775a9bff46e Mon Sep 17 00:00:00 2001 From: Thomson Comer Date: Wed, 30 Nov 2022 17:04:09 -0600 Subject: [PATCH 29/43] Drop repeat --- modules/cudf/src/column.cpp | 2 -- modules/cudf/src/column.ts | 10 ---------- modules/cudf/src/node_cudf/column.hpp | 5 ----- modules/cudf/src/series.ts | 21 --------------------- 4 files changed, 38 deletions(-) diff --git a/modules/cudf/src/column.cpp b/modules/cudf/src/column.cpp index d07304eef..e1005631d 100644 --- a/modules/cudf/src/column.cpp +++ b/modules/cudf/src/column.cpp @@ -49,8 +49,6 @@ Napi::Function Column::Init(Napi::Env const& env, Napi::Object exports) { // column/filling.cpp InstanceMethod<&Column::fill>("fill"), InstanceMethod<&Column::fill_in_place>("fillInPlace"), - // column/repeat.cpp - InstanceMethod<&Column::repeat>("repeat"), // column/binaryop.cpp InstanceMethod<&Column::add>("add"), InstanceMethod<&Column::sub>("sub"), diff --git a/modules/cudf/src/column.ts b/modules/cudf/src/column.ts index 64e32c50a..60ecdb715 100644 --- a/modules/cudf/src/column.ts +++ b/modules/cudf/src/column.ts @@ -249,16 +249,6 @@ export interface Column { */ fill(value: Scalar, begin?: number, end?: number, memoryResource?: MemoryResource): Column; - /** - * Repeats the values of this n times. - * - * @param repeats The number of times to repeat the column. - * @param memoryResource The optional MemoryResource used to allocate the result Column's device - * memory. - */ - repeat(value: Scalar, begin?: number, end?: number, memoryResource?: MemoryResource): - Column; - /** * Fills a range of elements in-place in a column with a scalar value. * diff --git a/modules/cudf/src/node_cudf/column.hpp b/modules/cudf/src/node_cudf/column.hpp index 2d598b76c..f277ec846 100644 --- a/modules/cudf/src/node_cudf/column.hpp +++ b/modules/cudf/src/node_cudf/column.hpp @@ -717,11 +717,6 @@ struct Column : public EnvLocalObjectWrap { cudf::scalar const& value, rmm::mr::device_memory_resource* mr = rmm::mr::get_current_device_resource()); - // column/filling/repeat.cpp - Column::wrapper_t repeat( - cudf::size_type repeats, - rmm::mr::device_memory_resource* mr = rmm::mr::get_current_device_resource()); - // column/replace.cpp Column::wrapper_t replace_nulls( cudf::column_view const& replacement, diff --git a/modules/cudf/src/series.ts b/modules/cudf/src/series.ts index 6820cbfb3..cc626093b 100644 --- a/modules/cudf/src/series.ts +++ b/modules/cudf/src/series.ts @@ -811,27 +811,6 @@ export class AbstractSeries { this._col.fill(new Scalar({type: this.type, value}), begin, end, memoryResource)); } - /** - * Repeats a Series n times, returning a new Series. - * - * @param repeats The number of times to repeat this. - * - * @example - * ```typescript - * import {Series} from '@rapidsai/cudf'; - * - * // Float64Series - * Series.new([1, 2, 3]).repeat(2) // [1, 2, 3, 1, 2, 3] - * // StringSeries - * Series.new(["foo", "bar", "test"]).repeat(2) // ["foo", "bar", "test", "foo", "bar", "test"] - * // Bool8Series - * Series.new([true, true, true]).repeat(2) // [true, false, false, true, false, false] - * ``` - */ - repeat(repeats: T['scalarType'], memoryResource?: MemoryResource): Series { - return this.__construct(this._col.repeat(repeats, memoryResource)); - } - /** * Fills a range of elements in-place in a column with a scalar value. * From fe8679fea2f76317e40a313dc8f0d970d283e1cf Mon Sep 17 00:00:00 2001 From: "H. Thomson Comer" Date: Wed, 30 Nov 2022 17:07:50 -0600 Subject: [PATCH 30/43] Really important comma needed to go. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 7f28d1ad1..cdc790d78 100644 --- a/package.json +++ b/package.json @@ -94,7 +94,7 @@ "modules/demo/client-server", "modules/demo/api-server", "modules/demo/viz-app", - "modules/demo/sql/*", + "modules/demo/sql/*" ], "dependencies": { "@typescript-eslint/eslint-plugin": "5.30.0", From 1be7b71a88449c444c71420adf9dd27a90ea95c2 Mon Sep 17 00:00:00 2001 From: "H. Thomson Comer" Date: Wed, 4 Jan 2023 12:49:17 -0600 Subject: [PATCH 31/43] Update modules/demo/api-server/routes/gpu/index.js --- modules/demo/api-server/routes/gpu/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/demo/api-server/routes/gpu/index.js b/modules/demo/api-server/routes/gpu/index.js index a0e6f3df0..342180f87 100644 --- a/modules/demo/api-server/routes/gpu/index.js +++ b/modules/demo/api-server/routes/gpu/index.js @@ -71,7 +71,7 @@ module.exports = async function(fastify, opts) { sourceType: 'files', sources: [path], }); - const name = request.body.filename; // request.body.replace('/\//g', '_'); + const name = request.body.filename; await fastify.cacheObject(name, cacheObject); result.success = true; result.message = 'CSV file in GPU memory.'; From f6874209537672e25edfaffc829fd92bbca70697 Mon Sep 17 00:00:00 2001 From: "H. Thomson Comer" Date: Wed, 4 Jan 2023 12:58:53 -0600 Subject: [PATCH 32/43] Update modules/demo/api-server/test/routes/graphology.test.js --- modules/demo/api-server/test/routes/graphology.test.js | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/demo/api-server/test/routes/graphology.test.js b/modules/demo/api-server/test/routes/graphology.test.js index 648c2252d..2a7a5a091 100644 --- a/modules/demo/api-server/test/routes/graphology.test.js +++ b/modules/demo/api-server/test/routes/graphology.test.js @@ -136,7 +136,6 @@ test('read_json file good', async (t) => { const res = await app.inject({method: 'POST', url: '/graphology/read_json?filename=' + rpath}); const release = await app.inject({method: 'POST', url: '/graphology/release'}); const payload = JSON.parse(res.payload); - console.log(payload); t.equal(payload.message, 'File read onto GPU.'); t.equal(payload.success, true); }); From 5a5efb540dd9cb55939dd742a277b5e50036bea8 Mon Sep 17 00:00:00 2001 From: "H. Thomson Comer" Date: Wed, 4 Jan 2023 13:03:52 -0600 Subject: [PATCH 33/43] Update modules/demo/api-server/test/routes/graphology.test.js --- modules/demo/api-server/test/routes/graphology.test.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/demo/api-server/test/routes/graphology.test.js b/modules/demo/api-server/test/routes/graphology.test.js index 2a7a5a091..d39500a65 100644 --- a/modules/demo/api-server/test/routes/graphology.test.js +++ b/modules/demo/api-server/test/routes/graphology.test.js @@ -128,7 +128,8 @@ test('read_json incorrect format', async (t) => { test('read_json file good', async (t) => { const dir = t.testdir(json_good); const rpath = '../../test/routes/' + dir.substring(dir.lastIndexOf('/')) + '/json_good.txt'; - /* + /* This comment is left for working out mocking with tap in fastify + see: https://github.com/tapjs/node-tap/issues/846 const build = t.mock('../../routes/graphology/index.js', {'../../util/gpu_cache.js': {publicPath: () => rpath}}); */ From 2d3ccb23a42a935e2a36db0579d981b39443f15d Mon Sep 17 00:00:00 2001 From: "H. Thomson Comer" Date: Tue, 3 Jan 2023 13:01:55 -0600 Subject: [PATCH 34/43] For some reason the route path changed. Upstream changes? --- modules/demo/api-server/util/gpu_cache.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/demo/api-server/util/gpu_cache.js b/modules/demo/api-server/util/gpu_cache.js index c376b5f0e..bcc2ff1c8 100644 --- a/modules/demo/api-server/util/gpu_cache.js +++ b/modules/demo/api-server/util/gpu_cache.js @@ -67,7 +67,7 @@ function json_aoa_to_dataframe(str, dtypes) { return result; }; -let _publicPath = Path.join(__dirname, '../../public'); +let _publicPath = Path.join(__dirname, '../public'); const cacheObject = async (name, data) => { if (timeout) { clearTimeout(timeout); } From 7ebef5948ea21144968393dc3f5858b6853279bd Mon Sep 17 00:00:00 2001 From: "H. Thomson Comer" Date: Tue, 3 Jan 2023 13:34:39 -0600 Subject: [PATCH 35/43] Trying to figure out issue with eslint improperly formatting in here. --- modules/demo/api-server/util/gpu_cache.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/demo/api-server/util/gpu_cache.js b/modules/demo/api-server/util/gpu_cache.js index bcc2ff1c8..d64bd612d 100644 --- a/modules/demo/api-server/util/gpu_cache.js +++ b/modules/demo/api-server/util/gpu_cache.js @@ -97,9 +97,9 @@ module.exports = { async readLargeGraphDemo(path) { console.log('readLargeGraphDemo'); - const dataset= Series.readText(path, ''); - let split = dataset.split('"options":'); - if (split.length <= 1) { throw 'Bad readLargeGraphDemo format: options not found.'; }; + const dataset = Series.readText(path, ''); + let split = dataset.split('"options":'); + if (split.length <= 1) { throw 'Bad readLargeGraphDemo format: options not found.'; } const toptions= split.gather([1], false); let rest = split.gather([0], false); split = rest.split('"edges":'); From ccb42f8993b22bf6eca62fa778d1681f32a41eef Mon Sep 17 00:00:00 2001 From: "H. Thomson Comer" Date: Tue, 3 Jan 2023 14:52:13 -0600 Subject: [PATCH 36/43] Reorder exports to make clang-format happy. --- modules/demo/api-server/util/gpu_cache.js | 92 +++++++++++------------ 1 file changed, 46 insertions(+), 46 deletions(-) diff --git a/modules/demo/api-server/util/gpu_cache.js b/modules/demo/api-server/util/gpu_cache.js index d64bd612d..4c5959dfb 100644 --- a/modules/demo/api-server/util/gpu_cache.js +++ b/modules/demo/api-server/util/gpu_cache.js @@ -79,68 +79,52 @@ const cacheObject = async (name, data) => { }; module.exports = { - async cacheObject(name, data) { cacheObject(name, data); }, - - async getData(name) { return datasets != null ? datasets[name] : undefined; }, - getDataSync(name) { return datasets != null ? datasets[name] : undefined; }, - - async listDataframes() { return datasets != null ? Object.keys(datasets) : []; }, - - async clearDataframes() { - clearCachedGPUData(); - clearTimeout(timeout); - datasets = null; - }, - - _setPathForTesting(path) { _publicPath= path; }, - publicPath() { return _publicPath; }, - async readLargeGraphDemo(path) { console.log('readLargeGraphDemo'); const dataset = Series.readText(path, ''); let split = dataset.split('"options":'); if (split.length <= 1) { throw 'Bad readLargeGraphDemo format: options not found.'; } - const toptions= split.gather([1], false); - let rest = split.gather([0], false); - split = rest.split('"edges":'); + const toptions = split.gather([1], false); + let rest = split.gather([0], false); + split = rest.split('"edges":'); if (split.length <= 1) { throw 'Bad readLargeGraphDemo format: edges not found.'; }; - const tedges= split.gather([1], false); - rest = split.gather([0], false); - split = rest.split('"nodes":'); + const tedges = split.gather([1], false); + rest = split.gather([0], false); + split = rest.split('"nodes":'); if (split.length <= 1) { throw 'Bad readLargeGraphDemo format: nodes not found.'; }; - const tnodes= split.gather([1], false); - const nodes= json_key_attributes_to_dataframe(tnodes); - const edges= json_aos_to_dataframe( + const tnodes = split.gather([1], false); + const nodes = json_key_attributes_to_dataframe(tnodes); + const edges = json_aos_to_dataframe( tedges, ['key', 'source', 'target'], [new Utf8String, new Int64, new Int64]); - let optionsArr= {}; - optionsArr['type']= Series.new(toptions.getJSONObject('.type')); - optionsArr['multi']= Series.new(toptions.getJSONObject('.multi')); - optionsArr['allowSelfLoops']= Series.new(toptions.getJSONObject('.allowSelfLoops')); - const options= new DataFrame(optionsArr); + let optionsArr = {}; + optionsArr['type'] = Series.new(toptions.getJSONObject('.type')); + optionsArr['multi'] = Series.new(toptions.getJSONObject('.multi')); + optionsArr['allowSelfLoops'] = Series.new(toptions.getJSONObject('.allowSelfLoops')); + const options = new DataFrame(optionsArr); return {nodes: nodes, edges: edges, options: options}; }, async readGraphology(path) { console.log('readGraphology'); - const dataset= Series.readText(path, ''); + const dataset = Series.readText(path, ''); if (dataset.length == 0) { throw 'File does not exist or is empty.' } - let split = dataset.split('"tags":'); + let split = dataset.split('"tags":'); if (split.length <= 1) { throw 'Bad graphology format: tags not found.'; } - const ttags= split.gather([1], false); - let rest = split.gather([0], false); - split = rest.split('"clusters":'); + const ttags = split.gather([1], false); + let rest = split.gather([0], false); + split = rest.split('"clusters":'); if (split.length <= 1) { throw 'Bad graphology format: clusters not found.'; } - const tclusters= split.gather([1], false); - rest = split.gather([0], false); - split = rest.split('"edges":'); + const tclusters = split.gather([1], false); + rest = split.gather([0], false); + split = rest.split('"edges":'); if (split.length <= 1) { throw 'Bad graphology format: edges not found.'; } - const tedges= split.gather([1], false); - rest = split.gather([0], false); - split = rest.split('"nodes":'); + const tedges = split.gather([1], false); + rest = split.gather([0], false); + split = rest.split('"nodes":'); if (split.length <= 1) { throw 'Bad graphology format: nodes not found.'; } - const tnodes= split.gather([1], false); - const tags = json_aos_to_dataframe(ttags, ['key', 'image'], [new Utf8String, new Utf8String]); - const clusters= json_aos_to_dataframe( + const tnodes = split.gather([1], false); + const tags = json_aos_to_dataframe(ttags, ['key', 'image'], [new Utf8String, new Utf8String]); + const clusters = json_aos_to_dataframe( tclusters, ['key', 'color', 'clusterLabel'], [new Int64, new Utf8String, new Utf8String]); const nodes = json_aos_to_dataframe(tnodes, ['key', 'label', 'tag', 'URL', 'cluster', 'x', 'y', 'score'], [ @@ -153,12 +137,28 @@ module.exports = { new Float64, new Int32 ]); - const edges= json_aoa_to_dataframe(tedges, [new Utf8String, new Utf8String]); + const edges = json_aoa_to_dataframe(tedges, [new Utf8String, new Utf8String]); return {nodes: nodes, edges: edges, tags: tags, clusters: clusters}; }, + async cacheObject(name, data) { cacheObject(name, data); }, + + async getData(name) { return datasets != null ? datasets[name] : undefined; }, + getDataSync(name) { return datasets != null ? datasets[name] : undefined; }, + + async listDataframes() { return datasets != null ? Object.keys(datasets) : []; }, + + async clearDataframes() { + clearCachedGPUData(); + clearTimeout(timeout); + datasets = null; + }, + + _setPathForTesting(path) { _publicPath = path; }, + publicPath() { return _publicPath; }, + async readCSV(options) { - const result= await DataFrame.readCSV(options); + const result = await DataFrame.readCSV(options); return result; } } From a537ad2900b040dc2ba51914c1a99d93d11a76a3 Mon Sep 17 00:00:00 2001 From: "H. Thomson Comer" Date: Wed, 4 Jan 2023 11:05:33 -0600 Subject: [PATCH 37/43] Writing docs for api-server --- .../api-server/routes/graphology/index.js | 36 +++++++ .../demo/api-server/routes/particles/index.js | 26 +++-- .../demo/api-server/routes/quadtree/index.js | 101 ++++++++++++++++++ 3 files changed, 149 insertions(+), 14 deletions(-) diff --git a/modules/demo/api-server/routes/graphology/index.js b/modules/demo/api-server/routes/graphology/index.js index bffcfcea9..3c5ebf77d 100644 --- a/modules/demo/api-server/routes/graphology/index.js +++ b/modules/demo/api-server/routes/graphology/index.js @@ -109,6 +109,11 @@ module.exports = async function(fastify, opts) { } }, handler: async (request, reply) => { + /** + * /graphology/read_json reads a graphology formatted json file from + * public storage, storing it in the `nodes`, `edges`, `clusters`, and + * `tags` DataFrames. + */ const query = request.query; let result = { 'params': JSON.stringify(query), @@ -159,6 +164,10 @@ module.exports = async function(fastify, opts) { method: 'GET', url: '/list_tables', handler: async (request, reply) => { + /** + * /graphology/list_tables returns a list of DataFrames stored in + * the GPU cache. + */ let message = 'Error'; let result = {success: false, message: message}; const list = await fastify.listDataframes(); @@ -171,6 +180,10 @@ module.exports = async function(fastify, opts) { url: '/get_column/:table/:column', schema: {querystring: {table: {type: 'string'}, 'column': {type: 'string'}}}, handler: async (request, reply) => { + /** + * /graphology/get_column/:table/:column returns a column of a DataFrame + * in the GPU cache as an Arrow Table. + */ let message = 'Error'; let result = {'params': JSON.stringify(request.params), success: false, message: message}; const table = await fastify.getData(request.params.table); @@ -206,6 +219,10 @@ module.exports = async function(fastify, opts) { url: '/get_table/:table', schema: {querystring: {table: {type: 'string'}}}, handler: async (request, reply) => { + /** + * /graphology/get_table/:table returns a DataFrame from the GPU cache + * as an Arrow Table. + */ let message = 'Error'; let result = {'params': JSON.stringify(request.params), success: false, message: message}; const table = await fastify.getData(request.params.table); @@ -223,6 +240,10 @@ module.exports = async function(fastify, opts) { method: 'GET', url: '/nodes/bounds', handler: async (request, reply) => { + /** + * /graphology/nodes/bounds returns the min/max of the x and y columns + * of the `nodes` DataFrame. + */ let message = 'Error'; let result = {success: false, message: message}; const df = await fastify.getData('nodes'); @@ -248,6 +269,10 @@ module.exports = async function(fastify, opts) { method: 'GET', url: '/nodes', handler: async (request, reply) => { + /** + * /graphology/nodes returns the `nodes` DataFrame, tiled into a single column + * with offset x,y,scale,color values. + */ let message = 'Error'; let result = {success: false, message: message}; const df = await fastify.getData('nodes'); @@ -287,6 +312,10 @@ module.exports = async function(fastify, opts) { method: 'GET', url: '/edges', handler: async (request, reply) => { + /** + * /graphology/edges returns the edges table, tiled into a single column + * of x,y,size,color offset values. + */ let message = 'Error'; let result = {success: false, message: message}; /** @type DataFrame<{x: Float32, y: Float32}> */ @@ -374,6 +403,13 @@ module.exports = async function(fastify, opts) { method: 'POST', url: '/release', handler: async (request, reply) => { + /** + * /graphology/release clears the dataframes from memory. + * This is useful for testing, but should not be used in production. + * In production, the dataframes should be cached in memory and reused. + * This solution allows unit tests to pass without timing out, as the + * cached GPU objects are not cleared between tests. + */ await fastify.clearDataFrames(); await reply.code(200).send({message: 'OK'}) } diff --git a/modules/demo/api-server/routes/particles/index.js b/modules/demo/api-server/routes/particles/index.js index 4ec761cee..fcf0af077 100644 --- a/modules/demo/api-server/routes/particles/index.js +++ b/modules/demo/api-server/routes/particles/index.js @@ -53,6 +53,18 @@ module.exports = async function(fastify, opts) { } const handler = async (request, reply) => { + /** + * /particles/get_shader_column/:table/:xmin/:xmax/:ymin/:ymax returns a bounds-limited + * set of interleaved longitude/latitude pairs as an Arrow Table containing a single column. + * The column is a Float32Array of length 2 * N, where N is the number of points. + * :table is the name of the table to query. + * (optional) :xmin, :xmax, :ymin, :ymax are the bounds to limit the query to. + * If no bounds are provided, the entire table is returned. + * + * The returned Arrow Table is a single column of Float32 values, where the first + * two values are the first point's longitude and latitude, the next two values are + * the second point's longitude and latitude, and so on. + */ let message = 'Error'; let result = {'params': request.params, success: false, message: message}; const table = await fastify.getData(request.params.table); @@ -121,18 +133,4 @@ module.exports = async function(fastify, opts) { schema: {querystring: {table: {type: 'string'}}}, handler: handler }); - fastify.route({ - method: 'POST', - url: '/quadtree/create/:table', - schema: {querystring: {table: {type: 'string'}}}, - handler: async (request, reply) => { - let message = 'Error'; - let result = {'params': request.params, success: false, message: message}; - const table = await fastify.getData(request.params.table); - if (table == undefined) { - result.message = 'Table not found'; - await reply.code(404).send(result); - } - } - }); } diff --git a/modules/demo/api-server/routes/quadtree/index.js b/modules/demo/api-server/routes/quadtree/index.js index 942dc2530..117cadbc6 100644 --- a/modules/demo/api-server/routes/quadtree/index.js +++ b/modules/demo/api-server/routes/quadtree/index.js @@ -51,6 +51,36 @@ module.exports = async function(fastify, opts) { url: '/create/:table', schema: {querystring: {table: {type: 'string'}}}, handler: async (request, reply) => { + /** + * @api {post} /quadtree/create/:table Create Quadtree + * @apiName CreateQuadtree + * @apiGroup Quadtree + * @apiDescription Create a quadtree from a table + * @apiParam {String} table Table name + * @apiParam {String} xAxisName Column name for x-axis + * @apiParam {String} yAxisName Column name for y-axis + * @apiParamExample {json} Request-Example: + * { + * "xAxisName": "x", + * "yAxisName": "y" + * } + * @apiSuccessExample {json} Success-Response: + * { + * "params": { + * "table": "test" + * }, + * "success": true, + * "message": "Quadtree created" + * } + * @apiErrorExample {json} Error-Response: + * { + * "params": { + * "table": "test" + * }, + * "success": false, + * "message": "Error" + * } + */ let message = 'Error'; let result = {'params': request.params, success: false, message: message}; const table = await fastify.getData(request.params.table); @@ -111,6 +141,45 @@ module.exports = async function(fastify, opts) { } }, handler: async (request, reply) => { + /** + * @api {post} /quadtree/set_polygons_quadtree Set Polygons Quadtree + * @apiName SetPolygonsQuadtree + * @apiGroup Quadtree + * @apiDescription Set polygons for quadtree + * @apiParam {String} name Name of quadtree + * @apiParam {Array} polygon_offset Array of polygon offsets + * @apiParam {Array} ring_offset Array of ring offsets + * @apiParam {Array} points Array of points + * @apiParamExample {json} Request-Example: + * { + * "name": "test_quadtree", + * "polygon_offset": [0, 4, 8], + * "ring_offset": [0, 4, 8, 12], + * "points": [0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 0, 1, 1, 0, 1] + * } + * @apiSuccessExample {json} Success-Response: + * { + * "params": { + * "name": "test_quadtree", + * "polygon_offset": [0, 4, 8], + * "ring_offset": [0, 4, 8, 12], + * "points": [0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 0, 1, 1, 0, 1] + * }, + * "success": true, + * "message": "Set polygon test_quadtree" + * } + * @apiErrorExample {json} Error-Response: + * { + * "params": { + * "name": "test_quadtree", + * "polygon_offset": [0, 4, 8], + * "ring_offset": [0, 4, 8, 12], + * "points": [0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 0, 1, 1, 0, 1] + * }, + * "success": false, + * "message": "Error" + * } + */ let message = 'Error'; let result = {'params': request.params, success: false, message: message}; try { @@ -138,6 +207,38 @@ module.exports = async function(fastify, opts) { url: '/get_points/:quadtree/:polygon', schema: {querystring: {quadtree: {type: 'string'}, polygon: {type: 'string'}}}, handler: async (request, reply) => { + /** + * @api {get} /quadtree/get_points/:quadtree/:polygon Get Points + * @apiName GetPoints + * @apiGroup Quadtree + * @apiDescription This API returns uses the quadtree to return only the points that are in + * the polygon. + * @apiParam {String} quadtree Name of quadtree created with /quadtree/create/:table + * @apiParam {String} polygon Name of polygon created with /quadtree/set_polygons_quadtree + * @apiParamExample {json} Request-Example: + * { + * "quadtree": "test_quadtree", + * "polygon": "test_polygon" + * } + * @apiSuccessExample {json} Success-Response: + * { + * "params": { + * "quadtree": "test_quadtree", + * "polygon": "test_polygon" + * }, + * "success": true, + * "message": "Get points from test_quadtree" + * } + * @apiErrorExample {json} Error-Response: + * { + * "params": { + * "quadtree": "test_quadtree", + * "polygon": "test_polygon" + * }, + * "success": false, + * "message": "Error" + * } + */ let message = 'Error'; let result = {'params': request.params, success: false, message: message}; try { From 695832b1b7b35e75de8b4b7712b53be05ff0b530 Mon Sep 17 00:00:00 2001 From: "H. Thomson Comer" Date: Wed, 4 Jan 2023 15:04:44 -0600 Subject: [PATCH 38/43] Add docs, refactor gpu calls. --- modules/demo/api-server/routes/gpu/index.js | 33 +++++++++++++++++++ .../api-server/routes/graphology/index.js | 31 ----------------- .../demo/api-server/test/routes/gpu.test.js | 24 ++++++++++++++ .../api-server/test/routes/graphology.test.js | 24 +++++++------- .../api-server/test/routes/particles.test.js | 4 +-- .../api-server/test/routes/quadtree.test.js | 6 ++-- 6 files changed, 74 insertions(+), 48 deletions(-) diff --git a/modules/demo/api-server/routes/gpu/index.js b/modules/demo/api-server/routes/gpu/index.js index 342180f87..514698f28 100644 --- a/modules/demo/api-server/routes/gpu/index.js +++ b/modules/demo/api-server/routes/gpu/index.js @@ -32,6 +32,8 @@ module.exports = async function(fastify, opts) { fastify.decorate('getData', gpu_cache.getData); fastify.decorate('readCSV', gpu_cache.readCSV); fastify.decorate('publicPath', gpu_cache.publicPath); + fastify.decorate('listDataframes', gpu_cache.listDataframes); + fastify.decorate('clearDataFrames', gpu_cache.clearDataframes); const get_schema = { logLevel: 'debug', @@ -119,4 +121,35 @@ module.exports = async function(fastify, opts) { } } }); + + fastify.route({ + method: 'GET', + url: '/list_tables', + handler: async (request, reply) => { + /** + * /graphology/list_tables returns a list of DataFrames stored in + * the GPU cache. + */ + let message = 'Error'; + let result = {success: false, message: message}; + const list = await fastify.listDataframes(); + return list; + } + }); + + fastify.route({ + method: 'POST', + url: '/release', + handler: async (request, reply) => { + /** + * /graphology/release clears the dataframes from memory. + * This is useful for testing, but should not be used in production. + * In production, the dataframes should be cached in memory and reused. + * This solution allows unit tests to pass without timing out, as the + * cached GPU objects are not cleared between tests. + */ + await fastify.clearDataFrames(); + await reply.code(200).send({message: 'OK'}) + } + }); } diff --git a/modules/demo/api-server/routes/graphology/index.js b/modules/demo/api-server/routes/graphology/index.js index 3c5ebf77d..a7f7a2ee9 100644 --- a/modules/demo/api-server/routes/graphology/index.js +++ b/modules/demo/api-server/routes/graphology/index.js @@ -160,21 +160,6 @@ module.exports = async function(fastify, opts) { } }); - fastify.route({ - method: 'GET', - url: '/list_tables', - handler: async (request, reply) => { - /** - * /graphology/list_tables returns a list of DataFrames stored in - * the GPU cache. - */ - let message = 'Error'; - let result = {success: false, message: message}; - const list = await fastify.listDataframes(); - return list; - } - }); - fastify.route({ method: 'GET', url: '/get_column/:table/:column', @@ -398,20 +383,4 @@ module.exports = async function(fastify, opts) { } } }); - - fastify.route({ - method: 'POST', - url: '/release', - handler: async (request, reply) => { - /** - * /graphology/release clears the dataframes from memory. - * This is useful for testing, but should not be used in production. - * In production, the dataframes should be cached in memory and reused. - * This solution allows unit tests to pass without timing out, as the - * cached GPU objects are not cleared between tests. - */ - await fastify.clearDataFrames(); - await reply.code(200).send({message: 'OK'}) - } - }); } diff --git a/modules/demo/api-server/test/routes/gpu.test.js b/modules/demo/api-server/test/routes/gpu.test.js index 1f3148d4c..591641214 100644 --- a/modules/demo/api-server/test/routes/gpu.test.js +++ b/modules/demo/api-server/test/routes/gpu.test.js @@ -37,3 +37,27 @@ test('read_csv', async (t) => { params: {filename: 'csv_base.csv'} }); }); + +test('list_tables', async (t) => { + const dir = t.testdir(csv_base); + const rpath = 'test/routes/' + dir.substring(dir.lastIndexOf('/')); + const app = await build(t); + gpu_cache._setPathForTesting(rpath); + const res = await app.inject( + {method: 'POST', url: '/gpu/DataFrame/readCSV', body: {filename: 'csv_base.csv'}}); + const tables = await app.inject({method: 'GET', url: '/gpu/list_tables'}); + const release = await app.inject({method: 'POST', url: '/gpu/release'}); + t.same(JSON.parse(tables.payload), ['csv_base.csv']); +}); + +test('release', async (t) => { + const dir = t.testdir(csv_base); + const rpath = 'test/routes/' + dir.substring(dir.lastIndexOf('/')); + const app = await build(t); + gpu_cache._setPathForTesting(rpath); + const res = await app.inject( + {method: 'POST', url: '/gpu/DataFrame/readCSV', body: {filename: 'csv_base.csv'}}); + const release = await app.inject({method: 'POST', url: '/gpu/release'}); + const tables = await app.inject({method: 'GET', url: '/gpu/list_tables'}); + t.same(JSON.parse(tables.payload), []); +}); diff --git a/modules/demo/api-server/test/routes/graphology.test.js b/modules/demo/api-server/test/routes/graphology.test.js index d39500a65..43665fa12 100644 --- a/modules/demo/api-server/test/routes/graphology.test.js +++ b/modules/demo/api-server/test/routes/graphology.test.js @@ -119,7 +119,7 @@ test('read_json incorrect format', async (t) => { const rpath = '../../test/routes/' + dir.substring(dir.lastIndexOf('/')) + '/json_bad.txt'; const app = await build(t); const res = await app.inject({method: 'POST', url: '/graphology/read_json?filename=' + rpath}); - const release = await app.inject({method: 'POST', url: '/graphology/release'}); + const release = await app.inject({method: 'POST', url: '/gpu/release'}); const payload = JSON.parse(res.payload); t.equal(payload.message, 'Bad graphology format: nodes not found.'); t.equal(payload.success, false); @@ -135,7 +135,7 @@ test('read_json file good', async (t) => { */ const app = await build(t); const res = await app.inject({method: 'POST', url: '/graphology/read_json?filename=' + rpath}); - const release = await app.inject({method: 'POST', url: '/graphology/release'}); + const release = await app.inject({method: 'POST', url: '/gpu/release'}); const payload = JSON.parse(res.payload); t.equal(payload.message, 'File read onto GPU.'); t.equal(payload.success, true); @@ -146,8 +146,8 @@ test('list_tables', async (t) => { const rpath = '../../test/routes/' + dir.substring(dir.lastIndexOf('/')) + '/json_good.txt'; const app = await build(t); const load = await app.inject({method: 'POST', url: '/graphology/read_json?filename=' + rpath}); - const res = await app.inject({method: 'GET', url: '/graphology/list_tables'}); - const release = await app.inject({method: 'POST', url: '/graphology/release'}); + const res = await app.inject({method: 'GET', url: '/gpu/list_tables'}); + const release = await app.inject({method: 'POST', url: '/gpu/release'}); const payload = JSON.parse(res.payload); t.ok(payload.includes('nodes')); }); @@ -164,7 +164,7 @@ test('get_table', async (t) => { }); t.same(res.statusCode, 200); const table = tableFromIPC(res.rawPayload); - const release = await app.inject({method: 'POST', url: '/graphology/release'}); + const release = await app.inject({method: 'POST', url: '/gpu/release'}); t.same(table.schema.names, ['key', 'label', 'tag', 'URL', 'cluster', 'x', 'y', 'score']); t.equal(table.numRows, 2); t.equal(table.numCols, 8); @@ -182,7 +182,7 @@ test('get_column', async (t) => { }); t.same(res.statusCode, 200); const table = tableFromIPC(res.rawPayload); - const release = await app.inject({method: 'POST', url: '/graphology/release'}); + const release = await app.inject({method: 'POST', url: '/gpu/release'}); t.same(table.schema.names, ['score']); t.equal(table.numRows, 2); t.equal(table.numCols, 1); @@ -209,7 +209,7 @@ test('nodes', async (t) => { 2, -5.515159729197043e+28 ])) - const release = await app.inject({method: 'POST', url: '/graphology/release'}); + const release = await app.inject({method: 'POST', url: '/gpu/release'}); }); test('nodes/bounds', async (t) => { @@ -229,7 +229,7 @@ test('nodes/bounds', async (t) => { 'ymax': 4.134339332580566, } }); - const release = await app.inject({method: 'POST', url: '/graphology/release'}); + const release = await app.inject({method: 'POST', url: '/gpu/release'}); }); test('nodes then nodes/bounds', async (t) => { @@ -264,7 +264,7 @@ test('nodes then nodes/bounds', async (t) => { 'ymax': 4.134339332580566, } }); - const release = await app.inject({method: 'POST', url: '/graphology/release'}); + const release = await app.inject({method: 'POST', url: '/gpu/release'}); }); test('edges', async (t) => { @@ -277,7 +277,7 @@ test('edges', async (t) => { {method: 'GET', url: '/graphology/edges', header: {'accepts': 'application/octet-stream'}}); t.equal(res.statusCode, 200); const table = tableFromIPC(res.rawPayload); - const release = await app.inject({method: 'POST', url: '/graphology/release'}); + const release = await app.inject({method: 'POST', url: '/gpu/release'}); t.ok(table.getChild('edges')); t.same(table.getChild('edges').toArray(), new Float32Array([ 0.9705526828765869, @@ -304,7 +304,7 @@ test('edges and nodes do not begin with 0', async (t) => { await app.inject({method: 'POST', url: '/graphology/read_large_demo?filename=' + rpath}); const res = await app.inject( {method: 'GET', url: '/graphology/edges', header: {'accepts': 'application/octet-stream'}}); - const release = await app.inject({method: 'POST', url: '/graphology/release'}); + const release = await app.inject({method: 'POST', url: '/gpu/release'}); t.equal(res.statusCode, 200); const table = tableFromIPC(res.rawPayload); t.ok(table.getChild('edges')); @@ -332,7 +332,7 @@ test('edge keys do not match node keys', async (t) => { await app.inject({method: 'POST', url: '/graphology/read_large_demo?filename=' + rpath}); const res = await app.inject( {method: 'GET', url: '/graphology/edges', header: {'accepts': 'application/octet-stream'}}); - const release = await app.inject({method: 'POST', url: '/graphology/release'}); + const release = await app.inject({method: 'POST', url: '/gpu/release'}); t.equal(res.statusCode, 422); t.same(JSON.parse(res.payload), {success: false, message: 'Edge sources do not match node keys', statusCode: 422}); diff --git a/modules/demo/api-server/test/routes/particles.test.js b/modules/demo/api-server/test/routes/particles.test.js index 673b34627..efe1b4c80 100644 --- a/modules/demo/api-server/test/routes/particles.test.js +++ b/modules/demo/api-server/test/routes/particles.test.js @@ -43,7 +43,7 @@ test('get_shader_column/:table', async (t) => { await app.inject({method: 'GET', url: '/particles/get_shader_column/csv_particles.csv'}); const expected = [-105, 40, -106, 41, -107, 42, -108, 43, -109, 44, -110, 45]; const got = tableFromIPC(res.rawPayload).getChild('gpu_buffer').toArray(); - const release = await app.inject({method: 'POST', url: '/graphology/release'}); + const release = await app.inject({method: 'POST', url: '/gpu/release'}); t.same(got, expected); }); @@ -58,6 +58,6 @@ test('get_shader_column/:table/:xmin/:xmax/:ymin/:ymax', async (t) => { {method: 'GET', url: '/particles/get_shader_column/csv_particles.csv/-109/-106/41/44'}); const expected = [-107, 42, -108, 43]; const got = tableFromIPC(res.rawPayload).getChild('gpu_buffer').toArray(); - const release = await app.inject({method: 'POST', url: '/graphology/release'}); + const release = await app.inject({method: 'POST', url: '/gpu/release'}); t.same(got, expected); }); diff --git a/modules/demo/api-server/test/routes/quadtree.test.js b/modules/demo/api-server/test/routes/quadtree.test.js index 7c9b8444a..0fee30698 100644 --- a/modules/demo/api-server/test/routes/quadtree.test.js +++ b/modules/demo/api-server/test/routes/quadtree.test.js @@ -33,7 +33,7 @@ test('quadtree/create/:table', async (t) => { url: '/quadtree/create/csv_quadtree.csv', body: {xAxisName: 'x', yAxisName: 'y'} }); - const release = await app.inject({method: 'POST', url: '/graphology/release'}); + const release = await app.inject({method: 'POST', url: '/gpu/release'}); const result = JSON.parse(res.payload); t.same(result, { statusCode: 200, @@ -56,7 +56,7 @@ test('quadtree/set_polygons', async (t) => { body: {name: 'test', polygon_offset: [0, 1], ring_offset: [0, 4], points: [0, 0, 1, 1, 2, 2, 3, 3]} }); - const release = await app.inject({method: 'POST', url: '/graphology/release'}); + const release = await app.inject({method: 'POST', url: '/gpu/release'}); const result = JSON.parse(res.payload); t.same(result, { statusCode: 200, @@ -95,7 +95,7 @@ test('quadtree/get_points', {only: true}, async (t) => { method: 'GET', url: 'quadtree/get_points/' + quadtree_name + '/' + polygons_name, }) - const release = await app.inject({method: 'POST', url: '/graphology/release'}); + const release = await app.inject({method: 'POST', url: '/gpu/release'}); const table = tableFromIPC(res.rawPayload); const got = table.getChild('points_in_polygon').toArray(); const expected = [1, -1, -1, 1, 0, 0]; From 4c3e0fb8dfc98f06dfd11ee6c9a2e8a3bc4744ef Mon Sep 17 00:00:00 2001 From: "H. Thomson Comer" Date: Mon, 9 Jan 2023 12:02:08 -0600 Subject: [PATCH 39/43] Quadtree tweaks. --- .../demo/api-server/routes/quadtree/index.js | 45 +++++++------------ .../api-server/test/routes/quadtree.test.js | 41 +++++++++++++++-- 2 files changed, 55 insertions(+), 31 deletions(-) diff --git a/modules/demo/api-server/routes/quadtree/index.js b/modules/demo/api-server/routes/quadtree/index.js index 117cadbc6..ef1b80ede 100644 --- a/modules/demo/api-server/routes/quadtree/index.js +++ b/modules/demo/api-server/routes/quadtree/index.js @@ -12,11 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -const {Int32, Float32, DataFrame, Series} = require('@rapidsai/cudf'); -const {RecordBatchStreamWriter} = require('apache-arrow'); -const fastify = require('fastify')({logger: {level: 'debug'}}); -const fastifyCors = require('@fastify/cors'); -const cuspatial = require('@rapidsai/cuspatial'); +const {Float64, Float32, DataFrame, Series} = require('@rapidsai/cudf'); +const {RecordBatchStreamWriter} = require('apache-arrow'); +const fastify = require('fastify')({logger: {level: 'debug'}}); +const fastifyCors = require('@fastify/cors'); +const cuspatial = require('@rapidsai/cuspatial'); const arrowPlugin = require('fastify-arrow'); const gpu_cache = require('../../util/gpu_cache.js'); @@ -94,24 +94,12 @@ module.exports = async function(fastify, opts) { result.message = 'Table not found'; await reply.code(404).send(result); } else { - const [xMin, xMax, yMin, yMax] = [ - parseFloat(table.get(request.body.xAxisName).min()), - parseFloat(table.get(request.body.xAxisName).max()), - parseFloat(table.get(request.body.yAxisName).min()), - parseFloat(table.get(request.body.yAxisName).max()), - ]; + xCol = table.get(request.body.xAxisName).cast(new Float64); + yCol = table.get(request.body.yAxisName).cast(new Float64); + const [xMin, xMax, yMin, yMax] = [xCol.min(), xCol.max(), yCol.min(), yCol.max()]; try { - const quadtree = cuspatial.Quadtree.new({ - x: table.get(request.body.xAxisName), - y: table.get(request.body.yAxisName), - xMin, - xMax, - yMin, - yMax, - scale: 0, - maxDepth: 15, - minSize: 1e5 - }); + const quadtree = cuspatial.Quadtree.new( + {x: xCol, y: yCol, xMin, xMax, yMin, yMax, scale: 0, maxDepth: 15, minSize: 1e5}); const quadtree_name = request.params.table + '_quadtree'; request.params.quadtree = quadtree_name const saved = await fastify.cacheObject(quadtree_name, quadtree); @@ -131,7 +119,7 @@ module.exports = async function(fastify, opts) { fastify.route({ method: 'POST', - url: '/set_polygons_quadtree', + url: '/set_polygons', schema: { querystring: { name: {type: 'string'}, @@ -253,13 +241,14 @@ module.exports = async function(fastify, opts) { const polyPointPairs = quadtree.pointInPolygon(polygons); const resultPoints = quadtree.points.gather(polyPointPairs.get('point_index')); const numPoints = resultPoints.get('x').length - let result_col = Series.sequence({size: numPoints * 2, step: 0, init: 0}); - result_col = result_col.scatter(resultPoints.get('x'), + let result_col = + Series.sequence({size: numPoints * 2, type: new Float32, step: 0, init: 0}); + result_col = result_col.scatter(resultPoints.get('x'), Series.sequence({size: numPoints, step: 2, init: 0})); - result_col = result_col.scatter(resultPoints.get('y'), + result_col = result_col.scatter(resultPoints.get('y'), Series.sequence({size: numPoints, step: 2, init: 1})); - result = new DataFrame({'points_in_polygon': result_col}) - const writer = RecordBatchStreamWriter.writeAll(result.toArrow()); + result = new DataFrame({'points_in_polygon': result_col}) + const writer = RecordBatchStreamWriter.writeAll(result.toArrow()); writer.close(); await reply.code(200).send(writer.toNodeStream()); } catch (e) { diff --git a/modules/demo/api-server/test/routes/quadtree.test.js b/modules/demo/api-server/test/routes/quadtree.test.js index 0fee30698..d823798e9 100644 --- a/modules/demo/api-server/test/routes/quadtree.test.js +++ b/modules/demo/api-server/test/routes/quadtree.test.js @@ -52,7 +52,7 @@ test('quadtree/set_polygons', async (t) => { {method: 'POST', url: '/gpu/DataFrame/readCSV', body: {filename: 'csv_quadtree.csv'}}); const res = await app.inject({ method: 'POST', - url: '/quadtree/set_polygons_quadtree', + url: '/quadtree/set_polygons', body: {name: 'test', polygon_offset: [0, 1], ring_offset: [0, 4], points: [0, 0, 1, 1, 2, 2, 3, 3]} }); @@ -67,7 +67,7 @@ test('quadtree/set_polygons', async (t) => { }) }); -test('quadtree/get_points', {only: true}, async (t) => { +test('quadtree/get_points_int', {only: true}, async (t) => { const dir = t.testdir(csv_quadtree); const rpath = 'test/routes/' + dir.substring(dir.lastIndexOf('/')); const app = await build(t); @@ -82,7 +82,42 @@ test('quadtree/get_points', {only: true}, async (t) => { const quadtree_name = JSON.parse(create.payload).params.quadtree; const set_poly = await app.inject({ method: 'POST', - url: '/quadtree/set_polygons_quadtree', + url: '/quadtree/set_polygons', + body: { + name: 'test', + polygon_offset: [0, 1], + ring_offset: [0, 4], + points: [-2, -2, -2, 2, 2, 2, 2, -2] + } + }); + const polygons_name = JSON.parse(set_poly.payload).params.name; + const res = await app.inject({ + method: 'GET', + url: 'quadtree/get_points/' + quadtree_name + '/' + polygons_name, + }) + const release = await app.inject({method: 'POST', url: '/gpu/release'}); + const table = tableFromIPC(res.rawPayload); + const got = table.getChild('points_in_polygon').toArray(); + const expected = [1.0, -1.0, -1.0, 1.0, 0.0, 0.0]; + t.same(expected, got); +}); + +test('quadtree/get_points_int', {only: true}, async (t) => { + const dir = t.testdir(csv_quadtree); + const rpath = 'test/routes/' + dir.substring(dir.lastIndexOf('/')); + const app = await build(t); + gpu_cache._setPathForTesting(rpath); + const load = await app.inject( + {method: 'POST', url: '/gpu/DataFrame/readCSV', body: {filename: 'csv_quadtree.csv'}}); + const create = await app.inject({ + method: 'POST', + url: '/quadtree/create/csv_quadtree.csv', + body: {xAxisName: 'x', yAxisName: 'y'} + }); + const quadtree_name = JSON.parse(create.payload).params.quadtree; + const set_poly = await app.inject({ + method: 'POST', + url: '/quadtree/set_polygons', body: { name: 'test', polygon_offset: [0, 1], From fa9447cf6eb0f233eff04b1c4c21f797e8b8468d Mon Sep 17 00:00:00 2001 From: "H. Thomson Comer" Date: Fri, 13 Jan 2023 09:32:38 -0600 Subject: [PATCH 40/43] Add quadtree count endpoint, plus clean up schema. --- .../demo/api-server/routes/quadtree/index.js | 60 ++++++ .../api-server/test/routes/quadtree.test.js | 43 +++- .../demo/api-server/test/routes/root.test.js | 189 +++++++++++------- modules/demo/api-server/util/schema.js | 46 +++++ 4 files changed, 268 insertions(+), 70 deletions(-) diff --git a/modules/demo/api-server/routes/quadtree/index.js b/modules/demo/api-server/routes/quadtree/index.js index ef1b80ede..aaa8fe450 100644 --- a/modules/demo/api-server/routes/quadtree/index.js +++ b/modules/demo/api-server/routes/quadtree/index.js @@ -259,4 +259,64 @@ module.exports = async function(fastify, opts) { } } }); + + fastify.route({ + method: 'GET', + url: '/:quadtree/:polygon/count', + schema: {querystring: {quadtree: {type: 'string'}, polygon: {type: 'string'}}}, + handler: async (request, reply) => { + /** + * @api {get} /quadtree/:quadtree/:polygon/count Count Points + * @apiName CountPoints + * @apiGroup Quadtree + * @apiDescription This API returns uses the quadtree to return only the points that are in + * the polygon. + * @apiParam {String} quadtree Name of quadtree created with /quadtree/create/:table + * @apiParam {String} polygon Name of polygon created with /quadtree/set_polygons_quadtree + * @apiParamExample {json} Request-Example: + * { + * "quadtree": "test_quadtree", + * "polygon": "test_polygon" + * } + * @apiSuccessExample {json} Success-Response: + * { + * "count": 100 + * } + * @apiErrorExample {json} Error-Response: + * { + * "params": { + * "quadtree": "test_quadtree", + * "polygon": "test_polygon" + * }, + * "success": false, + * "message": "Error" + * } + */ + let message = 'Error'; + let result = {'params': request.params, success: false, message: message}; + try { + const quadtree = await fastify.getData(request.params.quadtree); + const {polygon_offset, ring_offset, points} = await fastify.getData(request.params.polygon); + const data = await fastify.listDataframes(); + const pts = cuspatial.makePoints( + points.gather(Series.sequence({size: points.length, step: 2, init: 0})), + points.gather(Series.sequence({size: points.length, step: 2, init: 1}))); + const polylines = cuspatial.makePolylines(pts, ring_offset); + const polygons = cuspatial.makePolygons(polylines, polygon_offset); + // TODO: This is a good place to put the polygons object into the cache, + // and check for it before creating it. Is it worth benchmarking? + const polyPointPairs = quadtree.pointInPolygon(polygons); + result.count = polyPointPairs.get('point_index').length; + result.message = 'Counted points in polygon'; + result.success = true; + result.statusCode = 200; + await reply.code(200).send(result); + } catch (e) { + result.message = e; + result.success = false; + result.statusCode = 500; + await reply.code(result.statusCode).send(result); + } + } + }); } diff --git a/modules/demo/api-server/test/routes/quadtree.test.js b/modules/demo/api-server/test/routes/quadtree.test.js index d823798e9..74e94a752 100644 --- a/modules/demo/api-server/test/routes/quadtree.test.js +++ b/modules/demo/api-server/test/routes/quadtree.test.js @@ -67,7 +67,7 @@ test('quadtree/set_polygons', async (t) => { }) }); -test('quadtree/get_points_int', {only: true}, async (t) => { +test('quadtree/get_points_float', async (t) => { const dir = t.testdir(csv_quadtree); const rpath = 'test/routes/' + dir.substring(dir.lastIndexOf('/')); const app = await build(t); @@ -102,7 +102,7 @@ test('quadtree/get_points_int', {only: true}, async (t) => { t.same(expected, got); }); -test('quadtree/get_points_int', {only: true}, async (t) => { +test('quadtree/get_points_int', async (t) => { const dir = t.testdir(csv_quadtree); const rpath = 'test/routes/' + dir.substring(dir.lastIndexOf('/')); const app = await build(t); @@ -136,3 +136,42 @@ test('quadtree/get_points_int', {only: true}, async (t) => { const expected = [1, -1, -1, 1, 0, 0]; t.same(expected, got); }); + +test('quadtree/:quadtree/:polygon/count', async (t) => { + const dir = t.testdir(csv_quadtree); + const rpath = 'test/routes/' + dir.substring(dir.lastIndexOf('/')); + const app = await build(t); + gpu_cache._setPathForTesting(rpath); + const load = await app.inject( + {method: 'POST', url: '/gpu/DataFrame/readCSV', body: {filename: 'csv_quadtree.csv'}}); + const create = await app.inject({ + method: 'POST', + url: '/quadtree/create/csv_quadtree.csv', + body: {xAxisName: 'x', yAxisName: 'y'} + }); + const quadtree_name = JSON.parse(create.payload).params.quadtree; + const set_poly = await app.inject({ + method: 'POST', + url: '/quadtree/set_polygons', + body: { + name: 'test', + polygon_offset: [0, 1], + ring_offset: [0, 4], + points: [-2, -2, -2, 2, 2, 2, 2, -2] + } + }); + const polygons_name = JSON.parse(set_poly.payload).params.name; + const res = await app.inject({ + method: 'GET', + url: 'quadtree/' + quadtree_name + '/' + polygons_name + '/count', + }) + const release = await app.inject({method: 'POST', url: '/gpu/release'}); + const result = JSON.parse(res.payload); + t.same(result, { + statusCode: 200, + message: 'Counted points in polygon', + params: {quadtree: quadtree_name, polygon: polygons_name}, + count: 3, + success: true + }) +}); diff --git a/modules/demo/api-server/test/routes/root.test.js b/modules/demo/api-server/test/routes/root.test.js index 04624a64f..ce499977c 100644 --- a/modules/demo/api-server/test/routes/root.test.js +++ b/modules/demo/api-server/test/routes/root.test.js @@ -21,90 +21,143 @@ test('root returns API description', async (t) => { const app = await build(t); const res = await app.inject({url: '/'}); t.same(JSON.parse(res.payload), { - gpu: { - description: 'An abstract interface to the node-rapids api, supported by a server.', - schema: { - '/': { - method: 'The name of the method to apply to gpu_cache data.', - caller: 'Either an object that has been stored in the gpu_cache or a static module name.', - arguments: 'Correctly specified arguments to the gpu_cache method.', - result: 'Either a result code specifying success or failure or an arrow data buffer.', - }, - 'DataFrame/readCSV': { - method: 'POST', - params: { - filename: 'The name of the file, stored in the server\'s public/ folder, of the csv file.' + gpu: { + description: 'An abstract interface to the node-rapids api, supported by a server.', + schema: { + '/': { + method: 'The name of the method to apply to gpu_cache data.', + caller: 'Either an object that has been stored in the gpu_cache or a static module name.', + arguments: 'Correctly specified arguments to the gpu_cache method.', + result: 'Either a result code specifying success or failure or an arrow data buffer.', }, - result: `Causes the node-rapids backend to attempt to load the csv file specified + 'DataFrame/readCSV': { + method: 'POST', + params: { + filename: + 'The name of the file, stored in the server\'s public/ folder, of the csv file.' + }, + result: `Causes the node-rapids backend to attempt to load the csv file specified by :filename. The GPU will attempt to parse the CSV file asynchronously and will return OK/ Not Found/ or Fail based on the file status.`, - returns: '500/404/200' - }, - 'get_column/:table/:column': { - method: 'GET', - params: { - ':table': - 'The filename of a previously loaded dataset, for example with `DataFrame/readCSV`', - ':column': 'A valid column name in a DataFrame that has been previously loaded.' + returns: '500/404/200' }, - returns: 'An Arrow `RecordBatchStreamWriter` stream of the columnar data.' + 'get_column/:table/:column': { + method: 'GET', + params: { + ':table': + 'The filename of a previously loaded dataset, for example with `DataFrame/readCSV`', + ':column': 'A valid column name in a DataFrame that has been previously loaded.' + }, + returns: 'An Arrow `RecordBatchStreamWriter` stream of the columnar data.' + } } - } - }, - graphology: { - description: 'The graphology api provides GPU acceleration of graphology datasets.', - schema: { - read_json: { - filename: 'A URI to a graphology json dataset file.', - result: `Causes the node-rapids backend to attempt to load the json object specified + }, + graphology: { + description: 'The graphology api provides GPU acceleration of graphology datasets.', + schema: { + read_json: { + filename: 'A URI to a graphology json dataset file.', + result: `Causes the node-rapids backend to attempt to load the json object specified by :filename. The GPU will attempt to parse the json file asynchronously and will return OK/ Not Found/ or Fail based on the file status. If the load is successful, four tables will be created in the node-rapids backend: nodes, edges, clusters, and tags. The root objects in the json target must match these names and order.`, - returns: 'Result OK/Not Found/Fail' - }, - read_large_demo: { - filename: - 'A URI to a graphology json dataset file matching the sigma.js/examples/large-demos spec.', - result: `Produces the same result as 'read_json'. + returns: 'Result OK/Not Found/Fail' + }, + read_large_demo: { + filename: + 'A URI to a graphology json dataset file matching the sigma.js/examples/large-demos spec.', + result: `Produces the same result as 'read_json'. If the load is successful, three tables will be created in the node-rapids backend: nodes, edges, and options.`, - returns: 'Result OK/Not Found/Fail' - }, - list_tables: {returns: 'Tables that are available presently in GPU memory.'}, - get_table: { - ':table': - {table: 'The name of the table that has been allocated previously into GPU memory.'} - }, - get_column: {':table': {':column': {table: 'The table name', column: 'The column name'}}}, - nodes: { - returns: - 'Returns the existing nodes table after applying normalization functions for sigma.js' - }, - nodes: {bounds: {returns: 'Returns the x and y bounds to be used in rendering.'}}, - edges: {return: 'Returns the existing edges table after applying normalization for sigma.js'} - } - }, - particles: { - description: - 'The API responsible for parsing particles CSV files for the point-budget API demo.', - schema: { - 'get_shader_column/:table/:xmin/:xmax/:ymin/:ymax': { - method: 'POST', - params: { - ':table': 'The name of the CSV file previously loaded with `DataFrame/readCSV`', - 'xmin (optional)': 'Don\'t return results outside of xmin', - 'xmax (optional)': 'Don\'t return results outside of xmax', - 'ymin (optional)': 'Don\'t return results outside of ymin', - 'ymax (optional)': 'Don\'t return results outside of ymax' + returns: 'Result OK/Not Found/Fail' + }, + list_tables: {returns: 'Tables that are available presently in GPU memory.'}, + get_table: { + ':table': + {table: 'The name of the table that has been allocated previously into GPU memory.'} }, - result: `Returns the Longitude and Latitude columns of a table that has been read previously + get_column: {':table': {':column': {table: 'The table name', column: 'The column name'}}}, + nodes: { + returns: + 'Returns the existing nodes table after applying normalization functions for sigma.js' + }, + nodes: {bounds: {returns: 'Returns the x and y bounds to be used in rendering.'}}, + edges: + {return: 'Returns the existing edges table after applying normalization for sigma.js'} + } + }, + particles: { + description: + 'The API responsible for parsing particles CSV files for the point-budget API demo.', + schema: { + 'get_shader_column/:table/:xmin/:xmax/:ymin/:ymax': { + method: 'POST', + params: { + ':table': 'The name of the CSV file previously loaded with `DataFrame/readCSV`', + 'xmin (optional)': 'Don\'t return results outside of xmin', + 'xmax (optional)': 'Don\'t return results outside of xmax', + 'ymin (optional)': 'Don\'t return results outside of ymin', + 'ymax (optional)': 'Don\'t return results outside of ymax' + }, + result: + `Returns the Longitude and Latitude columns of a table that has been read previously with DataFrame/readCSV. The Longitude and Latitude will be packed into a a single column and interleaved.`, - return: 'Returns an Arrow stream of lon/lat values as a Table containing a single column.' + return: 'Returns an Arrow stream of lon/lat values as a Table containing a single column.' + } + } + }, + quadtree: { + description: 'The API responsible for making quadtree API server requests.', + schema: { + 'create/:table': { + method: 'POST', + params: {':table': 'The name of the CSV file previously loaded with `DataFrame/readCSV`'}, + result: 'Create a quadtree from the table specified by :table.', + return: { + '200': 'Quadtree created successfully.', + '404': 'Table not found.', + '500': 'Quadtree creation failed.' + } + }, + 'set_polygons': { + method: 'POST', + params: { + 'name': 'The name of the polygon set.', + 'points': 'A list of points that define the polygons.', + 'polygon_offset': 'The GeoArrow offset defining the polygons in the points list.', + 'ring_offset': + 'The GeoArrow offset defining the rings of the polygons in the points list.', + }, + result: 'Create a polygon set from the points specified.', + return: + {'200': 'Polygon set created successfully.', '500': 'Polygon set creation failed.'} + }, + 'get_points/:quadtree/:polygon': { + method: 'GET', + params: { + ':quadtree': 'The name of the quadtree previously created with `quadtree/create`', + ':polygon': + 'The name of the polygon set previously created with `quadtree/set_polygons`' + }, + result: + 'Returns the points that are contained within the polygons specified by :polygon.', + return: 'Returns an Arrow stream of points that are contained within the polygons.' + }, + ':quadtree/:polygon/count': { + method: 'GET', + params: { + ':quadtree': 'The name of the quadtree previously created with `quadtree/create`', + ':polygon': + 'The name of the polygon set previously created with `quadtree/set_polygons`' + }, + result: + 'Returns the number of points that are contained within the polygons specified by :polygon.', + return: {count: 'The number of points that are contained within the polygons.'} + } } } - } }); }); diff --git a/modules/demo/api-server/util/schema.js b/modules/demo/api-server/util/schema.js index 0ff7555ab..703acfa0c 100644 --- a/modules/demo/api-server/util/schema.js +++ b/modules/demo/api-server/util/schema.js @@ -99,6 +99,52 @@ const schema = { return: 'Returns an Arrow stream of lon/lat values as a Table containing a single column.' } } + }, + quadtree: { + description: 'The API responsible for making quadtree API server requests.', + schema: { + 'create/:table': { + method: 'POST', + params: {':table': 'The name of the CSV file previously loaded with `DataFrame/readCSV`'}, + result: 'Create a quadtree from the table specified by :table.', + return: { + '200': 'Quadtree created successfully.', + '404': 'Table not found.', + '500': 'Quadtree creation failed.' + } + }, + 'set_polygons': { + method: 'POST', + params: { + 'name': 'The name of the polygon set.', + 'points': 'A list of points that define the polygons.', + 'polygon_offset': 'The GeoArrow offset defining the polygons in the points list.', + 'ring_offset': + 'The GeoArrow offset defining the rings of the polygons in the points list.', + }, + result: 'Create a polygon set from the points specified.', + return: {'200': 'Polygon set created successfully.', '500': 'Polygon set creation failed.'} + }, + 'get_points/:quadtree/:polygon': { + method: 'GET', + params: { + ':quadtree': 'The name of the quadtree previously created with `quadtree/create`', + ':polygon': 'The name of the polygon set previously created with `quadtree/set_polygons`' + }, + result: 'Returns the points that are contained within the polygons specified by :polygon.', + return: 'Returns an Arrow stream of points that are contained within the polygons.' + }, + ':quadtree/:polygon/count': { + method: 'GET', + params: { + ':quadtree': 'The name of the quadtree previously created with `quadtree/create`', + ':polygon': 'The name of the polygon set previously created with `quadtree/set_polygons`' + }, + result: + 'Returns the number of points that are contained within the polygons specified by :polygon.', + return: {count: 'The number of points that are contained within the polygons.'} + } + } } }; From b1740283ef75cd237db3ad8f4016e4c9e7d03503 Mon Sep 17 00:00:00 2001 From: "H. Thomson Comer" Date: Tue, 17 Jan 2023 14:58:00 -0600 Subject: [PATCH 41/43] Endpoint to return just the first n points. --- .../demo/api-server/routes/quadtree/index.js | 75 ++++++++++++++++++- .../api-server/test/routes/quadtree.test.js | 34 +++++++++ 2 files changed, 108 insertions(+), 1 deletion(-) diff --git a/modules/demo/api-server/routes/quadtree/index.js b/modules/demo/api-server/routes/quadtree/index.js index aaa8fe450..2067ec835 100644 --- a/modules/demo/api-server/routes/quadtree/index.js +++ b/modules/demo/api-server/routes/quadtree/index.js @@ -190,6 +190,79 @@ module.exports = async function(fastify, opts) { } }); + fastify.route({ + method: 'GET', + url: '/get_points/:quadtree/:polygon/:n', + schema: + {querystring: {quadtree: {type: 'string'}, polygon: {type: 'string'}, n: {type: 'number'}}}, + handler: async (request, reply) => { + /** + * @api {get} /quadtree/get_points/:quadtree/:polygon Get Points + * @apiName GetPoints + * @apiGroup Quadtree + * @apiDescription This API returns uses the quadtree to return only the points that are in + * the polygon. This API only returns the first n points. + * @apiParam {String} quadtree Name of quadtree created with /quadtree/create/:table + * @apiParam {String} polygon Name of polygon created with /quadtree/set_polygons_quadtree + * @apiParam {Number} n Number of points to return + * @apiParamExample {json} Request-Example: + * { + * "quadtree": "test_quadtree", + * "polygon": "test_polygon" + * "n": 100 + * } + * @apiSuccessExample {json} Success-Response: + * { + * "params": { + * "quadtree": "test_quadtree", + * "polygon": "test_polygon" + * }, + * "success": true, + * "message": "Get points from test_quadtree" + * } + * @apiErrorExample {json} Error-Response: + * { + * "params": { + * "quadtree": "test_quadtree", + * "polygon": "test_polygon" + * }, + * "success": false, + * "message": "Error" + * } + */ + let message = 'Error'; + let result = {'params': request.params, success: false, message: message}; + try { + const quadtree = await fastify.getData(request.params.quadtree); + const {polygon_offset, ring_offset, points} = await fastify.getData(request.params.polygon); + const data = await fastify.listDataframes(); + const pts = cuspatial.makePoints( + points.gather(Series.sequence({size: points.length, step: 2, init: 0})), + points.gather(Series.sequence({size: points.length, step: 2, init: 1}))); + const polylines = cuspatial.makePolylines(pts, ring_offset); + const polygons = cuspatial.makePolygons(polylines, polygon_offset); + const polyPointPairs = quadtree.pointInPolygon(polygons); + const resultPoints = quadtree.points.gather(polyPointPairs.get('point_index')); + const numPoints = parseInt(request.params.n); + let result_col = + Series.sequence({size: numPoints * 2, type: new Float32, step: 0, init: 0}); + result_col = result_col.scatter(resultPoints.get('x'), + Series.sequence({size: numPoints, step: 2, init: 0})); + result_col = result_col.scatter(resultPoints.get('y'), + Series.sequence({size: numPoints, step: 2, init: 1})); + result = new DataFrame({'points_in_polygon': result_col}) + const writer = RecordBatchStreamWriter.writeAll(result.toArrow()); + writer.close(); + await reply.code(200).send(writer.toNodeStream()); + } catch (e) { + result.message = JSON.stringify(e); + result.success = false; + result.statusCode = 500; + await reply.code(result.statusCode).send(result); + } + } + }); + fastify.route({ method: 'GET', url: '/get_points/:quadtree/:polygon', @@ -252,7 +325,7 @@ module.exports = async function(fastify, opts) { writer.close(); await reply.code(200).send(writer.toNodeStream()); } catch (e) { - result.message = e; + result.message = JSON.stringify(e); result.success = false; result.statusCode = 500; await reply.code(result.statusCode).send(result); diff --git a/modules/demo/api-server/test/routes/quadtree.test.js b/modules/demo/api-server/test/routes/quadtree.test.js index 74e94a752..8d48a77fe 100644 --- a/modules/demo/api-server/test/routes/quadtree.test.js +++ b/modules/demo/api-server/test/routes/quadtree.test.js @@ -175,3 +175,37 @@ test('quadtree/:quadtree/:polygon/count', async (t) => { success: true }) }); + +test('quadtree/:quadtree/:polygon/:n', {only: true}, async (t) => { + const dir = t.testdir(csv_quadtree); + const rpath = 'test/routes/' + dir.substring(dir.lastIndexOf('/')); + const app = await build(t); + gpu_cache._setPathForTesting(rpath); + const load = await app.inject( + {method: 'POST', url: '/gpu/DataFrame/readCSV', body: {filename: 'csv_quadtree.csv'}}); + const create = await app.inject({ + method: 'POST', + url: '/quadtree/create/csv_quadtree.csv', + body: {xAxisName: 'x', yAxisName: 'y'} + }); + const quadtree_name = JSON.parse(create.payload).params.quadtree; + const set_poly = await app.inject({ + method: 'POST', + url: '/quadtree/set_polygons', + body: { + name: 'test', + polygon_offset: [0, 1], + ring_offset: [0, 4], + points: [-2, -2, -2, 2, 2, 2, 2, -2] + } + }); + const polygons_name = JSON.parse(set_poly.payload).params.name; + const res = await app.inject({ + method: 'GET', + url: 'quadtree/get_points/' + quadtree_name + '/' + polygons_name + '/2', + }) + const release = await app.inject({method: 'POST', url: '/gpu/release'}); + const table = tableFromIPC(res.rawPayload); + const got = table.getChild('points_in_polygon').toArray(); + const expected = [1.0, -1.0, -1.0, 1.0]; +}); From 6b23188bc199915495998c28300d9bc678d6a96e Mon Sep 17 00:00:00 2001 From: "H. Thomson Comer" Date: Wed, 18 Jan 2023 15:36:06 -0600 Subject: [PATCH 42/43] Next and clear endpoints. --- .../demo/api-server/routes/quadtree/index.js | 103 ++++++++- .../api-server/test/routes/quadtree.test.js | 195 +++++++++++++++++- 2 files changed, 290 insertions(+), 8 deletions(-) diff --git a/modules/demo/api-server/routes/quadtree/index.js b/modules/demo/api-server/routes/quadtree/index.js index 2067ec835..92c6c7584 100644 --- a/modules/demo/api-server/routes/quadtree/index.js +++ b/modules/demo/api-server/routes/quadtree/index.js @@ -190,6 +190,90 @@ module.exports = async function(fastify, opts) { } }); + fastify.route({ + method: 'GET', + url: '/get_points/:quadtree/:polygon/clear', + schema: + {querystring: {quadtree: {type: 'string'}, polygon: {type: 'string'}, n: {type: 'number'}}}, + handler: async (request, reply) => { + let message = 'Error'; + let result = {'params': request.params, success: false, message: message}; + try { + const quadtree = await fastify.getData(request.params.quadtree); + const {polygon_offset, ring_offset, points} = await fastify.getData(request.params.polygon); + const polygons_and_served = await fastify.getData(points.toArray()); + let polygons = polygons_and_served ? polygons_and_served.polygons : undefined; + let served = polygons_and_served ? polygons_and_served.served : 0; + if (polygons === undefined) { + result.message = 'No cached polygons found'; + result.success = false; + result.statusCode = 404; + } else { + await fastify.cacheObject(points.toArray(), {polygons: polygons, served: 0}); + result.message = 'Cleared points from cache'; + result.success = true; + result.statusCode = 200; + } + await reply.code(result.statusCode).send(result); + } catch (e) { + result.message = JSON.stringify(e); + result.success = false; + result.statusCode = 500; + await reply.code(result.statusCode).send(result); + } + } + }); + + fastify.route({ + method: 'GET', + url: '/get_points/:quadtree/:polygon/:n/next', + schema: + {querystring: {quadtree: {type: 'string'}, polygon: {type: 'string'}, n: {type: 'number'}}}, + handler: async (request, reply) => { + let message = 'Error'; + let result = {'params': request.params, success: false, message: message}; + try { + const quadtree = await fastify.getData(request.params.quadtree); + const {polygon_offset, ring_offset, points} = await fastify.getData(request.params.polygon); + const polygons_and_served = await fastify.getData(points.toArray()); + let polygons = polygons_and_served ? polygons_and_served.polygons : undefined; + let served = polygons_and_served ? polygons_and_served.served : 0; + if (polygons === undefined) { + const pts = cuspatial.makePoints( + points.gather(Series.sequence({size: points.length, step: 2, init: 0})), + points.gather(Series.sequence({size: points.length, step: 2, init: 1}))); + const polylines = cuspatial.makePolylines(pts, ring_offset); + polygons = cuspatial.makePolygons(polylines, polygon_offset); + await fastify.cacheObject(points.toArray(), {polygons: polygons, served: 0}); + } + const polyPointPairs = quadtree.pointInPolygon(polygons); + const resultPoints = quadtree.points.gather(polyPointPairs.get('point_index')); + // Either return the number of requested points or the number of points left + const numPoints = + Math.min(parseInt(request.params.n), resultPoints.get('x').length - served); + let result_col = + Series.sequence({size: numPoints * 2, type: new Float32, step: 0, init: 0}); + result_col = result_col.scatter( + resultPoints.get('x').gather(Series.sequence({size: numPoints, step: 1, init: served})), + Series.sequence({size: numPoints, step: 2, init: served})); + result_col = result_col.scatter( + resultPoints.get('y').gather(Series.sequence({size: numPoints, step: 1, init: served})), + Series.sequence({size: numPoints, step: 2, init: served + 1})); + await fastify.cacheObject(points.toArray(), + {polygons: polygons, served: served + numPoints}); + result = new DataFrame({'points_in_polygon': result_col}) + const writer = RecordBatchStreamWriter.writeAll(result.toArrow()); + writer.close(); + await reply.code(200).send(writer.toNodeStream()); + } catch (e) { + result.message = JSON.stringify(e); + result.success = false; + result.statusCode = 500; + await reply.code(result.statusCode).send(result); + } + } + }); + fastify.route({ method: 'GET', url: '/get_points/:quadtree/:polygon/:n', @@ -235,15 +319,20 @@ module.exports = async function(fastify, opts) { try { const quadtree = await fastify.getData(request.params.quadtree); const {polygon_offset, ring_offset, points} = await fastify.getData(request.params.polygon); - const data = await fastify.listDataframes(); - const pts = cuspatial.makePoints( - points.gather(Series.sequence({size: points.length, step: 2, init: 0})), - points.gather(Series.sequence({size: points.length, step: 2, init: 1}))); - const polylines = cuspatial.makePolylines(pts, ring_offset); - const polygons = cuspatial.makePolygons(polylines, polygon_offset); + const polygons_and_served = await fastify.getData(points.toArray()); + let polygons = polygons_and_served ? polygons_and_served.polygons : undefined; + let served = polygons_and_served ? polygons_and_served.served : 0; + if (polygons === undefined) { + const pts = cuspatial.makePoints( + points.gather(Series.sequence({size: points.length, step: 2, init: 0})), + points.gather(Series.sequence({size: points.length, step: 2, init: 1}))); + const polylines = cuspatial.makePolylines(pts, ring_offset); + polygons = cuspatial.makePolygons(polylines, polygon_offset); + await fastify.cacheObject(points.toArray(), {polygons: polygons, served: 0}); + } const polyPointPairs = quadtree.pointInPolygon(polygons); const resultPoints = quadtree.points.gather(polyPointPairs.get('point_index')); - const numPoints = parseInt(request.params.n); + const numPoints = Math.min(parseInt(request.params.n), resultPoints.get('x').length); let result_col = Series.sequence({size: numPoints * 2, type: new Float32, step: 0, init: 0}); result_col = result_col.scatter(resultPoints.get('x'), diff --git a/modules/demo/api-server/test/routes/quadtree.test.js b/modules/demo/api-server/test/routes/quadtree.test.js index 8d48a77fe..35b9b69ef 100644 --- a/modules/demo/api-server/test/routes/quadtree.test.js +++ b/modules/demo/api-server/test/routes/quadtree.test.js @@ -176,7 +176,7 @@ test('quadtree/:quadtree/:polygon/count', async (t) => { }) }); -test('quadtree/:quadtree/:polygon/:n', {only: true}, async (t) => { +test('quadtree/:quadtree/:polygon/:n', async (t) => { const dir = t.testdir(csv_quadtree); const rpath = 'test/routes/' + dir.substring(dir.lastIndexOf('/')); const app = await build(t); @@ -208,4 +208,197 @@ test('quadtree/:quadtree/:polygon/:n', {only: true}, async (t) => { const table = tableFromIPC(res.rawPayload); const got = table.getChild('points_in_polygon').toArray(); const expected = [1.0, -1.0, -1.0, 1.0]; + t.same(got, expected); +}); + +test('quadtree/:quadtree/:polygon/:n/next', {only: true}, async (t) => { + const dir = t.testdir(csv_quadtree); + const rpath = 'test/routes/' + dir.substring(dir.lastIndexOf('/')); + const app = await build(t); + gpu_cache._setPathForTesting(rpath); + const load = await app.inject( + {method: 'POST', url: '/gpu/DataFrame/readCSV', body: {filename: 'csv_quadtree.csv'}}); + const create = await app.inject({ + method: 'POST', + url: '/quadtree/create/csv_quadtree.csv', + body: {xAxisName: 'x', yAxisName: 'y'} + }); + const quadtree_name = JSON.parse(create.payload).params.quadtree; + const set_poly = await app.inject({ + method: 'POST', + url: '/quadtree/set_polygons', + body: { + name: 'test', + polygon_offset: [0, 1], + ring_offset: [0, 4], + points: [-4, -4, -4, 4, 4, 4, 4, -4] + } + }); + const polygons_name = JSON.parse(set_poly.payload).params.name; + let res = await app.inject({ + method: 'GET', + url: 'quadtree/get_points/' + quadtree_name + '/' + polygons_name + '/2/next', + }) + let table = tableFromIPC(res.rawPayload); + let got = table.getChild('points_in_polygon').toArray(); + let expected = [3.0, -3.0, 1.0, -1.0]; + t.same(got, expected); + res = await app.inject({ + method: 'GET', + url: 'quadtree/get_points/' + quadtree_name + '/' + polygons_name + '/2/next', + }) + table = tableFromIPC(res.rawPayload); + got = table.getChild('points_in_polygon').toArray(); + expected = [-1.0, 1.0, 2.0, -2.0]; + t.same(got, expected); + res = await app.inject({ + method: 'GET', + url: 'quadtree/get_points/' + quadtree_name + '/' + polygons_name + '/2/next', + }) + table = tableFromIPC(res.rawPayload); + got = table.getChild('points_in_polygon').toArray(); + expected = [-3.0, 3.0, -2.0, 2.0]; + t.same(got, expected); + res = await app.inject({ + method: 'GET', + url: 'quadtree/get_points/' + quadtree_name + '/' + polygons_name + '/2/next', + }) + table = tableFromIPC(res.rawPayload); + got = table.getChild('points_in_polygon').toArray(); + expected = [0.0, 0.0]; + t.same(got, expected); + res = await app.inject({ + method: 'GET', + url: 'quadtree/get_points/' + quadtree_name + '/' + polygons_name + '/2/next', + }) + table = tableFromIPC(res.rawPayload); + got = table.getChild('points_in_polygon').toArray(); + expected = []; + t.same(got, expected); + const release = await app.inject({method: 'POST', url: '/gpu/release'}); +}); + +test('quadtree/:quadtree/:polygon/:n max', {only: true}, async (t) => { + const dir = t.testdir(csv_quadtree); + const rpath = 'test/routes/' + dir.substring(dir.lastIndexOf('/')); + const app = await build(t); + gpu_cache._setPathForTesting(rpath); + const load = await app.inject( + {method: 'POST', url: '/gpu/DataFrame/readCSV', body: {filename: 'csv_quadtree.csv'}}); + const create = await app.inject({ + method: 'POST', + url: '/quadtree/create/csv_quadtree.csv', + body: {xAxisName: 'x', yAxisName: 'y'} + }); + const quadtree_name = JSON.parse(create.payload).params.quadtree; + const set_poly = await app.inject({ + method: 'POST', + url: '/quadtree/set_polygons', + body: { + name: 'test', + polygon_offset: [0, 1], + ring_offset: [0, 4], + points: [-4, -4, -4, 4, 4, 4, 4, -4] + } + }); + const polygons_name = JSON.parse(set_poly.payload).params.name; + let res = await app.inject({ + method: 'GET', + url: 'quadtree/get_points/' + quadtree_name + '/' + polygons_name + '/20', + }) + let table = tableFromIPC(res.rawPayload); + let got = table.getChild('points_in_polygon').length; + let expected = 14; + const release = await app.inject({method: 'POST', url: '/gpu/release'}); + t.same(got, expected); +}); + +test('quadtree/:quadtree/:polygon/:n/next max', {only: true}, async (t) => { + const dir = t.testdir(csv_quadtree); + const rpath = 'test/routes/' + dir.substring(dir.lastIndexOf('/')); + const app = await build(t); + gpu_cache._setPathForTesting(rpath); + const load = await app.inject( + {method: 'POST', url: '/gpu/DataFrame/readCSV', body: {filename: 'csv_quadtree.csv'}}); + const create = await app.inject({ + method: 'POST', + url: '/quadtree/create/csv_quadtree.csv', + body: {xAxisName: 'x', yAxisName: 'y'} + }); + const quadtree_name = JSON.parse(create.payload).params.quadtree; + const set_poly = await app.inject({ + method: 'POST', + url: '/quadtree/set_polygons', + body: { + name: 'test', + polygon_offset: [0, 1], + ring_offset: [0, 4], + points: [-4, -4, -4, 4, 4, 4, 4, -4] + } + }); + const polygons_name = JSON.parse(set_poly.payload).params.name; + let res = await app.inject({ + method: 'GET', + url: 'quadtree/get_points/' + quadtree_name + '/' + polygons_name + '/20/next', + }) + let table = tableFromIPC(res.rawPayload); + let got = table.getChild('points_in_polygon').length; + let expected = 14; + const release = await app.inject({method: 'POST', url: '/gpu/release'}); + t.same(got, expected); +}); + +test('quadtree/:quadtree/:polygon/clear', {only: true}, async (t) => { + const dir = t.testdir(csv_quadtree); + const rpath = 'test/routes/' + dir.substring(dir.lastIndexOf('/')); + const app = await build(t); + gpu_cache._setPathForTesting(rpath); + const load = await app.inject( + {method: 'POST', url: '/gpu/DataFrame/readCSV', body: {filename: 'csv_quadtree.csv'}}); + const create = await app.inject({ + method: 'POST', + url: '/quadtree/create/csv_quadtree.csv', + body: {xAxisName: 'x', yAxisName: 'y'} + }); + const quadtree_name = JSON.parse(create.payload).params.quadtree; + const set_poly = await app.inject({ + method: 'POST', + url: '/quadtree/set_polygons', + body: { + name: 'test', + polygon_offset: [0, 1], + ring_offset: [0, 4], + points: [-4, -4, -4, 4, 4, 4, 4, -4] + } + }); + const polygons_name = JSON.parse(set_poly.payload).params.name; + let res = await app.inject({ + method: 'GET', + url: 'quadtree/get_points/' + quadtree_name + '/' + polygons_name + '/20/next', + }) + let table = tableFromIPC(res.rawPayload); + let got = table.getChild('points_in_polygon').length; + let expected = 14; + t.same(got, expected); + res = await app.inject({ + method: 'GET', + url: 'quadtree/get_points/' + quadtree_name + '/' + polygons_name + '/clear', + }) + got = JSON.parse(res.payload); + expected = { + statusCode: 200, + success: true, + message: 'Cleared points from cache', + params: {quadtree: quadtree_name, polygon: polygons_name} + }; + t.same(got, expected); + res = await app.inject({ + method: 'GET', + url: 'quadtree/get_points/' + quadtree_name + '/' + polygons_name + '/20/next', + }) + table = tableFromIPC(res.rawPayload); + got = table.getChild('points_in_polygon').length; + expected = 14; + const release = await app.inject({method: 'POST', url: '/gpu/release'}); + t.same(got, expected); }); From 230d6a3a569b37289452fd3df75cd4a1ec6280e0 Mon Sep 17 00:00:00 2001 From: "H. Thomson Comer" Date: Mon, 23 Jan 2023 09:48:11 -0600 Subject: [PATCH 43/43] Fix bug with error in next endpoint. --- modules/demo/api-server/routes/quadtree/index.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/demo/api-server/routes/quadtree/index.js b/modules/demo/api-server/routes/quadtree/index.js index 92c6c7584..b9d87f20e 100644 --- a/modules/demo/api-server/routes/quadtree/index.js +++ b/modules/demo/api-server/routes/quadtree/index.js @@ -235,7 +235,7 @@ module.exports = async function(fastify, opts) { try { const quadtree = await fastify.getData(request.params.quadtree); const {polygon_offset, ring_offset, points} = await fastify.getData(request.params.polygon); - const polygons_and_served = await fastify.getData(points.toArray()); + const polygons_and_served = await fastify.getData(points); let polygons = polygons_and_served ? polygons_and_served.polygons : undefined; let served = polygons_and_served ? polygons_and_served.served : 0; if (polygons === undefined) { @@ -266,7 +266,7 @@ module.exports = async function(fastify, opts) { writer.close(); await reply.code(200).send(writer.toNodeStream()); } catch (e) { - result.message = JSON.stringify(e); + result.message = e.toString(); result.success = false; result.statusCode = 500; await reply.code(result.statusCode).send(result);