diff --git a/modules/demo/client-server/components/server/cache.js b/modules/demo/client-server/components/server/cache.js index d7179da74..cb9f002ab 100644 --- a/modules/demo/client-server/components/server/cache.js +++ b/modules/demo/client-server/components/server/cache.js @@ -93,11 +93,11 @@ readMortgageData() { sources: [path.resolve('./public', 'data/mortgage.csv')], dataTypes: { index: new Int16, - zip: new Int32, + zip: new Uint32, dti: new Float32, current_actual_upb: new Float32, borrower_credit_score: new Int16, - load_id: new Int32, + load_id: new Uint32, delinquency_12_prediction: new Float32, seller_name: new Int16 } diff --git a/modules/demo/graph/src/app.jsx b/modules/demo/graph/src/app.jsx index 25b51073b..abab8181b 100644 --- a/modules/demo/graph/src/app.jsx +++ b/modules/demo/graph/src/app.jsx @@ -190,7 +190,7 @@ function getNodeLabels({ x, y, coordinate, nodeId, props, layer }) { return labels; } -function getEdgeLabels({ x, y, coordinate, edgeId, props, layer }) { +function getEdgeLabels({ x, y, coordinate, edgeId, props, layer, sourceNodeId, targetNodeId }) { let size = 14; const color = [255, 255, 255]; const labels = [{ diff --git a/modules/demo/ssr/graph/index.js b/modules/demo/ssr/graph/index.js index c038048d1..75d6c6e93 100755 --- a/modules/demo/ssr/graph/index.js +++ b/modules/demo/ssr/graph/index.js @@ -15,10 +15,22 @@ // limitations under the License. const fastify = require('fastify')(); +const fs = require('fs'); +const path = require('path'); + +// create `/data` directory if it does not exist +const basePath = path.join(__dirname, 'data/'); +fs.access(basePath, fs.constants.F_OK, (err, _) => () => { + if (!err) { fs.mkdir(basePath); } +}); fastify // .register(require('./plugins/webrtc'), require('./plugins/graph')(fastify)) .register(require('fastify-static'), {root: require('path').join(__dirname, 'public')}) - .get('/', (req, reply) => reply.sendFile('video.html')); + .register(require('fastify-multipart')) + .register(require('fastify-cors'), {}) + .register((require('fastify-arrow'))) + .register(require('./plugins/api')) + .get('/', (req, reply) => reply.sendFile('video.html')) fastify.listen(8080).then(() => console.log('server ready')); diff --git a/modules/demo/ssr/graph/package.json b/modules/demo/ssr/graph/package.json index c9cfb891d..7f8e50950 100644 --- a/modules/demo/ssr/graph/package.json +++ b/modules/demo/ssr/graph/package.json @@ -21,11 +21,17 @@ "fastify-socket.io": "2.0.0", "fastify-static": "4.4.1", "fastify": "3.20.2", + "fastify-multipart": "5.0.2", + "fastify-cors": "6.0.2", + "fastify-arrow": "0.1.0", + "apache-arrow": "^4.0.0", "nanoid": "3.1.31", "rxjs": "6.6.7", "shm-typed-array": "0.0.13", "simple-peer": "9.11.0", - "socket.io": "4.1.3" + "socket.io": "4.1.3", + "glob": "7.2.0", + "sharp": "0.29.2" }, "files": [ "render", diff --git a/modules/demo/ssr/graph/plugins/api/index.js b/modules/demo/ssr/graph/plugins/api/index.js new file mode 100644 index 000000000..ccba5fa66 --- /dev/null +++ b/modules/demo/ssr/graph/plugins/api/index.js @@ -0,0 +1,254 @@ +// Copyright (c) 2021, NVIDIA CORPORATION. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +const {graphs, clients} = require('../graph'); +const fs = require('fs') +const util = require('util') +const {pipeline} = require('stream') +const pump = util.promisify(pipeline) +const glob = require('glob'); +const {Float32Buffer} = require('@rapidsai/cuda'); +const {Graph} = require('@rapidsai/cugraph'); +const {DataFrame, Series, Int32, Uint8, Uint32, Uint64, Float32, Float64} = + require('@rapidsai/cudf'); +const {loadEdges, loadNodes} = require('../graph/loader'); +const {RecordBatchStreamWriter} = require('apache-arrow'); +const path = require('path'); + +function readDataFrame(path) { + if (path.indexOf('.csv', path.length - 4) !== -1) { + // csv file + return DataFrame.readCSV({sources: [path], header: 0, sourceType: 'files'}); + + } else if (path.indexOf('.parquet', path.length - 8) !== -1) { + // csv file + return DataFrame.readParquet({sources: [path]}); + } + // if (df.names.includes('Unnamed: 0')) { df = df.cast({'Unnamed: 0': new Uint32}); } + return new DataFrame({}); +} + +async function getNodesForGraph(asDeviceMemory, nodes, numNodes) { + let nodesRes = {}; + const pos = new Float32Buffer(Array.from( + {length: numNodes * 2}, + () => Math.random() * 1000 * (Math.random() < 0.5 ? -1 : 1), + )); + + if (nodes.x in nodes.dataframe.names) { + nodesRes.nodeXPositions = asDeviceMemory(nodes.dataframe.get(node.x).data); + } else { + nodesRes.nodeXPositions = pos.subarray(0, pos.length / 2); + } + if (nodes.y in nodes.dataframe.names) { + nodesRes.nodeYPositions = asDeviceMemory(nodes.dataframe.get(node.y).data); + } else { + nodesRes.nodeYPositions = pos.subarray(pos.length / 2); + } + if (nodes.dataframe.names.includes(nodes.size)) { + nodesRes.nodeRadius = asDeviceMemory(nodes.dataframe.get(nodes.size).cast(new Uint8).data); + } + if (nodes.dataframe.names.includes(nodes.color)) { + nodesRes.nodeFillColors = + asDeviceMemory(nodes.dataframe.get(nodes.color).cast(new Uint32).data); + } + if (nodes.dataframe.names.includes(nodes.id)) { + nodesRes.nodeElementIndices = + asDeviceMemory(nodes.dataframe.get(nodes.id).cast(new Uint32).data); + } + return nodesRes; +} + +async function getEdgesForGraph(asDeviceMemory, edges) { + let edgesRes = {}; + + if (edges.dataframe.names.includes(edges.color)) { + edgesRes.edgeColors = asDeviceMemory(edges.dataframe.get(edges.color).data); + } else { + edgesRes.edgeColors = asDeviceMemory( + Series + .sequence( + {type: new Uint64, size: edges.dataframe.numRows, init: 18443486512814075489n, step: 0}) + .data); + } + if (edges.dataframe.names.includes(edges.id)) { + edgesRes.edgeList = asDeviceMemory(edges.dataframe.get(edges.id).cast(new Uint64).data); + } + if (edges.dataframe.names.includes(edges.bundle)) { + edgesRes.edgeBundles = asDeviceMemory(edges.dataframe.get(edges.bundle).data); + } + return edgesRes; +} + +async function getPaginatedRows(df, pageIndex = 0, pageSize = 400, selected = []) { + if (selected.length != 0) { + const selectedSeries = Series.new({type: new Int32, data: selected}).unique(true); + const updatedDF = df.gather(selectedSeries); + const idxs = Series.sequence({ + type: new Int32, + init: (pageIndex - 1) * pageSize, + size: Math.min(pageSize, updatedDF.numRows), + step: 1 + }); + return [updatedDF.gather(idxs).toArrow(), updatedDF.numRows]; + } else { + const idxs = Series.sequence({ + type: new Int32, + init: (pageIndex - 1) * pageSize, + size: Math.min(pageSize, df.numRows), + step: 1 + }); + return [df.gather(idxs).toArrow(), df.numRows]; + } +} + +module.exports = function(fastify, opts, done) { + fastify.addHook('preValidation', (request, reply, done) => { + // handle upload validation after reading request.file() in the route function itself + if (request.url == '/datasets/upload') { + done(); + } else { + request.query.id = + (request.method == 'POST') ? `${request.body.id}:video` : `${request.query.id}:video`; + if (request.query.id in fastify[clients]) { + done(); + } else { + reply.code(500).send('client handshake not established'); + } + } + }); + + async function renderGraph(id, data) { + const asDeviceMemory = (buf) => new (buf[Symbol.species])(buf); + const src = data.edges.dataframe.get(data.edges.src); + const dst = data.edges.dataframe.get(data.edges.dst); + const graph = Graph.fromEdgeList(src, dst); + fastify[graphs][id] = { + refCount: 0, + nodes: await getNodesForGraph(asDeviceMemory, data.nodes, graph.numNodes), + edges: await getEdgesForGraph(asDeviceMemory, data.edges), + graph: graph, + }; + + ++fastify[graphs][id].refCount; + + return { + gravity: 0.0, + linLogMode: false, + scalingRatio: 5.0, + barnesHutTheta: 0.0, + jitterTolerance: 0.05, + strongGravityMode: false, + outboundAttraction: false, + graph: fastify[graphs][id].graph, + nodes: { + ...fastify[graphs][id].nodes, + length: fastify[graphs][id].graph.numNodes, + }, + edges: { + ...fastify[graphs][id].edges, + length: fastify[graphs][id].graph.numEdges, + }, + }; + } + + fastify.post('/datasets/upload', async function(req, reply) { + const data = await req.file(); + const id = `${data.fields.id.value}:video`; + if (id in fastify[clients]) { + const basePath = `${__dirname}/../../data/`; + const filepath = path.join(basePath, data.filename); + const target = fs.createWriteStream(filepath); + try { + await pump(data.file, target); + } catch (err) { console.log(err); } + reply.send(); + } else { + reply.code(500).send('client handshake not established'); + } + }); + + fastify.get('/datasets', async (request, reply) => { + glob(`*.{csv,parquet}`, + {cwd: `${__dirname}/../../data/`}, + (er, files) => { reply.send(JSON.stringify(files.concat(['defaultExample']))); }); + }); + + fastify.post('/dataframe/load', async (request, reply) => { + const filePath = `${__dirname}/../../data/` + if (fs.existsSync(`${filePath}${request.body.nodes}`) && + fs.existsSync(`${filePath}${request.body.edges}`)) { + fastify[clients][request.query.id].data.nodes.dataframe = + await readDataFrame(`${filePath}${request.body.nodes}`); + + fastify[clients][request.query.id].data.edges.dataframe = + await readDataFrame(`${filePath}${request.body.edges}`); + } + else { + fastify[clients][request.query.id].data.nodes.dataframe = await loadNodes(); + fastify[clients][request.query.id].data.edges.dataframe = await loadEdges(); + } + if (fastify[clients][request.query.id].data.nodes.dataframe.numRows == 0) { + reply.code(500).send('no dataframe loaded'); + } + reply.send(JSON.stringify({ + 'nodes': fastify[clients][request.query.id].data.nodes.dataframe.numRows, + 'edges': fastify[clients][request.query.id].data.edges.dataframe.numRows + })); + }) + + fastify.get('/dataframe/columnNames/read', async (request, reply) => { + reply.send(JSON.stringify({ + nodesParams: fastify[clients][request.query.id].data.nodes.dataframe.names.concat([null]), + edgesParams: fastify[clients][request.query.id].data.edges.dataframe.names.concat([null]) + })); + }); + + fastify.post('/dataframe/columnNames/update', async (request, reply) => { + try { + Object.assign(fastify[clients][request.query.id].data.nodes, request.body.nodes); + Object.assign(fastify[clients][request.query.id].data.edges, request.body.edges); + reply.code(200).send('successfully updated columnNames'); + } catch (err) { reply.code(500).send(err); } + }); + + fastify.post('/graph/render', async (request, reply) => { + try { + fastify[clients][request.query.id].graph = + await renderGraph('default', fastify[clients][request.query.id].data); + reply.code(200).send('successfully rendered graph'); + } catch (err) { reply.code(500).send(err); } + }) + + fastify.get('/dataframe/read', async (request, reply) => { + try { + const pageIndex = parseInt(request.query.pageIndex); + const pageSize = parseInt(request.query.pageSize); + const dataframe = request.query.dataframe; //{'nodes', 'edges'} + const [arrowTable, numRows] = + await getPaginatedRows(fastify[clients][request.query.id].data[dataframe].dataframe, + pageIndex, + pageSize, + fastify[clients][request.query.id].state.selectedInfo[dataframe]); + + arrowTable.schema.metadata.set('numRows', numRows); + RecordBatchStreamWriter.writeAll(arrowTable).pipe(reply.stream()); + } catch (err) { + request.log.error({err}, '/run_query error'); + reply.code(500).send(err); + } + }); + + done(); +} diff --git a/modules/demo/ssr/graph/plugins/graph/deck.js b/modules/demo/ssr/graph/plugins/graph/deck.js new file mode 100644 index 000000000..ca838cebf --- /dev/null +++ b/modules/demo/ssr/graph/plugins/graph/deck.js @@ -0,0 +1,267 @@ +// Copyright (c) 2021, NVIDIA CORPORATION. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +const {IpcMemory, Uint8Buffer} = require('@rapidsai/cuda'); + +/** + * makeDeck() returns a Deck and a render callable object to be consumed by the multi-worker + * Renderer class + * + * @returns { + * DeckSSR, + * render(layers = {}, boxSelectRectData = [], props = {}) + * } + */ +function makeDeck() { + const {log: deckLog} = require('@deck.gl/core'); + deckLog.level = 0; + deckLog.enable(false); + + const {OrthographicView} = require('@deck.gl/core'); + const {TextLayer, PolygonLayer} = require('@deck.gl/layers'); + const {DeckSSR, GraphLayer} = require('@rapidsai/deck.gl'); + const {OrthographicController} = require('@rapidsai/deck.gl'); + + const makeLayers = (deck, graph = {}) => { + const [viewport] = (deck?.viewManager?.getViewports() || []); + const [minX = Number.NEGATIVE_INFINITY, + minY = Number.NEGATIVE_INFINITY, + ] = viewport?.getBounds() || []; + return [ + new TextLayer({ + sizeScale: 1, + opacity: 0.9, + maxWidth: 2000, + pickable: false, + getTextAnchor: 'start', + getAlignmentBaseline: 'top', + getSize: ({size}) => size, + getColor: ({color}) => color, + getPixelOffset: ({offset}) => offset, + data: Array.from({length: +process.env.NUM_WORKERS}, + (_, i) => // + ({ + size: 15, + offset: [0, i * 15], + text: `Worker ${i}`, + position: [minX, minY], + color: +process.env.WORKER_ID === i // + ? [245, 171, 53, 255] + : [255, 255, 255, 255], + })) + }), + new GraphLayer({pickable: true, ...graph}) + ]; + }; + + const getPolygonLayer = (rectdata) => { + return new PolygonLayer({ + filled: true, + stroked: true, + getPolygon: d => d.polygon, + lineWidthUnits: 'pixels', + getLineWidth: 2, + getLineColor: [80, 80, 80], + getLineColor: [0, 0, 0, 150], + getFillColor: [255, 255, 255, 65], + data: rectdata + }) + }; + + const onDragStart = + (info, event) => { + if (deck.props.controller.dragPan) { return; } + const {x, y} = info; + const [px, py] = info.viewport.unproject([x, y]); + deck.boxSelectCoordinates.startPos = [x, y]; + deck.boxSelectCoordinates.rectdata = + [{polygon: [[px, py], [px, py], [px, py], [px, py]], show: true}]; + } + + const onDragEnd = + (info, event) => { + if (deck.props.controller.dragPan || !deck.boxSelectCoordinates.startPos || + !deck.boxSelectCoordinates.rectdata) { + return; + } + const {x, y} = info; + const sx = deck.boxSelectCoordinates.startPos[0]; + const sy = deck.boxSelectCoordinates.startPos[1]; + + deck.boxSelectCoordinates.rectdata = + [{polygon: deck.boxSelectCoordinates.rectdata[0].polygon || [], show: true}]; + deck.boxSelectCoordinates.startPos = null; + deck.selectedInfo.selectedCoordinates = { + x: Math.min(sx, x), + y: Math.min(sy, y), + width: Math.abs(x - sx), + height: Math.abs(y - sy), + layerIds: ['GraphLayer'] + }; + + deck.selectedInfo.nodes = deck.pickObjects(deck.selectedInfo.selectedCoordinates) + .filter(selected => selected.hasOwnProperty('nodeId')) + .map(n => n.nodeId); + + deck.selectedInfo.edges = deck.pickObjects(deck.selectedInfo.selectedCoordinates) + .filter(selected => selected.hasOwnProperty('edgeId')) + .map(n => n.edgeId); + } + + const onDrag = (info, event) => { + if (deck.props.controller.dragPan) { return; } + if (deck.boxSelectCoordinates.startPos) { + const {x, y} = info; + const [px, py] = info.viewport.unproject([x, y]); + const startPoint = deck.boxSelectCoordinates.rectdata[0].polygon[0]; + deck.boxSelectCoordinates.rectdata = + [{polygon: [startPoint, [startPoint[0], py], [px, py], [px, startPoint[1]]], show: true}]; + }; + }; + + const onClick = (info, event) => { + deck.selectedInfo.selectedCoordinates = { + x: info.x, + y: info.y, + radius: 1, + }; + deck.selectedInfo.nodes = [deck.pickObject(deck.selectedInfo.selectedCoordinates)] + .filter(selected => selected && selected.hasOwnProperty('nodeId')) + .map(n => n.nodeId); + + console.log(deck.selectedInfo.nodes, deck.selectedInfo.selectedCoordinates); + }; + + const deck = new DeckSSR({ + createFramebuffer: true, + initialViewState: { + zoom: 1, + target: [0, 0, 0], + minZoom: Number.NEGATIVE_INFINITY, + maxZoom: Number.POSITIVE_INFINITY, + }, + layers: [makeLayers(null, {})], + views: [ + new OrthographicView({ + clear: { + color: [...[46, 46, 46].map((x) => x / 255), 1], + }, + controller: { + keyboard: false, + doubleClickZoom: false, + type: OrthographicController, + scrollZoom: {speed: 0.01, smooth: false}, + // dragPan: false + } + }), + ], + onAfterAnimationFrameRender({_loop}) { _loop.pause(); }, + }); + + deck.selectedInfo = {selectedCoordinates: {}, selected: []}; + deck.boxSelectCoordinates = {rectdata: [{polygon: [[]], show: false}], startPos: null}; + deck.setProps({onClick, onDrag, onDragStart, onDragEnd}); + + return { + deck, + render(layers = {}, boxSelectRectData = [], props = {}) { + const done = deck.animationLoop.waitForRender(); + deck.setProps({ + layers: makeLayers(deck, layers) + .concat(boxSelectRectData[0].show ? getPolygonLayer(boxSelectRectData) : []), + ...props + }); + deck.animationLoop.start(); + return done; + }, + }; +} + +function openGraphIpcHandles({nodes, edges, ...graphLayerProps} = {}) { + const data = { + nodes: openNodeIpcHandles(nodes), + edges: openEdgeIpcHandles(edges), + }; + return { + pickable: true, + edgeOpacity: .5, + edgeStrokeWidth: 2, + nodesStroked: true, + nodeFillOpacity: .5, + nodeStrokeOpacity: .9, + nodeRadiusScale: 1 / 75, + nodeRadiusMinPixels: 5, + nodeRadiusMaxPixels: 150, + ...graphLayerProps, + data, + numNodes: data.nodes.length, + numEdges: data.edges.length, + }; +} + +function closeGraphIpcHandles(graph) { + closeIpcHandles(graph.data.nodes); + closeIpcHandles(graph.data.edges); +} + +function openNodeIpcHandles(attrs = {}) { + const attributes = { + nodeRadius: openIpcHandle(attrs.nodeRadius), + nodeXPositions: openIpcHandle(attrs.nodeXPositions), + nodeYPositions: openIpcHandle(attrs.nodeYPositions), + nodeFillColors: openIpcHandle(attrs.nodeFillColors), + nodeElementIndices: openIpcHandle(attrs.nodeElementIndices), + }; + return {offset: 0, length: attrs.length ?? (attributes.nodeRadius?.byteLength || 0), attributes}; +} + +function openEdgeIpcHandles(attrs = {}) { + const attributes = { + edgeList: openIpcHandle(attrs.edgeList), + edgeColors: openIpcHandle(attrs.edgeColors), + edgeBundles: openIpcHandle(attrs.edgeBundles), + }; + return { + offset: 0, + length: attrs.length ?? (attributes.edgeList?.byteLength || 0) / 8, + attributes + }; +} + +function openIpcHandle(obj) { + if (typeof obj === 'string') { obj = JSON.parse(obj); } + if (obj) { + const {byteOffset = 0} = obj; + const handle = Uint8Array.from(obj.handle.map(Number)); + return new Uint8Buffer(new IpcMemory(handle)).subarray(byteOffset); + } + return null; +} + +function closeIpcHandles(obj) { + for (const key in obj) { + const {buffer} = obj[key] || {}; + if (buffer && (buffer instanceof IpcMemory)) { // + buffer.close(); + } + } +} + +function serializeCustomLayer(layers = []) { + return layers?.find((layer) => layer.id === 'GraphLayer').serialize(); +} + +module.exports.makeDeck = makeDeck; +module.exports.openLayerIpcHandles = openGraphIpcHandles; +module.exports.closeLayerIpcHandles = closeGraphIpcHandles; +module.exports.serializeCustomLayer = serializeCustomLayer; diff --git a/modules/demo/ssr/graph/plugins/graph/index.js b/modules/demo/ssr/graph/plugins/graph/index.js index 7853448ac..ce571afef 100644 --- a/modules/demo/ssr/graph/plugins/graph/index.js +++ b/modules/demo/ssr/graph/plugins/graph/index.js @@ -12,15 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -const wrtc = require('wrtc'); -const {DeviceBuffer} = require('@rapidsai/rmm'); -const {MemoryView} = require('@rapidsai/cuda'); -const {Float32Buffer} = require('@rapidsai/cuda'); -const {Graph} = require('@rapidsai/cugraph'); -const {Series, Int32} = require('@rapidsai/cudf'); +const wrtc = require('wrtc'); +const {MemoryView} = require('@rapidsai/cuda'); +const {DataFrame, Series, Float32} = require('@rapidsai/cudf'); -const {loadNodes, loadEdges} = require('./loader'); -const {RenderCluster} = require('../../render/cluster'); +const {RenderCluster} = require('../../render/cluster'); const {create: shmCreate, detach: shmDetach} = require('shm-typed-array'); @@ -64,24 +60,14 @@ function graphSSRClients(fastify) { }, event: {}, props: {width, height, layout}, - graph: await loadGraph(graphId), + graph: {}, // await loadGraph(graphId), + data: { + nodes: {dataframe: new DataFrame({}), color: '', size: '', id: '', x: 'x', y: 'y'}, + edges: {dataframe: new DataFrame({}), color: '', id: '', bundle: '', src: 'src', dst: 'dst'} + }, frame: shmCreate(width * height * 3 / 2), peer: peer, }; - if (clients[stream.id].graph.dataframes[0]) { - const res = getPaginatedRows(clients[stream.id].graph.dataframes[0]); - peer.send(JSON.stringify({ - type: 'data', - data: {nodes: {data: res, length: clients[stream.id].graph.dataframes[0].numRows}} - })); - } - if (clients[stream.id].graph.dataframes[1]) { - const res = getPaginatedRows(clients[stream.id].graph.dataframes[1]); - peer.send(JSON.stringify({ - type: 'data', - data: {edges: {data: res, length: clients[stream.id].graph.dataframes[1].numRows}} - })); - } stream.addTrack(source.createTrack()); peer.streams.push(stream); @@ -127,79 +113,17 @@ function graphSSRClients(fastify) { } } } - - async function loadGraph(id) { - let dataframes = []; - - if (!(id in graphs)) { - const asDeviceMemory = (buf) => new (buf[Symbol.species])(buf); - dataframes = await Promise.all([loadNodes(id), loadEdges(id)]); - const src = dataframes[1].get('src'); - const dst = dataframes[1].get('dst'); - graphs[id] = { - refCount: 0, - nodes: { - nodeRadius: asDeviceMemory(dataframes[0].get('size').data), - nodeFillColors: asDeviceMemory(dataframes[0].get('color').data), - nodeElementIndices: asDeviceMemory(dataframes[0].get('id').data), - }, - edges: { - edgeList: asDeviceMemory(dataframes[1].get('edge').data), - edgeColors: asDeviceMemory(dataframes[1].get('color').data), - edgeBundles: asDeviceMemory(dataframes[1].get('bundle').data), - }, - graph: Graph.fromEdgeList(src, dst), - }; - } - - ++graphs[id].refCount; - - const pos = new Float32Buffer(Array.from( - {length: graphs[id].graph.numNodes * 2}, - () => Math.random() * 1000 * (Math.random() < 0.5 ? -1 : 1), - )); - - return { - gravity: 0.0, - linLogMode: false, - scalingRatio: 5.0, - barnesHutTheta: 0.0, - jitterTolerance: 0.05, - strongGravityMode: false, - outboundAttraction: false, - graph: graphs[id].graph, - nodes: { - ...graphs[id].nodes, - length: graphs[id].graph.numNodes, - nodeXPositions: pos.subarray(0, pos.length / 2), - nodeYPositions: pos.subarray(pos.length / 2), - }, - edges: { - ...graphs[id].edges, - length: graphs[id].graph.numEdges, - }, - dataframes: dataframes - }; - } } function layoutAndRenderGraphs(clients) { - const renderer = new RenderCluster({numWorkers: 1 && 4}); + const renderer = new RenderCluster( + {numWorkers: 1 && 4, deckLayersPath: require('path').join(__dirname, 'deck')}); return () => { for (const id in clients) { const client = clients[id]; - const sendToClient = - ([nodes, edges]) => { - client.peer.send(JSON.stringify( - {type: 'data', data: {nodes: {data: getPaginatedRows(nodes), length: nodes.numRows}}})); - client.peer.send(JSON.stringify( - {type: 'data', data: {edges: {data: getPaginatedRows(edges), length: edges.numRows}}})); - } - if (client.isRendering) { - continue; - } + if (client.isRendering) { continue; } const state = {...client.state}; const props = {...client.props}; @@ -247,7 +171,7 @@ function layoutAndRenderGraphs(clients) { props, event, frame: client.frame.key, - graph: { + layers: { ...client.graph, graph: undefined, edges: getIpcHandles(client.graph.edges), @@ -263,26 +187,16 @@ function layoutAndRenderGraphs(clients) { result.state.clearSelections = false; // reset selected state - result.state.selectedInfo.selectedNodes = []; - result.state.selectedInfo.selectedEdges = []; + result.state.selectedInfo.nodes = []; + result.state.selectedInfo.edges = []; result.state.selectedInfo.selectedCoordinates = {}; result.state.boxSelectCoordinates.rectdata = [{polygon: [[]], show: false}]; // send to client - if (client.graph.dataframes) { sendToClient(client.graph.dataframes); } + client.peer.send(JSON.stringify({type: 'data', data: 'newQuery'})); } else if (JSON.stringify(client.state.selectedInfo.selectedCoordinates) !== JSON.stringify(result.state.selectedInfo.selectedCoordinates)) { - // selections updated - const nodes = - Series.new({type: new Int32, data: result.state.selectedInfo.selectedNodes}); - const edges = - Series.new({type: new Int32, data: result.state.selectedInfo.selectedEdges}); - if (client.graph.dataframes) { - sendToClient([ - client.graph.dataframes[0].gather(nodes), - client.graph.dataframes[1].gather(edges) - ]); - } + client.peer.send(JSON.stringify({type: 'data', data: 'newQuery'})); } // copy result state to client's current state result?.state && Object.assign(client.state, result.state); @@ -294,16 +208,27 @@ function layoutAndRenderGraphs(clients) { } } -function getPaginatedRows(df, page = 1, rowsPerPage = 400) { - if (!df) { return {}; } - return df.head(page * rowsPerPage).tail(rowsPerPage).toArrow().toArray(); -} - function forceAtlas2({graph, nodes, edges, ...params}) { - graph.forceAtlas2({...params, positions: nodes.nodeXPositions.buffer}); + if (graph == undefined) { return {}; } + const asDeviceMemory = (buf) => new (buf[Symbol.species])(buf); + + const tempPositions = + Series.new({type: new Float32, data: nodes.nodeXPositions.buffer}) + .concat(Series.new({type: new Float32, data: nodes.nodeYPositions.buffer})); + + const positions = graph.forceAtlas2({...params, positions: tempPositions.data}); + + nodes.nodeXPositions = asDeviceMemory( + Series.new({type: new Float32, length: graph.numNodes, offset: 0, data: positions.buffer}) + .data); + nodes.nodeYPositions = asDeviceMemory( + Series + .new( + {type: new Float32, length: graph.numNodes, offset: graph.numNodes, data: positions.buffer}) + .data); return { - graph, + graph: graph, ...params, nodes: {...nodes, length: graph.numNodes}, edges: {...edges, length: graph.numEdges}, diff --git a/modules/demo/ssr/graph/plugins/graph/loader.js b/modules/demo/ssr/graph/plugins/graph/loader.js index 023e0209d..29d0e07a3 100644 --- a/modules/demo/ssr/graph/plugins/graph/loader.js +++ b/modules/demo/ssr/graph/plugins/graph/loader.js @@ -1,5 +1,5 @@ -const {Utf8Vector} = require('apache-arrow'); -const {DataFrame, Series, Uint32, Uint64, Uint8} = require('@rapidsai/cudf'); +const {Utf8Vector} = require('apache-arrow'); +const {DataFrame, Series, Int32, Uint32, Uint64, Uint8} = require('@rapidsai/cudf'); function loadNodes(graphId) { if (graphId === 'default') { return getDefaultNodes(); } @@ -84,7 +84,7 @@ function getDefaultEdges() { return new DataFrame({ name: Series.new(Utf8Vector.from(Array.from({length: 312}, (_, i) => `${i}`))), src: Series.new({ - type: new Uint32, + type: new Int32, data: [ 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, @@ -103,7 +103,7 @@ function getDefaultEdges() { ] }), dst: Series.new({ - type: new Uint32, + type: new Int32, data: [ 2, 3, 4, 5, 6, 6, 7, 7, 8, 8, 8, 8, 9, 10, 11, 12, 14, 15, 16, 13, 17, 18, 17, 17, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 35, 35, 35, 36, 37, diff --git a/modules/demo/ssr/graph/public/video.html b/modules/demo/ssr/graph/public/video.html index c2c25edbb..8ffb9c85b 100644 --- a/modules/demo/ssr/graph/public/video.html +++ b/modules/demo/ssr/graph/public/video.html @@ -1,3 +1,14 @@ + + + + + + + + + + + @@ -18,11 +29,12 @@
+
-