diff --git a/cli/package.json b/cli/package.json index 021621214..c72a2b631 100644 --- a/cli/package.json +++ b/cli/package.json @@ -37,10 +37,12 @@ "dependencies": { "@dodona/dolos-lib": "3.3.1", "@dodona/dolos-web": "2.7.1", + "apache-arrow": "^17.0.0", "chalk": "^5.3.0", "cliui": "^8.0.1", "commander": "^12.1.0", "csv-stringify": "^6.5.0", + "duckdb-async": "^0.9.1", "open": "^10.1.0", "tree-sitter": "^0.21.1" }, diff --git a/cli/src/cli/commands/run.ts b/cli/src/cli/commands/run.ts index fbe5adf61..88e0a7de6 100644 --- a/cli/src/cli/commands/run.ts +++ b/cli/src/cli/commands/run.ts @@ -6,12 +6,14 @@ import { } from "../util/utils.js"; import { DEFAULT_HOST, DEFAULT_PORT } from "../server.js"; +import { DbView } from "../views/dbView.js"; import { TerminalView } from "../views/terminalView.js"; import { FileView } from "../views/fileView.js"; import { WebView } from "../views/webView.js"; import { Command } from "commander"; import * as Utils from "../util/utils.js"; import { Dolos, Options } from "@dodona/dolos-lib"; +import { DbWebView } from "../views/dbWebView.js"; export function runCommand(program: Command): Command { return new Command("run") @@ -219,12 +221,17 @@ export async function run(locations: string[], options: RunOptions): Promise new FileView(report, options), "html": () => new WebView(report, options), "web": () => new WebView(report, options), + "db": () => new DbView(report, options), + "dbweb": () => new DbWebView(report, options), }); if (view == null) { throw new Error(`Invalid output format: ${options.outputFormat}`); } + const startTime = new Date().getTime(); await view().show(); + const endTime = new Date().getTime(); + console.log(`Write-out took ${endTime - startTime} ms`); }); } diff --git a/cli/src/cli/server.ts b/cli/src/cli/server.ts index 089db0801..022229134 100644 --- a/cli/src/cli/server.ts +++ b/cli/src/cli/server.ts @@ -14,6 +14,8 @@ export interface Options { } const MIME: { [k: string]: string } = { + "db": "application/octet-stream", + "wasm": "application/wasm", "html": "text/html", "css": "text/css", "js": "text/javascript", @@ -32,7 +34,8 @@ function notFound(response: http.ServerResponse): void { export default async function runServer( reportDir: string, - options: Options + options: Options, + doneCallback?: () => void ): Promise { const port = options.port || DEFAULT_PORT; const host = options.host || DEFAULT_HOST; @@ -48,7 +51,14 @@ export default async function runServer( const reqPath = path.normalize(new URL(request.url, baseURL).pathname); let filePath; - if (reqPath.startsWith("/data")) { + if (reqPath === "/data/dolos.db") { + filePath = reportDir; // actually path to dolos.db + } else if (reqPath.startsWith("/data/done")) { + if (doneCallback) doneCallback(); + response.writeHead(202); + response.end(); + return; + } else if (reqPath.startsWith("/data")) { filePath = path.join(reportDir, reqPath.slice(5)); } else if (reqPath.endsWith("/")) { filePath = path.join(webDir, reqPath, "index.html"); diff --git a/cli/src/cli/views/dbView.ts b/cli/src/cli/views/dbView.ts new file mode 100644 index 000000000..5133818bf --- /dev/null +++ b/cli/src/cli/views/dbView.ts @@ -0,0 +1,146 @@ +import { View } from "./view.js"; +import { Database } from "duckdb-async"; +import * as arrow from "apache-arrow"; +import { promises as fs } from "fs"; +import { + Region, + Report, +} from "@dodona/dolos-lib"; + + +export interface Options { + outputDestination?: string; +} + +export class DbView extends View { + + protected outputDestination: string; + + constructor(protected report: Report, options: Options) { + super(); + this.outputDestination = + options.outputDestination || this.createName(); + } + + private createName(): string { + const dashedName = this.report.name.replace(/ /g, "-").replace(/[^a-zA-Z0-9-]/g, ""); + const timestamp = new Date().toISOString().replace(/[.:-]/g, ""); + return `dolos-report-${ timestamp }-${ dashedName }.db`; + } + + public async writePairs(db: Database): Promise { + const pairs = this.report.allPairs(); + + const table = arrow.tableFromArrays({ + id: new Uint32Array(pairs.map(p => p.id)), + leftFileId: new Uint32Array(pairs.map(p => p.leftFile.id)), + leftFilePath: pairs.map(p => p.leftFile.path), + rightFileId: new Uint32Array(pairs.map(p => p.rightFile.id)), + rightFilePath: pairs.map(p => p.rightFile.path), + similarity: pairs.map(p => p.similarity), + totalOverlap: new Uint32Array(pairs.map(p => p.overlap)), + longestFragment: new Uint32Array(pairs.map(p => p.longest)), + leftCovered: new Uint32Array(pairs.map(p => p.leftCovered)), + rightCovered: new Uint32Array(pairs.map(p => p.rightCovered)), + }); + + await db.register_buffer("arrow_pairs", [arrow.tableToIPC(table)], true); + await db.exec("CREATE TABLE pairs AS SELECT * FROM arrow_pairs"); + } + + public async writeKgrams(db: Database): Promise { + const fingerprints = this.report.sharedFingerprints(); + + const table = arrow.tableFromArrays({ + id: new Uint32Array(fingerprints.map(f => f.id)), + hash: new BigUint64Array(fingerprints.map(f => BigInt(f.hash))), + ignored: fingerprints.map(f => f.ignored), + data: fingerprints.map(f => f.kgram?.join(" ") || ""), + files: fingerprints.map(f => JSON.stringify(f.files().map(f => f.id))) + }); + + + await db.register_buffer("arrow_kgrams", [arrow.tableToIPC(table)], true); + await db.exec("CREATE TABLE kgrams AS SELECT id, hash, ignored, data, list_transform(files->'$[*]', f -> f::INTEGER) AS files FROM arrow_kgrams"); + } + + public async writeFiles(db: Database): Promise { + const entries = this.report.entries().concat(this.report.ignoredEntries()); + + const table = arrow.tableFromArrays({ + id: new Uint32Array(entries.map(e => e.file.id)), + ignored: entries.map(e => e.isIgnored), + path: entries.map(e => e.file.path), + content: entries.map(e => e.file.content), + kgramCount: new Uint32Array(entries.map(e => e.kgrams.length)), + ast: entries.map(e => JSON.stringify(e.file.tokens)), + mapping: entries.map(e => JSON.stringify(Region.toUInt16(e.file.mapping))), + extra: entries.map(e => JSON.stringify(e.file.extra || {})) + }); + + await db.register_buffer("arrow_files", [arrow.tableToIPC(table)], true); + await db.exec(` + CREATE TABLE files + AS SELECT + id, + ignored, + path, + content, + kgramCount, + list_transform(ast->>'$[*]', s->s::STRING) AS ast, + list_transform(mapping->'$[*]', m -> m::INT2) AS mapping, + extra->'$' as extra + FROM arrow_files + `); + } + + public async writeMetadata(db: Database): Promise { + await db.exec(` + CREATE TABLE metadata ( + key STRING, + value STRING, + type STRING + ); + `); + const stmt = await db.prepare(`INSERT INTO metadata VALUES (?, ?, ?)`); + for (const [key, value] of Object.entries(this.report.metadata())) { + await stmt.run( + key, + value == null ? "null" : value.toString(), + typeof value + ); + } + stmt.finalize(); + } + + async writeToDb(): Promise { + const dbName = this.outputDestination; + if (await fs.stat(dbName).catch(() => false)) { + throw new Error(`File ${dbName} already exists. Please specify a different output destination.`); + } + const db = await Database.create(dbName); + await db.exec('INSTALL arrow; LOAD arrow; BEGIN TRANSACTION;'); + + console.log(`Writing results to database: ${dbName}`); + await this.writeMetadata(db); + console.log("Metadata written."); + await this.writePairs(db); + + console.log("Pairs written."); + + await this.writeKgrams(db); + console.log("Kgrams written."); + await this.writeFiles(db); + console.log("Files written."); + + await db.exec('COMMIT;'); + console.log("Completed"); + await db.close(); + return dbName; + } + + async show(): Promise { + await this.writeToDb(); + } + +} diff --git a/cli/src/cli/views/dbWebView.ts b/cli/src/cli/views/dbWebView.ts new file mode 100644 index 000000000..58491d398 --- /dev/null +++ b/cli/src/cli/views/dbWebView.ts @@ -0,0 +1,26 @@ +import { FileView, Options as FVOptions } from "./fileView.js"; +import runServer, { Options as ServerOptions } from "../server.js"; +import { Report } from "@dodona/dolos-lib"; +import {DbView} from "./dbView.js"; + +/** + * This {@link View} will launch a webserver which hosts a web application to + * display results of an analysis written to CSV-files using the + * {@link FileView}. + */ +export class DbWebView extends DbView { + + constructor(report: Report, private options: ServerOptions & FVOptions) { + super(report, options); + } + + async show(): Promise { + const reportDir = await this.writeToDb(); + const start = Date.now(); + const done = () => { + const stop = Date.now(); + console.log(`Shown in ${stop - start}ms`); + } + setTimeout(async () => await runServer(reportDir, this.options, done), 1000); + } +} diff --git a/cli/src/cli/views/webView.ts b/cli/src/cli/views/webView.ts index 662a517bf..c7a2fe76f 100644 --- a/cli/src/cli/views/webView.ts +++ b/cli/src/cli/views/webView.ts @@ -15,6 +15,11 @@ export class WebView extends FileView { async show(): Promise { const reportDir = await this.writeToDirectory(); - setTimeout(async () => await runServer(reportDir, this.options), 1000); + const start = Date.now(); + const done = () => { + const stop = Date.now(); + console.log(`Shown in ${stop - start}ms`); + } + setTimeout(async () => await runServer(reportDir, this.options, done), 1000); } } diff --git a/core/src/algorithm/fingerprintIndex.ts b/core/src/algorithm/fingerprintIndex.ts index 0210871bb..45b5f1dd4 100644 --- a/core/src/algorithm/fingerprintIndex.ts +++ b/core/src/algorithm/fingerprintIndex.ts @@ -241,4 +241,21 @@ export class FingerprintIndex { } return pairs; } + + public pairsByOverlap(n: number = 100): Array { + const byShared = Array.from(this.files.values()); + byShared.sort((a, b) => a.shared.size - b.shared.size); + + let j = 0, k = 1; + const pairs = []; + while (pairs.length < n && k < byShared.length) { + pairs.push(new Pair(byShared[j], byShared[k])); + j += 1; + if (j === k) { + k += 1; + j = 0; + } + } + return pairs; + } } diff --git a/core/src/util/region.ts b/core/src/util/region.ts index 4e60f3592..9ff53b1b7 100644 --- a/core/src/util/region.ts +++ b/core/src/util/region.ts @@ -59,6 +59,26 @@ export class Region { return new Region(startRow, startCol, endRow, endCol); } + public static toUInt16(array: Region[]): number[] { + const out = []; + for (let i = 0; i < array.length; i++) { + const idx = i * 4; + out[idx] = array[i].startRow; + out[idx + 1] = array[i].startCol; + out[idx + 2] = array[i].endRow; + out[idx + 3] = array[i].endCol; + } + return out; + } + + public static fromUInt16(array: Uint16Array): Region[] { + const out = []; + for (let i = 0; i < array.length; i += 4) { + out.push(new Region(array[i], array[i + 1], array[i + 2], array[i + 3])); + } + return out; + } + constructor( public startRow: number, public startCol: number, diff --git a/lib/package.nix b/lib/package.nix new file mode 100644 index 000000000..02ee65b57 --- /dev/null +++ b/lib/package.nix @@ -0,0 +1,10 @@ +{ lib +, mkYarnPackage +, nodejs +, pkg-config +}: +let + package-json = lib.importJSON ./package.json; +in mkYarnPackage { + inherit (package-json) name version; +} diff --git a/lib/src/lib/dolos.ts b/lib/src/lib/dolos.ts index 015b9b824..54569f984 100644 --- a/lib/src/lib/dolos.ts +++ b/lib/src/lib/dolos.ts @@ -19,7 +19,7 @@ export class Dolos { private languageDetected = false; private language: Language | null = null; private tokenizer: Tokenizer | null = null; - private index: FingerprintIndex | null = null; + public index: FingerprintIndex | null = null; private readonly languagePicker = new LanguagePicker(); diff --git a/lib/src/test/index.test.ts b/lib/src/test/index.test.ts new file mode 100644 index 000000000..6607c727b --- /dev/null +++ b/lib/src/test/index.test.ts @@ -0,0 +1,16 @@ +import test from "ava"; +import { Dolos } from "../lib/dolos.js"; + +test("rank by total overlap", async t => { + const dolos = new Dolos(); + const report = await dolos.analyzePaths(["../cli/exercise - Pyramidal constants/info.csv"]); + t.is(report.files.length, 392); + const index = dolos.index!; + + const ranked = index.pairsByOverlap(); + for (let i = 0; i < ranked.length; i++) { + t.true(ranked[i].overlap >= ranked[i + 1].overlap, `Fail for pair ${i}: ${ranked[i].overlap} < ${ranked[i + 1].overlap}`); + } + + +}); diff --git a/package-lock.json b/package-lock.json index 4679bd7d6..3fa7685bf 100644 --- a/package-lock.json +++ b/package-lock.json @@ -21,10 +21,12 @@ "dependencies": { "@dodona/dolos-lib": "3.3.1", "@dodona/dolos-web": "2.7.1", + "apache-arrow": "^17.0.0", "chalk": "^5.3.0", "cliui": "^8.0.1", "commander": "^12.1.0", "csv-stringify": "^6.5.0", + "duckdb-async": "^0.9.1", "open": "^10.1.0", "tree-sitter": "^0.21.1" }, @@ -42,6 +44,30 @@ "node": ">=18" } }, + "cli/node_modules/duckdb": { + "version": "0.9.2", + "resolved": "https://registry.npmjs.org/duckdb/-/duckdb-0.9.2.tgz", + "integrity": "sha512-POZ37Vf5cHVmk4W8z0PsdGi67VeQsr9HRrBbmivDt9AQ8C1XLESVE79facydbroNiqm7+n7fx6jqjyxyrBFYlQ==", + "hasInstallScript": true, + "dependencies": { + "@mapbox/node-pre-gyp": "^1.0.0", + "node-addon-api": "^7.0.0", + "node-gyp": "^9.3.0" + } + }, + "cli/node_modules/duckdb-async": { + "version": "0.9.2", + "resolved": "https://registry.npmjs.org/duckdb-async/-/duckdb-async-0.9.2.tgz", + "integrity": "sha512-o33zBds2asp20/Mcw4ZJ1G9InD1TnOJ0U3XyIElLzim4sjlmgHc4R8NnLxdqZa/X9bjhzn364cA0XZ2fEKHPfw==", + "dependencies": { + "duckdb": "0.9.2" + } + }, + "cli/node_modules/node-addon-api": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-7.1.1.tgz", + "integrity": "sha512-5m3bsyrjFWE1xf7nz7YXdN4udnVtXK6/Yfgn5qnahL6bCkf2yKt4k3nuTKAtT4r3IG8JNR2ncsIMdZuAzJjHQQ==" + }, "core": { "name": "@dodona/dolos-core", "version": "1.1.0", @@ -81,6 +107,26 @@ "node": ">=18" } }, + "node_modules/@75lb/deep-merge": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@75lb/deep-merge/-/deep-merge-1.1.1.tgz", + "integrity": "sha512-xvgv6pkMGBA6GwdyJbNAnDmfAIR/DfWhrj9jgWh3TY7gRm3KO46x/GPjRg6wJ0nOepwqrNxFfojebh0Df4h4Tw==", + "dependencies": { + "lodash.assignwith": "^4.2.0", + "typical": "^7.1.1" + }, + "engines": { + "node": ">=12.17" + } + }, + "node_modules/@75lb/deep-merge/node_modules/typical": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/typical/-/typical-7.1.1.tgz", + "integrity": "sha512-T+tKVNs6Wu7IWiAce5BgMd7OZfNYUndHwc5MknN+UHOudi7sGZzuHdCadllRuqJ3fPtgFtIH9+lt9qRv6lmpfA==", + "engines": { + "node": ">=12.17" + } + }, "node_modules/@antfu/utils": { "version": "0.7.8", "resolved": "https://registry.npmjs.org/@antfu/utils/-/utils-0.7.8.tgz", @@ -172,6 +218,93 @@ "resolved": "web", "link": true }, + "node_modules/@duckdb/duckdb-wasm": { + "version": "1.28.1-dev99.0", + "resolved": "https://registry.npmjs.org/@duckdb/duckdb-wasm/-/duckdb-wasm-1.28.1-dev99.0.tgz", + "integrity": "sha512-EgIRjUFAos4oT4QeFLK6g7IDPWFWu8dP/mj2/b0pyiN5dDL+xxKW2KyLbeAs26L6Y6yeYy0O4OJSezFOqTbOmg==", + "dev": true, + "dependencies": { + "apache-arrow": "^14.0.1" + } + }, + "node_modules/@duckdb/duckdb-wasm/node_modules/@types/command-line-args": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@types/command-line-args/-/command-line-args-5.2.0.tgz", + "integrity": "sha512-UuKzKpJJ/Ief6ufIaIzr3A/0XnluX7RvFgwkV89Yzvm77wCh1kFaFmqN8XEnGcN62EuHdedQjEMb8mYxFLGPyA==", + "dev": true + }, + "node_modules/@duckdb/duckdb-wasm/node_modules/@types/command-line-usage": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/@types/command-line-usage/-/command-line-usage-5.0.2.tgz", + "integrity": "sha512-n7RlEEJ+4x4TS7ZQddTmNSxP+zziEG0TNsMfiRIxcIVXt71ENJ9ojeXmGO3wPoTdn7pJcU2xc3CJYMktNT6DPg==", + "dev": true + }, + "node_modules/@duckdb/duckdb-wasm/node_modules/@types/node": { + "version": "20.3.0", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.3.0.tgz", + "integrity": "sha512-cumHmIAf6On83X7yP+LrsEyUOf/YlociZelmpRYaGFydoaPdxdt80MAbu6vWerQT2COCp2nPvHdsbD7tHn/YlQ==", + "dev": true + }, + "node_modules/@duckdb/duckdb-wasm/node_modules/apache-arrow": { + "version": "14.0.2", + "resolved": "https://registry.npmjs.org/apache-arrow/-/apache-arrow-14.0.2.tgz", + "integrity": "sha512-EBO2xJN36/XoY81nhLcwCJgFwkboDZeyNQ+OPsG7bCoQjc2BT0aTyH/MR6SrL+LirSNz+cYqjGRlupMMlP1aEg==", + "dev": true, + "dependencies": { + "@types/command-line-args": "5.2.0", + "@types/command-line-usage": "5.0.2", + "@types/node": "20.3.0", + "@types/pad-left": "2.1.1", + "command-line-args": "5.2.1", + "command-line-usage": "7.0.1", + "flatbuffers": "23.5.26", + "json-bignum": "^0.0.3", + "pad-left": "^2.1.0", + "tslib": "^2.5.3" + }, + "bin": { + "arrow2csv": "bin/arrow2csv.js" + } + }, + "node_modules/@duckdb/duckdb-wasm/node_modules/array-back": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/array-back/-/array-back-6.2.2.tgz", + "integrity": "sha512-gUAZ7HPyb4SJczXAMUXMGAvI976JoK3qEx9v1FTmeYuJj0IBiaKttG1ydtGKdkfqWkIkouke7nG8ufGy77+Cvw==", + "dev": true, + "engines": { + "node": ">=12.17" + } + }, + "node_modules/@duckdb/duckdb-wasm/node_modules/command-line-usage": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/command-line-usage/-/command-line-usage-7.0.1.tgz", + "integrity": "sha512-NCyznE//MuTjwi3y84QVUGEOT+P5oto1e1Pk/jFPVdPPfsG03qpTIl3yw6etR+v73d0lXsoojRpvbru2sqePxQ==", + "dev": true, + "dependencies": { + "array-back": "^6.2.2", + "chalk-template": "^0.4.0", + "table-layout": "^3.0.0", + "typical": "^7.1.1" + }, + "engines": { + "node": ">=12.20.0" + } + }, + "node_modules/@duckdb/duckdb-wasm/node_modules/flatbuffers": { + "version": "23.5.26", + "resolved": "https://registry.npmjs.org/flatbuffers/-/flatbuffers-23.5.26.tgz", + "integrity": "sha512-vE+SI9vrJDwi1oETtTIFldC/o9GsVKRM+s6EL0nQgxXlYV1Vc4Tk30hj4xGICftInKQKj1F3up2n8UbIVobISQ==", + "dev": true + }, + "node_modules/@duckdb/duckdb-wasm/node_modules/typical": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/typical/-/typical-7.1.1.tgz", + "integrity": "sha512-T+tKVNs6Wu7IWiAce5BgMd7OZfNYUndHwc5MknN+UHOudi7sGZzuHdCadllRuqJ3fPtgFtIH9+lt9qRv6lmpfA==", + "dev": true, + "engines": { + "node": ">=12.17" + } + }, "node_modules/@esbuild/android-arm": { "version": "0.18.20", "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.18.20.tgz", @@ -610,7 +743,6 @@ "version": "1.1.3", "resolved": "https://registry.npmjs.org/@gar/promisify/-/promisify-1.1.3.tgz", "integrity": "sha512-k2Ty1JcVojjJFwrg/ThKi2ujJ7XNLYaFGNB/bWT9wGR+oSMJHMa5w+CUq6p/pVrKeNNgA7pCqEcjSnHVoqJQFw==", - "dev": true, "license": "MIT" }, "node_modules/@humanwhocodes/config-array": { @@ -658,6 +790,129 @@ "dev": true, "license": "MIT" }, + "node_modules/@mapbox/node-pre-gyp": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/@mapbox/node-pre-gyp/-/node-pre-gyp-1.0.11.tgz", + "integrity": "sha512-Yhlar6v9WQgUp/He7BdgzOz8lqMQ8sU+jkCq7Wx8Myc5YFJLbEe7lgui/V7G1qB1DJykHSGwreceSaD60Y0PUQ==", + "dependencies": { + "detect-libc": "^2.0.0", + "https-proxy-agent": "^5.0.0", + "make-dir": "^3.1.0", + "node-fetch": "^2.6.7", + "nopt": "^5.0.0", + "npmlog": "^5.0.1", + "rimraf": "^3.0.2", + "semver": "^7.3.5", + "tar": "^6.1.11" + }, + "bin": { + "node-pre-gyp": "bin/node-pre-gyp" + } + }, + "node_modules/@mapbox/node-pre-gyp/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "engines": { + "node": ">=8" + } + }, + "node_modules/@mapbox/node-pre-gyp/node_modules/are-we-there-yet": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-2.0.0.tgz", + "integrity": "sha512-Ci/qENmwHnsYo9xKIcUJN5LeDKdJ6R1Z1j9V/J5wyq8nh/mYPEpIKJbBZXtZjG04HiK7zV/p6Vs9952MrMeUIw==", + "deprecated": "This package is no longer supported.", + "dependencies": { + "delegates": "^1.0.0", + "readable-stream": "^3.6.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/@mapbox/node-pre-gyp/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" + }, + "node_modules/@mapbox/node-pre-gyp/node_modules/gauge": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/gauge/-/gauge-3.0.2.tgz", + "integrity": "sha512-+5J6MS/5XksCuXq++uFRsnUd7Ovu1XenbeuIuNRJxYWjgQbPuFhT14lAvsWfqfAmnwluf1OwMjz39HjfLPci0Q==", + "deprecated": "This package is no longer supported.", + "dependencies": { + "aproba": "^1.0.3 || ^2.0.0", + "color-support": "^1.1.2", + "console-control-strings": "^1.0.0", + "has-unicode": "^2.0.1", + "object-assign": "^4.1.1", + "signal-exit": "^3.0.0", + "string-width": "^4.2.3", + "strip-ansi": "^6.0.1", + "wide-align": "^1.1.2" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/@mapbox/node-pre-gyp/node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "engines": { + "node": ">=8" + } + }, + "node_modules/@mapbox/node-pre-gyp/node_modules/nopt": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/nopt/-/nopt-5.0.0.tgz", + "integrity": "sha512-Tbj67rffqceeLpcRXrT7vKAN8CwfPeIBgM7E6iBkmKLV7bEMwpGgYLGv0jACUsECaa/vuxP0IjEont6umdMgtQ==", + "dependencies": { + "abbrev": "1" + }, + "bin": { + "nopt": "bin/nopt.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/@mapbox/node-pre-gyp/node_modules/npmlog": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-5.0.1.tgz", + "integrity": "sha512-AqZtDUWOMKs1G/8lwylVjrdYgqA4d9nu8hc+0gzRxlDb1I10+FHBGMXs6aiQHFdCUUlqH99MUMuLfzWDNDtfxw==", + "deprecated": "This package is no longer supported.", + "dependencies": { + "are-we-there-yet": "^2.0.0", + "console-control-strings": "^1.1.0", + "gauge": "^3.0.0", + "set-blocking": "^2.0.0" + } + }, + "node_modules/@mapbox/node-pre-gyp/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/@mapbox/node-pre-gyp/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/@mdi/font": { "version": "7.4.47", "resolved": "https://registry.npmjs.org/@mdi/font/-/font-7.4.47.tgz", @@ -707,7 +962,6 @@ "version": "2.1.2", "resolved": "https://registry.npmjs.org/@npmcli/fs/-/fs-2.1.2.tgz", "integrity": "sha512-yOJKRvohFOaLqipNtwYB9WugyZKhC/DZC4VYPmpaCzDBrA8YpK3qHZ8/HGscMnE4GqbkLNuVcCnxkeQEdGt6LQ==", - "dev": true, "license": "ISC", "dependencies": { "@gar/promisify": "^1.1.3", @@ -722,7 +976,6 @@ "resolved": "https://registry.npmjs.org/@npmcli/move-file/-/move-file-2.0.1.tgz", "integrity": "sha512-mJd2Z5TjYWq/ttPLLGqArdtnC74J6bOzg4rMDnN+p1xTacZ2yPRCk2y0oSWQtygLR9YVQXgOcONrwtnk3JupxQ==", "deprecated": "This functionality has been moved to @npmcli/fs", - "dev": true, "license": "MIT", "dependencies": { "mkdirp": "^1.0.4", @@ -755,16 +1008,33 @@ } } }, + "node_modules/@swc/helpers": { + "version": "0.5.12", + "resolved": "https://registry.npmjs.org/@swc/helpers/-/helpers-0.5.12.tgz", + "integrity": "sha512-KMZNXiGibsW9kvZAO1Pam2JPTDBm+KSHMMHWdsyI/1DbIZjT2A6Gy3hblVXUMEDvUAKq+e0vL0X0o54owWji7g==", + "dependencies": { + "tslib": "^2.4.0" + } + }, "node_modules/@tootallnate/once": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-2.0.0.tgz", "integrity": "sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A==", - "dev": true, "license": "MIT", "engines": { "node": ">= 10" } }, + "node_modules/@types/command-line-args": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/@types/command-line-args/-/command-line-args-5.2.3.tgz", + "integrity": "sha512-uv0aG6R0Y8WHZLTamZwtfsDLVRnOa+n+n5rEvFWL5Na5gZ8V2Teab/duDPFzIIIhs9qizDpcavCusCLJZu62Kw==" + }, + "node_modules/@types/command-line-usage": { + "version": "5.0.4", + "resolved": "https://registry.npmjs.org/@types/command-line-usage/-/command-line-usage-5.0.4.tgz", + "integrity": "sha512-BwR5KP3Es/CSht0xqBcUXS3qCAUVXwpRKsV2+arxeb65atasuXG9LykC9Ab10Cw3s2raH92ZqOeILaQbsB2ACg==" + }, "node_modules/@types/d3": { "version": "7.4.3", "resolved": "https://registry.npmjs.org/@types/d3/-/d3-7.4.3.tgz", @@ -1081,12 +1351,17 @@ "version": "20.14.8", "resolved": "https://registry.npmjs.org/@types/node/-/node-20.14.8.tgz", "integrity": "sha512-DO+2/jZinXfROG7j7WKFn/3C6nFwxy2lLpgLjEXJz+0XKphZlTLJ14mo8Vfg8X5BWN6XjyESXq+LcYdT7tR3bA==", - "dev": true, "license": "MIT", "dependencies": { "undici-types": "~5.26.4" } }, + "node_modules/@types/pad-left": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/@types/pad-left/-/pad-left-2.1.1.tgz", + "integrity": "sha512-Xd22WCRBydkGSApl5Bw0PhAOHKSVjNL3E3AwzKaps96IMraPqy5BvZIsBVK6JLwdybUzjHnuWVwpDd0JjTfHXA==", + "dev": true + }, "node_modules/@types/papaparse": { "version": "5.3.14", "resolved": "https://registry.npmjs.org/@types/papaparse/-/papaparse-5.3.14.tgz", @@ -1633,7 +1908,6 @@ "version": "1.1.1", "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==", - "dev": true, "license": "ISC" }, "node_modules/acorn": { @@ -1676,7 +1950,6 @@ "version": "6.0.2", "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", - "dev": true, "license": "MIT", "dependencies": { "debug": "4" @@ -1689,7 +1962,6 @@ "version": "4.5.0", "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.5.0.tgz", "integrity": "sha512-5GG/5IbQQpC9FpkRGsSvZI5QYeSCzlJHdpBQntCsuTOxhKD8lqKhrleg2Yi7yvMIf82Ycmmqln9U8V9qwEiJew==", - "dev": true, "license": "MIT", "dependencies": { "humanize-ms": "^1.2.1" @@ -1772,11 +2044,29 @@ "node": ">= 8" } }, + "node_modules/apache-arrow": { + "version": "17.0.0", + "resolved": "https://registry.npmjs.org/apache-arrow/-/apache-arrow-17.0.0.tgz", + "integrity": "sha512-X0p7auzdnGuhYMVKYINdQssS4EcKec9TCXyez/qtJt32DrIMGbzqiaMiQ0X6fQlQpw8Fl0Qygcv4dfRAr5Gu9Q==", + "dependencies": { + "@swc/helpers": "^0.5.11", + "@types/command-line-args": "^5.2.3", + "@types/command-line-usage": "^5.0.4", + "@types/node": "^20.13.0", + "command-line-args": "^5.2.1", + "command-line-usage": "^7.0.1", + "flatbuffers": "^24.3.25", + "json-bignum": "^0.0.3", + "tslib": "^2.6.2" + }, + "bin": { + "arrow2csv": "bin/arrow2csv.cjs" + } + }, "node_modules/aproba": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/aproba/-/aproba-2.0.0.tgz", "integrity": "sha512-lYe4Gx7QT+MKGbDsA+Z+he/Wtef0BiwDOlK/XkBrdfsh9J/jPPXbX0tE9x9cl27Tmu5gg3QUbUrQYa/y+KOHPQ==", - "dev": true, "license": "ISC" }, "node_modules/are-we-there-yet": { @@ -1784,7 +2074,6 @@ "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-3.0.1.tgz", "integrity": "sha512-QZW4EDmGwlYur0Yyf/b2uGucHQMa8aFUP7eu9ddR73vvhFyt4V0Vl3QHPcTNJ8l6qYOBdxgXdnBXQrHilfRQBg==", "deprecated": "This package is no longer supported.", - "dev": true, "license": "ISC", "dependencies": { "delegates": "^1.0.0", @@ -1801,6 +2090,14 @@ "dev": true, "license": "Python-2.0" }, + "node_modules/array-back": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/array-back/-/array-back-3.1.0.tgz", + "integrity": "sha512-TkuxA4UCOvxuDK6NZYXCalszEzj+TLszyASooky+i742l9TqsOdYCMJJupxRic61hwquNtppB3hgcuq9SVSH1Q==", + "engines": { + "node": ">=6" + } + }, "node_modules/array-find-index": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/array-find-index/-/array-find-index-1.0.2.tgz", @@ -1966,7 +2263,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", - "dev": true, "license": "MIT" }, "node_modules/base64-js": { @@ -2040,7 +2336,6 @@ "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", - "dev": true, "license": "MIT", "dependencies": { "balanced-match": "^1.0.0", @@ -2103,7 +2398,6 @@ "version": "16.1.3", "resolved": "https://registry.npmjs.org/cacache/-/cacache-16.1.3.tgz", "integrity": "sha512-/+Emcj9DAXxX4cwlLmRI9c166RuL3w30zp4R7Joiv2cQTtTtA+jeuCAjH3ZlGnYS3tKENSrKhAzVVP9GVyzeYQ==", - "dev": true, "license": "ISC", "dependencies": { "@npmcli/fs": "^2.1.0", @@ -2133,7 +2427,6 @@ "version": "3.1.0", "resolved": "https://registry.npmjs.org/aggregate-error/-/aggregate-error-3.1.0.tgz", "integrity": "sha512-4I7Td01quW/RpocfNayFdFVk1qSuoh0E7JrbRJ16nH01HhKFQ88INq9Sd+nd72zqRySlr9BmDA8xlEJ6vJMrYA==", - "dev": true, "license": "MIT", "dependencies": { "clean-stack": "^2.0.0", @@ -2147,7 +2440,6 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", - "dev": true, "license": "MIT", "dependencies": { "balanced-match": "^1.0.0" @@ -2157,7 +2449,6 @@ "version": "2.2.0", "resolved": "https://registry.npmjs.org/clean-stack/-/clean-stack-2.2.0.tgz", "integrity": "sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A==", - "dev": true, "license": "MIT", "engines": { "node": ">=6" @@ -2168,7 +2459,6 @@ "resolved": "https://registry.npmjs.org/glob/-/glob-8.1.0.tgz", "integrity": "sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ==", "deprecated": "Glob versions prior to v9 are no longer supported", - "dev": true, "license": "ISC", "dependencies": { "fs.realpath": "^1.0.0", @@ -2188,7 +2478,6 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-4.0.0.tgz", "integrity": "sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg==", - "dev": true, "license": "MIT", "engines": { "node": ">=8" @@ -2198,7 +2487,6 @@ "version": "5.1.6", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz", "integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==", - "dev": true, "license": "ISC", "dependencies": { "brace-expansion": "^2.0.1" @@ -2211,7 +2499,6 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/p-map/-/p-map-4.0.0.tgz", "integrity": "sha512-/bjOqmgETBYB5BoEeGVea8dmvHb2m9GLy1E9W43yeyfP6QQCZGFNa+XRceJEuDB6zqr+gKpIAmlLebMpykw/MQ==", - "dev": true, "license": "MIT", "dependencies": { "aggregate-error": "^3.0.0" @@ -2261,6 +2548,49 @@ "url": "https://github.com/chalk/chalk?sponsor=1" } }, + "node_modules/chalk-template": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/chalk-template/-/chalk-template-0.4.0.tgz", + "integrity": "sha512-/ghrgmhfY8RaSdeo43hNXxpoHAtxdbskUHjPpfqUWGttFgycUhYPGx3YZBCnUCvOa7Doivn1IZec3DEGFoMgLg==", + "dependencies": { + "chalk": "^4.1.2" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/chalk-template?sponsor=1" + } + }, + "node_modules/chalk-template/node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/chalk-template/node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, "node_modules/chokidar": { "version": "3.6.0", "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.6.0.tgz", @@ -2290,7 +2620,6 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/chownr/-/chownr-2.0.0.tgz", "integrity": "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==", - "dev": true, "license": "ISC", "engines": { "node": ">=10" @@ -2468,7 +2797,6 @@ "version": "1.1.3", "resolved": "https://registry.npmjs.org/color-support/-/color-support-1.1.3.tgz", "integrity": "sha512-qiBjkpbMLO/HL68y+lh4q0/O1MZFj2RX6X/KmMa3+gJD3z+WwI1ZzDHysvqHGS3mP6mznPckpXmw1nI9cJjyRg==", - "dev": true, "license": "ISC", "bin": { "color-support": "bin.js" @@ -2494,6 +2822,50 @@ "dev": true, "license": "Apache-2.0" }, + "node_modules/command-line-args": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/command-line-args/-/command-line-args-5.2.1.tgz", + "integrity": "sha512-H4UfQhZyakIjC74I9d34fGYDwk3XpSr17QhEd0Q3I9Xq1CETHo4Hcuo87WyWHpAF1aSLjLRf5lD9ZGX2qStUvg==", + "dependencies": { + "array-back": "^3.1.0", + "find-replace": "^3.0.0", + "lodash.camelcase": "^4.3.0", + "typical": "^4.0.0" + }, + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/command-line-usage": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/command-line-usage/-/command-line-usage-7.0.2.tgz", + "integrity": "sha512-MwNFB8nxi3IVnzir+nkSIbDTU4H6ne26zqicO2eTt1wPrvdOAphPhnYqWOjxXKWYLNYDu4Z/r2ESEziEqKuOVg==", + "dependencies": { + "array-back": "^6.2.2", + "chalk-template": "^0.4.0", + "table-layout": "^3.0.2", + "typical": "^7.1.1" + }, + "engines": { + "node": ">=12.20.0" + } + }, + "node_modules/command-line-usage/node_modules/array-back": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/array-back/-/array-back-6.2.2.tgz", + "integrity": "sha512-gUAZ7HPyb4SJczXAMUXMGAvI976JoK3qEx9v1FTmeYuJj0IBiaKttG1ydtGKdkfqWkIkouke7nG8ufGy77+Cvw==", + "engines": { + "node": ">=12.17" + } + }, + "node_modules/command-line-usage/node_modules/typical": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/typical/-/typical-7.1.1.tgz", + "integrity": "sha512-T+tKVNs6Wu7IWiAce5BgMd7OZfNYUndHwc5MknN+UHOudi7sGZzuHdCadllRuqJ3fPtgFtIH9+lt9qRv6lmpfA==", + "engines": { + "node": ">=12.17" + } + }, "node_modules/commander": { "version": "12.1.0", "resolved": "https://registry.npmjs.org/commander/-/commander-12.1.0.tgz", @@ -2527,7 +2899,6 @@ "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", - "dev": true, "license": "MIT" }, "node_modules/concordance": { @@ -2561,7 +2932,6 @@ "version": "1.1.0", "resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz", "integrity": "sha512-ty/fTekppD2fIwRvnZAVdeOiGd1c7YXEixbgJTNzqcxJWKQnjJ/V1bNEEE6hygpM3WjwHFUVK6HTjWSzV4a8sQ==", - "dev": true, "license": "ISC" }, "node_modules/convert-to-spaces": { @@ -3103,7 +3473,6 @@ "version": "4.3.5", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz", "integrity": "sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==", - "dev": true, "license": "MIT", "dependencies": { "ms": "2.1.2" @@ -3121,7 +3490,6 @@ "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", - "dev": true, "license": "MIT" }, "node_modules/decompress-response": { @@ -3219,7 +3587,6 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz", "integrity": "sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ==", - "dev": true, "license": "MIT" }, "node_modules/detect-libc": { @@ -3288,7 +3655,6 @@ "version": "0.1.13", "resolved": "https://registry.npmjs.org/encoding/-/encoding-0.1.13.tgz", "integrity": "sha512-ETBauow1T35Y/WZMkio9jiM0Z5xjHHmJ4XmjZOq1l/dXz3lr2sRn87nJy20RupqSh1F2m3HHPSp8ShIPQJrJ3A==", - "dev": true, "license": "MIT", "optional": true, "dependencies": { @@ -3321,7 +3687,6 @@ "version": "2.2.1", "resolved": "https://registry.npmjs.org/env-paths/-/env-paths-2.2.1.tgz", "integrity": "sha512-+h1lkLKhZMTYjog1VEpJNG7NZJWcuc2DDk/qsqSTRRCOXiLjeQ1d1/udrUGhqMxUgAlwKNZ0cf2uqan5GLuS2A==", - "dev": true, "license": "MIT", "engines": { "node": ">=6" @@ -3331,7 +3696,6 @@ "version": "2.0.3", "resolved": "https://registry.npmjs.org/err-code/-/err-code-2.0.3.tgz", "integrity": "sha512-2bmlRpNKBxT/CRmPOlyISQpNj+qSeYvcym/uT0Jx2bMOlKLtSy1ZmLuVxSEKKyor/N5yhvp/ZiG1oE3DEYMSFA==", - "dev": true, "license": "MIT" }, "node_modules/esbuild": { @@ -3723,7 +4087,6 @@ "version": "3.1.1", "resolved": "https://registry.npmjs.org/exponential-backoff/-/exponential-backoff-3.1.1.tgz", "integrity": "sha512-dX7e/LHVJ6W3DE1MHWi9S1EYzDESENfLrYohG2G++ovZrYOkm4Knwa0mc1cn84xJOR4KEU0WSchhLbd0UklbHw==", - "dev": true, "license": "Apache-2.0" }, "node_modules/fast-deep-equal": { @@ -3848,6 +4211,17 @@ "url": "https://github.com/avajs/find-cache-dir?sponsor=1" } }, + "node_modules/find-replace": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/find-replace/-/find-replace-3.0.0.tgz", + "integrity": "sha512-6Tb2myMioCAgv5kfvP5/PkZZ/ntTpVK39fHY7WkWBgvbeE+VHd/tZuZ4mrC+bxh4cfOZeYKVPaJIZtZXV7GNCQ==", + "dependencies": { + "array-back": "^3.0.1" + }, + "engines": { + "node": ">=4.0.0" + } + }, "node_modules/find-up": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", @@ -3880,6 +4254,11 @@ "node": "^10.12.0 || >=12.0.0" } }, + "node_modules/flatbuffers": { + "version": "24.3.25", + "resolved": "https://registry.npmjs.org/flatbuffers/-/flatbuffers-24.3.25.tgz", + "integrity": "sha512-3HDgPbgiwWMI9zVB7VYBHaMrbOO7Gm0v+yD2FV/sCKj+9NDeVL7BOBYUuhWAQGKWOzBo8S9WdMvV0eixO233XQ==" + }, "node_modules/flatted": { "version": "3.3.1", "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.3.1.tgz", @@ -3933,7 +4312,6 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz", "integrity": "sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==", - "dev": true, "license": "ISC", "dependencies": { "minipass": "^3.0.0" @@ -3946,7 +4324,6 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", - "dev": true, "license": "ISC" }, "node_modules/fsevents": { @@ -3979,7 +4356,6 @@ "resolved": "https://registry.npmjs.org/gauge/-/gauge-4.0.4.tgz", "integrity": "sha512-f9m+BEN5jkg6a0fZjleidjN51VE1X+mPFQ2DJ0uv1V39oCLCbsGe6yjbBnp7eK7z/+GAon99a3nHuqbuuthyPg==", "deprecated": "This package is no longer supported.", - "dev": true, "license": "ISC", "dependencies": { "aproba": "^1.0.3 || ^2.0.0", @@ -3999,7 +4375,6 @@ "version": "5.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "dev": true, "license": "MIT", "engines": { "node": ">=8" @@ -4009,14 +4384,12 @@ "version": "8.0.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", - "dev": true, "license": "MIT" }, "node_modules/gauge/node_modules/is-fullwidth-code-point": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", - "dev": true, "license": "MIT", "engines": { "node": ">=8" @@ -4026,7 +4399,6 @@ "version": "4.2.3", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "dev": true, "license": "MIT", "dependencies": { "emoji-regex": "^8.0.0", @@ -4041,7 +4413,6 @@ "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dev": true, "license": "MIT", "dependencies": { "ansi-regex": "^5.0.1" @@ -4084,7 +4455,6 @@ "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", "deprecated": "Glob versions prior to v9 are no longer supported", - "dev": true, "license": "ISC", "dependencies": { "fs.realpath": "^1.0.0", @@ -4155,7 +4525,6 @@ "version": "4.2.11", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", - "dev": true, "license": "ISC" }, "node_modules/graphemer": { @@ -4169,7 +4538,6 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", - "dev": true, "license": "MIT", "engines": { "node": ">=8" @@ -4179,7 +4547,6 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz", "integrity": "sha512-8Rf9Y83NBReMnx0gFzA8JImQACstCYWUplepDa9xprwwtmgEZUF0h/i5xSA625zB/I37EtrswSST6OXxwaaIJQ==", - "dev": true, "license": "ISC" }, "node_modules/hasown": { @@ -4208,14 +4575,12 @@ "version": "4.1.1", "resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.1.1.tgz", "integrity": "sha512-er295DKPVsV82j5kw1Gjt+ADA/XYHsajl82cGNQG2eyoPkvgUhX+nDIyelzhIWbbsXP39EHcI6l5tYs2FYqYXQ==", - "dev": true, "license": "BSD-2-Clause" }, "node_modules/http-proxy-agent": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-5.0.0.tgz", "integrity": "sha512-n2hY8YdoRE1i7r6M0w9DIw5GgZN0G25P8zLCRQ8rjXtTU3vsNFBI/vWK/UIeE6g5MUUz6avwAPXmL6Fy9D/90w==", - "dev": true, "license": "MIT", "dependencies": { "@tootallnate/once": "2", @@ -4230,7 +4595,6 @@ "version": "5.0.1", "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", - "dev": true, "license": "MIT", "dependencies": { "agent-base": "6", @@ -4254,7 +4618,6 @@ "version": "1.2.1", "resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz", "integrity": "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==", - "dev": true, "license": "MIT", "dependencies": { "ms": "^2.0.0" @@ -4340,7 +4703,6 @@ "version": "0.1.4", "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", - "dev": true, "license": "MIT", "engines": { "node": ">=0.8.19" @@ -4363,7 +4725,6 @@ "version": "1.0.4", "resolved": "https://registry.npmjs.org/infer-owner/-/infer-owner-1.0.4.tgz", "integrity": "sha512-IClj+Xz94+d7irH5qRyfJonOdfTzuDaifE6ZPWfx0N0+/ATZCbuTPq2prFl526urkQd90WyUKIh1DfBQ2hMz9A==", - "dev": true, "license": "ISC" }, "node_modules/inflight": { @@ -4371,7 +4732,6 @@ "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", "deprecated": "This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful.", - "dev": true, "license": "ISC", "dependencies": { "once": "^1.3.0", @@ -4404,7 +4764,6 @@ "version": "9.0.5", "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-9.0.5.tgz", "integrity": "sha512-zHtQzGojZXTwZTHQqra+ETKd4Sn3vgi7uBmlPoXVWZqYvuKmtI0l/VZTjqGmJY9x88GGOaZ9+G9ES8hC4T4X8g==", - "dev": true, "license": "MIT", "dependencies": { "jsbn": "1.1.0", @@ -4533,7 +4892,6 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/is-lambda/-/is-lambda-1.0.1.tgz", "integrity": "sha512-z7CMFGNrENq5iFB9Bqo64Xk6Y9sg+epq1myIcdHaGnbMTYOxvzsEtdYqQUylB7LxfkvgrrjP32T6Ywciio9UIQ==", - "dev": true, "license": "MIT" }, "node_modules/is-number": { @@ -4618,7 +4976,6 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", - "dev": true, "license": "ISC" }, "node_modules/js-string-escape": { @@ -4648,9 +5005,16 @@ "version": "1.1.0", "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-1.1.0.tgz", "integrity": "sha512-4bYVV3aAMtDTTu4+xsDYa6sy9GyJ69/amsu9sYF2zqjiEoZA5xJi3BrfX3uY+/IekIu7MwdObdbDWpoZdBv3/A==", - "dev": true, "license": "MIT" }, + "node_modules/json-bignum": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/json-bignum/-/json-bignum-0.0.3.tgz", + "integrity": "sha512-2WHyXj3OfHSgNyuzDbSxI1w2jgw5gkWSWhS7Qg4bWXx1nLk3jnbwfUeS0PSba3IzpTUWdHxBieELUzXRjQB2zg==", + "engines": { + "node": ">=0.8" + } + }, "node_modules/json-buffer": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", @@ -4762,6 +5126,16 @@ "dev": true, "license": "MIT" }, + "node_modules/lodash.assignwith": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/lodash.assignwith/-/lodash.assignwith-4.2.0.tgz", + "integrity": "sha512-ZznplvbvtjK2gMvnQ1BR/zqPFZmS6jbK4p+6Up4xcRYA7yMIwxHCfbTcrYxXKzzqLsQ05eJPVznEW3tuwV7k1g==" + }, + "node_modules/lodash.camelcase": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz", + "integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==" + }, "node_modules/lodash.merge": { "version": "4.6.2", "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", @@ -4773,7 +5147,6 @@ "version": "7.18.3", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-7.18.3.tgz", "integrity": "sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==", - "dev": true, "license": "ISC", "engines": { "node": ">=12" @@ -4803,7 +5176,6 @@ "version": "3.1.0", "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-3.1.0.tgz", "integrity": "sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw==", - "dev": true, "license": "MIT", "dependencies": { "semver": "^6.0.0" @@ -4819,7 +5191,6 @@ "version": "6.3.1", "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", - "dev": true, "license": "ISC", "bin": { "semver": "bin/semver.js" @@ -4829,7 +5200,6 @@ "version": "10.2.1", "resolved": "https://registry.npmjs.org/make-fetch-happen/-/make-fetch-happen-10.2.1.tgz", "integrity": "sha512-NgOPbRiaQM10DYXvN3/hhGVI2M5MtITFryzBGxHM5p4wnFxsVCbxkrBrDsk+EZ5OB4jEOT7AjDxtdF+KVEFT7w==", - "dev": true, "license": "ISC", "dependencies": { "agentkeepalive": "^4.2.1", @@ -4995,7 +5365,6 @@ "version": "3.1.2", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", - "dev": true, "license": "ISC", "dependencies": { "brace-expansion": "^1.1.7" @@ -5017,7 +5386,6 @@ "version": "3.3.6", "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", - "dev": true, "license": "ISC", "dependencies": { "yallist": "^4.0.0" @@ -5030,7 +5398,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/minipass-collect/-/minipass-collect-1.0.2.tgz", "integrity": "sha512-6T6lH0H8OG9kITm/Jm6tdooIbogG9e0tLgpY6mphXSm/A9u8Nq1ryBG+Qspiub9LjWlBPsPS3tWQ/Botq4FdxA==", - "dev": true, "license": "ISC", "dependencies": { "minipass": "^3.0.0" @@ -5043,7 +5410,6 @@ "version": "2.1.2", "resolved": "https://registry.npmjs.org/minipass-fetch/-/minipass-fetch-2.1.2.tgz", "integrity": "sha512-LT49Zi2/WMROHYoqGgdlQIZh8mLPZmOrN2NdJjMXxYe4nkN6FUyuPuOAOedNJDrx0IRGg9+4guZewtp8hE6TxA==", - "dev": true, "license": "MIT", "dependencies": { "minipass": "^3.1.6", @@ -5061,7 +5427,6 @@ "version": "1.0.5", "resolved": "https://registry.npmjs.org/minipass-flush/-/minipass-flush-1.0.5.tgz", "integrity": "sha512-JmQSYYpPUqX5Jyn1mXaRwOda1uQ8HP5KAT/oDSLCzt1BYRhQU0/hDtsB1ufZfEEzMZ9aAVmsBw8+FWsIXlClWw==", - "dev": true, "license": "ISC", "dependencies": { "minipass": "^3.0.0" @@ -5074,7 +5439,6 @@ "version": "1.2.4", "resolved": "https://registry.npmjs.org/minipass-pipeline/-/minipass-pipeline-1.2.4.tgz", "integrity": "sha512-xuIq7cIOt09RPRJ19gdi4b+RiNvDFYe5JH+ggNvBqGqpQXcru3PcRmOZuHBKWK1Txf9+cQ+HMVN4d6z46LZP7A==", - "dev": true, "license": "ISC", "dependencies": { "minipass": "^3.0.0" @@ -5087,7 +5451,6 @@ "version": "1.0.3", "resolved": "https://registry.npmjs.org/minipass-sized/-/minipass-sized-1.0.3.tgz", "integrity": "sha512-MbkQQ2CTiBMlA2Dm/5cY+9SWFEN8pzzOXi6rlM5Xxq0Yqbda5ZQy9sU75a673FE9ZK0Zsbr6Y5iP6u9nktfg2g==", - "dev": true, "license": "ISC", "dependencies": { "minipass": "^3.0.0" @@ -5100,7 +5463,6 @@ "version": "2.1.2", "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-2.1.2.tgz", "integrity": "sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==", - "dev": true, "license": "MIT", "dependencies": { "minipass": "^3.0.0", @@ -5114,7 +5476,6 @@ "version": "1.0.4", "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz", "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", - "dev": true, "license": "MIT", "bin": { "mkdirp": "bin/cmd.js" @@ -5153,7 +5514,6 @@ "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "dev": true, "license": "MIT" }, "node_modules/muggle-string": { @@ -5204,7 +5564,6 @@ "version": "0.6.3", "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==", - "dev": true, "license": "MIT", "engines": { "node": ">= 0.6" @@ -5231,11 +5590,29 @@ "node": "^18 || ^20 || >= 21" } }, + "node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, "node_modules/node-gyp": { "version": "9.4.1", "resolved": "https://registry.npmjs.org/node-gyp/-/node-gyp-9.4.1.tgz", "integrity": "sha512-OQkWKbjQKbGkMf/xqI1jjy3oCTgMKJac58G2+bjZb3fza6gW2YrCSdMQYaoTb70crvE//Gngr4f0AgVHmqHvBQ==", - "dev": true, "license": "MIT", "dependencies": { "env-paths": "^2.2.0", @@ -5282,7 +5659,6 @@ "version": "6.0.0", "resolved": "https://registry.npmjs.org/nopt/-/nopt-6.0.0.tgz", "integrity": "sha512-ZwLpbTgdhuZUnZzjd7nb1ZV+4DoiC6/sfiVKok72ym/4Tlf+DFdlHYmT2JPmcNNWV6Pi3SDf1kT+A4r9RTuT9g==", - "dev": true, "license": "ISC", "dependencies": { "abbrev": "^1.0.0" @@ -5338,7 +5714,6 @@ "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-6.0.2.tgz", "integrity": "sha512-/vBvz5Jfr9dT/aFWd0FIRf+T/Q2WBsLENygUaFUqstqsycmZAP/t5BvFJTK0viFmSUxiUKTUplWy5vt+rvKIxg==", "deprecated": "This package is no longer supported.", - "dev": true, "license": "ISC", "dependencies": { "are-we-there-yet": "^3.0.0", @@ -5363,6 +5738,14 @@ "url": "https://github.com/fb55/nth-check?sponsor=1" } }, + "node_modules/object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", @@ -5521,6 +5904,18 @@ "node": ">=6" } }, + "node_modules/pad-left": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/pad-left/-/pad-left-2.1.0.tgz", + "integrity": "sha512-HJxs9K9AztdIQIAIa/OIazRAUW/L6B9hbQDxO4X07roW3eo9XqZc2ur9bn1StH9CnbbI9EgvejHQX7CBpCF1QA==", + "dev": true, + "dependencies": { + "repeat-string": "^1.5.4" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/papaparse": { "version": "5.4.1", "resolved": "https://registry.npmjs.org/papaparse/-/papaparse-5.4.1.tgz", @@ -5584,7 +5979,6 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", - "dev": true, "license": "MIT", "engines": { "node": ">=0.10.0" @@ -5982,14 +6376,12 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/promise-inflight/-/promise-inflight-1.0.1.tgz", "integrity": "sha512-6zWPyEOFaQBJYcGMHBKTKJ3u6TBsnMFOIZSa6ce1e/ZrrsOlnHRHbabMjLiBYKp+n44X9eUI6VUPaukCXHuG4g==", - "dev": true, "license": "ISC" }, "node_modules/promise-retry": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/promise-retry/-/promise-retry-2.0.1.tgz", "integrity": "sha512-y+WKFlBR8BGXnsNlIHFGPZmyDf3DFMoLhaflAnyZgV6rG6xu+JwesTo2Q9R6XwYmtmwAFCkAk3e35jEdoeh/3g==", - "dev": true, "license": "MIT", "dependencies": { "err-code": "^2.0.2", @@ -6098,6 +6490,15 @@ "node": ">=8.10.0" } }, + "node_modules/repeat-string": { + "version": "1.6.1", + "resolved": "https://registry.npmjs.org/repeat-string/-/repeat-string-1.6.1.tgz", + "integrity": "sha512-PV0dzCYDNfRi1jCDbJzpW7jNNDRuCOG/jI5ctQcGKt/clZD+YcPS3yIlWuTJMmESC8aevCFmWJy5wjAFgNqN6w==", + "dev": true, + "engines": { + "node": ">=0.10" + } + }, "node_modules/require-directory": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", @@ -6173,7 +6574,6 @@ "version": "0.12.0", "resolved": "https://registry.npmjs.org/retry/-/retry-0.12.0.tgz", "integrity": "sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==", - "dev": true, "license": "MIT", "engines": { "node": ">= 4" @@ -6195,7 +6595,6 @@ "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", "deprecated": "Rimraf versions prior to v4 are no longer supported", - "dev": true, "license": "ISC", "dependencies": { "glob": "^7.1.3" @@ -6369,7 +6768,6 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", "integrity": "sha512-KiKBS8AnWGEyLzofFfmvKwpdPzqiy16LvQfK3yv/fVH7Bj13/wl3JSR1J+rfgRE9q7xUJK4qvgS8raSOeLUehw==", - "dev": true, "license": "ISC" }, "node_modules/shebang-command": { @@ -6399,7 +6797,6 @@ "version": "3.0.7", "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", - "dev": true, "license": "ISC" }, "node_modules/simple-concat": { @@ -6488,7 +6885,6 @@ "version": "4.2.0", "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz", "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==", - "dev": true, "license": "MIT", "engines": { "node": ">= 6.0.0", @@ -6499,7 +6895,6 @@ "version": "2.8.3", "resolved": "https://registry.npmjs.org/socks/-/socks-2.8.3.tgz", "integrity": "sha512-l5x7VUUWbjVFbafGLxPWkYsHIhEvmF85tbIeFZWc8ZPtoMyybuEhL7Jye/ooC4/d48FgOjSJXgsF/AJPYCW8Zw==", - "dev": true, "license": "MIT", "dependencies": { "ip-address": "^9.0.5", @@ -6514,7 +6909,6 @@ "version": "7.0.0", "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-7.0.0.tgz", "integrity": "sha512-Fgl0YPZ902wEsAyiQ+idGd1A7rSFx/ayC1CQVMw5P+EQx2V0SgpGtf6OKFhVjPflPUl9YMmEOnmfjCdMUsygww==", - "dev": true, "license": "MIT", "dependencies": { "agent-base": "^6.0.2", @@ -6547,14 +6941,12 @@ "version": "1.1.3", "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.3.tgz", "integrity": "sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==", - "dev": true, "license": "BSD-3-Clause" }, "node_modules/ssri": { "version": "9.0.1", "resolved": "https://registry.npmjs.org/ssri/-/ssri-9.0.1.tgz", "integrity": "sha512-o57Wcn66jMQvfHG1FlYbWeZWW/dHZhJXjpIcTfXldXEk5nz5lStPo3mK0OJQfGR3RbZUlbISexbljkJzuEj/8Q==", - "dev": true, "license": "ISC", "dependencies": { "minipass": "^3.1.1" @@ -6586,6 +6978,14 @@ "node": ">=8" } }, + "node_modules/stream-read-all": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/stream-read-all/-/stream-read-all-3.0.1.tgz", + "integrity": "sha512-EWZT9XOceBPlVJRrYcykW8jyRSZYbkb/0ZK36uLEmoWVO5gxBOnntNTseNzfREsqxqdfEGQrD8SXQ3QWbBmq8A==", + "engines": { + "node": ">=10" + } + }, "node_modules/string_decoder": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", @@ -6706,7 +7106,6 @@ "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", - "dev": true, "license": "MIT", "dependencies": { "has-flag": "^4.0.0" @@ -6728,11 +7127,46 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/table-layout": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/table-layout/-/table-layout-3.0.2.tgz", + "integrity": "sha512-rpyNZYRw+/C+dYkcQ3Pr+rLxW4CfHpXjPDnG7lYhdRoUcZTUt+KEsX+94RGp/aVp/MQU35JCITv2T/beY4m+hw==", + "dependencies": { + "@75lb/deep-merge": "^1.1.1", + "array-back": "^6.2.2", + "command-line-args": "^5.2.1", + "command-line-usage": "^7.0.0", + "stream-read-all": "^3.0.1", + "typical": "^7.1.1", + "wordwrapjs": "^5.1.0" + }, + "bin": { + "table-layout": "bin/cli.js" + }, + "engines": { + "node": ">=12.17" + } + }, + "node_modules/table-layout/node_modules/array-back": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/array-back/-/array-back-6.2.2.tgz", + "integrity": "sha512-gUAZ7HPyb4SJczXAMUXMGAvI976JoK3qEx9v1FTmeYuJj0IBiaKttG1ydtGKdkfqWkIkouke7nG8ufGy77+Cvw==", + "engines": { + "node": ">=12.17" + } + }, + "node_modules/table-layout/node_modules/typical": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/typical/-/typical-7.1.1.tgz", + "integrity": "sha512-T+tKVNs6Wu7IWiAce5BgMd7OZfNYUndHwc5MknN+UHOudi7sGZzuHdCadllRuqJ3fPtgFtIH9+lt9qRv6lmpfA==", + "engines": { + "node": ">=12.17" + } + }, "node_modules/tar": { "version": "6.2.1", "resolved": "https://registry.npmjs.org/tar/-/tar-6.2.1.tgz", "integrity": "sha512-DZ4yORTwrbTj/7MZYq2w+/ZFdI6OZ/f9SFHR+71gIVUZhOQPHzVCLpvRnPgyaMpfWxxk/4ONva3GQSyNIKRv6A==", - "dev": true, "license": "ISC", "dependencies": { "chownr": "^2.0.0", @@ -6784,7 +7218,6 @@ "version": "5.0.0", "resolved": "https://registry.npmjs.org/minipass/-/minipass-5.0.0.tgz", "integrity": "sha512-3FnjYuehv9k6ovOEbyOswadCDPX1piCfhV8ncmYtHOjuPwylVWsghTLo7rabjC3Rx5xD4HDx8Wm1xnMF7S5qFQ==", - "dev": true, "license": "ISC", "engines": { "node": ">=8" @@ -6840,6 +7273,11 @@ "node": ">=8.0" } }, + "node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" + }, "node_modules/tree-sitter": { "version": "0.21.1", "resolved": "https://registry.npmjs.org/tree-sitter/-/tree-sitter-0.21.1.tgz", @@ -6907,6 +7345,11 @@ "typescript": ">=4.2.0" } }, + "node_modules/tslib": { + "version": "2.6.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.3.tgz", + "integrity": "sha512-xNvxJEOUiWPGhUuUdQgAJPKOOJfGnIyKySOc09XkKsgdUV/3E2zvwZYdejjmRgPCgcym1juLH3226yA7sEFJKQ==" + }, "node_modules/tunnel-agent": { "version": "0.6.0", "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", @@ -6959,6 +7402,14 @@ "node": ">=14.17" } }, + "node_modules/typical": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/typical/-/typical-4.0.0.tgz", + "integrity": "sha512-VAH4IvQ7BDFYglMd7BPRDfLgxZZX4O4TFcRDA6EN5X7erNJJq+McIEp8np9aVtxrCJ6qx4GTYVfOWNjcqwZgRw==", + "engines": { + "node": ">=8" + } + }, "node_modules/ufo": { "version": "1.5.3", "resolved": "https://registry.npmjs.org/ufo/-/ufo-1.5.3.tgz", @@ -6970,14 +7421,12 @@ "version": "5.26.5", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", - "dev": true, "license": "MIT" }, "node_modules/unique-filename": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/unique-filename/-/unique-filename-2.0.1.tgz", "integrity": "sha512-ODWHtkkdx3IAR+veKxFV+VBkUMcN+FaqzUUd7IZzt+0zhDZFPFxhlqwPF3YQvMHx1TD0tdgYl+kuPnJ8E6ql7A==", - "dev": true, "license": "ISC", "dependencies": { "unique-slug": "^3.0.0" @@ -7000,7 +7449,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/unique-slug/-/unique-slug-3.0.0.tgz", "integrity": "sha512-8EyMynh679x/0gqE9fT9oilG+qEt+ibFyqjuVTsZn1+CMxH+XLlpvr2UZx4nVcCwTpx81nICr2JQFkM+HPLq4w==", - "dev": true, "license": "ISC", "dependencies": { "imurmurhash": "^0.1.4" @@ -7384,6 +7832,11 @@ "dev": true, "license": "Apache-2.0" }, + "node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" + }, "node_modules/webpack-sources": { "version": "3.2.3", "resolved": "https://registry.npmjs.org/webpack-sources/-/webpack-sources-3.2.3.tgz", @@ -7411,11 +7864,19 @@ "node": ">=6" } }, + "node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, "node_modules/which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", - "dev": true, "license": "ISC", "dependencies": { "isexe": "^2.0.0" @@ -7431,7 +7892,6 @@ "version": "1.1.5", "resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.5.tgz", "integrity": "sha512-eDMORYaPNZ4sQIuuYPDHdQvf4gyCF9rEEV/yPxGfwPkRodwEgiMUUXTx/dex+Me0wxx53S+NgUHaP7y3MGlDmg==", - "dev": true, "license": "ISC", "dependencies": { "string-width": "^1.0.2 || 2 || 3 || 4" @@ -7441,7 +7901,6 @@ "version": "5.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "dev": true, "license": "MIT", "engines": { "node": ">=8" @@ -7451,14 +7910,12 @@ "version": "8.0.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", - "dev": true, "license": "MIT" }, "node_modules/wide-align/node_modules/is-fullwidth-code-point": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", - "dev": true, "license": "MIT", "engines": { "node": ">=8" @@ -7468,7 +7925,6 @@ "version": "4.2.3", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "dev": true, "license": "MIT", "dependencies": { "emoji-regex": "^8.0.0", @@ -7483,7 +7939,6 @@ "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dev": true, "license": "MIT", "dependencies": { "ansi-regex": "^5.0.1" @@ -7502,6 +7957,14 @@ "node": ">=0.10.0" } }, + "node_modules/wordwrapjs": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/wordwrapjs/-/wordwrapjs-5.1.0.tgz", + "integrity": "sha512-JNjcULU2e4KJwUNv6CHgI46UvDGitb6dGryHajXTDiLgg1/RiGoPSDw4kZfYnwGtEXf2ZMeIewDQgFGzkCB2Sg==", + "engines": { + "node": ">=12.17" + } + }, "node_modules/wrap-ansi": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", @@ -7641,7 +8104,6 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", - "dev": true, "license": "ISC" }, "node_modules/yargs": { @@ -7744,6 +8206,7 @@ "parsers": { "name": "@dodona/dolos-parsers", "version": "1.2.0", + "hasInstallScript": true, "license": "MIT", "dependencies": { "nan": "^2.19.0", @@ -7762,6 +8225,7 @@ "devDependencies": { "@babel/types": "^7.24.7", "@dodona/dolos-core": "1.1.0", + "@duckdb/duckdb-wasm": "^1.28.1-dev246.0", "@mdi/font": "7.4.47", "@types/d3": "7.4.3", "@types/luxon": "^3.4.2", diff --git a/shell.nix b/shell.nix index 05f46cc11..38d4eeedc 100644 --- a/shell.nix +++ b/shell.nix @@ -10,6 +10,7 @@ pkgs.devshell.mkShell { nixpkgs-fmt docker docker-compose + duckdb ]; env = [{ name = "PYTHON"; diff --git a/web/package.json b/web/package.json index 151252b72..5f9ffb313 100644 --- a/web/package.json +++ b/web/package.json @@ -21,6 +21,7 @@ "devDependencies": { "@babel/types": "^7.24.7", "@dodona/dolos-core": "1.1.0", + "@duckdb/duckdb-wasm": "^1.28.1-dev246.0", "@mdi/font": "7.4.47", "@types/d3": "7.4.3", "@types/luxon": "^3.4.2", diff --git a/web/public/data/dolos.db b/web/public/data/dolos.db new file mode 100644 index 000000000..6a06b496f Binary files /dev/null and b/web/public/data/dolos.db differ diff --git a/web/src/api/stores/api.store.ts b/web/src/api/stores/api.store.ts index 62778cdae..a13c411fd 100644 --- a/web/src/api/stores/api.store.ts +++ b/web/src/api/stores/api.store.ts @@ -9,17 +9,23 @@ import { } from "@/api/stores"; import { refDebounced } from "@vueuse/shared"; import { guessSimilarityThreshold } from "../utils"; +import { useDuckDB } from "@/composables/useDuckDB"; +import { computed, ShallowRef } from "vue"; +import { AsyncDuckDBConnection } from "@duckdb/duckdb-wasm"; /** * Store managing the API. */ export const useApiStore = defineStore("api", () => { + // API Stores const fileStore = useFileStore(); const kgramStore = useKgramStore(); const metadataStore = useMetadataStore(); const pairStore = usePairStore(); + const duckdb: ShallowRef = shallowRef(); + // If the data is loaded. const loading = shallowRef(true); // Loading text. @@ -38,19 +44,22 @@ export const useApiStore = defineStore("api", () => { // Hydrate the API stores. const hydrate = async (): Promise => { loading.value = true; - + console.time("Hydrate"); try { + const conn = await useDuckDB(dataUrl.value + '/dolos.db'); + // Hydrate all stores (fetch data) loadingText.value = "Fetching & parsing metadata..."; - await metadataStore.hydrate(); + await metadataStore.hydrate(conn); loadingText.value = "Fetching & parsing files..."; - await fileStore.hydrate(); + await fileStore.hydrate(conn); + loadingText.value = "Fetching & parsing k-grams..."; - await kgramStore.hydrate(); + await kgramStore.hydrate(conn); loadingText.value = "Fetching & parsing pairs..."; - await pairStore.hydrate(); + await pairStore.hydrate(conn); // Calculate the initial cut-off value. loadingText.value = "Calculating initial cut-off..."; @@ -61,6 +70,8 @@ export const useApiStore = defineStore("api", () => { } loading.value = false; + console.timeEnd("Hydrate"); + await fetch(dataUrl.value + '/done'); }; const { dataUrl } = useAppMode(); diff --git a/web/src/api/stores/file.store.ts b/web/src/api/stores/file.store.ts index bfa3dd0fa..c91e386de 100644 --- a/web/src/api/stores/file.store.ts +++ b/web/src/api/stores/file.store.ts @@ -9,6 +9,8 @@ import { FileScoring, SimilarityScore, } from "@/util/FileInterestingness"; +import { Region } from "@dodona/dolos-core"; +import { AsyncDuckDBConnection } from "@duckdb/duckdb-wasm"; /** * Store containing the file data & helper functions. @@ -104,8 +106,8 @@ export const useFileStore = defineStore("file", () => { // Functions - async function hydrate(): Promise { - const parsed = parse(await fetch()); + async function hydrate(conn: AsyncDuckDBConnection): Promise { + const parsed = parse(await fetch(conn)); filesById.value = parsed.files; filesActiveById.value = parsed.files; ignoredFile.value = parsed.ignoredFile; @@ -117,9 +119,10 @@ export const useFileStore = defineStore("file", () => { } async function fetch( - url: string = apiStore.url + "/files.csv" + conn: AsyncDuckDBConnection ): Promise { - return await parseCsv(url); + const result = await conn.query("SELECT * from files"); + return result.toArray(); } // Parse the files from a CSV string. @@ -154,7 +157,7 @@ export const useFileStore = defineStore("file", () => { let hasTimestamps = false; for (const row of fileData) { - const file = row as File; + const file = {...row } as File; const filePathSplit = row.path.split("."); const filePathExtension = filePathSplit[filePathSplit.length - 1]; const extra = JSON.parse(row.extra || "{}"); @@ -162,10 +165,10 @@ export const useFileStore = defineStore("file", () => { hasTimestamps = hasTimestamps || !!extra.timestamp; file.ignored = row.ignored == "true" file.extra = extra; - file.ast = JSON.parse(row.ast); - file.mapping = JSON.parse(row.mapping); + file.ast = row.ast.toArray(); + file.mapping = Region.fromUInt16(row.mapping.toArray()); file.astAndMappingLoaded = true; - file.amountOfKgrams = file.amountOfKgrams || file.ast.length; + file.amountOfKgrams = row.kgramCount || file.ast.length; if (extra.labels && extra.labels.length > 0) { hasLabels = true; diff --git a/web/src/api/stores/kgram.store.ts b/web/src/api/stores/kgram.store.ts index 822406370..98a03237a 100644 --- a/web/src/api/stores/kgram.store.ts +++ b/web/src/api/stores/kgram.store.ts @@ -3,6 +3,7 @@ import { shallowRef } from "vue"; import { Kgram, File } from "@/api/models"; import { parseCsv } from "@/api/utils"; import { useFileStore, useApiStore } from "@/api/stores"; +import { AsyncDuckDBConnection } from "@duckdb/duckdb-wasm"; /** * Store containing the k-grams data & helper functions. @@ -26,12 +27,12 @@ export const useKgramStore = defineStore("kgrams", () => { const kgrams: Kgram[] = []; const ignored: Kgram[] = []; for (const row of kgramData) { - const id = parseInt(row.id); - const fileIds: number[] = JSON.parse(row.files); + const id = row.id; + const fileIds: number[] = row.files.toArray(); const files: File[] = fileIds.map((id) => filesById[id]); const kgram = { id, - hash: parseInt(row.hash), + hash: Number(row.hash), ignored: row.ignored == "true", data: row.data, files, @@ -54,13 +55,13 @@ export const useKgramStore = defineStore("kgrams", () => { const fileStore = useFileStore(); // Fetch the k-grams from the CSV file. - async function fetch(): Promise { - const url = apiStore.url + "/kgrams.csv"; - return await parseCsv(url); + async function fetch(conn: AsyncDuckDBConnection): Promise { + const result = await conn.query("SELECT * from kgrams"); + return result.toArray(); } // Hydrate the store - async function hydrate(): Promise { + async function hydrate(conn: AsyncDuckDBConnection): Promise { // Make sure the file store is hydrated. if (!fileStore.hydrated) { throw new Error( @@ -68,7 +69,7 @@ export const useKgramStore = defineStore("kgrams", () => { ); } - const parsed = parse(await fetch(), fileStore.filesById); + const parsed = parse(await fetch(conn), fileStore.filesById); kgrams.value = parsed.kgrams; ignoredKgrams.value = parsed.ignored; hydrated.value = true; diff --git a/web/src/api/stores/metadata.store.ts b/web/src/api/stores/metadata.store.ts index d875dc78a..dd743e268 100644 --- a/web/src/api/stores/metadata.store.ts +++ b/web/src/api/stores/metadata.store.ts @@ -3,6 +3,7 @@ import { shallowRef } from "vue"; import { Metadata } from "@/api/models"; import { castToType, parseCsv } from "@/api/utils"; import { useApiStore } from "@/api/stores"; +import { AsyncDuckDBConnection } from "@duckdb/duckdb-wasm"; /** * Store containing the metadata & helper functions. @@ -17,7 +18,7 @@ export const useMetadataStore = defineStore("metadata", () => { // Parse the metadata from a CSV string. function parse(data: any[]): Metadata { return Object.fromEntries( - data.map(row => [row.property, castToType(row).value]) + data.map(row => [row.key, castToType(row).value]) ); } @@ -25,14 +26,14 @@ export const useMetadataStore = defineStore("metadata", () => { const apiStore = useApiStore(); // Fetch the metadata from the CSV file. - async function fetch(): Promise { - const url = apiStore.url + "/metadata.csv"; - return await parseCsv(url); + async function fetch(conn: AsyncDuckDBConnection): Promise { + const result = await conn.query('SELECT * from metadata'); + return result.toArray(); } // Hydrate the store - async function hydrate(): Promise { - metadata.value = parse(await fetch()); + async function hydrate(conn: AsyncDuckDBConnection): Promise { + metadata.value = parse(await fetch(conn)); hydrated.value = true; } diff --git a/web/src/api/stores/pair.store.ts b/web/src/api/stores/pair.store.ts index f1e497a2e..f972c2369 100644 --- a/web/src/api/stores/pair.store.ts +++ b/web/src/api/stores/pair.store.ts @@ -2,6 +2,7 @@ import { defineStore } from "pinia"; import { parseCsv } from "@/api/utils"; import { shallowRef, computed } from "vue"; import { Cluster } from "@/util/clustering-algorithms/ClusterTypes"; +import { AsyncDuckDBConnection } from "@duckdb/duckdb-wasm"; import { singleLinkageCluster } from "@/util/clustering-algorithms/SingleLinkageClustering"; @@ -72,19 +73,19 @@ export const usePairStore = defineStore("pairs", () => { function parse(pairData: any[], files: File[]): Pair[] { const pairs: Pair[] = []; for (const row of pairData) { - const id = parseInt(row.id); - const similarity = parseFloat(row.similarity); - const longestFragment = parseFloat(row.longestFragment); - const totalOverlap = parseFloat(row.totalOverlap); - const leftCovered = parseFloat(row.leftCovered); - const rightCovered = parseFloat(row.rightCovered); + const id = row.id; + const similarity = row.similarity; + const longestFragment = row.longestFragment; + const totalOverlap = row.totalOverlap; + const leftCovered = row.leftCovered; + const rightCovered = row.rightCovered; pairs[id] = { id, similarity, longestFragment, totalOverlap, - leftFile: files[parseInt(row.leftFileId)], - rightFile: files[parseInt(row.rightFileId)], + leftFile: files[row.leftFileId], + rightFile: files[row.rightFileId], fragments: null, leftCovered, rightCovered, @@ -94,19 +95,21 @@ export const usePairStore = defineStore("pairs", () => { } // Fetch the pairs from the CSV file. - async function fetch(): Promise { - const url = apiStore.url + "/pairs.csv"; - return await parseCsv(url); + async function fetch(conn: AsyncDuckDBConnection): Promise { + const result = await conn.query("SELECT * from pairs"); + return result.toArray(); } // Hydrate the store - async function hydrate(): Promise { + async function hydrate(conn: AsyncDuckDBConnection): Promise { // Make sure the file store is hydrated. if (!fileStore.hydrated) { throw new Error("The file store must be hydrated before the pair store."); } + console.time("Hydrate pairs"); + pairsById.value = parse(await fetch(conn), fileStore.filesActiveById); + console.timeEnd("Hydrate pairs"); - pairsById.value = parse(await fetch(), fileStore.filesActiveById); hydrated.value = true; } @@ -116,7 +119,6 @@ export const usePairStore = defineStore("pairs", () => { const kgrams = kgramStore.kgrams; const ignoredKgrams = kgramStore.ignoredKgrams; const ignoredFile = fileStore.ignoredFile; - const pairWithFragments = await dataWorker.populateFragments(pair, customOptions, kgrams, ignoredKgrams, ignoredFile); pairsById.value[pair.id] = pairWithFragments; return pairWithFragments; diff --git a/web/src/composables/useDuckDB.ts b/web/src/composables/useDuckDB.ts new file mode 100644 index 000000000..0b6bc824d --- /dev/null +++ b/web/src/composables/useDuckDB.ts @@ -0,0 +1,37 @@ +import * as duckdb from '@duckdb/duckdb-wasm'; +import duckdb_wasm from '@duckdb/duckdb-wasm/dist/duckdb-mvp.wasm?url'; +import mvp_worker from '@duckdb/duckdb-wasm/dist/duckdb-browser-mvp.worker.js?url'; +import duckdb_wasm_eh from '@duckdb/duckdb-wasm/dist/duckdb-eh.wasm?url'; +import eh_worker from '@duckdb/duckdb-wasm/dist/duckdb-browser-eh.worker.js?url'; + +const MANUAL_BUNDLES: duckdb.DuckDBBundles = { + mvp: { + mainModule: duckdb_wasm, + mainWorker: mvp_worker, + }, + eh: { + mainModule: duckdb_wasm_eh, + mainWorker: eh_worker, + }, +}; + + +export async function useDuckDB(dbURL: string): Promise { + // Select a bundle based on browser checks + const bundle = await duckdb.selectBundle(MANUAL_BUNDLES); + // Instantiate the asynchronus version of DuckDB-wasm + const worker = new Worker(bundle.mainWorker!); + const logger = new duckdb.ConsoleLogger(); + const db = new duckdb.AsyncDuckDB(logger, worker); + await db.instantiate(bundle.mainModule, bundle.pthreadWorker); + const conn = await db.connect(); + const res = await fetch(dbURL); + await db.registerFileBuffer('dolos.db', new Uint8Array(await res.arrayBuffer())) + await conn.query(` + ATTACH DATABASE 'dolos.db' AS dolos_db; + USE dolos_db; + `); + return conn; +} + +