Skip to content

Commit c0265aa

Browse files
author
web3blind
committed
Use SQLite for Viz Magic archive storage
1 parent 9ae7e8e commit c0265aa

4 files changed

Lines changed: 191 additions & 89 deletions

File tree

‎tools/archive-node/README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ It indexes public VIZ blocks, extracts only game-relevant operations, and serves
1414
- `custom` protocol `VE`;
1515
- `award` operations used for blessings/rewards.
1616

17-
The current implementation is dependency-free and stores data under `data/archive-node/` as JSON/JSONL files. `sqlite-schema.sql` documents the intended SQLite schema for a future storage backend.
17+
The implementation stores data in SQLite (`data/archive-node/archive.sqlite`) using Node 22's built-in `node:sqlite` module. `sqlite-schema.sql` documents the live schema.
1818

1919
## Run locally
2020

‎tools/archive-node/server.js‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ function createServer(options) {
6363
dataDir: cfg.dataDir,
6464
lastIndexedBlock: status.lastIndexedBlock || 0,
6565
cursorUpdatedAt: status.cursorUpdatedAt || null,
66-
storage: 'flat-file-jsonl',
66+
storage: 'sqlite',
6767
readOnly: true
6868
}, { 'Cache-Control': 'no-store' });
6969
return;
@@ -74,7 +74,7 @@ function createServer(options) {
7474
st.ok = st.ok !== false;
7575
st.service = 'viz-magic-game-archive';
7676
st.readOnly = true;
77-
st.storage = 'flat-file-jsonl';
77+
st.storage = 'sqlite';
7878
json(res, 200, st, { 'Cache-Control': 'no-store' });
7979
return;
8080
}

‎tools/archive-node/sqlite-schema.sql‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,5 @@
1-
-- Viz Magic archive-node reference schema.
2-
-- The current dependency-free implementation stores the same entities as
3-
-- flat JSON/JSONL files so community mirrors can run without native modules.
4-
-- This schema documents the intended SQLite layout for operators who later
5-
-- migrate the store to SQLite.
1+
-- Viz Magic archive-node SQLite schema.
2+
-- This is the live storage layout used by tools/archive-node/storage.js.
63

74
CREATE TABLE IF NOT EXISTS blocks (
85
block_num INTEGER PRIMARY KEY,

‎tools/archive-node/storage.js‎

Lines changed: 186 additions & 81 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,15 @@
22

33
var fs = require('fs');
44
var path = require('path');
5+
var originalEmitWarning = process.emitWarning;
6+
process.emitWarning = function(warning) {
7+
var text = String(warning && warning.message || warning || '');
8+
if (text.indexOf('SQLite is an experimental feature') !== -1) return;
9+
return originalEmitWarning.apply(process, arguments);
10+
};
11+
var sqlite = require('node:sqlite');
12+
process.emitWarning = originalEmitWarning;
13+
var DatabaseSync = sqlite.DatabaseSync;
514

615
function ensureDir(dir) {
716
fs.mkdirSync(dir, { recursive: true });
@@ -26,67 +35,94 @@ function writeJsonAtomic(file, data) {
2635
fs.renameSync(tmp, file);
2736
}
2837

29-
function readJsonLines(file) {
30-
try {
31-
var text = fs.readFileSync(file, 'utf8');
32-
if (!text.trim()) return [];
33-
return text.split(/\n+/).filter(Boolean).map(function(line) { return JSON.parse(line); });
34-
} catch (err) {
35-
return [];
36-
}
37-
}
38-
39-
function appendJsonLine(file, item) {
40-
ensureDir(path.dirname(file));
41-
fs.appendFileSync(file, JSON.stringify(item) + '\n', 'utf8');
42-
}
43-
4438
function ArchiveStore(rootDir) {
4539
this.rootDir = rootDir || path.join(process.cwd(), 'data', 'archive-node');
46-
this.blocksDir = path.join(this.rootDir, 'blocks');
47-
this.eventsDir = path.join(this.rootDir, 'events');
48-
ensureDir(this.blocksDir);
49-
ensureDir(this.eventsDir);
40+
ensureDir(this.rootDir);
41+
this.dbPath = path.join(this.rootDir, 'archive.sqlite');
42+
this.db = new DatabaseSync(this.dbPath);
43+
this.db.exec('PRAGMA journal_mode = WAL');
44+
this.db.exec('PRAGMA synchronous = NORMAL');
45+
this.db.exec('PRAGMA temp_store = MEMORY');
46+
this._initSchema();
5047
}
5148

52-
ArchiveStore.prototype.cursorPath = function() {
53-
return path.join(this.rootDir, 'cursor.json');
54-
};
55-
56-
ArchiveStore.prototype.statusPath = function() {
57-
return path.join(this.rootDir, 'status.json');
58-
};
59-
60-
ArchiveStore.prototype.blockPath = function(blockNum) {
61-
return path.join(this.blocksDir, String(blockNum) + '.json');
62-
};
63-
64-
ArchiveStore.prototype.allEventsPath = function() {
65-
return path.join(this.eventsDir, 'all.jsonl');
49+
ArchiveStore.prototype._initSchema = function() {
50+
this.db.exec([
51+
'CREATE TABLE IF NOT EXISTS blocks (',
52+
' block_num INTEGER PRIMARY KEY,',
53+
' block_id TEXT,',
54+
' previous TEXT,',
55+
' timestamp TEXT,',
56+
' source_node TEXT,',
57+
' indexed_at TEXT,',
58+
' event_count INTEGER NOT NULL DEFAULT 0,',
59+
' raw_json TEXT NOT NULL',
60+
')',
61+
';',
62+
'CREATE TABLE IF NOT EXISTS events (',
63+
' id TEXT PRIMARY KEY,',
64+
' block_num INTEGER NOT NULL,',
65+
' block_id TEXT,',
66+
' previous TEXT,',
67+
' timestamp TEXT,',
68+
' tx_index INTEGER NOT NULL,',
69+
' op_index INTEGER NOT NULL,',
70+
' op_type TEXT NOT NULL,',
71+
' protocol TEXT NOT NULL,',
72+
' type TEXT,',
73+
' sender TEXT,',
74+
' account TEXT,',
75+
' accounts_json TEXT,',
76+
' payload_json TEXT,',
77+
' raw_json TEXT NOT NULL',
78+
')',
79+
';',
80+
'CREATE TABLE IF NOT EXISTS event_accounts (',
81+
' event_id TEXT NOT NULL,',
82+
' account TEXT NOT NULL,',
83+
' protocol TEXT NOT NULL,',
84+
' block_num INTEGER NOT NULL,',
85+
' PRIMARY KEY(event_id, account)',
86+
')',
87+
';',
88+
'CREATE TABLE IF NOT EXISTS meta (',
89+
' key TEXT PRIMARY KEY,',
90+
' value TEXT NOT NULL',
91+
')',
92+
';',
93+
'CREATE INDEX IF NOT EXISTS idx_events_block ON events(block_num)',
94+
';',
95+
'CREATE INDEX IF NOT EXISTS idx_events_protocol_block ON events(protocol, block_num)',
96+
';',
97+
'CREATE INDEX IF NOT EXISTS idx_event_accounts_account_protocol ON event_accounts(account, protocol, block_num)',
98+
';'
99+
].join('\n'));
66100
};
67101

68-
ArchiveStore.prototype.protocolEventsPath = function(protocol) {
69-
return path.join(this.eventsDir, 'protocol-' + sanitizePart(protocol) + '.jsonl');
102+
ArchiveStore.prototype._getMeta = function(key, fallback) {
103+
var row = this.db.prepare('SELECT value FROM meta WHERE key = ?').get(key);
104+
if (!row) return fallback;
105+
try { return JSON.parse(row.value); } catch (err) { return fallback; }
70106
};
71107

72-
ArchiveStore.prototype.accountProtocolEventsPath = function(account, protocol) {
73-
return path.join(this.eventsDir, 'account-' + sanitizePart(account) + '-protocol-' + sanitizePart(protocol) + '.jsonl');
108+
ArchiveStore.prototype._setMeta = function(key, value) {
109+
this.db.prepare('INSERT OR REPLACE INTO meta(key, value) VALUES(?, ?)').run(key, JSON.stringify(value));
74110
};
75111

76112
ArchiveStore.prototype.getCursor = function() {
77-
return readJson(this.cursorPath(), { lastIndexedBlock: 0, updatedAt: null });
113+
return this._getMeta('cursor', { lastIndexedBlock: 0, updatedAt: null });
78114
};
79115

80116
ArchiveStore.prototype.setCursor = function(lastIndexedBlock) {
81-
writeJsonAtomic(this.cursorPath(), {
117+
this._setMeta('cursor', {
82118
lastIndexedBlock: Number(lastIndexedBlock) || 0,
83119
updatedAt: new Date().toISOString()
84120
});
85121
};
86122

87123
ArchiveStore.prototype.getStatus = function() {
88124
var cursor = this.getCursor();
89-
var status = readJson(this.statusPath(), {});
125+
var status = this._getMeta('status', {});
90126
status.lastIndexedBlock = cursor.lastIndexedBlock || 0;
91127
status.cursorUpdatedAt = cursor.updatedAt || null;
92128
return status;
@@ -95,70 +131,140 @@ ArchiveStore.prototype.getStatus = function() {
95131
ArchiveStore.prototype.setStatus = function(status) {
96132
status = status || {};
97133
status.updatedAt = new Date().toISOString();
98-
writeJsonAtomic(this.statusPath(), status);
134+
this._setMeta('status', status);
99135
};
100136

101137
ArchiveStore.prototype.hasBlock = function(blockNum) {
102-
return fs.existsSync(this.blockPath(blockNum));
138+
return !!this.db.prepare('SELECT 1 AS ok FROM blocks WHERE block_num = ?').get(Number(blockNum));
103139
};
104140

105141
ArchiveStore.prototype.getBlockRecord = function(blockNum) {
106-
return readJson(this.blockPath(blockNum), null);
142+
var row = this.db.prepare('SELECT * FROM blocks WHERE block_num = ?').get(Number(blockNum));
143+
if (!row) return null;
144+
return {
145+
blockNum: row.block_num,
146+
block_id: row.block_id || '',
147+
previous: row.previous || '',
148+
timestamp: row.timestamp || '',
149+
sourceNode: row.source_node || '',
150+
indexedAt: row.indexed_at || '',
151+
eventCount: row.event_count || 0,
152+
block: JSON.parse(row.raw_json)
153+
};
107154
};
108155

109156
ArchiveStore.prototype.putBlock = function(blockNum, block, sourceNode, events) {
110-
var record = {
111-
blockNum: Number(blockNum),
112-
block_id: block && (block.block_id || block.id || ''),
113-
previous: block && (block.previous || block.previous_block_id || ''),
114-
timestamp: block && block.timestamp || '',
115-
sourceNode: sourceNode || '',
116-
indexedAt: new Date().toISOString(),
117-
eventCount: events ? events.length : 0,
118-
block: block
119-
};
120-
writeJsonAtomic(this.blockPath(blockNum), record);
157+
var now = new Date().toISOString();
158+
var bn = Number(blockNum);
159+
var blockId = block && (block.block_id || block.id || '') || '';
160+
var previous = block && (block.previous || block.previous_block_id || '') || '';
161+
var timestamp = block && block.timestamp || '';
162+
this.db.exec('BEGIN IMMEDIATE');
163+
try {
164+
this.db.prepare('DELETE FROM event_accounts WHERE block_num = ?').run(bn);
165+
this.db.prepare('DELETE FROM events WHERE block_num = ?').run(bn);
166+
this.db.prepare([
167+
'INSERT OR REPLACE INTO blocks(block_num, block_id, previous, timestamp, source_node, indexed_at, event_count, raw_json)',
168+
'VALUES(?, ?, ?, ?, ?, ?, ?, ?)'
169+
].join(' ')).run(bn, blockId, previous, timestamp, sourceNode || '', now, events ? events.length : 0, JSON.stringify(block || {}));
170+
this.db.exec('COMMIT');
171+
} catch (err) {
172+
this.db.exec('ROLLBACK');
173+
throw err;
174+
}
121175
};
122176

123177
ArchiveStore.prototype.eventKey = function(event) {
124178
return [event.blockNum, event.txIndex, event.opIndex, event.protocol, event.type, event.sender || event.account || ''].join(':');
125179
};
126180

127181
ArchiveStore.prototype.putEventsForBlock = function(events) {
182+
var insertEvent = this.db.prepare([
183+
'INSERT OR REPLACE INTO events(id, block_num, block_id, previous, timestamp, tx_index, op_index, op_type, protocol, type, sender, account, accounts_json, payload_json, raw_json)',
184+
'VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)'
185+
].join(' '));
186+
var insertAccount = this.db.prepare('INSERT OR REPLACE INTO event_accounts(event_id, account, protocol, block_num) VALUES(?, ?, ?, ?)');
128187
var self = this;
129-
events.forEach(function(event) {
130-
event.id = event.id || self.eventKey(event);
131-
appendJsonLine(self.allEventsPath(), event);
132-
if (event.protocol) appendJsonLine(self.protocolEventsPath(event.protocol), event);
133-
var accounts = event.accounts || [];
134-
for (var i = 0; i < accounts.length; i += 1) {
135-
appendJsonLine(self.accountProtocolEventsPath(accounts[i], event.protocol || '_'), event);
136-
}
137-
});
188+
this.db.exec('BEGIN IMMEDIATE');
189+
try {
190+
events.forEach(function(event) {
191+
event.id = event.id || self.eventKey(event);
192+
var accounts = event.accounts || [];
193+
insertEvent.run(
194+
event.id,
195+
Number(event.blockNum) || 0,
196+
event.block_id || '',
197+
event.previous || '',
198+
event.timestamp || '',
199+
Number(event.txIndex) || 0,
200+
Number(event.opIndex) || 0,
201+
event.opType || '',
202+
event.protocol || '',
203+
event.type || '',
204+
event.sender || '',
205+
event.account || '',
206+
JSON.stringify(accounts),
207+
JSON.stringify(event.payload || null),
208+
JSON.stringify(event.raw || null)
209+
);
210+
for (var i = 0; i < accounts.length; i += 1) {
211+
insertAccount.run(event.id, accounts[i], event.protocol || '_', Number(event.blockNum) || 0);
212+
}
213+
});
214+
this.db.exec('COMMIT');
215+
} catch (err) {
216+
this.db.exec('ROLLBACK');
217+
throw err;
218+
}
219+
};
220+
221+
ArchiveStore.prototype._eventFromRow = function(row) {
222+
return {
223+
blockNum: row.block_num,
224+
block_id: row.block_id || '',
225+
previous: row.previous || '',
226+
timestamp: row.timestamp || '',
227+
txIndex: row.tx_index,
228+
opIndex: row.op_index,
229+
opType: row.op_type || '',
230+
protocol: row.protocol || '',
231+
type: row.type || '',
232+
sender: row.sender || '',
233+
account: row.account || '',
234+
accounts: JSON.parse(row.accounts_json || '[]'),
235+
payload: JSON.parse(row.payload_json || 'null'),
236+
raw: JSON.parse(row.raw_json || 'null'),
237+
id: row.id
238+
};
138239
};
139240

140241
ArchiveStore.prototype.queryEvents = function(options) {
141242
options = options || {};
142-
var file;
143-
if (options.account && options.protocol) file = this.accountProtocolEventsPath(options.account, options.protocol);
144-
else if (options.protocol) file = this.protocolEventsPath(options.protocol);
145-
else file = this.allEventsPath();
146243
var start = Number(options.start || options.from || 0);
147244
var end = Number(options.end || options.to || 2147483647);
148245
var limit = Math.max(1, Math.min(Number(options.limit || 500), 5000));
149-
var protocols = null;
150-
if (options.protocols && options.protocols.length) protocols = options.protocols;
151-
var rows = readJsonLines(file).filter(function(ev) {
152-
if (ev.blockNum < start || ev.blockNum > end) return false;
153-
if (protocols && protocols.indexOf(ev.protocol) === -1) return false;
154-
return true;
155-
});
156-
rows.sort(function(a, b) {
157-
if (a.blockNum !== b.blockNum) return a.blockNum - b.blockNum;
158-
if (a.txIndex !== b.txIndex) return a.txIndex - b.txIndex;
159-
return a.opIndex - b.opIndex;
160-
});
161-
if (rows.length > limit) rows = rows.slice(rows.length - limit);
246+
var params = [];
247+
var where = ['e.block_num >= ?', 'e.block_num <= ?'];
248+
params.push(start, end);
249+
var sql = 'SELECT e.* FROM events e';
250+
if (options.account && options.protocol) {
251+
sql += ' JOIN event_accounts ea ON ea.event_id = e.id';
252+
where.push('ea.account = ?');
253+
where.push('ea.protocol = ?');
254+
params.push(String(options.account), String(options.protocol));
255+
} else if (options.protocol) {
256+
where.push('e.protocol = ?');
257+
params.push(String(options.protocol));
258+
} else if (options.protocols && options.protocols.length) {
259+
where.push('e.protocol IN (' + options.protocols.map(function() { return '?'; }).join(',') + ')');
260+
options.protocols.forEach(function(protocol) { params.push(String(protocol)); });
261+
}
262+
sql += ' WHERE ' + where.join(' AND ');
263+
sql += ' ORDER BY e.block_num DESC, e.tx_index DESC, e.op_index DESC LIMIT ?';
264+
params.push(limit);
265+
var stmt = this.db.prepare(sql);
266+
var rows = stmt.all.apply(stmt, params);
267+
rows = rows.map(this._eventFromRow).reverse();
162268
return rows;
163269
};
164270

@@ -167,6 +273,5 @@ module.exports = {
167273
ensureDir: ensureDir,
168274
readJson: readJson,
169275
writeJsonAtomic: writeJsonAtomic,
170-
readJsonLines: readJsonLines,
171276
sanitizePart: sanitizePart
172277
};

0 commit comments

Comments
 (0)