Index parent chains and process sequencer batches (#431)

* Add Orbit chain transaction state tracking and configuration

Co-authored-by: antoine <[email protected]>

* Add Orbit transaction processing with enhanced monitoring and security

Co-authored-by: antoine <[email protected]>

* Add parent chain support for Orbit chain infrastructure contracts

Co-authored-by: antoine <[email protected]>

* fix provider

* update

* Checkpoint before follow-up message

* Improve RPC event querying with chunking and error handling

Co-authored-by: antoine <[email protected]>

* orbit

* Add Orbit batch indexing and UI for tracking sequencer batches

Co-authored-by: antoine <[email protected]>

* Checkpoint before follow-up message

Co-authored-by: antoine <[email protected]>

* Add Orbit batch discovery and enhanced parsing mechanism

Co-authored-by: antoine <[email protected]>

* Enhance Orbit transaction processing with improved logging and error handling

Co-authored-by: antoine <[email protected]>

* fixes

* fix column error

* fixes

* Implement orbit batch queue management with rate limiting and deduplication

Co-authored-by: antoine <[email protected]>

* orbit: improve batch event detection
- use BigNumber for indexed arg and hexZeroPad
- configurable SEARCH_BLOCK_WINDOW, default 50k
- fallback to provider.getLogs with explicit topics
- remove console.log and add better debug logs

* iteration

* Add parent workspace support for Orbit batch discovery and chain metadata

Co-authored-by: antoine <[email protected]>

* Simplify orbit batch discovery to use parent workspace indexed logs

Co-authored-by: antoine <[email protected]>

* introduce ndos

* Add Orbit node indexing and batch-node mapping support

Co-authored-by: antoine <[email protected]>

* Update Orbit node indexing for Nitro rollup assertion events

Co-authored-by: antoine <[email protected]>

* Improve batch sequence number lookup for orbit nodes using afterAcc

Co-authored-by: antoine <[email protected]>

* rebase

* iterations

* cleanup

* impr

* fix conflict

* finalization

* deleted extra files

* tests + cleanup

* tests + cleanup

* tests + cleanup

* tests + cleanup

* more tests

* more tests

* frontend tests

* remove useless fns

* cleanup

* orbit config

* add tests

* frontend tests

* fix reset fn + tests

* remove tests

* remove console log

* fix migrations

---------

Co-authored-by: Cursor Agent <[email protected]>
Co-authored-by: antoine <[email protected]>

Author: 3 people

.cursor/rules/frontend-tester.mdc

@@ -16,6 +16,7 @@ The linter for tests is tests/.eslintrc.js
 - When you create or update snapshots, always analyze them following instructions in the "Snapshots" sections of this file
 - Indent with 4 spaces
 - Make sure all imports are used. Remove unused ones.
+- It's fine to only have one test in a file if the tested file is simple. Each test should add some unique value.
 
 # Imports
 - Never import mount, vi, createTestingPinia, flushPromises

pm2-server/app.js

@@ -1,24 +1,33 @@
 const express = require('express');
 const app = express();
 const pm2 = require('./lib/pm2.js');
+const { secretMiddleware } = require('./lib/middleware.js');
 
 const bodyParser = require('body-parser')
 app.use(bodyParser.json());
 
 const commands = ['stop', 'reload', 'restart', 'delete', 'resume'];
 
-const secretMiddleware = (req, res, next) => {
-    if (req.query.secret == process.env.SECRET)
-        next();
-    else
-        return res.status(401).send('Invalid secret');
-};
-
 const handleError = (res, error) => {
     console.log(error);
     return res.status(400).send(error.message);
 };
 
+app.post('/log-listener', secretMiddleware, async (req, res) => {
+    const data = req.body;
+
+    try {
+        if (!data.slug || !data.jsonArgs)
+            throw new Error('Missing parameter');
+
+        const pm2Process = await pm2.startLogListener(data.slug, data.jsonArgs);
+
+        return res.status(200).send(pm2Process);
+    } catch(error) {
+        handleError(res, error);
+    }
+});
+
 app.get('/processes', secretMiddleware, async (req, res) => {
     try {
         const processes = await pm2.list()

pm2-server/ecosystem.config.js

@@ -1,12 +1,14 @@
 module.exports = {
-    apps : [{
-        name: "PM2 Server",
-        script: "./index.js",
-        env: {
-            NODE_ENV: "development",
+    apps : [
+        {
+            name: 'PM2 Server',
+            script: './index.js',
+            env: {
+                NODE_ENV: 'development',
+            },
+            env_production: {
+                NODE_ENV: 'production',
+            }
         },
-        env_production: {
-            NODE_ENV: "production",
-        }
-    }]
+    ]
 }

pm2-server/index.js

@@ -16,4 +16,5 @@ const triggerSync = () => {
 app.listen(port, () => {
     console.log(`App is listening on port ${port}`);
     triggerSync();
+    startL1Trackers();
 });

pm2-server/lib/client.js

@@ -0,0 +1,49 @@
+const { createPublicClient, http, defineChain, webSocket } = require('viem');
+
+const getProvider = (_rpcServer) => {
+    try {
+        const rpcServer = new URL(_rpcServer);
+        return rpcServer.protocol == 'ws:' || rpcServer.protocol == 'wss:' ? { http: [], webSocket: [_rpcServer] } : { http: [_rpcServer], webSocket: [] };
+    } catch (error) {
+        console.log(error);
+        process.exit(1);
+    }
+};
+
+const fetchOptions = (_rpcServer) => {
+    const rpcServer = new URL(_rpcServer);
+    if (rpcServer.username.length || rpcServer.password.length) {
+        const base64Credentials = btoa(`${rpcServer.username}:${rpcServer.password}`);
+        return { headers: { 'Authorization': `Basic ${base64Credentials}` }};
+    }
+    else
+        return {};
+};
+
+const getClient = (workspace) => {
+    const chain = defineChain({
+        id: workspace.networkId,
+        name: workspace.name,
+        network: workspace.name,
+        nativeCurrency: {
+            decimals: 18,
+            name: 'Ether',
+            symbol: 'ETH'
+        },
+        rpcUrls: {
+            default: getProvider(workspace.rpcServer),
+            public: getProvider(workspace.rpcServer)
+        }
+    });
+
+    const url = new URL(workspace.rpcServer).origin + new URL(workspace.rpcServer).pathname + new URL(workspace.rpcServer).search;
+    const transport = workspace.rpcServer.startsWith('ws') ?
+        webSocket(url) :
+        http(url, { fetchOptions: fetchOptions(workspace.rpcServer) });
+
+    return createPublicClient({ chain, transport });
+};
+
+module.exports = {
+    getClient
+};

pm2-server/lib/env.js

@@ -0,0 +1,5 @@
+module.exports = {
+    getEthernalSecret: () => process.env.ETHERNAL_SECRET,
+    getApiHost: () => process.env.ETHERNAL_HOST || 'http://localhost:8888',
+    getSecret: () => process.env.SECRET
+};

pm2-server/lib/middleware.js

@@ -0,0 +1,12 @@
+const { getSecret } = require('./env.js');
+
+const secretMiddleware = (req, res, next) => {
+    if (req.query.secret == getSecret())
+        next();
+    else
+        return res.status(401).send('Invalid secret');
+};
+
+module.exports = {
+    secretMiddleware
+};

pm2-server/lib/pm2.js

@@ -131,6 +131,34 @@ const _delete = (slug) => {
     });
 };
 
+const startLogListener = (slug, jsonArgs) => {
+    return new Promise((resolve, reject) => {
+        if (!slug || !jsonArgs) reject(new Error('Missing parameter'));
+
+        pm2.connect(error => {
+            if (error) reject(new Error(error));
+
+            const options = {
+                name: slug,
+                script: 'node',
+                args: ['logListener.js', jsonArgs],
+                interpreter: 'none',
+                log_type: 'json'
+            };
+
+            pm2.start(options, (error) => {
+                if (error) reject(new Error(error));
+
+                pm2.describe(slug, (error, process) => {
+                    if (error) reject(new Error(error));
+
+                    resolve(process[0]);
+                });
+            });
+        });
+    });
+};
+
 const start = (slug, workspaceId) => {
     return new Promise((resolve, reject) => {
         if (!slug || !workspaceId) reject(new Error('Missing parameter'));
@@ -159,4 +187,4 @@ const start = (slug, workspaceId) => {
     });
 };
 
-module.exports = { list, show, stop, reload, restart, delete: _delete, start, resume };
+module.exports = { list, show, stop, reload, restart, delete: _delete, start, resume, startLogListener };

pm2-server/lib/queue.js

@@ -0,0 +1,28 @@
+const Redis = require('ioredis');
+const { Queue } = require('bullmq');
+
+const redisUrl = process.env.ETHERNAL_REDIS_URL
+
+const defaultJobOptions = {
+    attempts: 50,
+    removeOnComplete: {
+        count: 100,
+        age: 4 * 60
+    },
+    timeout: 30000,
+    backoff: {
+        type: 'exponential',
+        delay: 1000
+    }
+};
+
+const connection = new Redis(redisUrl);
+
+const enqueue = (queueName, jobName, data, priority = 1) => {
+    const queue = new Queue(queueName, { connection, defaultJobOptions });
+    return queue.add(jobName, data, { priority });
+};
+
+module.exports = {
+    enqueue
+}

pm2-server/lib/workspace.js

@@ -0,0 +1,19 @@
+const axios = require('axios');
+const { getEthernalSecret, getApiHost } = require('./env');
+
+const secret = getEthernalSecret();
+const apiHost = getApiHost();
+
+if (!secret) {
+    console.log(`Pass the secret with the ETHERNAL_SECRET env variable.`);
+    process.exit(1);
+}
+
+const getWorkspace = async (workspaceId) => {
+    const { data: workspace } = await axios.get(`${apiHost}/api/workspaces/${workspaceId}`, { params: { secret } });
+    return { workspace };
+};
+
+module.exports = {
+    getWorkspace
+}