Skip to content

Commit b94f99d

Browse files
igorlsigorls
authored
Merge pull request #142 from eosrio/large-tx-fix
Large tx-sig fix
2 parents c59005b + 76807c6 commit b94f99d

File tree

7 files changed

+77
-48
lines changed

7 files changed

+77
-48
lines changed

‎connections/amqp.ts

+8-9
Original file line numberDiff line numberDiff line change
@@ -17,17 +17,16 @@ export async function createConnection(config): Promise<Connection> {
1717
}
1818

1919
export function getAmpqUrl(config): string {
20-
let frameMaxValue = '0x10000';
21-
if (config.frameMax) {
22-
frameMaxValue = config.frameMax;
23-
}
24-
const u = encodeURIComponent(config.user);
25-
const p = encodeURIComponent(config.pass);
26-
const v = encodeURIComponent(config.vhost)
27-
return `amqp://${u}:${p}@${config.host}/${v}?frameMax=${frameMaxValue}`;
20+
let frameMaxValue = '0x10000';
21+
if (config.frameMax) {
22+
frameMaxValue = config.frameMax;
23+
}
24+
const u = encodeURIComponent(config.user);
25+
const p = encodeURIComponent(config.pass);
26+
const v = encodeURIComponent(config.vhost);
27+
return `amqp://${u}:${p}@${config.host}/${v}?frameMax=${frameMaxValue}`;
2828
}
2929

30-
3130
async function createChannels(connection) {
3231
try {
3332
const channel = await connection.createChannel();

‎ecosystem.config.js

+7
Original file line numberDiff line numberDiff line change
@@ -20,4 +20,11 @@ readdirSync(chainsRoot)
2020
}
2121
});
2222

23+
apps.push({
24+
name: 'hyperion-governor',
25+
namespace: 'hyperion',
26+
script: 'governor/server/index.js',
27+
watch: false,
28+
});
29+
2330
module.exports = {apps};

‎modules/parsers/base-parser.ts

-1
Original file line numberDiff line numberDiff line change
@@ -192,7 +192,6 @@ export abstract class BaseParser {
192192
if (ds_act) {
193193

194194
if (ds_act.account && ds_act.name && ds_act.authorization) {
195-
console.log(ds_act);
196195
action.act.data = ds_act.data;
197196
}
198197

‎package-lock.json

+16-24
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎package.json

+4-4
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "hyperion-history",
3-
"version": "3.3.9-7",
3+
"version": "3.3.9-8",
44
"description": "Scalable Full History API Solution for EOSIO based blockchains",
55
"main": "launcher.js",
66
"scripts": {
@@ -60,11 +60,11 @@
6060
"socket.io-redis": "^6.1.1",
6161
"telegraf": "^4.12.3-canary.1",
6262
"typescript": "^4.5.2",
63-
"uWebSockets.js": "github:uNetworking/uWebSockets.js#v20.24.0",
64-
"ws": "^8.12.1"
63+
"uWebSockets.js": "github:uNetworking/uWebSockets.js#v20.31.0",
64+
"ws": "^8.14.2"
6565
},
6666
"devDependencies": {
67-
"@types/amqplib": "^0.10.1",
67+
"@types/amqplib": "^0.10.2",
6868
"@types/async": "^3.2.18",
6969
"@types/global-agent": "^2.1.1",
7070
"@types/ioredis": "^4.28.10",

‎workers/deserializer.ts

+40-8
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import {Type} from "../addons/eosjs-native/eosjs-serialize";
88
import {debugLog, hLog} from "../helpers/common_functions";
99
import {createHash} from "crypto";
1010
import flatstr from 'flatstr';
11+
import {Options} from "amqplib";
1112

1213
const FJS = require('fast-json-stringify');
1314

@@ -124,6 +125,9 @@ export default class MainDSWorker extends HyperionWorker {
124125

125126
allowedDynamicContracts: Set<string> = new Set<string>();
126127

128+
backpressureQueue: any[] = [];
129+
waitToSend = false;
130+
127131
constructor() {
128132

129133
super();
@@ -290,6 +294,16 @@ export default class MainDSWorker extends HyperionWorker {
290294
this.ch.consume(process.env['worker_queue'], (data) => {
291295
this.consumerQueue.push(data).catch(console.log);
292296
});
297+
this.ch.on('drain', args => {
298+
this.waitToSend = false;
299+
while (this.backpressureQueue.length > 0) {
300+
const msg = this.backpressureQueue.shift();
301+
const status = this.controlledSendToQueue(msg.queue, msg.payload, msg.options);
302+
if (!status) {
303+
break;
304+
}
305+
}
306+
});
293307
}
294308
}
295309

@@ -478,15 +492,15 @@ export default class MainDSWorker extends HyperionWorker {
478492
hLog(`${block_num} was filtered with ${inline_count} actions!`);
479493
}
480494
try {
495+
trace[1].signatures = signatures;
481496
this.routeToPool(trace[1], {
482497
block_num,
483498
block_id,
484499
producer,
485500
ts,
486501
inline_count,
487502
filtered,
488-
live: process.env['live_mode'],
489-
signatures
503+
live: process.env['live_mode']
490504
});
491505
} catch (e) {
492506
hLog(e);
@@ -622,15 +636,33 @@ export default class MainDSWorker extends HyperionWorker {
622636
}
623637

624638
const pool_queue = `${this.chain}:ds_pool:${selected_q}`;
625-
if (this.ch_ready) {
626-
// console.log('selected_q', pool_queue);
627-
this.ch.sendToQueue(pool_queue, bufferFromJson(trace, true), {headers});
628-
return true;
639+
const payload = bufferFromJson(trace, true);
640+
641+
if (!this.waitToSend) {
642+
if (this.ch_ready) {
643+
this.controlledSendToQueue(pool_queue, payload, {headers});
644+
return true;
645+
} else {
646+
return false;
647+
}
629648
} else {
649+
this.backpressureQueue.push({
650+
queue: pool_queue,
651+
payload: payload,
652+
options: {headers}
653+
});
630654
return false;
631655
}
632656
}
633657

658+
controlledSendToQueue(pool_queue: string, payload: Buffer, options: Options.Publish): boolean {
659+
const enqueueResult = this.ch.sendToQueue(pool_queue, payload, options);
660+
if (!enqueueResult) {
661+
this.waitToSend = true;
662+
}
663+
return enqueueResult;
664+
}
665+
634666
createSerialBuffer(inputArray) {
635667
return new Serialize.SerialBuffer({textEncoder: this.txEnc, textDecoder: this.txDec, array: inputArray});
636668
}
@@ -1160,7 +1192,7 @@ export default class MainDSWorker extends HyperionWorker {
11601192
let jsonRow = await this.processContractRowNative(payload, block_num);
11611193

11621194
if (jsonRow?.value && !jsonRow['_blacklisted']) {
1163-
console.log(jsonRow);
1195+
debugLog(jsonRow);
11641196
debugLog('Delta DS failed ->>', jsonRow);
11651197
jsonRow = await this.processContractRowNative(payload, block_num - 1);
11661198
debugLog('Retry with previous ABI ->>', jsonRow);
@@ -1495,7 +1527,7 @@ export default class MainDSWorker extends HyperionWorker {
14951527
} catch (e) {
14961528
hLog(`Delta struct [${key}] processing error: ${e.message}`);
14971529
hLog(e);
1498-
console.log(data[1]);
1530+
hLog(data[1]);
14991531
}
15001532
}
15011533
}

‎workers/ds-pool.ts

+2-2
Original file line numberDiff line numberDiff line change
@@ -454,8 +454,8 @@ export default class DSPoolWorker extends HyperionWorker {
454454
}
455455

456456
async processTraces(transaction_trace, extra) {
457-
const {cpu_usage_us, net_usage_words} = transaction_trace;
458-
const {block_num, block_id, producer, ts, inline_count, filtered, live, signatures} = extra;
457+
const {cpu_usage_us, net_usage_words, signatures} = transaction_trace;
458+
const {block_num, block_id, producer, ts, inline_count, filtered, live} = extra;
459459

460460
if (transaction_trace.status === 0) {
461461
let action_count = 0;

0 commit comments

Comments
 (0)