Skip to content

Commit dd8c847

Browse files
wphanwphan
authored
bundleSender reconnect, refactor filler logs (#317)
* bundleSender reconnect, refactor filler logs * fix sim * add jito_connected metric, check if bundle sender is connected * switchboard crank: jito check
1 parent b9ffe11 commit dd8c847

File tree

8 files changed

+350
-296
lines changed

8 files changed

+350
-296
lines changed

‎src/bots/filler.ts‎

Lines changed: 35 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,7 @@ import {
8989
validRebalanceSettledPnlThreshold,
9090
} from '../utils';
9191
import { selectMakers } from '../makerSelection';
92-
import { BundleSender } from '../bundleSender';
92+
import { BundleSender, JITO_METRIC_TYPES } from '../bundleSender';
9393
import { Metrics } from '../metrics';
9494
import { LRUCache } from 'lru-cache';
9595
import { bs58 } from '@project-serum/anchor/dist/cjs/utils/bytes';
@@ -147,11 +147,6 @@ enum METRIC_TYPES {
147147
simulate_tx_duration_histogram = 'simulate_tx_duration_histogram',
148148
expired_nodes_set_size = 'expired_nodes_set_size',
149149

150-
jito_bundles_accepted = 'jito_bundles_accepted',
151-
jito_bundles_simulation_failure = 'jito_simulation_failure',
152-
jito_dropped_bundle = 'jito_dropped_bundle',
153-
jito_landed_tips = 'jito_landed_tips',
154-
jito_bundle_count = 'jito_bundle_count',
155150
clock_subscriber_ts = 'clock_subscriber_ts',
156151
wall_clock_ts = 'wall_clock_ts',
157152
}
@@ -225,6 +220,7 @@ export class FillerBot extends TxThreaded implements Bot {
225220
protected mutexBusyCounter?: CounterValue;
226221
protected attemptedTriggersCounter?: CounterValue;
227222
protected txSimErrorCounter?: CounterValue;
223+
protected jitoConnectedGauge?: GaugeValue;
228224
protected jitoBundlesAcceptedGauge?: GaugeValue;
229225
protected jitoBundlesSimulationFailureGauge?: GaugeValue;
230226
protected jitoDroppedBundleGauge?: GaugeValue;
@@ -450,24 +446,28 @@ export class FillerBot extends TxThreaded implements Bot {
450446
METRIC_TYPES.tx_sim_error_count,
451447
'Count of errors from simulating transactions'
452448
);
449+
this.jitoConnectedGauge = this.metrics.addGauge(
450+
JITO_METRIC_TYPES.jito_connected,
451+
'Whether the jito bundle sender is connected'
452+
);
453453
this.jitoBundlesAcceptedGauge = this.metrics.addGauge(
454-
METRIC_TYPES.jito_bundles_accepted,
454+
JITO_METRIC_TYPES.jito_bundles_accepted,
455455
'Count of jito bundles that were accepted'
456456
);
457457
this.jitoBundlesSimulationFailureGauge = this.metrics.addGauge(
458-
METRIC_TYPES.jito_bundles_simulation_failure,
458+
JITO_METRIC_TYPES.jito_bundles_simulation_failure,
459459
'Count of jito bundles that failed simulation'
460460
);
461461
this.jitoDroppedBundleGauge = this.metrics.addGauge(
462-
METRIC_TYPES.jito_dropped_bundle,
462+
JITO_METRIC_TYPES.jito_dropped_bundle,
463463
'Count of jito bundles that were dropped'
464464
);
465465
this.jitoLandedTipsGauge = this.metrics.addGauge(
466-
METRIC_TYPES.jito_landed_tips,
466+
JITO_METRIC_TYPES.jito_landed_tips,
467467
'Gauge of historic bundle tips that landed'
468468
);
469469
this.jitoBundleCount = this.metrics.addGauge(
470-
METRIC_TYPES.jito_bundle_count,
470+
JITO_METRIC_TYPES.jito_bundle_count,
471471
'Count of jito bundles that were sent, and their status'
472472
);
473473
this.clockSubscriberTs = this.metrics.addGauge(
@@ -1453,15 +1453,16 @@ export class FillerBot extends TxThreaded implements Bot {
14531453
marketType,
14541454
} = await this.getNodeFillInfo(nodeToFill);
14551455

1456-
logger.info(
1457-
logMessageForNodeToFill(
1458-
nodeToFill,
1459-
takerUserPubKey,
1460-
takerUserSlot,
1461-
makerInfos,
1462-
this.getMaxSlot(),
1463-
`Filling multi maker perp node with ${nodeToFill.makerNodes.length} makers (fillTxId: ${fillTxId})`
1464-
)
1456+
logMessageForNodeToFill(
1457+
nodeToFill,
1458+
takerUserPubKey,
1459+
takerUserSlot,
1460+
makerInfos,
1461+
this.getMaxSlot(),
1462+
fillTxId,
1463+
'multiMakerPerpFill',
1464+
this.revertOnFailure ?? false,
1465+
false
14651466
);
14661467

14671468
if (!isVariant(marketType, 'perp')) {
@@ -1645,7 +1646,7 @@ export class FillerBot extends TxThreaded implements Bot {
16451646
const nodesSent: Array<NodeToFill> = [];
16461647
const fillTxId = this.fillTxId++;
16471648

1648-
for (const [idx, nodeToFill] of nodesToFill.entries()) {
1649+
for (const [_idx, nodeToFill] of nodesToFill.entries()) {
16491650
let ixs: Array<TransactionInstruction> = [
16501651
ComputeBudgetProgram.setComputeUnitLimit({
16511652
units: 1_400_000,
@@ -1681,15 +1682,16 @@ export class FillerBot extends TxThreaded implements Bot {
16811682
marketType,
16821683
} = await this.getNodeFillInfo(nodeToFill);
16831684

1684-
logger.info(
1685-
logMessageForNodeToFill(
1686-
nodeToFill,
1687-
takerUserPubKey,
1688-
takerUserSlot,
1689-
makerInfos,
1690-
this.getMaxSlot(),
1691-
`Filling perp node ${idx} (fillTxId: ${fillTxId})`
1692-
)
1685+
logMessageForNodeToFill(
1686+
nodeToFill,
1687+
takerUserPubKey,
1688+
takerUserSlot,
1689+
makerInfos,
1690+
this.getMaxSlot(),
1691+
fillTxId,
1692+
'fillPerpNode',
1693+
this.revertOnFailure ?? false,
1694+
false
16931695
);
16941696
this.logSlots();
16951697

@@ -2260,6 +2262,9 @@ export class FillerBot extends TxThreaded implements Bot {
22602262
}
22612263
return slotsUntilJito < SLOTS_UNTIL_JITO_LEADER_TO_SEND;
22622264
}
2265+
if (!this.bundleSender?.connected()) {
2266+
return false;
2267+
}
22632268
return true;
22642269
}
22652270

‎src/bots/spotFiller.ts‎

Lines changed: 36 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,7 @@ import {
8484
validMinimumGasAmount,
8585
validRebalanceSettledPnlThreshold,
8686
} from '../utils';
87-
import { BundleSender } from '../bundleSender';
87+
import { JITO_METRIC_TYPES, BundleSender } from '../bundleSender';
8888
import {
8989
CACHED_BLOCKHASH_OFFSET,
9090
CONFIRM_TX_INTERVAL_MS,
@@ -139,11 +139,6 @@ enum METRIC_TYPES {
139139
simulate_tx_duration_histogram = 'simulate_tx_duration_histogram',
140140
expired_nodes_set_size = 'expired_nodes_set_size',
141141

142-
jito_bundles_accepted = 'jito_bundles_accepted',
143-
jito_bundles_simulation_failure = 'jito_simulation_failure',
144-
jito_dropped_bundle = 'jito_dropped_bundle',
145-
jito_landed_tips = 'jito_landed_tips',
146-
jito_bundle_count = 'jito_bundle_count',
147142
clock_subscriber_ts = 'clock_subscriber_ts',
148143
wall_clock_ts = 'wall_clock_ts',
149144
}
@@ -309,6 +304,7 @@ export class SpotFillerBot implements Bot {
309304
protected pendingTxSigsLoopRateLimitedCounter?: CounterValue;
310305
protected evictedPendingTxSigsToConfirmCounter?: CounterValue;
311306
protected expiredNodesSetSize?: GaugeValue;
307+
protected jitoConnectedGauge?: GaugeValue;
312308
protected jitoBundlesAcceptedGauge?: GaugeValue;
313309
protected jitoBundlesSimulationFailureGauge?: GaugeValue;
314310
protected jitoDroppedBundleGauge?: GaugeValue;
@@ -583,24 +579,28 @@ export class SpotFillerBot implements Bot {
583579
METRIC_TYPES.expired_nodes_set_size,
584580
'Count of nodes that are expired'
585581
);
582+
this.jitoConnectedGauge = this.metrics.addGauge(
583+
JITO_METRIC_TYPES.jito_connected,
584+
'Whether the jito bundle sender is connected'
585+
);
586586
this.jitoBundlesAcceptedGauge = this.metrics.addGauge(
587-
METRIC_TYPES.jito_bundles_accepted,
587+
JITO_METRIC_TYPES.jito_bundles_accepted,
588588
'Count of jito bundles that were accepted'
589589
);
590590
this.jitoBundlesSimulationFailureGauge = this.metrics.addGauge(
591-
METRIC_TYPES.jito_bundles_simulation_failure,
591+
JITO_METRIC_TYPES.jito_bundles_simulation_failure,
592592
'Count of jito bundles that failed simulation'
593593
);
594594
this.jitoDroppedBundleGauge = this.metrics.addGauge(
595-
METRIC_TYPES.jito_dropped_bundle,
595+
JITO_METRIC_TYPES.jito_dropped_bundle,
596596
'Count of jito bundles that were dropped'
597597
);
598598
this.jitoLandedTipsGauge = this.metrics.addGauge(
599-
METRIC_TYPES.jito_landed_tips,
599+
JITO_METRIC_TYPES.jito_landed_tips,
600600
'Gauge of historic bundle tips that landed'
601601
);
602602
this.jitoBundleCount = this.metrics.addGauge(
603-
METRIC_TYPES.jito_bundle_count,
603+
JITO_METRIC_TYPES.jito_bundle_count,
604604
'Count of jito bundles that were sent, and their status'
605605
);
606606
this.clockSubscriberTs = this.metrics.addGauge(
@@ -1571,6 +1571,9 @@ export class SpotFillerBot implements Bot {
15711571
}
15721572
return slotsUntilJito < SLOTS_UNTIL_JITO_LEADER_TO_SEND;
15731573
}
1574+
if (!this.bundleSender?.connected()) {
1575+
return false;
1576+
}
15741577
return true;
15751578
}
15761579

@@ -1764,17 +1767,17 @@ export class SpotFillerBot implements Bot {
17641767
marketType,
17651768
} = await this.getNodeFillInfo(nodeToFill);
17661769

1767-
logger.info(
1768-
logMessageForNodeToFill(
1769-
nodeToFill,
1770-
takerUserPubKey,
1771-
takerUserSlot,
1772-
makerInfos,
1773-
this.getMaxSlot(),
1774-
`Filling multi maker spot node with ${nodeToFill.makerNodes.length} makers (fillTxId: ${fillTxId})`,
1775-
spotPrecision,
1776-
'SHOULD_NOT_HAVE_NO_MAKERS'
1777-
)
1770+
logMessageForNodeToFill(
1771+
nodeToFill,
1772+
takerUserPubKey,
1773+
takerUserSlot,
1774+
makerInfos,
1775+
this.getMaxSlot(),
1776+
fillTxId,
1777+
'multiMakerSpotFill',
1778+
this.revertOnFailure ?? false,
1779+
false,
1780+
spotPrecision
17781781
);
17791782

17801783
if (!isVariant(marketType, 'spot')) {
@@ -2015,17 +2018,17 @@ export class SpotFillerBot implements Bot {
20152018
}
20162019
}
20172020

2018-
logger.info(
2019-
logMessageForNodeToFill(
2020-
nodeToFill,
2021-
takerUserPubKey,
2022-
takerUserSlot,
2023-
makerInfos,
2024-
this.getMaxSlot(),
2025-
`Filling spot node with ${nodeToFill.makerNodes.length} makers (fillTxId: ${fillTxId})`,
2026-
spotMarketPrecision,
2027-
fallbackSource as string
2028-
)
2021+
logMessageForNodeToFill(
2022+
nodeToFill,
2023+
takerUserPubKey,
2024+
takerUserSlot,
2025+
makerInfos,
2026+
this.getMaxSlot(),
2027+
fillTxId,
2028+
'fillSpotNode',
2029+
this.revertOnFailure ?? false,
2030+
false,
2031+
spotMarketPrecision
20292032
);
20302033

20312034
const ixs = [

‎src/bots/switchboardCranker.ts‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,16 @@ export class SwitchboardCrankerBot implements Bot {
106106
return recentBlockhash.blockhash;
107107
}
108108

109+
private shouldBuildForBundle(): boolean {
110+
if (!this.globalConfig.useJito) {
111+
return false;
112+
}
113+
if (!this.bundleSender?.connected()) {
114+
return false;
115+
}
116+
return true;
117+
}
118+
109119
async runCrankLoop() {
110120
const pullFeedAliases = chunks(
111121
shuffle(Object.keys(this.crankConfigs.pullFeedConfigs)),
@@ -119,7 +129,8 @@ export class SwitchboardCrankerBot implements Bot {
119129
}),
120130
];
121131

122-
if (this.globalConfig.useJito) {
132+
const shouldBuildForBundle = this.shouldBuildForBundle();
133+
if (shouldBuildForBundle) {
123134
ixs.push(this.bundleSender!.getTipIx());
124135
} else {
125136
const priorityFees =
@@ -143,7 +154,7 @@ export class SwitchboardCrankerBot implements Bot {
143154
await this.getBlockhashForTx()
144155
);
145156

146-
if (this.globalConfig.useJito) {
157+
if (shouldBuildForBundle) {
147158
tx.sign([
148159
// @ts-ignore;
149160
this.driftClient.wallet.payer,

0 commit comments

Comments
 (0)