Skip to content

Commit 0c1b6ed

Browse files
jasonishjasonish
committed
threads: remove global packet pools
Replace global thread local packet pools with packet pools owned by thread vars. The idea is to remove global data such that a library user can have multiple instances of the Suricata engine that don't share any data. As library users can provide their own threads, and may utilize different threading infrastucture such as thread pools, we should do our best not to leak resources from one instance into another. The downside is that we might lose some optimizations that thread local storage can provide, such as as a faster path to returning a packet to another threads pool.
1 parent 8542017 commit 0c1b6ed

40 files changed

Lines changed: 254 additions & 141 deletions

‎examples/lib/custom/main.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,7 @@ static void *SimpleWorker(void *arg)
9696
goto done;
9797
}
9898

99-
Packet *p = PacketGetFromQueueOrAlloc();
99+
Packet *p = PacketGetFromQueueOrAlloc(tv);
100100
if (unlikely(p == NULL)) {
101101
/* Memory allocation error. */
102102
goto done;

‎examples/plugins/ci-capture/source.c‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -65,8 +65,8 @@ static TmEcode ReceiveLoop(ThreadVars *tv, void *data, void *slot)
6565
/* Notify we are running and processing packets. */
6666
TmThreadsSetFlag(tv, THV_RUNNING);
6767

68-
PacketPoolWait();
69-
Packet *p = PacketGetFromQueueOrAlloc();
68+
PacketPoolWait(tv);
69+
Packet *p = PacketGetFromQueueOrAlloc(tv);
7070
if (unlikely(p == NULL)) {
7171
return TM_ECODE_FAILED;
7272
}

‎plugins/napatech/source-napatech.c‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -847,7 +847,7 @@ TmEcode NapatechPacketLoop(ThreadVars *tv, void *data, void *slot)
847847
while (!(suricata_ctl_flags & SURICATA_STOP)) {
848848
/* make sure we have at least one packet in the packet pool, to prevent
849849
* us from alloc'ing packets at line rate */
850-
PacketPoolWait();
850+
PacketPoolWait(tv);
851851

852852
/* Napatech returns packets 1 at a time */
853853
status = NT_NetRxGet(ntv->rx_stream, &packet_buffer, 1000);
@@ -863,7 +863,7 @@ TmEcode NapatechPacketLoop(ThreadVars *tv, void *data, void *slot)
863863
break;
864864
}
865865

866-
Packet *p = PacketGetFromQueueOrAlloc();
866+
Packet *p = PacketGetFromQueueOrAlloc(tv);
867867
if (unlikely(p == NULL)) {
868868
NT_NetRxRelease(ntv->rx_stream, packet_buffer);
869869
SCReturnInt(TM_ECODE_FAILED);

‎plugins/pfring/source-pfring.c‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -330,9 +330,9 @@ TmEcode ReceivePfringLoop(ThreadVars *tv, void *data, void *slot)
330330

331331
/* make sure we have at least one packet in the packet pool, to prevent
332332
* us from alloc'ing packets at line rate */
333-
PacketPoolWait();
333+
PacketPoolWait(tv);
334334

335-
p = PacketGetFromQueueOrAlloc();
335+
p = PacketGetFromQueueOrAlloc(tv);
336336
if (p == NULL) {
337337
SCReturnInt(TM_ECODE_FAILED);
338338
}

‎src/decode-geneve.c‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@
4343
#include "pkt-var.h"
4444
#include "util-profiling.h"
4545
#include "host.h"
46+
#include "tmqh-packetpool.h"
4647

4748
#define VALID_GENEVE_VERSIONS \
4849
{ \
@@ -301,6 +302,7 @@ static int DecodeGeneveTest01(void)
301302

302303
memset(&tv, 0, sizeof(ThreadVars));
303304
memset(&dtv, 0, sizeof(DecodeThreadVars));
305+
PacketPoolInit(&tv);
304306

305307
FlowInitConfig(FLOW_QUIET);
306308
DecodeUDP(&tv, &dtv, p, raw_geneve, sizeof(raw_geneve));
@@ -315,6 +317,7 @@ static int DecodeGeneveTest01(void)
315317
FlowShutdown();
316318
PacketFree(p);
317319
PacketFree(tp);
320+
PacketPoolDestroy(&tv);
318321
PASS;
319322
}
320323

@@ -343,6 +346,7 @@ static int DecodeGeneveTest02(void)
343346

344347
memset(&tv, 0, sizeof(ThreadVars));
345348
memset(&dtv, 0, sizeof(DecodeThreadVars));
349+
PacketPoolInit(&tv);
346350

347351
FlowInitConfig(FLOW_QUIET);
348352
DecodeUDP(&tv, &dtv, p, raw_geneve, sizeof(raw_geneve));
@@ -357,6 +361,7 @@ static int DecodeGeneveTest02(void)
357361
FlowShutdown();
358362
PacketFree(p);
359363
PacketFree(tp);
364+
PacketPoolDestroy(&tv);
360365
PASS;
361366
}
362367

@@ -390,6 +395,7 @@ static int DecodeGeneveTest03(void)
390395

391396
memset(&tv, 0, sizeof(ThreadVars));
392397
memset(&dtv, 0, sizeof(DecodeThreadVars));
398+
PacketPoolInit(&tv);
393399

394400
FlowInitConfig(FLOW_QUIET);
395401
DecodeUDP(&tv, &dtv, p, raw_geneve, sizeof(raw_geneve));
@@ -404,6 +410,7 @@ static int DecodeGeneveTest03(void)
404410
FlowShutdown();
405411
PacketFree(p);
406412
PacketFree(tp);
413+
PacketPoolDestroy(&tv);
407414
PASS;
408415
}
409416

@@ -434,6 +441,7 @@ static int DecodeGeneveTest04(void)
434441

435442
memset(&tv, 0, sizeof(ThreadVars));
436443
memset(&dtv, 0, sizeof(DecodeThreadVars));
444+
PacketPoolInit(&tv);
437445

438446
FlowInitConfig(FLOW_QUIET);
439447
DecodeUDP(&tv, &dtv, p, raw_geneve, sizeof(raw_geneve));
@@ -444,6 +452,7 @@ static int DecodeGeneveTest04(void)
444452
DecodeGeneveConfigPorts(GENEVE_DEFAULT_PORT_S); /* Reset Geneve port list for future calls */
445453
FlowShutdown();
446454
PacketFree(p);
455+
PacketPoolDestroy(&tv);
447456
PASS;
448457
}
449458

@@ -474,6 +483,7 @@ static int DecodeGeneveTest05(void)
474483

475484
memset(&tv, 0, sizeof(ThreadVars));
476485
memset(&dtv, 0, sizeof(DecodeThreadVars));
486+
PacketPoolInit(&tv);
477487

478488
FlowInitConfig(FLOW_QUIET);
479489
DecodeUDP(&tv, &dtv, p, raw_geneve, sizeof(raw_geneve));
@@ -483,6 +493,7 @@ static int DecodeGeneveTest05(void)
483493

484494
FlowShutdown();
485495
PacketFree(p);
496+
PacketPoolDestroy(&tv);
486497
PASS;
487498
}
488499
#endif /* UNITTESTS */

‎src/decode-ipv4.c‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737
#include "defrag.h"
3838
#include "flow.h"
3939
#include "util-print.h"
40+
#include "tmqh-packetpool.h"
4041

4142
/* Generic validation
4243
*
@@ -1293,6 +1294,7 @@ static int DecodeIPV4DefragTest01(void)
12931294

12941295
memset(&tv, 0, sizeof(ThreadVars));
12951296
memset(&dtv, 0, sizeof(DecodeThreadVars));
1297+
PacketPoolInit(&tv);
12961298

12971299
FlowInitConfig(FLOW_QUIET);
12981300
DefragInit();
@@ -1327,6 +1329,7 @@ static int DecodeIPV4DefragTest01(void)
13271329
PacketRecycle(p);
13281330
FlowShutdown();
13291331
PacketFree(p);
1332+
PacketPoolDestroy(&tv);
13301333
PASS;
13311334
}
13321335

@@ -1389,6 +1392,7 @@ static int DecodeIPV4DefragTest02(void)
13891392

13901393
memset(&tv, 0, sizeof(ThreadVars));
13911394
memset(&dtv, 0, sizeof(DecodeThreadVars));
1395+
PacketPoolInit(&tv);
13921396

13931397
FlowInitConfig(FLOW_QUIET);
13941398
DefragInit();
@@ -1424,6 +1428,7 @@ static int DecodeIPV4DefragTest02(void)
14241428
PacketRecycle(p);
14251429
FlowShutdown();
14261430
PacketFree(p);
1431+
PacketPoolDestroy(&tv);
14271432
PASS;
14281433
}
14291434

@@ -1480,6 +1485,7 @@ static int DecodeIPV4DefragTest03(void)
14801485
DecodeThreadVars dtv;
14811486
memset(&tv, 0, sizeof(ThreadVars));
14821487
memset(&dtv, 0, sizeof(DecodeThreadVars));
1488+
PacketPoolInit(&tv);
14831489

14841490
FlowInitConfig(FLOW_QUIET);
14851491
DefragInit();
@@ -1524,6 +1530,7 @@ static int DecodeIPV4DefragTest03(void)
15241530
PacketRecycle(p);
15251531
FlowShutdown();
15261532
PacketFree(p);
1533+
PacketPoolDestroy(&tv);
15271534
PASS;
15281535
}
15291536

‎src/decode-ipv6.c‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737
#include "flow-hash.h"
3838
#include "util-print.h"
3939
#include "util-validate.h"
40+
#include "tmqh-packetpool.h"
4041

4142
/**
4243
* \brief Function to decode IPv4 in IPv6 packets
@@ -768,6 +769,7 @@ static int DecodeIPV6FragTest01 (void)
768769

769770
memset(&tv, 0, sizeof(ThreadVars));
770771
memset(&dtv, 0, sizeof(DecodeThreadVars));
772+
PacketPoolInit(&tv);
771773

772774
PacketCopyData(p1, raw_frag1, sizeof(raw_frag1));
773775
PacketCopyData(p2, raw_frag2, sizeof(raw_frag2));
@@ -805,6 +807,7 @@ static int DecodeIPV6FragTest01 (void)
805807
}
806808
DefragDestroy();
807809
FlowShutdown();
810+
PacketPoolDestroy(&tv);
808811
return result;
809812
}
810813

‎src/decode-vxlan.c‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@
3939
#include "util-validate.h"
4040
#include "util-unittest.h"
4141
#include "util-debug.h"
42+
#include "tmqh-packetpool.h"
4243

4344
#define VXLAN_HEADER_LEN sizeof(VXLANHeader)
4445

@@ -249,6 +250,7 @@ static int DecodeVXLANtest01 (void)
249250
DecodeThreadVars dtv;
250251
memset(&tv, 0, sizeof(ThreadVars));
251252
memset(&dtv, 0, sizeof(DecodeThreadVars));
253+
PacketPoolInit(&tv);
252254

253255
DecodeVXLANConfigPorts(VXLAN_DEFAULT_PORT_S);
254256
FlowInitConfig(FLOW_QUIET);
@@ -264,6 +266,7 @@ static int DecodeVXLANtest01 (void)
264266
FlowShutdown();
265267
PacketFree(p);
266268
PacketFreeOrRelease(tp);
269+
PacketPoolDestroy(&tv);
267270
PASS;
268271
}
269272

@@ -288,6 +291,7 @@ static int DecodeVXLANtest02 (void)
288291
DecodeThreadVars dtv;
289292
memset(&tv, 0, sizeof(ThreadVars));
290293
memset(&dtv, 0, sizeof(DecodeThreadVars));
294+
PacketPoolInit(&tv);
291295

292296
DecodeVXLANConfigPorts("1");
293297
FlowInitConfig(FLOW_QUIET);
@@ -299,6 +303,7 @@ static int DecodeVXLANtest02 (void)
299303
DecodeVXLANConfigPorts(VXLAN_DEFAULT_PORT_S); /* reset */
300304
FlowShutdown();
301305
PacketFree(p);
306+
PacketPoolDestroy(&tv);
302307
PASS;
303308
}
304309

@@ -342,6 +347,7 @@ decoder:\n\
342347
DecodeThreadVars dtv;
343348
memset(&tv, 0, sizeof(ThreadVars));
344349
memset(&dtv, 0, sizeof(DecodeThreadVars));
350+
PacketPoolInit(&tv);
345351

346352
DecodeVXLANConfig();
347353
DecodeVXLANConfigPorts(VXLAN_DEFAULT_PORT_S);
@@ -360,6 +366,7 @@ decoder:\n\
360366
PacketFreeOrRelease(tp);
361367
SCConfDeInit();
362368
SCConfRestoreContextBackup();
369+
PacketPoolDestroy(&tv);
363370
PASS;
364371
}
365372

@@ -399,6 +406,7 @@ decoder:\n\
399406
DecodeThreadVars dtv;
400407
memset(&tv, 0, sizeof(ThreadVars));
401408
memset(&dtv, 0, sizeof(DecodeThreadVars));
409+
PacketPoolInit(&tv);
402410

403411
DecodeVXLANConfig();
404412
DecodeVXLANConfigPorts(VXLAN_DEFAULT_PORT_S);
@@ -417,6 +425,7 @@ decoder:\n\
417425
PacketFreeOrRelease(tp);
418426
SCConfDeInit();
419427
SCConfRestoreContextBackup();
428+
PacketPoolDestroy(&tv);
420429
PASS;
421430
}
422431

@@ -457,6 +466,7 @@ decoder:\n\
457466
DecodeThreadVars dtv;
458467
memset(&tv, 0, sizeof(ThreadVars));
459468
memset(&dtv, 0, sizeof(DecodeThreadVars));
469+
PacketPoolInit(&tv);
460470

461471
DecodeVXLANConfig();
462472
DecodeVXLANConfigPorts(VXLAN_DEFAULT_PORT_S);
@@ -471,6 +481,7 @@ decoder:\n\
471481
PacketFree(p);
472482
SCConfDeInit();
473483
SCConfRestoreContextBackup();
484+
PacketPoolDestroy(&tv);
474485
PASS;
475486
}
476487

@@ -511,6 +522,7 @@ decoder:\n\
511522
DecodeThreadVars dtv;
512523
memset(&tv, 0, sizeof(ThreadVars));
513524
memset(&dtv, 0, sizeof(DecodeThreadVars));
525+
PacketPoolInit(&tv);
514526

515527
DecodeVXLANConfig();
516528
DecodeVXLANConfigPorts(VXLAN_DEFAULT_PORT_S);
@@ -529,6 +541,7 @@ decoder:\n\
529541
PacketFreeOrRelease(tp);
530542
SCConfDeInit();
531543
SCConfRestoreContextBackup();
544+
PacketPoolDestroy(&tv);
532545
PASS;
533546
}
534547

@@ -551,6 +564,7 @@ static int DecodeVXLANtest07(void)
551564
DecodeThreadVars dtv;
552565
memset(&tv, 0, sizeof(ThreadVars));
553566
memset(&dtv, 0, sizeof(DecodeThreadVars));
567+
PacketPoolInit(&tv);
554568

555569
DecodeVXLANConfigPorts(VXLAN_DEFAULT_PORT_S);
556570
FlowInitConfig(FLOW_QUIET);
@@ -565,6 +579,7 @@ static int DecodeVXLANtest07(void)
565579

566580
FlowShutdown();
567581
PacketFree(p);
582+
PacketPoolDestroy(&tv);
568583
PASS;
569584
}
570585
#endif /* UNITTESTS */

‎src/decode.c‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -290,10 +290,10 @@ void PacketFreeOrRelease(Packet *p)
290290
*
291291
* \retval p packet, NULL on error
292292
*/
293-
Packet *PacketGetFromQueueOrAlloc(void)
293+
Packet *PacketGetFromQueueOrAlloc(ThreadVars *tv)
294294
{
295295
/* try the pool first */
296-
Packet *p = PacketPoolGetPacket();
296+
Packet *p = PacketPoolGetPacket(tv);
297297

298298
if (p == NULL) {
299299
/* non fatal, we're just not processing a packet then */
@@ -403,7 +403,7 @@ Packet *PacketTunnelPktSetup(ThreadVars *tv, DecodeThreadVars *dtv, Packet *pare
403403
}
404404

405405
/* get us a packet */
406-
Packet *p = PacketGetFromQueueOrAlloc();
406+
Packet *p = PacketGetFromQueueOrAlloc(tv);
407407
if (unlikely(p == NULL)) {
408408
SCReturnPtr(NULL, "Packet");
409409
}
@@ -470,12 +470,13 @@ Packet *PacketTunnelPktSetup(ThreadVars *tv, DecodeThreadVars *dtv, Packet *pare
470470
*
471471
* \retval p the pseudo packet or NULL if out of memory
472472
*/
473-
Packet *PacketDefragPktSetup(Packet *parent, const uint8_t *pkt, uint32_t len, uint8_t proto)
473+
Packet *PacketDefragPktSetup(
474+
ThreadVars *tv, Packet *parent, const uint8_t *pkt, uint32_t len, uint8_t proto)
474475
{
475476
SCEnter();
476477

477478
/* get us a packet */
478-
Packet *p = PacketGetFromQueueOrAlloc();
479+
Packet *p = PacketGetFromQueueOrAlloc(tv);
479480
if (unlikely(p == NULL)) {
480481
SCReturnPtr(NULL, "Packet");
481482
}

‎src/decode.h‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1115,10 +1115,11 @@ enum DecodeTunnelProto {
11151115

11161116
Packet *PacketTunnelPktSetup(ThreadVars *tv, DecodeThreadVars *dtv, Packet *parent,
11171117
const uint8_t *pkt, uint32_t len, enum DecodeTunnelProto proto);
1118-
Packet *PacketDefragPktSetup(Packet *parent, const uint8_t *pkt, uint32_t len, uint8_t proto);
1118+
Packet *PacketDefragPktSetup(
1119+
ThreadVars *tv, Packet *parent, const uint8_t *pkt, uint32_t len, uint8_t proto);
11191120
void PacketDefragPktSetupParent(Packet *parent);
11201121
void DecodeRegisterPerfCounters(DecodeThreadVars *, ThreadVars *);
1121-
Packet *PacketGetFromQueueOrAlloc(void);
1122+
Packet *PacketGetFromQueueOrAlloc(ThreadVars *tv);
11221123
Packet *PacketGetFromAlloc(void);
11231124
void PacketDecodeFinalize(ThreadVars *tv, DecodeThreadVars *dtv, Packet *p);
11241125
void PacketUpdateEngineEventCounters(ThreadVars *tv,

0 commit comments

Comments
 (0)